From 4e9bf5d6e7db81b5b4ef62dcd702965075652098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Madar=C3=A1sz?= Date: Mon, 1 Jan 2024 22:15:15 +0100 Subject: [PATCH] update lua support --- tools/cparser.lua | 1725 ++++++++++++++++++++++++++++ tools/luajit_lcpp.lua | 1960 -------------------------------- tools/luajit_make_bindings.lua | 34 +- 3 files changed, 1750 insertions(+), 1969 deletions(-) create mode 100644 tools/cparser.lua delete mode 100644 tools/luajit_lcpp.lua diff --git a/tools/cparser.lua b/tools/cparser.lua new file mode 100644 index 0000000..b47d2b5 --- /dev/null +++ b/tools/cparser.lua @@ -0,0 +1,1725 @@ +-- Copyright (c) Facebook, Inc. and its affiliates. +-- This source code is licensed under the MIT license found in the +-- LICENSE file in the root directory of this source tree. +-- +-- Lua module to preprocess and parse C declarations. +-- (Leon Bottou, 2015) + +-- @r-lyeh: del lcdecl module +-- @r-lyeh: add cppString method +-- @r-lyeh: add __COUNTER__ macro +-- @r-lyeh: del empty #line directives from output + +-- standard libs +local string = require 'string' +local coroutine = require 'coroutine' +local table = require 'table' +local io = require 'io' + +-- Lua 5.1 to 5.3 compatibility +local unpack = unpack or table.unpack + +-- Debugging +local DEBUG = true +if DEBUG then pcall(require,'strict') end + +-- luacheck: globals cparser +-- luacheck: ignore 43 4/ti 4/li +-- luacheck: ignore 212/.*_ +-- luacheck: ignore 211/is[A-Z].* 211/Type +-- luacheck: ignore 542 + + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- ALL UGLY HACKS SHOULD BE HERE + + +-- Sometimes we cannot find system include files but need to know at +-- least things about them. For instance, certain system include files +-- define alternate forms for keywords. + +local knownIncludeQuirks = {} + +knownIncludeQuirks[""] = { -- c99 + "#ifndef complex", "# define complex _Complex", "#endif" +} + +knownIncludeQuirks[""] = { -- c99 + "#ifndef bool", "# define bool _Bool", "#endif" +} + +knownIncludeQuirks[""] = { -- c11 + "#ifndef alignof", "# define alignof _Alignof", "#endif", + "#ifndef alignas", "# define alignas _Alignas", "#endif" +} + +knownIncludeQuirks[""] = { -- c11 + "#ifndef noreturn", "# define noreturn _Noreturn", "#endif" +} + +knownIncludeQuirks[""] = { -- c11 + "#ifndef thread_local", "# define thread_local _Thread_local", "#endif" +} + +knownIncludeQuirks[""] = { -- c++ + "#define and &&", "#define and_eq &=", "#define bitand &", "#define bitor |", + "#define compl ~", "#define not !", "#define not_eq !=", "#define or ||", + "#define or_eq |=", "#define xor ^", "#define xor_eq ^=" +} + + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- TAGGED TABLES + + +-- Utilities to produce and print tagged tables. +-- The tag name is simply the contents of table key . +-- Function returns a node constructor +-- +-- Example: +-- +-- > Foo = newTag('Foo') +-- > Bar = newTag('Bar') +-- +-- > print( Foo{const=true,next=Bar{name="Hello"}} ) +-- Foo{next=Bar{name="Hello"},const=true} +-- +-- > print( Bar{name="hi!", Foo{1}, Foo{2}, Foo{3}} ) +-- Bar{Foo{1},Foo{2},Foo{3},name="hi!"} + +local function newTag(tag) + -- the printing function + local function tostr(self) + local function str(x) + if type(x)=='string' then + return string.format("%q",x):gsub("\\\n","\\n") + elseif type(x)=='table' and not getmetatable(x) then + return "{..}" + else + return tostring(x) + end + end + local p = string.format("%s{", self.tag or "Node") + local s = {} + local seqlen = 0 + for i=1,#self do + if self[i] then seqlen=i else break end end + for i=1,seqlen do + s[1+#s] = str(self[i]) end + for k,v in pairs(self) do + if type(k) == 'number' then + if k<1 or k>seqlen then + s[1+#s] = string.format("[%s]=%s",k,str(v)) end + elseif type(k) ~= 'string' then + s.extra = true + elseif k:find("^_") and type(v)=='table' then + s[1+#s] = string.format("%s={..}",k) -- hidden + elseif k ~= 'tag' then + s[1+#s] = string.format("%s=%s",k,str(v)) end + end + if s.extra then s[1+#s] = "..." end + return p .. table.concat(s,',') .. '}' + end + -- the constructor + return function(t) -- must be followed by a table constructor + t = t or {} + assert(type(t)=='table') + setmetatable(t, { __tostring=tostr } ) + t.tag = tag + return t + end +end + +-- hack to print any table: print(Node(nn)) +local Node = newTag(nil) -- luacheck: ignore 211 + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- UTILITIES + + +-- Many functions below have an optional argument 'options' which is +-- simply an array of compiler-like options that are specified in the +-- toplevel call and passed to nearly all functions. Because it +-- provides a good communication channel across the code components, +-- many named fields are also used for multiple purposes. The +-- following function is called at the beginning of the user facing +-- functions to make a copy of the user provided option array and +-- setup some of these fields. + +local function copyOptions(options) + options = options or {} + assert(type(options)=='table') + local noptions = {} + -- copy options + for k,v in ipairs(options) do noptions[k]=v end + -- copy user modifiable named fields + noptions.sizeof = options.sizeof -- not used yet + noptions.alignof = options.alignof -- not used yet + -- create reversed hash + noptions.hash = {} + for i,v in ipairs(options) do + noptions.hash[v] = i + end + -- compute dialect flags + local dialect = 'gnu99' + for _,v in ipairs(options) do + if v:find("^%-std=%s*[^%s]") then + dialect = v:match("^%-std=%s*(.-)%s*$") + end + end + noptions.dialect = dialect + noptions.dialectGnu = dialect:find("^gnu") + noptions.dialect99 = dialect:find("9[9x]$") + noptions.dialect11 = dialect:find("1[1x]$") + noptions.dialectAnsi = not noptions.dialectGnu + noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect99 + noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect11 + -- return + return noptions +end + + +-- This function tests whether a particular option has been given. + +local function hasOption(options, opt) + assert(options) + assert(options.silent or options.hash) + return options.hash and options.hash[opt] +end + + +-- Generic functions for error messages + +local function xmessage(err, options, lineno, message, ...) + local msg = string.format("cparser: (%s) ",lineno) + msg = msg .. string.format(message,...) + if options.silent then + if err == 'error' then error(msg, 0) end + else + if err == 'warning' and hasOption(options, "-Werror") then err = 'error' end + if err == 'error' or not hasOption(options, "-w") then print(msg) end + if err == 'error' then error("cparser: aborted",0) end + end +end + +local function xwarning(options, lineno, message, ...) + xmessage('warning', options, lineno, message, ...) +end + +local function xerror(options, lineno, message, ...) + xmessage('error', options, lineno, message, ...) +end + +local function xassert(cond, ...) + if not cond then xerror(...) end +end + +local function xdebug(lineno,message,...) + local msg = string.format("\t\t[%s] ", lineno) + msg = msg .. string.format(message,...) + print(msg) +end + + +-- Nil-safe max + +local function max(a,b) + a = a or b + b = b or a + return a > b and a or b +end + + +-- Deep table comparison +-- (not very efficient, no loop detection) + +local function tableCompare(a,b) + if a == b then + return true + elseif type(a) == 'table' and type(b) == 'table' then + for k,v in pairs(a) do + if not tableCompare(v,b[k]) then return false end + end + for k,v in pairs(b) do + if not tableCompare(a[k],v) then return false end + end + return true + else + return false + end +end + + +-- Concatenate two possibly null arrays + +local function tableAppend(a1, a2) + if not a1 then + return a2 + elseif not a2 then + return a1 + else + local a = {} + for _,v in ipairs(a1) do a[1+#a] = v end + for _,v in ipairs(a2) do a[1+#a] = v end + return a + end +end + +-- Concatenate strings from table (skipping non-string content.) + +local function tableConcat(a) + local b = {} + for _,v in ipairs(a) do + if type(v) == 'string' then b[1+#b]=v end end + return table.concat(b) +end + + +-- Evaluate a lua expression, return nil on error. + +local function evalLuaExpression(s) + assert(type(s)=='string') + local f = load(string.gmatch(s,".*")) + local function r(status,...) + if status then return ... end end + return r(pcall(f or error)) +end + + +-- Bitwise manipulations +-- try lua53 operators otherwise revert to iterative version + +local bit = evalLuaExpression([[ + local bit = {} + function bit.bnot(a) return ~a end + function bit.bor(a,b) return a | b end + function bit.band(a,b) return a & b end + function bit.bxor(a,b) return a ~ b end + function bit.lshift(a,b) return a < 0 and b < 0 and ~((~a) << b) or a << b end + return bit +]]) + +if not bit then + local function bor(a,b) + local r, c, d = 0, 1, -1 + while a > 0 or b > 0 or a < -1 or b < -1 do + if a % 2 > 0 or b % 2 > 0 then r = r + c end + a, b, c, d = math.floor(a / 2), math.floor(b / 2), c * 2, d * 2 end + if a < 0 or b < 0 then r = r + d end + return r end + bit = {} + function bit.bnot(a) return -1-a end + function bit.bor(a,b) return bor(a,b) end + function bit.band(a,b) return -1-bor(-1-a,-1-b) end + function bit.bxor(a,b) return bor(-1-bor(a,-1-b),-1-bor(-1-a,b)) end + function bit.lshift(a,b) return math.floor(a * 2 ^ b) end +end + + +-- Coroutine helpers. +-- This code uses many coroutines that yield lines or tokens. +-- All functions that can yield take an options table as first argument. + + +-- Wrap a coroutine f into an iterator +-- The options and all the extra arguments are passed +-- to the coroutine when it starts. Together with the +-- above calling convention, this lets us specify +-- coroutine pipelines (see example in function "cpp".) + +local function wrap(options, f, ...) + local function g(...) coroutine.yield(nil) f(...) end + local c = coroutine.create(g) + coroutine.resume(c, options, ...) + local function r(s,...) + if not s then local m = ... ; error(m, 0) end + return ... + end + return function() + if coroutine.status(c) ~= 'dead' then + return r(coroutine.resume(c)) + end + end +end + + +-- Collect coroutine outputs into an array +-- The options and the extra arguments are passed to the coroutine. + +local function callAndCollect(options, f, ...) -- Bell Labs nostalgia + local collect = {} + for s in wrap(options, f, ...) do + collect[1+#collect] = s + end + return collect +end + + +-- Yields all outputs from iterator iter. +-- Argument options is ignored. + +local function yieldFromIterator(options_, iter) + local function yes(v,...) coroutine.yield(v,...) return v end + while yes(iter()) do end +end + + +-- Yields all values from array . +-- This function successively yields all values in the table. +-- Every yield is augmented with all extra arguments passed to the function. +-- Argument options is ignored. + +local function yieldFromArray(options_, arr, ...) + for _,v in ipairs(arr) do + coroutine.yield(v, ...) + end +end + + + + + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- INITIAL PREPROCESSING + + + +-- A routine that pulls lines from a line iterator +-- and yields them together with a location +-- composed of the optional prefix, a colon, and a line number. +-- Argument options is ignored. +-- Lua provides good line iterators such as: +-- io.lines(filename) filedesc:lines() str:gmatch("[^\n]+") + +local function yieldLines(options_,lineIterator,prefix) + prefix = prefix or "" + assert(type(prefix)=='string') + local n = 0 + for s in lineIterator do + n = n + 1 + coroutine.yield(s, string.format("%s:%d", prefix, n)) + end +end + + +-- A routine that obtains lines from coroutine , +-- joins lines terminated by a backslash, and yield the +-- resulting lines. The coroutine is initialized with +-- argument and all extra arguments. +-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (3) + +local function joinLines(options, lines, ...) + local li = wrap(options, lines, ...) + for s, n in li do + while type(s) == 'string' and s:find("\\%s*$") do + local t = li() or "" + s = s:gsub("\\%s*$", "") .. t + end + coroutine.yield(s, n) + end +end + + +-- A routine that obtain lines from coroutine , eliminate the +-- comments and yields the resulting lines. The coroutine is +-- initialized with argument and all extra arguments. +-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (4) + +local function eliminateComments(options, lines, ...) + local lineIterator = wrap(options, lines, ...) + local s,n = lineIterator() + while type(s) == 'string' do + local inString = false + local q = s:find("[\'\"\\/]", 1) + while q ~= nil do + if hasOption(options,"-d:comments") then + xdebug(n, "comment: [%s][%s] %s",s:sub(1,q-1),s:sub(q),inString) + end + local c = s:byte(q) + if inString then + if c == 92 then -- \ + q = q + 1 + elseif c == inString then + inString = false + end + else + if c == 34 or c == 39 then -- " or ' + inString = c + elseif c == 47 and s:byte(q+1) == 47 then -- "//" + s = s:sub(1,q-1) + elseif c == 47 and s:byte(q+1) == 42 then -- "/*" + local p = s:find("%*/",q+2) + if p ~= nil then + s = s:sub(1,q-1) .. " " .. s:sub(p+2) + else + s = s:sub(1,q-1) + local ss,pp + repeat + ss = lineIterator() + xassert(ss ~= nil, options, n, "Unterminated comment") + pp = ss:find("%*/") + until pp + s = s .. " " .. ss:sub(pp+2) + end + end + end + q = s:find("[\'\"\\/]", q+1) + end + coroutine.yield(s, n) + s, n = lineIterator() + end +end + + + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- TOKENIZER + + + +local keywordTable = { + ------ Standard keywords + "auto", "break", "case", "char", "const", "continue", "default", "do", + "double", "else", "enum", "extern", "float", "for", "goto", "if", "int", + "long", "register", "return", "short", "signed", "sizeof", "static", "struct", + "switch", "typedef", "union", "unsigned", "void", "volatile", "while", + ------ Nonstandard or dialect specific keywords do not belong here + ------ because the main function of this table is to say which + ------ identifiers cannot be variable names. +} + +local punctuatorTable = { + "+", "-", "*", "/", "%", "&", "|", "^", ">>", "<<", "~", + "=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", ">>=", "<<=", + "(", ")", "[", "]", "{", "}", "++", "--", + "==", "!=", ">=", "<=", ">", "<", "&&", "||", "!", + ".", "->", "*", "&", "?", ":", "::", "->*", ".*", ";", ",", + "#", "##", "...", "@", "\\" -- preprocessor stuff +} + +local keywordHash = {} +for _,v in ipairs(keywordTable) do + keywordHash[v] = true +end + +local punctuatorHash = {} +for _,v in ipairs(punctuatorTable) do + local l = v:len() + local b = v:byte() + punctuatorHash[v] = true + punctuatorHash[b] = max(l,punctuatorHash[b]) +end + + +-- The following functions test the types of the tokens returned by the tokenizer. +-- They should not be applied to arbitrary strings. + +local function isSpace(tok) + return type(tok) == 'string' and tok:find("^%s") ~= nil end +local function isNewline(tok) -- Subtype of space + return type(tok) == 'string' and tok:find("^\n") ~= nil end +local function isNumber(tok) + return type(tok) == 'string' and tok:find("^[.0-9]") ~= nil end +local function isString(tok) + if type(tok) ~= 'string' then return false end + return tok:find("^[\'\"]") ~= nil end +local function isHeaderName(tok) + if type(tok) ~= 'string' then return false end + return tok:find("^\"") or tok:find("^<") and tok:find(">$") end +local function isPunctuator(tok) + return type(tok) == 'string' and punctuatorHash[tok] ~= nil end +local function isIdentifier(tok) + return type(tok) == 'string' and tok:find("^[A-Za-z_$]") ~= nil end +local function isKeyword(tok) -- Subtype of identifier + return keywordHash[tok] ~= nil end +local function isName(tok) -- Subtype of identifier + return isIdentifier(tok) and not keywordHash[tok] end + +-- Magic tokens are used to mark macro expansion boundaries (see expandMacros.) +local function isMagic(tok) + return tok and type(tok) ~= 'string' end +local function isBlank(tok) -- Treats magic token as space. + return isMagic(tok) or isSpace(tok) end + +-- The tokenizeLine() function takes a line, splits it into tokens, +-- and yields tokens and locations. The number tokens are the weird +-- preprocessor numbers defined by ansi c. The string tokens include +-- character constants and angle-bracket delimited strings occuring +-- after an include directive. Every line begins with a newline +-- token giving the proper indentation. All subsequent spaces +-- are reduced to a single space character. + +local function tokenizeLine(options, s, n, notNewline) + -- little optimization for multiline macros + -- s may be an array of precomputed tokens + if type(s) == 'table' then + return yieldFromArray(options, s, n) + end + -- normal operation + assert(type(s) == 'string') + local p = s:find("[^%s]") + -- produce a newline token + if p and not notNewline then + local r = '\n' .. s:sub(1,p-1) + coroutine.yield(r, n) + end + -- produce one token + local function token() + local b, l, r + if hasOption(options, "-d:tokenize") then + xdebug(n, "[%s][%s]",s:sub(1,p-1),s:sub(p)) + end + -- space + l = s:find("[^%s]", p) + if l == nil then + return nil + elseif l > p then + p = l + return " ", n + end + -- identifier + r = s:match("^[a-zA-Z_$][a-zA-Z0-9_$]*", p) + if r ~= nil then + p = p + r:len() + return r, n + end + -- preprocessor numbers + r = s:match("^%.?[0-9][0-9a-zA-Z._]*", p) + if r ~= nil then + l = r:len() + while r:find("[eEpP]$") and s:find("^[-+]", p+l) do + r = r .. s:match("^[-+][0-9a-zA-Z._]*", p+l) + l = r:len() + end + p = p + l + return r, n + end + -- angle-delimited strings in include directives + b = s:byte(p) + if b == 60 and s:find("^%s*#%s*include") then + r = s:match("^<[^>]+>", p) + if r ~= nil then + p = p + r:len() + return r, n + end + end + -- punctuator + l = punctuatorHash[b] + if l ~= nil then + while l > 0 do + r = s:sub(p,p+l-1) + if punctuatorHash[r] then + p = p + l + return r, n + end + l = l - 1 + end + end + -- string + if b == 34 or b == 39 then -- quotes + local q = p + repeat + q = s:find("[\'\"\\]", q + 1) + l = s:byte(q) + xassert(q ~= nil, options, n, "Unterminated string or character constant") + if l == 92 then + q = q + 1 + end + until l == b + r = s:sub(p,q) + p = q + 1 + return r, n + end + -- other stuff (we prefer to signal an error here) + xerror(options, n,"Unrecognized character (%s)", s:sub(p)) + end + -- loop + if p then + for tok,tokn in token do + coroutine.yield(tok, tokn) + end + end +end + + +-- Obtain lines from coroutine , +-- and yields their tokens. The coroutine is initialized with +-- argument and all extra arguments. + +local function tokenize(options, lines, ...) + for s,n in wrap(options, lines, ...) do + tokenizeLine(options, s, n) + end +end + + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- PREPROCESSING + + +-- Preprocessing is performed by two coroutines. The first one +-- processes all the preprocessor directives and yields the remaining +-- lines. The second one processes tokens from the remaining lines and +-- perform macro expansions. Both take a table of macro definitions as +-- argument. The first one writes into the table and the second one +-- reads from it. +-- +-- Each macro definition is an array of tokens (for a single line +-- macro) or a table whose entry <"lines"> contains an array of arrays +-- of tokens (#defmacro). If the macro takes arguments, the entry +-- <"args"> contains a list of argument names. If the macro is +-- recursive (#defrecmacro), the entry is set. +-- Alternatively, the macro definition may be a function called at +-- macro-expansion time. This provides for complicated situations. + + +-- forward declarations +local function expandMacros() end +local function processDirectives() end + + +-- Starting with the second coroutine which takes a token producing +-- coroutine and yields the preprocessed tokens. Argument macros is +-- the macro definition table. + +-- The standard mandates that the result of a macro-expansion must be +-- scanned for further macro invocations whose argunent list possibly +-- consume tokens that follow the macro-expansion. This means that one +-- cannot recursively call expandMacros but one must prepend the +-- macro-expansion in front of the remaining tokens. The standard also +-- mandates that the result of any macro-expansion must be marked to +-- prevent recursive invocation of the macro that generated it, +-- whether when expanding macro arguments or expanding the macro +-- itself. We achieve this by bracketing every macro-expansion with +-- magic tokens that track which macro definitions must be disabled. +-- These magic tokens are removed later in the coroutines +-- or . + +expandMacros = function(options, macros, tokens, ...) + -- basic iterator + local ti = wrap(options, tokens, ...) + -- prepending tokens in front of the token stream + local prepend = {} + local function prependToken(s,n) + table.insert(prepend,{s,n}) end + local function prependTokens(pti) + local pos = 1+#prepend + for s,n in pti do table.insert(prepend,pos,{s,n}) end end + local ti = function() + if #prepend > 0 then return unpack(table.remove(prepend)) + else return ti() end end + -- iterator that handles magic tokens to update macro definition table + local ti = function() + local s,n = ti() + while type(s) == 'table' do + if s.tag == 'push' then + local nmacros = {} + setmetatable(nmacros, {__index=macros}) + if s.symb then nmacros[s.symb] = false end + macros = nmacros + elseif s.tag == 'pop' then + local mt = getmetatable(macros) + if mt and mt.__index then macros = mt.__index end + end + coroutine.yield(s,n) + s,n = ti() + end + return s,n + end + -- redefine ti() to ensure tok,n remain up-to-date + local tok,n = ti() + local ti = function() tok,n=ti() return tok,n end + -- collect one macro arguments into an array + -- stop when reaching a closing parenthesis or a comma + local function collectArgument(ti, varargs) + local count = 0 + local tokens = {} + ti() + while isSpace(tok) do + tok = ti() + end + while tok do + if tok == ')' and count == 0 then + break + elseif tok == ')' then + count = count - 1 + elseif tok == '(' then + count = count + 1 + elseif tok == ',' and count == 0 and not varargs then + break + end + if isSpace(tok) then tok = " " end + tokens[1+#tokens] = tok + tok = ti() + end + if #tokens > 0 and isSpace(tokens[#tokens]) then + tokens[#tokens] = nil + end + return tokens + end + -- collects all macro arguments + local function collectArguments(ti,def,ntok,nn) + local args = def.args + local nargs = { [0]={} } + if #args == 0 then ti() end + for _,name in ipairs(args) do + if tok == ')' and name == "__VA_ARGS__" then + nargs[0][name] = { negComma=true } + nargs[name] = { negComma=true } + else + xassert(tok=='(' or tok==',', options, nn, "not enough arguments for macro '%s'", ntok) + local arg = collectArgument(ti, name == "__VA_ARGS__") + nargs[0][name] = arg + nargs[name] = callAndCollect(options, expandMacros, macros, yieldFromArray, arg, nn) + end + end + if def.nva then -- named variadic argument (implies dialectGnu) + nargs[def.nva] = nargs["__VA_ARGS__"] + nargs[0][def.nva] = nargs[0]["__VA_ARGS__"] + end + xassert(tok, options, nn, "unterminated arguments for macro '%s'", ntok) + xassert(tok==')', options, nn, "too many arguments for macro '%s'", ntok) + return nargs + end + -- coroutine that substitute the macro arguments + -- and stringification and concatenation are handled here + local function substituteArguments(options, def, nargs, n, inDirective) + local uargs = nargs[0] or nargs -- unexpanded argument values + if inDirective then nargs = uargs end -- use unexpanded arguments in directives + -- prepare loop + local i,j,k = 1,1,1 + while def[i] do + if isBlank(def[i]) then + -- copy blanks + coroutine.yield(def[i], n) + else + -- positions j and k on next non-space tokens + local function updateJandK() + if j <= i then j=i + repeat j=j+1 until def[j] == nil or not isBlank(def[j]) end + if k <= j then k=j + repeat k=k+1 until def[k] == nil or not isBlank(def[k]) end + end + updateJandK() + -- alternatives + if def[i]=='#' and def[j] and nargs[def[j]] then + -- stringification (with the weird quoting rules) + local v = { '\"' } + for _,t in ipairs(uargs[def[j]]) do + if type(t)=='string' then + if t:find("^%s+$") then t = ' ' end + if t:find("^[\'\"]") then t = string.format("%q", t):sub(2,-2) end + v[1+#v] = t end end + v[1+#v] = '\"' + coroutine.yield(tableConcat(v), n) + i = j + elseif def.nva and def[i]==',' and def[j]=='##' and def[k]==def.nva then + -- named variadic macro argument with ## to signal negative comma (gcc crap) + if nargs[def.nva].negComma then i=i+1 end + while i < j do coroutine.yield(def[i], n) ; i=i+1 end + elseif def[i]==',' and def[j]=='__VA_ARGS__' and def[k]==')' then + -- __VA_ARGS__ with implied negative comma semantics + if nargs[def[j]].negComma then i=i+1 end + while i < j do coroutine.yield(def[i], n) ; i=i+1 end + i = j-1 + elseif def[j]=='##' and def[k] and not inDirective then + -- concatenation + local u = {} + local function addToU(s) + if nargs[s] then for _,v in ipairs(uargs[s]) do u[1+#u] = v end + else u[1+#u]=s end end + addToU(def[i]) + while def[j] == '##' and def[k] do + addToU(def[k]) + i = k + updateJandK() + end + tokenizeLine(options, tableConcat(u), n, true) + elseif nargs[def[i]] then + -- substitution + yieldFromArray(options, nargs[def[i]], n) + else + -- copy + coroutine.yield(def[i], n) + end + end + i = i + 1 + end + end + -- main loop + local newline, directive = true, false + while tok ~= nil do + -- detects Zpassed directives + if newline and tok == '#' then + newline, directive = false, true + elseif not isBlank(tok) then + newline = false + elseif isNewline(tok) then + newline, directive = true, false + end + -- process code + local def = macros[tok] + if not def or directive then + -- not a macro + coroutine.yield(tok, n) + elseif type(def) == 'function' then + -- magic macro + def(ti,tok,n) + elseif def.args == nil then + -- object-like macro + prependToken({tag='pop'},n) + prependTokens(wrap(options, substituteArguments, def, {}, n)) + prependToken({tag='push', symb=tok},n) + else + -- function-like macro + local ntok, nn = tok,n + local spc = false + ti() + if isSpace(tok) then spc=true ti() end + if (tok ~= '(') then + coroutine.yield(ntok, nn) + if spc then coroutine.yield(' ', n) end + if tok then prependToken(tok,n) end + else + local nargs = collectArguments(ti,def,ntok,nn) + if def.lines == nil then + -- single-line function-like macro + prependToken({tag='pop'},n) + prependTokens(wrap(options, substituteArguments, def, nargs, nn)) + prependToken({tag='push', symb=ntok},nn) + else + -- multi-line function-like macro + local lines = def.lines + -- a coroutine that yields the macro definition + local function yieldMacroLines() + local count = 0 + for i=1,#lines,2 do + local ls,ln = lines[i], lines[i+1] + -- are we possibly in a cpp directive + local dir = false + if ls[2] and ls[2]:find('^#') then + dir = isIdentifier(ls[3]) and ls[3] or ls[4] + end + if dir and nargs[dir] then + dir = false -- leading stringification + elseif dir == 'defmacro' then + count = count + 1 -- entering a multiline macto + elseif dir == 'endmacro' then + count = count - 1 -- leaving a multiline macro + end + dir = dir or count > 0 + -- substitute + ls = callAndCollect(options,substituteArguments,ls,nargs,ln,dir) + -- compute lines (optimize speed by passing body lines as tokens) + local j=1 + while isBlank(ls[j]) do j=j+1 end + if ls[j] and ls[j]:find("^#") then -- but not directives + ls = ls[1]:sub(2) .. tableConcat(ls, nil, 2) + end + coroutine.yield(ls,ln) + end + end + -- recursively reenters preprocessing subroutines in order to handle + -- preprocessor directives located inside the macro expansion. As a result + -- we cannot expand macro invocations that extend beyond the macro-expansion. + local nmacros = {} + setmetatable(nmacros,{__index=macros}) + if not def.recursive then nmacros[ntok]=false end + if not def.recursive then coroutine.yield({tag='push',symb=ntok}) end + expandMacros(options, nmacros, tokenize, processDirectives, nmacros, yieldMacroLines) + if not def.recursive then coroutine.yield({tag='pop'}) end + end + end + end + ti() + end +end + + +-- Processing conditional directive requires evaluating conditions +-- This function takes an iterator on preprocessed expression tokens +-- and computes the value. This does not handle defined(X) expressions. +-- Optional argument resolver is a function that takes an indentifer +-- name and returns a value. Otherwise zero is assumed + +local function evaluateCppExpression(options, tokenIterator, n, resolver) + -- redefine token iterator to skip spaces and update tok + local tok + local function ti() + repeat tok = tokenIterator() + until not isBlank(tok) return tok + end + -- operator tables + local unaryOps = { + ["!"] = function(v) return v == 0 and 1 or 0 end, + ["~"] = function(v) return bit.bnot(v) end, + ["+"] = function(v) return v end, + ["-"] = function(v) return -v end, + ["L"] = function(v) return v end + } + local binaryOps = { + ["*"] = function(a,b) return a * b end, + ["/"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return math.floor(a / b) end, + ["%"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return a % b end, + ["+"] = function(a,b) return a + b end, + ["-"] = function(a,b) return a - b end, + [">>"] = function(a,b) return bit.lshift(a, -b) end, + ["<<"] = function(a,b) return bit.lshift(a, b) end, + [">="] = function(a,b) return a >= b and 1 or 0 end, + ["<="] = function(a,b) return a <= b and 1 or 0 end, + [">"] = function(a,b) return a > b and 1 or 0 end, + ["<"] = function(a,b) return a < b and 1 or 0 end, + ["=="] = function(a,b) return a == b and 1 or 0 end, + ["!="] = function(a,b) return a ~= b and 1 or 0 end, + ["&"] = function(a,b) return bit.band(a,b) end, + ["^"] = function(a,b) return bit.bxor(a,b) end, + ["|"] = function(a,b) return bit.bor(a,b) end, + ["&&"] = function(a,b) return (a ~= 0 and b ~= 0) and 1 or 0 end, + ["||"] = function(a,b) return (a ~= 0 or b ~= 0) and 1 or 0 end, + } + local binaryPrec = { + ["*"] = 1, ["/"] = 1, ["%"] = 1, + ["+"] = 2, ["-"] = 2, + [">>"] = 3, ["<<"] = 3, + [">="] = 4, ["<="] = 4, ["<"] = 4, [">"] = 4, + ["=="] = 5, ["!="] = 5, + ["&"] = 6, ["^"] = 7, ["|"] = 8, + ["&&"] = 9, ["||"] = 10 + } + -- forward + local function evaluate() end + -- unary operations + local function evalUnary() + if unaryOps[tok] then + local op = unaryOps[tok] + ti(); return op(evalUnary()) + elseif tok == '(' then + ti(); local v = evaluate() + xassert(tok == ')', options, n, "missing closing parenthesis") + ti(); return v + elseif tok == 'defined' then -- magic macro should have removed this + xerror(options, n, "syntax error after ") + elseif isIdentifier(tok) then + local v = type(resolver) == 'function' and resolver(tok,ti) + ti(); return v or 0 + elseif isNumber(tok) then + local v = tok:gsub("[ULul]+$","") + if v:find("^0[0-7]+$") then + v = tonumber(v,8) -- octal + elseif v:find("^0[bB][01]+") then + v = tonumber(v:sub(3),2) -- binary + else + v = tonumber(v) -- lua does the rest + end + xassert(v and v == math.floor(v), options, n, "syntax error (invalid integer '%s')", tok) + ti(); return v + elseif isString(tok) then + local v = '""' + if tok:find("^'") then -- interpret character constant as number + v = evalLuaExpression(string.format("return string.byte(%s)", tok)) + xassert(type(v)=='number', options, n, "syntax error (invalid value '%s')", tok) + ti() + else + while isString(tok) do + xassert(tok:find('^"'), options, n, "syntax error (invalid value '%s')", tok) + v = v:gsub('"$','') .. tok:gsub('^"','') + ti() + end + end + return v + end + xerror(options, n, "syntax error (invalid value '%s')", tok) + end + -- binary operations + local function evalBinary(p) + if p == 0 then + return evalUnary() + else + local val = evalBinary(p-1) + while binaryPrec[tok] == p do + local op = binaryOps[tok] ti() + local oval = evalBinary(p-1) + xassert(p==4 or p==5 or type(val)=='number', options, n, + "expression uses arithmetic operators on strings") + xassert(type(val) == type(oval), options, n, + "expression compares numbers and strings") + val = op(val,oval) + end + return val + end + end + -- eval ternary conditonal + local function evalTernary() + local c = evalBinary(10) + if tok ~= '?' then return c end + ti() local v1 = evalBinary(10) + xassert(tok == ':', options, n, "expecting ':' after '?'") + ti() local v2 = evalBinary(10) + if c == 0 then return v2 else return v1 end + end + + -- actual definition of evaluate + evaluate = function() + return evalTernary() + end + -- main function + ti() + xassert(tok, options, n, "constant expression expected"); + local result = evaluate() + if hasOption(options, "-d:eval") then + xdebug(n, "eval %s", result) + end + -- warn about garbage when called from cpp (but not when called with resolver) + while isBlank(tok) do ti() end + xassert(resolver or not tok, options, n, "garbage after conditional expression"); + return result +end + + +-- Now dealing with the coroutine that processes all directives. +-- This coroutine obtains lines from coroutine , +-- processes all directives, and yields remaining lines + +processDirectives = function(options, macros, lines, ...) + local li = wrap(options, lines, ...) + local s, n = li() + -- redefine li to make sure vars s and n are up-to-date + local li = function() s,n=li() return s,n end + -- directives store their current token in these vars + local dirtok, tok, spc + -- forward declaration + local processLine + -- the captureTable mechanism communicates certain preprocessor + -- events to the declaration parser in order to report them + -- to the user (the parsing does not depend on this). + -- if macros[1] is a table, the preprocessor will append + -- records to this table for the parser to process. + local function hasCaptureTable() + local captable = rawget(macros,1) + return captable and type(captable)=='table' and captable + end + local function addToCaptureTable(record) + local captable = hasCaptureTable() + if captable and record then captable[1+#captable] = record end + end + + -- simple directives + local function doIgnore() + if hasOption(options, "-Zpass") then coroutine.yield(s, n) end + end + local function doError() + xerror(options, n, "unexpected preprocessor directive #%s", dirtok) + end + local function doMessage() + local msg = s:match("^%s*#+%s*[a-z]*%s+([^%s].*)") + xmessage(dirtok, options, n, msg or '#' .. dirtok) + end + -- undef + local function doUndef(ti) + ti() + local nam = tok + xassert(isIdentifier(nam), options, n, "symbol expected after #undef") + if ti() then xwarning(options, n, "garbage after #undef directive") end + if hasOption(options, "-d:defines") then xdebug(n, "undef %s", nam) end + if hasCaptureTable() and macros[nam] and macros[nam].captured then + addToCaptureTable{directive='undef',name=nam, where=n} + end + macros[nam] = false -- false overrides inherited definitions + end + -- define + local function getMacroArguments(ti) + local args = {} + local msg = "argument list in function-like macro" + local nva = nil -- named variadic argument + ti() + while tok and tok ~= ')' do + local nam = tok + ti() + if options.dialectGnu and isIdentifier(nam) and tok == '...' then nam,nva=tok,nam ; ti() end + xassert(nam ~= "__VA_ARGS__", options, n, "name __VA_ARGS__ is not allowed here") + xassert(tok == ')' or nam ~= '...', options, n, "ellipsis in argument list must appear last") + xassert(tok == ')' or tok == ',', options, n, "bad " .. msg) + if tok == ',' then ti() end + if nam == '...' then nam = "__VA_ARGS__" end + xassert(isIdentifier(nam), options, n, "bad " .. msg) + args[1+#args] = nam + end + xassert(tok == ')', options, n, "unterminated " .. msg) + ti() + return args, nva + end + local function doDefine(ti) + xassert(isIdentifier(ti()), options, n, "symbol expected after #define") + local nam, args, nva = tok, nil, nil + -- collect arguments + if ti() == '(' and not spc then + args, nva = getMacroArguments(ti) + end + -- collect definition + local def = { tok, args = args, nva = nva } + while ti(true) do + def[1+#def] = tok + end + -- define macro + if macros[nam] and not tableCompare(def,macros[nam]) then + xwarning(options, n,"redefinition of preprocessor symbol '%s'", nam) + end + macros[nam] = def + -- debug + if hasOption(options, "-d:defines") then + if args then args = "(" .. tableConcat(args,",") .. ")" end + xdebug(n, "define %s%s = %s", nam, args or "", tableConcat(def,' ')) + end + -- capture integer macro definitions + if hasCaptureTable() and args == nil then + local i = 0 + local v = callAndCollect(options, expandMacros, macros, yieldFromArray, def, n) + local function ti() i=i+1 return v[i] end + local ss,r = pcall(evaluateCppExpression,{silent=true}, ti, n, error) + if ss and type(r)=='number' then + def.captured = true + addToCaptureTable{directive='define', name=nam, intval=r, where=n} + end + end + end + -- defmacro + local function checkDirective(stop) + xassert(s, options, n, "unterminated macro (missing #%s)", stop) + local r = type(s) == 'string' and s:match("^%s*#%s*([a-z]+)") + if r == "endmacro" or r == "endif" then + if s:find(r .. "%s*[^%s]") then + xwarning(options, n, "garbage after #%s directive", r) + end + end + return r + end + local function doMacroLines(lines, stop) + while true do + li() + local ss = callAndCollect(options,tokenizeLine,s,n) + if #ss > 0 then lines[1+#lines] = ss ; lines[1+#lines] = n end + local r = checkDirective(stop) + if r == "endmacro" or r == "endif" then + xassert(r == stop, options,n, "unbalanced directives (got #%s instead of #%s)",r,stop) + return r + elseif r=="defmacro" then + doMacroLines(lines,"endmacro") + elseif r == "if" or r == "ifdef" or r == "ifndef" then + doMacroLines(lines,"endif") + end + end + end + local function doDefmacro(ti) + xassert(isIdentifier(ti()), options, n, "symbol expected after #defmacro") + local nam,nn = tok,n + xassert(ti()=='(', options, n, "argument list expected in #defmacro") + local args, nva = getMacroArguments(ti) + xassert(not tok, options, n, "garbage after argument list in #defmacro") + -- collect definition + local lines = {} + local def = { args=args, nva=nva, lines=lines, recursive=(dirtok=="defrecursivemacro") } + doMacroLines(lines,"endmacro") + lines[#lines] = nil + lines[#lines] = nil + if hasOption(options,"-d:directives") then + xdebug(n, "directive: #endmacro") + end + if macros[nam] and not tableCompare(def,macros[nam]) then + xwarning(options, n, "redefinition of preprocessor symbol '%s'", nam) + end + if hasOption(options, "-d:defines") then + xdebug(nn, "defmacro %s(%s) =", nam, tableConcat(args,',')) + for i=1,#lines,2 do + xdebug(lines[i+1], "\t%s", tableConcat(lines[i]):gsub("^\n","")) end + end + macros[nam] = def + end + -- include + local function doInclude(ti) + -- get filename + local pti = wrap(options, expandMacros, macros, yieldFromIterator, ti) + local tok = pti() + while isBlank(tok) do tok=pti() end + if tok == '<' then -- computed include + repeat local tok2 = pti() + tok = tok .. tostring(tok2) + until tok2==nil or tok2=='>' or isNewline(tok2) + tok = tok:gsub('%s>$','>') -- gcc does this + end + xassert(isHeaderName(tok), options, n, "malformed header name after #include") + local ttok = pti() + while isBlank(ttok) do ttok=pti() end + if ttok then xwarning(options, n, "garbage after #include directive") end + -- interpret filename + local sys = tok:byte() == 60 + local min = dirtok=="include_next" and options.includedir or 0 + local fname = evalLuaExpression(string.format("return '%s'", tok:sub(2,-2))) + local pname, fd, fdi + for i,v in ipairs(options) do + if v == "-I-" then + sys=false + elseif i > min and v:find("^%-I") and not sys then + pname = v:match("^%-I%s*(.*)") .. '/' .. fname + fdi, fd = i, io.open(pname, "r") + if fd then break end + end + end + if fd then + -- include file + if hasOption(options, "-d:include") then xdebug(n, "including %q", pname) end + local savedfdi = options.includedir + options.includedir = fdi -- saved index to implement include_next + processDirectives(options, macros, eliminateComments, joinLines, + yieldLines, fd:lines(), pname) + options.includedir = savedfdi + else + -- include file not found + if hasOption(options, "-Zpass") then + coroutine.yield(string.format("#include %s",tok), n) + else + xwarning(options, n, "include directive (%s) was unresolved", tok) + end + -- quirks + if knownIncludeQuirks[tok] then + processDirectives(options, macros, eliminateComments, joinLines, + yieldFromArray, knownIncludeQuirks[tok], n) + end + -- capture + if hasCaptureTable() then + addToCaptureTable{directive='include',name=tok, where=n} + end + end + end + -- conditionals + local function doConditionalBranch(execute) + checkDirective("endif") + while true do + li() + local r = checkDirective("endif") + if r == "else" or r == "elif" or r == "endif" then + return r + elseif execute then + processLine() + elseif r == "if" or r == "ifdef" or r == "ifndef" then + while doConditionalBranch(false) ~= "endif" do end + end + end + end + local function doConditional(result) + local r = doConditionalBranch(result) + if r == 'elif' and not result then + return processLine(true) + end + while r ~= "endif" do + r = doConditionalBranch(not result) + end + if hasOption(options,"-d:directives") then + xdebug(n, "directive: %s",s:gsub('^%s*','')) + end + end + local function doIfdef(ti) + ti() + xassert(isIdentifier(tok), options, n, "symbol expected after #%s", dirtok) + local result = macros[tok] + if ti() then xwarning(options, n, "garbage after #undef directive") end + if dirtok == 'ifndef' then result = not result end + doConditional(result) + end + local function doIf(ti) + -- magic macro for 'defined' + local nmacros = {} + setmetatable(nmacros,{__index=macros}) + nmacros['defined'] = function(ti) + local tok,n = ti() + if tok == '(' then tok = ti() + if ti() ~= ')' then tok = nil end end + if isIdentifier(tok) then + coroutine.yield(macros[tok] and "1" or "0", n) + else + coroutine.yield("defined", n) -- error + end + end + -- evaluate and branch + local pti = wrap(options, expandMacros, nmacros, yieldFromIterator, ti) + local result = evaluateCppExpression(options, pti, n) + doConditional(result ~= 0) + end + -- table of directives + local directives = { + ["else"] = doError, ["elif"] = doError, ["endif"] = doError, + ["pragma"] = doIgnore, ["ident"] = doIgnore, ["line"] = doIgnore, + ["error"] = doMessage, ["warning"] = doMessage, + ["if"] = doIf, ["ifdef"] = doIfdef, ["ifndef"] = doIfdef, + ["define"] = doDefine, ["undef"] = doUndef, + ["defmacro"] = doDefmacro, ["defrecursivemacro"] = doDefmacro, + ["endmacro"] = doError, + ["include"] = doInclude, ["include_next"] = doInclude, + } + -- process current line + processLine = function(okElif) + if type(s) == 'table' then + -- optimization for multiline macros: + -- When s is an an array of precomputed tokens, code is assumed. + coroutine.yield(s, n) + elseif not s:find("^%s*#") then + -- code + coroutine.yield(s, n) + elseif s:find("^%s*##") and hasOption(options, "-Zpass") then + -- pass + local ns = s:gsub("^(%s*)##","%1#") + coroutine.yield(ns, n) + else + if hasOption(options, "-d:directives") then + xdebug(n, "directive: %s",s:gsub('^%s*','')) + end + -- tokenize directive + local ti = wrap(options, tokenizeLine, s, n) + -- a token iterator that skips spaces unless told otherwise + local ti = function(keepSpaces) + tok = ti() + spc = isSpace(tok) + while not keepSpaces and isBlank(tok) do + tok = ti() + spc = spc or isSpace(tok) + end + return tok, n + end + -- start parsing directives + ti() + assert(tok=='#' or tok=='##') + if tok == '##' then + xwarning(options, n, "directive starts with ## without -Zpass") end + dirtok = ti() + if isIdentifier(tok) then + local f = directives[dirtok] + if okElif and dirtok == "elif" then f = doIf end + xassert(f, options, n, "unrecognized preprocessor directive #%s", tok) + f(ti) + elseif tok ~= nil then + xerror(options, n, "unrecognized preprocessor directive '#%s'", s:gsub("^%s*","")) + end + end + end + -- main loop + while s ~= nil do + processLine() + li() + end +end + + +-- This function yields initialization lines + +local function initialDefines(options) + -- cpp-extracted definitions + if hasOption(options, "-Zcppdef") then + local fd = io.popen("cpp -dM < /dev/null","r") + yieldLines(options, fd:lines(), "") + fd:close() + end + -- builtin definitions + local sb = { "#define __CPARSER__ 1" } + local function addDef(s,v) + sb[1+#sb] = string.format("#ifndef %s",s) + sb[1+#sb] = string.format("# define %s %s",s,v) + sb[1+#sb] = string.format("#endif") + end + addDef("__STDC__", "1") + local stdc = "199409L" + if options.dialect11 then stdc = "201112L" end + if options.dialect99 then stdc = "199901L" end + addDef("__STDC_VERSION__", stdc) + if options.dialectGnu then + addDef("__GNUC__", 4) + addDef("__GNUC_MINOR__", 2) + end + yieldLines(options, wrap(options, yieldFromArray, sb), "") + -- command line definitions + local sc = {} + for _,v in ipairs(options) do + local d + if v:find("^%-D(.*)=") then + d = v:gsub("^%-D%s*(.*)%s*=%s*(.-)%s*$","#define %1 %2") + elseif v:find("^%-D") then + d = v:gsub("^%-D%s*(.-)%s*$","#define %1 1") + elseif v:find("^%-U") then + d = v:gsub("^%-U%s*(.-)%s*$","#undef %1") + end + if d then sc[1+#sc] = d end + end + yieldLines(options, wrap(options, yieldFromArray, sc), "") +end + + +-- This function creates the initial macro directory + +local function initialMacros(options) + local macros = {} + -- magic macros + macros["__FILE__"] = function(_,_,n) + local f + if type(n) == 'string' then f=n:match("^[^:]*") end + coroutine.yield(string.format("%q", f or ""), n) + end + macros["__LINE__"] = function(_,_,n) + local d = n + if type(d) == 'string' then d=tonumber(d:match("%d*$")) end + coroutine.yield(string.format("%d", d or 0), n) + end + macros["__DATE__"] = function(_,_,n) + coroutine.yield(string.format("%q", os.date("%b %e %Y")), n) + end + macros["__TIME__"] = function(_,_,n) + coroutine.yield(string.format("%q", os.date("%T")), n) + end + --< @r-lyeh: add __COUNTER__ + macros["__COUNTER__"] = function(_,_,n) + counter = counter or 0 + counter = counter + 1 + coroutine.yield(string.format("%d", counter), n) + end + --< + -- initial macros + local li = wrap(options,processDirectives,macros,initialDefines) + for _ in li do end + -- return + return macros +end + + +-- This function prepares a string containing the definition of the +-- macro named in macro definition table , or nil if no +-- such definition exists. + +local function macroToString(macros, name) + local v = macros[name] + if type(v) == 'table' then + local dir = 'define' + if v.recursive and v.lines then + dir = 'defrecursivemacro' + elseif v.lines then + dir = 'defmacro' + end + local arr = {"#", dir, ' ', name } + if v.args then + arr[1+#arr] = '(' + for i,s in ipairs(v.args) do + if i ~= 1 then arr[1+#arr] = ',' end + if s == '__VA_ARGS__' then s = (v.nva or '') .. '...' end + arr[1+#arr] = s + end + arr[1+#arr] = ') ' + else + arr[1+#arr] = ' ' + end + for _,s in ipairs(v) do + arr[1+#arr] = s + end + if v.lines then + for i = 1, #v.lines, 2 do + local vl = v.lines[i] + arr[1+#arr] = '\n' + if type(vl)=='table' then vl=tableConcat(vl) end + arr[1+#arr] = vl:gsub('^%s?%s?',' '):gsub('^\n','') + end + arr[1+#arr] = '\n' + arr[1+#arr] = "#endmacro" + end + return tableConcat(arr) + end +end + +-- This function dumps all macros to file descriptor outputfile + +local function dumpMacros(macros, outputfile) + outputfile = outputfile or io.output() + assert(type(macros) == 'table') + assert(io.type(outputfile) == 'file') + for k,_ in pairs(macros) do + local s = macroToString(macros,k) + if s then outputfile:write(string.format("%s\n", s)) end + end +end + + + +-- A coroutine that filters out spaces, directives, and magic tokens + +local function filterSpaces(options, tokens, ...) + local ti = wrap(options, tokens, ...) + local tok,n = ti() + while tok do + -- skip directives + while isNewline(tok) do + tok,n = ti() + while isBlank(tok) do tok,n = ti() end + if tok == '#' then + while not isNewline(tok) do + tok, n = ti() + end + end + end + -- output nonspaces + if not isBlank(tok) then + coroutine.yield(tok, n) + end + tok,n = ti() + end +end + +-- This function takes a line iterator and an optional location prefix. +-- It returns a token iterator for the preprocessed tokens +-- and a table of macro definitions. + +local function cppTokenIterator(options, lines, prefix) + options = copyOptions(options) + prefix = prefix or "" + assert(type(options)=='table') + assert(type(lines)=='function') + assert(type(prefix)=='string') + local macros = initialMacros(options) + local ti = wrap(options, + filterSpaces, + expandMacros, macros, + tokenize, + processDirectives, macros, + eliminateComments, + joinLines, + yieldLines, lines, prefix) + return ti, macros +end + + +-- A coroutine that reconstructs lines from the preprocessed tokens + +local function preprocessedLines(options, tokens, ...) + local ti = wrap(options, tokens, ...) + local tok,n = ti() + while tok do + local curn = n + local curl = {} + if isNewline(tok) then + curn = n + curl[1+#curl] = tok:sub(2) + tok, n = ti() + end + while tok and not isNewline(tok) do + if not isMagic(tok) then curl[1+#curl] = tok end + tok, n = ti() + end + coroutine.yield(table.concat(curl), curn) + end +end + + +-- This function preprocesses file . +-- The optional argument specifies where to write the +-- preprocessed file and may be a string or a file descriptor. +-- The optional argument contains an array of option strings. +-- Note that option "-Zpass" is added, unless the option "-Znopass" is present. + +--< @r-lyeh: iterator+string methods +local function cppIterator(lines, outputfile, options, prefix) + -- handle optional arguments + prefix = prefix or '' + options = copyOptions(options) + outputfile = outputfile or "-" + assert(type(lines)=='function') + assert(type(options)=='table') + local closeoutputfile = false + if io.type(outputfile) ~= 'file' then + assert(type(outputfile) == 'string') + if outputfile == '-' then + outputfile = io.output() + else + closeoutputfile = true + outputfile = io.open(outputfile,"w") + end + end + assert(io.type(outputfile) == 'file') + -- makes option -Zpass on by default + if not hasOption(options,"-Znopass") then + options.hash["-Zpass"] = true + end + -- prepare iterator + local dM = hasOption(options, "-dM") + local macros = initialMacros(options) + local li = wrap(options, + preprocessedLines, + expandMacros, macros, + tokenize, + processDirectives, macros, + eliminateComments, + joinLines, + yieldLines, lines, prefix) + -- iterate, inserting line markers + local lm = hasOption(options,"-Zpass") and "line" or "" + local cf, cn + for s,n in li do + if not dM and s:find("[^%s]") then + local xf, xn + if type(n) == 'number' then + xn = n + elseif type(n) == 'string' then + xf, xn = n:match("^([^:]*).-(%d*)$") + xn = tonumber(xn) + end + if cf ~= xf or cn ~= xn then + cf, cn = xf, xn + if lm ~= "" then --< @r-lyeh: do not output empty #line directives: ie, `#line 19 "file.c"` -> `# 19 ""` + outputfile:write(string.format("#%s %d %q\n", lm, cn, cf)) + end + end + outputfile:write(s) + outputfile:write("\n") + cn = cn + 1 + end + end + if dM then + dumpMacros(macros, outputfile) + end + if closeoutputfile then outputfile:close() end +end + +local function cppString(str, outputfile, options) + assert(type(str)=='string') + + return cppIterator(str:gmatch("[^\r\n]+"), outputfile, options) +end + +local function cpp(filename, outputfile, options) + assert(type(filename)=='string') + + return cppIterator(io.lines(filename), outputfile, options, filename) +end +--< + +--------------------------------------------------- +--------------------------------------------------- +--------------------------------------------------- +-- EXPORTS + +cparser = {} + +cparser.cpp = cpp +cparser.cppTokenIterator = cppTokenIterator +cparser.macroToString = macroToString + +--< @r-lyeh: add string methods +cparser.cppString = cppString +cparser.parseString = parseString +--< + +return cparser diff --git a/tools/luajit_lcpp.lua b/tools/luajit_lcpp.lua deleted file mode 100644 index b0d9f19..0000000 --- a/tools/luajit_lcpp.lua +++ /dev/null @@ -1,1960 +0,0 @@ ----------------------------------------------------------------------------- ---## lcpp - a C-PreProcessor for Lua 5.1 and LuaJIT ffi integration --- --- Copyright (C) 2012-2014 Michael Schmoock --- ---### Links --- * GitHub page: [http://github.com/willsteel/lcpp](http://github.com/willsteel/lcpp) --- * Project page: [http://lcpp.schmoock.net](http://lcpp.schmoock.net) --- * Lua: [http://www.lua.org](http://www.lua.org) --- * LuaJIT: [http://luajit.org](http://luajit.org) --- * Sponsored by: [http://mmbbq.org](http://mmbbq.org) --- --- It can be used to pre-process LuaJIT ffi C header file input. --- It can also be used to preprocess any other code (i.e. Lua itself) --- --- git clone https://github.com/willsteel/lcpp.git ----------------------------------------------------------------------------- ---## USAGE --- -- load lcpp --- local lcpp = require("lcpp") --- --- -- use LuaJIT ffi and lcpp to parse cpp code --- local ffi = require("ffi") --- ffi.cdef("#include ") --- --- -- use lcpp manually but add some predefines --- local lcpp = require("lcpp"); --- local out = lcpp.compileFile("your_header.h", {UNICODE=1}); --- print(out); --- --- -- compile some input manually --- local out = lcpp.compile([[ --- #include "myheader.h" --- #define MAXPATH 260 --- typedef struct somestruct_t { --- void* base; --- size_t size; --- wchar_t path[MAXPATH]; --- } t_exe; --- ]]) --- -- the result should be like this --- out == [[ --- // --- typedef struct somestruct_t { --- void* base; --- size_t size; --- wchar_t path[260]; --- } t_exe; --- ]] --- --- -- access lcpp defines dynamically (i.e. if used with ffi) --- local ffi = require("ffi") --- local lcpp = require("lcpp") --- ffi.cdef("#include ") --- =ffi.lcpp_defs.YOUR_DEFINE --- --- ---## This CPPs BNF: --- RULES: --- CODE := {LINE} --- LINE := {STUFF NEWML} STUFF NEWL --- STUFF := DIRECTIVE | IGNORED_CONTENT --- DIRECTIVE := OPTSPACES CMD OPTSPACES DIRECTIVE_NAME WHITESPACES DIRECTIVE_CONTENT WHITESPACES NEWL --- --- LEAVES: --- NEWL := "\n" --- NEWL_ESC := "\\n" --- WHITESPACES := "[ \t]+" --- OPTSPACES := "[ \t]*" --- COMMENT := "//(.-)$" --- MLCOMMENT := "/[*](.-)[*]/" --- IGNORED_CONTENT := "[^#].*" --- CMD := "#" --- DIRECTIVE_NAME := "include"|"define"|"undef"|"if"|"else"|"elif"|"else if"|"endif"|"ifdef"|"ifndef"|"pragma" --- DIRECTIVE_CONTENT := ".*?" --- ---## TODOs: --- - lcpp.LCPP_LUA for: load, loadfile --- ---## License (MIT) --- ----------------------------------------------------------------------------- --- Permission is hereby granted, free of charge, to any person obtaining a copy --- of this software and associated documentation files (the "Software"), to deal --- in the Software without restriction, including without limitation the rights --- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell --- copies of the Software, and to permit persons to whom the Software is --- furnished to do so, subject to the following conditions: --- --- The above copyright notice and this permission notice shall be included in --- all copies or substantial portions of the Software. --- --- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR --- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, --- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE --- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER --- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, --- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN --- THE SOFTWARE. --- --- MIT license: http://www.opensource.org/licenses/mit-license.php --- ----------------------------------------------------------------------------- --- --- @module lcpp -local lcpp = {} --- check bit is avail or not -local ok, bit = pcall(require, 'bit') -if not ok then -bit = { - lshift = function (x, y) - if y < 0 then return bit.rshift(x,-y) end - return (x * 2^y) % (2^32) - end, - rshift = function (x, y) - if y < 0 then return bit.lshift(x,-y) end - return math.floor(x % (2^32) / (2^y)) - end, - bxor = function (x, y) - -- from http://lua-users.org/wiki/BitUtils - local z = 0 - for i = 0, 31 do - if (x % 2 == 0) then -- x had a '0' in bit i - if ( y % 2 == 1) then -- y had a '1' in bit i - y = y - 1 - z = z + 2 ^ i -- set bit i of z to '1' - end - else -- x had a '1' in bit i - x = x - 1 - if (y % 2 == 0) then -- y had a '0' in bit i - z = z + 2 ^ i -- set bit i of z to '1' - else - y = y - 1 - end - end - y = y / 2 - x = x / 2 - end - return z - end, - bnot = function (x) - -- if word size is not defined, I think it better than 0xFFFFFFFF - x. - return -1 - x - end, - band = function (x, y) - return ((x + y) - bit.bxor(x, y)) / 2 - end, - bor = function (x, y) - return bit.bnot(bit.band(bit.bnot(x), bit.bnot(y))) - end, -} -end - --- CONFIG -lcpp.LCPP_LUA = false -- whether to use lcpp to preprocess Lua code (load, loadfile, loadstring...) -lcpp.LCPP_FFI = true -- whether to use lcpp as LuaJIT ffi PreProcessor (if used in luaJIT) -lcpp.LCPP_TEST = false -- whether to run lcpp unit tests when loading lcpp module -lcpp.ENV = {} -- static predefines (env-like) -lcpp.FAST = false -- perf. tweaks when enabled. con: breaks minor stuff like __LINE__ macros -lcpp.DEBUG = false - --- PREDEFINES -local __FILE__ = "__FILE__" -local __LINE__ = "__LINE__" -local __DATE__ = "__DATE__" -local __TIME__ = "__TIME__" -local __LCPP_INDENT__ = "__LCPP_INDENT__" - --- BNF LEAVES -local ENDL = "$" -local STARTL = "^" -local NEWL = "\n" -local NEWL_BYTE = NEWL:byte(1) -local NEWL_ESC = "\\" -local NEWML = "\\\n" -local CMD = "#" -local CMD_BYTE = CMD:byte(1) -local COMMENT = "^(.-)//.-$" -local MLCOMMENT = "/[*].-[*]/" -local WHITESPACES = "%s+" -local OPTSPACES = "%s*" -local IDENTIFIER = "[_%a][_%w]*" -local NOIDENTIFIER = "[^%w_]+" -local FILENAME = "[0-9a-zA-Z.%-_/\\]+" -local TEXT = ".+" -local STRINGIFY = "#" -local STRINGIFY_BYTE = STRINGIFY:byte(1) -local STRING_LITERAL = ".*" - --- BNF WORDS -local _INCLUDE = "include" -local _INCLUDE_NEXT = "include_next" -local _DEFINE = "define" -local _IFDEF = "ifdef" -local _IFNDEF = "ifndef" -local _ENDIF = "endif" -local _UNDEF = "undef" -local _IF = "if" -local _ELSE = "else" -local _ELIF = "elif" -local _NOT = "!" -local _ERROR = "error" -local _WARNING = "warning" -local _PRAGMA = "pragma" - --- BNF RULES -local INCLUDE = STARTL.._INCLUDE..WHITESPACES.."[<]("..FILENAME..")[>]"..OPTSPACES..ENDL -local LOCAL_INCLUDE = STARTL.._INCLUDE..WHITESPACES.."[\"]("..FILENAME..")[\"]"..OPTSPACES..ENDL -local INCLUDE_NEXT = STARTL.._INCLUDE_NEXT..WHITESPACES.."[\"<]("..FILENAME..")[\">]"..OPTSPACES..ENDL -local DEFINE = STARTL.._DEFINE -local IFDEF = STARTL.._IFDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL -local IFNDEF = STARTL.._IFNDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL -local ENDIF = STARTL.._ENDIF..OPTSPACES..ENDL -local UNDEF = STARTL.._UNDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL -local IF = STARTL.._IF..WHITESPACES.."(.*)"..ENDL -local ELSE = STARTL.._ELSE..OPTSPACES..ENDL -local ELIF = STARTL.._ELIF..WHITESPACES.."(.*)"..ENDL -local ELSEIF = STARTL.._ELSE..WHITESPACES.._IF..WHITESPACES.."(.*)"..ENDL -local ERROR = STARTL.._ERROR..WHITESPACES.."("..TEXT..")"..OPTSPACES..ENDL -local WARNING = STARTL.._WARNING..WHITESPACES.."("..TEXT..")"..OPTSPACES..ENDL -local ERROR_NOTEXT = STARTL.._ERROR..OPTSPACES..ENDL --> not required when we have POSIX regex -local PRAGMA = STARTL.._PRAGMA - --- speedups -local TRUEMACRO = STARTL.."("..IDENTIFIER..")%s*$" -local REPLMACRO = STARTL.."("..IDENTIFIER..")"..WHITESPACES.."(.+)$" -local FUNCMACRO = STARTL.."("..IDENTIFIER..")%(([_%s%w,]*)%)%s*(.*)" - - --- ------------ --- LOCAL UTILS --- ------------ -lcpp.STATE = {lineno = 0} -- current state for debugging the last operation -local function error(msg) _G.print(debug.traceback()); _G.error(string.format("lcpp ERR [%04i] %s", lcpp.STATE.lineno, msg)) end -local function print(msg) _G.print(string.format("//lcpp INF [%04i] %s", lcpp.STATE.lineno, msg)) end - --- splits a string using a pattern into a table of substrings -local function gsplit(str, pat) - local function _split(str, pat) - local t = {} -- NOTE: use {n = 0} in Lua-5.0 - local fpat = "(.-)"..pat - local last_end = 1 - local s, e, cap = str:find(fpat, 1) - while s do - if s ~= 1 or cap ~= "" then - coroutine.yield(cap) - end - last_end = e + 1 - s, e, cap = str:find(fpat, last_end) - end - if last_end <= #str then - cap = str:sub(last_end) - coroutine.yield(cap) - end - end - return coroutine.wrap(function() _split(str, pat) end) -end -local function split(str, pat) - local t = {} - for str in gsplit(str, pat) do table.insert(t, str) end - return t -end - --- Checks whether a string starts with a given substring --- offset is optional -local function strsw(str, pat, offset) - if not str then return false end - if not offset then offset = 0 end - return string.sub(str, 1+offset, string.len(pat)+offset) == pat -end - --- Checks whether a string ends with a given substring -local function strew(str, pat) - if not str then return false end - return pat=='' or string.sub(str,-string.len(pat)) == pat -end - --- string trim12 from lua wiki -local function trim(str) - local from = str:match"^%s*()" - return from > #str and "" or str:match(".*%S", from) -end - --- returns the number of string occurrences -local function findn(input, what) - local count = 0 - local offset = 0 - local _ - while true do - _, offset = string.find(input, what, offset+1, true) - if not offset then return count end - count = count + 1 - end -end - --- C literal string concatenation -local function concatStringLiteral(input) - -- screener does remove multiline definition, so just check ".*"%s*".*" pattern - return input:gsub("\"("..STRING_LITERAL..")\""..OPTSPACES.."\"("..STRING_LITERAL..")\"", "\"%1%2\"") -end - --- c style boolean check (thus, 0 will be false) -local function CBoolean(value) - return value and (value ~= 0) -end - --- eval with c style number parse (UL, LL, L) -local function CEval(expr) - local ok, r = pcall(loadstring, "return " .. parseCInteger(expr)) - if ok and r then - return r() - else - error(r) - end -end - --- a lightweight and flexible tokenizer -local function _tokenizer(str, setup) - local defsetup = { - -- EXAMPLE patterns have to be pretended with "^" for the tokenizer - ["identifier"] = '^[_%a][_%w]*', - ["number"] = '^[%+%-]?%d+[%.]?%d*[UL]*', - ["ignore"] = '^%s+', - ["string"] = true, - ["keywords"] = { - -- ["NAME"] = '^pattern', - -- ... - }, - } - if not setup then - setup = defsetup - end - setup.identifier = setup.identifier or defsetup.identifier - setup.number = setup.number or defsetup.number - setup.ignore = setup.ignore or defsetup.ignore - if nil == setup.string then setup.string = true end - setup.keywords = setup.keywords or {} - - local strlen = #str - local i = 1 - local i1, i2 - local keyword - - local function find(pat) - i1, i2 = str:find(pat,i) - return i1 ~= nil - end - - local function cut() - return str:sub(i, i2) - end - - local findKeyword - if setup.keywords_order then - findKeyword = function () - for _, name in ipairs(setup.keywords_order) do - assert(setup.keywords[name]) - local pat = setup.keywords[name] - local result = find(pat) - if result then - keyword = name - return true - end - end - end - else - findKeyword = function () - for name, pat in pairs(setup.keywords) do - local result = find(pat) - if result then - keyword = name - return true - end - end - end - end - - while true do - if i > strlen then return 'eof', nil, strlen, strlen end - if findKeyword() then - coroutine.yield(keyword, cut(), i1, i2) - elseif find(setup.ignore) then - coroutine.yield("ignore", cut(), i1, i2) - elseif find(setup.number) then - coroutine.yield('number', tonumber(cut()), i1, i2) - elseif find(setup.identifier) then - coroutine.yield('identifier', cut(), i1, i2) - elseif setup.string and (find('^"[^"]*"') or find("^'[^']*'")) then - -- strip the quotes - coroutine.yield('string', cut():sub(2,-2), i1, i2) - else -- any other unknown character - i1 = i - i2 = i - coroutine.yield('unknown', cut(), i1, i2) - end - i = i2+1 - end -end -local function tokenizer(str, setup) - return coroutine.wrap(function() _tokenizer(str, setup) end) -end - - --- ------------ --- PARSER --- ------------ - -local LCPP_TOKENIZE_COMMENT = { - string = false, - keywords = { - MLCOMMENT = "^/%*.-%*/", - SLCOMMENT = "^//.-\n", - STRING_LITERAL = '^"[^"]*"', - }, -} --- hint: LuaJIT ffi does not rely on us to remove the comments, but maybe other usecases -local function removeComments(input) - local out = {} - for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_COMMENT) do - if k == "MLCOMMENT" then - local newlineCount = findn(input:sub(start, end_), "\n") - local newlines = string.rep("\n", newlineCount) - table.insert(out, newlines) - elseif k == "SLCOMMENT" then - table.insert(out, "\n") - else - table.insert(out, input:sub(start, end_)) - end - end - return table.concat(out) -end - --- C style number parse (UL, LL, L) and (octet, hex, binary) -local LCPP_TOKENIZE_INTEGER = { - string = false, - keywords_order = { - "STRING_LITERAL", - "CHAR_LITERAL", - "HEX_LITERAL", - "BIN_LITERAL", - "OCT_LITERAL", - "FPNUM_LITERAL", - "NUMBER_LITERAL", - }, - keywords = { - STRING_LITERAL = '^"[^"]*"', - CHAR_LITERAL = "^L'.*'", - HEX_LITERAL = '^[%+%-]?%s*0x[a-fA-F%d]+[UL]*', - BIN_LITERAL = '^[%+%-]?%s*0b%d+[UL]*', - OCT_LITERAL = '^[%+%-]?%s*0%d+[UL]*', - FPNUM_LITERAL = '^[%+%-]?%s*%d+[%.]?%d*e[%+%-]%d*', - NUMBER_LITERAL = '^[%+%-]?%s*%d+[%.]?%d*[UL]+', - }, -} -local function parseCInteger(input) - -- print('parseCInteger:input:' .. input) - local out = {} - local unary - for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_INTEGER) do - -- print('parseCInteger:' .. k .. "|" .. v) - if k == "CHAR_LITERAL" then - table.insert(out, tostring(string.byte(loadstring("return \"" .. v:gsub("^L%'(.+)%'", "%1") .. "\"")()))) - elseif k == "HEX_LITERAL" then - unary, v = v:match('([%+%-]?)0x([a-fA-F%d]+)[UL]*') - local n = tonumber(v, 16) - table.insert(out, unary..tostring(n)) - elseif k == "NUMBER_LITERAL" then - v = v:match('([^UL]+)[UL]+') - table.insert(out, v) - elseif k == "BIN_LITERAL" then - unary, v = v:match('([%+%-]?)0b([01]+)[UL]*') - local n = tonumber(v, 2) - table.insert(out, unary..tostring(n)) - elseif k == "OCT_LITERAL" then - unary, v = v:match('([%+%-]?)(0%d+)[UL]*') - local n = tonumber(v, 8) - table.insert(out, unary..tostring(n)) - else - table.insert(out, input:sub(start, end_)) - end - end - local str = table.concat(out) - -- print('parseCInteger:result:'..str) - return str -end - --- screener: revmoce comments, trim, ml concat... --- it only splits to cpp input lines and removes comments. it does not tokenize. -local function screener(input) - local function _screener(input) - input = removeComments(input) - - -- concat mulit-line input. - local count = 1 - while count > 0 do input, count = string.gsub(input, "^(.-)\\\n(.-)$", "%1 %2\n") end - - -- trim and join blocks not starting with "#" - local buffer = {} - for line in gsplit(input, NEWL) do - --print('newline:'..line) - line = trim(line) - if #line > 0 then - if line:byte(1) == CMD_BYTE then - line = line:gsub("#%s*(.*)", "#%1") -- remove optinal whitespaces after "#". reduce triming later. - if #buffer > 0 then - coroutine.yield(table.concat(buffer, NEWL)) - buffer = {} - end - coroutine.yield(line) - else - if lcpp.FAST then - table.insert(buffer, line) - else - coroutine.yield(line) - end - end - elseif not lcpp.FAST then - coroutine.yield(line) - end - end - if #buffer > 0 then - coroutine.yield(table.concat(buffer, NEWL)) - end - end - - return coroutine.wrap(function() _screener(input) end) -end - --- apply currently known macros to input (and returns it) -local LCPP_TOKENIZE_APPLY_MACRO = { - keywords = { - DEFINED = "^defined%s*%(%s*"..IDENTIFIER.."%s*%)" , - }, -} -local function apply(state, input) - while true do - local out = {} - local functions = {} - local expand - - for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_APPLY_MACRO) do - -- print('tokenize:'..tostring(k).."|"..tostring(v)) - if k == "identifier" then - local repl = v - local macro = state.defines[v] - if macro then - if type(macro) == "boolean" then - repl = "" - expand = true - elseif type(macro) == "string" then - repl = macro - expand = (repl ~= v) - elseif type(macro) == "number" then - repl = tostring(macro) - expand = (repl ~= v) - elseif type(macro) == "function" then - local decl,cnt = input:sub(start):gsub("^[_%a][_%w]*%s*%b()", "%1") - -- print('matching:'..input.."|"..decl.."|"..cnt) - if cnt > 0 then - repl = macro(decl) - -- print("d&r:"..decl.."|"..repl) - expand = true - table.insert(out, repl) - table.insert(out, input:sub(end_ + #decl)) - break - else - if input:sub(start):find("^[_%a][_%w]*%s*%(") then - -- that is part of functional macro declaration. - -- print(v ..': cannot replace:<'..input..'> read more line') - return input,true - else - -- on macro name is also used as the symbol of some C declaration - -- (e.g. /usr/include/spawn.h, /usr/include/sys/select.h on centos 6.4) - -- no need to preprocess. - print(v .. ': macro name but used as C declaration in:' .. input) - end - end - end - end - table.insert(out, repl) - elseif k == "DEFINED" then - table.insert(out, input:sub(start, end_)) - else - table.insert(out, input:sub(start, end_)) - end - end - input = table.concat(out) - if not expand then - break - end - end - - -- C liberal string concatenation, processing U,L,UL,LL - return parseCInteger(concatStringLiteral(input)),false -end - --- processes an input line. called from lcpp doWork loop -local function processLine(state, line) - if not line or #line == 0 then return line end - local cmd = nil - if line:byte(1) == CMD_BYTE then cmd = line:sub(2) end - -- print("process:"..line)--.."|"..tostring(state:skip())) - - --[[ IF/THEN/ELSE STRUCTURAL BLOCKS ]]-- - if cmd then - local ifdef = cmd:match(IFDEF) - local ifexp = cmd:match(IF) - local ifndef = cmd:match(IFNDEF) - local elif = cmd:match(ELIF) - local elseif_ = cmd:match(ELSEIF) - local else_ = cmd:match(ELSE) - local endif = cmd:match(ENDIF) - local struct = ifdef or ifexp or ifndef or elif or elseif_ or else_ or endif - - if struct then - local skip = state:skip() - if ifdef then state:openBlock(state:defined(ifdef)) end - -- if skipped, it may have undefined expression. so not parse them - if ifexp then state:openBlock(skip and true or CBoolean(state:parseExpr(ifexp))) end - if ifndef then state:openBlock(not state:defined(ifndef)) end - if elif then state:elseBlock((skip and skip < #state.stack) and true or CBoolean(state:parseExpr(elif))) end - if elseif_ then state:elseBlock((skip and skip < #state.stack) and true or CBoolean(state:parseExpr(elseif_))) end - if else_ then state:elseBlock(true) end - if endif then state:closeBlock() end - return -- remove structural directives - end - end - - - --[[ SKIPPING ]]-- - if state:skip() then - -- print('skip:' .. line) - return - end - - - --[[ READ NEW DIRECTIVES ]]-- - if cmd then - -- handle #undef ... - local key = cmd:match(UNDEF) - if type(key) == "string" then - state:undefine(key) - return - end - - -- read "#define >FooBar...<" directives - if cmd:match(DEFINE) then - local define = trim(cmd:sub(DEFINE:len()+1)) - local macroname, replacement - - -- simple "true" defines - macroname = define:match(TRUEMACRO) - if macroname then - state:define(macroname, true) - else - - -- replace macro defines - macroname, replacement = define:match(REPLMACRO) - if macroname and replacement then - state:define(macroname, replacement) - else - - -- read functional macros - macroname, replacement, source = state:parseFunction(define) - if macroname and replacement then - -- add original text for definition to check identify - state:define(macroname, replacement, false, source) - end - - end - end - - return - end - - -- handle #include ... - local filename = cmd:match(INCLUDE) - if filename then - return state:includeFile(filename) - end - local filename = cmd:match(LOCAL_INCLUDE) - if filename then - return state:includeFile(filename, false, true) - end - local filename = cmd:match(INCLUDE_NEXT) - if filename then - --print("include_next:"..filename) - return state:includeFile(filename, true) - end - - -- ignore, because we dont have any pragma directives yet - if cmd:match(PRAGMA) then return end - - -- handle #error - local errMsg = cmd:match(ERROR) - local errNoTxt = cmd:match(ERROR_NOTEXT) - local warnMsg = cmd:match(WARNING) - if errMsg then error(errMsg) end - if errNoTxt then error("") end - if warnMsg then - print(warnMsg) - return - end - - -- abort on unknown keywords - error("unknown directive: "..line) - end - - if state.incompleteLine then - --print('merge with incompleteLine:'..state.incompleteLine) - line = (state.incompleteLine .. line) - state.incompleteLine = nil - end - - - --[[ APPLY MACROS ]]-- - -- print(line) - local _line,more = state:apply(line); - -- print('endprocess:'.._line) - if more then - state.incompleteLine = line - return "" - else - return _line - end - - return line -end - -local function doWork(state) - local function _doWork(state) - if not state:defined(__FILE__) then state:define(__FILE__, "", true) end - local oldIndent = state:getIndent() - while true do - local input = state:getLine() - if not input then break end - local output = processLine(state, input) - if not lcpp.FAST and not output then output = "" end -- output empty skipped lines - if lcpp.DEBUG then output = output.." -- "..input end -- input as comment when DEBUG - if output then coroutine.yield(output) end - end - if (oldIndent ~= state:getIndent()) then error("indentation level must be balanced within a file. was:"..oldIndent.." is:"..state:getIndent()) end - end - return coroutine.wrap(function() _doWork(state) end) -end - -local function includeFile(state, filename, next, _local) - local result, result_state = lcpp.compileFile(filename, state.defines, state.macro_sources, next, _local) - -- now, we take the define table of the sub file for further processing - state.defines = result_state.defines - -- and return the compiled result - return result -end - --- sets a global define -local function define(state, key, value, override, macro_source) - --print("define:"..key.." type:"..tostring(value).." value:"..tostring(pval)) - if value and not override then - if type(value) == 'function' then - assert(macro_source, "macro source should specify to check identity") - local pval = state.macro_sources[key] - if pval and (pval ~= macro_source) then error("already defined: "..key) end - state.macro_sources[key] = macro_source - else - local pval = state.defines[key] - if pval and (pval ~= value) then error("already defined: "..key) end - end - end - state.defines[key] = state:prepareMacro(value) -end - --- parses CPP exressions --- i.e.: #if !defined(_UNICODE) && !defined(UNICODE) --- ---BNF: --- EXPR -> (BRACKET_OPEN)(EXPR)(BRACKET_CLOSE) --- EXPR -> (EXPR)(OR)(EXPR) --- EXPR -> (EXPR)(AND)(EXPR) --- EXPR -> (NOT)(EXPR) --- EXPR -> (FUNCTION) --- FUNCTION -> (IDENTIFIER)(BRACKET_OPEN)(ARGS)(BRACKET_CLOSE) --- ARGS -> ((IDENTIFIER)[(COMMA)(IDENTIFIER)])? ---LEAVES: --- IGNORE -> " \t" --- BRACKET_OPEN -> "(" --- BRACKET_CLOSE -> ")" --- OR -> "||" --- AND -> "&&" --- NOT -> "!" --- IDENTIFIER -> "[0-9a-zA-Z_]" --- - -local LCPP_TOKENIZE_MACRO = { - string = true, - keywords_order = { - "CONCAT", - "SPACE", - }, - keywords = { - CONCAT = "^%s*##%s*", - SPACE = "^%s", - }, -} -local LCPP_TOKENIZE_MACRO_ARGS = { - string = true, - keywords_order = { - "STRING_LITERAL", - "PARENTHESE", - "FUNCTIONAL", - "ARGS", - "SINGLE_CHARACTER_ARGS", - "COMMA", - }, - keywords = { - PARENTHESE = "^%s*%b()", - FUNCTIONAL = "^".. IDENTIFIER .. "%s*%b()", - STRING_LITERAL = '^"[^"]*"', - ARGS = "^[^,%s][^,]*[^,%s]", - SINGLE_CHARACTER_ARGS = "^[^,%s]", - COMMA = "^,", - }, -} -local LCPP_TOKENIZE_EXPR = { - string = false, - keywords_order = { - "DEFINED", - "FUNCTIONAL_MACRO", - "BROPEN", - "BRCLOSE", - - "TENARY_START", - "TENARY_MIDDLE", - -- binary operators - "EQUAL", - "NOT_EQUAL", - "AND", - "OR", - "BAND", - "BOR", - "BXOR", - "PLUS", - "MINUS", - "MULTIPLY", - "DIV", - "MOD", - "LTE", - "MTE", - "LSHIFT", - "RSHIFT", - "LT", - "MT", - -- unary operator - "NOT", - "BNOT", - -- literal - "STRING_LITERAL", - "CHAR_LITERAL", - "HEX_LITERAL", - "FPNUM_LITERAL", - "NUMBER_LITERAL", - }, - keywords = { - DEFINED = '^defined', - FUNCTIONAL_MACRO = '^' .. IDENTIFIER .. "%s*%b()", - BROPEN = '^[(]', - BRCLOSE = '^[)]', - - TENARY_START = '^%?', - TENARY_MIDDLE = '^%:', - - EQUAL = '^==', - NOT_EQUAL = '^!=', - AND = '^&&', - OR = '^||', - BAND = '^&', - BOR = '^|', - BXOR = '^%^', - PLUS = '^%+', - MINUS = '^%-', - MULTIPLY = '^%*', - DIV = '^%/', - MOD = '^%%', - LTE = '^<=', - MTE = '^>=', - LSHIFT = '^<<', - RSHIFT = '^>>', - LT = '^<', - MT = '^>', - - NOT = '^!', - BNOT = '^~', - - STRING_LITERAL = '^L?"[^"]*"', - CHAR_LITERAL = "^L?'.*'", - HEX_LITERAL = '^[%+%-]?0?x[a-fA-F%d]+[UL]*', - FPNUM_LITERAL = '^[%+%-]?%d+[%.]?%d*e[%+%-]%d*', - NUMBER_LITERAL = '^[%+%-]?0?b?%d+[%.]?%d*[UL]*', - }, -} - -local function parseDefined(state, input) - local result = false - local bropen = false - local brclose = false - local ident = nil - - for key, value in input do - if key == "BROPEN" then - bropen = true - end - if key == "identifier" then - ident = value - if not bropen then break end - end - if key == "BRCLOSE" and ident then - brclose = true - break - end - end - - -- wiht and w/o brackets allowed - if ident and ((bropen and brclose) or (not bropen and not brclose)) then - return state:defined(ident) - end - - error("expression parse error: defined(ident)") -end - - ---[[ -order : smaller is higher priority -1 () [] -> . -2 ! ~ - + * & sizeof type cast ++ -- -3 * / % -4 + - -5 << >> -6 < <= > >= -7 == != -8 & -9 ^ -10 | -11 && -12 || -13 ?: = += -= *= /= %= &= |= ^= <<= >>= -14 , -]] -local combination_order = function (op, unary) - if unary then - if op == '-' or op == '!' or op == '~' then - return 2 - else - assert(false, 'unsupported unary operator:' .. op) - end - else - if op == '*' or op == '/' or op == '%' then - return 3 - elseif op == '+' or op == '-' then - return 4 - elseif op == '>>' or op == '<<' then - return 5 - elseif op == '<' or op == '>' or op == '<=' or op == '>=' then - return 6 - elseif op == '==' or op == '!=' then - return 7 - elseif op == '&' then - return 8 - elseif op == '^' then - return 9 - elseif op == '|' then - return 10 - elseif op == '&&' then - return 11 - elseif op == '||' then - return 12 - elseif op == '?' or op == ':' then - return 13 - else - assert(false, 'unsupported operator:' .. op) - end - end -end - -local evaluate -evaluate = function (node) - if not node.op then -- leaf node or leaf node with unary operators - local v = node.v - if node.uops then - for _, uop in ipairs(node.uops) do - -- print('apply uop:'..uop.."|"..tostring(v)) - if uop == '-' then - v = -v - elseif uop == '!' then - v = (not v) - elseif uop == '~' then - v = bit.bnot(v) - else - assert(false, 'invalid uop:' .. tostring(uop)) - end - end - end - -- print('after apply:'..tostring(v)) - return v - end - -- print(node.op..':'..tostring(node.l.v or node.l.op).."("..type(node.l.v)..")|"..tostring(node.r.v or node.r.op).."("..type(node.r.v)..")") - if node.op == '+' then -- binary operators - return (evaluate(node.l) + evaluate(node.r)) - elseif node.op == '-' then - return (evaluate(node.l) - evaluate(node.r)) - elseif node.op == '*' then - return (evaluate(node.l) * evaluate(node.r)) - elseif node.op == '/' then - return (evaluate(node.l) / evaluate(node.r)) - elseif node.op == '%' then - return (evaluate(node.l) % evaluate(node.r)) - elseif node.op == '==' then - return (evaluate(node.l) == evaluate(node.r)) - elseif node.op == '!=' then - return (evaluate(node.l) ~= evaluate(node.r)) - elseif node.op == '<<' then - return bit.lshift(evaluate(node.l), evaluate(node.r)) - elseif node.op == '>>' then - return bit.rshift(evaluate(node.l), evaluate(node.r)) - elseif node.op == '&&' then - return (CBoolean(evaluate(node.l)) and CBoolean(evaluate(node.r))) - elseif node.op == '||' then - return (CBoolean(evaluate(node.l)) or CBoolean(evaluate(node.r))) - elseif node.op == '&' then - return bit.band(evaluate(node.l), evaluate(node.r)) - elseif node.op == '|' then - return bit.bor(evaluate(node.l), evaluate(node.r)) - elseif node.op == '^' then - return bit.bxor(evaluate(node.l), evaluate(node.r)) - elseif node.op == '<=' then - return (evaluate(node.l) <= evaluate(node.r)) - elseif node.op == '>=' then - return (evaluate(node.l) >= evaluate(node.r)) - elseif node.op == '<' then - return (evaluate(node.l) < evaluate(node.r)) - elseif node.op == '>' then - return (evaluate(node.l) > evaluate(node.r)) - else - assert(false, 'invalid op:' .. tostring(node.op)) - end -end - -local function setValue(node, v) - -- print('setValue:' .. tostring(v).."|"..tostring(node.uops))-- .. "\t" .. debug.traceback()) - if not node.op then - assert(not node.v, debug.traceback()) - node.v = v - else - assert(node.l and (not node.r)) - node.r = {v = v, uops = node.uops} - end -end - -local function setUnaryOp(node, uop) - -- print('setUnaryOp:' .. tostring(uop))-- .. "\t" .. debug.traceback()) - if not node.uops then node.uops = {} end - table.insert(node.uops, 1, uop) -end - -local function parseExpr(state, input) - local node = {} - local root = node - -- first call gets string input. rest uses tokenizer - if type(input) == "string" then - -- print('parse:' .. input) - input = tokenizer(input, LCPP_TOKENIZE_EXPR) - end - - for type, value in input do - -- print("type:"..type.." value:"..value) - -- unary operator - if type == "NOT" or - type == "BNOT" then - setUnaryOp(node, value) - end - if type == "BROPEN" then - setValue(node, state:parseExpr(input)) - end - if type == "BRCLOSE" then - --print('BRCLOSE:' .. tostring(result)) - break - end - if type == "STRING_LITERAL" then - setValue(node, value:sub(value[1] == 'L' and 3 or 2,-2)) - end - if type == "NUMBER_LITERAL" or type == "HEX_LITERAL" or type == "FPNUM_LITERAL" or type == "CHAR_LITERAL" then - setValue(node, tonumber(parseCInteger(value))) - end - -- tenary operator - -- tenary has lowest priority, so any other operation can be calculate now. - if type == "TENARY_START" then - local l = state:parseExpr(input) - local r = state:parseExpr(input) - if evaluate(root) then - return l - else - return r - end - end - if type == "TENARY_MIDDLE" then - break - end - -- binary operator - if type == "EQUAL" or - type == "NOT_EQUAL" or - type == "AND" or - type == "OR" or - type == "BAND" or - type == "BOR" or - type == "BXOR" or - type == "PLUS" or - type == "MINUS" or - type == "MULTIPLY" or - type == "DIV" or - type == "MOD" or - type == "LTE" or - type == "MTE" or - type == "LSHIFT" or - type == "RSHIFT" or - type == "LT" or - type == "MT" then - if node.op then - if not node.r then -- during parse right operand : uop1 uop2 ... uopN operand1 op1 uop(N+1) uop(N+2) ... [uop(N+K)] - assert(type == "MINUS", "error: operators come consequently: " .. tostring(node.op) .. " and " .. tostring(value)) - -- unary operater after binary operator - setUnaryOp(node, value) - else -- uop1 uop2 ... uopN operand1 op1 uop(N+1) uop(N+2) ... uop(N+M) operand2 [op2] - -- print("operator processing:" .. tostring(node.op) .. "|" .. value .. "|" .. tostring(node.l) .. "|" .. tostring(node.r)) - local tmp = node - while tmp do - -- print('compare ' .. value..' and ' .. tmp.op) - if combination_order(tmp.op) > combination_order(value) then - -- print(value..' is stronger than ' .. tmp.op) - break - end - tmp = tmp.parent - end - if tmp then - node = { - op = value, - l = tmp.r, - parent = tmp - } - tmp.r.parent = node - tmp.r = node - else - node = { - op = value, - l = root, - } - root.parent = node - root = node - end - end - elseif node.v ~= nil then -- uop1 uop2 ... uopN operand1 [op] - local devided - if node.uops then - for _, uop in ipairs(node.uops) do - if combination_order(uop, true) > combination_order(value) then - -- there is a binary operator which has stronger than any of the unary - devided = uop - end - end - end - if devided then - assert(false, "TODO: can we do something about this case??:"..value.." is stronger than "..devided) - else - node.l = { v = node.v, uops = node.uops } - node.v = nil - node.uops = nil - node.op = value - end - else -- unary operator : uop1 uop2 ... [uopN] - assert(type == "MINUS", "error: invalid unary operator:" .. value) - setUnaryOp(node, value) - end - end - if type == "DEFINED" then - setValue(node, parseDefined(state, input)) - elseif type == "identifier" or type == "FUNCTIONAL_MACRO" then - -- print('ident:' .. value) - local eval = state:apply(value) - -- print('apply result ' .. eval .. "|" .. tostring(unprocessed)) - if eval ~= value then - eval = state:parseExpr(eval) - -- print('re-evaluate expr ' .. tostring(eval)) - setValue(node, eval) - else - -- undefined macro symbol is always treated as 0. - -- http://gcc.gnu.org/onlinedocs/cpp/If.html#If - setValue(node, 0) - end - end - end - - local r = evaluate(root) - -- print('evaluate:' .. tostring(r)) - return r -end - --- apply string ops "##" -local function prepareMacro(state, input) - if type(input) ~= "string" then return input end - repeat - local out = {} - local concat - for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_MACRO) do - if k == "CONCAT" then - -- remove concat op "##" - concat = true - else - table.insert(out, input:sub(start, end_)) - end - end - input = table.concat(out) - until not concat - return input -end - --- macro args replacement function slower but more torelant for pathological case -local function replaceArgs(argsstr, repl) - local args = {} - argsstr = argsstr:sub(2,-2) - -- print('argsstr:'..argsstr) - local comma - for k, v, start, end_ in tokenizer(argsstr, LCPP_TOKENIZE_MACRO_ARGS) do - -- print("replaceArgs:" .. k .. "|" .. v) - if k == "ARGS" or k == "PARENTHESE" or k == "STRING_LITERAL" or - k == "FUNCTIONAL" or k == "SINGLE_CHARACTER_ARGS" then - table.insert(args, v) - comma = false - elseif k == "COMMA" then - if comma then - -- continued comma means empty parameter - table.insert(args, "") - end - comma = true - end - end - local v = repl:gsub("%$(%d+)", function (m) return args[tonumber(m)] or "" end) - -- print("replaceArgs:" .. repl .. "|" .. tostring(#args) .. "|" .. v) - return v -end - --- i.e.: "MAX(x, y) (((x) > (y)) ? (x) : (y))" -local function parseFunction(state, input) - if not input then return end - local concat - local name, argsstr, repl = input:match(FUNCMACRO) - if not name or not argsstr or not repl then return end - - -- rename args to $1,$2... for later gsub - local noargs = 0 - for argname in argsstr:gmatch(IDENTIFIER) do - noargs = noargs + 1 - -- avoid matching substring of another identifier (eg. attrib matches __attribute__ and replace it) - repl = repl:gsub("(#*)(%s*)("..argname..")([_%w]?)", function (s1, s2, s3, s4) - if #s4 <= 0 then - return (#s1 == 1) and ("\"$"..noargs.."\"") or (s1..s2.."$"..noargs) - else - return s1..s2..s3..s4 - end - end) - end - -- remove concat (after replace matching argument name to $1, $2, ...) - repl = repl:gsub("%s*##%s*", "") - - -- build macro funcion - local func = function(input) - return input:gsub(name.."%s*(%b())", function (match) - return replaceArgs(match, repl) - end) - end - - return name, func, repl -end - - --- ------------ --- LCPP INTERFACE --- ------------ - ---- initialies a lcpp state. not needed manually. handy for testing -function lcpp.init(input, predefines, macro_sources) - -- create sate var - local state = {} -- init the state object - state.defines = {} -- the table of known defines and replacements - state.screener = screener(input) - state.lineno = 0 -- the current line number - state.stack = {} -- stores wether the current stack level is to be included - state.once = {} -- stack level was once true (first if that evals to true) - state.macro_sources = macro_sources or {} -- original replacement text for functional macro - - -- funcs - state.define = define - state.undefine = function(state, key) - state:define(key, nil) - state.macro_sources[key] = nil - end - state.defined = function(state, key) - return state.defines[key] ~= nil - end - state.apply = apply - state.includeFile = includeFile - state.doWork = doWork - state.getIndent = function(state) - return #state.stack - end - state.openBlock = function(state, bool) - state.stack[#state.stack+1] = bool - state.once [#state.once+1] = bool - state:define(__LCPP_INDENT__, state:getIndent(), true) - end - state.elseBlock = function(state, bool) - if state.once[#state.once] then - state.stack[#state.stack] = false - else - state.stack[#state.stack] = bool - if bool then state.once[#state.once] = true end - end - end - state.closeBlock = function(state) - state.stack[#state.stack] = nil - state.once [#state.once] = nil - state:define(__LCPP_INDENT__, state:getIndent(), true) - if state:getIndent() < 0 then error("Unopened block detected. Indentaion problem.") end - end - state.skip = function(state) - for i = 1, #state.stack do - if not state.stack[i] then return i end - end - return false - end - state.getLine = function(state) - state.lineno = state.lineno + 1 - state:define(__LINE__, state.lineno, true) - return state.screener() - end - state.prepareMacro = prepareMacro - state.parseExpr = parseExpr - state.parseFunction = parseFunction - - -- predefines - state:define(__DATE__, os.date("%B %d %Y"), true) - state:define(__TIME__, os.date("%H:%M:%S"), true) - state:define(__LINE__, state.lineno, true) - state:define(__LCPP_INDENT__, state:getIndent(), true) - predefines = predefines or {} - for k,v in pairs(lcpp.ENV) do state:define(k, v, true) end -- static ones - for k,v in pairs(predefines) do state:define(k, v, true) end - - if lcpp.LCPP_TEST then lcpp.STATE = state end -- activate static state debugging - - return state -end - ---- the preprocessors main function. --- returns the preprocessed output as a string. --- @param code data as string --- @param predefines OPTIONAL a table of predefined variables --- @usage lcpp.compile("#define bar 0x1337\nstatic const int foo = bar;") --- @usage lcpp.compile("#define bar 0x1337\nstatic const int foo = bar;", {["bar"] = "0x1338"}) -function lcpp.compile(code, predefines, macro_sources) - local state = lcpp.init(code, predefines, macro_sources) - local buf = {} - for output in state:doWork() do - table.insert(buf, output) - end - local output = table.concat(buf, NEWL) - if lcpp.DEBUG then print(output) end - return output, state -end - ---- preprocesses a file --- @param filename the file to read --- @param predefines OPTIONAL a table of predefined variables --- @usage out, state = lcpp.compileFile("../odbg/plugin.h", {["MAX_PAH"]=260, ["UNICODE"]=true}) -function lcpp.compileFile(filename, predefines, macro_sources, next, _local) - if not filename then error("processFile() arg1 has to be a string") end - local file = io.open(filename, 'r') - if not file then error("file not found: "..filename) end - local code = file:read('*a') - predefines = predefines or {} - predefines[__FILE__] = filename - return lcpp.compile(code, predefines, macro_sources) -end - - --- ------------ --- SATIC UNIT TESTS --- ------------ -function lcpp.test(suppressMsg) - local testLabelCount = 0 - local function getTestLabel() - testLabelCount = testLabelCount + 1 - return " lcpp_assert_"..testLabelCount - end - - -- this ugly global is required so our testcode can find it - _G.lcpp_test = { - assertTrueCalls = 0; - assertTrueCount = 0; - assertTrue = function() - lcpp_test.assertTrueCount = lcpp_test.assertTrueCount + 1; - end - } - - local testlcpp = [[ - assert(__LINE__ == 1, "_LINE_ macro test 1: __LINE__") - // This test uses LCPP with lua code (uncommon but possible) - assert(__LINE__ == 3, "_LINE_ macro test 3: __LINE__") - /* - * It therefore asserts any if/else/macro functions and various syntaxes - * (including this comment, that would cause errors if not filtered) - */ - assert(__LINE__ == 8, "_LINE_ macro test 8: __LINE__") - /* - assert(false, "multi-line comment not removed") - */ - /* pathological case which contains single line comment start in multiline comments. - * e.g. this multiline comment should be finish next line. - * http://foobar.com */ // comment - /* if singleline comment processes first, sometimes indicator of end of multiline loss */ #define THIS_SHOULD_ENABLE 111 /* - continuous multiline comment after macro definition - //*/ - ///* this removed. - assert(THIS_SHOULD_ENABLE == 111, "pathological multiline comment test") - - - #define TRUE - #define ONE (1) - #define TWO (2) - #define THREE_FUNC(x) (3) - #define LEET 0x1337 - #define CLONG 123456789L - #define CLONGLONG 123456789123456789LL - #define CULONG 12345678UL - #define CUINT 123456U - #define BINARY -0b1001 /* -9 */ - #define OCTET 075 /* 61 */ - #define NON_OCTET 75 - #define HEX 0xffffU - #define __P(x) x - #define WCHAR_ZERO L'\0' - # define NCURSES_IMPEXP - # define NCURSES_API - # define NCURSES_EXPORT(type) NCURSES_IMPEXP type NCURSES_API - #define MACRO_TO_ITSELF MACRO_TO_ITSELF - local MACRO_TO_ITSELF = 111 - assert(MACRO_TO_ITSELF == 111, "can process macro to itself") - assert(WCHAR_ZERO == 0, "wchar evaluate fails") - assert(CLONG == 123456789, "read *L fails") - assert(CLONGLONG == 123456789123456789, "read *LL fails") - assert(CULONG == 12345678, "read *UL fails") - assert(HEX == 65535, "read hex fails") - #if CUINT != 123456U - assert(false, "cannot evaluate number which contains Unsinged postfix correctly") - #else - assert(CUINT == 123456, "read *U fails") - #endif - #pragma ignored - assert __P((BINARY == -9, "parse, binary literal fails")) - assert(OCTET == 61 and NON_OCTET == 75, "parse octet literal fails") - - lcpp_test.assertTrue() - assert(LEET == 0x1337, "simple #define replacement") - local msg - /* function to check macro expand to empty */ - local function check_argnum(...) - return select('#', ...) - end - NCURSES_EXPORT(function) test_export(a) - return a + 1 - end - assert(test_export(2) == 3, "EXPORT style macro") - local macrofunc = function __P(( - a, b)) - return a + b - end - assert(macrofunc(1, 2) == 3, "macro arg contains parenthese") - - msg = "tenary operator test" - #if (HEX % 2 == 1 ? CUINT : CULONG) == 123456 - lcpp_test.assertTrue() - #else - assert(false, msg.."1") - #endif - #if (OCTET % 2 == 0 ? CUINT : CULONG) == 123456 - assert(false, msg.."1") - #else - lcpp_test.assertTrue() - #endif - - - - - # if defined TRUE - lcpp_test.assertTrue() -- valid strange syntax test (spaces and missing brackets) - # endif - - - msg = "#define if/else test" - #ifdef TRUE - lcpp_test.assertTrue() - #else - assert(false, msg.."1") - #endif - #ifdef NOTDEFINED - assert(false, msg.."2") - #else - lcpp_test.assertTrue() - #endif - #ifndef NOTDEFINED - lcpp_test.assertTrue() - #else - assert(false, msg.."3") - #endif - - - msg = "#if defined statement test" - #if defined(TRUE) - lcpp_test.assertTrue() - #else - assert(false, msg.."1") - #endif - #if !defined(LEET) && !defined(TRUE) - assert(false, msg.."2") - #endif - #if !defined(NOTLEET) && !defined(NOTDEFINED) - lcpp_test.assertTrue() - #else - assert(false, msg.."3") - #endif - #if !(defined(LEET) && defined(TRUE)) - assert(false, msg.."4") - #else - lcpp_test.assertTrue() - #endif - #if !defined(LEET) && !defined(TRUE) - assert(false, msg.."5") - #endif - #if defined(LEET) && defined(TRUE) && defined(NOTDEFINED) - assert(false, msg.."6") - #endif - #if ONE + TWO * TWO == 5 - lcpp_test.assertTrue() - #else - assert(false, msg.."7") - #endif - #if (ONE + TWO) * TWO == 0x6 - lcpp_test.assertTrue() - #else - assert(false, msg.."8") - #endif - #if ONE * TWO + ONE / TWO == 2.5 - lcpp_test.assertTrue() - #else - assert(false, msg.."9") - #endif - #if ONE + ONE * TWO / TWO == 2 - lcpp_test.assertTrue() - #else - assert(false, msg.."10") - #endif - #if TWO - - TWO == 4 - lcpp_test.assertTrue() - #else - assert(false, msg.."11") - #endif - #if (TWO - - TWO) % (ONE + TWO) == 1 - lcpp_test.assertTrue() - #else - assert(false, msg.."12") - #endif - #if ONE << TWO + TWO >> ONE == 8 - lcpp_test.assertTrue() - #else - assert(false, msg.."13") - #endif - #if (ONE << TWO) + (TWO >> ONE) == 5 - lcpp_test.assertTrue() - #else - assert(false, msg.."14") - #endif - #if (ONE << TWO) + TWO >> ONE == 3 - lcpp_test.assertTrue() - #else - assert(false, msg.."15") - #endif - #if (THREE_FUNC(0xfffffU) & 4) == 0 - lcpp_test.assertTrue() - #else - assert(false, msg.."16") - #endif - #if (0x3 & THREE_FUNC("foobar")) == 0b11 - lcpp_test.assertTrue() - #else - assert(false, msg.."17") - #endif - #if defined(TWO) && ((TWO-0) < 3) - lcpp_test.assertTrue() - #else - assert(false, msg.."17") - #endif - #if TWO == 1--1 - lcpp_test.assertTrue() - #else - assert(false, msg.."18") - #endif - #if HEX > 0xfFfFU - assert(false, msg.."18") - #else - lcpp_test.assertTrue() - #endif - #define TRUE_DEFINED defined(TRUE) - #if TRUE_DEFINED - lcpp_test.assertTrue() - #else - assert(false, msg.."19") - #endif - #define NOTDEFINED_DEFINED defined(TRUE) && defined(NOTDEFINED) - #if NOTDEFINED_DEFINED - assert(false, msg.."20") - #else - lcpp_test.assertTrue() - #endif - #if LEET && LEET > 0x1336 - lcpp_test.assertTrue() - #else - assert(false, msg.."20") - #endif - #if NOTLEET && NOTLEET > 0x1336 - assert(false, msg.."21") - #else - lcpp_test.assertTrue() - #endif - #if defined(NOTLEET) || BINARY + 0 >= 10L || !defined(TRUE) - assert(false, msg.."22") - #else - lcpp_test.assertTrue() - #endif - - - - msg = "macro chaining" - #define FOO 0x7B - #define BAR (FOO+0x7B) - assert(-BAR == -0x7B*2, msg) - #define BAZ 456 - #define BLUR BA##Z - assert(BLUR == 456, msg) - local testfunc = function (x) return "["..tostring(x).."]" end - #define FOOBAR(x) testfunc(x) - assert(FOOBAR(1) == "[1]", msg) - #define testfunc(x) - #define FOOBAZ(x) testfunc(x) - assert(check_argnum(FOOBAR(1)) == 0, msg) - assert(check_argnum(FOOBAZ(1)) == 0, msg) - #undef testfunc - assert(FOOBAR(1) == "[1]", msg) - assert(FOOBAZ(1) == "[1]", msg) - - - msg = "indentation test" - assert(__LCPP_INDENT__ == 0, msg.."1") - #if defined(TRUE) - assert(__LCPP_INDENT__ == 1, msg.."2") - #endif - assert(__LCPP_INDENT__ == 0, msg.."3") - - - #define LCPP_FUNCTION_1(x, y) (x and not y) - assert(LCPP_FUNCTION_1(true, false), "function macro") - #define LCPP_FUNCTION_2(x, y) \ - (not x \ - and y) - assert(LCPP_FUNCTION_2(false, true), "multiline function macro") - #define LCPP_FUNCTION_3(_x, _y) LCPP_FUNCTION_2(_x, _y) - assert(LCPP_FUNCTION_3(false, true), "function macro with argname contains _") - - #define LCPP_FUNCTION_4_CHILD() false - #define LCPP_FUNCTION_4(_x) - - assert(check_argnum(LCPP_FUNCTION_4(LCPP_FUNCTION_4_CHILD())) == 0, "functional macro which receives functional macro as argument") - assert(check_argnum(LCPP_FUNCTION_4(LCPP_FUNCTION_3(true, true))) == 0, "functional macro which receives functional macro as argument2") - - #define LCPP_FUNCTION_5(x, y) (x) + (x) + (y) + (y) - assert(LCPP_FUNCTION_5(10, 20) == 60, "macro argument multiple usage") - - #define LCPP_NOT_FUNCTION (BLUR) - assert(LCPP_NOT_FUNCTION == 456, "if space between macro name and argument definition exists, it is regarded as replacement macro") - - #define __CONCAT(x, y, z, w) x ## y ## z (w) - local __fncall = function (x) return x + 111 end - assert(111 == __CONCAT( __ , fnc , all, 0 ), "funcall fails") - assert(222 == __CONCAT( __ ,, fncall, 111 ), "funcall fails2") - #define __ATTRIB_CALL(x, y, attrib) __attribute__(x, y, attrib) - local __attribute__ = function (x, y, attr) - return attr * (x + y) - end - assert(__ATTRIB_CALL( 1, 2 , 100 ) == 300, "funcall fails3") - - - msg = "#elif test" - #if defined(NOTDEFINED) - -- it should not be processed - #if NOTDEFINED - #else - assert(false, msg.."1") - #endif - #elif defined(NOTDEFINED) - assert(false, msg.."2") - #elif defined(TRUE) - lcpp_test.assertTrue() - #else - assert(false, msg.."3") - #endif - - - msg = "#else if test" - #if defined(NOTDEFINED) - assert(false, msg.."1") - #else if defined(NOTDEFINED) - assert(false, msg.."2") - #else if defined(TRUE) - lcpp_test.assertTrue() - #else - assert(false, msg.."3") - #endif - - - msg = "bock stack tree test" - #ifdef TRUE - #ifdef NOTDEFINED - assert(false, msg.."1") - #elif defined(TRUE) - lcpp_test.assertTrue() - #else - assert(false, msg.."2") - #endif - #else - assert(false, msg.."3") - #endif - #ifdef NOTDEFINED - #ifdef TRUE - assert(false, msg.."4") - #endif - #endif - - msg = "test concat ## operator" - #define CONCAT_TEST1 foo##bar - local CONCAT_TEST1 = "blubb" - assert(foobar == "blubb", msg) - #define CONCAT_TEST2() bar##foo - local CONCAT_TEST2() = "blubb" - assert(barfoo == "blubb", msg) - -- dont process ## within strings - #define CONCAT_TEST3 "foo##bar" - assert(CONCAT_TEST3 == "foo##bar", msg) - msg = "test concat inside func type macro" - #define CONCAT_TEST4(baz) CONCAT_TEST##baz - local CONCAT_TEST4(1) = "bazbaz" - assert(foobar == "bazbaz", msg) - - msg = "#undef test" - #define UNDEF_TEST - #undef UNDEF_TEST - #ifdef UNDEF_TEST - assert(false, msg) - #endif - - msg = "stringify operator(#) test" - #define STRINGIFY(str) #str - assert(STRINGIFY(abcde) == "abcde", msg) - #define STRINGIFY_AND_CONCAT(str1, str2) #str1 ## \ - #str2 - assert(STRINGIFY_AND_CONCAT(fgh, ij) == "fghij", msg) - - #define msg_concat(msg1, msg2) msg1 ## msg2 - assert("I, am, lcpp" == msg_concat("I, am", ", lcpp"), "processing macro argument which includes ,") - - #define FUNC__ARG 500 - #define __ARG 100 - #define FUNC(x) FUNC##x - assert(FUNC(__ARG) == 500, "create new macro symbol by concat") - - msg = "same macro definition which has exactly same value allows. (checked with gcc 4.4.7 __WORDSIZE)" - #define DUP_MACRO_DEF (111) - #define DUP_MACRO_DEF (111) - #define DUP_FUNC_MACRO(x, y) x + y - #define DUP_FUNC_MACRO(x, y) x + y - - - msg = "check #if conditional check" - #define VALUE1 (123) - #if VALUE1 != 123 - assert(false, msg .." #if " .. tostring(VALUE1) .. " != 123") - #endif - #if 123 != VALUE1 - assert(false, msg .." #if " .. tostring(VALUE1) .. " != 123 (2)") - #endif - - #define VALUE2 ("hoge") - #if VALUE2 == "hoge" - #define VALUE3 (true) - #endif - assert(VALUE3 == true, msg .. " #if " .. tostring(VALUE3) .. " == true") - - #define VALUE4 (VALUE1 + VALUE1) - #if VALUE4 != 246 - assert(false, msg .." #if check for nested definition:" .. tostring(VALUE4)) - #endif - - msg = "+-*/<> in #if expression:" - #define CALC_VALUE_A (1) - #define CALC_VALUE_B (2) - #if (CALC_VALUE_A + CALC_VALUE_B) != 3 - assert(false, msg .. " + not work:") - #endif - #if (CALC_VALUE_A * CALC_VALUE_B) != 2 - assert(false, msg .. " * not work:") - #endif - #if (CALC_VALUE_A - CALC_VALUE_B) != -1 - assert(false, msg .. " - not work:") - #endif - #if (CALC_VALUE_A / CALC_VALUE_B) != 0.5 - assert(false, msg .. " / not work:") - #endif - - #if (CALC_VALUE_A >= CALC_VALUE_A) - #else - assert(false, msg .. " >= not work1") - #endif - #if (CALC_VALUE_B >= CALC_VALUE_A) - #else - assert(false, msg .. " >= not work2") - #endif - - #if (CALC_VALUE_B <= CALC_VALUE_B) - #else - assert(false, msg .. " <= not work1") - #endif - #if (CALC_VALUE_A <= CALC_VALUE_B) - #else - assert(false, msg .. " <= not work2") - #endif - - #if (CALC_VALUE_B > CALC_VALUE_A) - #else - assert(false, msg .. " > not work1") - #endif - #if (CALC_VALUE_A > CALC_VALUE_A) - assert(false, msg .. " > not work2") - #endif - - #if (CALC_VALUE_A < CALC_VALUE_B) - #else - assert(false, msg .. " < not work1") - #endif - #if (CALC_VALUE_B < CALC_VALUE_B) - assert(false, msg .. " < not work2") - #endif - #if (CALC_VALUE_B | CALC_VALUE_A) != 3 - assert(false, msg .. " | not work") - #endif - #if (CALC_VALUE_B & CALC_VALUE_A) != 0 - assert(false, msg .. " & not work") - #endif - #if (CALC_VALUE_B ^ CALC_VALUE_A) != 3 - assert(false, msg .. " ^ not work") - #endif - #if -CALC_VALUE_B != -2 - assert(false, msg .. " unary - not work") - #endif - #if ~CALC_VALUE_B != -3 - assert(false, msg .. " unary ~ not work") - #endif - ]] - lcpp.FAST = false -- enable full valid output for testing - lcpp.SELF_TEST = true - local testlua = lcpp.compile(testlcpp) - lcpp.SELF_TEST = nil - -- print(testlua) - assert(loadstring(testlua, "testlua"))() - lcpp_test.assertTrueCalls = findn(testlcpp, "lcpp_test.assertTrue()") - assert(lcpp_test.assertTrueCount == lcpp_test.assertTrueCalls, "assertTrue calls:"..lcpp_test.assertTrueCalls.." count:"..lcpp_test.assertTrueCount) - _G.lcpp_test = nil -- delete ugly global hack - if not suppressMsg then print("Test run suscessully") end -end -if lcpp.LCPP_TEST then lcpp.test(true) end - - --- ------------ --- REGISTER LCPP --- ------------ - ---- disable lcpp processing for ffi, loadstring and such -lcpp.disable = function() - if lcpp.LCPP_LUA then - -- activate LCPP_LUA actually does anything useful - -- _G.loadstring = _G.loadstring_lcpp_backup - end - - if lcpp.LCPP_FFI and pcall(require, "ffi") then - ffi = require("ffi") - if ffi.lcpp_cdef_backup then - ffi.cdef = ffi.lcpp_cdef_backup - ffi.lcpp_cdef_backup = nil - end - end -end - ---- (re)enable lcpp processing for ffi, loadstring and such -lcpp.enable = function() - -- Use LCPP to process Lua code (load, loadfile, loadstring...) - if lcpp.LCPP_LUA then - -- TODO: make it properly work on all functions - error("lcpp.LCPP_LUA = true -- not properly implemented yet"); - _G.loadstring_lcpp_backup = _G.loadstring - _G.loadstring = function(str, chunk) - return loadstring_lcpp_backup(lcpp.compile(str), chunk) - end - end - -- Use LCPP as LuaJIT PreProcessor if used inside LuaJIT. i.e. Hook ffi.cdef - if lcpp.LCPP_FFI and pcall(require, "ffi") then - ffi = require("ffi") - if not ffi.lcpp_cdef_backup then - if not ffi.lcpp_defs then ffi.lcpp_defs = {} end -- defs are stored and reused - ffi.lcpp = function(input) - local output, state = lcpp.compile(input, ffi.lcpp_defs, ffi.lcpp_macro_sources) - ffi.lcpp_defs = state.defines - ffi.lcpp_macro_sources = state.macro_sources - return output - end - ffi.lcpp_cdef_backup = ffi.cdef - ffi.cdef = function(input) - if true then - return ffi.lcpp_cdef_backup(ffi.lcpp(input)) - else - local fn,cnt = input:gsub('#include ["<].-([^/]+%.h)[">]', '%1') - input = ffi.lcpp(input) - if cnt > 0 then - local f = io.open("./tmp/"..fn, 'w') - if f then - f:write(input) - f:close() - else - assert(fn:find('/'), 'cannot open: ./tmp/'..fn) - end - end - return ffi.lcpp_cdef_backup(input) - end - end - end - end -end - -lcpp.enable() -return lcpp diff --git a/tools/luajit_make_bindings.lua b/tools/luajit_make_bindings.lua index 5db448c..3b6be32 100644 --- a/tools/luajit_make_bindings.lua +++ b/tools/luajit_make_bindings.lua @@ -1,9 +1,25 @@ -local lcpp = require("tools/luajit_lcpp") +local lcpp = require("tools/cparser") local glue = [[ #define __TINYC__ #define static #define __thread +#define STATIC_ASSERT(x) +//#define ifdef(a,b,...) ifdef_##a(b,__VA_ARGS__) +//#define ifndef(a,b,...) ifdef_##a(__VA_ARGS__,b) +//#define ifdef_true(b,...) b +//#define ifdef_false(b,...) __VA_ARGS__ +#define OBJ \ + struct { \ + ifdef(debug, const char *objname;) \ + uintptr_t objheader; \ + array(struct obj*) objchildren; \ + }; +#define ENTITY \ + struct { OBJ \ + uintptr_t cflags; \ + void *c[OBJCOMPONENTS_MAX]; \ + }; typedef struct FILE FILE; typedef long int ptrdiff_t; typedef long unsigned int size_t; @@ -18,17 +34,17 @@ function trim_multilines(str) return output end -io.input("./engine/v4k.h") -local v4k_h = io.read("*all") -v4k_h = v4k_h:gsub("#line", "//#line") -v4k_h = v4k_h:gsub("#include", "//#include") +io.input("./engine/fwk.h") +local fwk_h = io.read("*all") +fwk_h = fwk_h:gsub("#line", "//#line") +fwk_h = fwk_h:gsub("#include", "//#include") print('--autogenerated luajit bindings. do not edit.') -- .. os.date("%Y/%m/%d")) print('local ffi = require("ffi")') print('ffi.cdef([[') -local result = lcpp.compile(glue .. v4k_h) -print( trim_multilines(result) ) +local result = lcpp.cppString(glue .. fwk_h, '-', {"-Znopass"}, 'fwk') +--print( trim_multilines(result) ) print(']])') @@ -68,11 +84,11 @@ function _M.mat44() return m end -local fwk = ffi.load("v4k") +local fwk = ffi.load("fwk") return setmetatable( _M, { __index = function( table, key ) return fwk[ key ] end } ) -]]) +]]) \ No newline at end of file