1726 lines
58 KiB
Lua
1726 lines
58 KiB
Lua
-- Copyright (c) Facebook, Inc. and its affiliates.
|
|
-- This source code is licensed under the MIT license found in the
|
|
-- LICENSE file in the root directory of this source tree.
|
|
--
|
|
-- Lua module to preprocess and parse C declarations.
|
|
-- (Leon Bottou, 2015)
|
|
|
|
-- @r-lyeh: del lcdecl module
|
|
-- @r-lyeh: add cppString method
|
|
-- @r-lyeh: add __COUNTER__ macro
|
|
-- @r-lyeh: del empty #line directives from output
|
|
|
|
-- standard libs
|
|
local string = require 'string'
|
|
local coroutine = require 'coroutine'
|
|
local table = require 'table'
|
|
local io = require 'io'
|
|
|
|
-- Lua 5.1 to 5.3 compatibility
|
|
local unpack = unpack or table.unpack
|
|
|
|
-- Debugging
|
|
local DEBUG = true
|
|
if DEBUG then pcall(require,'strict') end
|
|
|
|
-- luacheck: globals cparser
|
|
-- luacheck: ignore 43 4/ti 4/li
|
|
-- luacheck: ignore 212/.*_
|
|
-- luacheck: ignore 211/is[A-Z].* 211/Type
|
|
-- luacheck: ignore 542
|
|
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- ALL UGLY HACKS SHOULD BE HERE
|
|
|
|
|
|
-- Sometimes we cannot find system include files but need to know at
|
|
-- least things about them. For instance, certain system include files
|
|
-- define alternate forms for keywords.
|
|
|
|
local knownIncludeQuirks = {}
|
|
|
|
knownIncludeQuirks["<complex.h>"] = { -- c99
|
|
"#ifndef complex", "# define complex _Complex", "#endif"
|
|
}
|
|
|
|
knownIncludeQuirks["<stdbool.h>"] = { -- c99
|
|
"#ifndef bool", "# define bool _Bool", "#endif"
|
|
}
|
|
|
|
knownIncludeQuirks["<stdalign.h>"] = { -- c11
|
|
"#ifndef alignof", "# define alignof _Alignof", "#endif",
|
|
"#ifndef alignas", "# define alignas _Alignas", "#endif"
|
|
}
|
|
|
|
knownIncludeQuirks["<stdnoreturn.h>"] = { -- c11
|
|
"#ifndef noreturn", "# define noreturn _Noreturn", "#endif"
|
|
}
|
|
|
|
knownIncludeQuirks["<threads.h>"] = { -- c11
|
|
"#ifndef thread_local", "# define thread_local _Thread_local", "#endif"
|
|
}
|
|
|
|
knownIncludeQuirks["<iso646.h>"] = { -- c++
|
|
"#define and &&", "#define and_eq &=", "#define bitand &", "#define bitor |",
|
|
"#define compl ~", "#define not !", "#define not_eq !=", "#define or ||",
|
|
"#define or_eq |=", "#define xor ^", "#define xor_eq ^="
|
|
}
|
|
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- TAGGED TABLES
|
|
|
|
|
|
-- Utilities to produce and print tagged tables.
|
|
-- The tag name is simply the contents of table key <tag>.
|
|
-- Function <newTag> returns a node constructor
|
|
--
|
|
-- Example:
|
|
--
|
|
-- > Foo = newTag('Foo')
|
|
-- > Bar = newTag('Bar')
|
|
--
|
|
-- > print( Foo{const=true,next=Bar{name="Hello"}} )
|
|
-- Foo{next=Bar{name="Hello"},const=true}
|
|
--
|
|
-- > print( Bar{name="hi!", Foo{1}, Foo{2}, Foo{3}} )
|
|
-- Bar{Foo{1},Foo{2},Foo{3},name="hi!"}
|
|
|
|
local function newTag(tag)
|
|
-- the printing function
|
|
local function tostr(self)
|
|
local function str(x)
|
|
if type(x)=='string' then
|
|
return string.format("%q",x):gsub("\\\n","\\n")
|
|
elseif type(x)=='table' and not getmetatable(x) then
|
|
return "{..}"
|
|
else
|
|
return tostring(x)
|
|
end
|
|
end
|
|
local p = string.format("%s{", self.tag or "Node")
|
|
local s = {}
|
|
local seqlen = 0
|
|
for i=1,#self do
|
|
if self[i] then seqlen=i else break end end
|
|
for i=1,seqlen do
|
|
s[1+#s] = str(self[i]) end
|
|
for k,v in pairs(self) do
|
|
if type(k) == 'number' then
|
|
if k<1 or k>seqlen then
|
|
s[1+#s] = string.format("[%s]=%s",k,str(v)) end
|
|
elseif type(k) ~= 'string' then
|
|
s.extra = true
|
|
elseif k:find("^_") and type(v)=='table' then
|
|
s[1+#s] = string.format("%s={..}",k) -- hidden
|
|
elseif k ~= 'tag' then
|
|
s[1+#s] = string.format("%s=%s",k,str(v)) end
|
|
end
|
|
if s.extra then s[1+#s] = "..." end
|
|
return p .. table.concat(s,',') .. '}'
|
|
end
|
|
-- the constructor
|
|
return function(t) -- must be followed by a table constructor
|
|
t = t or {}
|
|
assert(type(t)=='table')
|
|
setmetatable(t, { __tostring=tostr } )
|
|
t.tag = tag
|
|
return t
|
|
end
|
|
end
|
|
|
|
-- hack to print any table: print(Node(nn))
|
|
local Node = newTag(nil) -- luacheck: ignore 211
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- UTILITIES
|
|
|
|
|
|
-- Many functions below have an optional argument 'options' which is
|
|
-- simply an array of compiler-like options that are specified in the
|
|
-- toplevel call and passed to nearly all functions. Because it
|
|
-- provides a good communication channel across the code components,
|
|
-- many named fields are also used for multiple purposes. The
|
|
-- following function is called at the beginning of the user facing
|
|
-- functions to make a copy of the user provided option array and
|
|
-- setup some of these fields.
|
|
|
|
local function copyOptions(options)
|
|
options = options or {}
|
|
assert(type(options)=='table')
|
|
local noptions = {}
|
|
-- copy options
|
|
for k,v in ipairs(options) do noptions[k]=v end
|
|
-- copy user modifiable named fields
|
|
noptions.sizeof = options.sizeof -- not used yet
|
|
noptions.alignof = options.alignof -- not used yet
|
|
-- create reversed hash
|
|
noptions.hash = {}
|
|
for i,v in ipairs(options) do
|
|
noptions.hash[v] = i
|
|
end
|
|
-- compute dialect flags
|
|
local dialect = 'gnu99'
|
|
for _,v in ipairs(options) do
|
|
if v:find("^%-std=%s*[^%s]") then
|
|
dialect = v:match("^%-std=%s*(.-)%s*$")
|
|
end
|
|
end
|
|
noptions.dialect = dialect
|
|
noptions.dialectGnu = dialect:find("^gnu")
|
|
noptions.dialect99 = dialect:find("9[9x]$")
|
|
noptions.dialect11 = dialect:find("1[1x]$")
|
|
noptions.dialectAnsi = not noptions.dialectGnu
|
|
noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect99
|
|
noptions.dialectAnsi = noptions.dialectAnsi and not noptions.dialect11
|
|
-- return
|
|
return noptions
|
|
end
|
|
|
|
|
|
-- This function tests whether a particular option has been given.
|
|
|
|
local function hasOption(options, opt)
|
|
assert(options)
|
|
assert(options.silent or options.hash)
|
|
return options.hash and options.hash[opt]
|
|
end
|
|
|
|
|
|
-- Generic functions for error messages
|
|
|
|
local function xmessage(err, options, lineno, message, ...)
|
|
local msg = string.format("cparser: (%s) ",lineno)
|
|
msg = msg .. string.format(message,...)
|
|
if options.silent then
|
|
if err == 'error' then error(msg, 0) end
|
|
else
|
|
if err == 'warning' and hasOption(options, "-Werror") then err = 'error' end
|
|
if err == 'error' or not hasOption(options, "-w") then print(msg) end
|
|
if err == 'error' then error("cparser: aborted",0) end
|
|
end
|
|
end
|
|
|
|
local function xwarning(options, lineno, message, ...)
|
|
xmessage('warning', options, lineno, message, ...)
|
|
end
|
|
|
|
local function xerror(options, lineno, message, ...)
|
|
xmessage('error', options, lineno, message, ...)
|
|
end
|
|
|
|
local function xassert(cond, ...)
|
|
if not cond then xerror(...) end
|
|
end
|
|
|
|
local function xdebug(lineno,message,...)
|
|
local msg = string.format("\t\t[%s] ", lineno)
|
|
msg = msg .. string.format(message,...)
|
|
print(msg)
|
|
end
|
|
|
|
|
|
-- Nil-safe max
|
|
|
|
local function max(a,b)
|
|
a = a or b
|
|
b = b or a
|
|
return a > b and a or b
|
|
end
|
|
|
|
|
|
-- Deep table comparison
|
|
-- (not very efficient, no loop detection)
|
|
|
|
local function tableCompare(a,b)
|
|
if a == b then
|
|
return true
|
|
elseif type(a) == 'table' and type(b) == 'table' then
|
|
for k,v in pairs(a) do
|
|
if not tableCompare(v,b[k]) then return false end
|
|
end
|
|
for k,v in pairs(b) do
|
|
if not tableCompare(a[k],v) then return false end
|
|
end
|
|
return true
|
|
else
|
|
return false
|
|
end
|
|
end
|
|
|
|
|
|
-- Concatenate two possibly null arrays
|
|
|
|
local function tableAppend(a1, a2)
|
|
if not a1 then
|
|
return a2
|
|
elseif not a2 then
|
|
return a1
|
|
else
|
|
local a = {}
|
|
for _,v in ipairs(a1) do a[1+#a] = v end
|
|
for _,v in ipairs(a2) do a[1+#a] = v end
|
|
return a
|
|
end
|
|
end
|
|
|
|
-- Concatenate strings from table (skipping non-string content.)
|
|
|
|
local function tableConcat(a)
|
|
local b = {}
|
|
for _,v in ipairs(a) do
|
|
if type(v) == 'string' then b[1+#b]=v end end
|
|
return table.concat(b)
|
|
end
|
|
|
|
|
|
-- Evaluate a lua expression, return nil on error.
|
|
|
|
local function evalLuaExpression(s)
|
|
assert(type(s)=='string')
|
|
local f = load(string.gmatch(s,".*"))
|
|
local function r(status,...)
|
|
if status then return ... end end
|
|
return r(pcall(f or error))
|
|
end
|
|
|
|
|
|
-- Bitwise manipulations
|
|
-- try lua53 operators otherwise revert to iterative version
|
|
|
|
local bit = evalLuaExpression([[
|
|
local bit = {}
|
|
function bit.bnot(a) return ~a end
|
|
function bit.bor(a,b) return a | b end
|
|
function bit.band(a,b) return a & b end
|
|
function bit.bxor(a,b) return a ~ b end
|
|
function bit.lshift(a,b) return a < 0 and b < 0 and ~((~a) << b) or a << b end
|
|
return bit
|
|
]])
|
|
|
|
if not bit then
|
|
local function bor(a,b)
|
|
local r, c, d = 0, 1, -1
|
|
while a > 0 or b > 0 or a < -1 or b < -1 do
|
|
if a % 2 > 0 or b % 2 > 0 then r = r + c end
|
|
a, b, c, d = math.floor(a / 2), math.floor(b / 2), c * 2, d * 2 end
|
|
if a < 0 or b < 0 then r = r + d end
|
|
return r end
|
|
bit = {}
|
|
function bit.bnot(a) return -1-a end
|
|
function bit.bor(a,b) return bor(a,b) end
|
|
function bit.band(a,b) return -1-bor(-1-a,-1-b) end
|
|
function bit.bxor(a,b) return bor(-1-bor(a,-1-b),-1-bor(-1-a,b)) end
|
|
function bit.lshift(a,b) return math.floor(a * 2 ^ b) end
|
|
end
|
|
|
|
|
|
-- Coroutine helpers.
|
|
-- This code uses many coroutines that yield lines or tokens.
|
|
-- All functions that can yield take an options table as first argument.
|
|
|
|
|
|
-- Wrap a coroutine f into an iterator
|
|
-- The options and all the extra arguments are passed
|
|
-- to the coroutine when it starts. Together with the
|
|
-- above calling convention, this lets us specify
|
|
-- coroutine pipelines (see example in function "cpp".)
|
|
|
|
local function wrap(options, f, ...)
|
|
local function g(...) coroutine.yield(nil) f(...) end
|
|
local c = coroutine.create(g)
|
|
coroutine.resume(c, options, ...)
|
|
local function r(s,...)
|
|
if not s then local m = ... ; error(m, 0) end
|
|
return ...
|
|
end
|
|
return function()
|
|
if coroutine.status(c) ~= 'dead' then
|
|
return r(coroutine.resume(c))
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
-- Collect coroutine outputs into an array
|
|
-- The options and the extra arguments are passed to the coroutine.
|
|
|
|
local function callAndCollect(options, f, ...) -- Bell Labs nostalgia
|
|
local collect = {}
|
|
for s in wrap(options, f, ...) do
|
|
collect[1+#collect] = s
|
|
end
|
|
return collect
|
|
end
|
|
|
|
|
|
-- Yields all outputs from iterator iter.
|
|
-- Argument options is ignored.
|
|
|
|
local function yieldFromIterator(options_, iter)
|
|
local function yes(v,...) coroutine.yield(v,...) return v end
|
|
while yes(iter()) do end
|
|
end
|
|
|
|
|
|
-- Yields all values from array <arr>.
|
|
-- This function successively yields all values in the table.
|
|
-- Every yield is augmented with all extra arguments passed to the function.
|
|
-- Argument options is ignored.
|
|
|
|
local function yieldFromArray(options_, arr, ...)
|
|
for _,v in ipairs(arr) do
|
|
coroutine.yield(v, ...)
|
|
end
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- INITIAL PREPROCESSING
|
|
|
|
|
|
|
|
-- A routine that pulls lines from a line iterator
|
|
-- and yields them together with a location
|
|
-- composed of the optional prefix, a colon, and a line number.
|
|
-- Argument options is ignored.
|
|
-- Lua provides good line iterators such as:
|
|
-- io.lines(filename) filedesc:lines() str:gmatch("[^\n]+")
|
|
|
|
local function yieldLines(options_,lineIterator,prefix)
|
|
prefix = prefix or ""
|
|
assert(type(prefix)=='string')
|
|
local n = 0
|
|
for s in lineIterator do
|
|
n = n + 1
|
|
coroutine.yield(s, string.format("%s:%d", prefix, n))
|
|
end
|
|
end
|
|
|
|
|
|
-- A routine that obtains lines from coroutine <lines>,
|
|
-- joins lines terminated by a backslash, and yield the
|
|
-- resulting lines. The coroutine is initialized with
|
|
-- argument <options> and all extra arguments.
|
|
-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (3)
|
|
|
|
local function joinLines(options, lines, ...)
|
|
local li = wrap(options, lines, ...)
|
|
for s, n in li do
|
|
while type(s) == 'string' and s:find("\\%s*$") do
|
|
local t = li() or ""
|
|
s = s:gsub("\\%s*$", "") .. t
|
|
end
|
|
coroutine.yield(s, n)
|
|
end
|
|
end
|
|
|
|
|
|
-- A routine that obtain lines from coroutine <lines>, eliminate the
|
|
-- comments and yields the resulting lines. The coroutine is
|
|
-- initialized with argument <options> and all extra arguments.
|
|
-- Reference: https://gcc.gnu.org/onlinedocs/cpp/Initial-processing.html (4)
|
|
|
|
local function eliminateComments(options, lines, ...)
|
|
local lineIterator = wrap(options, lines, ...)
|
|
local s,n = lineIterator()
|
|
while type(s) == 'string' do
|
|
local inString = false
|
|
local q = s:find("[\'\"\\/]", 1)
|
|
while q ~= nil do
|
|
if hasOption(options,"-d:comments") then
|
|
xdebug(n, "comment: [%s][%s] %s",s:sub(1,q-1),s:sub(q),inString)
|
|
end
|
|
local c = s:byte(q)
|
|
if inString then
|
|
if c == 92 then -- \
|
|
q = q + 1
|
|
elseif c == inString then
|
|
inString = false
|
|
end
|
|
else
|
|
if c == 34 or c == 39 then -- " or '
|
|
inString = c
|
|
elseif c == 47 and s:byte(q+1) == 47 then -- "//"
|
|
s = s:sub(1,q-1)
|
|
elseif c == 47 and s:byte(q+1) == 42 then -- "/*"
|
|
local p = s:find("%*/",q+2)
|
|
if p ~= nil then
|
|
s = s:sub(1,q-1) .. " " .. s:sub(p+2)
|
|
else
|
|
s = s:sub(1,q-1)
|
|
local ss,pp
|
|
repeat
|
|
ss = lineIterator()
|
|
xassert(ss ~= nil, options, n, "Unterminated comment")
|
|
pp = ss:find("%*/")
|
|
until pp
|
|
s = s .. " " .. ss:sub(pp+2)
|
|
end
|
|
end
|
|
end
|
|
q = s:find("[\'\"\\/]", q+1)
|
|
end
|
|
coroutine.yield(s, n)
|
|
s, n = lineIterator()
|
|
end
|
|
end
|
|
|
|
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- TOKENIZER
|
|
|
|
|
|
|
|
local keywordTable = {
|
|
------ Standard keywords
|
|
"auto", "break", "case", "char", "const", "continue", "default", "do",
|
|
"double", "else", "enum", "extern", "float", "for", "goto", "if", "int",
|
|
"long", "register", "return", "short", "signed", "sizeof", "static", "struct",
|
|
"switch", "typedef", "union", "unsigned", "void", "volatile", "while",
|
|
------ Nonstandard or dialect specific keywords do not belong here
|
|
------ because the main function of this table is to say which
|
|
------ identifiers cannot be variable names.
|
|
}
|
|
|
|
local punctuatorTable = {
|
|
"+", "-", "*", "/", "%", "&", "|", "^", ">>", "<<", "~",
|
|
"=", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", ">>=", "<<=",
|
|
"(", ")", "[", "]", "{", "}", "++", "--",
|
|
"==", "!=", ">=", "<=", ">", "<", "&&", "||", "!",
|
|
".", "->", "*", "&", "?", ":", "::", "->*", ".*", ";", ",",
|
|
"#", "##", "...", "@", "\\" -- preprocessor stuff
|
|
}
|
|
|
|
local keywordHash = {}
|
|
for _,v in ipairs(keywordTable) do
|
|
keywordHash[v] = true
|
|
end
|
|
|
|
local punctuatorHash = {}
|
|
for _,v in ipairs(punctuatorTable) do
|
|
local l = v:len()
|
|
local b = v:byte()
|
|
punctuatorHash[v] = true
|
|
punctuatorHash[b] = max(l,punctuatorHash[b])
|
|
end
|
|
|
|
|
|
-- The following functions test the types of the tokens returned by the tokenizer.
|
|
-- They should not be applied to arbitrary strings.
|
|
|
|
local function isSpace(tok)
|
|
return type(tok) == 'string' and tok:find("^%s") ~= nil end
|
|
local function isNewline(tok) -- Subtype of space
|
|
return type(tok) == 'string' and tok:find("^\n") ~= nil end
|
|
local function isNumber(tok)
|
|
return type(tok) == 'string' and tok:find("^[.0-9]") ~= nil end
|
|
local function isString(tok)
|
|
if type(tok) ~= 'string' then return false end
|
|
return tok:find("^[\'\"]") ~= nil end
|
|
local function isHeaderName(tok)
|
|
if type(tok) ~= 'string' then return false end
|
|
return tok:find("^\"") or tok:find("^<") and tok:find(">$") end
|
|
local function isPunctuator(tok)
|
|
return type(tok) == 'string' and punctuatorHash[tok] ~= nil end
|
|
local function isIdentifier(tok)
|
|
return type(tok) == 'string' and tok:find("^[A-Za-z_$]") ~= nil end
|
|
local function isKeyword(tok) -- Subtype of identifier
|
|
return keywordHash[tok] ~= nil end
|
|
local function isName(tok) -- Subtype of identifier
|
|
return isIdentifier(tok) and not keywordHash[tok] end
|
|
|
|
-- Magic tokens are used to mark macro expansion boundaries (see expandMacros.)
|
|
local function isMagic(tok)
|
|
return tok and type(tok) ~= 'string' end
|
|
local function isBlank(tok) -- Treats magic token as space.
|
|
return isMagic(tok) or isSpace(tok) end
|
|
|
|
-- The tokenizeLine() function takes a line, splits it into tokens,
|
|
-- and yields tokens and locations. The number tokens are the weird
|
|
-- preprocessor numbers defined by ansi c. The string tokens include
|
|
-- character constants and angle-bracket delimited strings occuring
|
|
-- after an include directive. Every line begins with a newline
|
|
-- token giving the proper indentation. All subsequent spaces
|
|
-- are reduced to a single space character.
|
|
|
|
local function tokenizeLine(options, s, n, notNewline)
|
|
-- little optimization for multiline macros
|
|
-- s may be an array of precomputed tokens
|
|
if type(s) == 'table' then
|
|
return yieldFromArray(options, s, n)
|
|
end
|
|
-- normal operation
|
|
assert(type(s) == 'string')
|
|
local p = s:find("[^%s]")
|
|
-- produce a newline token
|
|
if p and not notNewline then
|
|
local r = '\n' .. s:sub(1,p-1)
|
|
coroutine.yield(r, n)
|
|
end
|
|
-- produce one token
|
|
local function token()
|
|
local b, l, r
|
|
if hasOption(options, "-d:tokenize") then
|
|
xdebug(n, "[%s][%s]",s:sub(1,p-1),s:sub(p))
|
|
end
|
|
-- space
|
|
l = s:find("[^%s]", p)
|
|
if l == nil then
|
|
return nil
|
|
elseif l > p then
|
|
p = l
|
|
return " ", n
|
|
end
|
|
-- identifier
|
|
r = s:match("^[a-zA-Z_$][a-zA-Z0-9_$]*", p)
|
|
if r ~= nil then
|
|
p = p + r:len()
|
|
return r, n
|
|
end
|
|
-- preprocessor numbers
|
|
r = s:match("^%.?[0-9][0-9a-zA-Z._]*", p)
|
|
if r ~= nil then
|
|
l = r:len()
|
|
while r:find("[eEpP]$") and s:find("^[-+]", p+l) do
|
|
r = r .. s:match("^[-+][0-9a-zA-Z._]*", p+l)
|
|
l = r:len()
|
|
end
|
|
p = p + l
|
|
return r, n
|
|
end
|
|
-- angle-delimited strings in include directives
|
|
b = s:byte(p)
|
|
if b == 60 and s:find("^%s*#%s*include") then
|
|
r = s:match("^<[^>]+>", p)
|
|
if r ~= nil then
|
|
p = p + r:len()
|
|
return r, n
|
|
end
|
|
end
|
|
-- punctuator
|
|
l = punctuatorHash[b]
|
|
if l ~= nil then
|
|
while l > 0 do
|
|
r = s:sub(p,p+l-1)
|
|
if punctuatorHash[r] then
|
|
p = p + l
|
|
return r, n
|
|
end
|
|
l = l - 1
|
|
end
|
|
end
|
|
-- string
|
|
if b == 34 or b == 39 then -- quotes
|
|
local q = p
|
|
repeat
|
|
q = s:find("[\'\"\\]", q + 1)
|
|
l = s:byte(q)
|
|
xassert(q ~= nil, options, n, "Unterminated string or character constant")
|
|
if l == 92 then
|
|
q = q + 1
|
|
end
|
|
until l == b
|
|
r = s:sub(p,q)
|
|
p = q + 1
|
|
return r, n
|
|
end
|
|
-- other stuff (we prefer to signal an error here)
|
|
xerror(options, n,"Unrecognized character (%s)", s:sub(p))
|
|
end
|
|
-- loop
|
|
if p then
|
|
for tok,tokn in token do
|
|
coroutine.yield(tok, tokn)
|
|
end
|
|
end
|
|
end
|
|
|
|
|
|
-- Obtain lines from coroutine <lines>,
|
|
-- and yields their tokens. The coroutine is initialized with
|
|
-- argument <options> and all extra arguments.
|
|
|
|
local function tokenize(options, lines, ...)
|
|
for s,n in wrap(options, lines, ...) do
|
|
tokenizeLine(options, s, n)
|
|
end
|
|
end
|
|
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- PREPROCESSING
|
|
|
|
|
|
-- Preprocessing is performed by two coroutines. The first one
|
|
-- processes all the preprocessor directives and yields the remaining
|
|
-- lines. The second one processes tokens from the remaining lines and
|
|
-- perform macro expansions. Both take a table of macro definitions as
|
|
-- argument. The first one writes into the table and the second one
|
|
-- reads from it.
|
|
--
|
|
-- Each macro definition is an array of tokens (for a single line
|
|
-- macro) or a table whose entry <"lines"> contains an array of arrays
|
|
-- of tokens (#defmacro). If the macro takes arguments, the entry
|
|
-- <"args"> contains a list of argument names. If the macro is
|
|
-- recursive (#defrecmacro), the entry <recursive> is set.
|
|
-- Alternatively, the macro definition may be a function called at
|
|
-- macro-expansion time. This provides for complicated situations.
|
|
|
|
|
|
-- forward declarations
|
|
local function expandMacros() end
|
|
local function processDirectives() end
|
|
|
|
|
|
-- Starting with the second coroutine which takes a token producing
|
|
-- coroutine and yields the preprocessed tokens. Argument macros is
|
|
-- the macro definition table.
|
|
|
|
-- The standard mandates that the result of a macro-expansion must be
|
|
-- scanned for further macro invocations whose argunent list possibly
|
|
-- consume tokens that follow the macro-expansion. This means that one
|
|
-- cannot recursively call expandMacros but one must prepend the
|
|
-- macro-expansion in front of the remaining tokens. The standard also
|
|
-- mandates that the result of any macro-expansion must be marked to
|
|
-- prevent recursive invocation of the macro that generated it,
|
|
-- whether when expanding macro arguments or expanding the macro
|
|
-- itself. We achieve this by bracketing every macro-expansion with
|
|
-- magic tokens that track which macro definitions must be disabled.
|
|
-- These magic tokens are removed later in the coroutines
|
|
-- <filterSpaces> or <preprocessedLines>.
|
|
|
|
expandMacros = function(options, macros, tokens, ...)
|
|
-- basic iterator
|
|
local ti = wrap(options, tokens, ...)
|
|
-- prepending tokens in front of the token stream
|
|
local prepend = {}
|
|
local function prependToken(s,n)
|
|
table.insert(prepend,{s,n}) end
|
|
local function prependTokens(pti)
|
|
local pos = 1+#prepend
|
|
for s,n in pti do table.insert(prepend,pos,{s,n}) end end
|
|
local ti = function()
|
|
if #prepend > 0 then return unpack(table.remove(prepend))
|
|
else return ti() end end
|
|
-- iterator that handles magic tokens to update macro definition table
|
|
local ti = function()
|
|
local s,n = ti()
|
|
while type(s) == 'table' do
|
|
if s.tag == 'push' then
|
|
local nmacros = {}
|
|
setmetatable(nmacros, {__index=macros})
|
|
if s.symb then nmacros[s.symb] = false end
|
|
macros = nmacros
|
|
elseif s.tag == 'pop' then
|
|
local mt = getmetatable(macros)
|
|
if mt and mt.__index then macros = mt.__index end
|
|
end
|
|
coroutine.yield(s,n)
|
|
s,n = ti()
|
|
end
|
|
return s,n
|
|
end
|
|
-- redefine ti() to ensure tok,n remain up-to-date
|
|
local tok,n = ti()
|
|
local ti = function() tok,n=ti() return tok,n end
|
|
-- collect one macro arguments into an array
|
|
-- stop when reaching a closing parenthesis or a comma
|
|
local function collectArgument(ti, varargs)
|
|
local count = 0
|
|
local tokens = {}
|
|
ti()
|
|
while isSpace(tok) do
|
|
tok = ti()
|
|
end
|
|
while tok do
|
|
if tok == ')' and count == 0 then
|
|
break
|
|
elseif tok == ')' then
|
|
count = count - 1
|
|
elseif tok == '(' then
|
|
count = count + 1
|
|
elseif tok == ',' and count == 0 and not varargs then
|
|
break
|
|
end
|
|
if isSpace(tok) then tok = " " end
|
|
tokens[1+#tokens] = tok
|
|
tok = ti()
|
|
end
|
|
if #tokens > 0 and isSpace(tokens[#tokens]) then
|
|
tokens[#tokens] = nil
|
|
end
|
|
return tokens
|
|
end
|
|
-- collects all macro arguments
|
|
local function collectArguments(ti,def,ntok,nn)
|
|
local args = def.args
|
|
local nargs = { [0]={} }
|
|
if #args == 0 then ti() end
|
|
for _,name in ipairs(args) do
|
|
if tok == ')' and name == "__VA_ARGS__" then
|
|
nargs[0][name] = { negComma=true }
|
|
nargs[name] = { negComma=true }
|
|
else
|
|
xassert(tok=='(' or tok==',', options, nn, "not enough arguments for macro '%s'", ntok)
|
|
local arg = collectArgument(ti, name == "__VA_ARGS__")
|
|
nargs[0][name] = arg
|
|
nargs[name] = callAndCollect(options, expandMacros, macros, yieldFromArray, arg, nn)
|
|
end
|
|
end
|
|
if def.nva then -- named variadic argument (implies dialectGnu)
|
|
nargs[def.nva] = nargs["__VA_ARGS__"]
|
|
nargs[0][def.nva] = nargs[0]["__VA_ARGS__"]
|
|
end
|
|
xassert(tok, options, nn, "unterminated arguments for macro '%s'", ntok)
|
|
xassert(tok==')', options, nn, "too many arguments for macro '%s'", ntok)
|
|
return nargs
|
|
end
|
|
-- coroutine that substitute the macro arguments
|
|
-- and stringification and concatenation are handled here
|
|
local function substituteArguments(options, def, nargs, n, inDirective)
|
|
local uargs = nargs[0] or nargs -- unexpanded argument values
|
|
if inDirective then nargs = uargs end -- use unexpanded arguments in directives
|
|
-- prepare loop
|
|
local i,j,k = 1,1,1
|
|
while def[i] do
|
|
if isBlank(def[i]) then
|
|
-- copy blanks
|
|
coroutine.yield(def[i], n)
|
|
else
|
|
-- positions j and k on next non-space tokens
|
|
local function updateJandK()
|
|
if j <= i then j=i
|
|
repeat j=j+1 until def[j] == nil or not isBlank(def[j]) end
|
|
if k <= j then k=j
|
|
repeat k=k+1 until def[k] == nil or not isBlank(def[k]) end
|
|
end
|
|
updateJandK()
|
|
-- alternatives
|
|
if def[i]=='#' and def[j] and nargs[def[j]] then
|
|
-- stringification (with the weird quoting rules)
|
|
local v = { '\"' }
|
|
for _,t in ipairs(uargs[def[j]]) do
|
|
if type(t)=='string' then
|
|
if t:find("^%s+$") then t = ' ' end
|
|
if t:find("^[\'\"]") then t = string.format("%q", t):sub(2,-2) end
|
|
v[1+#v] = t end end
|
|
v[1+#v] = '\"'
|
|
coroutine.yield(tableConcat(v), n)
|
|
i = j
|
|
elseif def.nva and def[i]==',' and def[j]=='##' and def[k]==def.nva then
|
|
-- named variadic macro argument with ## to signal negative comma (gcc crap)
|
|
if nargs[def.nva].negComma then i=i+1 end
|
|
while i < j do coroutine.yield(def[i], n) ; i=i+1 end
|
|
elseif def[i]==',' and def[j]=='__VA_ARGS__' and def[k]==')' then
|
|
-- __VA_ARGS__ with implied negative comma semantics
|
|
if nargs[def[j]].negComma then i=i+1 end
|
|
while i < j do coroutine.yield(def[i], n) ; i=i+1 end
|
|
i = j-1
|
|
elseif def[j]=='##' and def[k] and not inDirective then
|
|
-- concatenation
|
|
local u = {}
|
|
local function addToU(s)
|
|
if nargs[s] then for _,v in ipairs(uargs[s]) do u[1+#u] = v end
|
|
else u[1+#u]=s end end
|
|
addToU(def[i])
|
|
while def[j] == '##' and def[k] do
|
|
addToU(def[k])
|
|
i = k
|
|
updateJandK()
|
|
end
|
|
tokenizeLine(options, tableConcat(u), n, true)
|
|
elseif nargs[def[i]] then
|
|
-- substitution
|
|
yieldFromArray(options, nargs[def[i]], n)
|
|
else
|
|
-- copy
|
|
coroutine.yield(def[i], n)
|
|
end
|
|
end
|
|
i = i + 1
|
|
end
|
|
end
|
|
-- main loop
|
|
local newline, directive = true, false
|
|
while tok ~= nil do
|
|
-- detects Zpassed directives
|
|
if newline and tok == '#' then
|
|
newline, directive = false, true
|
|
elseif not isBlank(tok) then
|
|
newline = false
|
|
elseif isNewline(tok) then
|
|
newline, directive = true, false
|
|
end
|
|
-- process code
|
|
local def = macros[tok]
|
|
if not def or directive then
|
|
-- not a macro
|
|
coroutine.yield(tok, n)
|
|
elseif type(def) == 'function' then
|
|
-- magic macro
|
|
def(ti,tok,n)
|
|
elseif def.args == nil then
|
|
-- object-like macro
|
|
prependToken({tag='pop'},n)
|
|
prependTokens(wrap(options, substituteArguments, def, {}, n))
|
|
prependToken({tag='push', symb=tok},n)
|
|
else
|
|
-- function-like macro
|
|
local ntok, nn = tok,n
|
|
local spc = false
|
|
ti()
|
|
if isSpace(tok) then spc=true ti() end
|
|
if (tok ~= '(') then
|
|
coroutine.yield(ntok, nn)
|
|
if spc then coroutine.yield(' ', n) end
|
|
if tok then prependToken(tok,n) end
|
|
else
|
|
local nargs = collectArguments(ti,def,ntok,nn)
|
|
if def.lines == nil then
|
|
-- single-line function-like macro
|
|
prependToken({tag='pop'},n)
|
|
prependTokens(wrap(options, substituteArguments, def, nargs, nn))
|
|
prependToken({tag='push', symb=ntok},nn)
|
|
else
|
|
-- multi-line function-like macro
|
|
local lines = def.lines
|
|
-- a coroutine that yields the macro definition
|
|
local function yieldMacroLines()
|
|
local count = 0
|
|
for i=1,#lines,2 do
|
|
local ls,ln = lines[i], lines[i+1]
|
|
-- are we possibly in a cpp directive
|
|
local dir = false
|
|
if ls[2] and ls[2]:find('^#') then
|
|
dir = isIdentifier(ls[3]) and ls[3] or ls[4]
|
|
end
|
|
if dir and nargs[dir] then
|
|
dir = false -- leading stringification
|
|
elseif dir == 'defmacro' then
|
|
count = count + 1 -- entering a multiline macto
|
|
elseif dir == 'endmacro' then
|
|
count = count - 1 -- leaving a multiline macro
|
|
end
|
|
dir = dir or count > 0
|
|
-- substitute
|
|
ls = callAndCollect(options,substituteArguments,ls,nargs,ln,dir)
|
|
-- compute lines (optimize speed by passing body lines as tokens)
|
|
local j=1
|
|
while isBlank(ls[j]) do j=j+1 end
|
|
if ls[j] and ls[j]:find("^#") then -- but not directives
|
|
ls = ls[1]:sub(2) .. tableConcat(ls, nil, 2)
|
|
end
|
|
coroutine.yield(ls,ln)
|
|
end
|
|
end
|
|
-- recursively reenters preprocessing subroutines in order to handle
|
|
-- preprocessor directives located inside the macro expansion. As a result
|
|
-- we cannot expand macro invocations that extend beyond the macro-expansion.
|
|
local nmacros = {}
|
|
setmetatable(nmacros,{__index=macros})
|
|
if not def.recursive then nmacros[ntok]=false end
|
|
if not def.recursive then coroutine.yield({tag='push',symb=ntok}) end
|
|
expandMacros(options, nmacros, tokenize, processDirectives, nmacros, yieldMacroLines)
|
|
if not def.recursive then coroutine.yield({tag='pop'}) end
|
|
end
|
|
end
|
|
end
|
|
ti()
|
|
end
|
|
end
|
|
|
|
|
|
-- Processing conditional directive requires evaluating conditions
|
|
-- This function takes an iterator on preprocessed expression tokens
|
|
-- and computes the value. This does not handle defined(X) expressions.
|
|
-- Optional argument resolver is a function that takes an indentifer
|
|
-- name and returns a value. Otherwise zero is assumed
|
|
|
|
local function evaluateCppExpression(options, tokenIterator, n, resolver)
|
|
-- redefine token iterator to skip spaces and update tok
|
|
local tok
|
|
local function ti()
|
|
repeat tok = tokenIterator()
|
|
until not isBlank(tok) return tok
|
|
end
|
|
-- operator tables
|
|
local unaryOps = {
|
|
["!"] = function(v) return v == 0 and 1 or 0 end,
|
|
["~"] = function(v) return bit.bnot(v) end,
|
|
["+"] = function(v) return v end,
|
|
["-"] = function(v) return -v end,
|
|
["L"] = function(v) return v end
|
|
}
|
|
local binaryOps = {
|
|
["*"] = function(a,b) return a * b end,
|
|
["/"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return math.floor(a / b) end,
|
|
["%"] = function(a,b) xassert(b~=0,options,n,"division by zero"); return a % b end,
|
|
["+"] = function(a,b) return a + b end,
|
|
["-"] = function(a,b) return a - b end,
|
|
[">>"] = function(a,b) return bit.lshift(a, -b) end,
|
|
["<<"] = function(a,b) return bit.lshift(a, b) end,
|
|
[">="] = function(a,b) return a >= b and 1 or 0 end,
|
|
["<="] = function(a,b) return a <= b and 1 or 0 end,
|
|
[">"] = function(a,b) return a > b and 1 or 0 end,
|
|
["<"] = function(a,b) return a < b and 1 or 0 end,
|
|
["=="] = function(a,b) return a == b and 1 or 0 end,
|
|
["!="] = function(a,b) return a ~= b and 1 or 0 end,
|
|
["&"] = function(a,b) return bit.band(a,b) end,
|
|
["^"] = function(a,b) return bit.bxor(a,b) end,
|
|
["|"] = function(a,b) return bit.bor(a,b) end,
|
|
["&&"] = function(a,b) return (a ~= 0 and b ~= 0) and 1 or 0 end,
|
|
["||"] = function(a,b) return (a ~= 0 or b ~= 0) and 1 or 0 end,
|
|
}
|
|
local binaryPrec = {
|
|
["*"] = 1, ["/"] = 1, ["%"] = 1,
|
|
["+"] = 2, ["-"] = 2,
|
|
[">>"] = 3, ["<<"] = 3,
|
|
[">="] = 4, ["<="] = 4, ["<"] = 4, [">"] = 4,
|
|
["=="] = 5, ["!="] = 5,
|
|
["&"] = 6, ["^"] = 7, ["|"] = 8,
|
|
["&&"] = 9, ["||"] = 10
|
|
}
|
|
-- forward
|
|
local function evaluate() end
|
|
-- unary operations
|
|
local function evalUnary()
|
|
if unaryOps[tok] then
|
|
local op = unaryOps[tok]
|
|
ti(); return op(evalUnary())
|
|
elseif tok == '(' then
|
|
ti(); local v = evaluate()
|
|
xassert(tok == ')', options, n, "missing closing parenthesis")
|
|
ti(); return v
|
|
elseif tok == 'defined' then -- magic macro should have removed this
|
|
xerror(options, n, "syntax error after <defined>")
|
|
elseif isIdentifier(tok) then
|
|
local v = type(resolver) == 'function' and resolver(tok,ti)
|
|
ti(); return v or 0
|
|
elseif isNumber(tok) then
|
|
local v = tok:gsub("[ULul]+$","")
|
|
if v:find("^0[0-7]+$") then
|
|
v = tonumber(v,8) -- octal
|
|
elseif v:find("^0[bB][01]+") then
|
|
v = tonumber(v:sub(3),2) -- binary
|
|
else
|
|
v = tonumber(v) -- lua does the rest
|
|
end
|
|
xassert(v and v == math.floor(v), options, n, "syntax error (invalid integer '%s')", tok)
|
|
ti(); return v
|
|
elseif isString(tok) then
|
|
local v = '""'
|
|
if tok:find("^'") then -- interpret character constant as number
|
|
v = evalLuaExpression(string.format("return string.byte(%s)", tok))
|
|
xassert(type(v)=='number', options, n, "syntax error (invalid value '%s')", tok)
|
|
ti()
|
|
else
|
|
while isString(tok) do
|
|
xassert(tok:find('^"'), options, n, "syntax error (invalid value '%s')", tok)
|
|
v = v:gsub('"$','') .. tok:gsub('^"','')
|
|
ti()
|
|
end
|
|
end
|
|
return v
|
|
end
|
|
xerror(options, n, "syntax error (invalid value '%s')", tok)
|
|
end
|
|
-- binary operations
|
|
local function evalBinary(p)
|
|
if p == 0 then
|
|
return evalUnary()
|
|
else
|
|
local val = evalBinary(p-1)
|
|
while binaryPrec[tok] == p do
|
|
local op = binaryOps[tok] ti()
|
|
local oval = evalBinary(p-1)
|
|
xassert(p==4 or p==5 or type(val)=='number', options, n,
|
|
"expression uses arithmetic operators on strings")
|
|
xassert(type(val) == type(oval), options, n,
|
|
"expression compares numbers and strings")
|
|
val = op(val,oval)
|
|
end
|
|
return val
|
|
end
|
|
end
|
|
-- eval ternary conditonal
|
|
local function evalTernary()
|
|
local c = evalBinary(10)
|
|
if tok ~= '?' then return c end
|
|
ti() local v1 = evalBinary(10)
|
|
xassert(tok == ':', options, n, "expecting ':' after '?'")
|
|
ti() local v2 = evalBinary(10)
|
|
if c == 0 then return v2 else return v1 end
|
|
end
|
|
|
|
-- actual definition of evaluate
|
|
evaluate = function()
|
|
return evalTernary()
|
|
end
|
|
-- main function
|
|
ti()
|
|
xassert(tok, options, n, "constant expression expected");
|
|
local result = evaluate()
|
|
if hasOption(options, "-d:eval") then
|
|
xdebug(n, "eval %s", result)
|
|
end
|
|
-- warn about garbage when called from cpp (but not when called with resolver)
|
|
while isBlank(tok) do ti() end
|
|
xassert(resolver or not tok, options, n, "garbage after conditional expression");
|
|
return result
|
|
end
|
|
|
|
|
|
-- Now dealing with the coroutine that processes all directives.
|
|
-- This coroutine obtains lines from coroutine <lines>,
|
|
-- processes all directives, and yields remaining lines
|
|
|
|
processDirectives = function(options, macros, lines, ...)
|
|
local li = wrap(options, lines, ...)
|
|
local s, n = li()
|
|
-- redefine li to make sure vars s and n are up-to-date
|
|
local li = function() s,n=li() return s,n end
|
|
-- directives store their current token in these vars
|
|
local dirtok, tok, spc
|
|
-- forward declaration
|
|
local processLine
|
|
-- the captureTable mechanism communicates certain preprocessor
|
|
-- events to the declaration parser in order to report them
|
|
-- to the user (the parsing does not depend on this).
|
|
-- if macros[1] is a table, the preprocessor will append
|
|
-- records to this table for the parser to process.
|
|
local function hasCaptureTable()
|
|
local captable = rawget(macros,1)
|
|
return captable and type(captable)=='table' and captable
|
|
end
|
|
local function addToCaptureTable(record)
|
|
local captable = hasCaptureTable()
|
|
if captable and record then captable[1+#captable] = record end
|
|
end
|
|
|
|
-- simple directives
|
|
local function doIgnore()
|
|
if hasOption(options, "-Zpass") then coroutine.yield(s, n) end
|
|
end
|
|
local function doError()
|
|
xerror(options, n, "unexpected preprocessor directive #%s", dirtok)
|
|
end
|
|
local function doMessage()
|
|
local msg = s:match("^%s*#+%s*[a-z]*%s+([^%s].*)")
|
|
xmessage(dirtok, options, n, msg or '#' .. dirtok)
|
|
end
|
|
-- undef
|
|
local function doUndef(ti)
|
|
ti()
|
|
local nam = tok
|
|
xassert(isIdentifier(nam), options, n, "symbol expected after #undef")
|
|
if ti() then xwarning(options, n, "garbage after #undef directive") end
|
|
if hasOption(options, "-d:defines") then xdebug(n, "undef %s", nam) end
|
|
if hasCaptureTable() and macros[nam] and macros[nam].captured then
|
|
addToCaptureTable{directive='undef',name=nam, where=n}
|
|
end
|
|
macros[nam] = false -- false overrides inherited definitions
|
|
end
|
|
-- define
|
|
local function getMacroArguments(ti)
|
|
local args = {}
|
|
local msg = "argument list in function-like macro"
|
|
local nva = nil -- named variadic argument
|
|
ti()
|
|
while tok and tok ~= ')' do
|
|
local nam = tok
|
|
ti()
|
|
if options.dialectGnu and isIdentifier(nam) and tok == '...' then nam,nva=tok,nam ; ti() end
|
|
xassert(nam ~= "__VA_ARGS__", options, n, "name __VA_ARGS__ is not allowed here")
|
|
xassert(tok == ')' or nam ~= '...', options, n, "ellipsis in argument list must appear last")
|
|
xassert(tok == ')' or tok == ',', options, n, "bad " .. msg)
|
|
if tok == ',' then ti() end
|
|
if nam == '...' then nam = "__VA_ARGS__" end
|
|
xassert(isIdentifier(nam), options, n, "bad " .. msg)
|
|
args[1+#args] = nam
|
|
end
|
|
xassert(tok == ')', options, n, "unterminated " .. msg)
|
|
ti()
|
|
return args, nva
|
|
end
|
|
local function doDefine(ti)
|
|
xassert(isIdentifier(ti()), options, n, "symbol expected after #define")
|
|
local nam, args, nva = tok, nil, nil
|
|
-- collect arguments
|
|
if ti() == '(' and not spc then
|
|
args, nva = getMacroArguments(ti)
|
|
end
|
|
-- collect definition
|
|
local def = { tok, args = args, nva = nva }
|
|
while ti(true) do
|
|
def[1+#def] = tok
|
|
end
|
|
-- define macro
|
|
if macros[nam] and not tableCompare(def,macros[nam]) then
|
|
xwarning(options, n,"redefinition of preprocessor symbol '%s'", nam)
|
|
end
|
|
macros[nam] = def
|
|
-- debug
|
|
if hasOption(options, "-d:defines") then
|
|
if args then args = "(" .. tableConcat(args,",") .. ")" end
|
|
xdebug(n, "define %s%s = %s", nam, args or "", tableConcat(def,' '))
|
|
end
|
|
-- capture integer macro definitions
|
|
if hasCaptureTable() and args == nil then
|
|
local i = 0
|
|
local v = callAndCollect(options, expandMacros, macros, yieldFromArray, def, n)
|
|
local function ti() i=i+1 return v[i] end
|
|
local ss,r = pcall(evaluateCppExpression,{silent=true}, ti, n, error)
|
|
if ss and type(r)=='number' then
|
|
def.captured = true
|
|
addToCaptureTable{directive='define', name=nam, intval=r, where=n}
|
|
end
|
|
end
|
|
end
|
|
-- defmacro
|
|
local function checkDirective(stop)
|
|
xassert(s, options, n, "unterminated macro (missing #%s)", stop)
|
|
local r = type(s) == 'string' and s:match("^%s*#%s*([a-z]+)")
|
|
if r == "endmacro" or r == "endif" then
|
|
if s:find(r .. "%s*[^%s]") then
|
|
xwarning(options, n, "garbage after #%s directive", r)
|
|
end
|
|
end
|
|
return r
|
|
end
|
|
local function doMacroLines(lines, stop)
|
|
while true do
|
|
li()
|
|
local ss = callAndCollect(options,tokenizeLine,s,n)
|
|
if #ss > 0 then lines[1+#lines] = ss ; lines[1+#lines] = n end
|
|
local r = checkDirective(stop)
|
|
if r == "endmacro" or r == "endif" then
|
|
xassert(r == stop, options,n, "unbalanced directives (got #%s instead of #%s)",r,stop)
|
|
return r
|
|
elseif r=="defmacro" then
|
|
doMacroLines(lines,"endmacro")
|
|
elseif r == "if" or r == "ifdef" or r == "ifndef" then
|
|
doMacroLines(lines,"endif")
|
|
end
|
|
end
|
|
end
|
|
local function doDefmacro(ti)
|
|
xassert(isIdentifier(ti()), options, n, "symbol expected after #defmacro")
|
|
local nam,nn = tok,n
|
|
xassert(ti()=='(', options, n, "argument list expected in #defmacro")
|
|
local args, nva = getMacroArguments(ti)
|
|
xassert(not tok, options, n, "garbage after argument list in #defmacro")
|
|
-- collect definition
|
|
local lines = {}
|
|
local def = { args=args, nva=nva, lines=lines, recursive=(dirtok=="defrecursivemacro") }
|
|
doMacroLines(lines,"endmacro")
|
|
lines[#lines] = nil
|
|
lines[#lines] = nil
|
|
if hasOption(options,"-d:directives") then
|
|
xdebug(n, "directive: #endmacro")
|
|
end
|
|
if macros[nam] and not tableCompare(def,macros[nam]) then
|
|
xwarning(options, n, "redefinition of preprocessor symbol '%s'", nam)
|
|
end
|
|
if hasOption(options, "-d:defines") then
|
|
xdebug(nn, "defmacro %s(%s) =", nam, tableConcat(args,','))
|
|
for i=1,#lines,2 do
|
|
xdebug(lines[i+1], "\t%s", tableConcat(lines[i]):gsub("^\n","")) end
|
|
end
|
|
macros[nam] = def
|
|
end
|
|
-- include
|
|
local function doInclude(ti)
|
|
-- get filename
|
|
local pti = wrap(options, expandMacros, macros, yieldFromIterator, ti)
|
|
local tok = pti()
|
|
while isBlank(tok) do tok=pti() end
|
|
if tok == '<' then -- computed include
|
|
repeat local tok2 = pti()
|
|
tok = tok .. tostring(tok2)
|
|
until tok2==nil or tok2=='>' or isNewline(tok2)
|
|
tok = tok:gsub('%s>$','>') -- gcc does this
|
|
end
|
|
xassert(isHeaderName(tok), options, n, "malformed header name after #include")
|
|
local ttok = pti()
|
|
while isBlank(ttok) do ttok=pti() end
|
|
if ttok then xwarning(options, n, "garbage after #include directive") end
|
|
-- interpret filename
|
|
local sys = tok:byte() == 60
|
|
local min = dirtok=="include_next" and options.includedir or 0
|
|
local fname = evalLuaExpression(string.format("return '%s'", tok:sub(2,-2)))
|
|
local pname, fd, fdi
|
|
for i,v in ipairs(options) do
|
|
if v == "-I-" then
|
|
sys=false
|
|
elseif i > min and v:find("^%-I") and not sys then
|
|
pname = v:match("^%-I%s*(.*)") .. '/' .. fname
|
|
fdi, fd = i, io.open(pname, "r")
|
|
if fd then break end
|
|
end
|
|
end
|
|
if fd then
|
|
-- include file
|
|
if hasOption(options, "-d:include") then xdebug(n, "including %q", pname) end
|
|
local savedfdi = options.includedir
|
|
options.includedir = fdi -- saved index to implement include_next
|
|
processDirectives(options, macros, eliminateComments, joinLines,
|
|
yieldLines, fd:lines(), pname)
|
|
options.includedir = savedfdi
|
|
else
|
|
-- include file not found
|
|
if hasOption(options, "-Zpass") then
|
|
coroutine.yield(string.format("#include %s",tok), n)
|
|
else
|
|
xwarning(options, n, "include directive (%s) was unresolved", tok)
|
|
end
|
|
-- quirks
|
|
if knownIncludeQuirks[tok] then
|
|
processDirectives(options, macros, eliminateComments, joinLines,
|
|
yieldFromArray, knownIncludeQuirks[tok], n)
|
|
end
|
|
-- capture
|
|
if hasCaptureTable() then
|
|
addToCaptureTable{directive='include',name=tok, where=n}
|
|
end
|
|
end
|
|
end
|
|
-- conditionals
|
|
local function doConditionalBranch(execute)
|
|
checkDirective("endif")
|
|
while true do
|
|
li()
|
|
local r = checkDirective("endif")
|
|
if r == "else" or r == "elif" or r == "endif" then
|
|
return r
|
|
elseif execute then
|
|
processLine()
|
|
elseif r == "if" or r == "ifdef" or r == "ifndef" then
|
|
while doConditionalBranch(false) ~= "endif" do end
|
|
end
|
|
end
|
|
end
|
|
local function doConditional(result)
|
|
local r = doConditionalBranch(result)
|
|
if r == 'elif' and not result then
|
|
return processLine(true)
|
|
end
|
|
while r ~= "endif" do
|
|
r = doConditionalBranch(not result)
|
|
end
|
|
if hasOption(options,"-d:directives") then
|
|
xdebug(n, "directive: %s",s:gsub('^%s*',''))
|
|
end
|
|
end
|
|
local function doIfdef(ti)
|
|
ti()
|
|
xassert(isIdentifier(tok), options, n, "symbol expected after #%s", dirtok)
|
|
local result = macros[tok]
|
|
if ti() then xwarning(options, n, "garbage after #undef directive") end
|
|
if dirtok == 'ifndef' then result = not result end
|
|
doConditional(result)
|
|
end
|
|
local function doIf(ti)
|
|
-- magic macro for 'defined'
|
|
local nmacros = {}
|
|
setmetatable(nmacros,{__index=macros})
|
|
nmacros['defined'] = function(ti)
|
|
local tok,n = ti()
|
|
if tok == '(' then tok = ti()
|
|
if ti() ~= ')' then tok = nil end end
|
|
if isIdentifier(tok) then
|
|
coroutine.yield(macros[tok] and "1" or "0", n)
|
|
else
|
|
coroutine.yield("defined", n) -- error
|
|
end
|
|
end
|
|
-- evaluate and branch
|
|
local pti = wrap(options, expandMacros, nmacros, yieldFromIterator, ti)
|
|
local result = evaluateCppExpression(options, pti, n)
|
|
doConditional(result ~= 0)
|
|
end
|
|
-- table of directives
|
|
local directives = {
|
|
["else"] = doError, ["elif"] = doError, ["endif"] = doError,
|
|
["pragma"] = doIgnore, ["ident"] = doIgnore, ["line"] = doIgnore,
|
|
["error"] = doMessage, ["warning"] = doMessage,
|
|
["if"] = doIf, ["ifdef"] = doIfdef, ["ifndef"] = doIfdef,
|
|
["define"] = doDefine, ["undef"] = doUndef,
|
|
["defmacro"] = doDefmacro, ["defrecursivemacro"] = doDefmacro,
|
|
["endmacro"] = doError,
|
|
["include"] = doInclude, ["include_next"] = doInclude,
|
|
}
|
|
-- process current line
|
|
processLine = function(okElif)
|
|
if type(s) == 'table' then
|
|
-- optimization for multiline macros:
|
|
-- When s is an an array of precomputed tokens, code is assumed.
|
|
coroutine.yield(s, n)
|
|
elseif not s:find("^%s*#") then
|
|
-- code
|
|
coroutine.yield(s, n)
|
|
elseif s:find("^%s*##") and hasOption(options, "-Zpass") then
|
|
-- pass
|
|
local ns = s:gsub("^(%s*)##","%1#")
|
|
coroutine.yield(ns, n)
|
|
else
|
|
if hasOption(options, "-d:directives") then
|
|
xdebug(n, "directive: %s",s:gsub('^%s*',''))
|
|
end
|
|
-- tokenize directive
|
|
local ti = wrap(options, tokenizeLine, s, n)
|
|
-- a token iterator that skips spaces unless told otherwise
|
|
local ti = function(keepSpaces)
|
|
tok = ti()
|
|
spc = isSpace(tok)
|
|
while not keepSpaces and isBlank(tok) do
|
|
tok = ti()
|
|
spc = spc or isSpace(tok)
|
|
end
|
|
return tok, n
|
|
end
|
|
-- start parsing directives
|
|
ti()
|
|
assert(tok=='#' or tok=='##')
|
|
if tok == '##' then
|
|
xwarning(options, n, "directive starts with ## without -Zpass") end
|
|
dirtok = ti()
|
|
if isIdentifier(tok) then
|
|
local f = directives[dirtok]
|
|
if okElif and dirtok == "elif" then f = doIf end
|
|
xassert(f, options, n, "unrecognized preprocessor directive #%s", tok)
|
|
f(ti)
|
|
elseif tok ~= nil then
|
|
xerror(options, n, "unrecognized preprocessor directive '#%s'", s:gsub("^%s*",""))
|
|
end
|
|
end
|
|
end
|
|
-- main loop
|
|
while s ~= nil do
|
|
processLine()
|
|
li()
|
|
end
|
|
end
|
|
|
|
|
|
-- This function yields initialization lines
|
|
|
|
local function initialDefines(options)
|
|
-- cpp-extracted definitions
|
|
if hasOption(options, "-Zcppdef") then
|
|
local fd = io.popen("cpp -dM < /dev/null","r")
|
|
yieldLines(options, fd:lines(), "<cppdef>")
|
|
fd:close()
|
|
end
|
|
-- builtin definitions
|
|
local sb = { "#define __CPARSER__ 1" }
|
|
local function addDef(s,v)
|
|
sb[1+#sb] = string.format("#ifndef %s",s)
|
|
sb[1+#sb] = string.format("# define %s %s",s,v)
|
|
sb[1+#sb] = string.format("#endif")
|
|
end
|
|
addDef("__STDC__", "1")
|
|
local stdc = "199409L"
|
|
if options.dialect11 then stdc = "201112L" end
|
|
if options.dialect99 then stdc = "199901L" end
|
|
addDef("__STDC_VERSION__", stdc)
|
|
if options.dialectGnu then
|
|
addDef("__GNUC__", 4)
|
|
addDef("__GNUC_MINOR__", 2)
|
|
end
|
|
yieldLines(options, wrap(options, yieldFromArray, sb), "<builtin>")
|
|
-- command line definitions
|
|
local sc = {}
|
|
for _,v in ipairs(options) do
|
|
local d
|
|
if v:find("^%-D(.*)=") then
|
|
d = v:gsub("^%-D%s*(.*)%s*=%s*(.-)%s*$","#define %1 %2")
|
|
elseif v:find("^%-D") then
|
|
d = v:gsub("^%-D%s*(.-)%s*$","#define %1 1")
|
|
elseif v:find("^%-U") then
|
|
d = v:gsub("^%-U%s*(.-)%s*$","#undef %1")
|
|
end
|
|
if d then sc[1+#sc] = d end
|
|
end
|
|
yieldLines(options, wrap(options, yieldFromArray, sc), "<cmdline>")
|
|
end
|
|
|
|
|
|
-- This function creates the initial macro directory
|
|
|
|
local function initialMacros(options)
|
|
local macros = {}
|
|
-- magic macros
|
|
macros["__FILE__"] = function(_,_,n)
|
|
local f
|
|
if type(n) == 'string' then f=n:match("^[^:]*") end
|
|
coroutine.yield(string.format("%q", f or "<unknown>"), n)
|
|
end
|
|
macros["__LINE__"] = function(_,_,n)
|
|
local d = n
|
|
if type(d) == 'string' then d=tonumber(d:match("%d*$")) end
|
|
coroutine.yield(string.format("%d", d or 0), n)
|
|
end
|
|
macros["__DATE__"] = function(_,_,n)
|
|
coroutine.yield(string.format("%q", os.date("%b %e %Y")), n)
|
|
end
|
|
macros["__TIME__"] = function(_,_,n)
|
|
coroutine.yield(string.format("%q", os.date("%T")), n)
|
|
end
|
|
--< @r-lyeh: add __COUNTER__
|
|
macros["__COUNTER__"] = function(_,_,n)
|
|
counter = counter or 0
|
|
counter = counter + 1
|
|
coroutine.yield(string.format("%d", counter), n)
|
|
end
|
|
--<
|
|
-- initial macros
|
|
local li = wrap(options,processDirectives,macros,initialDefines)
|
|
for _ in li do end
|
|
-- return
|
|
return macros
|
|
end
|
|
|
|
|
|
-- This function prepares a string containing the definition of the
|
|
-- macro named <name> in macro definition table <macros>, or nil if no
|
|
-- such definition exists.
|
|
|
|
local function macroToString(macros, name)
|
|
local v = macros[name]
|
|
if type(v) == 'table' then
|
|
local dir = 'define'
|
|
if v.recursive and v.lines then
|
|
dir = 'defrecursivemacro'
|
|
elseif v.lines then
|
|
dir = 'defmacro'
|
|
end
|
|
local arr = {"#", dir, ' ', name }
|
|
if v.args then
|
|
arr[1+#arr] = '('
|
|
for i,s in ipairs(v.args) do
|
|
if i ~= 1 then arr[1+#arr] = ',' end
|
|
if s == '__VA_ARGS__' then s = (v.nva or '') .. '...' end
|
|
arr[1+#arr] = s
|
|
end
|
|
arr[1+#arr] = ') '
|
|
else
|
|
arr[1+#arr] = ' '
|
|
end
|
|
for _,s in ipairs(v) do
|
|
arr[1+#arr] = s
|
|
end
|
|
if v.lines then
|
|
for i = 1, #v.lines, 2 do
|
|
local vl = v.lines[i]
|
|
arr[1+#arr] = '\n'
|
|
if type(vl)=='table' then vl=tableConcat(vl) end
|
|
arr[1+#arr] = vl:gsub('^%s?%s?',' '):gsub('^\n','')
|
|
end
|
|
arr[1+#arr] = '\n'
|
|
arr[1+#arr] = "#endmacro"
|
|
end
|
|
return tableConcat(arr)
|
|
end
|
|
end
|
|
|
|
-- This function dumps all macros to file descriptor outputfile
|
|
|
|
local function dumpMacros(macros, outputfile)
|
|
outputfile = outputfile or io.output()
|
|
assert(type(macros) == 'table')
|
|
assert(io.type(outputfile) == 'file')
|
|
for k,_ in pairs(macros) do
|
|
local s = macroToString(macros,k)
|
|
if s then outputfile:write(string.format("%s\n", s)) end
|
|
end
|
|
end
|
|
|
|
|
|
|
|
-- A coroutine that filters out spaces, directives, and magic tokens
|
|
|
|
local function filterSpaces(options, tokens, ...)
|
|
local ti = wrap(options, tokens, ...)
|
|
local tok,n = ti()
|
|
while tok do
|
|
-- skip directives
|
|
while isNewline(tok) do
|
|
tok,n = ti()
|
|
while isBlank(tok) do tok,n = ti() end
|
|
if tok == '#' then
|
|
while not isNewline(tok) do
|
|
tok, n = ti()
|
|
end
|
|
end
|
|
end
|
|
-- output nonspaces
|
|
if not isBlank(tok) then
|
|
coroutine.yield(tok, n)
|
|
end
|
|
tok,n = ti()
|
|
end
|
|
end
|
|
|
|
-- This function takes a line iterator and an optional location prefix.
|
|
-- It returns a token iterator for the preprocessed tokens
|
|
-- and a table of macro definitions.
|
|
|
|
local function cppTokenIterator(options, lines, prefix)
|
|
options = copyOptions(options)
|
|
prefix = prefix or ""
|
|
assert(type(options)=='table')
|
|
assert(type(lines)=='function')
|
|
assert(type(prefix)=='string')
|
|
local macros = initialMacros(options)
|
|
local ti = wrap(options,
|
|
filterSpaces,
|
|
expandMacros, macros,
|
|
tokenize,
|
|
processDirectives, macros,
|
|
eliminateComments,
|
|
joinLines,
|
|
yieldLines, lines, prefix)
|
|
return ti, macros
|
|
end
|
|
|
|
|
|
-- A coroutine that reconstructs lines from the preprocessed tokens
|
|
|
|
local function preprocessedLines(options, tokens, ...)
|
|
local ti = wrap(options, tokens, ...)
|
|
local tok,n = ti()
|
|
while tok do
|
|
local curn = n
|
|
local curl = {}
|
|
if isNewline(tok) then
|
|
curn = n
|
|
curl[1+#curl] = tok:sub(2)
|
|
tok, n = ti()
|
|
end
|
|
while tok and not isNewline(tok) do
|
|
if not isMagic(tok) then curl[1+#curl] = tok end
|
|
tok, n = ti()
|
|
end
|
|
coroutine.yield(table.concat(curl), curn)
|
|
end
|
|
end
|
|
|
|
|
|
-- This function preprocesses file <filename>.
|
|
-- The optional argument <outputfile> specifies where to write the
|
|
-- preprocessed file and may be a string or a file descriptor.
|
|
-- The optional argument <options> contains an array of option strings.
|
|
-- Note that option "-Zpass" is added, unless the option "-Znopass" is present.
|
|
|
|
--< @r-lyeh: iterator+string methods
|
|
local function cppIterator(lines, outputfile, options, prefix)
|
|
-- handle optional arguments
|
|
prefix = prefix or ''
|
|
options = copyOptions(options)
|
|
outputfile = outputfile or "-"
|
|
assert(type(lines)=='function')
|
|
assert(type(options)=='table')
|
|
local closeoutputfile = false
|
|
if io.type(outputfile) ~= 'file' then
|
|
assert(type(outputfile) == 'string')
|
|
if outputfile == '-' then
|
|
outputfile = io.output()
|
|
else
|
|
closeoutputfile = true
|
|
outputfile = io.open(outputfile,"w")
|
|
end
|
|
end
|
|
assert(io.type(outputfile) == 'file')
|
|
-- makes option -Zpass on by default
|
|
if not hasOption(options,"-Znopass") then
|
|
options.hash["-Zpass"] = true
|
|
end
|
|
-- prepare iterator
|
|
local dM = hasOption(options, "-dM")
|
|
local macros = initialMacros(options)
|
|
local li = wrap(options,
|
|
preprocessedLines,
|
|
expandMacros, macros,
|
|
tokenize,
|
|
processDirectives, macros,
|
|
eliminateComments,
|
|
joinLines,
|
|
yieldLines, lines, prefix)
|
|
-- iterate, inserting line markers
|
|
local lm = hasOption(options,"-Zpass") and "line" or ""
|
|
local cf, cn
|
|
for s,n in li do
|
|
if not dM and s:find("[^%s]") then
|
|
local xf, xn
|
|
if type(n) == 'number' then
|
|
xn = n
|
|
elseif type(n) == 'string' then
|
|
xf, xn = n:match("^([^:]*).-(%d*)$")
|
|
xn = tonumber(xn)
|
|
end
|
|
if cf ~= xf or cn ~= xn then
|
|
cf, cn = xf, xn
|
|
if lm ~= "" then --< @r-lyeh: do not output empty #line directives: ie, `#line 19 "file.c"` -> `# 19 ""`
|
|
outputfile:write(string.format("#%s %d %q\n", lm, cn, cf))
|
|
end
|
|
end
|
|
outputfile:write(s)
|
|
outputfile:write("\n")
|
|
cn = cn + 1
|
|
end
|
|
end
|
|
if dM then
|
|
dumpMacros(macros, outputfile)
|
|
end
|
|
if closeoutputfile then outputfile:close() end
|
|
end
|
|
|
|
local function cppString(str, outputfile, options)
|
|
assert(type(str)=='string')
|
|
|
|
return cppIterator(str:gmatch("[^\r\n]+"), outputfile, options)
|
|
end
|
|
|
|
local function cpp(filename, outputfile, options)
|
|
assert(type(filename)=='string')
|
|
|
|
return cppIterator(io.lines(filename), outputfile, options, filename)
|
|
end
|
|
--<
|
|
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
---------------------------------------------------
|
|
-- EXPORTS
|
|
|
|
cparser = {}
|
|
|
|
cparser.cpp = cpp
|
|
cparser.cppTokenIterator = cppTokenIterator
|
|
cparser.macroToString = macroToString
|
|
|
|
--< @r-lyeh: add string methods
|
|
cparser.cppString = cppString
|
|
cparser.parseString = parseString
|
|
--<
|
|
|
|
return cparser
|