----------------------------------------------------------------------------
--## lcpp - a C-PreProcessor for Lua 5.1 and LuaJIT ffi integration
--
-- Copyright (C) 2012-2014 Michael Schmoock <michael@schmoock.net>
--
--### Links
-- * GitHub page:   [http://github.com/willsteel/lcpp](http://github.com/willsteel/lcpp)
-- * Project page:  [http://lcpp.schmoock.net](http://lcpp.schmoock.net)
-- * Lua:           [http://www.lua.org](http://www.lua.org)
-- * LuaJIT:        [http://luajit.org](http://luajit.org)
-- * Sponsored by:  [http://mmbbq.org](http://mmbbq.org)
--
-- It can be used to pre-process LuaJIT ffi C header file input.
-- It can also be used to preprocess any other code (i.e. Lua itself)
--
--  git clone https://github.com/willsteel/lcpp.git
----------------------------------------------------------------------------
--## USAGE
--  -- load lcpp
--  local lcpp = require("lcpp")
--
--  -- use LuaJIT ffi and lcpp to parse cpp code
--  local ffi = require("ffi")
--  ffi.cdef("#include <your_header.h>")
--
--  -- use lcpp manually but add some predefines
--  local lcpp = require("lcpp");
--  local out = lcpp.compileFile("your_header.h", {UNICODE=1});
--  print(out);
--
--  -- compile some input manually
--  local out = lcpp.compile([[
--      #include "myheader.h"
--      #define MAXPATH 260
--      typedef struct somestruct_t {
--          void*          base;
--          size_t         size;
--          wchar_t        path[MAXPATH];
--      } t_exe;
--  ]])
--  -- the result should be like this
--  out == [[
--      // <preprocessed content of file "myheader.h">
--      typedef struct somestruct_t {
--          void*          base;
--          size_t         size;
--          wchar_t        path[260];
--      } t_exe;
--  ]]
--
--  -- access lcpp defines dynamically (i.e. if used with ffi)
--  local ffi = require("ffi")
--  local lcpp = require("lcpp")
--  ffi.cdef("#include <your_header.h>")
--  =ffi.lcpp_defs.YOUR_DEFINE
--
--
--## This CPPs BNF:
--  RULES:
--  CODE              := {LINE}
--  LINE              := {STUFF NEWML} STUFF  NEWL
--  STUFF             := DIRECTIVE | IGNORED_CONTENT
--  DIRECTIVE         := OPTSPACES CMD OPTSPACES DIRECTIVE_NAME WHITESPACES DIRECTIVE_CONTENT WHITESPACES NEWL
--
--  LEAVES:
--  NEWL              := "\n"
--  NEWL_ESC          := "\\n"
--  WHITESPACES       := "[ \t]+"
--  OPTSPACES         := "[ \t]*"
--  COMMENT           := "//(.-)$"
--  MLCOMMENT         := "/[*](.-)[*]/"
--  IGNORED_CONTENT   := "[^#].*"
--  CMD               := "#"
--  DIRECTIVE_NAME    := "include"|"define"|"undef"|"if"|"else"|"elif"|"else if"|"endif"|"ifdef"|"ifndef"|"pragma"
--  DIRECTIVE_CONTENT := ".*?"
--
--## TODOs:
--  - lcpp.LCPP_LUA for: load, loadfile
--
--## License (MIT)
-- -----------------------------------------------------------------------------
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to deal
-- in the Software without restriction, including without limitation the rights
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-- copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-- THE SOFTWARE.
--
-- MIT license: http://www.opensource.org/licenses/mit-license.php
-- -----------------------------------------------------------------------------
--
-- @module lcpp
local lcpp = {}
-- check bit is avail or not
local ok, bit = pcall(require, 'bit')
if not ok then
bit = {
    lshift = function (x, y)
        if y < 0 then return bit.rshift(x,-y) end
        return (x * 2^y) % (2^32)
    end,
    rshift = function (x, y)
        if y < 0 then return bit.lshift(x,-y) end
        return math.floor(x % (2^32) / (2^y))
    end,
    bxor = function (x, y)
        -- from http://lua-users.org/wiki/BitUtils
        local z = 0
        for i = 0, 31 do
            if (x % 2 == 0) then                      -- x had a '0' in bit i
                if ( y % 2 == 1) then                  -- y had a '1' in bit i
                    y = y - 1
                    z = z + 2 ^ i                       -- set bit i of z to '1'
                end
            else                                      -- x had a '1' in bit i
                x = x - 1
                if (y % 2 == 0) then                  -- y had a '0' in bit i
                    z = z + 2 ^ i                       -- set bit i of z to '1'
                else
                    y = y - 1
                end
            end
            y = y / 2
            x = x / 2
        end
        return z
    end,
    bnot = function (x)
        -- if word size is not defined, I think it better than 0xFFFFFFFF - x.
        return -1 - x
    end,
    band = function (x, y)
        return ((x + y) - bit.bxor(x, y)) / 2
    end,
    bor = function (x, y)
        return bit.bnot(bit.band(bit.bnot(x), bit.bnot(y)))
    end,
}
end

-- CONFIG
lcpp.LCPP_LUA         = false   -- whether to use lcpp to preprocess Lua code (load, loadfile, loadstring...)
lcpp.LCPP_FFI         = true    -- whether to use lcpp as LuaJIT ffi PreProcessor (if used in luaJIT)
lcpp.LCPP_TEST        = false   -- whether to run lcpp unit tests when loading lcpp module
lcpp.ENV              = {}      -- static predefines (env-like)
lcpp.FAST             = false   -- perf. tweaks when enabled. con: breaks minor stuff like __LINE__ macros
lcpp.DEBUG            = false

-- PREDEFINES
local __FILE__        = "__FILE__"
local __LINE__        = "__LINE__"
local __DATE__        = "__DATE__"
local __TIME__        = "__TIME__"
local __LCPP_INDENT__ = "__LCPP_INDENT__"

-- BNF LEAVES
local ENDL            = "$"
local STARTL          = "^"
local NEWL            = "\n"
local NEWL_BYTE       = NEWL:byte(1)
local NEWL_ESC        = "\\"
local NEWML           = "\\\n"
local CMD             = "#"
local CMD_BYTE        = CMD:byte(1)
local COMMENT         = "^(.-)//.-$"
local MLCOMMENT       = "/[*].-[*]/"
local WHITESPACES     = "%s+"
local OPTSPACES       = "%s*"
local IDENTIFIER      = "[_%a][_%w]*"
local NOIDENTIFIER    = "[^%w_]+"
local FILENAME        = "[0-9a-zA-Z.%-_/\\]+"
local TEXT            = ".+"
local STRINGIFY       = "#"
local STRINGIFY_BYTE  = STRINGIFY:byte(1)
local STRING_LITERAL  = ".*"

-- BNF WORDS
local _INCLUDE        = "include"
local _INCLUDE_NEXT   = "include_next"
local _DEFINE         = "define"
local _IFDEF          = "ifdef"
local _IFNDEF         = "ifndef"
local _ENDIF          = "endif"
local _UNDEF          = "undef"
local _IF             = "if"
local _ELSE           = "else"
local _ELIF           = "elif"
local _NOT            = "!"
local _ERROR          = "error"
local _WARNING        = "warning"
local _PRAGMA         = "pragma"

-- BNF RULES
local INCLUDE         = STARTL.._INCLUDE..WHITESPACES.."[<]("..FILENAME..")[>]"..OPTSPACES..ENDL
local LOCAL_INCLUDE   = STARTL.._INCLUDE..WHITESPACES.."[\"]("..FILENAME..")[\"]"..OPTSPACES..ENDL
local INCLUDE_NEXT    = STARTL.._INCLUDE_NEXT..WHITESPACES.."[\"<]("..FILENAME..")[\">]"..OPTSPACES..ENDL
local DEFINE          = STARTL.._DEFINE
local IFDEF           = STARTL.._IFDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL
local IFNDEF          = STARTL.._IFNDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL
local ENDIF           = STARTL.._ENDIF..OPTSPACES..ENDL
local UNDEF           = STARTL.._UNDEF..WHITESPACES.."("..IDENTIFIER..")"..OPTSPACES..ENDL
local IF              = STARTL.._IF..WHITESPACES.."(.*)"..ENDL
local ELSE            = STARTL.._ELSE..OPTSPACES..ENDL
local ELIF            = STARTL.._ELIF..WHITESPACES.."(.*)"..ENDL
local ELSEIF          = STARTL.._ELSE..WHITESPACES.._IF..WHITESPACES.."(.*)"..ENDL
local ERROR           = STARTL.._ERROR..WHITESPACES.."("..TEXT..")"..OPTSPACES..ENDL
local WARNING         = STARTL.._WARNING..WHITESPACES.."("..TEXT..")"..OPTSPACES..ENDL
local ERROR_NOTEXT    = STARTL.._ERROR..OPTSPACES..ENDL --> not required when we have POSIX regex
local PRAGMA          = STARTL.._PRAGMA

-- speedups
local TRUEMACRO = STARTL.."("..IDENTIFIER..")%s*$"
local REPLMACRO = STARTL.."("..IDENTIFIER..")"..WHITESPACES.."(.+)$"
local FUNCMACRO = STARTL.."("..IDENTIFIER..")%(([_%s%w,]*)%)%s*(.*)"


-- ------------
-- LOCAL UTILS
-- ------------
lcpp.STATE = {lineno = 0} -- current state for debugging the last operation
local function error(msg) _G.print(debug.traceback()); _G.error(string.format("lcpp ERR [%04i] %s", lcpp.STATE.lineno, msg)) end
local function print(msg) _G.print(string.format("//lcpp INF [%04i] %s", lcpp.STATE.lineno, msg)) end

-- splits a string using a pattern into a table of substrings
local function gsplit(str, pat)
    local function _split(str, pat)
        local t = {}  -- NOTE: use {n = 0} in Lua-5.0
        local fpat = "(.-)"..pat
        local last_end = 1
        local s, e, cap = str:find(fpat, 1)
        while s do
            if s ~= 1 or cap ~= "" then
                coroutine.yield(cap)
            end
            last_end = e + 1
            s, e, cap = str:find(fpat, last_end)
        end
        if last_end <= #str then
            cap = str:sub(last_end)
            coroutine.yield(cap)
        end
    end
    return coroutine.wrap(function() _split(str, pat) end)
end
local function split(str, pat)
    local t = {}
    for str in gsplit(str, pat) do table.insert(t, str) end
    return t
end

-- Checks whether a string starts with a given substring
-- offset is optional
local function strsw(str, pat, offset)
    if not str then return false end
    if not offset then offset = 0 end
    return string.sub(str, 1+offset, string.len(pat)+offset) == pat
end

-- Checks whether a string ends with a given substring
local function strew(str, pat)
    if not str then return false end
    return pat=='' or string.sub(str,-string.len(pat)) == pat
end

-- string trim12 from lua wiki
local function trim(str)
    local from = str:match"^%s*()"
    return from > #str and "" or str:match(".*%S", from)
end

-- returns the number of string occurrences
local function findn(input, what)
    local count = 0
    local offset = 0
    local _
    while true do
            _, offset = string.find(input, what, offset+1, true)
            if not offset then return count end
            count = count + 1
    end
end

-- C literal string concatenation
local function concatStringLiteral(input)
    -- screener does remove multiline definition, so just check ".*"%s*".*" pattern
    return input:gsub("\"("..STRING_LITERAL..")\""..OPTSPACES.."\"("..STRING_LITERAL..")\"", "\"%1%2\"")
end

-- c style boolean check (thus, 0 will be false)
local function CBoolean(value)
    return value and (value ~= 0)
end

-- eval with c style number parse (UL, LL, L)
local function CEval(expr)
    local ok, r = pcall(loadstring, "return " .. parseCInteger(expr))
    if ok and r then
        return r()
    else
        error(r)
    end
end

-- a lightweight and flexible tokenizer
local function _tokenizer(str, setup)
        local defsetup = {
            -- EXAMPLE patterns have to be pretended with "^" for the tokenizer
            ["identifier"] = '^[_%a][_%w]*',
            ["number"] = '^[%+%-]?%d+[%.]?%d*[UL]*',
            ["ignore"] = '^%s+',
            ["string"] = true,
            ["keywords"] = {
                -- ["NAME"] = '^pattern',
                -- ...
            },
        }
    if not setup then
        setup = defsetup
    end
    setup.identifier = setup.identifier or defsetup.identifier
    setup.number = setup.number or defsetup.number
    setup.ignore = setup.ignore or defsetup.ignore
    if nil == setup.string then setup.string = true end
    setup.keywords = setup.keywords or {}

    local strlen = #str
    local i = 1
    local i1, i2
    local keyword

    local function find(pat)
        i1, i2 = str:find(pat,i)
        return i1 ~= nil
    end

    local function cut()
        return str:sub(i, i2)
    end

    local findKeyword
    if setup.keywords_order then
        findKeyword = function ()
            for _, name in ipairs(setup.keywords_order) do
                assert(setup.keywords[name])
                local pat = setup.keywords[name]
                local result = find(pat)
                if result then
                    keyword = name
                    return true
                end
            end
        end
    else
        findKeyword = function ()
            for name, pat in pairs(setup.keywords) do
                local result = find(pat)
                if result then
                    keyword = name
                    return true
                end
            end
        end
    end

    while true do
        if i > strlen then return 'eof', nil, strlen, strlen end
        if findKeyword() then
            coroutine.yield(keyword, cut(), i1, i2)
        elseif find(setup.ignore) then
            coroutine.yield("ignore", cut(), i1, i2)
        elseif find(setup.number) then
            coroutine.yield('number', tonumber(cut()), i1, i2)
        elseif find(setup.identifier) then
            coroutine.yield('identifier', cut(), i1, i2)
        elseif setup.string and (find('^"[^"]*"') or find("^'[^']*'")) then
            -- strip the quotes
            coroutine.yield('string', cut():sub(2,-2), i1, i2)
        else -- any other unknown character
            i1 = i
            i2 = i
            coroutine.yield('unknown', cut(), i1, i2)
        end
        i = i2+1
    end
end
local function tokenizer(str, setup)
    return coroutine.wrap(function() _tokenizer(str, setup) end)
end


-- ------------
-- PARSER
-- ------------

local LCPP_TOKENIZE_COMMENT = {
    string = false,
    keywords = {
        MLCOMMENT = "^/%*.-%*/",
        SLCOMMENT = "^//.-\n",
        STRING_LITERAL = '^"[^"]*"',
    },
}
-- hint: LuaJIT ffi does not rely on us to remove the comments, but maybe other usecases
local function removeComments(input)
    local out = {}
    for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_COMMENT) do
        if k == "MLCOMMENT" then
            local newlineCount = findn(input:sub(start, end_), "\n")
            local newlines = string.rep("\n", newlineCount)
            table.insert(out, newlines)
        elseif k == "SLCOMMENT" then
            table.insert(out, "\n")
        else
            table.insert(out, input:sub(start, end_))
        end
    end
    return table.concat(out)
end

-- C style number parse (UL, LL, L) and (octet, hex, binary)
local LCPP_TOKENIZE_INTEGER = {
    string = false,
    keywords_order = {
        "STRING_LITERAL",
        "CHAR_LITERAL",
        "HEX_LITERAL",
        "BIN_LITERAL",
        "OCT_LITERAL",
        "FPNUM_LITERAL",
        "NUMBER_LITERAL",
    },
    keywords = {
        STRING_LITERAL = '^"[^"]*"',
        CHAR_LITERAL = "^L'.*'",
        HEX_LITERAL = '^[%+%-]?%s*0x[a-fA-F%d]+[UL]*',
        BIN_LITERAL = '^[%+%-]?%s*0b%d+[UL]*',
        OCT_LITERAL = '^[%+%-]?%s*0%d+[UL]*',
        FPNUM_LITERAL = '^[%+%-]?%s*%d+[%.]?%d*e[%+%-]%d*',
        NUMBER_LITERAL = '^[%+%-]?%s*%d+[%.]?%d*[UL]+',
    },
}
local function parseCInteger(input)
    -- print('parseCInteger:input:' .. input)
    local out = {}
    local unary
    for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_INTEGER) do
    -- print('parseCInteger:' .. k .. "|" .. v)
        if k == "CHAR_LITERAL" then
            table.insert(out, tostring(string.byte(loadstring("return \"" .. v:gsub("^L%'(.+)%'", "%1") .. "\"")())))
        elseif k == "HEX_LITERAL" then
            unary, v = v:match('([%+%-]?)0x([a-fA-F%d]+)[UL]*')
            local n = tonumber(v, 16)
            table.insert(out, unary..tostring(n))
        elseif k == "NUMBER_LITERAL" then
            v = v:match('([^UL]+)[UL]+')
            table.insert(out, v)
        elseif k == "BIN_LITERAL" then
            unary, v = v:match('([%+%-]?)0b([01]+)[UL]*')
            local n = tonumber(v, 2)
            table.insert(out, unary..tostring(n))
        elseif k == "OCT_LITERAL" then
            unary, v = v:match('([%+%-]?)(0%d+)[UL]*')
            local n = tonumber(v, 8)
            table.insert(out, unary..tostring(n))
        else
            table.insert(out, input:sub(start, end_))
        end
    end
    local str = table.concat(out)
    -- print('parseCInteger:result:'..str)
    return str
end

-- screener: revmoce comments, trim, ml concat...
-- it only splits to cpp input lines and removes comments. it does not tokenize.
local function screener(input)
    local function _screener(input)
        input = removeComments(input)

        -- concat mulit-line input.
        local count = 1
        while count > 0 do input, count = string.gsub(input, "^(.-)\\\n(.-)$", "%1 %2\n") end

        -- trim and join blocks not starting with "#"
        local buffer = {}
        for line in gsplit(input, NEWL) do
            --print('newline:'..line)
            line = trim(line)
            if #line > 0 then
                if line:byte(1) == CMD_BYTE then
                    line = line:gsub("#%s*(.*)", "#%1") -- remove optinal whitespaces after "#". reduce triming later.
                    if #buffer > 0 then
                        coroutine.yield(table.concat(buffer, NEWL))
                        buffer = {}
                    end
                    coroutine.yield(line)
                else
                    if lcpp.FAST then
                        table.insert(buffer, line)
                    else
                        coroutine.yield(line)
                    end
                end
            elseif not lcpp.FAST then
                coroutine.yield(line)
            end
        end
        if #buffer > 0 then
            coroutine.yield(table.concat(buffer, NEWL))
        end
    end

    return coroutine.wrap(function() _screener(input) end)
end

-- apply currently known macros to input (and returns it)
local LCPP_TOKENIZE_APPLY_MACRO = {
    keywords = {
        DEFINED = "^defined%s*%(%s*"..IDENTIFIER.."%s*%)"   ,
    },
}
local function apply(state, input)
    while true do
        local out = {}
        local functions = {}
        local expand

        for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_APPLY_MACRO) do
            -- print('tokenize:'..tostring(k).."|"..tostring(v))
            if k == "identifier" then
                local repl = v
                local macro = state.defines[v]
                if macro then
                    if type(macro)     == "boolean" then
                        repl = ""
                        expand = true
                    elseif type(macro) == "string" then
                        repl = macro
                        expand = (repl ~= v)
                    elseif type(macro) == "number" then
                        repl = tostring(macro)
                        expand = (repl ~= v)
                    elseif type(macro) == "function" then
                        local decl,cnt = input:sub(start):gsub("^[_%a][_%w]*%s*%b()", "%1")
                        -- print('matching:'..input.."|"..decl.."|"..cnt)
                        if cnt > 0 then
                            repl = macro(decl)
                            -- print("d&r:"..decl.."|"..repl)
                            expand = true
                            table.insert(out, repl)
                            table.insert(out, input:sub(end_ + #decl))
                            break
                        else
                            if input:sub(start):find("^[_%a][_%w]*%s*%(") then
                                -- that is part of functional macro declaration.
                                -- print(v ..': cannot replace:<'..input..'> read more line')
                                return input,true
                            else
                                -- on macro name is also used as the symbol of some C declaration
                                -- (e.g. /usr/include/spawn.h, /usr/include/sys/select.h on centos 6.4)
                                -- no need to preprocess.
                                print(v .. ': macro name but used as C declaration in:' .. input)
                            end
                        end
                    end
                end
                table.insert(out, repl)
            elseif k == "DEFINED" then
                table.insert(out, input:sub(start, end_))
            else
                table.insert(out, input:sub(start, end_))
            end
        end
        input = table.concat(out)
        if not expand then
            break
        end
    end

    -- C liberal string concatenation, processing U,L,UL,LL
    return parseCInteger(concatStringLiteral(input)),false
end

-- processes an input line. called from lcpp doWork loop
local function processLine(state, line)
    if not line or #line == 0 then return line end
    local cmd = nil
    if line:byte(1) == CMD_BYTE then cmd = line:sub(2) end
    -- print("process:"..line)--.."|"..tostring(state:skip()))

    --[[ IF/THEN/ELSE STRUCTURAL BLOCKS ]]--
    if cmd then
        local ifdef   = cmd:match(IFDEF)
        local ifexp   = cmd:match(IF)
        local ifndef  = cmd:match(IFNDEF)
        local elif    = cmd:match(ELIF)
        local elseif_ = cmd:match(ELSEIF)
        local else_   = cmd:match(ELSE)
        local endif   = cmd:match(ENDIF)
        local struct  = ifdef or ifexp or ifndef or elif or elseif_ or else_ or endif

        if struct then
            local skip = state:skip()
            if ifdef   then state:openBlock(state:defined(ifdef))      end
            -- if skipped, it may have undefined expression. so not parse them
            if ifexp   then state:openBlock(skip and true or CBoolean(state:parseExpr(ifexp)))    end
            if ifndef  then state:openBlock(not state:defined(ifndef)) end
            if elif    then state:elseBlock((skip and skip < #state.stack) and true or CBoolean(state:parseExpr(elif)))     end
            if elseif_ then state:elseBlock((skip and skip < #state.stack) and true or CBoolean(state:parseExpr(elseif_)))  end
            if else_   then state:elseBlock(true)                      end
            if endif   then state:closeBlock()                         end
            return -- remove structural directives
        end
    end


    --[[ SKIPPING ]]--
    if state:skip() then
        -- print('skip:' .. line)
        return
    end


    --[[ READ NEW DIRECTIVES ]]--
    if cmd then
        -- handle #undef ...
        local key = cmd:match(UNDEF)
        if type(key) == "string" then
            state:undefine(key)
            return
        end

        -- read "#define >FooBar...<" directives
        if cmd:match(DEFINE) then
            local define = trim(cmd:sub(DEFINE:len()+1))
            local macroname, replacement

            -- simple "true" defines
            macroname = define:match(TRUEMACRO)
            if macroname then
                state:define(macroname, true)
            else

            -- replace macro defines
            macroname, replacement = define:match(REPLMACRO)
            if macroname and replacement then
                state:define(macroname, replacement)
            else

            -- read functional macros
            macroname, replacement, source = state:parseFunction(define)
            if macroname and replacement then
                -- add original text for definition to check identify
                state:define(macroname, replacement, false, source)
            end

            end
            end

            return
        end

        -- handle #include ...
        local filename = cmd:match(INCLUDE)
        if filename then
            return state:includeFile(filename)
        end
        local filename = cmd:match(LOCAL_INCLUDE)
        if filename then
            return state:includeFile(filename, false, true)
        end
        local filename = cmd:match(INCLUDE_NEXT)
        if filename then
        --print("include_next:"..filename)
            return state:includeFile(filename, true)
        end

        -- ignore, because we dont have any pragma directives yet
        if cmd:match(PRAGMA) then return end

        -- handle #error
        local errMsg = cmd:match(ERROR)
        local errNoTxt = cmd:match(ERROR_NOTEXT)
        local warnMsg = cmd:match(WARNING)
        if errMsg then error(errMsg) end
        if errNoTxt then error("<ERROR MESSAGE NOT SET>") end
        if warnMsg then
            print(warnMsg)
            return
        end

        -- abort on unknown keywords
        error("unknown directive: "..line)
    end

    if state.incompleteLine then
        --print('merge with incompleteLine:'..state.incompleteLine)
        line = (state.incompleteLine .. line)
        state.incompleteLine = nil
    end


    --[[ APPLY MACROS ]]--
    -- print(line)
    local _line,more = state:apply(line);
    -- print('endprocess:'.._line)
    if more then
        state.incompleteLine = line
        return ""
    else
        return _line
    end

    return line
end

local function doWork(state)
    local function _doWork(state)
        if not state:defined(__FILE__) then state:define(__FILE__, "<USER_CHUNK>", true) end
        local oldIndent = state:getIndent()
        while true do
            local input = state:getLine()
            if not input then break end
            local output = processLine(state, input)
            if not lcpp.FAST and not output then output = "" end -- output empty skipped lines
            if lcpp.DEBUG then output = output.." -- "..input end -- input as comment when DEBUG
            if output then coroutine.yield(output) end
        end
        if (oldIndent ~= state:getIndent()) then error("indentation level must be balanced within a file. was:"..oldIndent.." is:"..state:getIndent()) end
    end
    return coroutine.wrap(function() _doWork(state) end)
end

local function includeFile(state, filename, next, _local)
    local result, result_state = lcpp.compileFile(filename, state.defines, state.macro_sources, next, _local)
    -- now, we take the define table of the sub file for further processing
    state.defines = result_state.defines
    -- and return the compiled result
    return result
end

-- sets a global define
local function define(state, key, value, override, macro_source)
    --print("define:"..key.." type:"..tostring(value).." value:"..tostring(pval))
    if value and not override then
        if type(value) == 'function' then
            assert(macro_source, "macro source should specify to check identity")
            local pval = state.macro_sources[key]
            if pval and (pval ~= macro_source) then error("already defined: "..key) end
            state.macro_sources[key] = macro_source
        else
            local pval = state.defines[key]
            if pval and (pval ~= value) then error("already defined: "..key) end
        end
    end
    state.defines[key] = state:prepareMacro(value)
end

-- parses CPP exressions
-- i.e.: #if !defined(_UNICODE) && !defined(UNICODE)
--
--BNF:
--  EXPR     -> (BRACKET_OPEN)(EXPR)(BRACKET_CLOSE)
--  EXPR     -> (EXPR)(OR)(EXPR)
--  EXPR     -> (EXPR)(AND)(EXPR)
--  EXPR     -> (NOT)(EXPR)
--  EXPR     -> (FUNCTION)
--  FUNCTION -> (IDENTIFIER)(BRACKET_OPEN)(ARGS)(BRACKET_CLOSE)
--  ARGS     -> ((IDENTIFIER)[(COMMA)(IDENTIFIER)])?
--LEAVES:
--  IGNORE -> " \t"
--  BRACKET_OPEN  -> "("
--  BRACKET_CLOSE -> ")"
--  OR -> "||"
--  AND -> "&&"
--  NOT -> "!"
--  IDENTIFIER -> "[0-9a-zA-Z_]"
--

local LCPP_TOKENIZE_MACRO = {
    string = true,
    keywords_order = {
        "CONCAT",
        "SPACE",
    },
    keywords = {
        CONCAT = "^%s*##%s*",
        SPACE = "^%s",
    },
}
local LCPP_TOKENIZE_MACRO_ARGS = {
    string = true,
    keywords_order = {
        "STRING_LITERAL",
        "PARENTHESE",
        "FUNCTIONAL",
        "ARGS",
        "SINGLE_CHARACTER_ARGS",
        "COMMA",
    },
    keywords = {
        PARENTHESE = "^%s*%b()",
        FUNCTIONAL = "^".. IDENTIFIER .. "%s*%b()",
        STRING_LITERAL = '^"[^"]*"',
        ARGS = "^[^,%s][^,]*[^,%s]",
        SINGLE_CHARACTER_ARGS = "^[^,%s]",
        COMMA = "^,",
    },
}
local LCPP_TOKENIZE_EXPR = {
    string = false,
    keywords_order = {
        "DEFINED",
        "FUNCTIONAL_MACRO",
        "BROPEN",
        "BRCLOSE",

        "TENARY_START",
        "TENARY_MIDDLE",
        -- binary operators
        "EQUAL",
        "NOT_EQUAL",
        "AND",
        "OR",
        "BAND",
        "BOR",
        "BXOR",
        "PLUS",
        "MINUS",
        "MULTIPLY",
        "DIV",
        "MOD",
        "LTE",
        "MTE",
        "LSHIFT",
        "RSHIFT",
        "LT",
        "MT",
        -- unary operator
        "NOT",
        "BNOT",
        -- literal
        "STRING_LITERAL",
        "CHAR_LITERAL",
        "HEX_LITERAL",
        "FPNUM_LITERAL",
        "NUMBER_LITERAL",
    },
    keywords = {
        DEFINED = '^defined',
        FUNCTIONAL_MACRO = '^' .. IDENTIFIER .. "%s*%b()",
        BROPEN = '^[(]',
        BRCLOSE = '^[)]',

        TENARY_START = '^%?',
        TENARY_MIDDLE = '^%:',

        EQUAL = '^==',
        NOT_EQUAL = '^!=',
        AND = '^&&',
        OR = '^||',
        BAND = '^&',
        BOR = '^|',
        BXOR = '^%^',
        PLUS = '^%+',
        MINUS = '^%-',
        MULTIPLY = '^%*',
        DIV = '^%/',
        MOD = '^%%',
        LTE = '^<=',
        MTE = '^>=',
        LSHIFT = '^<<',
        RSHIFT = '^>>',
        LT = '^<',
        MT = '^>',

        NOT = '^!',
        BNOT = '^~',

        STRING_LITERAL = '^L?"[^"]*"',
        CHAR_LITERAL = "^L?'.*'",
        HEX_LITERAL = '^[%+%-]?0?x[a-fA-F%d]+[UL]*',
        FPNUM_LITERAL = '^[%+%-]?%d+[%.]?%d*e[%+%-]%d*',
        NUMBER_LITERAL = '^[%+%-]?0?b?%d+[%.]?%d*[UL]*',
    },
}

local function parseDefined(state, input)
    local result = false
    local bropen = false
    local brclose = false
    local ident = nil

    for key, value in input do
        if key == "BROPEN" then
            bropen = true
        end
        if key == "identifier" then
             ident = value
             if not bropen then break end
        end
        if key == "BRCLOSE" and ident then
            brclose = true
            break
        end
    end

    -- wiht and w/o brackets allowed
    if ident and ((bropen and brclose) or (not bropen and not brclose)) then
        return state:defined(ident)
    end

    error("expression parse error: defined(ident)")
end


--[[
order : smaller is higher priority
1   ()   []   ->   .
2    !   ~   -   +   *   &   sizeof   type cast   ++   --
3   *   /   %
4   +   -
5   <<   >>
6   <   <=   >   >=
7   ==   !=
8   &
9   ^
10  |
11  &&
12  ||
13   ?:   =   +=   -=   *=   /=   %=   &=   |=   ^=   <<=   >>=
14  ,
]]
local combination_order = function (op, unary)
    if unary then
        if op == '-' or op == '!' or op == '~' then
            return 2
        else
            assert(false, 'unsupported unary operator:' .. op)
        end
    else
        if op == '*' or op == '/' or op == '%' then
            return 3
        elseif op == '+' or op == '-' then
            return 4
        elseif op == '>>' or op == '<<' then
            return 5
        elseif op == '<' or op == '>' or op == '<=' or op == '>=' then
            return 6
        elseif op == '==' or op == '!=' then
            return 7
        elseif op == '&' then
            return 8
        elseif op == '^' then
            return 9
        elseif op == '|' then
            return 10
        elseif op == '&&' then
            return 11
        elseif op == '||' then
            return 12
        elseif op == '?' or op == ':' then
            return 13
        else
            assert(false, 'unsupported operator:' .. op)
        end
    end
end

local evaluate
evaluate = function (node)
    if not node.op then -- leaf node or leaf node with unary operators
        local v = node.v
        if node.uops then
            for _, uop in ipairs(node.uops) do
                -- print('apply uop:'..uop.."|"..tostring(v))
                if uop == '-' then
                    v = -v
                elseif uop == '!' then
                    v = (not v)
                elseif uop == '~' then
                    v = bit.bnot(v)
                else
                    assert(false, 'invalid uop:' .. tostring(uop))
                end
            end
        end
        -- print('after apply:'..tostring(v))
        return v
    end
    -- print(node.op..':'..tostring(node.l.v or node.l.op).."("..type(node.l.v)..")|"..tostring(node.r.v or node.r.op).."("..type(node.r.v)..")")
    if node.op == '+' then -- binary operators
        return (evaluate(node.l) + evaluate(node.r))
    elseif node.op == '-' then
        return (evaluate(node.l) - evaluate(node.r))
    elseif node.op == '*' then
        return (evaluate(node.l) * evaluate(node.r))
    elseif node.op == '/' then
        return (evaluate(node.l) / evaluate(node.r))
    elseif node.op == '%' then
        return (evaluate(node.l) % evaluate(node.r))
    elseif node.op == '==' then
        return (evaluate(node.l) == evaluate(node.r))
    elseif node.op == '!=' then
        return (evaluate(node.l) ~= evaluate(node.r))
    elseif node.op == '<<' then
        return bit.lshift(evaluate(node.l), evaluate(node.r))
    elseif node.op == '>>' then
        return bit.rshift(evaluate(node.l), evaluate(node.r))
    elseif node.op == '&&' then
        return (CBoolean(evaluate(node.l)) and CBoolean(evaluate(node.r)))
    elseif node.op == '||' then
        return (CBoolean(evaluate(node.l)) or CBoolean(evaluate(node.r)))
    elseif node.op == '&' then
        return bit.band(evaluate(node.l), evaluate(node.r))
    elseif node.op == '|' then
        return bit.bor(evaluate(node.l), evaluate(node.r))
    elseif node.op == '^' then
        return bit.bxor(evaluate(node.l), evaluate(node.r))
    elseif node.op == '<=' then
        return (evaluate(node.l) <= evaluate(node.r))
    elseif node.op == '>=' then
        return (evaluate(node.l) >= evaluate(node.r))
    elseif node.op == '<' then
        return (evaluate(node.l) < evaluate(node.r))
    elseif node.op == '>' then
        return (evaluate(node.l) > evaluate(node.r))
    else
        assert(false, 'invalid op:' .. tostring(node.op))
    end
end

local function setValue(node, v)
    -- print('setValue:' .. tostring(v).."|"..tostring(node.uops))-- .. "\t" .. debug.traceback())
    if not node.op then
        assert(not node.v, debug.traceback())
        node.v = v
    else
        assert(node.l and (not node.r))
        node.r = {v = v, uops = node.uops}
    end
end

local function setUnaryOp(node, uop)
    -- print('setUnaryOp:' .. tostring(uop))-- .. "\t" .. debug.traceback())
    if not node.uops then node.uops = {} end
    table.insert(node.uops, 1, uop)
end

local function parseExpr(state, input)
    local node = {}
    local root = node
    -- first call gets string input. rest uses tokenizer
    if type(input) == "string" then
        -- print('parse:' .. input)
        input = tokenizer(input, LCPP_TOKENIZE_EXPR)
    end

    for type, value in input do
        -- print("type:"..type.." value:"..value)
        -- unary operator
        if type == "NOT" or
            type == "BNOT" then
            setUnaryOp(node, value)
        end
        if type == "BROPEN" then
            setValue(node, state:parseExpr(input))
        end
        if type == "BRCLOSE" then
            --print('BRCLOSE:' .. tostring(result))
            break
        end
        if type == "STRING_LITERAL" then
            setValue(node, value:sub(value[1] == 'L' and 3 or 2,-2))
        end
        if type == "NUMBER_LITERAL" or type == "HEX_LITERAL" or type == "FPNUM_LITERAL" or type == "CHAR_LITERAL" then
            setValue(node, tonumber(parseCInteger(value)))
        end
        -- tenary operator
        -- tenary has lowest priority, so any other operation can be calculate now.
        if type == "TENARY_START" then
            local l = state:parseExpr(input)
            local r = state:parseExpr(input)
            if evaluate(root) then
                return l
            else
                return r
            end
        end
        if type == "TENARY_MIDDLE" then
            break
        end
        -- binary operator
        if type == "EQUAL" or
            type == "NOT_EQUAL" or
            type == "AND" or
            type == "OR" or
            type == "BAND" or
            type == "BOR" or
            type == "BXOR" or
            type == "PLUS" or
            type == "MINUS" or
            type == "MULTIPLY" or
            type == "DIV" or
            type == "MOD" or
            type == "LTE" or
            type == "MTE" or
            type == "LSHIFT" or
            type == "RSHIFT" or
            type == "LT" or
            type == "MT" then
            if node.op then
                if not node.r then -- during parse right operand : uop1 uop2 ... uopN operand1 op1 uop(N+1) uop(N+2) ... [uop(N+K)]
                    assert(type == "MINUS",  "error: operators come consequently: " .. tostring(node.op) .. " and " .. tostring(value))
                    -- unary operater after binary operator
                    setUnaryOp(node, value)
                else -- uop1 uop2 ... uopN operand1 op1 uop(N+1) uop(N+2) ... uop(N+M) operand2 [op2]
                    -- print("operator processing:" .. tostring(node.op) .. "|" .. value .. "|" .. tostring(node.l) .. "|" .. tostring(node.r))
                    local tmp = node
                    while tmp do
                        -- print('compare ' .. value..' and ' .. tmp.op)
                        if combination_order(tmp.op) > combination_order(value) then
                            -- print(value..' is stronger than ' .. tmp.op)
                            break
                        end
                        tmp = tmp.parent
                    end
                    if tmp then
                        node = {
                            op = value,
                            l = tmp.r,
                            parent = tmp
                        }
                        tmp.r.parent = node
                        tmp.r = node
                    else
                        node = {
                            op = value,
                            l = root,
                        }
                        root.parent = node
                        root = node
                    end
                end
            elseif node.v ~= nil then -- uop1 uop2 ... uopN operand1 [op]
                local devided
                if node.uops then
                    for _, uop in ipairs(node.uops) do
                        if combination_order(uop, true) > combination_order(value) then
                            -- there is a binary operator which has stronger than any of the unary
                            devided = uop
                        end
                    end
                end
                if devided then
                    assert(false, "TODO: can we do something about this case??:"..value.." is stronger than "..devided)
                else
                    node.l = { v = node.v, uops = node.uops }
                    node.v = nil
                    node.uops = nil
                    node.op = value
                end
            else -- unary operator : uop1 uop2 ... [uopN]
                assert(type == "MINUS", "error: invalid unary operator:" .. value)
                setUnaryOp(node, value)
            end
        end
        if type == "DEFINED" then
            setValue(node, parseDefined(state, input))
        elseif type == "identifier" or type == "FUNCTIONAL_MACRO" then
            -- print('ident:' .. value)
            local eval = state:apply(value)
            -- print('apply result ' .. eval .. "|" .. tostring(unprocessed))
            if eval ~= value then
                eval = state:parseExpr(eval)
                -- print('re-evaluate expr ' .. tostring(eval))
                setValue(node, eval)
            else
                -- undefined macro symbol is always treated as 0.
                -- http://gcc.gnu.org/onlinedocs/cpp/If.html#If
                setValue(node, 0)
            end
        end
    end

    local r = evaluate(root)
    -- print('evaluate:' .. tostring(r))
    return r
end

-- apply string ops "##"
local function prepareMacro(state, input)
    if type(input) ~= "string" then return input end
    repeat
        local out = {}
        local concat
        for k, v, start, end_ in tokenizer(input, LCPP_TOKENIZE_MACRO) do
            if k == "CONCAT" then
                -- remove concat op "##"
                concat = true
            else
                table.insert(out, input:sub(start, end_))
            end
        end
        input = table.concat(out)
    until not concat
    return input
end

-- macro args replacement function slower but more torelant for pathological case
local function replaceArgs(argsstr, repl)
    local args = {}
    argsstr = argsstr:sub(2,-2)
    -- print('argsstr:'..argsstr)
    local comma
    for k, v, start, end_ in tokenizer(argsstr, LCPP_TOKENIZE_MACRO_ARGS) do
        -- print("replaceArgs:" .. k .. "|" .. v)
        if k == "ARGS" or k == "PARENTHESE" or k == "STRING_LITERAL" or
            k == "FUNCTIONAL" or k == "SINGLE_CHARACTER_ARGS" then
            table.insert(args, v)
            comma = false
        elseif k == "COMMA" then
            if comma then
                -- continued comma means empty parameter
                table.insert(args, "")
            end
            comma = true
        end
    end
    local v = repl:gsub("%$(%d+)", function (m) return args[tonumber(m)] or "" end)
    -- print("replaceArgs:" .. repl .. "|" .. tostring(#args) .. "|" .. v)
    return v
end

-- i.e.: "MAX(x, y) (((x) > (y)) ? (x) : (y))"
local function parseFunction(state, input)
    if not input then return end
    local concat
    local name, argsstr, repl = input:match(FUNCMACRO)
    if not name or not argsstr or not repl then return end

    -- rename args to $1,$2... for later gsub
    local noargs = 0
    for argname in argsstr:gmatch(IDENTIFIER) do
        noargs = noargs + 1
        -- avoid matching substring of another identifier (eg. attrib matches __attribute__ and replace it)
        repl = repl:gsub("(#*)(%s*)("..argname..")([_%w]?)", function (s1, s2, s3, s4)
            if #s4 <= 0 then
                return (#s1 == 1) and ("\"$"..noargs.."\"") or (s1..s2.."$"..noargs)
            else
                return s1..s2..s3..s4
            end
        end)
    end
    -- remove concat (after replace matching argument name to $1, $2, ...)
    repl = repl:gsub("%s*##%s*", "")

    -- build macro funcion
    local func = function(input)
        return input:gsub(name.."%s*(%b())", function (match)
            return replaceArgs(match, repl)
        end)
    end

    return name, func, repl
end


-- ------------
-- LCPP INTERFACE
-- ------------

--- initialies a lcpp state. not needed manually. handy for testing
function lcpp.init(input, predefines, macro_sources)
    -- create sate var
    local state     = {}              -- init the state object
    state.defines   = {}              -- the table of known defines and replacements
    state.screener  = screener(input)
    state.lineno    = 0               -- the current line number
    state.stack     = {}              -- stores wether the current stack level is to be included
    state.once      = {}              -- stack level was once true (first if that evals to true)
    state.macro_sources = macro_sources or {} -- original replacement text for functional macro

    -- funcs
    state.define = define
    state.undefine = function(state, key)
        state:define(key, nil)
        state.macro_sources[key] = nil
    end
    state.defined = function(state, key)
        return state.defines[key] ~= nil
    end
    state.apply = apply
    state.includeFile = includeFile
    state.doWork = doWork
    state.getIndent = function(state)
        return #state.stack
    end
    state.openBlock = function(state, bool)
        state.stack[#state.stack+1] = bool
        state.once [#state.once+1]  = bool
        state:define(__LCPP_INDENT__, state:getIndent(), true)
    end
    state.elseBlock = function(state, bool)
        if state.once[#state.once] then
            state.stack[#state.stack] = false
        else
            state.stack[#state.stack] = bool
            if bool then state.once[#state.once] = true end
        end
    end
    state.closeBlock = function(state)
        state.stack[#state.stack] = nil
        state.once [#state.once]  = nil
        state:define(__LCPP_INDENT__, state:getIndent(), true)
        if state:getIndent() < 0 then error("Unopened block detected. Indentaion problem.") end
    end
    state.skip = function(state)
        for i = 1, #state.stack do
            if not state.stack[i] then return i end
        end
        return false
    end
    state.getLine = function(state)
        state.lineno = state.lineno + 1
        state:define(__LINE__, state.lineno, true)
        return state.screener()
    end
    state.prepareMacro = prepareMacro
    state.parseExpr = parseExpr
    state.parseFunction = parseFunction

    -- predefines
    state:define(__DATE__, os.date("%B %d %Y"), true)
    state:define(__TIME__, os.date("%H:%M:%S"), true)
    state:define(__LINE__, state.lineno, true)
    state:define(__LCPP_INDENT__, state:getIndent(), true)
    predefines = predefines or {}
    for k,v in pairs(lcpp.ENV) do   state:define(k, v, true) end    -- static ones
    for k,v in pairs(predefines) do state:define(k, v, true) end

    if lcpp.LCPP_TEST then lcpp.STATE = state end -- activate static state debugging

    return state
end

--- the preprocessors main function.
-- returns the preprocessed output as a string.
-- @param code data as string
-- @param predefines OPTIONAL a table of predefined variables
-- @usage lcpp.compile("#define bar 0x1337\nstatic const int foo = bar;")
-- @usage lcpp.compile("#define bar 0x1337\nstatic const int foo = bar;", {["bar"] = "0x1338"})
function lcpp.compile(code, predefines, macro_sources)
    local state = lcpp.init(code, predefines, macro_sources)
    local buf = {}
    for output in state:doWork() do
        table.insert(buf, output)
    end
    local output = table.concat(buf, NEWL)
    if lcpp.DEBUG then print(output) end
    return output, state
end

--- preprocesses a file
-- @param filename the file to read
-- @param predefines OPTIONAL a table of predefined variables
-- @usage out, state = lcpp.compileFile("../odbg/plugin.h", {["MAX_PAH"]=260, ["UNICODE"]=true})
function lcpp.compileFile(filename, predefines, macro_sources, next, _local)
    if not filename then error("processFile() arg1 has to be a string") end
    local file = io.open(filename, 'r')
    if not file then error("file not found: "..filename) end
    local code = file:read('*a')
    predefines = predefines or {}
    predefines[__FILE__] = filename
    return lcpp.compile(code, predefines, macro_sources)
end


-- ------------
-- SATIC UNIT TESTS
-- ------------
function lcpp.test(suppressMsg)
    local testLabelCount = 0
    local function getTestLabel()
        testLabelCount = testLabelCount + 1
        return " lcpp_assert_"..testLabelCount
    end

    -- this ugly global is required so our testcode can find it
    _G.lcpp_test = {
        assertTrueCalls = 0;
        assertTrueCount = 0;
        assertTrue = function()
            lcpp_test.assertTrueCount = lcpp_test.assertTrueCount + 1;
        end
    }

    local testlcpp = [[
        assert(__LINE__ == 1, "_LINE_ macro test 1: __LINE__")
        // This test uses LCPP with lua code (uncommon but possible)
        assert(__LINE__ == 3, "_LINE_ macro test 3: __LINE__")
        /*
         * It therefore asserts any if/else/macro functions and various syntaxes
         * (including this comment, that would cause errors if not filtered)
         */
        assert(__LINE__ == 8, "_LINE_ macro test 8: __LINE__")
        /*
         assert(false, "multi-line comment not removed")
         */
        /* pathological case which contains single line comment start in multiline comments.
         * e.g. this multiline comment should be finish next line.
         * http://foobar.com */ // comment
        /* if singleline comment processes first, sometimes indicator of end of multiline loss */ #define THIS_SHOULD_ENABLE 111 /*
            continuous multiline comment after macro definition
        //*/
        ///* this removed.
        assert(THIS_SHOULD_ENABLE == 111, "pathological multiline comment test")


        #define TRUE
        #define ONE (1)
        #define TWO (2)
        #define THREE_FUNC(x) (3)
        #define LEET 0x1337
        #define CLONG 123456789L
        #define CLONGLONG 123456789123456789LL
        #define CULONG 12345678UL
        #define CUINT 123456U
        #define BINARY -0b1001 /* -9 */
        #define OCTET 075 /* 61 */
        #define NON_OCTET 75
        #define HEX 0xffffU
        #define __P(x) x
        #define WCHAR_ZERO L'\0'
        #  define NCURSES_IMPEXP
        #  define NCURSES_API
        #  define NCURSES_EXPORT(type) NCURSES_IMPEXP type NCURSES_API
        #define MACRO_TO_ITSELF MACRO_TO_ITSELF
        local MACRO_TO_ITSELF = 111
        assert(MACRO_TO_ITSELF == 111, "can process macro to itself")
        assert(WCHAR_ZERO == 0, "wchar evaluate fails")
        assert(CLONG == 123456789, "read *L fails")
        assert(CLONGLONG == 123456789123456789, "read *LL fails")
        assert(CULONG == 12345678, "read *UL fails")
        assert(HEX == 65535, "read hex fails")
        #if CUINT != 123456U
        assert(false, "cannot evaluate number which contains Unsinged postfix correctly")
        #else
        assert(CUINT == 123456, "read *U fails")
        #endif
        #pragma ignored
        assert __P((BINARY == -9, "parse, binary literal fails"))
        assert(OCTET == 61 and NON_OCTET == 75, "parse octet literal fails")

        lcpp_test.assertTrue()
        assert(LEET == 0x1337, "simple #define replacement")
        local msg
        /* function to check macro expand to empty */
        local function check_argnum(...)
            return select('#', ...)
        end
        NCURSES_EXPORT(function) test_export(a)
            return a + 1
        end
        assert(test_export(2) == 3, "EXPORT style macro")
        local macrofunc = function __P((
            a, b))
            return a + b
        end
        assert(macrofunc(1, 2) == 3, "macro arg contains parenthese")

        msg = "tenary operator test"
        #if (HEX % 2 == 1 ? CUINT : CULONG) == 123456
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."1")
        #endif
        #if (OCTET % 2 == 0 ? CUINT : CULONG) == 123456
            assert(false, msg.."1")
        #else
            lcpp_test.assertTrue()
        #endif




        # if defined TRUE
            lcpp_test.assertTrue() -- valid strange syntax test (spaces and missing brackets)
        # endif


        msg = "#define if/else test"
        #ifdef TRUE
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."1")
        #endif
        #ifdef NOTDEFINED
            assert(false, msg.."2")
        #else
            lcpp_test.assertTrue()
        #endif
        #ifndef NOTDEFINED
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."3")
        #endif


        msg = "#if defined statement test"
        #if defined(TRUE)
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."1")
        #endif
        #if !defined(LEET) && !defined(TRUE)
            assert(false, msg.."2")
        #endif
        #if !defined(NOTLEET) && !defined(NOTDEFINED)
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."3")
        #endif
        #if !(defined(LEET) && defined(TRUE))
            assert(false, msg.."4")
        #else
            lcpp_test.assertTrue()
        #endif
        #if !defined(LEET) && !defined(TRUE)
            assert(false, msg.."5")
        #endif
        #if defined(LEET) && defined(TRUE) && defined(NOTDEFINED)
            assert(false, msg.."6")
        #endif
        #if ONE + TWO * TWO == 5
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."7")
        #endif
        #if (ONE + TWO) * TWO == 0x6
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."8")
        #endif
        #if ONE * TWO + ONE / TWO == 2.5
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."9")
        #endif
        #if ONE + ONE * TWO / TWO == 2
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."10")
        #endif
        #if TWO - - TWO == 4
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."11")
        #endif
        #if (TWO - - TWO) % (ONE + TWO) == 1
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."12")
        #endif
        #if ONE << TWO + TWO >> ONE == 8
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."13")
        #endif
        #if (ONE << TWO) + (TWO >> ONE) == 5
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."14")
        #endif
        #if (ONE << TWO) + TWO >> ONE == 3
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."15")
        #endif
        #if (THREE_FUNC(0xfffffU) & 4) == 0
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."16")
        #endif
        #if (0x3 & THREE_FUNC("foobar")) == 0b11
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."17")
        #endif
        #if defined(TWO) && ((TWO-0) < 3)
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."17")
        #endif
        #if TWO == 1--1
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."18")
        #endif
        #if HEX > 0xfFfFU
            assert(false, msg.."18")
        #else
            lcpp_test.assertTrue()
        #endif
        #define TRUE_DEFINED defined(TRUE)
        #if TRUE_DEFINED
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."19")
        #endif
        #define NOTDEFINED_DEFINED defined(TRUE) && defined(NOTDEFINED)
        #if NOTDEFINED_DEFINED
            assert(false, msg.."20")
        #else
            lcpp_test.assertTrue()
        #endif
        #if LEET && LEET > 0x1336
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."20")
        #endif
        #if NOTLEET && NOTLEET > 0x1336
            assert(false, msg.."21")
        #else
            lcpp_test.assertTrue()
        #endif
        #if defined(NOTLEET) || BINARY + 0 >= 10L || !defined(TRUE)
            assert(false, msg.."22")
        #else
            lcpp_test.assertTrue()
        #endif



        msg = "macro chaining"
        #define FOO 0x7B
        #define BAR (FOO+0x7B)
        assert(-BAR == -0x7B*2, msg)
        #define BAZ 456
        #define BLUR BA##Z
        assert(BLUR == 456, msg)
        local testfunc = function (x) return "["..tostring(x).."]" end
        #define FOOBAR(x) testfunc(x)
        assert(FOOBAR(1) == "[1]", msg)
        #define testfunc(x)
        #define FOOBAZ(x) testfunc(x)
        assert(check_argnum(FOOBAR(1)) == 0, msg)
        assert(check_argnum(FOOBAZ(1)) == 0, msg)
        #undef testfunc
        assert(FOOBAR(1) == "[1]", msg)
        assert(FOOBAZ(1) == "[1]", msg)


        msg = "indentation test"
        assert(__LCPP_INDENT__ == 0, msg.."1")
        #if defined(TRUE)
            assert(__LCPP_INDENT__ == 1, msg.."2")
        #endif
        assert(__LCPP_INDENT__ == 0, msg.."3")


        #define LCPP_FUNCTION_1(x, y) (x and not y)
        assert(LCPP_FUNCTION_1(true, false), "function macro")
        #define LCPP_FUNCTION_2(x, y) \
            (not x \
             and y)
        assert(LCPP_FUNCTION_2(false, true), "multiline function macro")
        #define LCPP_FUNCTION_3(_x, _y) LCPP_FUNCTION_2(_x, _y)
        assert(LCPP_FUNCTION_3(false, true), "function macro with argname contains _")

        #define LCPP_FUNCTION_4_CHILD() false
        #define LCPP_FUNCTION_4(_x)

        assert(check_argnum(LCPP_FUNCTION_4(LCPP_FUNCTION_4_CHILD())) == 0, "functional macro which receives functional macro as argument")
        assert(check_argnum(LCPP_FUNCTION_4(LCPP_FUNCTION_3(true, true))) == 0, "functional macro which receives functional macro as argument2")

        #define LCPP_FUNCTION_5(x, y) (x) + (x) + (y) + (y)
        assert(LCPP_FUNCTION_5(10, 20) == 60, "macro argument multiple usage")

        #define LCPP_NOT_FUNCTION (BLUR)
        assert(LCPP_NOT_FUNCTION == 456, "if space between macro name and argument definition exists, it is regarded as replacement macro")

        #define __CONCAT(x, y, z, w) x ## y ## z (w)
        local __fncall = function (x) return x + 111 end
        assert(111 == __CONCAT( __  ,   fnc , all, 0 ), "funcall fails")
        assert(222 == __CONCAT( __  ,, fncall, 111  ), "funcall fails2")
        #define __ATTRIB_CALL(x, y, attrib) __attribute__(x, y, attrib)
        local __attribute__ = function (x, y, attr)
            return attr * (x + y)
        end
        assert(__ATTRIB_CALL(  1, 2  , 100 ) == 300, "funcall fails3")


        msg = "#elif test"
        #if defined(NOTDEFINED)
            -- it should not be processed
            #if NOTDEFINED
            #else
            assert(false, msg.."1")
            #endif
        #elif defined(NOTDEFINED)
            assert(false, msg.."2")
        #elif defined(TRUE)
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."3")
        #endif


        msg = "#else if test"
        #if defined(NOTDEFINED)
            assert(false, msg.."1")
        #else if defined(NOTDEFINED)
            assert(false, msg.."2")
        #else if defined(TRUE)
            lcpp_test.assertTrue()
        #else
            assert(false, msg.."3")
        #endif


        msg = "bock stack tree test"
        #ifdef TRUE
            #ifdef NOTDEFINED
                assert(false, msg.."1")
            #elif defined(TRUE)
                lcpp_test.assertTrue()
            #else
                assert(false, msg.."2")
            #endif
        #else
            assert(false, msg.."3")
        #endif
        #ifdef NOTDEFINED
            #ifdef TRUE
                assert(false, msg.."4")
            #endif
        #endif

        msg = "test concat ## operator"
        #define CONCAT_TEST1 foo##bar
        local CONCAT_TEST1 = "blubb"
        assert(foobar == "blubb", msg)
        #define CONCAT_TEST2() bar##foo
        local CONCAT_TEST2() = "blubb"
        assert(barfoo == "blubb", msg)
        -- dont process ## within strings
        #define CONCAT_TEST3 "foo##bar"
        assert(CONCAT_TEST3 == "foo##bar", msg)
        msg = "test concat inside func type macro"
        #define CONCAT_TEST4(baz) CONCAT_TEST##baz
        local CONCAT_TEST4(1) = "bazbaz"
        assert(foobar == "bazbaz", msg)

        msg = "#undef test"
        #define UNDEF_TEST
        #undef UNDEF_TEST
        #ifdef UNDEF_TEST
            assert(false, msg)
        #endif

        msg = "stringify operator(#) test"
        #define STRINGIFY(str) #str
        assert(STRINGIFY(abcde) == "abcde", msg)
        #define STRINGIFY_AND_CONCAT(str1, str2) #str1 ## \
        #str2
        assert(STRINGIFY_AND_CONCAT(fgh, ij) == "fghij", msg)

        #define msg_concat(msg1, msg2) msg1 ## msg2
        assert("I, am, lcpp" == msg_concat("I, am", ", lcpp"), "processing macro argument which includes ,")

        #define FUNC__ARG 500
        #define __ARG 100
        #define FUNC(x) FUNC##x
        assert(FUNC(__ARG) == 500, "create new macro symbol by concat")

        msg = "same macro definition which has exactly same value allows. (checked with gcc 4.4.7 __WORDSIZE)"
        #define DUP_MACRO_DEF (111)
        #define DUP_MACRO_DEF (111)
        #define DUP_FUNC_MACRO(x, y) x + y
        #define DUP_FUNC_MACRO(x, y) x + y


        msg = "check #if conditional check"
        #define VALUE1 (123)
        #if VALUE1 != 123
            assert(false, msg .." #if " .. tostring(VALUE1) .. " != 123")
        #endif
        #if 123 != VALUE1
            assert(false, msg .." #if " .. tostring(VALUE1) .. " != 123 (2)")
        #endif

        #define VALUE2 ("hoge")
        #if VALUE2 == "hoge"
            #define VALUE3 (true)
        #endif
        assert(VALUE3 == true, msg .. " #if " .. tostring(VALUE3) .. " == true")

        #define VALUE4 (VALUE1 + VALUE1)
        #if VALUE4 != 246
            assert(false, msg .." #if check for nested definition:" .. tostring(VALUE4))
        #endif

        msg = "+-*/<> in #if expression:"
        #define CALC_VALUE_A (1)
        #define CALC_VALUE_B (2)
        #if (CALC_VALUE_A + CALC_VALUE_B) != 3
            assert(false, msg .. " + not work:")
        #endif
        #if (CALC_VALUE_A * CALC_VALUE_B) != 2
            assert(false, msg .. " * not work:")
        #endif
        #if (CALC_VALUE_A - CALC_VALUE_B) != -1
            assert(false, msg .. " - not work:")
        #endif
        #if (CALC_VALUE_A / CALC_VALUE_B) != 0.5
            assert(false, msg .. " / not work:")
        #endif

        #if (CALC_VALUE_A >= CALC_VALUE_A)
        #else
            assert(false, msg .. " >= not work1")
        #endif
        #if (CALC_VALUE_B >= CALC_VALUE_A)
        #else
            assert(false, msg .. " >= not work2")
        #endif

        #if (CALC_VALUE_B <= CALC_VALUE_B)
        #else
            assert(false, msg .. " <= not work1")
        #endif
        #if (CALC_VALUE_A <= CALC_VALUE_B)
        #else
            assert(false, msg .. " <= not work2")
        #endif

        #if (CALC_VALUE_B > CALC_VALUE_A)
        #else
            assert(false, msg .. " > not work1")
        #endif
        #if (CALC_VALUE_A > CALC_VALUE_A)
            assert(false, msg .. " > not work2")
        #endif

        #if (CALC_VALUE_A < CALC_VALUE_B)
        #else
            assert(false, msg .. " < not work1")
        #endif
        #if (CALC_VALUE_B < CALC_VALUE_B)
            assert(false, msg .. " < not work2")
        #endif
        #if (CALC_VALUE_B | CALC_VALUE_A) != 3
            assert(false, msg .. " | not work")
        #endif
        #if (CALC_VALUE_B & CALC_VALUE_A) != 0
            assert(false, msg .. " &     not work")
        #endif
        #if (CALC_VALUE_B ^ CALC_VALUE_A) != 3
            assert(false, msg .. " ^ not work")
        #endif
        #if -CALC_VALUE_B != -2
            assert(false, msg .. " unary - not work")
        #endif
        #if ~CALC_VALUE_B != -3
            assert(false, msg .. " unary ~ not work")
        #endif
    ]]
    lcpp.FAST = false   -- enable full valid output for testing
    lcpp.SELF_TEST = true
    local testlua = lcpp.compile(testlcpp)
    lcpp.SELF_TEST = nil
    --  print(testlua)
    assert(loadstring(testlua, "testlua"))()
    lcpp_test.assertTrueCalls = findn(testlcpp, "lcpp_test.assertTrue()")
    assert(lcpp_test.assertTrueCount == lcpp_test.assertTrueCalls, "assertTrue calls:"..lcpp_test.assertTrueCalls.." count:"..lcpp_test.assertTrueCount)
    _G.lcpp_test = nil  -- delete ugly global hack
    if not suppressMsg then print("Test run suscessully") end
end
if lcpp.LCPP_TEST then lcpp.test(true) end


-- ------------
-- REGISTER LCPP
-- ------------

--- disable lcpp processing for ffi, loadstring and such
lcpp.disable = function()
    if lcpp.LCPP_LUA then
        -- activate LCPP_LUA actually does anything useful
        -- _G.loadstring = _G.loadstring_lcpp_backup
    end

    if lcpp.LCPP_FFI and pcall(require, "ffi") then
        ffi = require("ffi")
        if ffi.lcpp_cdef_backup then
            ffi.cdef = ffi.lcpp_cdef_backup
            ffi.lcpp_cdef_backup = nil
        end
    end
end

--- (re)enable lcpp processing for ffi, loadstring and such
lcpp.enable = function()
    -- Use LCPP to process Lua code (load, loadfile, loadstring...)
    if lcpp.LCPP_LUA then
        -- TODO: make it properly work on all functions
        error("lcpp.LCPP_LUA = true -- not properly implemented yet");
        _G.loadstring_lcpp_backup = _G.loadstring
        _G.loadstring = function(str, chunk)
            return loadstring_lcpp_backup(lcpp.compile(str), chunk)
        end
    end
    -- Use LCPP as LuaJIT PreProcessor if used inside LuaJIT. i.e. Hook ffi.cdef
    if lcpp.LCPP_FFI and pcall(require, "ffi") then
        ffi = require("ffi")
        if not ffi.lcpp_cdef_backup then
            if not ffi.lcpp_defs then ffi.lcpp_defs = {} end -- defs are stored and reused
            ffi.lcpp = function(input)
                local output, state = lcpp.compile(input, ffi.lcpp_defs, ffi.lcpp_macro_sources)
                ffi.lcpp_defs = state.defines
                ffi.lcpp_macro_sources = state.macro_sources
                return output
            end
            ffi.lcpp_cdef_backup = ffi.cdef
            ffi.cdef = function(input)
                if true then
                    return ffi.lcpp_cdef_backup(ffi.lcpp(input))
                else
                    local fn,cnt = input:gsub('#include ["<].-([^/]+%.h)[">]', '%1')
                    input = ffi.lcpp(input)
                    if cnt > 0 then
                        local f = io.open("./tmp/"..fn, 'w')
                        if f then
                            f:write(input)
                            f:close()
                        else
                            assert(fn:find('/'), 'cannot open: ./tmp/'..fn)
                        end
                    end
                    return ffi.lcpp_cdef_backup(input)
                end
            end
        end
    end
end

lcpp.enable()
return lcpp