+++ /dev/null
---[[--------------------------------------------------------------------
-
- llex.lua: Lua 5.1 lexical analyzer in Lua
- This file is part of LuaSrcDiet, based on Yueliang material.
-
- Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net>
- The COPYRIGHT file describes the conditions
- under which this software may be distributed.
-
- See the ChangeLog for more information.
-
-----------------------------------------------------------------------]]
-
---[[--------------------------------------------------------------------
--- NOTES:
--- * This is a version of the native 5.1.x lexer from Yueliang 0.4.0,
--- with significant modifications to handle LuaSrcDiet's needs:
--- (1) llex.error is an optional error function handler
--- (2) seminfo for strings include their delimiters and no
--- translation operations are performed on them
--- * ADDED shbang handling has been added to support executable scripts
--- * NO localized decimal point replacement magic
--- * NO limit to number of lines
--- * NO support for compatible long strings (LUA_COMPAT_LSTR)
--- * Please read technotes.txt for more technical details.
-----------------------------------------------------------------------]]
-
-local base = _G
-local string = require "string"
-module "llex"
-
-local find = string.find
-local match = string.match
-local sub = string.sub
-
-----------------------------------------------------------------------
--- initialize keyword list, variables
-----------------------------------------------------------------------
-
-local kw = {}
-for v in string.gmatch([[
-and break do else elseif end false for function if in
-local nil not or repeat return then true until while]], "%S+") do
- kw[v] = true
-end
-
--- NOTE: see init() for module variables (externally visible):
--- tok, seminfo, tokln
-
-local z, -- source stream
- sourceid, -- name of source
- I, -- position of lexer
- buff, -- buffer for strings
- ln -- line number
-
-----------------------------------------------------------------------
--- add information to token listing
-----------------------------------------------------------------------
-
-local function addtoken(token, info)
- local i = #tok + 1
- tok[i] = token
- seminfo[i] = info
- tokln[i] = ln
-end
-
-----------------------------------------------------------------------
--- handles line number incrementation and end-of-line characters
-----------------------------------------------------------------------
-
-local function inclinenumber(i, is_tok)
- local sub = sub
- local old = sub(z, i, i)
- i = i + 1 -- skip '\n' or '\r'
- local c = sub(z, i, i)
- if (c == "\n" or c == "\r") and (c ~= old) then
- i = i + 1 -- skip '\n\r' or '\r\n'
- old = old..c
- end
- if is_tok then addtoken("TK_EOL", old) end
- ln = ln + 1
- I = i
- return i
-end
-
-----------------------------------------------------------------------
--- initialize lexer for given source _z and source name _sourceid
-----------------------------------------------------------------------
-
-function init(_z, _sourceid)
- z = _z -- source
- sourceid = _sourceid -- name of source
- I = 1 -- lexer's position in source
- ln = 1 -- line number
- tok = {} -- lexed token list*
- seminfo = {} -- lexed semantic information list*
- tokln = {} -- line numbers for messages*
- -- (*) externally visible thru' module
- --------------------------------------------------------------------
- -- initial processing (shbang handling)
- --------------------------------------------------------------------
- local p, _, q, r = find(z, "^(#[^\r\n]*)(\r?\n?)")
- if p then -- skip first line
- I = I + #q
- addtoken("TK_COMMENT", q)
- if #r > 0 then inclinenumber(I, true) end
- end
-end
-
-----------------------------------------------------------------------
--- returns a chunk name or id, no truncation for long names
-----------------------------------------------------------------------
-
-function chunkid()
- if sourceid and match(sourceid, "^[=@]") then
- return sub(sourceid, 2) -- remove first char
- end
- return "[string]"
-end
-
-----------------------------------------------------------------------
--- formats error message and throws error
--- * a simplified version, does not report what token was responsible
-----------------------------------------------------------------------
-
-function errorline(s, line)
- local e = error or base.error
- e(string.format("%s:%d: %s", chunkid(), line or ln, s))
-end
-local errorline = errorline
-
-------------------------------------------------------------------------
--- count separators ("=") in a long string delimiter
-------------------------------------------------------------------------
-
-local function skip_sep(i)
- local sub = sub
- local s = sub(z, i, i)
- i = i + 1
- local count = #match(z, "=*", i) -- note, take the length
- i = i + count
- I = i
- return (sub(z, i, i) == s) and count or (-count) - 1
-end
-
-----------------------------------------------------------------------
--- reads a long string or long comment
-----------------------------------------------------------------------
-
-local function read_long_string(is_str, sep)
- local i = I + 1 -- skip 2nd '['
- local sub = sub
- local c = sub(z, i, i)
- if c == "\r" or c == "\n" then -- string starts with a newline?
- i = inclinenumber(i) -- skip it
- end
- local j = i
- while true do
- local p, q, r = find(z, "([\r\n%]])", i) -- (long range)
- if not p then
- errorline(is_str and "unfinished long string" or
- "unfinished long comment")
- end
- i = p
- if r == "]" then -- delimiter test
- if skip_sep(i) == sep then
- buff = sub(z, buff, I)
- I = I + 1 -- skip 2nd ']'
- return buff
- end
- i = I
- else -- newline
- buff = buff.."\n"
- i = inclinenumber(i)
- end
- end--while
-end
-
-----------------------------------------------------------------------
--- reads a string
-----------------------------------------------------------------------
-
-local function read_string(del)
- local i = I
- local find = find
- local sub = sub
- while true do
- local p, q, r = find(z, "([\n\r\\\"\'])", i) -- (long range)
- if p then
- if r == "\n" or r == "\r" then
- errorline("unfinished string")
- end
- i = p
- if r == "\\" then -- handle escapes
- i = i + 1
- r = sub(z, i, i)
- if r == "" then break end -- (EOZ error)
- p = find("abfnrtv\n\r", r, 1, true)
- ------------------------------------------------------
- if p then -- special escapes
- if p > 7 then
- i = inclinenumber(i)
- else
- i = i + 1
- end
- ------------------------------------------------------
- elseif find(r, "%D") then -- other non-digits
- i = i + 1
- ------------------------------------------------------
- else -- \xxx sequence
- local p, q, s = find(z, "^(%d%d?%d?)", i)
- i = q + 1
- if s + 1 > 256 then -- UCHAR_MAX
- errorline("escape sequence too large")
- end
- ------------------------------------------------------
- end--if p
- else
- i = i + 1
- if r == del then -- ending delimiter
- I = i
- return sub(z, buff, i - 1) -- return string
- end
- end--if r
- else
- break -- (error)
- end--if p
- end--while
- errorline("unfinished string")
-end
-
-------------------------------------------------------------------------
--- main lexer function
-------------------------------------------------------------------------
-
-function llex()
- local find = find
- local match = match
- while true do--outer
- local i = I
- -- inner loop allows break to be used to nicely section tests
- while true do--inner
- ----------------------------------------------------------------
- local p, _, r = find(z, "^([_%a][_%w]*)", i)
- if p then
- I = i + #r
- if kw[r] then
- addtoken("TK_KEYWORD", r) -- reserved word (keyword)
- else
- addtoken("TK_NAME", r) -- identifier
- end
- break -- (continue)
- end
- ----------------------------------------------------------------
- local p, _, r = find(z, "^(%.?)%d", i)
- if p then -- numeral
- if r == "." then i = i + 1 end
- local _, q, r = find(z, "^%d*[%.%d]*([eE]?)", i)
- i = q + 1
- if #r == 1 then -- optional exponent
- if match(z, "^[%+%-]", i) then -- optional sign
- i = i + 1
- end
- end
- local _, q = find(z, "^[_%w]*", i)
- I = q + 1
- local v = sub(z, p, q) -- string equivalent
- if not base.tonumber(v) then -- handles hex test also
- errorline("malformed number")
- end
- addtoken("TK_NUMBER", v)
- break -- (continue)
- end
- ----------------------------------------------------------------
- local p, q, r, t = find(z, "^((%s)[ \t\v\f]*)", i)
- if p then
- if t == "\n" or t == "\r" then -- newline
- inclinenumber(i, true)
- else
- I = q + 1 -- whitespace
- addtoken("TK_SPACE", r)
- end
- break -- (continue)
- end
- ----------------------------------------------------------------
- local r = match(z, "^%p", i)
- if r then
- buff = i
- local p = find("-[\"\'.=<>~", r, 1, true)
- if p then
- -- two-level if block for punctuation/symbols
- --------------------------------------------------------
- if p <= 2 then
- if p == 1 then -- minus
- local c = match(z, "^%-%-(%[?)", i)
- if c then
- i = i + 2
- local sep = -1
- if c == "[" then
- sep = skip_sep(i)
- end
- if sep >= 0 then -- long comment
- addtoken("TK_LCOMMENT", read_long_string(false, sep))
- else -- short comment
- I = find(z, "[\n\r]", i) or (#z + 1)
- addtoken("TK_COMMENT", sub(z, buff, I - 1))
- end
- break -- (continue)
- end
- -- (fall through for "-")
- else -- [ or long string
- local sep = skip_sep(i)
- if sep >= 0 then
- addtoken("TK_LSTRING", read_long_string(true, sep))
- elseif sep == -1 then
- addtoken("TK_OP", "[")
- else
- errorline("invalid long string delimiter")
- end
- break -- (continue)
- end
- --------------------------------------------------------
- elseif p <= 5 then
- if p < 5 then -- strings
- I = i + 1
- addtoken("TK_STRING", read_string(r))
- break -- (continue)
- end
- r = match(z, "^%.%.?%.?", i) -- .|..|... dots
- -- (fall through)
- --------------------------------------------------------
- else -- relational
- r = match(z, "^%p=?", i)
- -- (fall through)
- end
- end
- I = i + #r
- addtoken("TK_OP", r) -- for other symbols, fall through
- break -- (continue)
- end
- ----------------------------------------------------------------
- local r = sub(z, i, i)
- if r ~= "" then
- I = i + 1
- addtoken("TK_OP", r) -- other single-char tokens
- break
- end
- addtoken("TK_EOS", "") -- end of stream,
- return -- exit here
- ----------------------------------------------------------------
- end--while inner
- end--while outer
-end
-
-return base.getfenv()
projects / project / luci.git / blobdiff