From: Jo-Philipp Wich <jow@openwrt.org>
Date: Thu, 23 Jul 2009 00:41:44 +0000 (+0000)
Subject: libs/web: add template parser reimplemented in C
X-Git-Tag: 0.10.0~1326
X-Git-Url: https://git.archive.openwrt.org/?p=project%2Fluci.git;a=commitdiff_plain;h=80a2dd2cc27e2fc0c55f03f01a1d460a91954ab6;hp=9cf0abcb073454cc405a4757b04446dcdc98d399

libs/web: add template parser reimplemented in C
---

diff --git a/libs/web/Makefile b/libs/web/Makefile
index f7fac7740..cc3bc96fc 100644
--- a/libs/web/Makefile
+++ b/libs/web/Makefile
@@ -1,2 +1,28 @@
 include ../../build/config.mk
 include ../../build/module.mk
+include ../../build/gccconfig.mk
+
+TPL_LDFLAGS    =
+TPL_CFLAGS     =
+TPL_SO         = parser.so
+TPL_COMMON_OBJ = src/template_parser.o
+TPL_LUALIB_OBJ = src/template_lualib.o
+
+%.o: %.c
+	$(COMPILE) $(TPL_CFLAGS) $(LUA_CFLAGS) $(FPIC) -c -o $@ $< 
+
+compile: build-clean $(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ)
+	$(LINK) $(SHLIB_FLAGS) $(TPL_LDFLAGS) -o src/$(TPL_SO) \
+		$(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ)
+	mkdir -p dist$(LUCI_LIBRARYDIR)/template
+	cp src/$(TPL_SO) dist$(LUCI_LIBRARYDIR)/template/$(TPL_SO)
+
+install: build
+	cp -pR dist$(LUA_LIBRARYDIR)/* $(LUA_LIBRARYDIR)
+
+clean: build-clean
+
+build-clean:
+	rm -f src/*.o src/$(TPL_SO)
+
+
diff --git a/libs/web/src/template_lualib.c b/libs/web/src/template_lualib.c
new file mode 100644
index 000000000..685613fbd
--- /dev/null
+++ b/libs/web/src/template_lualib.c
@@ -0,0 +1,62 @@
+/*
+ * LuCI Template - Lua binding
+ *
+ *   Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include "template_lualib.h"
+
+int template_L_parse(lua_State *L)
+{
+	const char *file = luaL_checkstring(L, 1);
+	struct template_parser parser;
+	int lua_status;
+
+	if( (parser.fd = open(file, O_RDONLY)) > 0 )
+	{
+		parser.flags   = 0;
+		parser.bufsize = 0;
+		parser.state   = T_STATE_TEXT_NEXT;
+		
+		if( !(lua_status = lua_load(L, template_reader, &parser, file)) )
+		{
+			return 1;
+		}
+		else
+		{
+			lua_pushnil(L);
+			lua_pushinteger(L, lua_status);
+			lua_pushlstring(L, parser.out, parser.outsize);
+			return 3;
+		}
+	}
+
+	lua_pushnil(L);
+	lua_pushinteger(L, 255);
+	lua_pushstring(L, "No such file or directory");
+	return 3;
+}
+
+/* module table */
+static const luaL_reg R[] = {
+	{"parse",	template_L_parse},
+	{NULL,		NULL}
+};
+
+LUALIB_API int luaopen_luci_template_parser(lua_State *L) {
+	luaL_register(L, TEMPLATE_LUALIB_META, R);
+	return 1;
+}
+
diff --git a/libs/web/src/template_lualib.h b/libs/web/src/template_lualib.h
new file mode 100644
index 000000000..de915e1a2
--- /dev/null
+++ b/libs/web/src/template_lualib.h
@@ -0,0 +1,28 @@
+/*
+ * LuCI Template - Lua library header
+ *
+ *   Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef _TEMPLATE_LUALIB_H_
+#define _TEMPLATE_LUALIB_H_
+
+#include "template_parser.h"
+
+#define TEMPLATE_LUALIB_META  "template.parser"
+
+LUALIB_API int luaopen_luci_template_parser(lua_State *L);
+
+#endif
diff --git a/libs/web/src/template_parser.c b/libs/web/src/template_parser.c
new file mode 100644
index 000000000..58de5bb77
--- /dev/null
+++ b/libs/web/src/template_parser.c
@@ -0,0 +1,463 @@
+/*
+ * LuCI Template - Parser implementation
+ *
+ *   Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include "template_parser.h"
+
+
+/* leading and trailing code for different types */
+const char * gen_code[6][2] = {
+	{ "write(\"",			"\")"	},
+	{ NULL,					NULL	},
+	{ "write(tostring(",	"))"	},
+	{ "include(\"",			"\")"	},
+	{ "write(translate(\"",	"\"))"	},
+	{ NULL,					" "		}
+};
+
+/* Simple strstr() like function that takes len arguments for both haystack and needle. */
+static char *strfind(char *haystack, int hslen, const char *needle, int ndlen)
+{
+	int match = 0;
+	int i, j;
+
+	for( i = 0; i < hslen; i++ )
+	{
+		if( haystack[i] == needle[0] )
+		{
+			match = ((ndlen == 1) || ((i + ndlen) <= hslen));
+
+			for( j = 1; (j < ndlen) && ((i + j) < hslen); j++ )
+			{
+				if( haystack[i+j] != needle[j] )
+				{
+					match = 0;
+					break;
+				}
+			}
+
+			if( match )
+				return &haystack[i];
+		}
+	}
+
+	return NULL;
+}
+
+/* 
+ * Inspect current read buffer and find the number of "vague" characters at the end 
+ * which could indicate an opening token. Returns the number of "vague" chars.
+ * The last continuous sequence of whitespace, optionally followed by a "<" is
+ * treated as "vague" because whitespace may be discarded if the upcoming opening
+ * token indicates pre-whitespace-removal ("<%-"). A single remaining "<" char 
+ * can't be differentiated from an opening token ("<%"), so it's kept to be processed
+ * in the next cycle.
+ */
+static int stokscan(struct template_parser *data, int off, int no_whitespace)
+{
+	int i;
+	int skip = 0;
+	int tokoff = data->bufsize - 1;
+
+	for( i = tokoff; i >= off; i-- )
+	{
+		if( data->buf[i] == T_TOK_START[0] )
+		{
+			skip = tokoff - i + 1;
+			tokoff = i - 1;
+			break;
+		}
+	}
+
+	if( !no_whitespace )
+	{
+		for( i = tokoff; i >= off; i-- )
+		{
+			if( isspace(data->buf[i]) )
+				skip++;
+			else
+				break;
+		}
+	}
+
+	return skip;
+}
+
+/*
+ * Similar to stokscan() but looking for closing token indicators.
+ * Matches "-", optionally followed by a "%" char.
+ */
+static int etokscan(struct template_parser *data)
+{
+	int skip = 0;
+
+	if( (data->bufsize > 0) && (data->buf[data->bufsize-1] == T_TOK_END[0]) )
+		skip++;
+
+	if( (data->bufsize > skip) && (data->buf[data->bufsize-skip-1] == T_TOK_SKIPWS[0]) )
+		skip++;
+
+	return skip;
+}
+
+/*
+ * Generate Lua expressions from the given raw code, write it into the
+ * output buffer and set the lua_Reader specific size pointer.
+ * Takes parser-state, lua_Reader's size pointer and generator flags
+ * as parameter. The given flags indicate whether leading or trailing
+ * code should be added. Returns a pointer to the output buffer.
+ */
+static const char * generate_expression(struct template_parser *data, size_t *sz, int what)
+{
+	char tmp[T_OUTBUFSZ];
+	int i;
+	int size = 0;
+	int start = 0;
+	int i18n_hasdef = 0;
+
+	memset(tmp, 0, T_OUTBUFSZ);
+
+	/* Inject leading expression code (if any) */
+	if( (what & T_GEN_START) && (gen_code[data->type][0] != NULL) )
+	{
+		memcpy(tmp, gen_code[data->type][0], strlen(gen_code[data->type][0]));
+		size += strlen(gen_code[data->type][0]);
+	}
+
+	/* Parse source buffer */
+	for( i = 0; i < data->outsize; i++ )
+	{
+		/* Skip leading whitespace for non-raw and non-expr chunks */
+		if( !start && isspace(data->out[i]) && (data->type == T_TYPE_I18N || data->type == T_TYPE_INCLUDE) )
+			continue;
+		else if( !start )
+			start = 1;
+
+		/* Found whitespace after i18n key */
+		if( (data->type == T_TYPE_I18N) && (i18n_hasdef == 1) )
+		{
+			/* At non-whitespace char, inject seperator token */
+			if( !isspace(data->out[i]) )
+			{
+				memcpy(&tmp[size], T_TOK_I18NSEP, strlen(T_TOK_I18NSEP));
+				size += strlen(T_TOK_I18NSEP);
+				i18n_hasdef = 2;
+			}
+
+			/* At further whitespace, skip */
+			else
+			{
+				continue;
+			}
+		}
+
+		/* Escape quotes, backslashes and newlines for plain, i18n and include expressions */
+		if( (data->type == T_TYPE_TEXT || data->type == T_TYPE_I18N || data->type == T_TYPE_INCLUDE) &&
+		    (data->out[i] == '\\' || data->out[i] == '"' || data->out[i] == '\n' || data->out[i] == '\t') )
+		{
+			tmp[size++] = '\\';
+
+			switch(data->out[i])
+			{
+				case '\n':
+					tmp[size++] = 'n';
+					break;
+
+				case '\t':
+					tmp[size++] = 't';
+					break;
+
+				default:
+					tmp[size++] = data->out[i];
+			}
+		}
+
+		/* Found whitespace in i18n expression, raise flag */
+		else if( isspace(data->out[i]) && (data->type == T_TYPE_I18N) )
+		{
+			i18n_hasdef = 1;
+		}
+
+		/* Normal char */
+		else
+		{
+			tmp[size++] = data->out[i];
+		}
+	}
+
+	/* Processed i18n expression without default text, inject separator */
+	if( (data->type == T_TYPE_I18N) && (i18n_hasdef < 2) )
+	{
+		memcpy(&tmp[size], T_TOK_I18NSEP, strlen(T_TOK_I18NSEP));
+		size += strlen(T_TOK_I18NSEP);
+	}
+
+	/* Inject trailing expression code (if any) */
+	if( (what & T_GEN_END) && (gen_code[data->type][1] != NULL) )
+	{
+		memcpy(&tmp[size], gen_code[data->type][1], strlen(gen_code[data->type][1]));
+		size += strlen(gen_code[data->type][1]);
+	}
+
+	*sz = data->outsize = size;
+	memset(data->out, 0, T_OUTBUFSZ);
+	memcpy(data->out, tmp, size);
+
+	//printf("<<<%i|%i|%i|%s>>>\n", what, data->type, *sz, data->out);
+
+	return data->out;
+}
+
+/*
+ * Move the number of bytes specified in data->bufsize from the
+ * given source pointer to the beginning of the read buffer.
+ */
+static void bufmove(struct template_parser *data, const char *src)
+{
+	if( data->bufsize > 0 )
+		memmove(data->buf, src, data->bufsize);
+	else if( data->bufsize < 0 )
+		data->bufsize = 0;
+
+	data->buf[data->bufsize] = 0;
+}
+
+/*
+ * Move the given amount of bytes from the given source pointer
+ * to the output buffer and set data->outputsize.
+ */
+static void bufout(struct template_parser *data, const char *src, int len)
+{
+	if( len >= 0 )
+	{
+		memset(data->out, 0, T_OUTBUFSZ);
+		memcpy(data->out, src, len);
+		data->outsize = len;
+	}
+	else
+	{
+		data->outsize = 0;
+	}
+}
+
+/*
+ * lua_Reader compatible function that parses template code on demand from
+ * the given file handle.
+ */
+const char *template_reader(lua_State *L, void *ud, size_t *sz)
+{
+	struct template_parser *data = ud;
+	char *match = NULL;
+	int off = 0;
+	int ignore = 0;
+	int genflags = 0;
+	int readlen = 0;
+	int vague = 0;
+
+	while( !(data->flags & T_FLAG_EOF) || (data->bufsize > 0) )
+	{
+		/* Fill buffer */
+		if( !(data->flags & T_FLAG_EOF) && (data->bufsize < T_READBUFSZ) )
+		{
+			if( (readlen = read(data->fd, &data->buf[data->bufsize], T_READBUFSZ - data->bufsize)) > 0 )
+				data->bufsize += readlen;
+			else if( readlen == 0 )
+				data->flags |= T_FLAG_EOF;
+			else
+				return NULL;
+		}
+
+		/* Evaluate state */
+		switch(data->state)
+		{
+			/* Plain text chunk (before "<%") */
+			case T_STATE_TEXT_INIT:
+			case T_STATE_TEXT_NEXT:
+				off = 0; ignore = 0; *sz = 0;
+				data->type = T_TYPE_TEXT;
+
+				/* Skip leading whitespace if requested */
+				if( data->flags & T_FLAG_SKIPWS )
+				{
+					data->flags &= ~T_FLAG_SKIPWS;
+					while( (off < data->bufsize) && isspace(data->buf[off]) )
+						off++;
+				}
+
+				/* Found "<%" */
+				if( (match = strfind(&data->buf[off], data->bufsize - off - 1, T_TOK_START, strlen(T_TOK_START))) != NULL )
+				{
+					readlen = (int)(match - &data->buf[off]);
+					data->bufsize -= (readlen + strlen(T_TOK_START) + off);
+					match += strlen(T_TOK_START);
+
+					/* Check for leading '-' */
+					if( match[0] == T_TOK_SKIPWS[0] )
+					{
+						data->bufsize--;
+						match++;
+
+						while( (readlen > 1) && isspace(data->buf[off+readlen-1]) )
+						{
+							readlen--;
+						}
+					}
+
+					bufout(data, &data->buf[off], readlen);
+					bufmove(data, match);
+					data->state = T_STATE_CODE_INIT;
+				}
+
+				/* Maybe plain chunk */
+				else
+				{
+					/* Preserve trailing "<" or white space, maybe a start token */
+					vague = stokscan(data, off, 0);
+
+					/* We can process some bytes ... */
+					if( vague < data->bufsize )
+					{
+						readlen = data->bufsize - vague - off;
+					}
+
+					/* No bytes to process, so try to remove at least whitespace ... */
+					else
+					{
+						/* ... but try to preserve trailing "<" ... */
+						vague = stokscan(data, off, 1);
+
+						if( vague < data->bufsize )
+						{
+							readlen = data->bufsize - vague - off;
+						}
+
+						/* ... no chance, push out buffer */
+						else
+						{
+							readlen = vague - off;
+							vague   = 0;
+						}
+					}
+
+					bufout(data, &data->buf[off], readlen);
+
+					data->state   = T_STATE_TEXT_NEXT;
+					data->bufsize = vague;
+					bufmove(data, &data->buf[off+readlen]);
+				}
+
+				if( ignore || data->outsize == 0 )
+					continue;
+				else
+					return generate_expression(data, sz, T_GEN_START | T_GEN_END);
+
+				break;
+
+			/* Ignored chunk (inside "<%# ... %>") */
+			case T_STATE_SKIP:
+				ignore = 1;
+
+			/* Initial code chunk ("<% ...") */
+			case T_STATE_CODE_INIT:
+				off = 0;
+
+				/* Check for leading '-' */
+				if( data->buf[off] == T_TOK_SKIPWS[0] )
+					off++;
+
+				/* Determine code type */
+				switch(data->buf[off])
+				{
+					case '#':
+						ignore = 1;
+						off++;
+						data->type = T_TYPE_COMMENT;
+						break;
+
+					case '=':
+						off++;
+						data->type = T_TYPE_EXPR;
+						break;
+
+					case '+':
+						off++;
+						data->type = T_TYPE_INCLUDE;
+						break;
+
+					case ':':
+						off++;
+						data->type = T_TYPE_I18N;
+						break;
+
+					default:
+						data->type = T_TYPE_CODE;
+						break;
+				}
+
+			/* Subsequent code chunk ("..." or "... %>") */ 
+			case T_STATE_CODE_NEXT:
+				/* Found "%>" */
+				if( (match = strfind(&data->buf[off], data->bufsize - off, T_TOK_END, strlen(T_TOK_END))) != NULL )
+				{
+					genflags = ( data->state == T_STATE_CODE_INIT )
+						? (T_GEN_START | T_GEN_END) : T_GEN_END;
+
+					readlen = (int)(match - &data->buf[off]);
+
+					/* Check for trailing '-' */
+					if( (match > data->buf) && (*(match-1) == T_TOK_SKIPWS[0]) )
+					{
+						readlen--;
+						data->flags |= T_FLAG_SKIPWS;
+					}
+
+					bufout(data, &data->buf[off], readlen);
+
+					data->state = T_STATE_TEXT_INIT;
+					data->bufsize -= ((int)(match - &data->buf[off]) + strlen(T_TOK_END) + off);
+					bufmove(data, &match[strlen(T_TOK_END)]);
+				}
+
+				/* Code chunk */
+				else
+				{
+					genflags = ( data->state == T_STATE_CODE_INIT ) ? T_GEN_START : 0;
+
+					/* Preserve trailing "%" and "-", maybe an end token */
+					vague   = etokscan(data);
+					readlen = data->bufsize - off - vague;
+					bufout(data, &data->buf[off], readlen);
+
+					data->state   = T_STATE_CODE_NEXT;
+					data->bufsize = vague;
+					bufmove(data, &data->buf[readlen+off]);
+				}
+
+				if( ignore || (data->outsize == 0 && !genflags) )
+					continue;
+				else
+					return generate_expression(data, sz, genflags);
+
+				break;
+		}
+	}
+
+	*sz = 0;
+	return NULL;
+}
+
+
diff --git a/libs/web/src/template_parser.h b/libs/web/src/template_parser.h
new file mode 100644
index 000000000..42ebdff69
--- /dev/null
+++ b/libs/web/src/template_parser.h
@@ -0,0 +1,81 @@
+/*
+ * LuCI Template - Parser header
+ *
+ *   Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef _TEMPLATE_PARSER_H_
+#define _TEMPLATE_PARSER_H_
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <lua.h>
+#include <lualib.h>
+#include <lauxlib.h>
+
+
+#define T_READBUFSZ			1024
+#define T_OUTBUFSZ			T_READBUFSZ * 3
+
+/* parser states */
+#define T_STATE_TEXT_INIT	0
+#define T_STATE_TEXT_NEXT	1
+#define T_STATE_CODE_INIT	2
+#define T_STATE_CODE_NEXT	3
+#define T_STATE_SKIP 		4
+
+/* parser flags */
+#define T_FLAG_EOF			0x01
+#define T_FLAG_SKIPWS		0x02
+
+/* tokens used in matching and expression generation */
+#define T_TOK_START			"<%"
+#define T_TOK_END			"%>"
+#define T_TOK_SKIPWS		"-"
+#define T_TOK_I18NSEP		"\", \""
+
+/* generator flags */
+#define T_GEN_START			0x01
+#define T_GEN_END			0x02
+
+/* code types */
+#define T_TYPE_TEXT			0
+#define T_TYPE_COMMENT		1
+#define T_TYPE_EXPR			2
+#define T_TYPE_INCLUDE 		3
+#define T_TYPE_I18N			4
+#define T_TYPE_CODE			5
+
+/* parser state */
+struct template_parser {
+	int fd;
+	int bufsize;
+	int outsize;
+	int state;
+	int flags;
+	int type;
+	char buf[T_READBUFSZ];
+	char out[T_OUTBUFSZ];
+};
+
+
+const char *template_reader(lua_State *L, void *ud, size_t *sz);
+
+#endif