3 HTTP protocol implementation for LuCI
4 (c) 2008 Freifunk Leipzig / Jo-Philipp Wich <xm@leipzig.freifunk.net>
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
10 http://www.apache.org/licenses/LICENSE-2.0
16 module("luci.http.protocol", package.seeall)
18 local ltn12 = require("luci.ltn12")
20 HTTP_MAX_CONTENT = 1024*4 -- 4 kB maximum content size
21 HTTP_URLENC_MAXKEYLEN = 1024 -- maximum allowd size of urlencoded parameter names
24 -- Decode an urlencoded string.
25 -- Returns the decoded value.
26 function urldecode( str, no_plus )
28 local function __chrdec( hex )
29 return string.char( tonumber( hex, 16 ) )
32 if type(str) == "string" then
34 str = str:gsub( "+", " " )
37 str = str:gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
44 -- Extract and split urlencoded data pairs, separated bei either "&" or ";" from given url.
45 -- Returns a table value with urldecoded values.
46 function urldecode_params( url, tbl )
48 local params = tbl or { }
51 url = url:gsub( "^.+%?([^?]+)", "%1" )
54 for pair in url:gmatch( "[^&;]+" ) do
57 local key = urldecode( pair:match("^([^=]+)") )
58 local val = urldecode( pair:match("^[^=]+=(.+)$") )
61 if type(key) == "string" and key:len() > 0 then
62 if type(val) ~= "string" then val = "" end
64 if not params[key] then
66 elseif type(params[key]) ~= "table" then
67 params[key] = { params[key], val }
69 table.insert( params[key], val )
78 -- Encode given string in urlencoded format.
79 -- Returns the encoded string.
80 function urlencode( str )
82 local function __chrenc( chr )
84 "%%%02x", string.byte( chr )
88 if type(str) == "string" then
90 "([^a-zA-Z0-9$_%-%.%+!*'(),])",
99 -- Encode given table to urlencoded string.
100 -- Returns the encoded string.
101 function urlencode_params( tbl )
104 for k, v in pairs(tbl) do
105 enc = enc .. ( enc and "&" or "" ) ..
106 urlencode(k) .. "=" ..
115 local function __initval( tbl, key )
116 if tbl[key] == nil then
118 elseif type(tbl[key]) == "string" then
119 tbl[key] = { tbl[key], "" }
121 table.insert( tbl[key], "" )
125 local function __appendval( tbl, key, chunk )
126 if type(tbl[key]) == "table" then
127 tbl[key][#tbl[key]] = tbl[key][#tbl[key]] .. chunk
129 tbl[key] = tbl[key] .. chunk
133 local function __finishval( tbl, key, handler )
135 if type(tbl[key]) == "table" then
136 tbl[key][#tbl[key]] = handler( tbl[key][#tbl[key]] )
138 tbl[key] = handler( tbl[key] )
144 -- Table of our process states
145 local process_states = { }
147 -- Extract "magic", the first line of a http message.
148 -- Extracts the message type ("get", "post" or "response"), the requested uri
149 -- or the status code if the line descripes a http response.
150 process_states['magic'] = function( msg, chunk, err )
153 -- ignore empty lines before request
159 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
165 msg.request_method = method:lower()
166 msg.request_uri = uri
167 msg.http_version = tonumber( http_ver )
170 -- We're done, next state is header parsing
171 return true, function( chunk )
172 return process_states['headers']( msg, chunk )
178 local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
183 msg.type = "response"
184 msg.status_code = code
185 msg.status_message = message
186 msg.http_version = tonumber( http_ver )
189 -- We're done, next state is header parsing
190 return true, function( chunk )
191 return process_states['headers']( msg, chunk )
198 return nil, "Invalid HTTP message magic"
202 -- Extract headers from given string.
203 process_states['headers'] = function( msg, chunk )
207 -- Look for a valid header format
208 local hdr, val = chunk:match( "^([A-Z][A-Za-z0-9%-_]+): +(.+)$" )
210 if type(hdr) == "string" and hdr:len() > 0 and
211 type(val) == "string" and val:len() > 0
213 msg.headers[hdr] = val
215 -- Valid header line, proceed
218 elseif #chunk == 0 then
219 -- Empty line, we won't accept data anymore
223 return nil, "Invalid HTTP header received"
226 return nil, "Unexpected EOF"
231 -- Find first MIME boundary
232 process_states['mime-init'] = function( msg, chunk, filecb )
235 if #chunk >= #msg.mime_boundary + 2 then
236 local boundary = chunk:sub( 1, #msg.mime_boundary + 4 )
238 if boundary == "--" .. msg.mime_boundary .. "\r\n" then
240 -- Store remaining data in buffer
241 msg._mimebuffer = chunk:sub( #msg.mime_boundary + 5, #chunk )
243 -- Switch to header processing state
244 return true, function( chunk )
245 return process_states['mime-headers']( msg, chunk, filecb )
248 return nil, "Invalid MIME boundary"
254 return nil, "Unexpected EOF"
259 -- Read MIME part headers
260 process_states['mime-headers'] = function( msg, chunk, filecb )
264 -- Combine look-behind buffer with current chunk
265 chunk = msg._mimebuffer .. chunk
267 if not msg._mimeheaders then
268 msg._mimeheaders = { }
271 local function __storehdr( k, v )
272 msg._mimeheaders[k] = v
276 -- Read all header lines
277 local ok, count = 1, 0
279 chunk, ok = chunk:gsub( "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n", __storehdr )
283 -- Headers processed, check for empty line
284 chunk, ok = chunk:gsub( "^\r\n", "" )
286 -- Store remaining buffer contents
287 msg._mimebuffer = chunk
292 -- When no Content-Type header is given assume text/plain
293 if not msg._mimeheaders['Content-Type'] then
294 msg._mimeheaders['Content-Type'] = 'text/plain'
297 -- Check Content-Disposition
298 if msg._mimeheaders['Content-Disposition'] then
299 -- Check for "form-data" token
300 if msg._mimeheaders['Content-Disposition']:match("^form%-data; ") then
301 -- Check for field name, filename
302 local field = msg._mimeheaders['Content-Disposition']:match('name="(.-)"')
303 local file = msg._mimeheaders['Content-Disposition']:match('filename="(.+)"$')
305 -- Is a file field and we have a callback
306 if file and filecb then
307 msg.params[field] = file
308 msg._mimecallback = function(chunk,eof)
312 headers = msg._mimeheaders
316 -- Treat as form field
318 __initval( msg.params, field )
320 msg._mimecallback = function(chunk,eof)
321 __appendval( msg.params, field, chunk )
325 -- Header was valid, continue with mime-data
326 return true, function( chunk )
327 return process_states['mime-data']( msg, chunk, filecb )
330 -- Unknown Content-Disposition, abort
331 return nil, "Unexpected Content-Disposition MIME section header"
334 -- Content-Disposition is required, abort without
335 return nil, "Missing Content-Disposition MIME section header"
338 -- We parsed no headers yet and buffer is almost empty
339 elseif count > 0 or #chunk < 128 then
340 -- Keep feeding me with chunks
344 -- Buffer looks like garbage
345 return nil, "Malformed MIME section header"
347 return nil, "Unexpected EOF"
352 -- Read MIME part data
353 process_states['mime-data'] = function( msg, chunk, filecb )
357 -- Combine look-behind buffer with current chunk
358 local buffer = msg._mimebuffer .. chunk
360 -- Look for MIME boundary
361 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
365 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
368 msg._mimebuffer = buffer:sub( epos + 1, #buffer )
370 -- Next state is mime-header processing
371 return true, function( chunk )
372 return process_states['mime-headers']( msg, chunk, filecb )
376 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
380 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
382 -- We processed the final MIME boundary, cleanup
383 msg._mimebuffer = nil
384 msg._mimeheaders = nil
385 msg._mimecallback = nil
387 -- We won't accept data anymore
390 -- We're somewhere within a data section and our buffer is full
391 if #buffer > #chunk then
392 -- Flush buffered data
393 msg._mimecallback( buffer:sub( 1, #buffer - #chunk ), false )
396 msg._mimebuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
398 -- Buffer is not full yet, append new data
400 msg._mimebuffer = buffer
408 return nil, "Unexpected EOF"
413 -- Init urldecoding stream
414 process_states['urldecode-init'] = function( msg, chunk, filecb )
418 -- Check for Content-Length
419 if msg.env.CONTENT_LENGTH then
420 msg.content_length = tonumber(msg.env.CONTENT_LENGTH)
422 if msg.content_length <= HTTP_MAX_CONTENT then
424 msg._urldecbuffer = chunk
425 msg._urldeclength = 0
427 -- Switch to urldecode-key state
428 return true, function(chunk)
429 return process_states['urldecode-key']( msg, chunk, filecb )
432 return nil, "Request exceeds maximum allowed size"
435 return nil, "Missing Content-Length header"
438 return nil, "Unexpected EOF"
443 -- Process urldecoding stream, read and validate parameter key
444 process_states['urldecode-key'] = function( msg, chunk, filecb )
447 -- Prevent oversized requests
448 if msg._urldeclength >= msg.content_length then
449 return nil, "Request exceeds maximum allowed size"
452 -- Combine look-behind buffer with current chunk
453 local buffer = msg._urldecbuffer .. chunk
454 local spos, epos = buffer:find("=")
459 -- Check that key doesn't exceed maximum allowed key length
460 if ( spos - 1 ) <= HTTP_URLENC_MAXKEYLEN then
461 local key = urldecode( buffer:sub( 1, spos - 1 ) )
464 msg._urldeclength = msg._urldeclength + epos
465 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
467 -- Use file callback or store values inside msg.params
469 msg._urldeccallback = function( chunk, eof )
470 filecb( field, chunk, eof )
473 __initval( msg.params, key )
475 msg._urldeccallback = function( chunk, eof )
476 __appendval( msg.params, key, chunk )
478 -- FIXME: Use a filter
480 __finishval( msg.params, key, urldecode )
485 -- Proceed with urldecode-value state
486 return true, function( chunk )
487 return process_states['urldecode-value']( msg, chunk, filecb )
490 return nil, "POST parameter exceeds maximum allowed length"
493 return nil, "POST data exceeds maximum allowed length"
496 return nil, "Unexpected EOF"
501 -- Process urldecoding stream, read parameter value
502 process_states['urldecode-value'] = function( msg, chunk, filecb )
506 -- Combine look-behind buffer with current chunk
507 local buffer = msg._urldecbuffer .. chunk
511 -- Compare processed length
512 if msg._urldeclength == msg.content_length then
514 msg._urldeclength = nil
515 msg._urldecbuffer = nil
516 msg._urldeccallback = nil
518 -- We won't accept data anymore
521 return nil, "Content-Length mismatch"
525 -- Check for end of value
526 local spos, epos = buffer:find("[&;]")
529 -- Flush buffer, send eof
530 msg._urldeccallback( buffer:sub( 1, spos - 1 ), true )
531 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
532 msg._urldeclength = msg._urldeclength + epos
534 -- Back to urldecode-key state
535 return true, function( chunk )
536 return process_states['urldecode-key']( msg, chunk, filecb )
539 -- We're somewhere within a data section and our buffer is full
540 if #buffer > #chunk then
541 -- Flush buffered data
542 msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
545 msg._urldeclength = msg._urldeclength + #buffer - #chunk
546 msg._urldecbuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
548 -- Buffer is not full yet, append new data
550 msg._urldecbuffer = buffer
558 msg._urldeccallback( "", true )
564 -- Creates a header source from a given socket
565 function header_source( sock )
566 return ltn12.source.simplify( function()
568 local chunk, err, part = sock:receive("*l")
572 if err ~= "timeout" then
574 and "Line exceeds maximum allowed length"
581 elseif chunk ~= nil then
584 chunk = chunk:gsub("\r$","")
592 -- Decode MIME encoded data.
593 function mimedecode_message_body( source, msg, filecb )
595 -- Find mime boundary
596 if msg and msg.env.CONTENT_TYPE then
598 local bound = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)")
601 msg.mime_boundary = bound
603 return nil, "No MIME boundary found or invalid content type given"
607 -- Create an initial LTN12 sink
608 -- The whole MIME parsing process is implemented as fancy sink, sinks replace themself
609 -- depending on current processing state (init, header, data). Return the initial state.
610 local sink = ltn12.sink.simplify(
612 return process_states['mime-init']( msg, chunk, filecb )
616 -- Create a throttling LTN12 source
617 -- Frequent state switching in the mime parsing process leads to unwanted buffer aggregation.
618 -- This source checks wheather there's still data in our internal read buffer and returns an
619 -- empty string if there's already enough data in the processing queue. If the internal buffer
620 -- runs empty we're calling the original source to get the next chunk of data.
621 local tsrc = function()
623 -- XXX: we schould propably keep the maximum buffer size in sync with
624 -- the blocksize of our original source... but doesn't really matter
625 if msg._mimebuffer ~= nil and #msg._mimebuffer > 256 then
632 -- Pump input data...
635 local ok, err = ltn12.pump.step( tsrc, sink )
638 if not ok and err then
649 -- Decode urlencoded data.
650 function urldecode_message_body( source, msg )
652 -- Create an initial LTN12 sink
653 -- Return the initial state.
654 local sink = ltn12.sink.simplify(
656 return process_states['urldecode-init']( msg, chunk )
660 -- Create a throttling LTN12 source
661 -- See explaination in mimedecode_message_body().
662 local tsrc = function()
663 if msg._urldecbuffer ~= nil and #msg._urldecbuffer > 0 then
670 -- Pump input data...
673 local ok, err = ltn12.pump.step( tsrc, sink )
676 if not ok and err then
687 -- Parse a http message header
688 function parse_message_header( source )
693 local sink = ltn12.sink.simplify(
695 return process_states['magic']( msg, chunk )
699 -- Pump input data...
703 ok, err = ltn12.pump.step( source, sink )
706 if not ok and err then
712 -- Process get parameters
713 if ( msg.request_method == "get" or msg.request_method == "post" ) and
714 msg.request_uri:match("?")
716 msg.params = urldecode_params( msg.request_uri )
721 -- Populate common environment variables
723 CONTENT_LENGTH = msg.headers['Content-Length'];
724 CONTENT_TYPE = msg.headers['Content-Type'];
725 REQUEST_METHOD = msg.request_method:upper();
726 REQUEST_URI = msg.request_uri;
727 SCRIPT_NAME = msg.request_uri:gsub("?.+$","");
728 SCRIPT_FILENAME = ""; -- XXX implement me
729 SERVER_PROTOCOL = "HTTP/" .. string.format("%.1f", msg.http_version)
732 -- Populate HTTP_* environment variables
733 for i, hdr in ipairs( {
744 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
745 local val = msg.headers[hdr]
756 -- Parse a http message body
757 function parse_message_body( source, msg, filecb )
758 -- Is it multipart/mime ?
759 if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
760 msg.env.CONTENT_TYPE:match("^multipart/form%-data")
763 return mimedecode_message_body( source, msg, filecb )
765 -- Is it application/x-www-form-urlencoded ?
766 elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
767 msg.env.CONTENT_TYPE == "application/x-www-form-urlencoded"
769 return urldecode_message_body( source, msg, filecb )
772 -- Unhandled encoding
773 -- If a file callback is given then feed it chunk by chunk, else
774 -- store whole buffer in message.content
779 -- If we have a file callback then feed it
780 if type(filecb) == "function" then
783 -- ... else append to .content
786 msg.content_length = 0
788 sink = function( chunk )
789 if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
791 msg.content = msg.content .. chunk
792 msg.content_length = msg.content_length + #chunk
796 return nil, "POST data exceeds maximum allowed length"
803 local ok, err = ltn12.pump.step( source, sink )
805 if not ok and err then
817 [301] = "Moved Permanently",
818 [304] = "Not Modified",
819 [400] = "Bad Request",
822 [405] = "Method Not Allowed",
823 [411] = "Length Required",
824 [412] = "Precondition Failed",
825 [500] = "Internal Server Error",
826 [503] = "Server Unavailable",