3 HTTP protocol implementation for LuCI
4 (c) 2008 Freifunk Leipzig / Jo-Philipp Wich <xm@leipzig.freifunk.net>
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
10 http://www.apache.org/licenses/LICENSE-2.0
16 module("luci.http.protocol", package.seeall)
18 local ltn12 = require("luci.ltn12")
20 HTTP_MAX_CONTENT = 1024*4 -- 4 kB maximum content size
21 HTTP_URLENC_MAXKEYLEN = 1024 -- maximum allowd size of urlencoded parameter names
22 TSRC_BLOCKSIZE = 2048 -- target block size for throttling sources
25 -- Decode an urlencoded string.
26 -- Returns the decoded value.
27 function urldecode( str, no_plus )
29 local function __chrdec( hex )
30 return string.char( tonumber( hex, 16 ) )
33 if type(str) == "string" then
35 str = str:gsub( "+", " " )
38 str = str:gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
45 -- Extract and split urlencoded data pairs, separated bei either "&" or ";" from given url.
46 -- Returns a table value with urldecoded values.
47 function urldecode_params( url, tbl )
49 local params = tbl or { }
52 url = url:gsub( "^.+%?([^?]+)", "%1" )
55 for pair in url:gmatch( "[^&;]+" ) do
58 local key = urldecode( pair:match("^([^=]+)") )
59 local val = urldecode( pair:match("^[^=]+=(.+)$") )
62 if type(key) == "string" and key:len() > 0 then
63 if type(val) ~= "string" then val = "" end
65 if not params[key] then
67 elseif type(params[key]) ~= "table" then
68 params[key] = { params[key], val }
70 table.insert( params[key], val )
79 -- Encode given string in urlencoded format.
80 -- Returns the encoded string.
81 function urlencode( str )
83 local function __chrenc( chr )
85 "%%%02x", string.byte( chr )
89 if type(str) == "string" then
91 "([^a-zA-Z0-9$_%-%.%+!*'(),])",
100 -- Encode given table to urlencoded string.
101 -- Returns the encoded string.
102 function urlencode_params( tbl )
105 for k, v in pairs(tbl) do
106 enc = enc .. ( enc and "&" or "" ) ..
107 urlencode(k) .. "=" ..
116 local function __initval( tbl, key )
117 if tbl[key] == nil then
119 elseif type(tbl[key]) == "string" then
120 tbl[key] = { tbl[key], "" }
122 table.insert( tbl[key], "" )
126 local function __appendval( tbl, key, chunk )
127 if type(tbl[key]) == "table" then
128 tbl[key][#tbl[key]] = tbl[key][#tbl[key]] .. chunk
130 tbl[key] = tbl[key] .. chunk
134 local function __finishval( tbl, key, handler )
136 if type(tbl[key]) == "table" then
137 tbl[key][#tbl[key]] = handler( tbl[key][#tbl[key]] )
139 tbl[key] = handler( tbl[key] )
145 -- Table of our process states
146 local process_states = { }
148 -- Extract "magic", the first line of a http message.
149 -- Extracts the message type ("get", "post" or "response"), the requested uri
150 -- or the status code if the line descripes a http response.
151 process_states['magic'] = function( msg, chunk, err )
154 -- ignore empty lines before request
160 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
166 msg.request_method = method:lower()
167 msg.request_uri = uri
168 msg.http_version = tonumber( http_ver )
171 -- We're done, next state is header parsing
172 return true, function( chunk )
173 return process_states['headers']( msg, chunk )
179 local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
184 msg.type = "response"
185 msg.status_code = code
186 msg.status_message = message
187 msg.http_version = tonumber( http_ver )
190 -- We're done, next state is header parsing
191 return true, function( chunk )
192 return process_states['headers']( msg, chunk )
199 return nil, "Invalid HTTP message magic"
203 -- Extract headers from given string.
204 process_states['headers'] = function( msg, chunk )
208 -- Look for a valid header format
209 local hdr, val = chunk:match( "^([A-Z][A-Za-z0-9%-_]+): +(.+)$" )
211 if type(hdr) == "string" and hdr:len() > 0 and
212 type(val) == "string" and val:len() > 0
214 msg.headers[hdr] = val
216 -- Valid header line, proceed
219 elseif #chunk == 0 then
220 -- Empty line, we won't accept data anymore
224 return nil, "Invalid HTTP header received"
227 return nil, "Unexpected EOF"
232 -- Find first MIME boundary
233 process_states['mime-init'] = function( msg, chunk, filecb )
236 if #chunk >= #msg.mime_boundary + 2 then
237 local boundary = chunk:sub( 1, #msg.mime_boundary + 4 )
239 if boundary == "--" .. msg.mime_boundary .. "\r\n" then
241 -- Store remaining data in buffer
242 msg._mimebuffer = chunk:sub( #msg.mime_boundary + 5, #chunk )
244 -- Switch to header processing state
245 return true, function( chunk )
246 return process_states['mime-headers']( msg, chunk, filecb )
249 return nil, "Invalid MIME boundary"
255 return nil, "Unexpected EOF"
260 -- Read MIME part headers
261 process_states['mime-headers'] = function( msg, chunk, filecb )
265 -- Combine look-behind buffer with current chunk
266 chunk = msg._mimebuffer .. chunk
268 if not msg._mimeheaders then
269 msg._mimeheaders = { }
272 local function __storehdr( k, v )
273 msg._mimeheaders[k] = v
277 -- Read all header lines
278 local ok, count = 1, 0
280 chunk, ok = chunk:gsub( "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n", __storehdr )
284 -- Headers processed, check for empty line
285 chunk, ok = chunk:gsub( "^\r\n", "" )
287 -- Store remaining buffer contents
288 msg._mimebuffer = chunk
293 -- When no Content-Type header is given assume text/plain
294 if not msg._mimeheaders['Content-Type'] then
295 msg._mimeheaders['Content-Type'] = 'text/plain'
298 -- Check Content-Disposition
299 if msg._mimeheaders['Content-Disposition'] then
300 -- Check for "form-data" token
301 if msg._mimeheaders['Content-Disposition']:match("^form%-data; ") then
302 -- Check for field name, filename
303 local field = msg._mimeheaders['Content-Disposition']:match('name="(.-)"')
304 local file = msg._mimeheaders['Content-Disposition']:match('filename="(.+)"$')
306 -- Is a file field and we have a callback
307 if file and filecb then
308 msg.params[field] = file
309 msg._mimecallback = function(chunk,eof)
313 headers = msg._mimeheaders
317 -- Treat as form field
319 __initval( msg.params, field )
321 msg._mimecallback = function(chunk,eof)
322 __appendval( msg.params, field, chunk )
326 -- Header was valid, continue with mime-data
327 return true, function( chunk )
328 return process_states['mime-data']( msg, chunk, filecb )
331 -- Unknown Content-Disposition, abort
332 return nil, "Unexpected Content-Disposition MIME section header"
335 -- Content-Disposition is required, abort without
336 return nil, "Missing Content-Disposition MIME section header"
339 -- We parsed no headers yet and buffer is almost empty
340 elseif count > 0 or #chunk < 128 then
341 -- Keep feeding me with chunks
345 -- Buffer looks like garbage
346 return nil, "Malformed MIME section header"
348 return nil, "Unexpected EOF"
353 -- Read MIME part data
354 process_states['mime-data'] = function( msg, chunk, filecb )
358 -- Combine look-behind buffer with current chunk
359 local buffer = msg._mimebuffer .. chunk
361 -- Look for MIME boundary
362 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
366 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
369 msg._mimebuffer = buffer:sub( epos + 1, #buffer )
371 -- Next state is mime-header processing
372 return true, function( chunk )
373 return process_states['mime-headers']( msg, chunk, filecb )
377 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
381 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
383 -- We processed the final MIME boundary, cleanup
384 msg._mimebuffer = nil
385 msg._mimeheaders = nil
386 msg._mimecallback = nil
388 -- We won't accept data anymore
391 -- We're somewhere within a data section and our buffer is full
392 if #buffer > #chunk then
393 -- Flush buffered data
394 msg._mimecallback( buffer:sub( 1, #buffer - #chunk ), false )
397 msg._mimebuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
399 -- Buffer is not full yet, append new data
401 msg._mimebuffer = buffer
409 return nil, "Unexpected EOF"
414 -- Init urldecoding stream
415 process_states['urldecode-init'] = function( msg, chunk, filecb )
419 -- Check for Content-Length
420 if msg.env.CONTENT_LENGTH then
421 msg.content_length = tonumber(msg.env.CONTENT_LENGTH)
423 if msg.content_length <= HTTP_MAX_CONTENT then
425 msg._urldecbuffer = chunk
426 msg._urldeclength = 0
428 -- Switch to urldecode-key state
429 return true, function(chunk)
430 return process_states['urldecode-key']( msg, chunk, filecb )
433 return nil, "Request exceeds maximum allowed size"
436 return nil, "Missing Content-Length header"
439 return nil, "Unexpected EOF"
444 -- Process urldecoding stream, read and validate parameter key
445 process_states['urldecode-key'] = function( msg, chunk, filecb )
448 -- Prevent oversized requests
449 if msg._urldeclength >= msg.content_length then
450 return nil, "Request exceeds maximum allowed size"
453 -- Combine look-behind buffer with current chunk
454 local buffer = msg._urldecbuffer .. chunk
455 local spos, epos = buffer:find("=")
460 -- Check that key doesn't exceed maximum allowed key length
461 if ( spos - 1 ) <= HTTP_URLENC_MAXKEYLEN then
462 local key = urldecode( buffer:sub( 1, spos - 1 ) )
465 msg._urldeclength = msg._urldeclength + epos
466 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
468 -- Use file callback or store values inside msg.params
470 msg._urldeccallback = function( chunk, eof )
471 filecb( field, chunk, eof )
474 __initval( msg.params, key )
476 msg._urldeccallback = function( chunk, eof )
477 __appendval( msg.params, key, chunk )
479 -- FIXME: Use a filter
481 __finishval( msg.params, key, urldecode )
486 -- Proceed with urldecode-value state
487 return true, function( chunk )
488 return process_states['urldecode-value']( msg, chunk, filecb )
491 return nil, "POST parameter exceeds maximum allowed length"
494 return nil, "POST data exceeds maximum allowed length"
497 return nil, "Unexpected EOF"
502 -- Process urldecoding stream, read parameter value
503 process_states['urldecode-value'] = function( msg, chunk, filecb )
507 -- Combine look-behind buffer with current chunk
508 local buffer = msg._urldecbuffer .. chunk
512 -- Compare processed length
513 if msg._urldeclength == msg.content_length then
515 msg._urldeclength = nil
516 msg._urldecbuffer = nil
517 msg._urldeccallback = nil
519 -- We won't accept data anymore
522 return nil, "Content-Length mismatch"
526 -- Check for end of value
527 local spos, epos = buffer:find("[&;]")
530 -- Flush buffer, send eof
531 msg._urldeccallback( buffer:sub( 1, spos - 1 ), true )
532 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
533 msg._urldeclength = msg._urldeclength + epos
535 -- Back to urldecode-key state
536 return true, function( chunk )
537 return process_states['urldecode-key']( msg, chunk, filecb )
540 -- We're somewhere within a data section and our buffer is full
541 if #buffer > #chunk then
542 -- Flush buffered data
543 msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
546 msg._urldeclength = msg._urldeclength + #buffer - #chunk
547 msg._urldecbuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
549 -- Buffer is not full yet, append new data
551 msg._urldecbuffer = buffer
559 msg._urldeccallback( "", true )
565 -- Creates a header source from a given socket
566 function header_source( sock )
567 return ltn12.source.simplify( function()
569 local chunk, err, part = sock:receive("*l")
573 if err ~= "timeout" then
575 and "Line exceeds maximum allowed length"
582 elseif chunk ~= nil then
585 chunk = chunk:gsub("\r$","")
593 -- Decode MIME encoded data.
594 function mimedecode_message_body( source, msg, filecb )
596 -- Find mime boundary
597 if msg and msg.env.CONTENT_TYPE then
599 local bound = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)")
602 msg.mime_boundary = bound
604 return nil, "No MIME boundary found or invalid content type given"
608 -- Create an initial LTN12 sink
609 -- The whole MIME parsing process is implemented as fancy sink, sinks replace themself
610 -- depending on current processing state (init, header, data). Return the initial state.
611 local sink = ltn12.sink.simplify(
613 return process_states['mime-init']( msg, chunk, filecb )
617 -- Create a throttling LTN12 source
618 -- Frequent state switching in the mime parsing process leads to unwanted buffer aggregation.
619 -- This source checks wheather there's still data in our internal read buffer and returns an
620 -- empty string if there's already enough data in the processing queue. If the internal buffer
621 -- runs empty we're calling the original source to get the next chunk of data.
622 local tsrc = function()
624 -- XXX: we schould propably keep the maximum buffer size in sync with
625 -- the blocksize of our original source... but doesn't really matter
626 if msg._mimebuffer ~= nil and #msg._mimebuffer > TSRC_BLOCKSIZE then
633 -- Pump input data...
636 local ok, err = ltn12.pump.step( tsrc, sink )
639 if not ok and err then
650 -- Decode urlencoded data.
651 function urldecode_message_body( source, msg )
653 -- Create an initial LTN12 sink
654 -- Return the initial state.
655 local sink = ltn12.sink.simplify(
657 return process_states['urldecode-init']( msg, chunk )
661 -- Create a throttling LTN12 source
662 -- See explaination in mimedecode_message_body().
663 local tsrc = function()
664 if msg._urldecbuffer ~= nil and #msg._urldecbuffer > TSRC_BLOCKSIZE then
671 -- Pump input data...
674 local ok, err = ltn12.pump.step( tsrc, sink )
677 if not ok and err then
688 -- Parse a http message header
689 function parse_message_header( source )
694 local sink = ltn12.sink.simplify(
696 return process_states['magic']( msg, chunk )
700 -- Pump input data...
704 ok, err = ltn12.pump.step( source, sink )
707 if not ok and err then
713 -- Process get parameters
714 if ( msg.request_method == "get" or msg.request_method == "post" ) and
715 msg.request_uri:match("?")
717 msg.params = urldecode_params( msg.request_uri )
722 -- Populate common environment variables
724 CONTENT_LENGTH = msg.headers['Content-Length'];
725 CONTENT_TYPE = msg.headers['Content-Type'];
726 REQUEST_METHOD = msg.request_method:upper();
727 REQUEST_URI = msg.request_uri;
728 SCRIPT_NAME = msg.request_uri:gsub("?.+$","");
729 SCRIPT_FILENAME = ""; -- XXX implement me
730 SERVER_PROTOCOL = "HTTP/" .. string.format("%.1f", msg.http_version)
733 -- Populate HTTP_* environment variables
734 for i, hdr in ipairs( {
745 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
746 local val = msg.headers[hdr]
757 -- Parse a http message body
758 function parse_message_body( source, msg, filecb )
759 -- Is it multipart/mime ?
760 if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
761 msg.env.CONTENT_TYPE:match("^multipart/form%-data")
764 return mimedecode_message_body( source, msg, filecb )
766 -- Is it application/x-www-form-urlencoded ?
767 elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
768 msg.env.CONTENT_TYPE == "application/x-www-form-urlencoded"
770 return urldecode_message_body( source, msg, filecb )
773 -- Unhandled encoding
774 -- If a file callback is given then feed it chunk by chunk, else
775 -- store whole buffer in message.content
780 -- If we have a file callback then feed it
781 if type(filecb) == "function" then
784 -- ... else append to .content
787 msg.content_length = 0
789 sink = function( chunk )
790 if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
792 msg.content = msg.content .. chunk
793 msg.content_length = msg.content_length + #chunk
797 return nil, "POST data exceeds maximum allowed length"
804 local ok, err = ltn12.pump.step( source, sink )
806 if not ok and err then
818 [301] = "Moved Permanently",
819 [304] = "Not Modified",
820 [400] = "Bad Request",
823 [405] = "Method Not Allowed",
824 [411] = "Length Required",
825 [412] = "Precondition Failed",
826 [500] = "Internal Server Error",
827 [503] = "Server Unavailable",