3 HTTP protocol implementation for LuCI
4 (c) 2008 Freifunk Leipzig / Jo-Philipp Wich <xm@leipzig.freifunk.net>
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
10 http://www.apache.org/licenses/LICENSE-2.0
16 module("luci.http.protocol", package.seeall)
20 require("luci.http.protocol.filter")
22 HTTP_MAX_CONTENT = 1024*4 -- 4 kB maximum content size
23 HTTP_URLENC_MAXKEYLEN = 1024 -- maximum allowd size of urlencoded parameter names
26 -- Decode an urlencoded string.
27 -- Returns the decoded value.
28 function urldecode( str )
30 local function __chrdec( hex )
31 return string.char( tonumber( hex, 16 ) )
34 if type(str) == "string" then
35 str = str:gsub( "+", " " ):gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
42 -- Extract and split urlencoded data pairs, separated bei either "&" or ";" from given url.
43 -- Returns a table value with urldecoded values.
44 function urldecode_params( url, tbl )
46 local params = tbl or { }
49 url = url:gsub( "^.+%?([^?]+)", "%1" )
52 for i, pair in ipairs(luci.util.split( url, "[&;]+", nil, true )) do
55 local key = urldecode( pair:match("^([^=]+)") )
56 local val = urldecode( pair:match("^[^=]+=(.+)$") )
59 if type(key) == "string" and key:len() > 0 then
60 if type(val) ~= "string" then val = "" end
62 if not params[key] then
64 elseif type(params[key]) ~= "table" then
65 params[key] = { params[key], val }
67 table.insert( params[key], val )
76 -- Encode given string in urlencoded format.
77 -- Returns the encoded string.
78 function urlencode( str )
80 local function __chrenc( chr )
82 "%%%02x", string.byte( chr )
86 if type(str) == "string" then
88 "([^a-zA-Z0-9$_%-%.+!*'(),])",
97 -- Encode given table to urlencoded string.
98 -- Returns the encoded string.
99 function urlencode_params( tbl )
102 for k, v in pairs(tbl) do
103 enc = enc .. ( enc and "&" or "" ) ..
104 urlencode(k) .. "=" ..
112 -- Table of our process states
113 local process_states = { }
115 -- Extract "magic", the first line of a http message.
116 -- Extracts the message type ("get", "post" or "response"), the requested uri
117 -- or the status code if the line descripes a http response.
118 process_states['magic'] = function( msg, chunk )
123 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
129 msg.request_method = method:lower()
130 msg.request_uri = uri
131 msg.http_version = tonumber( http_ver )
134 -- We're done, next state is header parsing
135 return true, function( chunk )
136 return process_states['headers']( msg, chunk )
142 local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
147 msg.type = "response"
148 msg.status_code = code
149 msg.status_message = message
150 msg.http_version = tonumber( http_ver )
153 -- We're done, next state is header parsing
154 return true, function( chunk )
155 return process_states['headers']( msg, chunk )
162 return nil, "Invalid HTTP message magic"
166 -- Extract headers from given string.
167 process_states['headers'] = function( msg, chunk )
171 -- Look for a valid header format
172 local hdr, val = chunk:match( "^([A-Z][A-Za-z0-9%-_]+): +(.+)$" )
174 if type(hdr) == "string" and hdr:len() > 0 and
175 type(val) == "string" and val:len() > 0
177 msg.headers[hdr] = val
179 -- Valid header line, proceed
182 elseif #chunk == 0 then
183 -- Empty line, we won't accept data anymore
187 return nil, "Invalid HTTP header received"
190 return nil, "Unexpected EOF"
195 -- Find first MIME boundary
196 process_states['mime-init'] = function( msg, chunk, filecb )
199 if #chunk >= #msg.mime_boundary + 2 then
200 local boundary = chunk:sub( 1, #msg.mime_boundary + 4 )
202 if boundary == "--" .. msg.mime_boundary .. "\r\n" then
204 -- Store remaining data in buffer
205 msg._mimebuffer = chunk:sub( #msg.mime_boundary + 5, #chunk )
207 -- Switch to header processing state
208 return true, function( chunk )
209 return process_states['mime-headers']( msg, chunk, filecb )
212 return nil, "Invalid MIME boundary"
218 return nil, "Unexpected EOF"
223 -- Read MIME part headers
224 process_states['mime-headers'] = function( msg, chunk, filecb )
228 -- Combine look-behind buffer with current chunk
229 chunk = msg._mimebuffer .. chunk
231 if not msg._mimeheaders then
232 msg._mimeheaders = { }
235 local function __storehdr( k, v )
236 msg._mimeheaders[k] = v
240 -- Read all header lines
241 local ok, count = 1, 0
243 chunk, ok = chunk:gsub( "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n", __storehdr )
247 -- Headers processed, check for empty line
248 chunk, ok = chunk:gsub( "^\r\n", "" )
250 -- Store remaining buffer contents
251 msg._mimebuffer = chunk
256 -- When no Content-Type header is given assume text/plain
257 if not msg._mimeheaders['Content-Type'] then
258 msg._mimeheaders['Content-Type'] = 'text/plain'
261 -- Check Content-Disposition
262 if msg._mimeheaders['Content-Disposition'] then
263 -- Check for "form-data" token
264 if msg._mimeheaders['Content-Disposition']:match("^form%-data; ") then
265 -- Check for field name, filename
266 local field = msg._mimeheaders['Content-Disposition']:match('name="(.-)"')
267 local file = msg._mimeheaders['Content-Disposition']:match('filename="(.+)"$')
269 -- Is a file field and we have a callback
270 if file and filecb then
271 msg.params[field] = file
272 msg._mimecallback = function(chunk,eof)
276 headers = msg._mimeheaders
280 -- Treat as form field
282 msg.params[field] = ""
283 msg._mimecallback = function(chunk,eof)
284 msg.params[field] = msg.params[field] .. chunk
288 -- Header was valid, continue with mime-data
289 return true, function( chunk )
290 return process_states['mime-data']( msg, chunk, filecb )
293 -- Unknown Content-Disposition, abort
294 return nil, "Unexpected Content-Disposition MIME section header"
297 -- Content-Disposition is required, abort without
298 return nil, "Missing Content-Disposition MIME section header"
301 -- We parsed no headers yet and buffer is almost empty
302 elseif count > 0 or #chunk < 128 then
303 -- Keep feeding me with chunks
307 -- Buffer looks like garbage
308 return nil, "Malformed MIME section header"
310 return nil, "Unexpected EOF"
315 -- Read MIME part data
316 process_states['mime-data'] = function( msg, chunk, filecb )
320 -- Combine look-behind buffer with current chunk
321 local buffer = msg._mimebuffer .. chunk
323 -- Look for MIME boundary
324 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
328 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
331 msg._mimebuffer = buffer:sub( epos + 1, #buffer )
333 -- Next state is mime-header processing
334 return true, function( chunk )
335 return process_states['mime-headers']( msg, chunk, filecb )
339 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
343 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
345 -- We processed the final MIME boundary, cleanup
346 msg._mimebuffer = nil
347 msg._mimeheaders = nil
348 msg._mimecallback = nil
350 -- We won't accept data anymore
353 -- We're somewhere within a data section and our buffer is full
354 if #buffer > #chunk then
355 -- Flush buffered data
356 msg._mimecallback( buffer:sub( 1, #buffer - #chunk ), false )
359 msg._mimebuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
361 -- Buffer is not full yet, append new data
363 msg._mimebuffer = buffer
371 return nil, "Unexpected EOF"
376 -- Init urldecoding stream
377 process_states['urldecode-init'] = function( msg, chunk, filecb )
381 -- Check for Content-Length
382 if msg.env.CONTENT_LENGTH then
383 msg.content_length = tonumber(msg.env.CONTENT_LENGTH)
385 if msg.content_length <= HTTP_MAX_CONTENT then
387 msg._urldecbuffer = chunk
388 msg._urldeclength = 0
390 -- Switch to urldecode-key state
391 return true, function(chunk)
392 return process_states['urldecode-key']( msg, chunk, filecb )
395 return nil, "Request exceeds maximum allowed size"
398 return nil, "Missing Content-Length header"
401 return nil, "Unexpected EOF"
406 -- Process urldecoding stream, read and validate parameter key
407 process_states['urldecode-key'] = function( msg, chunk, filecb )
410 -- Prevent oversized requests
411 if msg._urldeclength >= msg.content_length then
412 return nil, "Request exceeds maximum allowed size"
415 -- Combine look-behind buffer with current chunk
416 local buffer = msg._urldecbuffer .. chunk
417 local spos, epos = buffer:find("=")
422 -- Check that key doesn't exceed maximum allowed key length
423 if ( spos - 1 ) <= HTTP_URLENC_MAXKEYLEN then
424 local key = urldecode( buffer:sub( 1, spos - 1 ) )
428 msg._urldeclength = msg._urldeclength + epos
429 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
431 -- Use file callback or store values inside msg.params
433 msg._urldeccallback = function( chunk, eof )
434 filecb( field, chunk, eof )
437 msg._urldeccallback = function( chunk, eof )
438 msg.params[key] = msg.params[key] .. chunk
440 -- FIXME: Use a filter
442 msg.params[key] = urldecode( msg.params[key] )
447 -- Proceed with urldecode-value state
448 return true, function( chunk )
449 return process_states['urldecode-value']( msg, chunk, filecb )
452 return nil, "POST parameter exceeds maximum allowed length"
455 return nil, "POST data exceeds maximum allowed length"
458 return nil, "Unexpected EOF"
463 -- Process urldecoding stream, read parameter value
464 process_states['urldecode-value'] = function( msg, chunk, filecb )
468 -- Combine look-behind buffer with current chunk
469 local buffer = msg._urldecbuffer .. chunk
473 -- Compare processed length
474 if msg._urldeclength == msg.content_length then
476 msg._urldeclength = nil
477 msg._urldecbuffer = nil
478 msg._urldeccallback = nil
480 -- We won't accept data anymore
483 return nil, "Content-Length mismatch"
487 -- Check for end of value
488 local spos, epos = buffer:find("[&;]")
491 -- Flush buffer, send eof
492 msg._urldeccallback( buffer:sub( 1, spos - 1 ), true )
493 msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
494 msg._urldeclength = msg._urldeclength + epos
496 -- Back to urldecode-key state
497 return true, function( chunk )
498 return process_states['urldecode-key']( msg, chunk, filecb )
501 -- We're somewhere within a data section and our buffer is full
502 if #buffer > #chunk then
503 -- Flush buffered data
504 msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
507 msg._urldeclength = msg._urldeclength + #buffer - #chunk
508 msg._urldecbuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
510 -- Buffer is not full yet, append new data
512 msg._urldecbuffer = buffer
519 return nil, "Unexpected EOF"
524 -- Decode MIME encoded data.
525 function mimedecode_message_body( source, msg, filecb )
527 -- Find mime boundary
528 if msg and msg.env.CONTENT_TYPE then
530 local bound = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)")
533 msg.mime_boundary = bound
535 return nil, "No MIME boundary found or invalid content type given"
539 -- Create an initial LTN12 sink
540 -- The whole MIME parsing process is implemented as fancy sink, sinks replace themself
541 -- depending on current processing state (init, header, data). Return the initial state.
542 local sink = ltn12.sink.simplify(
544 return process_states['mime-init']( msg, chunk, filecb )
548 -- Create a throttling LTN12 source
549 -- Frequent state switching in the mime parsing process leads to unwanted buffer aggregation.
550 -- This source checks wheather there's still data in our internal read buffer and returns an
551 -- empty string if there's already enough data in the processing queue. If the internal buffer
552 -- runs empty we're calling the original source to get the next chunk of data.
553 local tsrc = function()
555 -- XXX: we schould propably keep the maximum buffer size in sync with
556 -- the blocksize of our original source... but doesn't really matter
557 if msg._mimebuffer ~= null and #msg._mimebuffer > 256 then
564 -- Pump input data...
567 local ok, err = ltn12.pump.step( tsrc, sink )
570 if not ok and err then
581 -- Decode urlencoded data.
582 function urldecode_message_body( source, msg )
584 -- Create an initial LTN12 sink
585 -- Return the initial state.
586 local sink = ltn12.sink.simplify(
588 return process_states['urldecode-init']( msg, chunk )
592 -- Create a throttling LTN12 source
593 -- See explaination in mimedecode_message_body().
594 local tsrc = function()
595 if msg._urldecbuffer ~= null and #msg._urldecbuffer > 0 then
602 -- Pump input data...
605 local ok, err = ltn12.pump.step( tsrc, sink )
608 if not ok and err then
619 -- Parse a http message
620 function parse_message( data, filecb )
622 local reader = _linereader( data, HTTP_MAX_READBUF )
623 local message = parse_message_header( reader )
626 parse_message_body( reader, message, filecb )
633 -- Parse a http message header
634 function parse_message_header( source )
639 local sink = ltn12.sink.simplify(
641 return process_states['magic']( msg, chunk )
645 -- Pump input data...
649 ok, err = ltn12.pump.step( source, sink )
652 if not ok and err then
658 -- Process get parameters
659 if ( msg.request_method == "get" or msg.request_method == "post" ) and
660 msg.request_uri:match("?")
662 msg.params = urldecode_params( msg.request_uri )
667 -- Populate common environment variables
669 CONTENT_LENGTH = msg.headers['Content-Length'];
670 CONTENT_TYPE = msg.headers['Content-Type'];
671 REQUEST_METHOD = msg.request_method:upper();
672 REQUEST_URI = msg.request_uri;
673 SCRIPT_NAME = msg.request_uri:gsub("?.+$","");
674 SCRIPT_FILENAME = ""; -- XXX implement me
675 SERVER_PROTOCOL = "HTTP/" .. msg.http_version
678 -- Populate HTTP_* environment variables
679 for i, hdr in ipairs( {
690 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
691 local val = msg.headers[hdr]
702 -- Parse a http message body
703 function parse_message_body( source, msg, filecb )
705 -- Install an additional filter if we're operating on chunked transfer
706 -- coding and client is HTTP/1.1 capable
707 if msg.http_version == 1.1 and
708 msg.headers['Transfer-Encoding'] and
709 msg.headers['Transfer-Encoding']:find("chunked")
711 source = ltn12.source.chain(
712 source, luci.http.protocol.filter.decode_chunked
717 -- Is it multipart/mime ?
718 if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
719 msg.env.CONTENT_TYPE:match("^multipart/form%-data")
722 return mimedecode_message_body( source, msg, filecb )
724 -- Is it application/x-www-form-urlencoded ?
725 elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
726 msg.env.CONTENT_TYPE == "application/x-www-form-urlencoded"
728 return urldecode_message_body( source, msg, filecb )
731 -- Unhandled encoding
732 -- If a file callback is given then feed it line by line, else
733 -- store whole buffer in message.content
738 -- If we have a file callback then feed it
739 if type(filecb) == "function" then
742 -- ... else append to .content
745 msg.content_length = 0
747 sink = function( chunk )
748 if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
750 msg.content = msg.content .. chunk
751 msg.content_length = msg.content_length + #chunk
755 return nil, "POST data exceeds maximum allowed length"
762 local ok, err = ltn12.pump.step( source, sink )
764 if not ok and err then