modules/admin-full: Improved system configuration page
[project/luci.git] / libs / http / luasrc / http / protocol.lua
1 --[[
2
3 HTTP protocol implementation for LuCI
4 (c) 2008 Freifunk Leipzig / Jo-Philipp Wich <xm@leipzig.freifunk.net>
5
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9
10         http://www.apache.org/licenses/LICENSE-2.0
11
12 $Id$
13
14 ]]--
15
16 module("luci.http.protocol", package.seeall)
17
18 local ltn12 = require("luci.ltn12")
19
20 HTTP_MAX_CONTENT      = 1024*4          -- 4 kB maximum content size
21 HTTP_URLENC_MAXKEYLEN = 1024            -- maximum allowd size of urlencoded parameter names
22 TSRC_BLOCKSIZE        = 2048            -- target block size for throttling sources
23
24
25 -- Decode an urlencoded string.
26 -- Returns the decoded value.
27 function urldecode( str, no_plus )
28
29         local function __chrdec( hex )
30                 return string.char( tonumber( hex, 16 ) )
31         end
32
33         if type(str) == "string" then
34                 if not no_plus then
35                         str = str:gsub( "+", " " )
36                 end
37
38                 str = str:gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
39         end
40
41         return str
42 end
43
44
45 -- Extract and split urlencoded data pairs, separated bei either "&" or ";" from given url.
46 -- Returns a table value with urldecoded values.
47 function urldecode_params( url, tbl )
48
49         local params = tbl or { }
50
51         if url:find("?") then
52                 url = url:gsub( "^.+%?([^?]+)", "%1" )
53         end
54
55         for pair in url:gmatch( "[^&;]+" ) do
56
57                 -- find key and value
58                 local key = urldecode( pair:match("^([^=]+)")     )
59                 local val = urldecode( pair:match("^[^=]+=(.+)$") )
60
61                 -- store
62                 if type(key) == "string" and key:len() > 0 then
63                         if type(val) ~= "string" then val = "" end
64
65                         if not params[key] then
66                                 params[key] = val
67                         elseif type(params[key]) ~= "table" then
68                                 params[key] = { params[key], val }
69                         else
70                                 table.insert( params[key], val )
71                         end
72                 end
73         end
74
75         return params
76 end
77
78
79 -- Encode given string in urlencoded format.
80 -- Returns the encoded string.
81 function urlencode( str )
82
83         local function __chrenc( chr )
84                 return string.format(
85                         "%%%02x", string.byte( chr )
86                 )
87         end
88
89         if type(str) == "string" then
90                 str = str:gsub(
91                         "([^a-zA-Z0-9$_%-%.%+!*'(),])",
92                         __chrenc
93                 )
94         end
95
96         return str
97 end
98
99
100 -- Encode given table to urlencoded string.
101 -- Returns the encoded string.
102 function urlencode_params( tbl )
103         local enc = ""
104
105         for k, v in pairs(tbl) do
106                 enc = enc .. ( enc and "&" or "" ) ..
107                         urlencode(k) .. "="  ..
108                         urlencode(v)
109         end
110
111         return enc
112 end
113
114
115 -- Parameter helper
116 local function __initval( tbl, key )
117         if tbl[key] == nil then
118                 tbl[key] = ""
119         elseif type(tbl[key]) == "string" then
120                 tbl[key] = { tbl[key], "" }
121         else
122                 table.insert( tbl[key], "" )
123         end
124 end
125
126 local function __appendval( tbl, key, chunk )
127         if type(tbl[key]) == "table" then
128                 tbl[key][#tbl[key]] = tbl[key][#tbl[key]] .. chunk
129         else
130                 tbl[key] = tbl[key] .. chunk
131         end
132 end
133
134 local function __finishval( tbl, key, handler )
135         if handler then
136                 if type(tbl[key]) == "table" then
137                         tbl[key][#tbl[key]] = handler( tbl[key][#tbl[key]] )
138                 else
139                         tbl[key] = handler( tbl[key] )
140                 end
141         end
142 end
143
144
145 -- Table of our process states
146 local process_states = { }
147
148 -- Extract "magic", the first line of a http message.
149 -- Extracts the message type ("get", "post" or "response"), the requested uri
150 -- or the status code if the line descripes a http response.
151 process_states['magic'] = function( msg, chunk, err )
152
153         if chunk ~= nil then
154                 -- ignore empty lines before request
155                 if #chunk == 0 then
156                         return true, nil
157                 end
158
159                 -- Is it a request?
160                 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
161
162                 -- Yup, it is
163                 if method then
164
165                         msg.type           = "request"
166                         msg.request_method = method:lower()
167                         msg.request_uri    = uri
168                         msg.http_version   = tonumber( http_ver )
169                         msg.headers        = { }
170
171                         -- We're done, next state is header parsing
172                         return true, function( chunk )
173                                 return process_states['headers']( msg, chunk )
174                         end
175
176                 -- Is it a response?
177                 else
178
179                         local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
180
181                         -- Is a response
182                         if code then
183
184                                 msg.type           = "response"
185                                 msg.status_code    = code
186                                 msg.status_message = message
187                                 msg.http_version   = tonumber( http_ver )
188                                 msg.headers        = { }
189
190                                 -- We're done, next state is header parsing
191                                 return true, function( chunk )
192                                         return process_states['headers']( msg, chunk )
193                                 end
194                         end
195                 end
196         end
197
198         -- Can't handle it
199         return nil, "Invalid HTTP message magic"
200 end
201
202
203 -- Extract headers from given string.
204 process_states['headers'] = function( msg, chunk )
205
206         if chunk ~= nil then
207
208                 -- Look for a valid header format
209                 local hdr, val = chunk:match( "^([A-Z][A-Za-z0-9%-_]+): +(.+)$" )
210
211                 if type(hdr) == "string" and hdr:len() > 0 and
212                    type(val) == "string" and val:len() > 0
213                 then
214                         msg.headers[hdr] = val
215
216                         -- Valid header line, proceed
217                         return true, nil
218
219                 elseif #chunk == 0 then
220                         -- Empty line, we won't accept data anymore
221                         return false, nil
222                 else
223                         -- Junk data
224                         return nil, "Invalid HTTP header received"
225                 end
226         else
227                 return nil, "Unexpected EOF"
228         end
229 end
230
231
232 -- Init urldecoding stream
233 process_states['urldecode-init'] = function( msg, chunk, filecb )
234
235         if chunk ~= nil then
236
237                 -- Check for Content-Length
238                 if msg.env.CONTENT_LENGTH then
239                         msg.content_length = tonumber(msg.env.CONTENT_LENGTH)
240
241                         if msg.content_length <= HTTP_MAX_CONTENT then
242                                 -- Initialize buffer
243                                 msg._urldecbuffer = chunk
244                                 msg._urldeclength = 0
245
246                                 -- Switch to urldecode-key state
247                                 return true, function(chunk)
248                                         return process_states['urldecode-key']( msg, chunk, filecb )
249                                 end
250                         else
251                                 return nil, "Request exceeds maximum allowed size"
252                         end
253                 else
254                         return nil, "Missing Content-Length header"
255                 end
256         else
257                 return nil, "Unexpected EOF"
258         end
259 end
260
261
262 -- Process urldecoding stream, read and validate parameter key
263 process_states['urldecode-key'] = function( msg, chunk, filecb )
264         if chunk ~= nil then
265
266                 -- Prevent oversized requests
267                 if msg._urldeclength >= msg.content_length then
268                         return nil, "Request exceeds maximum allowed size"
269                 end
270
271                 -- Combine look-behind buffer with current chunk
272                 local buffer = msg._urldecbuffer .. chunk
273                 local spos, epos = buffer:find("=")
274
275                 -- Found param
276                 if spos then
277
278                         -- Check that key doesn't exceed maximum allowed key length
279                         if ( spos - 1 ) <= HTTP_URLENC_MAXKEYLEN then
280                                 local key = urldecode( buffer:sub( 1, spos - 1 ) )
281
282                                 -- Prepare buffers
283                                 msg._urldeclength   = msg._urldeclength + epos
284                                 msg._urldecbuffer   = buffer:sub( epos + 1, #buffer )
285
286                                 -- Use file callback or store values inside msg.params
287                                 if filecb then
288                                         msg._urldeccallback = function( chunk, eof )
289                                                 filecb( field, chunk, eof )
290                                         end
291                                 else
292                                         __initval( msg.params, key )
293
294                                         msg._urldeccallback = function( chunk, eof )
295                                                 __appendval( msg.params, key, chunk )
296
297                                                 -- FIXME: Use a filter
298                                                 if eof then
299                                                         __finishval( msg.params, key, urldecode )
300                                                 end
301                                         end
302                                 end
303
304                                 -- Proceed with urldecode-value state
305                                 return true, function( chunk )
306                                         return process_states['urldecode-value']( msg, chunk, filecb )
307                                 end
308                         else
309                                 return nil, "POST parameter exceeds maximum allowed length"
310                         end
311                 else
312                         return nil, "POST data exceeds maximum allowed length"
313                 end
314         else
315                 return nil, "Unexpected EOF"
316         end
317 end
318
319
320 -- Process urldecoding stream, read parameter value
321 process_states['urldecode-value'] = function( msg, chunk, filecb )
322
323         if chunk ~= nil then
324
325                 -- Combine look-behind buffer with current chunk
326                 local buffer = msg._urldecbuffer .. chunk
327
328                 -- Check for EOF
329                 if #buffer == 0 then
330                         -- Compare processed length
331                         if msg._urldeclength == msg.content_length then
332                                 -- Cleanup
333                                 msg._urldeclength   = nil
334                                 msg._urldecbuffer   = nil
335                                 msg._urldeccallback = nil
336
337                                 -- We won't accept data anymore
338                                 return false
339                         else
340                                 return nil, "Content-Length mismatch"
341                         end
342                 end
343
344                 -- Check for end of value
345                 local spos, epos = buffer:find("[&;]")
346                 if spos then
347
348                         -- Flush buffer, send eof
349                         msg._urldeccallback( buffer:sub( 1, spos - 1 ), true )
350                         msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
351                         msg._urldeclength = msg._urldeclength + epos
352
353                         -- Back to urldecode-key state
354                         return true, function( chunk )
355                                 return process_states['urldecode-key']( msg, chunk, filecb )
356                         end
357                 else
358                         -- We're somewhere within a data section and our buffer is full
359                         if #buffer > #chunk then
360                                 -- Flush buffered data
361                                 msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
362
363                                 -- Store new data
364                                 msg._urldeclength = msg._urldeclength + #buffer - #chunk
365                                 msg._urldecbuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
366
367                         -- Buffer is not full yet, append new data
368                         else
369                                 msg._urldecbuffer = buffer
370                         end
371
372                         -- Keep feeding me
373                         return true
374                 end
375         else
376                 -- Send EOF
377                 msg._urldeccallback( "", true )
378                 return false
379         end
380 end
381
382
383 -- Creates a header source from a given socket
384 function header_source( sock )
385         return ltn12.source.simplify( function()
386
387                 local chunk, err, part = sock:receive("*l")
388
389                 -- Line too long
390                 if chunk == nil then
391                         if err ~= "timeout" then
392                                 return nil, part
393                                         and "Line exceeds maximum allowed length"
394                                         or  "Unexpected EOF"
395                         else
396                                 return nil, err
397                         end
398
399                 -- Line ok
400                 elseif chunk ~= nil then
401
402                         -- Strip trailing CR
403                         chunk = chunk:gsub("\r$","")
404
405                         return chunk, nil
406                 end
407         end )
408 end
409
410
411 -- Decode MIME encoded data.
412 function mimedecode_message_body( src, msg, filecb )
413
414         if msg and msg.env.CONTENT_TYPE then
415                 msg.mime_boundary = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)$")
416         end
417
418         if not msg.mime_boundary then
419                 return nil, "Invalid Content-Type found"
420         end
421
422
423         local function parse_headers( chunk, field )
424
425                 local stat
426                 repeat
427                         chunk, stat = chunk:gsub(
428                                 "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n",
429                                 function(k,v)
430                                         field.headers[k] = v
431                                         return ""
432                                 end
433                         )
434                 until stat == 0
435
436                 chunk, stat = chunk:gsub("^\r\n","")
437
438                 -- End of headers
439                 if stat > 0 then
440                         if field.headers["Content-Disposition"] then
441                                 if field.headers["Content-Disposition"]:match("^form%-data; ") then
442                                         field.name = field.headers["Content-Disposition"]:match('name="(.-)"')
443                                         field.file = field.headers["Content-Disposition"]:match('filename="(.+)"$')
444                                 end
445                         end
446
447                         if not field.headers["Content-Type"] then
448                                 field.headers["Content-Type"] = "text/plain"
449                         end
450
451                         return chunk, true
452                 end
453
454                 return chunk, false
455         end
456
457
458         local field  = { headers = { } }
459         local inhdr  = false
460         local store  = nil
461         local lchunk = nil
462
463         local function snk( chunk )
464
465                 if chunk and not lchunk then
466                         lchunk = "\r\n" .. chunk
467
468                 elseif lchunk then
469                         local data = lchunk .. ( chunk or "" )
470                         local spos, epos, found
471
472                         repeat
473                                 spos, epos = data:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
474
475                                 if not spos then
476                                         spos, epos = data:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
477                                 end
478
479
480                                 if spos then
481                                         local predata = data:sub( 1, spos - 1 )
482
483                                         if inhdr then
484                                                 predata, eof = parse_headers( predata, field )
485
486                                                 if not eof then
487                                                         return nil, "Invalid MIME section header"
488                                                 end
489
490                                                 if not field.name then
491                                                         return nil, "Invalid Content-Disposition header"
492                                                 end
493                                         end
494
495                                         if store then
496                                                 store( field.headers, predata, true )
497                                         end
498
499
500                                         field = { headers = { } }
501                                         found = found or true
502
503                                         data, eof = parse_headers( data:sub( epos + 1, #data ), field )
504                                         inhdr = not eof
505
506                                         if eof then
507                                                 if field.file and filecb then
508                                                         msg.params[field.name] = field.file
509                                                         store = filecb
510                                                 else
511                                                         __initval( msg.params, field.name )
512
513                                                         store = function( hdr, buf, eof )
514                                                                 __appendval( msg.params, field.name, buf )
515                                                         end
516                                                 end
517                                         end
518                                 end
519                         until not spos
520
521
522                         if found then
523                                 if #data > 78 then
524                                         lchunk = data:sub( #data - 78 + 1, #data )
525                                         data   = data:sub( 1, #data - 78 )
526
527                                         store( field.headers, data )
528                                 else
529                                         lchunk, data = data, nil
530                                 end
531                         else
532                                 if inhdr then
533                                         lchunk, eof = parse_headers( data, field )
534                                         inhdr = not eof
535                                 else
536                                         store( field.headers, lchunk )
537                                         lchunk, chunk = chunk, nil
538                                 end
539                         end
540                 end
541
542                 return true
543         end
544
545         return luci.ltn12.pump.all( src, snk )
546 end
547
548
549 -- Decode urlencoded data.
550 function urldecode_message_body( source, msg )
551
552         -- Create an initial LTN12 sink
553         -- Return the initial state.
554         local sink = ltn12.sink.simplify(
555                 function( chunk )
556                         return process_states['urldecode-init']( msg, chunk )
557                 end
558         )
559
560         -- Create a throttling LTN12 source
561         -- See explaination in mimedecode_message_body().
562         local tsrc = function()
563                 if msg._urldecbuffer ~= nil and #msg._urldecbuffer > 0 then
564                         return ""
565                 else
566                         return source()
567                 end
568         end
569
570         -- Pump input data...
571         while true do
572                 -- get data
573                 local ok, err = ltn12.pump.step( tsrc, sink )
574
575                 -- step
576                 if not ok and err then
577                         return nil, err
578
579                 -- eof
580                 elseif not ok then
581                         return true
582                 end
583         end
584 end
585
586
587 -- Parse a http message header
588 function parse_message_header( source )
589
590         local ok   = true
591         local msg  = { }
592
593         local sink = ltn12.sink.simplify(
594                 function( chunk )
595                         return process_states['magic']( msg, chunk )
596                 end
597         )
598
599         -- Pump input data...
600         while ok do
601
602                 -- get data
603                 ok, err = ltn12.pump.step( source, sink )
604
605                 -- error
606                 if not ok and err then
607                         return nil, err
608
609                 -- eof
610                 elseif not ok then
611
612                         -- Process get parameters
613                         if ( msg.request_method == "get" or msg.request_method == "post" ) and
614                            msg.request_uri:match("?")
615                         then
616                                 msg.params = urldecode_params( msg.request_uri )
617                         else
618                                 msg.params = { }
619                         end
620
621                         -- Populate common environment variables
622                         msg.env = {
623                                 CONTENT_LENGTH    = msg.headers['Content-Length'];
624                                 CONTENT_TYPE      = msg.headers['Content-Type'];
625                                 REQUEST_METHOD    = msg.request_method:upper();
626                                 REQUEST_URI       = msg.request_uri;
627                                 SCRIPT_NAME       = msg.request_uri:gsub("?.+$","");
628                                 SCRIPT_FILENAME   = "";         -- XXX implement me
629                                 SERVER_PROTOCOL   = "HTTP/" .. string.format("%.1f", msg.http_version)
630                         }
631
632                         -- Populate HTTP_* environment variables
633                         for i, hdr in ipairs( {
634                                 'Accept',
635                                 'Accept-Charset',
636                                 'Accept-Encoding',
637                                 'Accept-Language',
638                                 'Connection',
639                                 'Cookie',
640                                 'Host',
641                                 'Referer',
642                                 'User-Agent',
643                         } ) do
644                                 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
645                                 local val = msg.headers[hdr]
646
647                                 msg.env[var] = val
648                         end
649                 end
650         end
651
652         return msg
653 end
654
655
656 -- Parse a http message body
657 function parse_message_body( source, msg, filecb )
658         -- Is it multipart/mime ?
659         if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
660            msg.env.CONTENT_TYPE:match("^multipart/form%-data")
661         then
662
663                 return mimedecode_message_body( source, msg, filecb )
664
665         -- Is it application/x-www-form-urlencoded ?
666         elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
667                msg.env.CONTENT_TYPE == "application/x-www-form-urlencoded"
668         then
669                 return urldecode_message_body( source, msg, filecb )
670
671
672         -- Unhandled encoding
673         -- If a file callback is given then feed it chunk by chunk, else
674         -- store whole buffer in message.content
675         else
676
677                 local sink
678
679                 -- If we have a file callback then feed it
680                 if type(filecb) == "function" then
681                         sink = filecb
682
683                 -- ... else append to .content
684                 else
685                         msg.content = ""
686                         msg.content_length = 0
687
688                         sink = function( chunk )
689                                 if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
690
691                                         msg.content        = msg.content        .. chunk
692                                         msg.content_length = msg.content_length + #chunk
693
694                                         return true
695                                 else
696                                         return nil, "POST data exceeds maximum allowed length"
697                                 end
698                         end
699                 end
700
701                 -- Pump data...
702                 while true do
703                         local ok, err = ltn12.pump.step( source, sink )
704
705                         if not ok and err then
706                                 return nil, err
707                         elseif not err then
708                                 return true
709                         end
710                 end
711         end
712 end
713
714 -- Status codes
715 statusmsg = {
716         [200] = "OK",
717         [301] = "Moved Permanently",
718         [304] = "Not Modified",
719         [400] = "Bad Request",
720         [403] = "Forbidden",
721         [404] = "Not Found",
722         [405] = "Method Not Allowed",
723         [411] = "Length Required",
724         [412] = "Precondition Failed",
725         [500] = "Internal Server Error",
726         [503] = "Server Unavailable",
727 }