luci-base: refactor luci.http
[project/luci.git] / libs / luci-lib-httpclient / luasrc / httpclient.lua
1 -- Copyright 2009 Steven Barth <steven@midlink.org>
2 -- Licensed to the public under the Apache License 2.0.
3
4 require "nixio.util"
5 local nixio = require "nixio"
6
7 local ltn12 = require "luci.ltn12"
8 local util = require "luci.util"
9 local table = require "table"
10 local http = require "luci.http.protocol"
11 local date = require "luci.http.protocol.date"
12
13 local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber
14 local unpack = unpack
15
16 module "luci.httpclient"
17
18 function chunksource(sock, buffer)
19         buffer = buffer or ""
20         return function()
21                 local output
22                 local _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
23                 while not count and #buffer <= 1024 do
24                         local newblock, code = sock:recv(1024 - #buffer)
25                         if not newblock then
26                                 return nil, code
27                         end
28                         buffer = buffer .. newblock  
29                         _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
30                 end
31                 count = tonumber(count, 16)
32                 if not count then
33                         return nil, -1, "invalid encoding"
34                 elseif count == 0 then
35                         return nil
36                 elseif count + 2 <= #buffer - endp then
37                         output = buffer:sub(endp+1, endp+count)
38                         buffer = buffer:sub(endp+count+3)
39                         return output
40                 else
41                         output = buffer:sub(endp+1, endp+count)
42                         buffer = ""
43                         if count - #output > 0 then
44                                 local remain, code = sock:recvall(count-#output)
45                                 if not remain then
46                                         return nil, code
47                                 end
48                                 output = output .. remain
49                                 count, code = sock:recvall(2)
50                         else
51                                 count, code = sock:recvall(count+2-#buffer+endp)
52                         end
53                         if not count then
54                                 return nil, code
55                         end
56                         return output
57                 end
58         end
59 end
60
61
62 function request_to_buffer(uri, options)
63         local source, code, msg = request_to_source(uri, options)
64         local output = {}
65         
66         if not source then
67                 return nil, code, msg
68         end
69         
70         source, code = ltn12.pump.all(source, (ltn12.sink.table(output)))
71         
72         if not source then
73                 return nil, code
74         end
75         
76         return table.concat(output)
77 end
78
79 function request_to_source(uri, options)
80         local status, response, buffer, sock = request_raw(uri, options)
81         if not status then
82                 return status, response, buffer
83         elseif status ~= 200 and status ~= 206 then
84                 return nil, status, buffer
85         end
86         
87         if response.headers["Transfer-Encoding"] == "chunked" then
88                 return chunksource(sock, buffer)
89         else
90                 return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource())
91         end
92 end
93
94 --
95 -- GET HTTP-resource
96 --
97 function request_raw(uri, options)
98         options = options or {}
99         local pr, auth, host, port, path
100
101         if options.params then
102                 uri = uri .. '?' .. http.urlencode_params(options.params)
103         end
104
105         if uri:find("%[") then
106                 if uri:find("@") then
107                         pr, auth, host, port, path = uri:match("(%w+)://(.+)@(%b[]):?([0-9]*)(.*)")
108                         host = host:sub(2,-2)
109                 else
110                         pr, host, port, path = uri:match("(%w+)://(%b[]):?([0-9]*)(.*)")
111                         host = host:sub(2,-2)
112                 end
113         else
114                 if uri:find("@") then
115                         pr, auth, host, port, path =
116                                 uri:match("(%w+)://(.+)@([%w-.]+):?([0-9]*)(.*)")
117                 else
118                         pr, host, port, path = uri:match("(%w+)://([%w-.]+):?([0-9]*)(.*)")
119                 end
120         end
121
122         if not host then
123                 return nil, -1, "unable to parse URI"
124         end
125         
126         if pr ~= "http" and pr ~= "https" then
127                 return nil, -2, "protocol not supported"
128         end
129         
130         port = #port > 0 and port or (pr == "https" and 443 or 80)
131         path = #path > 0 and path or "/"
132         
133         options.depth = options.depth or 10
134         local headers = options.headers or {}
135         local protocol = options.protocol or "HTTP/1.1"
136         headers["User-Agent"] = headers["User-Agent"] or "LuCI httpclient 0.1"
137         
138         if headers.Connection == nil then
139                 headers.Connection = "close"
140         end
141         
142         if auth and not headers.Authorization then
143                 headers.Authorization = "Basic " .. nixio.bin.b64encode(auth)
144         end
145
146         local sock, code, msg = nixio.connect(host, port)
147         if not sock then
148                 return nil, code, msg
149         end
150         
151         sock:setsockopt("socket", "sndtimeo", options.sndtimeo or 15)
152         sock:setsockopt("socket", "rcvtimeo", options.rcvtimeo or 15)
153         
154         if pr == "https" then
155                 local tls = options.tls_context or nixio.tls()
156                 sock = tls:create(sock)
157                 local stat, code, error = sock:connect()
158                 if not stat then
159                         return stat, code, error
160                 end
161         end
162
163         -- Pre assemble fixes   
164         if protocol == "HTTP/1.1" then
165                 headers.Host = headers.Host or host
166         end
167         
168         if type(options.body) == "table" then
169                 options.body = http.urlencode_params(options.body)
170         end
171
172         if type(options.body) == "string" then
173                 headers["Content-Length"] = headers["Content-Length"] or #options.body
174                 headers["Content-Type"] = headers["Content-Type"] or
175                         "application/x-www-form-urlencoded"
176                 options.method = options.method or "POST"
177         end
178         
179         if type(options.body) == "function" then
180                 options.method = options.method or "POST"
181         end
182
183         if options.cookies then
184                 local cookiedata = {}
185                 for _, c in ipairs(options.cookies) do
186                         local cdo = c.flags.domain
187                         local cpa = c.flags.path
188                         if   (cdo == host or cdo == "."..host or host:sub(-#cdo) == cdo) 
189                          and (cpa == path or cpa == "/" or cpa .. "/" == path:sub(#cpa+1))
190                          and (not c.flags.secure or pr == "https")
191                         then
192                                 cookiedata[#cookiedata+1] = c.key .. "=" .. c.value
193                         end 
194                 end
195                 if headers["Cookie"] then
196                         headers["Cookie"] = headers["Cookie"] .. "; " .. table.concat(cookiedata, "; ")
197                 else
198                         headers["Cookie"] = table.concat(cookiedata, "; ")
199                 end
200         end
201
202         -- Assemble message
203         local message = {(options.method or "GET") .. " " .. path .. " " .. protocol}
204         
205         for k, v in pairs(headers) do
206                 if type(v) == "string" or type(v) == "number" then
207                         message[#message+1] = k .. ": " .. v
208                 elseif type(v) == "table" then
209                         for i, j in ipairs(v) do
210                                 message[#message+1] = k .. ": " .. j
211                         end
212                 end
213         end
214
215         message[#message+1] = ""
216         message[#message+1] = ""
217         
218         -- Send request
219         sock:sendall(table.concat(message, "\r\n"))
220         
221         if type(options.body) == "string" then
222                 sock:sendall(options.body)
223         elseif type(options.body) == "function" then
224                 local res = {options.body(sock)}
225                 if not res[1] then
226                         sock:close()
227                         return unpack(res)
228                 end
229         end
230         
231         -- Create source and fetch response
232         local linesrc = sock:linesource()
233         local line, code, error = linesrc()
234         
235         if not line then
236                 sock:close()
237                 return nil, code, error
238         end
239         
240         local protocol, status, msg = line:match("^([%w./]+) ([0-9]+) (.*)")
241         
242         if not protocol then
243                 sock:close()
244                 return nil, -3, "invalid response magic: " .. line
245         end
246         
247         local response = {
248                 status = line, headers = {}, code = 0, cookies = {}, uri = uri
249         }
250         
251         line = linesrc()
252         while line and line ~= "" do
253                 local key, val = line:match("^([%w-]+)%s?:%s?(.*)")
254                 if key and key ~= "Status" then
255                         if type(response.headers[key]) == "string" then
256                                 response.headers[key] = {response.headers[key], val}
257                         elseif type(response.headers[key]) == "table" then
258                                 response.headers[key][#response.headers[key]+1] = val
259                         else
260                                 response.headers[key] = val
261                         end
262                 end
263                 line = linesrc()
264         end
265         
266         if not line then
267                 sock:close()
268                 return nil, -4, "protocol error"
269         end
270         
271         -- Parse cookies
272         if response.headers["Set-Cookie"] then
273                 local cookies = response.headers["Set-Cookie"]
274                 for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do
275                         local cobj = cookie_parse(c)
276                         cobj.flags.path = cobj.flags.path or path:match("(/.*)/?[^/]*")
277                         if not cobj.flags.domain or cobj.flags.domain == "" then
278                                 cobj.flags.domain = host
279                                 response.cookies[#response.cookies+1] = cobj
280                         else
281                                 local hprt, cprt = {}, {}
282                                 
283                                 -- Split hostnames and save them in reverse order
284                                 for part in host:gmatch("[^.]*") do
285                                         table.insert(hprt, 1, part)
286                                 end
287                                 for part in cobj.flags.domain:gmatch("[^.]*") do
288                                         table.insert(cprt, 1, part)
289                                 end
290                                 
291                                 local valid = true
292                                 for i, part in ipairs(cprt) do
293                                         -- If parts are different and no wildcard
294                                         if hprt[i] ~= part and #part ~= 0 then
295                                                 valid = false
296                                                 break
297                                         -- Wildcard on invalid position
298                                         elseif hprt[i] ~= part and #part == 0 then
299                                                 if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then
300                                                         valid = false
301                                                         break
302                                                 end
303                                         end
304                                 end
305                                 -- No TLD cookies
306                                 if valid and #cprt > 1 and #cprt[2] > 0 then
307                                         response.cookies[#response.cookies+1] = cobj
308                                 end
309                         end
310                 end
311         end
312         
313         -- Follow 
314         response.code = tonumber(status)
315         if response.code and options.depth > 0 then
316                 if (response.code == 301 or response.code == 302 or response.code == 307)
317                  and response.headers.Location then
318                         local nuri = response.headers.Location or response.headers.location
319                         if not nuri then
320                                 return nil, -5, "invalid reference"
321                         end
322                         if not nuri:find("https?://") then
323                                 nuri = pr .. "://" .. host .. ":" .. port .. nuri
324                         end
325                         
326                         options.depth = options.depth - 1
327                         if options.headers then
328                                 options.headers.Host = nil
329                         end
330                         sock:close()
331                         
332                         return request_raw(nuri, options)
333                 end
334         end
335         
336         return response.code, response, linesrc(true)..sock:readall(), sock
337 end
338
339 function cookie_parse(cookiestr)
340         local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)")
341         if not key then
342                 return nil
343         end
344
345         local cookie = {key = key, value = val, flags = {}}
346         for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do
347                 fkey = fkey:lower()
348                 if fkey == "expires" then
349                         fval = date.to_unix(fval:gsub("%-", " "))
350                 end
351                 cookie.flags[fkey] = fval
352         end
353
354         return cookie
355 end
356
357 function cookie_create(cookie)
358         local cookiedata = {cookie.key .. "=" .. cookie.value}
359
360         for k, v in pairs(cookie.flags) do
361                 if k == "expires" then
362                         v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ")
363                 end
364                 cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "")
365         end
366
367         return table.concat(cookiedata, "; ")
368 end