[NSE] URL encoding for url.parse_query(), url.build_query()

[nnposter () users sourceforge net]

The patch below adds automatic URL encoding/decoding as part of query
string parsing or construction. It seems logical to implemented it this
way since http.post() also does the same transparent encoding for the
POST body name/value pairs..

There are only 5 scripts that use these functions from nselib/url.lua.
The changes are summarized below.


url.lua:

* Added the automatic URL encoding/decoding to url.parse_query() and
url.build_query()

* Added parsing support for empty parameters, such as ...&foo&...
instead of ...&foo=&...


http-form-fuzzer.nse:

* Replaced local generate_get_string() with effectively identical
url.build_query().


http-open-redirect.nse:

* Inherently remediated a source of false-negatives due to comparing
(presumably URL encoded) query values to HTTP Location headers (which
are not encoded).

* Improved URL comparison accuracy by replacing pattern matching with
string comparison. (Pattern magic characters in query values were
getting interpreted, as opposed to being taken literally.)

* Abstracted out the redirection test URL as a script constant.


http-rfi-spider.nse:

* Inherently remediated a query string formation with unencoded
reserved characters, such as "=" and "?".

* Replaced local generate_get_string() with effectively identical
url.build_query().


http-sql-injection.nse:

* Just replaced URL-encoded value of an injected payload with its
native value.


http-unsafe-output-escaping.nse:

* Just replaced URL-encoded value of an injected payload with its
native value.


Please let me know if you see any technical issues with the patch or
you philosophically disagree with this integration.



Cheers,
nnposter



Patch against r33655 follows:


--- nselib/url.lua.orig 2014-09-07 16:12:03.726753500 -0600
+++ nselib/url.lua      2014-09-07 14:33:19.293003600 -0600
@@ -319,7 +319,8 @@
 -- This function takes a <code><query></code> of the form
 -- <code>"name1=value1&name2=value2"</code>
 -- and returns a table containing the name-value pairs, with the name as the key
--- and the value as its associated value.
+-- and the value as its associated value. Both the name and the value are
+-- subject to URL decoding.
 -- @param query Query string.
 -- @return A table of name-value pairs following the pattern
 -- <code>table["name"]</code> = <code>value</code>.
@@ -333,9 +334,11 @@
   query = string.gsub(query, "&gt;", ">")
 
   local function ginsert(qstr)
-    local first, last = string.find(qstr, "=")
-    if first then
-      parsed[string.sub(qstr, 0, first-1)] = string.sub(qstr, first+1)
+    local pos = qstr:find("=", 1, true)
+    if pos then
+      parsed[unescape(qstr:sub(1, pos - 1))] = unescape(qstr:sub(pos + 1))
+    else
+      parsed[unescape(qstr)] = ""
     end
   end
 
@@ -355,7 +358,8 @@
 ---
 -- Builds a query string from a table.
 --
--- This is the inverse of <code>parse_query</code>.
+-- This is the inverse of <code>parse_query</code>. Both the parameter name
+-- and value are subject to URL encoding.
 -- @param query A dictionary table where <code>table['name']</code> =
 -- <code>value</code>.
 -- @return A query string (like <code>"name=value2&name=value2"</code>).
@@ -364,7 +368,7 @@
   local qstr = ""
 
   for i,v in pairs(query) do
-    qstr = qstr .. i .. '=' .. v .. '&'
+    qstr = qstr .. escape(i) .. '=' .. escape(v) .. '&'
   end
   return string.sub(qstr, 0, #qstr-1)
 end


--- scripts/http-form-fuzzer.nse.orig   2014-09-07 16:51:19.826835900 -0600
+++ scripts/http-form-fuzzer.nse        2014-09-07 16:51:00.365890000 -0600
@@ -97,14 +97,6 @@
   return postdata
 end
 
-local function generate_get_string(data)
-  local get_str = {"?"}
-  for name,value in pairs(data) do
-    get_str[#get_str+1]=url.escape(name).."="..url.escape(value).."&"
-  end
-  return table.concat(get_str)
-end
-
 -- generate a charset of characters with ascii codes from 33 to 126
 -- you can use http://www.asciitable.com/ to see which characters those actually are
 local charset = generate_charset(33,126)
@@ -156,7 +148,7 @@
   if form["method"]=="post" then
     sending_function = function(data) return http.post(host, port, form_submission_path, nil, nil, data) end
   else
-    sending_function = function(data) return http.get(host, port, form_submission_path..generate_get_string(data), 
{no_cache=true, bypass_cache=true}) end
+    sending_function = function(data) return http.get(host, port, form_submission_path.."?"..url.build_query(data), 
{no_cache=true, bypass_cache=true}) end
   end
 
   for _,field in ipairs(form["fields"]) do


--- scripts/http-open-redirect.nse.orig 2014-09-07 16:13:47.871753500 -0600
+++ scripts/http-open-redirect.nse      2014-09-07 14:33:16.350709400 -0600
@@ -43,6 +43,8 @@
 
 portrule = shortport.http
 
+local redirect_canary = "http://scanme.nmap.org/";
+
 local function dbg(str,...)
   stdnse.debug2(str, ...)
 end
@@ -74,10 +76,9 @@
   local q = url.parse_query(query);
   -- Check the values (and keys) and see if they are reflected in the location header
   for k,v in pairs(q) do
-    local s,f = string.find(destination, v)
-    if s == 1 then
+    if destination:sub(1, #v) == v then
       -- Build a new URL
-      q[k] = "http%3A%2f%2fscanme.nmap.org%2f";
+      q[k] = redirect_canary;
       return url.build_query(q)
     end
   end
@@ -123,7 +124,7 @@
           dbg("Checking potential open redirect: %s:%s%s", host,port,url);
           local testResponse = http.get(host, port, url);
           --dbgt(testResponse)
-          if isRedirect(testResponse.status) and testResponse.header.location == "http://scanme.nmap.org/"; then
+          if isRedirect(testResponse.status) and testResponse.header.location == redirect_canary then
             table.insert(results, ("%s://%s:%s%s"):format(parsed.scheme, host, port,url))
           end
         end


--- scripts/http-rfi-spider.nse.orig    2014-09-07 16:13:57.119753500 -0600
+++ scripts/http-rfi-spider.nse 2014-09-07 15:29:26.464903100 -0600
@@ -69,14 +69,6 @@
   return postdata
 end
 
-local function generate_get_string(data)
-  local get_str = {"?"}
-  for name,value in pairs(data) do
-    get_str[#get_str+1]=url.escape(name).."="..url.escape(value).."&"
-  end
-  return table.concat(get_str)
-end
-
 -- checks each field of a form to see if it's vulnerable to rfi
 local function check_form(form, host, port, path)
   local vulnerable_fields = {}
@@ -96,7 +88,7 @@
   if form["method"]=="post" then
     sending_function = function(data) return http.post(host, port, form_submission_path, nil, nil, data) end
   else
-    sending_function = function(data) return http.get(host, port, form_submission_path..generate_get_string(data), 
nil) end
+    sending_function = function(data) return http.get(host, port, form_submission_path.."?"..url.build_query(data), 
nil) end
   end
 
   for _,field in ipairs(form["fields"]) do
@@ -208,7 +200,7 @@
       end --for
     end --if
 
-    -- now try inclusion by parameters
+    -- now try inclusion by query parameters
     local injectable = {}
     -- search for injectable links (as in sql-injection.nse)
     if r.response.status and r.response.body then


--- scripts/http-sql-injection.nse.orig 2014-09-07 16:14:08.361753500 -0600
+++ scripts/http-sql-injection.nse      2014-09-07 15:39:31.540404600 -0600
@@ -102,7 +102,7 @@
 
       for k, v in pairs(qtab) do
         old_qtab = qtab[k];
-        qtab[k] = qtab[k] ..  "'%20OR%20sqlspider"
+        qtab[k] = qtab[k] ..  "' OR sqlspider"
 
         utab.query = url.build_query(qtab)
         urlstr = url.build(utab)


--- scripts/http-unsafe-output-escaping.nse.orig        2014-09-07 16:14:18.310753500 -0600
+++ scripts/http-unsafe-output-escaping.nse     2014-09-07 15:41:01.326382300 -0600
@@ -78,7 +78,7 @@
 end
 
 local function addPayload(v)
-  return v.."ghz%3Ehzx%22zxc%27xcv"
+  return v.."ghz>hzx\"zxc'xcv"
 end
 
 local function createMinedLinks(reflected_values, all_values)
_______________________________________________
Sent through the dev mailing list
http://nmap.org/mailman/listinfo/dev
Archived at http://seclists.org/nmap-dev/