Nmap Development mailing list archives

Previous By Date Next
Previous By Thread Next

Re: [NSE] NSE HTTP library

From: Sven Klemm <sven () c3d2 de>
Date: Thu, 31 Jan 2008 16:43:19 +0100
Fyodor wrote:

My only lingering concern is the issue of multiple header fields of
the same name.  Sven had a pretty convincing rationale for using the
comma separated list as described in the HTTP 1.1 RFC at
http://tools.ietf.org/html/rfc2616#section-4.2 .  While parsing the
WWW-Authenticate fields returned by IIS may be harder using the
comma-separated list approach, I worry that putting in our hack just
for that may lead to other problems if/when we find behavior which
depends on the CSL handling.

Maybe we can just make the HTTP Auth script a bit smarter with its
parsing--even if that means special casing some auth type keywords and
the like.

Or am I missing a good reason for dumping the CSL behavior?


You are right. As some servers might send the header comma-separated
the script would have to handle both situations to work under all
circumstances. I changed the code to separate them by comma again.
I've also adjusted the HTTP Auth script to handle this.

I adjusted the header handling so that multiline headers are now
properly handled. Earlier versions ignored those lines.

You can now also set arbitrary headers for the http.get() function
e.g.: http.get(host, port, '/', {header={Authorization="Basic
YWRtaW46C"}})

This way you can also overwrite the User-Agent or the Host used in the
GET request.

Cheers,
Sven

-- 
Sven Klemm
http://cthulhu.c3d2.de/~sven/


Index: nselib/http.lua
===================================================================
--- nselib/http.lua     (revision 0)
+++ nselib/http.lua     (revision 0)
@@ -0,0 +1,149 @@
+-- See nmaps COPYING for licence
+module(...,package.seeall)
+
+require 'stdnse'
+require 'url'
+
+--
+-- http.get( host, port, path, options )
+-- http.request( host, port, request, options )
+-- http.get_url( url, options )
+--
+-- host may either be a string or table
+-- port may either be a number or a table
+--
+-- the format of the return value is a table with the following structure:
+-- {status = 200, header = {}, body ="<html>...</html>"}
+-- the header table has an entry for each received header with the header name being the key
+-- the table also has an entry named "status" which contains the http status code of the request
+-- in case of an error status is nil
+
+
+-- fetch relative URL with get request
+get = function( host, port, path, options )
+  options = options or {}
+  local presets = {Host=host,Connection="close",['User-Agent']="Nmap NSE"}
+  if type(host) == 'table' then
+    presets['Host'] = ( host.name ~= '' and host.name ) or host.ip
+  end
+
+  local header = options.header or {}
+  for key,value in pairs(presets) do
+    header[key] = header[key] or value
+  end
+
+  local data = "GET "..path.." HTTP/1.1\r\n"
+  for key,value in pairs(header) do
+    data = data .. key .. ": " .. value .. "\r\n"
+  end
+  data = data .. "\r\n"
+
+  return request( host, port, data, options )
+end
+
+-- fetch URL with get request
+get_url = function( u, options )
+  local parsed = url.parse( u )
+  local port = {}
+
+  port.service = parsed.scheme
+  port.number = parsed.port
+
+  if not port.number then
+    if parsed.scheme == 'https' then
+      port.number = 443
+    else
+      port.number = 80
+    end
+  end
+
+  local path = parsed.path or "/"
+  if parsed.query then
+    path = path .. "?" .. parsed.query
+  end
+
+  return get( parsed.host, port, path )
+end
+
+-- send http request and return the result as table
+-- host may be a table or the hostname
+-- port may be a table or the portnumber
+request = function( host, port, data, options )
+  options = options or {}
+
+  if type(host) == 'table' then
+    host = ( host.name ~= '' and host.name ) or host.ip
+  end
+
+  local protocol = 'tcp'
+  if type(port) == 'table' then
+    if nmap.have_ssl() and ( port.service == 'https' or ( port.version and port.version.service_tunnel == 'ssl' ) ) 
then
+      protocol = 'ssl'
+    end
+    port = port.number
+  end
+
+  local result = {status=nil,header={},body=""}
+  local socket = nmap.new_socket()
+  if options.timeout then
+    socket:set_timeout( options.timeout )
+  end
+  if not socket:connect( host, port, protocol ) then
+    return result
+  end
+  if not socket:send( data ) then
+    return result
+  end
+
+  local buffer = stdnse.make_buffer( socket, "\r?\n" )
+
+  local status, line, _
+  local header, body = {}, {}
+
+  -- header loop
+  while true do
+    status, line = buffer()
+    if (not status or line == "") then break end
+    table.insert(header,line)
+  end
+
+  -- build nicer table for header
+  local last_header, match
+  for number, line in pairs( header ) do
+    if number == 1 then
+      local code
+      _, _, code = string.find( line, "HTTP/%d\.%d (%d+)")
+      result.status = tonumber(code)
+    else
+      match, _, key, value = string.find( line, "(.+): (.*)" )
+      if match and key and value then
+        key = key:lower()
+        if result.header[key] then
+          result.header[key] = result.header[key] .. ',' .. value
+        else
+          result.header[key] = value
+        end
+        last_header = key
+      else
+        match, _, value = string.find( line, " +(.*)" )
+        if match and value then
+          result.header[last_header] = result.header[last_header] .. ',' .. value
+        end
+      end
+    end
+  end
+
+  -- body loop
+  while true do
+    status, line = buffer()
+    if (not status) then break end
+    table.insert(body,line)
+  end
+
+  socket:close()
+  result.body = table.concat( body, "\n" )
+
+  return result
+
+end
+
Index: scripts/HTTPAuth.nse
===================================================================
--- scripts/HTTPAuth.nse        (revision 6773)
+++ scripts/HTTPAuth.nse        (working copy)
@@ -14,121 +14,52 @@
 categories = {"intrusive"}
 
 require "shortport"
+require "http"
 
-portrule = shortport.port_or_service({80, 8080}, "http")
+portrule = shortport.port_or_service({80, 443, 8080}, {"http","https"})
 
 action = function(host, port)
-       local socket
-       local catch = function()
-               socket:close()
-       end
+  local realm,scheme,result
+  local basic = false
 
-       local try = nmap.new_try(catch)
+  local answer = http.get( host, port, "/" )
 
-       local get_http_headers = function(dst, dst_port, query_string)
-               socket = nmap.new_socket()
+  --- check for 401 response code
+  if answer.status == 401 then
+    result = "HTTP Service requires authentication\n"
 
-               try(socket:connect(dst, dst_port))
-               try(socket:send(query_string))
+    -- split www-authenticate header
+    local auth_headers = {}
+    local pcre = pcre.new('\\w+( (\\w+=("[^"]+"|\\w+), *)*(\\w+=("[^"]+"|\\w+)))?',0,"C")
+    local match = function( match ) table.insert(auth_headers, match) end
+    pcre:gmatch( answer.header['www-authenticate'], match )
 
-               local response = ""
-               local lines
-               local status
+    for _, value in pairs( auth_headers ) do
+      result = result .. "  Auth type: "
+      scheme, realm = string.match(value, "(%a+).-[Rr]ealm=\"(.-)\"")
+      if scheme == "Basic" then
+        basic = true
+      end
+      if realm ~= nil then
+        result = result .. scheme .. ", realm = " .. realm .. "\n"
+      else
+        result = result .. string.match(value, "(%a+)") .. "\n"
+      end
+    end
+  end
 
-               while true do
-                       status, lines = socket:receive_lines(1)
+  if basic then
+    answer = http.get(host, port, '/', {header={Authorization="Basic YWRtaW46C"}})
+    if answer.status ~= 401 and answer.status ~= 403 then
+      result = result .. "  HTTP server may accept user=\"admin\" with blank password for Basic authentication\n"
+    end
 
-                       if not status then
-                               break
-                       end
+    answer = http.get(host, port, '/', {header={Authorization="Basic YWRtaW46YWRtaW4"}})
+    if answer.status ~= 401 and answer.status ~= 403 then
+      result = result .. "  HTTP server may accept user=\"admin\" with password=\"admin\" for Basic authentication\n"
+    end
+  end
 
-                       response = response .. lines
-               end
-
-               try(socket:close())
-
-               local tags = {"(.-)<![Dd][Oo][Cc][Tt][Yy][Pp][Ee]", "(.-)<[Hh][Tt][Mm][Ll]", "(.-)<[Hh][Ee][Aa][Dd]", 
"(.-)<[Bb][Oo][Dd][Yy]"}
-               local hdrs
-
-               for I = 1, #tags do
-                       hdrs = string.match(response, tags[I])
-                       if hdrs ~= nil and hdrs ~= response and hdrs ~= "" then
-                               return hdrs
-                       end
-               end
-
-               return response
-       end
-
-       local auth
-       local value
-       local realm
-       local scheme
-       local result
-       local basic = false
-
-       local query = "GET / HTTP/1.1\r\n"
-       query = query .. "Accept: */*\r\n"
-       query = query .. "Accept-Language: en\r\n"
-       query = query .. "User-Agent: Nmap NSE\r\n"
-       query = query .. "Connection: close\r\n"
-       query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-       local headers = get_http_headers(host.ip, port.number, query)
-
-       --- check for 401 response code
-       auth = string.match(headers, "HTTP/1.- 401")
-       if auth ~= nil then
-               result = "HTTP Service requires authentication\n"
-               -- loop through any WWW-Authenticate: headers to determine valid authentication schemes
-               for value in string.gmatch(headers, "[Aa]uthenticate:(.-)\n") do
-                       result = result .. "  Auth type: "
-                       scheme, realm = string.match(value, "(%a+).-[Rr]ealm=\"(.-)\"")
-                       if scheme == "Basic" then
-                               basic = true
-                       end
-                       if realm ~= nil then
-                               result = result .. scheme .. ", realm = " .. realm .. "\n"
-                       else
-                               result = result .. string.match(value, "(%a+)") .. "\n"
-                       end
-               end
-       end
-
-       if basic then
-               query = "GET / HTTP/1.1\r\n"
-               query = query .. "Authorization: Basic YWRtaW46C\r\n"
-               query = query .. "Accept: */*\r\n"
-               query = query .. "Accept-Language: en\r\n"
-               query = query .. "User-Agent: Nmap NSE\r\n"
-               query = query .. "Connection: close\r\n"
-               query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-               auth = ""
-               headers = get_http_headers(host.ip, port.number, query)
-
-               auth = string.match(headers, "HTTP/1.- 40[013]")
-               if auth == nil then
-                       result = result .. "  HTTP server may accept user=\"admin\" with blank password for Basic 
authentication\n"
-               end
-
-               query = "GET / HTTP/1.1\r\n"
-               query = query .. "Authorization: Basic YWRtaW46YWRtaW4\r\n"
-               query = query .. "Accept: */*\r\n"
-               query = query .. "Accept-Language: en\r\n"
-               query = query .. "User-Agent: Nmap NSE\r\n"
-               query = query .. "Connection: close\r\n"
-               query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-               auth = ""
-               headers = get_http_headers(host.ip, port.number, query)
-
-               auth = string.match(headers, "HTTP/1.- 40[013]")
-               if auth == nil then
-                       result = result .. "  HTTP server may accept user=\"admin\" with password=\"admin\" for Basic 
authentication\n"
-               end
-       end
-
-       return result
+  return result
 end
 
Index: scripts/showHTMLTitle.nse
===================================================================
--- scripts/showHTMLTitle.nse   (revision 6773)
+++ scripts/showHTMLTitle.nse   (working copy)
@@ -11,7 +11,7 @@
 
 categories = {"demo", "safe"}
 
-require "stdnse"
+require 'http'
 
 portrule = function(host, port)
        if not (port.service == 'http' or port.service == 'https') then
@@ -26,41 +26,20 @@
 end
 
 action = function(host, port)
-       local socket, request, result, status, s, title, protocol
+       local data, result, title, protocol
 
-       socket = nmap.new_socket()
+       data = http.get( host, port, '/' )
+       result = data.body
 
-       if port.service == 'https' or port.version.service_tunnel == 'ssl' then
-               protocol = "ssl"
-       else
-               protocol = "tcp"
-       end
-
-       socket:connect(host.ip, port.number, protocol )
-       request = "GET / HTTP/1.0\r\n\r\n"
-       socket:send(request)
-
-       result = ""
-       while true do
-               status, s = socket:receive_lines(1)
-               if not status then
-                       break
-               end
-
-               result = result .. s
-       end
-       socket:close()
-       
        -- watch out, this doesn't really work for all html tags
-       -- also string.lower consumes the /
-       result = string.gsub(result, "</?(%a+)>", function(c) return "<" .. string.lower(c) .. ">" end)
-       
-       title = string.match(result, "<title>(.+)<title>")
+       result = string.gsub(result, "<(/?%a+)>", function(c) return "<" .. string.lower(c) .. ">" end)
 
+       title = string.match(result, "<title>(.+)</title>")
+
        if title ~= nil then
                result = string.gsub(title , "[\n\r\t]", "")
                if string.len(title) > 50 then
-                       stdnse.print_debug("showHTMLTitle.nse: Title got truncated!");  
+                       stdnse.print_debug("showHTMLTitle.nse: Title got truncated!");
                        result = string.sub(result, 1, 62) .. "..."
                end
        else
Index: scripts/robots.nse
===================================================================
--- scripts/robots.nse  (revision 6773)
+++ scripts/robots.nse  (working copy)
@@ -1,6 +1,7 @@
 require('shortport')
 require('strbuf')
 require('listop')
+require('http')
 
 id = "robots.txt"
 author = "Eddie Bell <ejlbell () gmail com>"
@@ -9,7 +10,7 @@
 categories = {"safe"}
 runlevel = 1.0
 
-portrule = shortport.port_or_service(80, "http")
+portrule = shortport.port_or_service({80,443}, {"http","https"})
 local last_len = 0
 
 -- split the output in 40 character lines 
@@ -32,40 +33,15 @@
 end
 
 action = function(host, port)
-       local soc, lines, status
+       local answer = http.get( host, port, "/robots.txt" )
 
-       local catch = function() soc:close() end
-       local try = nmap.new_try(catch)
-
-       -- connect to webserver 
-       soc = nmap.new_socket()
-       soc:set_timeout(4000)
-       try(soc:connect(host.ip, port.number))
-
-       local query = strbuf.new()
-       query = query .. "GET /robots.txt HTTP/1.1"
-       query = query .. "Accept: */*"
-       query = query .. "Accept-Language: en"
-       query = query .. "User-Agent: Nmap NSE"
-       query = query .. "Host: " .. host.ip .. ":" .. port.number
-       query = query .. "Connection: close"
-       query = query .. '\r\n\r\n';
-       try(soc:send(strbuf.dump(query, '\r\n')))
-
-       local response = strbuf.new()
-       while true do
-               status, lines = soc:receive_lines(1)
-               if not status then break end
-               response = response .. lines
-       end
-
-       if not string.find(strbuf.dump(response), "HTTP/1.1 200 OK") then
+       if answer.status ~= 200 then
                return nil
        end
 
        -- parse all disallowed entries and remove comments
        local output = strbuf.new()
-       for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+       for w in string.gmatch(answer.body, "Disallow:%s*([^\n]*)\n") do
                        w = w:gsub("%s*#.*", "")
                        buildOutput(output, w)
        end

Attachment: signature.asc
Description: OpenPGP digital signature


_______________________________________________
Sent through the nmap-dev mailing list
http://cgi.insecure.org/mailman/listinfo/nmap-dev
Archived at http://SecLists.Org
Previous By Date Next
Previous By Thread Next

Current thread: