Call for testing: http-form-brute major upgrade r35542

[Daniel Miller <bonsaiviking () gmail com>]

Gritty details are below, but here's the short version: http-form-brute.nse
was just overhauled to support complex logins, and should work great for
just about anything.

I especially need testing (and --script-trace output if it fails) for
Joomla, and new database entries for other CMS and web apps (Drupal,
Tomcat, others?)

Happy New Year!
Dan

On Thu, Dec 31, 2015 at 10:21 AM, <commit-mailer () nmap org> wrote:

> Author: dmiller
> Date: Thu Dec 31 08:21:03 2015
> New Revision: 35542
>
> Log:
> Major upgrade to http-form-brute
>
> Now can correctly handle complex token-based CSRF protections by grabbing
> all
> fields from the login form and using session cookies from the form request.
> These cookies are sent with subsequent redirect-following requests for
> better
> failure detection. On success, cookies are cleared and re-acquired.
>
> A database of known web apps' login forms is used for form detection,
> helpful
> for those whose username and password fields do not begin with "user" and
> "pass". Form mangling (to remove cookie checks in one instance) and custom
> onsuccess functions are also supported. Tested: django, wordpress,
> mediawiki.
>
> This *may* be a replacement (upon suitable testing) for
> http-wordpress-brute and
> http-joomla-brute.
>
>
> Modified:
>    nmap/scripts/http-form-brute.nse
>
> Modified: nmap/scripts/http-form-brute.nse
>
> ==============================================================================
> --- nmap/scripts/http-form-brute.nse    (original)
> +++ nmap/scripts/http-form-brute.nse    Thu Dec 31 08:21:03 2015
> @@ -104,6 +104,73 @@
>  local max_rcount = 2    -- how many times a form submission can be
> redirected
>  local form_debug = 1    -- debug level for printing form components
>
> +--- Database of known web apps for form detection
> +--
> +local known_apps = {
> +  joomla = {
> +    match = {
> +      action = "/administrator/index.php",
> +    },
> +    uservar = "username",
> +    passvar = "passwd",
> +    -- http-joomla-brute just checks for name="passwd" to indicate
> failure,
> +    -- so default onfailure should work. TODO: get onsuccess for this app.
> +  },
> +  django = {
> +    match = {
> +      action = "/login/",
> +      id = "login-form"
> +    },
> +    uservar = "username",
> +    passvar = "password",
> +    onsuccess = "Set%-Cookie:%s*sessionid=",
> +  },
> +  mediawiki = {
> +    match = {
> +      action = "action=submitlogin"
> +    },
> +    uservar = "wpName",
> +    passvar = "wpPassword",
> +    onsuccess = "Set%-Cookie:[^\n]*%wUserID=%d",
> +  },
> +  wordpress = {
> +    match = {
> +      action = "wp%-login%.php$",
> +    },
> +    uservar = "log",
> +    passvar = "pwd",
> +    onsuccess = "Location:[^\n]*/wp%-admin/",
> +    mangle = function(form)
> +      for i, f in ipairs(form.fields) do
> +        if f.name and f.name == "testcookie" then
> +          table.remove(form.fields, i)
> +          break
> +        end
> +      end
> +    end,
> +    sessioncookies = false,
> +  },
> +  websphere = {
> +    match = {
> +      action = "/ibm/console/j_security_check"
> +    },
> +    uservar = "j_username",
> +    passvar = "j_password",
> +    onfailure = function(response)
> +      local body = response.body
> +      local rpath = response.header.location
> +      return response.status < 300 and body and not (
> +        (rpath and rpath:match('logonError%.jsp'))
> +        or (
> +          body:match('Unable to login%.') or
> +          body:match('Login failed%.') or
> +          body:match('Invalid User ID or password')
> +          )
> +        )
> +    end,
> +    sessioncookies = false,
> +  },
> +}
>
>  ---
>  -- Test whether a given string (presumably a HTML fragment) contains
> @@ -122,6 +189,21 @@
>    return false
>  end
>
> +local function urlencode_form(fields, uservar, username, passvar,
> password)
> +  local parts = {}
> +  for _, field in ipairs(fields) do
> +    if field.name then
> +      local val = field.value or ""
> +      if field.name == uservar then
> +        val = username
> +      elseif field.name == passvar then
> +        val = password
> +      end
> +      parts[#parts+1] = url.escape(field.name) .. "=" .. url.escape(val)
> +    end
> +  end
> +  return table.concat(parts, "&")
> +end
>
>  ---
>  -- Detect a login form in a given HTML page
> @@ -132,14 +214,54 @@
>  -- @return Form object (see http.parse_form() for description)
>  --         or nil (if the operation failed)
>  -- @return Error string that describes any failure
> -local detect_form = function (host, port, path)
> -  local response = http.get(host, port, path)
> +-- @return cookies that were set by the request
> +local detect_form = function (host, port, path, hostname)
> +  local response = http.get(host, port, path, {
> +    bypass_cache = true,
> +    header = {Host = hostname}
> +  })
>    if not (response and response.body and response.status == 200) then
>      return nil, string.format("Unable to retrieve a login form from path
> %q", path)
>    end
>
>    for _, f in pairs(http.grab_forms(response.body)) do
>      local form = http.parse_form(f)
> +    for app, val in pairs(known_apps) do
> +      local match = true
> +      -- first check the 'match' table and be sure all values match
> +      for k, v in pairs(val.match) do
> +        match = match and string.match(form[k], v)
> +      end
> +      -- then check that uservar and passvar are in this form
> +      if match then
> +        -- how many field names must match?
> +        match = 2 - (val.uservar and 1 or 0) - (val.passvar and 1 or 0)
> +        for _, field in pairs(form.fields) do
> +          if field.name and
> +            field.name == val.uservar or field.name == val.passvar then
> +            -- found one, decrement
> +            match = match - 1
> +          end
> +          -- Have we found them all?
> +          if match <= 0 then break end
> +        end
> +        if match <= 0 then
> +          stdnse.debug1("Detected %s login form.", app)
> +          -- copy uservar, passvar, etc. from the fingerprint
> +          for k, v in pairs(val) do
> +            form[k] = v
> +          end
> +          -- apply any special mangling
> +          if val.mangle then
> +            val.mangle(form)
> +          end
> +          return form, nil, response.cookies
> +        end
> +        -- failed to match uservar and passvar
> +      end
> +      -- failed to match form
> +    end
> +    -- No known apps match, try generic matching
>      local unfld, pnfld, ptfld
>      for _, fld in pairs(form.fields) do
>        if fld.name then
> @@ -159,13 +281,57 @@
>        form.method = form.method or "GET"
>        form.uservar = (unfld or {}).name
>        form.passvar = (ptfld or pnfld).name
> -      return form
> +      return form, nil, response.cookies
>      end
>    end
>
>    return nil, string.format("Unable to detect a login form at path %q",
> path)
>  end
>
> +-- Recursively copy a table.
> +-- Only recurs when a value is a table, other values are copied by
> assignment.
> +local function tcopy (t)
> +  local tc = {};
> +  for k,v in pairs(t) do
> +    if type(v) == "table" then
> +      tc[k] = tcopy(v);
> +    else
> +      tc[k] = v;
> +    end
> +  end
> +  return tc;
> +end
> +
> +-- TODO: expire cookies
> +local function update_cookies (old, new)
> +  for i, c in ipairs(new) do
> +    local add = true
> +    for j, oc in ipairs(old) do
> +      if oc.name == c.name then
> +        old[j] = c
> +        add = false
> +        break
> +      end
> +    end
> +    if add then
> +      table.insert(old, c)
> +    end
> +  end
> +end
> +
> +-- make sure this path is ok as a form action.
> +-- Also make sure we stay on the same host.
> +local function path_ok (path, hostname, port)
> +  local pparts = url.parse(path)
> +  if pparts.authority then
> +    if pparts.userinfo
> +      or ( pparts.host ~= hostname )
> +      or ( pparts.port and tonumber(pparts.port) ~= port.number ) then
> +      return false
> +    end
> +  end
> +  return true
> +end
>
>  Driver = {
>
> @@ -173,9 +339,26 @@
>      local o = {}
>      setmetatable(o, self)
>      self.__index = self
> -    o.host = nmap.registry.args['http-form-brute.hostname'] or host
> +    if not options.http_options then
> +      -- we need to supply the no_cache directive, or else the http
> library
> +      -- incorrectly tells us that the authentication was successful
> +      options.http_options = {
> +        no_cache = true,
> +        bypass_cache = true,
> +        redirect_ok = false,
> +        cookies = options.cookies,
> +        header = {
> +          -- nil just means not set, so default http.lua behavior
> +          Host = options.hostname,
> +          ["Content-Type"] = "application/x-www-form-urlencoded"
> +        }
> +      }
> +    end
> +    o.host = host
>      o.port = port
>      o.options = options
> +    -- each thread may store its params table here under its thread id
> +    options.threads = options.threads or {}
>      return o
>    end,
>
> @@ -187,26 +370,49 @@
>    end,
>
>    submit_form = function (self, username, password)
> -    -- we need to supply the no_cache directive, or else the http library
> -    -- incorrectly tells us that the authentication was successful
>      local path = self.options.path
> -    local opts = {no_cache = true, redirect_ok = false}
> -    local params = {[self.options.passvar] = password}
> -    if self.options.uservar then params[self.options.uservar] = username
> end
> +    local tid = stdnse.gettid()
> +    local thread = self.options.threads[tid]
> +    if not thread then
> +      thread = {
> +        -- copy of form fields so we don't clobber another thread's
> passvar
> +        params = tcopy(self.options.formfields),
> +        -- copy of options so we don't clobber another thread's cookies
> +        opts = tcopy(self.options.http_options),
> +      }
> +      self.options.threads[tid] = thread
> +    end
> +    if self.options.sessioncookies and not (thread.opts.cookies and
> next(thread.opts.cookies)) then
> +      -- grab new session cookies
> +      local form, errmsg, cookies = detect_form(self.host, self.port,
> path, self.options.hostname)
> +      if not form then
> +        stdnse.debug1("Failed to get new session cookies: %s", errmsg)
> +      else
> +        thread.opts.cookies = cookies
> +        thread.params = form.fields
> +      end
> +    end
> +    local params = thread.params
> +    local opts = thread.opts
>      local response
>      if self.options.method == "POST" then
> -      response = http.post(self.host, self.port, path, opts, nil, params)
> +      response = http.post(self.host, self.port, path, opts, nil,
> +      urlencode_form(params, self.options.uservar, username,
> self.options.passvar, password))
>      else
>        local uri = path
> -                  .. (path:find("?", 1, true) and "&" or "?")
> -                  .. url.build_query(params)
> +        .. (path:find("?", 1, true) and "&" or "?")
> +        .. urlencode_form(params, self.options.uservar, username,
> self.options.passvar, password)
>        response = http.get(self.host, self.port, uri, opts)
>      end
>      local rcount = 0
>      while response do
>        if self.options.is_success and self.options.is_success(response)
> then
> +        -- "log out"
> +        opts.cookies = nil
>          return response, true
>        end
> +      -- set cookies
> +      update_cookies(opts.cookies, response.cookies)
>        if self.options.is_failure and self.options.is_failure(response)
> then
>          return response, false
>        end
> @@ -217,7 +423,19 @@
>        end
>        rcount = rcount + 1
>        path = url.absolute(path, rpath)
> -      response = http.get(self.host, self.port, path, opts)
> +      if path_ok(path, self.options.hostname, self.port) then
> +        -- clean up the url (cookie check fails if path contains hostname)
> +        -- this strips off the smallest prefix followed by a non-doubled /
> +        path = path:gsub("^.-%f[/](/%f[^/])","%1")
> +        response = http.get(self.host, self.port, path, opts)
> +      else
> +        -- being redirected off-host. Stop and assume failure.
> +        response = nil
> +      end
> +    end
> +    if response and self.options.is_failure then
> +      -- "log out" to avoid dumb login attempt limits
> +      opts.cookies = nil
>      end
>      -- Neither is_success nor is-failure condition applied. The login is
> deemed
>      -- a success if the script is looking for a failure (which did not
> occur).
> @@ -255,9 +473,18 @@
>    local passvar = stdnse.get_script_args('http-form-brute.passvar')
>    local onsuccess = stdnse.get_script_args('http-form-brute.onsuccess')
>    local onfailure = stdnse.get_script_args('http-form-brute.onfailure')
> +  local hostname = stdnse.get_script_args('http-form-brute.hostname') or
> stdnse.get_hostname(host)
> +  local sessioncookies =
> stdnse.get_script_args('http-form-brute.sessioncookies')
> +  if not sessioncookies then
> +    sessioncookies = true
> +  elseif sessioncookies == "false" then
> +    sessioncookies = false
> +  end
>
> +  local formfields = {}
> +  local cookies = {}
>    if not passvar then
> -    local form, errmsg = detect_form(host, port, path)
> +    local form, errmsg, dcookies = detect_form(host, port, path, hostname)
>      if not form then
>        return stdnse.format_output(false, errmsg)
>      end
> @@ -265,11 +492,15 @@
>      method = method or form.method
>      uservar = uservar or form.uservar
>      passvar = passvar or form.passvar
> +    onsuccess = onsuccess or form.onsuccess
> +    onfailure = onfailure or form.onfailure
> +    formfields = form.fields or formfields
> +    cookies = dcookies or cookies
> +    sessioncookies = form.sessioncookies == nil and sessioncookies or
> form.sessioncookies
>    end
>
>    -- path should not change the origin
> -  local pparts = url.parse(path)
> -  if pparts.scheme or pparts.authority then
> +  if not path_ok(path, hostname, port) then
>      return stdnse.format_output(false, string.format("Unusable form
> action %q", path))
>    end
>    stdnse.debug(form_debug, "Form submission path: " .. path)
> @@ -293,12 +524,18 @@
>    end
>
>    -- convert onsuccess and onfailure to functions
> -  local is_success = onsuccess and function (response)
> -                                     return
> http.response_contains(response, onsuccess, true)
> -                                   end
> -  local is_failure = onfailure and function (response)
> -                                     return
> http.response_contains(response, onfailure, true)
> -                                   end
> +  local is_success = onsuccess and (
> +    type(onsuccess) == "function" and onsuccess
> +    or function (response)
> +      return http.response_contains(response, onsuccess, true)
> +    end
> +    )
> +  local is_failure = onfailure and (
> +    type(onfailure) == "function" and onfailure
> +    or function (response)
> +      return http.response_contains(response, onfailure, true)
> +    end
> +    )
>    -- the fallback test is to look for passvar field in the response
>    if not (is_success or is_failure) then
>      is_failure = function (response)
> @@ -312,7 +549,11 @@
>                    uservar = uservar,
>                    passvar = passvar,
>                    is_success = is_success,
> -                  is_failure = is_failure
> +                  is_failure = is_failure,
> +                  hostname = hostname,
> +                  formfields = formfields,
> +                  cookies = cookies,
> +                  sessioncookies = sessioncookies,
>                    }
>
>    -- validate that the form submission behaves as expected
> @@ -330,6 +571,7 @@
>    local engine = brute.Engine:new(Driver, host, port, options)
>    -- there's a bug in http.lua that does not allow it to be called by
>    -- multiple threads
> +  -- TODO: is this even true any more? We should fix it if not.
>    engine:setMaxThreads(1)
>    engine.options.script_name = SCRIPT_NAME
>    engine.options:setOption("passonly", not uservar)
>
> _______________________________________________
> Sent through the svn mailing list
> https://nmap.org/mailman/listinfo/svn
_______________________________________________
Sent through the dev mailing list
https://nmap.org/mailman/listinfo/dev
Archived at http://seclists.org/nmap-dev/