Re: [NSE] Web application detection

[Diman Todorov <diman.todorov () chello at>]

great! I will integrate it as soon as i have 2 hours i can spend on  
leisure :)

cheers,
Diman

On 10.07.2007, at 19:10, Sven Klemm wrote:

> Diman Todorov wrote:
> > this brings up an interesting point. Have you had a look at the
> > nselib? So far it has a very small dynamically loadable module
> > written in c for bitwise operations which serves as a concept
> > demonstration. Do you think you could easily make a loadable NSE
> > module out of your libxml2 wrapper? I could imagine that it would
> > only be compiled if users have libxml2 installed.
>
> The attached patch adds it as a new library. I left out the libxml2
> detection stuff as I don't know how to do that.
> The only thing that is currently wrapped is parsing html and xml
> documents and querying those parsed documents with xpath.
> I attached the sedusa.nse script again because I changed the case for
> xml table to lowercase.
>
> Cheers,
> sven
> Index: xml.c
> ===================================================================
> --- xml.c     (revision 0)
> +++ xml.c     (revision 0)
> @@ -0,0 +1,121 @@
> +
> +#include "xml.h"
> +
> +#include <libxml/HTMLparser.h>
> +#include <libxml/xpath.h>
> +
> +typedef struct XmlDocumentData {
> +  xmlDocPtr document;
> +} XmlDocumentData;
> +
> +// create a lua XmlDocument
> +int create_document( lua_State * L, xmlDocPtr doc ) {
> +  if ( doc ) {
> +    // parsing successful
> +    XmlDocumentData * doc_data;
> +    doc_data = (XmlDocumentData *) lua_newuserdata( L, sizeof 
> (XmlDocumentData));
> +    // set metatable for userdata
> +    luaL_getmetatable( L, "XmlDocument" );
> +    lua_setmetatable( L, -2 );
> +    doc_data->document = doc;
> +  } else  {
> +    // parsing failed
> +    luaL_error( L , "parsing document failed." );
> +  }
> +  return 1;
> +}
> +
> +// takes an html document as string and returns a XmlDocument
> +int xml_parse_html( lua_State * L ) {
> +  const char * doc_string = luaL_checkstring( L, 1 );
> +  lua_pop(L, 1);
> +  char * url = NULL;
> +  char * encoding = NULL;
> +
> +  int options = HTML_PARSE_RECOVER | HTML_PARSE_NOERROR |  
> HTML_PARSE_NOWARNING | HTML_PARSE_NONET;
> +  htmlDocPtr doc = htmlReadDoc( doc_string, url, encoding, options );
> +
> +  return create_document( L, doc );
> +}
> +
> +// takes an xml document as string and returns a XmlDocument
> +int xml_parse_xml( lua_State * L ) {
> +  const char * doc_string = luaL_checkstring( L, 1 );
> +  lua_pop(L, 1);
> +  char * url = NULL;
> +  char * encoding = NULL;
> +
> +  int options = XML_PARSE_RECOVER | XML_PARSE_NOERROR |  
> XML_PARSE_NOWARNING | XML_PARSE_NONET;
> +  xmlDocPtr doc = xmlReadDoc( doc_string, url, encoding, options );
> +
> +  return create_document( L, doc );
> +}
> +
> +
> +static const struct luaL_reg xml_methods[] = {
> +  { "parse_html", xml_parse_html },
> +  { "parse_xml", xml_parse_xml },
> +  { NULL, NULL }
> +};
> +
> +// takes an xpath expression and return the match(es) as string(s)
> +int xmldoc_xpath( lua_State * L ) {
> +  XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L,  
> 1, "XmlDocument");
> +  const char * xpath = luaL_checkstring( L, 2 );
> +  xmlXPathContextPtr context = xmlXPathNewContext( doc->document );
> +
> +  if ( !context ) luaL_error( L, "Error creating context." );
> +
> +  xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context);
> +  xmlXPathFreeContext(context);
> +  if ( !result ) luaL_error( L, "Error while evaluating XPath  
> expression." );
> +
> +  if( xmlXPathNodeSetIsEmpty( result->nodesetval ) ) {
> +    // empty resultset
> +    lua_pushnil( L );
> +    xmlXPathFreeNodeSetList( result );
> +    return 1;
> +  } else {
> +    int i;
> +    // we found something
> +    xmlNodeSetPtr nodeset = result->nodesetval;
> +    for ( i = 0; i < nodeset->nodeNr; i++) {
> +      char * tmp = xmlXPathCastNodeToString( nodeset->nodeTab[i] );
> +      lua_pushstring( L, tmp );
> +      free( tmp );
> +    }
> +    int matches = nodeset->nodeNr;
> +    xmlXPathFreeNodeSetList( result );
> +    return matches;
> +  }
> +}
> +
> +int xmldoc_free( lua_State * L ) {
> +  XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L,  
> 1, "XmlDocument");
> +  free( doc->document );
> +  return 0;
> +}
> +
> +// XmlDocument methods
> +static const struct luaL_reg xmldoc_methods[] = {
> +  { "xpath", xmldoc_xpath },
> +  { "__gc", xmldoc_free },
> +  { NULL, NULL }
> +};
> +
> +
> +// initializer function, called when library is required
> +int luaopen_xml( lua_State * L ) {
> +
> +  // create metatable
> +  luaL_newmetatable( L, "XmlDocument" );
> +  // metatable.__index = metatable
> +  lua_pushvalue( L, -1 );
> +  lua_setfield( L, -2, "__index" );
> +  // register methods
> +  luaL_register( L, NULL, xmldoc_methods );
> +
> +  luaL_register( L, "xml", xml_methods );
> +  return 1;
> +}
> +
> Index: Makefile.in
> ===================================================================
> --- Makefile.in       (revision 5165)
> +++ Makefile.in       (working copy)
> @@ -12,15 +12,20 @@
>  LIBTOOL= ./libtool
>  LTFLAGS = --tag=CC --silent
>
> -all: bit.so
> +all: bit.so xml.so
>
>  bit.so: bit.c @LIBTOOL_DEPS@
>       $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $(CFLAGS)  
> -c bit.c
>       $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -avoid-version -module - 
> rpath /usr/local/lib -o bit.la bit.lo
>       mv .libs/bit.so bit.so
>
> +xml.so: xml.c @LIBTOOL_DEPS@
> +     $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $ 
> (LIBXML_INCLUDE) $(CFLAGS) -c xml.c
> +     $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -lxml2 -avoid-version - 
> module -rpath /usr/local/lib -o xml.la xml.lo
> +     mv .libs/xml.so xml.so
> +
>  clean:
> -     rm -f bit.so *.la *.lo
> +     rm -f bit.so xml.so *.la *.lo
>       rm -rf .libs
>
>  distclean: clean
> Index: xml.h
> ===================================================================
> --- xml.h     (revision 0)
> +++ xml.h     (revision 0)
> @@ -0,0 +1,13 @@
> +
> +#ifndef XMLLIB
> +#define XMLLIB
> +
> +#define XMLLIBNAME "xml"
> +
> +#include "lauxlib.h"
> +#include "lua.h"
> +
> +LUALIB_API int luaopen_xml(lua_State *L);
> +
> +#endif
> +
> Index: nselib.h
> ===================================================================
> --- nselib.h  (revision 5165)
> +++ nselib.h  (working copy)
> @@ -2,6 +2,7 @@
>  #define NSE_LIB
>
>  #define NSE_BITLIBNAME "bit"
> +#define NSE_XMLLIBNAME "xml"
>
>
>  #endif
> id = "Sedusa"
> description = "Connects to an HTTP server and tries to guess the  
> running web application."
> author = "Sven Klemm <sven () c3d2 de>"
> license = "See nmaps COPYING for licence"
> categories = {"safe"}
>
> require "stdnse"
> require "shortport"
> require "url"
> require "xml"
>
> portrule = shortport.service({'http', 'https', 'ssl/http'})
>
> action = function(host, port)
>   local scheme, hostname, app, exps, index, xpath, u, doc
>   local target = {}
>
>   if port.service == 'https' or port.version.service_tunnel ==  
> 'ssl' then
>     target.scheme = "https"
>     if port.number ~= 443 then target.port = port.number end
>   else
>     target.scheme = "http"
>     if port.number ~= 80 then target.port = port.number end
>   end
>
>   if (host.name and not host.name == "") then
>     target.host = host.name
>   else
>     target.host = host.ip
>   end
>   target.path = "/"
>
>   u = url.build( target )
>   doc = Sedusa.get_document( u )
>
>   for app, exps in pairs( Sedusa.hints ) do
>     for index, xpath in pairs( exps ) do
>       if doc.xml:xpath( xpath ) then
>         if Sedusa.verify[app] then
>           local version = Sedusa.verify[app]( u )
>           if version then
>             return app .. " " .. version
>           else
>             return app .. " version not identified."
>           end
>         else
>           return "No verify function for " .. app .. " found."
>         end
>         break
>       end
>     end
>   end
>
> end
>
> Sedusa = {
>
>   hints = {},
>   verify = {},
>
>   document_cache = {},
>
>   Document = {
>     new = function( h, b )
>       local parsed = nil
>       if string.len( b ) > 0 then
>         parsed = xml.parse_html( b )
>       end
>       return { header = h, body = b, xml = parsed }
>     end
>   },
>
>   get_document = function( url )
>     local document
>
>     if not Sedusa.document_cache[ url ] then
>       document = Sedusa.http_get( url )
>       Sedusa.document_cache[ url ] = document
>     else
>       document = Sedusa.document_cache[ url ]
>     end
>     if document.header['Status'] == 301 or
>        document.header['Status'] == 302
>      then
>       document = Sedusa.get_document( document.header['Location'] )
>     end
>
>     return document
>   end,
>
>   http_get = function( u )
>     local protocol, port, request, query, socket
>     local parsed = url.parse( u )
>
>     if parsed.scheme == 'https' then
>       protocol = "ssl"
>       port = 443
>     else
>       protocol = "tcp"
>       port = 80
>     end
>
>     if ( parsed.port ) then port = parsed.port end
>
>     query = parsed.path
>     if ( parsed.query ) then
>       query = query .. '?' .. parsed.query
>     end
>
>     request = "GET "..query.." HTTP/1.1\r\nHost: "..parsed.host.."\r 
> \n\r\n"
>
>     socket = nmap.new_socket()
>     socket:connect( parsed.host, port, protocol )
>     socket:send(request)
>
>     local buffer = stdnse.make_buffer( socket, "\r?\n")
>
>     local status, line, key, value, head, header
>     head, header = {}, {}
>     -- head loop
>     while true do
>       status, line = buffer()
>       if (not status or line == "") then break end
>       table.insert(header,line)
>     end
>
>     -- build nicer table for header
>     for key, value in pairs( header ) do
>       if key == 1 then
>         local code = select( 3, string.find( value, "HTTP/%d\.%d (%d 
> +)") )
>         head['Status'] = tonumber(code)
>       else
>         key, value = select( 3, string.find( value, "(.+): (.*)" ) )
>         if key and value then
>           head[key] = value:gsub( '[\r\n]+', '' )
>         end
>       end
>     end
>
>     local body = {}
>     while true do
>       status, line = buffer()
>       if (not status) then break end
>       table.insert(body,line)
>     end
>
>     socket:close()
>     return Sedusa.Document.new( head, table.concat(body) )
>   end,
>
> }
>
>
>
> -- setup default detector for some web applications
> for key, app in pairs( {"b2evolution", "bBlog", "C3D2-Web",  
> "DokuWiki", "gitweb", "Midgard", "Pentabarf", "PhpWiki", "Plone",  
> "PostNuke", "TYPO3", "vBulletin"} ) do
>   Sedusa.hints[app] = { '//meta[@name="generator" and starts-with 
> (@content,"' .. app .. '")]' }
>   Sedusa.verify[app] = function( url )
>     local document = Sedusa.get_document( url )
>     -- look in generator meta tag
>     local generator = document.xml:xpath( '//meta 
> [@name="generator"]/@content' )
>     if generator and string.match( generator, app .. " (.*)" ) then
>       return string.match( generator, app .. " (.*)" )
>     end
>   end
>
> end
>
> Sedusa.verify["PhpWiki"] = function( url )
>   local document = Sedusa.get_document( url )
>
>   local generator = document.xml:xpath( '//meta 
> [@name="PHPWIKI_VERSION"]/@content' )
>   if generator then
>     return generator
>   end
>
> end
>
>
> Sedusa.hints["Drupal"] = {
>   '//div[@id="block-user-0" and h2[text()="User login"]]/div 
> [@class="content"]/form[@id="user-login-form"]/div[div[@class="form- 
> item"]/input[@type="text" and @class="form-text required"]]',
> }
>
> Sedusa.hints["Joomla!"] = {
>   '//meta[@name="Generator" and starts-with(@content,"Joomla!")]'
> }
>
> Sedusa.hints["MediaWiki"] = {
>   '//body[contains(@class, "mediawiki")]',
>   '//div[@id="footer"]/div[@id="f-poweredbyico"]/a[@href="http:// 
> www.mediawiki.org/"]/img'
> }
>
> Sedusa.verify["MediaWiki"] = function( u )
>   local document = Sedusa.get_document( u )
>   local link = document.xml:xpath( '//a[contains(@href,"index.php? 
> title=") and starts-with(@href, "/")]/@href' )
>   if link then
>     link = link:match( "^(.*/index.php[?]title=).*")
>   else
>     link = document.xml:xpath( '//script[@type="text/javascript"  
> and contains(text(), "var wgScriptPath = ")]' )
>     link = link:gsub( "[\r\n]", "" )
>     if link then
>       link = link:match( 'var wgScriptPath = "([^"]+)"') .. '/ 
> index.php?title='
>     end
>   end
>
>   -- get version from special:version
>   local version = Sedusa.get_document( url.absolute( u, link ..  
> "Special:Version" ) )
>   if version.xml:xpath('//div/ul/li[a[@href="http:// 
> www.mediawiki.org/" and text()="MediaWiki"]]') then
>     return string.match( version.xml:xpath('//div/ul/li[a 
> [@href="http://www.mediawiki.org/"; and text()="MediaWiki"]]'),  
> "MediaWiki: ([^\r\n]+)" )
>   end
>
>   -- get version from atom feed
>   local atom = Sedusa.get_document( url.absolute( u, link ..  
> "Special:Recentchanges&feed=atom" ) )
>   if atom.xml:xpath( '//generator[starts-with(text(),  
> "MediaWiki")]' ) then
>     return string.match( atom.xml:xpath( '//generator/text()' ),  
> "^MediaWiki (.*)$" )
>   end
>
> end
>
> Sedusa.hints["WordPress"] = {
>   '//meta[@name="generator" and starts-with(@content,"WordPress")]',
>   '//head/link[@rel="stylesheet" and @type="text/css" and contains 
> ( @href, "/wp-content/")]',
>   '//div[@id="content"]/div[@class="post" and starts-with(@id,  
> "post-") and div[@class="posttitle"] and div[@class="postmeta"] and  
> div[@class="postbody"] and div[@class="postfooter"]]',
> }
>
> Sedusa.verify["WordPress"] = function( u )
>   local document = Sedusa.get_document( u )
>
>   -- look in generator meta tag
>   local generator = document.xml:xpath( '//meta[@name="generator"]/ 
> @content' )
>   if generator and string.match( generator, "WordPress (.*)" ) then
>     return string.match( generator, "WordPress (.*)" )
>   end
>
>   -- look in atom feed
>   local atom = document.xml:xpath( '//link[@rel="alternate" and  
> @type="application/atom+xml"]/@href' )
>   local feed = Sedusa.get_document( atom )
>   if feed.xml:xpath( '//generator[text()="WordPress"]/@version' ) then
>     return feed.xml:xpath( '//generator[text()="WordPress"]/ 
> @version' )
>   end
>
>   return "Version not identified."
> end
>
> Sedusa.hints["Serendipity"] = {
>   '//meta[@name="Powered-By" and starts-with(@content,  
> "Serendipity")]'
> }
>
> Sedusa.verify["Serendipity"] = function( u )
>   local document = Sedusa.get_document( u )
>
>   local generator = document.xml:xpath( '//meta[@name="Powered-By"]/ 
> @content' )
>   if generator and string.match( generator, "Serendipity v[.] 
> (.*)" ) then
>     return string.match( generator, "Serendipity v[.](.*)" )
>   end
>
> end
>
> Sedusa.hints["Trac"] = {
>   '//div[@id="footer"]/a[@id="tracpowered" and @href="http:// 
> trac.edgewall.org/"]/img[@alt="Trac Powered"]',
> }
>
> Sedusa.verify["Trac"] = function( u )
>   local document = Sedusa.get_document( u )
>
>   local version = document.xml:xpath( '//div[@id="footer" and a 
> [@id="tracpowered" and @href="http://trac.edgewall.org/"]]/p 
> [@class="left"]/a/strong/text()' )
>   if version and version:match( "Trac (.*)" ) then
>     return version:match( "Trac (.*)" )
>   end
>
> end
>
>
> _______________________________________________
> Sent through the nmap-dev mailing list
> http://cgi.insecure.org/mailman/listinfo/nmap-dev
> Archived at http://SecLists.Org


_______________________________________________
Sent through the nmap-dev mailing list
http://cgi.insecure.org/mailman/listinfo/nmap-dev
Archived at http://SecLists.Org