Nmap Development mailing list archives
Re: [NSE] Web application detection
From: Diman Todorov <diman.todorov () chello at>
Date: Tue, 10 Jul 2007 19:29:57 +0200
great! I will integrate it as soon as i have 2 hours i can spend on leisure :) cheers, Diman On 10.07.2007, at 19:10, Sven Klemm wrote:
Diman Todorov wrote:this brings up an interesting point. Have you had a look at the nselib? So far it has a very small dynamically loadable module written in c for bitwise operations which serves as a concept demonstration. Do you think you could easily make a loadable NSE module out of your libxml2 wrapper? I could imagine that it would only be compiled if users have libxml2 installed.The attached patch adds it as a new library. I left out the libxml2 detection stuff as I don't know how to do that. The only thing that is currently wrapped is parsing html and xml documents and querying those parsed documents with xpath. I attached the sedusa.nse script again because I changed the case for xml table to lowercase. Cheers, sven Index: xml.c =================================================================== --- xml.c (revision 0) +++ xml.c (revision 0) @@ -0,0 +1,121 @@ + +#include "xml.h" + +#include <libxml/HTMLparser.h> +#include <libxml/xpath.h> + +typedef struct XmlDocumentData { + xmlDocPtr document; +} XmlDocumentData; + +// create a lua XmlDocument +int create_document( lua_State * L, xmlDocPtr doc ) { + if ( doc ) { + // parsing successful + XmlDocumentData * doc_data; + doc_data = (XmlDocumentData *) lua_newuserdata( L, sizeof (XmlDocumentData)); + // set metatable for userdata + luaL_getmetatable( L, "XmlDocument" ); + lua_setmetatable( L, -2 ); + doc_data->document = doc; + } else { + // parsing failed + luaL_error( L , "parsing document failed." ); + } + return 1; +} + +// takes an html document as string and returns a XmlDocument +int xml_parse_html( lua_State * L ) { + const char * doc_string = luaL_checkstring( L, 1 ); + lua_pop(L, 1); + char * url = NULL; + char * encoding = NULL; + + int options = HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_NONET; + htmlDocPtr doc = htmlReadDoc( doc_string, url, encoding, options ); + + return create_document( L, doc ); +} + +// takes an xml document as string and returns a XmlDocument +int xml_parse_xml( lua_State * L ) { + const char * doc_string = luaL_checkstring( L, 1 ); + lua_pop(L, 1); + char * url = NULL; + char * encoding = NULL; + + int options = XML_PARSE_RECOVER | XML_PARSE_NOERROR | XML_PARSE_NOWARNING | XML_PARSE_NONET; + xmlDocPtr doc = xmlReadDoc( doc_string, url, encoding, options ); + + return create_document( L, doc ); +} + + +static const struct luaL_reg xml_methods[] = { + { "parse_html", xml_parse_html }, + { "parse_xml", xml_parse_xml }, + { NULL, NULL } +}; + +// takes an xpath expression and return the match(es) as string(s) +int xmldoc_xpath( lua_State * L ) { + XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L, 1, "XmlDocument"); + const char * xpath = luaL_checkstring( L, 2 ); + xmlXPathContextPtr context = xmlXPathNewContext( doc->document ); + + if ( !context ) luaL_error( L, "Error creating context." ); + + xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context); + xmlXPathFreeContext(context); + if ( !result ) luaL_error( L, "Error while evaluating XPath expression." ); + + if( xmlXPathNodeSetIsEmpty( result->nodesetval ) ) { + // empty resultset + lua_pushnil( L ); + xmlXPathFreeNodeSetList( result ); + return 1; + } else { + int i; + // we found something + xmlNodeSetPtr nodeset = result->nodesetval; + for ( i = 0; i < nodeset->nodeNr; i++) { + char * tmp = xmlXPathCastNodeToString( nodeset->nodeTab[i] ); + lua_pushstring( L, tmp ); + free( tmp ); + } + int matches = nodeset->nodeNr; + xmlXPathFreeNodeSetList( result ); + return matches; + } +} + +int xmldoc_free( lua_State * L ) { + XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L, 1, "XmlDocument"); + free( doc->document ); + return 0; +} + +// XmlDocument methods +static const struct luaL_reg xmldoc_methods[] = { + { "xpath", xmldoc_xpath }, + { "__gc", xmldoc_free }, + { NULL, NULL } +}; + + +// initializer function, called when library is required +int luaopen_xml( lua_State * L ) { + + // create metatable + luaL_newmetatable( L, "XmlDocument" ); + // metatable.__index = metatable + lua_pushvalue( L, -1 ); + lua_setfield( L, -2, "__index" ); + // register methods + luaL_register( L, NULL, xmldoc_methods ); + + luaL_register( L, "xml", xml_methods ); + return 1; +} + Index: Makefile.in =================================================================== --- Makefile.in (revision 5165) +++ Makefile.in (working copy) @@ -12,15 +12,20 @@ LIBTOOL= ./libtool LTFLAGS = --tag=CC --silent -all: bit.so +all: bit.so xml.so bit.so: bit.c @LIBTOOL_DEPS@ $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $(CFLAGS) -c bit.c $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -avoid-version -module - rpath /usr/local/lib -o bit.la bit.lo mv .libs/bit.so bit.so +xml.so: xml.c @LIBTOOL_DEPS@ + $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $ (LIBXML_INCLUDE) $(CFLAGS) -c xml.c + $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -lxml2 -avoid-version - module -rpath /usr/local/lib -o xml.la xml.lo + mv .libs/xml.so xml.so + clean: - rm -f bit.so *.la *.lo + rm -f bit.so xml.so *.la *.lo rm -rf .libs distclean: clean Index: xml.h =================================================================== --- xml.h (revision 0) +++ xml.h (revision 0) @@ -0,0 +1,13 @@ + +#ifndef XMLLIB +#define XMLLIB + +#define XMLLIBNAME "xml" + +#include "lauxlib.h" +#include "lua.h" + +LUALIB_API int luaopen_xml(lua_State *L); + +#endif + Index: nselib.h =================================================================== --- nselib.h (revision 5165) +++ nselib.h (working copy) @@ -2,6 +2,7 @@ #define NSE_LIB #define NSE_BITLIBNAME "bit" +#define NSE_XMLLIBNAME "xml" #endif id = "Sedusa" description = "Connects to an HTTP server and tries to guess the running web application." author = "Sven Klemm <sven () c3d2 de>" license = "See nmaps COPYING for licence" categories = {"safe"} require "stdnse" require "shortport" require "url" require "xml" portrule = shortport.service({'http', 'https', 'ssl/http'}) action = function(host, port) local scheme, hostname, app, exps, index, xpath, u, doc local target = {} if port.service == 'https' or port.version.service_tunnel == 'ssl' then target.scheme = "https" if port.number ~= 443 then target.port = port.number end else target.scheme = "http" if port.number ~= 80 then target.port = port.number end end if (host.name and not host.name == "") then target.host = host.name else target.host = host.ip end target.path = "/" u = url.build( target ) doc = Sedusa.get_document( u ) for app, exps in pairs( Sedusa.hints ) do for index, xpath in pairs( exps ) do if doc.xml:xpath( xpath ) then if Sedusa.verify[app] then local version = Sedusa.verify[app]( u ) if version then return app .. " " .. version else return app .. " version not identified." end else return "No verify function for " .. app .. " found." end break end end end end Sedusa = { hints = {}, verify = {}, document_cache = {}, Document = { new = function( h, b ) local parsed = nil if string.len( b ) > 0 then parsed = xml.parse_html( b ) end return { header = h, body = b, xml = parsed } end }, get_document = function( url ) local document if not Sedusa.document_cache[ url ] then document = Sedusa.http_get( url ) Sedusa.document_cache[ url ] = document else document = Sedusa.document_cache[ url ] end if document.header['Status'] == 301 or document.header['Status'] == 302 then document = Sedusa.get_document( document.header['Location'] ) end return document end, http_get = function( u ) local protocol, port, request, query, socket local parsed = url.parse( u ) if parsed.scheme == 'https' then protocol = "ssl" port = 443 else protocol = "tcp" port = 80 end if ( parsed.port ) then port = parsed.port end query = parsed.path if ( parsed.query ) then query = query .. '?' .. parsed.query end request = "GET "..query.." HTTP/1.1\r\nHost: "..parsed.host.."\r \n\r\n" socket = nmap.new_socket() socket:connect( parsed.host, port, protocol ) socket:send(request) local buffer = stdnse.make_buffer( socket, "\r?\n") local status, line, key, value, head, header head, header = {}, {} -- head loop while true do status, line = buffer() if (not status or line == "") then break end table.insert(header,line) end -- build nicer table for header for key, value in pairs( header ) do if key == 1 then local code = select( 3, string.find( value, "HTTP/%d\.%d (%d +)") ) head['Status'] = tonumber(code) else key, value = select( 3, string.find( value, "(.+): (.*)" ) ) if key and value then head[key] = value:gsub( '[\r\n]+', '' ) end end end local body = {} while true do status, line = buffer() if (not status) then break end table.insert(body,line) end socket:close() return Sedusa.Document.new( head, table.concat(body) ) end, } -- setup default detector for some web applications for key, app in pairs( {"b2evolution", "bBlog", "C3D2-Web", "DokuWiki", "gitweb", "Midgard", "Pentabarf", "PhpWiki", "Plone", "PostNuke", "TYPO3", "vBulletin"} ) do Sedusa.hints[app] = { '//meta[@name="generator" and starts-with (@content,"' .. app .. '")]' } Sedusa.verify[app] = function( url ) local document = Sedusa.get_document( url ) -- look in generator meta tag local generator = document.xml:xpath( '//meta [@name="generator"]/@content' ) if generator and string.match( generator, app .. " (.*)" ) then return string.match( generator, app .. " (.*)" ) end end end Sedusa.verify["PhpWiki"] = function( url ) local document = Sedusa.get_document( url ) local generator = document.xml:xpath( '//meta [@name="PHPWIKI_VERSION"]/@content' ) if generator then return generator end end Sedusa.hints["Drupal"] = { '//div[@id="block-user-0" and h2[text()="User login"]]/div [@class="content"]/form[@id="user-login-form"]/div[div[@class="form- item"]/input[@type="text" and @class="form-text required"]]', } Sedusa.hints["Joomla!"] = { '//meta[@name="Generator" and starts-with(@content,"Joomla!")]' } Sedusa.hints["MediaWiki"] = { '//body[contains(@class, "mediawiki")]', '//div[@id="footer"]/div[@id="f-poweredbyico"]/a[@href="http:// www.mediawiki.org/"]/img' } Sedusa.verify["MediaWiki"] = function( u ) local document = Sedusa.get_document( u ) local link = document.xml:xpath( '//a[contains(@href,"index.php? title=") and starts-with(@href, "/")]/@href' ) if link then link = link:match( "^(.*/index.php[?]title=).*") else link = document.xml:xpath( '//script[@type="text/javascript" and contains(text(), "var wgScriptPath = ")]' ) link = link:gsub( "[\r\n]", "" ) if link then link = link:match( 'var wgScriptPath = "([^"]+)"') .. '/ index.php?title=' end end -- get version from special:version local version = Sedusa.get_document( url.absolute( u, link .. "Special:Version" ) ) if version.xml:xpath('//div/ul/li[a[@href="http:// www.mediawiki.org/" and text()="MediaWiki"]]') then return string.match( version.xml:xpath('//div/ul/li[a [@href="http://www.mediawiki.org/" and text()="MediaWiki"]]'), "MediaWiki: ([^\r\n]+)" ) end -- get version from atom feed local atom = Sedusa.get_document( url.absolute( u, link .. "Special:Recentchanges&feed=atom" ) ) if atom.xml:xpath( '//generator[starts-with(text(), "MediaWiki")]' ) then return string.match( atom.xml:xpath( '//generator/text()' ), "^MediaWiki (.*)$" ) end end Sedusa.hints["WordPress"] = { '//meta[@name="generator" and starts-with(@content,"WordPress")]', '//head/link[@rel="stylesheet" and @type="text/css" and contains ( @href, "/wp-content/")]', '//div[@id="content"]/div[@class="post" and starts-with(@id, "post-") and div[@class="posttitle"] and div[@class="postmeta"] and div[@class="postbody"] and div[@class="postfooter"]]', } Sedusa.verify["WordPress"] = function( u ) local document = Sedusa.get_document( u ) -- look in generator meta tag local generator = document.xml:xpath( '//meta[@name="generator"]/ @content' ) if generator and string.match( generator, "WordPress (.*)" ) then return string.match( generator, "WordPress (.*)" ) end -- look in atom feed local atom = document.xml:xpath( '//link[@rel="alternate" and @type="application/atom+xml"]/@href' ) local feed = Sedusa.get_document( atom ) if feed.xml:xpath( '//generator[text()="WordPress"]/@version' ) then return feed.xml:xpath( '//generator[text()="WordPress"]/ @version' ) end return "Version not identified." end Sedusa.hints["Serendipity"] = { '//meta[@name="Powered-By" and starts-with(@content, "Serendipity")]' } Sedusa.verify["Serendipity"] = function( u ) local document = Sedusa.get_document( u ) local generator = document.xml:xpath( '//meta[@name="Powered-By"]/ @content' ) if generator and string.match( generator, "Serendipity v[.] (.*)" ) then return string.match( generator, "Serendipity v[.](.*)" ) end end Sedusa.hints["Trac"] = { '//div[@id="footer"]/a[@id="tracpowered" and @href="http:// trac.edgewall.org/"]/img[@alt="Trac Powered"]', } Sedusa.verify["Trac"] = function( u ) local document = Sedusa.get_document( u ) local version = document.xml:xpath( '//div[@id="footer" and a [@id="tracpowered" and @href="http://trac.edgewall.org/"]]/p [@class="left"]/a/strong/text()' ) if version and version:match( "Trac (.*)" ) then return version:match( "Trac (.*)" ) end end _______________________________________________ Sent through the nmap-dev mailing list http://cgi.insecure.org/mailman/listinfo/nmap-dev Archived at http://SecLists.Org
_______________________________________________ Sent through the nmap-dev mailing list http://cgi.insecure.org/mailman/listinfo/nmap-dev Archived at http://SecLists.Org
Current thread:
- [NSE] Web application detection Sven Klemm (Jul 10)
- Re: [NSE] Web application detection Diman Todorov (Jul 10)
- Re: [NSE] Web application detection Sven Klemm (Jul 10)
- Re: [NSE] Web application detection Diman Todorov (Jul 10)
- Re: [NSE] Web application detection Sven Klemm (Jul 10)
- Re: [NSE] Web application detection Diman Todorov (Jul 10)