source: trunk/plugins/nikto_robots.plugin @ 457

#VERSION,2.02
# $Id$
###############################################################################
#  Copyright (C) 2004 CIRT, Inc.
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; version 2
#  of the License only.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################
# PURPOSE:
# Check out the robots.txt file
###############################################################################
sub nikto_robots_init {
    my $id = {
        name      => "robots",
        full_name => "Robots",
        author    => "Sullo",
        description =>
          "Checks whether there's anything within the robots.txt file and analyses it for other paths to pass to other scripts.",
        recon_method => \&nikto_robots,
        recon_weight => 49,
        copyright    => "2008 CIRT Inc."
        };
    return $id;
}

sub nikto_robots {
    my ($mark) = @_;
    my %headers;
    (my $RES, $CONTENT) = nfetch($mark, "/robots.txt", "GET", "", \%headers, "", "robots");

    if (($RES eq 200) || ($RES eq $FoF{'okay'}{'response'})) {
        if (is_404("robots.txt", $CONTENT, $RES, $headers{'location'})) { return; }

        my ($DIRS, $RFILES) = "";
        my $DISCTR = 0;
        my @DOC = split(/\n/, $CONTENT);
        foreach my $line (@DOC) {
            $line = quotemeta($line);
            if ($line =~ /allow/i) {
                chomp($line);
                $line =~ s/\#.*$//;
                $line =~ s/(?:^\s+|\s+$)//g;
                $line =~ s/\s+/ /g;
                $line =~ s/\\t/ /g;
                $line =~ s/(?:dis)?allow(?:\\:)?\s?//i;
                $line =~ s/\*//g;
                $line =~ s/\/+/\//g;

                if ($line eq "") { next; }

                # try to figure out file vs dir... just guess...
                if (($line !~ /\./) && ($line !~ /\/$/)) { $line .= "/"; }

                $line = LW2::uri_normalize($line);

                # figure out dirs/files...
                my $realdir  = LW2::uri_get_dir($line);
                my $realfile = $line;
                $realfile =~ s/^$realdir//;

                nprint("- robots.txt entry dir:$realdir -- file:$realfile", "d");
                if (($realdir  ne "") && ($realdir  ne "/")) { $DIRS{$realdir}++; }
                if (($realfile ne "") && ($realfile ne "/")) { $RFILES{$realfile}++; }
                $DISCTR++;
            }    # end if $line =~ disallow
        }    # end foreach my $line (@DOC)_

        # add them  to mutate dir/file
        my $raw;
        foreach $raw (sort keys %DIRS) {
            $raw =~ s/\\//g;
            if ($VARIABLES{"\@MUTATEDIRS"} !~ /$raw/) {
                $VARIABLES{"\@MUTATEDIRS"} .= " $raw";
            }
            if ($raw =~ /cgi/ && $VARIABLES{"\@CGIDIRS"} !~ /$raw/) {
                $VARIABLES{"\@CGIDIRS"} .= " $raw";
            }
            if ($raw =~ /forum/ && $VARIABLES{"\@NUKE"} !~ /$raw/) {
                $VARIABLES{"\@NUKE"} .= " $raw";
            }
            if ($raw =~ /pass/ && $VARIABLES{"\@PASSWORDDIRS"} !~ /$raw/) {
                $VARIABLES{"\@PASSWORDDIRS"} .= " $raw";
            }
        }

        foreach $raw (sort keys %RFILES) {
            $raw =~ s/\\//g;
            if ($VARIABLES{"\@MUTATEFILES"} !~ /$raw/) {
                $VARIABLES{"\@MUTATEFILES"} .= " $raw";
            }
            if ($raw =~ /pass/ && $VARIABLES{"\@PASSWORDFILES"} !~ /$raw/) {
                $VARIABLES{"\@PASSWORDFILES"} .= " $raw";
            }
        }

        my $msg;
        if ($DISCTR eq 1) { $msg = "contains $DISCTR entry which should be manually viewed."; }
        elsif ($DISCTR > 1) { $msg = "contains $DISCTR entries which should be manually viewed."; }
        else { $msg = "retrieved but it does not contain any 'disallow' entries (which is odd)."; }

        add_vulnerability($mark, "robots.txt $msg", 999996, 0, "GET", "/robots.txt", \%result);
    }
}

1;