#!/usr/pkg/bin/python3.13

# Dependencies:
# * Python 3
# * urllib3 (for HTTP requests with Keep Alive support)
# * beautifulsoup (for HTML parsing)
# * dnspython (for DNS queries)


import argparse
import sys
import random
import string
import re
import socket
import urllib.parse
import warnings
import json

import bs4
import dns.resolver
import dns.query
import dns.zone
import urllib3


STANDARD_PHP_FILES = ['index.php', 'wp-config.php', 'configuration.php',
                      'config.php', 'config.inc.php', 'settings.php']


# initializing global state variables
global_what404 = {}
duplicate_preventer = []
mainpage_cache = {}
dns_cache = {}

# This prevents BeautifulSoup from causing warnings when the
# content of a webpage looks suspicious, i.e. when it only
# contains a single URL or a filename.
warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

# This disables warnings about the lack of certificate verification.
# Usually this is a bad idea, but for this tool we want to find HTTPS leaks
# even if they are shipped with invalid certificates.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)  # noqa: DUO131


def DEFAULT(f):
    setattr(f, '_is_default_test', True)
    return f


def INFO(f):
    setattr(f, '_is_info_test', True)
    return f


def HOSTNAME(f):
    setattr(f, '_is_hostname_test', True)
    return f


def pdebug(msg):
    if args.debug:
        print(f"[[debug]] {msg}")


def pout(cause, url, misc="", noisymsg=False):
    if noisymsg and not args.noisy:
        return
    # we're storing URL without protocol/www-prefix and cause to avoid
    # duplicates on same host
    dup_check = cause + "__" + re.sub(r'http[s]?://(www\.)?', '', url) + misc
    if dup_check not in duplicate_preventer:
        duplicate_preventer.append(dup_check)
        if args.json:
            json_out.append({"cause": cause, "url": url, "misc": misc})
        else:
            if misc:
                print(f"[{cause}] {url} {misc}")
            else:
                print(f"[{cause}] {url}")


def randstring():
    return "".join(random.SystemRandom().choice(string.ascii_lowercase) for i in range(8))


def escape(msg):
    return repr(msg)[1:-1]


def fetcher(fullurl, binary=False, getredir=False, geterrpage=False):
    data = ""
    redir = ""
    try:
        r = pool.request('GET', fullurl, retries=False, redirect=False)
        if getredir:
            headers = {k.lower(): v for k, v in r.headers.items()}
            if 'location' in headers:
                redir = headers['location']
        elif (r.status != 200 and not geterrpage):
            data = ""
        elif binary:
            data = r.data
        else:
            data = r.data.decode('ascii', errors='ignore')
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        data = ""
    if getredir:
        return data, redir
    return data


def fetchpartial(fullurl, size, returnsize=False, binary=False):
    try:
        r = pool.request('GET', fullurl, retries=False, redirect=False,
                         preload_content=False)
        if r.status == 200:
            ret = r.read(size)
            if binary:
                rv = ret
            else:
                rv = ret.decode('ascii', errors='ignore')
            if returnsize:
                size = r.headers.getlist('content-length')
                if size == []:
                    size = 0
                else:
                    size = int(size[0])
                r.release_conn()
                return (rv, size)
            r.release_conn()
            return rv
        r.release_conn()
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass
    if returnsize:
        return ("", 0)
    return ""


def check404(url):
    if url in global_what404:
        return global_what404[url]
    rndurl = url + "/" + randstring() + ".htm"

    pdebug(f"Checking 404 page state of {rndurl}")
    try:
        r = pool.request('GET', rndurl, retries=False, redirect=False)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        return False
    what404 = {}
    what404['state'] = (r.status != 200)
    what404['content'] = r.data.decode('ascii', errors='ignore')
    if any(m in what404['content'] for m in ['<?php', '<?=']):
        # we're getting php code for 404 pages
        pout("errorpage_with_php", rndurl)
        what404['php'] = True
    else:
        what404['php'] = False
    if 'INSERT INTO' in what404['content']:
        # we're getting sql code for 404 pages
        pout("errorpage_with_sql", rndurl)
        what404['sql'] = True
    else:
        what404['sql'] = False
    if 'you have <code>DEBUG = True</code>' in what404['content']:
        pout("django_debug", rndurl)
    if '<title>No route found' in what404['content'] and \
       'Symfony Exception' in what404['content']:
        pout("symfony_debug", rndurl)

    global_what404[url] = what404
    return what404


def getmainpage(url):
    if url in mainpage_cache:
        return mainpage_cache[url]['content']

    try:
        r = pool.request('GET', url, retries=False, redirect=True)
        if r.status == 200:
            data = r.data.decode('ascii', errors='ignore')
        else:
            data = ""
        headers = {k.lower(): v for k, v in r.headers.items()}
        if 'location' in headers:
            redir = headers['location']
        else:
            redir = ""
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        mainpage_cache[url] = {}
        mainpage_cache[url]['httpcode'] = 0
        mainpage_cache[url]['location'] = ""
        mainpage_cache[url]['content'] = ""
        return ""

    mainpage_cache[url] = {}
    mainpage_cache[url]['httpcode'] = r.status
    mainpage_cache[url]['location'] = redir
    mainpage_cache[url]['content'] = data
    return data


def dnscache(qhost):
    if qhost in dns_cache:
        return dns_cache[qhost]
    try:
        socket.inet_aton(qhost)
        dns_cache[qhost] = qhost
        return qhost
    except OSError:
        pass
    try:
        if 'resolve' in dir(dns.resolver):
            dnsanswer = dns.resolver.resolve(qhost, 'A')
        else:  # dnspython before 2.0
            dnsanswer = dns.resolver.query(qhost, 'A')
    except (dns.exception.DNSException, ConnectionResetError):
        dns_cache[qhost] = None
        return None
    dns_cache[qhost] = dnsanswer.rrset
    return dnsanswer.rrset


@DEFAULT
def test_lfm_php(url):
    r = fetcher(url + '/lfm.php')
    if 'Lazy File Manager' in r:
        pout("lfm_php", url + "/lfm.php")


@DEFAULT
def test_idea(url):
    r = fetcher(url + '/.idea/WebServers.xml')
    if 'name="WebServers"' in r:
        pout("idea", url + "/.idea/WebServers.xml")


@DEFAULT
def test_symfony_databases_yml(url):
    r = fetcher(url + '/config/databases.yml')
    if 'class:' in r and 'param:' in r:
        pout("symfony_databases_yml", url + "/config/databases.yml")


@DEFAULT
def test_rails_database_yml(url):
    r = fetcher(url + '/config/database.yml')
    if 'adapter:' in r and 'database:' in r:
        pout("rails_database_yml", url + "/config/database.yml")


@DEFAULT
def test_git_dir(url):
    r = fetcher(url + '/.git/config')
    if '[core]' in r:
        pout("git_dir", url + "/.git/config")


@DEFAULT
def test_svn_dir(url):
    r = fetcher(url + '/.svn/entries')
    try:
        if (str)((int)(r)) + "\n" == r:
            pout("svn_dir", url + "/.svn/entries")
    except ValueError:
        pass


@DEFAULT
def test_apache_server_status(url):
    r = fetcher(url + '/server-status')
    if 'Apache Status' in r:
        pout("apache_server_status", url + "/server-status")


@DEFAULT
def test_apache_server_info(url):
    r = fetcher(url + '/server-info')
    if 'Apache Server Information</h1>' in r:
        pout("apache_server_info", url + "/server-info")


@DEFAULT
def test_coredump(url):
    r = fetchpartial(url + "/core", 20, binary=True)
    if r and r[0:4] == b'\x7fELF':
        pout("coredump", url + "/core")


@DEFAULT
def test_sftp_config(url):
    r = fetcher(url + '/sftp-config.json')
    if '"type":' in r and 'ftp' in r and '"save_before_upload"' in r:
        pout("sftp_config", url + "/sftp-config.json")


@DEFAULT
def test_wsftp_ini(url):
    for fn in ["WS_FTP.ini", "ws_ftp.ini", "WS_FTP.INI"]:
        r = fetcher(url + "/" + fn)
        if '[_config_]' in r:
            pout("wsftp_ini", url + "/" + fn)


@DEFAULT
def test_filezilla_xml(url):
    for fn in ["filezilla.xml", "sitemanager.xml", "FileZilla.xml"]:
        r = fetcher(url + "/" + fn)
        if '<FileZilla' in r:
            pout("filezilla_xml", url + "/" + fn)


@DEFAULT
def test_winscp_ini(url):
    for fn in ["winscp.ini", "WinSCP.ini"]:
        r = fetcher(url + "/" + fn)
        if '[Configuration]' in r:
            pout("winscp_ini", f"{url}/{fn}")


@DEFAULT
def test_ds_store(url):
    r = fetcher(url + '/.DS_Store', binary=True)
    if r[0:8] == b'\x00\x00\x00\x01Bud1':
        pout("ds_store", url + "/.DS_Store")


@DEFAULT
def test_php_cs_fixer(url):
    r = fetcher(url + '/.php_cs.cache')
    if r[0:8] == '{"php":"':
        pout("php_cs_cache", url + "/.php_cs.cache")
    r = fetcher(url + '/.php-cs-fixer.cache')
    if r[0:8] == '{"php":"':
        pout("php_cs_cache", url + "/.php-cs-fixer.cache")


@DEFAULT
def test_backupfiles(url):
    what404 = check404(url)
    if not what404:
        return
    if what404['php'] and not what404['state']:
        # we don't get proper 404 replies and the default page contains PHP
        # code, check doesn't make sense.
        return
    for f in STANDARD_PHP_FILES:
        for ps in ['_FILE_.bak', '_FILE_~', '._FILE_.swp', '%23_FILE_%23', '_FILE_.save',
                   '_FILE_.orig']:
            furl = url + "/" + ps.replace('_FILE_', f)
            r = fetcher(furl)
            if any(m in r for m in ['<?php', '<?=']):
                pout("backupfiles", furl)


@DEFAULT
def test_backup_archive(url):
    apexhost = re.sub('^www.', '', re.sub('(.*//|/.*)', "", url))
    wwwhost = "www." + apexhost

    for fn in ['backup', 'www', 'wwwdata', 'db', 'htdocs', apexhost, wwwhost]:
        r = fetchpartial(url + "/" + fn + ".zip", 2, binary=True)
        if r == b'PK':
            pout("backup_archive", url + "/" + fn + ".zip")
        r = fetchpartial(url + "/" + fn + ".tar.gz", 3, binary=True)
        if r == b'\x1f\x8b\x08':
            pout("backup_archive", url + "/" + fn + ".tar.gz")
        r = fetchpartial(url + "/" + fn + ".tar.bz2", 3, binary=True)
        if r in [b'BZh', b'BZ0']:
            pout("backup_archive", url + "/" + fn + ".tar.bz2")
        r = fetchpartial(url + "/" + fn + ".tar.xz", 6, binary=True)
        if r == b'\xFD7zXZ\x00':
            pout("backup_archive", url + "/" + fn + ".tar.xz")


@DEFAULT
def test_deadjoe(url):
    r = fetcher(url + '/DEADJOE')
    if 'in JOE when it aborted' in r:
        pout("deadjoe", url + "/DEADJOE")


@DEFAULT
def test_sql_dump(url):
    what404 = check404(url)
    if not what404:
        return
    for f in ['dump.sql', 'database.sql', '1.sql', 'backup.sql', 'data.sql',
              'db_backup.sql', 'dbdump.sql', 'db.sql', 'localhost.sql',
              'mysql.sql', 'site.sql', 'sql.sql', 'temp.sql', 'users.sql',
              'translate.sql', 'mysqldump.sql']:
        if not what404['sql'] or what404['state']:
            # if we don't get proper 404 replies and the default page contains
            # SQL code, check doesn't make sense.
            r = fetchpartial(url + "/" + f, 4000)
            if any(m in r for m in ['INSERT INTO']):
                pout("sql_dump", url + "/" + f)
        r = fetchpartial(url + "/" + f + ".gz", 3, binary=True)
        if r == b'\x1f\x8b\x08':
            pout("sql_dump_gz", url + "/" + f + ".gz")
        r = fetchpartial(url + "/" + f + ".bz2", 3, binary=True)
        if r in [b'BZh', b'BZ0']:
            pout("sql_dump_bz", url + "/" + f + ".bz2")
        r = fetchpartial(url + "/" + f + ".xz", 6, binary=True)
        if r == b'\xFD7zXZ\x00':
            pout("sql_dump_xz", url + "/" + f + ".xz")


@DEFAULT
def test_bitcoin_wallet(url):
    r = fetchpartial(url + "/wallet.dat", 16, binary=True)
    if r and len(r) == 16 and r[12:] == b'b1\x05\x00':
        pout("bitcoin_wallet", url + "/wallet.dat")


@DEFAULT
def test_drupal_backup_migrate(url):
    r = fetcher(url + '/sites/default/private/files/backup_migrate/scheduled/test.txt')
    if 'this file should not be publicly accessible' in r:
        pout("drupal_backup_migrate",
             url + "/sites/default/private/files/backup_migrate/scheduled/test.txt")


@DEFAULT
def test_magento_config(url):
    r = fetcher(url + '/app/etc/local.xml')
    if '<config' in r and 'Mage' in r:
        pout("magento_config", url + "/app/etc/local.xml")


@DEFAULT
def test_xaa(url):
    r, s = fetchpartial(url + "/xaa", 4000, returnsize=True, binary=True)
    if r is None:
        return
    # we need some heuristics here to avoid false positives.
    # If output is larger than 10 MB it's likely a true positive.
    if s > 10000000:
        pout("xaa", url + "/xaa")
        return
    # Check for signs of common compression formats (gz, bzip2, xz, zstd, zip).
    if (r[0:3] == b'\x1f\x8b\x08'
            or r[0:3] in [b'BZh', b'BZ0']
            or r[0:6] == b'\xFD7zXZ\x00'
            or r[0:4] in [b'\x28\xB5\x2F\xFD', b'PK\x03\x04']):
        pout("xaa", url + "/xaa")


@DEFAULT
def test_optionsbleed(url):
    try:
        r = pool.request('OPTIONS', url, retries=False, redirect=False)
    except (ConnectionRefusedError, urllib3.exceptions.HTTPError,
            UnicodeError):
        return
    try:
        allow = str(r.headers["Allow"])
    except KeyError:
        return
    if allow == "":
        pout("options_empty", url, noisymsg=True)
        return

    # catch some obvious cases first
    if (",," in allow) or (allow[0] == ",") or (allow[-1] == ","):
        pout("optionsbleed", url, escape(allow))
        return

    z = [x.strip() for x in allow.split(',')]
    if re.match("^[a-zA-Z-]*$", "".join(z)):
        if len(z) > len(set(z)):
            pout("options_duplicates", url, escape(allow), noisymsg=True)
        return
    if re.match("^[a-zA-Z- ]*$", allow):
        pout("options_spaces", url, escape(allow))
        return
    pout("optionsbleed", url, escape(allow))


@DEFAULT
def test_privatekey(url):
    hostkey = re.sub('^www.', '', re.sub('(.*//|/.*)', "", url)) + ".key"
    wwwkey = "www." + hostkey
    for fn in ["server.key", "privatekey.key", "myserver.key", "key.pem",
               hostkey, wwwkey]:
        r = fetcher(url + "/" + fn)
        if 'BEGIN PRIVATE KEY' in r:
            pout("privatekey_pkcs8", f"{url}/{fn}")
        if 'BEGIN RSA PRIVATE KEY' in r:
            pout("privatekey_rsa", f"{url}/{fn}")
        if 'BEGIN DSA PRIVATE KEY' in r:
            pout("privatekey_dsa", f"{url}/{fn}")
        if 'BEGIN EC PRIVATE KEY' in r:
            pout("privatekey_ec", f"{url}/{fn}")


@DEFAULT
def test_sshkey(url):
    for fn in ["id_rsa", "id_dsa", ".ssh/id_rsa", ".ssh/id_dsa"]:
        r = fetcher(url + "/" + fn)
        if 'BEGIN' in r and 'PRIVATE KEY' in r:
            pout("sshkey", f"{url}/{fn}")


@DEFAULT
def test_dotenv(url):
    r = fetcher(url + "/.env")
    if "APP_ENV=" in r or "DB_PASSWORD=" in r:
        pout("dotenv", url + "/.env")


@DEFAULT
def test_invalidsrc(url):
    r = getmainpage(url)
    try:
        p = bs4.BeautifulSoup(r, 'html.parser')
    except (NotImplementedError, TypeError, AssertionError):
        # This is due to a python bug, please remove in the future.
        # https://bugs.python.org/issue32876
        pdebug("Can't parse due to python bug")
        return
    srcs = []
    for tag in p.findAll(attrs={"src": True}):
        srcs.append(tag['src'])
    srcs = list(set(srcs))
    srcs.sort()

    checkeddomains = []
    for src in srcs:
        try:
            realurl = urllib.parse.urljoin(url, src)
            domain = urllib.parse.urlparse(realurl).hostname
            protocol = urllib.parse.urlparse(realurl).scheme
        except ValueError:
            pout("invalidsrc_brokenurl", escape(url), src)
            continue
        if domain is None:
            continue
        if protocol not in ['https', 'http']:
            continue

        # We avoid double-checking multiple requests to the same host.
        # This is a compromise between speed and in-depth scanning.
        if domain in checkeddomains:
            continue
        checkeddomains.append(domain)
        pdebug(f"Checking url {realurl}")

        if dnscache(domain) is None:
            pout("invalidsrc_dns", url, escape(src))
            continue

        try:
            r = pool.request('GET', realurl, retries=False, redirect=False)
            if r.status >= 400:
                pout("invalidsrc_http", url, escape(src))
        except UnicodeEncodeError:
            pass
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pout("invalidsrc_http_connfail", url, escape(src))


@DEFAULT
def test_ilias_defaultpw(url):
    getmainpage(url)
    if (url + "/ilias.php" in mainpage_cache[url]['location']
            or (url + "/login.php" in mainpage_cache[url]['location']
                and 'powered by ILIAS' in fetcher(mainpage_cache[url]['location']))):
        # we're confident we found an ILIAS installation
        pdebug("Ilias found")
        try:
            login = pool.request("POST", url + "/ilias.php?cmd=post&baseClass=ilStartUpGUI",
                                 fields={'username': 'root',
                                         'password': 'homer',
                                         'cmd[doStandardAuthentication]': 'Login'},
                                 headers={'Cookie': 'iltest=;PHPSESSID=' + randstring()})
            data = login.data.decode('ascii', errors='ignore')
            if (('class="ilFailureMessage"' not in data)
                    and ('name="il_message_focus"' not in data)
                    and (('class="ilBlockContent"' in data)
                         or ('class="ilAdminRow"' in data))):
                pout("ilias_defaultpw", url, "root/homer")
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pass


@DEFAULT
def test_cgiecho(url):
    for pre in [url + "/cgi-bin/cgiecho", url + "/cgi-sys/cgiecho"]:
        try:
            r = pool.request("GET", pre + "/" + randstring())
            if r.status == 500 and '<P><EM>cgiemail' in r.data.decode('ascii', errors='ignore'):
                pout("cgiecho", pre)
        except (ConnectionRefusedError, ConnectionResetError,
                urllib3.exceptions.HTTPError):
            pass


@DEFAULT
def test_phpunit_eval(url):
    try:
        r = pool.request("POST", url + "/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php",
                         body='<?php echo(substr_replace("hello", "12", 2, 2));')
        data = r.data.decode('ascii', errors='ignore')
        if data == "he12o":
            pout("phpunit_eval", url + "/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php")
    except (ConnectionRefusedError, ConnectionResetError,
            urllib3.exceptions.HTTPError):
        pass


@DEFAULT
def test_acmereflect(url):
    reflect = randstring()
    try:
        r = pool.request('GET', url + "/.well-known/acme-challenge/<html>" + reflect,
                         retries=False, redirect=False)
        if not r.data.decode('ascii', errors='ignore').startswith("<html>" + reflect):
            return
        headers = {k.lower(): v for k, v in r.headers.items()}
        if ('content-type' in headers) and headers['content-type'].startswith("text/plain"):
            return
        pout("acmereflect", url + "/.well-known/acme-challenge/reflect")

    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        return


@DEFAULT
def test_drupaldb(url):
    r = fetchpartial(url + "/sites/default/files/.ht.sqlite", 20, binary=True)
    if r and r[0:13] == b'SQLite format':
        pout("drupaldb", url + "/sites/default/files/.ht.sqlite")


@DEFAULT
def test_phpwarnings(url):
    try:
        r = pool.request("GET", url, headers={"Cookie": "PHPSESSID=in_vålíd"})
        if ('The session id is too long or contains illegal characters'
                in r.data.decode('ascii', errors='ignore')):
            pout("phpwarnings", url)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_adminer(url):
    r = fetcher(url + "/adminer.php")
    if "adminer.org" in r:
        pout("adminer", url + "/adminer.php")


@DEFAULT
def test_elmah(url):
    r = fetcher(url + "/elmah.axd")
    if "Error Log for" in r:
        pout("elmah", url + "/elmah.axd")
    r = fetcher(url + "/scripts/elmah.axd")
    if "Error Log for" in r:
        pout("elmah", url + "/scripts/elmah.axd")


@DEFAULT
def test_citrix_rce(url):
    try:
        r = pool.request('GET', url + "/vpn/../vpns/portal/tips.html",
                         retries=False, redirect=False,
                         headers={"NSC_USER": "x", "NSC_NONCE": "x"})
        if '<div id="nexttip">' in r.data.decode('ascii', errors='ignore'):
            pout("citrix_rce", url + "/vpn/../vpns/portal/tips.html")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_installer(url):
    r = getmainpage(url)
    if (mainpage_cache[url]['location'].endswith("wp-admin/setup-config.php")
       or 'href="wp-admin/css/install.css"' in r):
        pout("installer_wordpress", url)
    elif mainpage_cache[url]['location'].endswith("installation/index.php"):
        pout("installer_joomla", url)
    elif mainpage_cache[url]['location'].endswith("typo3/install.php"):
        pout("installer_typo3", url)
    elif mainpage_cache[url]['location'].endswith("install.php"):
        pout("installer_drupal", url)
    elif mainpage_cache[url]['location'].endswith("serendipity_admin.php"):
        pout("installer_s9y", url)
    elif 'LocalSettings.php not found' in r:
        pout("installer_mediawiki", url)
    elif '8 easy steps and will take around 5 minutes' in r:
        pout("installer_matomo", url)
    elif 'Create an <strong>admin account</strong>' in r:
        pout("installer_nextcloud", url)


@DEFAULT
def test_wpsubdir(url):
    r, redir = fetcher(url + "/wordpress/", getredir=True)
    if (redir.endswith("wp-admin/setup-config.php")
       or 'href="wp-admin/css/install.css"' in r):
        pout("wpsubdir", url + "/wordpress/")


@DEFAULT
def test_telescope(url):
    r = fetcher(url + "/telescope", geterrpage=True)
    if '<strong>Laravel</strong> Telescope' in r:
        pout("telescope", url + "/telescope")
    elif 'The Telescope assets are not published' in r:
        pout("telescope_inactive", url + "/telescope")


@DEFAULT
def test_vb_test(url):
    r = fetcher(url + "/vb_test.php")
    if '<title>vBulletin Test Script' in r:
        pout("vb_test", url + "/vb_test.php")


@DEFAULT
def test_headerinject(url):
    rnd = randstring()
    try:
        r = pool.request('GET', url + '/%%0D%%0A%s:1' % rnd, retries=False, redirect=False)
        if rnd in r.headers:
            pout("headerinject", url + '/%%0D%%0A%s:1' % rnd)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
def test_wpdebug(url):
    r = fetcher(url + "/wp-content/debug.log")
    if re.match(r'^\[\d\d-\w\w\w-\d\d\d\d ', r):
        pout("wpdebug", url + "/wp-content/debug.log")


@DEFAULT
def test_thumbsdb(url):
    r = fetcher(url + "/Thumbs.db", binary=True)
    if r and r[0:8] == b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1':
        pout("thumbsdb", url + "/Thumbs.db")


@DEFAULT
def test_duplicator(url):
    for fn in ['installer.php', 'installer-backup.php']:
        r = fetcher(f"{url}/{fn}")
        if '/dup-installer/main.installer.php' in r:
            pout("duplicator", f"{url}/{fn}")
    for fn in ['backups-dup-pro', 'backups-dup-lite']:
        r = fetcher(f"{url}/wp-content/{fn}/")
        if '>Index of /' in r:
            pout("duplicator_dirlisting", f"{url}/wp-content/{fn}/")


@DEFAULT
def test_desktopini(url):
    r = fetcher(url + "/desktop.ini")
    if '[\x00.\x00S\x00h\x00e\x00l\x00l\x00C\x00l\x00a\x00s\x00s' in r:
        pout("desktopini", url + "/desktop.ini")
    r = fetcher(url + "/Desktop.ini")
    if '[\x00.\x00S\x00h\x00e\x00l\x00l\x00C\x00l\x00a\x00s\x00s' in r:
        pout("desktopini", url + "/Desktop.ini")


@DEFAULT
@HOSTNAME
def test_axfr(qhost):
    try:
        if 'resolve' in dir(dns.resolver):
            ns = dns.resolver.resolve(qhost, 'NS')
        else:  # dnspython before 2.0
            ns = dns.resolver.query(qhost, 'NS')
    except (dns.exception.DNSException, dns.exception.Timeout,
            ConnectionResetError, ConnectionRefusedError,
            EOFError, socket.gaierror, TimeoutError, OSError):
        return
    for r in ns.rrset:
        r = str(r)
        ipv4 = []
        ipv6 = []
        try:
            if 'resolve' in dir(dns.resolver):
                ipv4 = dns.resolver.resolve(r, 'a').rrset
                ipv6 = dns.resolver.resolve(r, 'aaaa').rrset
            else:  # dnspython before 2.0
                ipv4 = dns.resolver.query(r, 'a').rrset
                ipv6 = dns.resolver.query(r, 'aaaa').rrset
        except dns.resolver.NoAnswer:
            pass
        ips = []
        for ip in ipv4:
            ips.append(str(ip))
        for ip in ipv6:
            ips.append(str(ip))
        for ip in ips:
            try:
                axfr = dns.zone.from_xfr(dns.query.xfr(ip, qhost))
                if axfr:
                    pout("axfr", qhost, r)
            except (dns.exception.DNSException, dns.exception.Timeout,
                    ConnectionResetError, ConnectionRefusedError,
                    EOFError, socket.gaierror, TimeoutError, OSError):
                pass


@DEFAULT
@HOSTNAME
def test_openmonit(qhost):
    url = f"http://{qhost}:2812/"
    headers = urllib3.util.make_headers(basic_auth='admin:monit')
    try:
        r = pool.request('GET', url, headers=headers)
        if '<title>Monit:' in r.data.decode('ascii', errors='ignore'):
            pout("openmonit", url)
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@DEFAULT
@HOSTNAME
def test_openelasticsearch(qhost):
    headers = urllib3.util.make_headers(basic_auth='admin:admin')
    try:
        r = pool.request("GET", f"http://{qhost}:9200", headers=headers)
        if '"cluster_name" :' in r.data.decode('ascii', errors='ignore'):
            pout("openelasticsearch", f"http://{qhost}:9200")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass
    try:
        r = pool.request("GET", f"https://{qhost}:9200", headers=headers)
        if '"cluster_name" :' in r.data.decode('ascii', errors='ignore'):
            pout("openelasticsearch", f"https://{qhost}:9200")
    except (urllib3.exceptions.HTTPError, UnicodeError,
            ConnectionRefusedError):
        pass


@INFO
def test_drupal(url):
    r = fetcher(url + "/CHANGELOG.txt")
    try:
        if r != "":
            result = re.findall("Drupal [0-9.]*", r)
            version = result[0][7:]
            pout("drupal", url, version)
        r = fetcher(url + "/core/CHANGELOG.txt")
        if r != "":
            result = re.findall("Drupal [0-9.]*", r)
            version = result[0][7:]
            pout("drupal", url, version)
    except IndexError:
        pass


@INFO
def test_wordpress(url):
    r = getmainpage(url)
    try:
        p = bs4.BeautifulSoup(r, 'html.parser')
        g = p.findAll("meta", {"name": "generator"})
        if g and g[0]['content'][:9] == "WordPress":
            version = g[0]['content'][10:]
            if not set(version).issubset("0123456789."):
                return
            pout("wordpress", url, version)
    except (NotImplementedError, TypeError, AssertionError):
        # Necessary due to a python bug (remove in the future):
        # https://bugs.python.org/issue32876
        return
    except KeyError:
        return


@INFO
def test_mailman(url):
    murl = f"{url}/mailman/listinfo"
    r = fetcher(murl)
    if 'Delivered by Mailman' in r:
        ver = re.findall('version ([0-9.]+)', r)
        if len(ver) > 0:
            ver = ver[0]
        else:
            ver = "unknown"
        if 'There currently are no publicly-advertised' in r:
            pout("mailman_unused", f"{murl} {ver}")
        else:
            pout("mailman", f"{murl} {ver}")


@INFO
def test_composer(url):
    for c in ['composer.json', 'composer.lock']:
        furl = url + '/' + c
        r = fetcher(furl)
        if '"require":' in r or '"packages":' in r:
            pout("composer", furl)


def test_phpinfo(url):
    for fn in ["phpinfo.php", "info.php", "i.php", "test.php"]:
        r = fetcher(url + "/" + fn)
        if 'phpinfo()' in r:
            pout("phpinfo", url + "/" + fn)


def new_excepthook(etype, value, traceback):
    if etype == KeyboardInterrupt:
        pdebug("Interrupted by user...")
        sys.exit(1)

    print("Oh oh... an unhandled exception has happened. This shouldn't be.")
    print("Please report a bug and include all output.")
    print("")
    print("called with")
    print(" ".join(sys.argv))
    print("")
    sys.__excepthook__(etype, value, traceback)


sys.excepthook = new_excepthook


parser = argparse.ArgumentParser()
parser.add_argument("hosts", nargs='+', help="hostname to scan")
parser.add_argument("-t", "--tests", help="Comma-separated tests to run.")
parser.add_argument("--useragent", help="User agent to send")
parser.add_argument("--nowww", action="store_true",
                    help="Skip scanning www.[host]")
parser.add_argument("--nohttp", action="store_true",
                    help="Don't scan http")
parser.add_argument("--nohttps", action="store_true",
                    help="Don't scan https")
parser.add_argument("-i", "--info", action="store_true",
                    help="Enable all info tests (no bugs/security vulnerabilities)")
parser.add_argument("-n", "--noisy", action="store_true",
                    help="Show noisy messages that indicate boring bugs, but no security issue")
parser.add_argument("-p", "--path", default='', action="store", type=str,
                    help="Base path on server (scans root dir by default)")
parser.add_argument("-j", "--json", action="store_true",
                    help="Produce JSON output")
parser.add_argument("-d", "--debug", action="store_true",
                    help="Show detailed debugging info")
args = parser.parse_args()

# Initializing global pool manager
user_agent = {'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0'}
if args.useragent:
    user_agent = {'user-agent': args.useragent}
pool = urllib3.PoolManager(10, headers=user_agent, cert_reqs='CERT_NONE',  # noqa: DUO132
                           retries=False, timeout=2)

# This is necessary for directory traversal attacks like citrix_cve
urllib3.util.url.NORMALIZABLE_SCHEMES = ()

if args.tests is None:
    tests = [g for f, g in locals().items() if hasattr(g, '_is_default_test')]
else:
    tests = []
    try:
        for x in args.tests.split(','):
            tests.append(locals()["test_" + x])
    except KeyError:
        print(f"Test {x} does not exist")
        sys.exit(1)

if args.info:
    tests += [g for f, g in locals().items() if hasattr(g, '_is_info_test')]

path = args.path.rstrip("/")
if path != "" and path[0] != "/":
    path = "/" + path
if path != "":
    pdebug(f"Path: {path}")

hosts = list(args.hosts)
if not args.nowww:
    for h in args.hosts:
        hosts.append("www." + h)

for i, h in enumerate(hosts):
    if h.startswith("http://") or h.startswith("https://"):
        print("ERROR: Please run snallygaster with a hostname, not a URL.")
        sys.exit(1)
    try:
        hosts[i] = h.encode("idna").decode("ascii")
    except UnicodeError:
        print("ERROR: Invalid hostname")
        sys.exit(1)
    if h != hosts[i]:
        pdebug(f"Converted {h} to {hosts[i]}")

pdebug("All hosts: %s" % ",".join(hosts))


json_out = []
for host in hosts:
    pdebug(f"Scanning {host}")
    for test in tests:
        pdebug(f"Running {test.__name__} test")
        if hasattr(test, '_is_hostname_test'):
            test(host)
        else:
            if not args.nohttp:
                test("http://" + host + path)
            if not args.nohttps:
                test("https://" + host + path)

# clear all sockets
pool.clear()

if args.json:
    print(json.dumps(json_out))
