#!/usr/pkg/bin/python3.13


__license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'

import re

from lxml.etree import XMLSyntaxError

from calibre import force_unicode, human_readable, prepare_string_for_xml
from calibre.ebooks.chardet import replace_encoding_declarations, find_declared_encoding
from calibre.utils.xml_parse import safe_xml_fromstring
from calibre.ebooks.html_entities import html5_entities
from calibre.ebooks.oeb.polish.pretty import pretty_script_or_style as fix_style_tag
from calibre.ebooks.oeb.polish.utils import PositionFinder, guess_type
from calibre.ebooks.oeb.polish.check.base import BaseError, WARN, ERROR, INFO
from calibre.ebooks.oeb.base import OEB_DOCS, XHTML_NS, urlquote, URL_SAFE, XHTML
from polyglot.builtins import iteritems, error_message

HTML_ENTITTIES = frozenset(html5_entities)
XML_ENTITIES = {'lt', 'gt', 'amp', 'apos', 'quot'}
ALL_ENTITIES = HTML_ENTITTIES | XML_ENTITIES

replace_pat = re.compile('&(%s);' % '|'.join(re.escape(x) for x in sorted(HTML_ENTITTIES - XML_ENTITIES)))
mismatch_pat = re.compile(r'tag mismatch:.+?line (\d+).+?line \d+')


class EmptyFile(BaseError):

    HELP = _('This file is empty, it contains nothing, you should probably remove it.')
    INDIVIDUAL_FIX = _('Remove this file')

    def __init__(self, name):
        BaseError.__init__(self, _('The file %s is empty') % name, name)

    def __call__(self, container):
        container.remove_item(self.name)
        return True


class DecodeError(BaseError):

    is_parsing_error = True

    HELP = _('A decoding errors means that the contents of the file could not'
             ' be interpreted as text. This usually happens if the file has'
             ' an incorrect character encoding declaration or if the file is actually'
             ' a binary file, like an image or font that is mislabelled with'
             ' an incorrect media type in the OPF.')

    def __init__(self, name):
        BaseError.__init__(self, _('Parsing of %s failed, could not decode') % name, name)


class XMLParseError(BaseError):

    is_parsing_error = True

    HELP = _('A parsing error in an XML file means that the XML syntax in the file is incorrect.'
             ' Such a file will most probably not open in an e-book reader. These errors can '
             ' usually be fixed automatically, however, automatic fixing can sometimes '
             ' "do the wrong thing".')

    def __init__(self, msg, *args, **kwargs):
        msg = msg or ''
        BaseError.__init__(self, 'Parsing failed: ' + msg, *args, **kwargs)
        m = mismatch_pat.search(msg)
        if m is not None:
            self.has_multiple_locations = True
            self.all_locations = [(self.name, int(m.group(1)), None), (self.name, self.line, self.col)]


class HTMLParseError(XMLParseError):

    HELP = _('A parsing error in an HTML file means that the HTML syntax is incorrect.'
             ' Most readers will automatically ignore such errors, but they may result in '
             ' incorrect display of content. These errors can usually be fixed automatically,'
             ' however, automatic fixing can sometimes "do the wrong thing".')


class PrivateEntities(XMLParseError):

    HELP = _('This HTML file uses private entities.'
    ' These are not supported. You can try running "Fix HTML" from the Tools menu,'
    ' which will try to automatically resolve the private entities.')


class NamedEntities(BaseError):

    level = WARN
    INDIVIDUAL_FIX = _('Replace all named entities with their character equivalents in this book')
    HELP = _('Named entities are often only incompletely supported by various book reading software.'
             ' Therefore, it is best to not use them, replacing them with the actual characters they'
             ' represent. This can be done automatically.')

    def __init__(self, name):
        BaseError.__init__(self, _('Named entities present'), name)

    def __call__(self, container):
        changed = False
        from calibre.ebooks.oeb.polish.check.main import XML_TYPES
        check_types = XML_TYPES | OEB_DOCS
        for name, mt in iteritems(container.mime_map):
            if mt in check_types:
                raw = container.raw_data(name)
                nraw = replace_pat.sub(lambda m:html5_entities[m.group(1)], raw)
                if raw != nraw:
                    changed = True
                    with container.open(name, 'wb') as f:
                        f.write(nraw.encode('utf-8'))
        return changed


def make_filename_safe(name):
    from calibre.utils.filenames import ascii_filename

    def esc(n):
        return ''.join(x if x in URL_SAFE else '_' for x in n)
    return '/'.join(esc(ascii_filename(x)) for x in name.split('/'))


class EscapedName(BaseError):

    level = WARN

    def __init__(self, name):
        BaseError.__init__(self, _('Filename contains unsafe characters'), name)
        qname = urlquote(name)

        self.sname = make_filename_safe(name)
        self.HELP = _(
            'The filename {0} contains unsafe characters, that must be escaped, like'
            ' this {1}. This can cause problems with some e-book readers. To be'
            ' absolutely safe, use only the English alphabet [a-z], the numbers [0-9],'
            ' underscores and hyphens in your file names. While many other characters'
            ' are allowed, they may cause problems with some software.').format(name, qname)
        self.INDIVIDUAL_FIX = _(
            'Rename the file {0} to {1}').format(name, self.sname)

    def __call__(self, container):
        from calibre.ebooks.oeb.polish.replace import rename_files
        all_names = set(container.name_path_map)
        bn, ext = self.sname.rpartition('.')[0::2]
        c = 0
        while self.sname in all_names:
            c += 1
            self.sname = '%s_%d.%s' % (bn, c, ext)
        rename_files(container, {self.name:self.sname})
        return True


class TooLarge(BaseError):

    level = INFO
    MAX_SIZE = 260 *1024
    HELP = _('This HTML file is larger than %s. Too large HTML files can cause performance problems'
             ' on some e-book readers. Consider splitting this file into smaller sections.') % human_readable(MAX_SIZE)

    def __init__(self, name):
        BaseError.__init__(self, _('File too large'), name)


class BadEntity(BaseError):

    HELP = _('This is an invalid (unrecognized) entity. Replace it with whatever'
             ' text it is supposed to have represented.')

    def __init__(self, ent, name, lnum, col):
        BaseError.__init__(self, _('Invalid entity: %s') % ent, name, lnum, col)


class BadNamespace(BaseError):

    INDIVIDUAL_FIX = _(
        'Run fix HTML on this file, which will automatically insert the correct namespace')

    def __init__(self, name, namespace):
        BaseError.__init__(self, _('Invalid or missing namespace'), name)
        self.HELP = prepare_string_for_xml(_(
            'This file has {0}. Its namespace must be {1}. Set the namespace by defining the xmlns'
            ' attribute on the <html> element, like this <html xmlns="{1}">').format(
                (_('incorrect namespace %s') % namespace) if namespace else _('no namespace'),
                XHTML_NS))

    def __call__(self, container):
        container.parsed(self.name)
        container.dirty(self.name)
        return True


class NonUTF8(BaseError):

    level = WARN
    INDIVIDUAL_FIX = _("Change this file's encoding to UTF-8")

    def __init__(self, name, enc):
        BaseError.__init__(self, _('Non UTF-8 encoding declaration'), name)
        self.HELP = _('This file has its encoding declared as %s. Some'
                      ' reader software cannot handle non-UTF8 encoded files.'
                      ' You should change the encoding to UTF-8.') % enc

    def __call__(self, container):
        raw = container.raw_data(self.name)
        if isinstance(raw, str):
            raw, changed = replace_encoding_declarations(raw)
            if changed:
                container.open(self.name, 'wb').write(raw.encode('utf-8'))
                return True


class EntitityProcessor:

    def __init__(self, mt):
        self.entities = ALL_ENTITIES if mt in OEB_DOCS else XML_ENTITIES
        self.ok_named_entities = []
        self.bad_entities = []

    def __call__(self, m):
        val = m.group(1).decode('ascii')
        if val in XML_ENTITIES:
            # Leave XML entities alone
            return m.group()

        if val.startswith('#'):
            nval = val[1:]
            try:
                if nval.startswith('x'):
                    int(nval[1:], 16)
                else:
                    int(nval, 10)
            except ValueError:
                # Invalid numerical entity
                self.bad_entities.append((m.start(), m.group()))
                return b' ' * len(m.group())
            return m.group()

        if val in self.entities:
            # Known named entity, report it
            self.ok_named_entities.append(m.start())
        else:
            self.bad_entities.append((m.start(), m.group()))
        return b' ' * len(m.group())


def check_html_size(name, mt, raw):
    errors = []
    if len(raw) > TooLarge.MAX_SIZE:
        errors.append(TooLarge(name))
    return errors


entity_pat = re.compile(br'&(#{0,1}[a-zA-Z0-9]{1,8});')


def check_encoding_declarations(name, container):
    errors = []
    enc = find_declared_encoding(container.raw_data(name))
    if enc is not None and enc.lower() != 'utf-8':
        errors.append(NonUTF8(name, enc))
    return errors


def check_for_private_entities(name, raw):
    if re.search(br'<!DOCTYPE\s+.+?<!ENTITY\s+.+?]>', raw, flags=re.DOTALL) is not None:
        return True


def check_xml_parsing(name, mt, raw):
    if not raw:
        return [EmptyFile(name)]
    if check_for_private_entities(name, raw):
        return [PrivateEntities(_('Private entities found'), name)]
    raw = raw.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
    # Get rid of entities as named entities trip up the XML parser
    eproc = EntitityProcessor(mt)
    eraw = entity_pat.sub(eproc, raw)
    errcls = HTMLParseError if mt in OEB_DOCS else XMLParseError
    errors = []
    if eproc.ok_named_entities:
        errors.append(NamedEntities(name))
    if eproc.bad_entities:
        position = PositionFinder(raw)
        for offset, ent in eproc.bad_entities:
            lnum, col = position(offset)
            errors.append(BadEntity(ent, name, lnum, col))

    try:
        root = safe_xml_fromstring(eraw, recover=False)
    except UnicodeDecodeError:
        return errors + [DecodeError(name)]
    except XMLSyntaxError as err:
        try:
            line, col = err.position
        except:
            line = col = None
        return errors + [errcls(error_message(err), name, line, col)]
    except Exception as err:
        return errors + [errcls(error_message(err), name)]

    if mt in OEB_DOCS:
        if root.nsmap.get(root.prefix, None) != XHTML_NS:
            errors.append(BadNamespace(name, root.nsmap.get(root.prefix, None)))

    return errors


class CSSError(BaseError):

    is_parsing_error = True

    def __init__(self, level, msg, name, line, col):
        self.level = level
        prefix = 'CSS: '
        BaseError.__init__(self, prefix + msg, name, line, col)
        if level == WARN:
            self.HELP = _('This CSS construct is not recognized. That means that it'
                          ' most likely will not work on reader devices. Consider'
                          ' replacing it with something else.')
        else:
            self.HELP = _('Some reader programs are very'
                          ' finicky about CSS stylesheets and will ignore the whole'
                          ' sheet if there is an error. These errors can often'
                          ' be fixed automatically, however, automatic fixing will'
                          ' typically remove unrecognized items, instead of correcting them.')
            self.INDIVIDUAL_FIX = _('Try to fix parsing errors in this stylesheet automatically')

    def __call__(self, container):
        root = container.parsed(self.name)
        container.dirty(self.name)
        if container.mime_map[self.name] in OEB_DOCS:
            for style in root.xpath('//*[local-name()="style"]'):
                if style.get('type', 'text/css') == 'text/css' and style.text and style.text.strip():
                    fix_style_tag(container, style)
            for elem in root.xpath('//*[@style]'):
                raw = elem.get('style')
                if raw:
                    elem.set('style', force_unicode(container.parse_css(raw, is_declaration=True).cssText, 'utf-8').replace('\n', ' '))
        return True


pos_pats = (re.compile(r'\[(\d+):(\d+)'), re.compile(r'(\d+), (\d+)\)'))


class DuplicateId(BaseError):

    has_multiple_locations = True

    INDIVIDUAL_FIX = _(
        'Remove the duplicate ids from all but the first element')

    def __init__(self, name, eid, locs):
        BaseError.__init__(self, _('Duplicate id: %s') % eid, name)
        self.HELP = _(
            'The id {0} is present on more than one element in {1}. This is'
            ' not allowed. Remove the id from all but one of the elements').format(eid, name)
        self.all_locations = [(name, lnum, None) for lnum in sorted(locs)]
        self.duplicate_id = eid

    def __call__(self, container):
        elems = [e for e in container.parsed(self.name).xpath('//*[@id]') if e.get('id') == self.duplicate_id]
        for e in elems[1:]:
            e.attrib.pop('id')
        container.dirty(self.name)
        return True


class InvalidId(BaseError):

    level = WARN
    INDIVIDUAL_FIX = _(
        'Replace this id with a randomly generated valid id')

    def __init__(self, name, line, eid):
        BaseError.__init__(self, _('Invalid id: %s') % eid, name, line)
        self.HELP = _(
            'The id {0} is not a valid id. IDs must start with a letter ([A-Za-z]) and may be'
            ' followed by any number of letters, digits ([0-9]), hyphens ("-"), underscores ("_")'
            ', colons (":"), and periods ("."). This is to ensure maximum compatibility'
            ' with a wide range of devices.').format(eid)
        self.invalid_id = eid

    def __call__(self, container):
        from calibre.ebooks.oeb.base import uuid_id
        from calibre.ebooks.oeb.polish.replace import replace_ids
        newid = uuid_id()
        changed = False
        elems = (e for e in container.parsed(self.name).xpath('//*[@id]') if e.get('id') == self.invalid_id)
        for e in elems:
            e.set('id', newid)
            changed = True
            container.dirty(self.name)
        if changed:
            replace_ids(container, {self.name:{self.invalid_id:newid}})
        return changed


class BareTextInBody(BaseError):

    INDIVIDUAL_FIX = _('Wrap the bare text in a p tag')
    HELP = _('You cannot have bare text inside the body tag. The text must be placed inside some other tag, such as p or div')
    has_multiple_locations = True

    def __init__(self, name, lines):
        BaseError.__init__(self, _('Bare text in body tag'), name)
        self.all_locations = [(name, l, None) for l in sorted(lines)]

    def __call__(self, container):
        root = container.parsed(self.name)
        for body in root.xpath('//*[local-name() = "body"]'):
            children = tuple(body.iterchildren('*'))
            if body.text and body.text.strip():
                p = body.makeelement(XHTML('p'))
                p.text, body.text = body.text.strip(), '\n  '
                p.tail = '\n'
                if children:
                    p.tail += '  '
                body.insert(0, p)
            for child in children:
                if child.tail and child.tail.strip():
                    p = body.makeelement(XHTML('p'))
                    p.text, child.tail = child.tail.strip(), '\n  '
                    p.tail = '\n'
                    body.insert(body.index(child) + 1, p)
                    if child is not children[-1]:
                        p.tail += '  '
        container.dirty(self.name)
        return True


class ErrorHandler:

    ' Replacement logger to get useful error/warning info out of css_parser during parsing '

    def __init__(self, name):
        # may be disabled during setting of known valid items
        self.name = name
        self.errors = []

    def __noop(self, *args, **kwargs):
        pass
    info = debug = setLevel = getEffectiveLevel = addHandler = removeHandler = __noop

    def __handle(self, level, *args):
        msg = ' '.join(map(str, args))
        line = col = None
        for pat in pos_pats:
            m = pat.search(msg)
            if m is not None:
                line, col = int(m.group(1)), int(m.group(2))
        if msg and line is not None:
            # Ignore error messages with no line numbers as these are usually
            # summary messages for an underlying error with a line number
            if 'panose-1' in msg and 'unknown property name' in msg.lower():
                return  # panose-1 is allowed in CSS 2.1 and is generated by calibre
            self.errors.append(CSSError(level, msg, self.name, line, col))

    def error(self, *args):
        self.__handle(ERROR, *args)

    def warn(self, *args):
        self.__handle(WARN, *args)
    warning = warn


def check_filenames(container):
    errors = []
    all_names = set(container.name_path_map) - container.names_that_must_not_be_changed
    for name in all_names:
        if urlquote(name) != name:
            errors.append(EscapedName(name))
    return errors


valid_id = re.compile(r'^[a-zA-Z][a-zA-Z0-9_:.-]*$')


def check_ids(container):
    errors = []
    mts = set(OEB_DOCS) | {guess_type('a.opf'), guess_type('a.ncx')}
    for name, mt in iteritems(container.mime_map):
        if mt in mts:
            root = container.parsed(name)
            seen_ids = {}
            dups = {}
            for elem in root.xpath('//*[@id]'):
                eid = elem.get('id')
                if eid in seen_ids:
                    if eid not in dups:
                        dups[eid] = [seen_ids[eid]]
                    dups[eid].append(elem.sourceline)
                else:
                    seen_ids[eid] = elem.sourceline
                if eid and valid_id.match(eid) is None:
                    errors.append(InvalidId(name, elem.sourceline, eid))
            errors.extend(DuplicateId(name, eid, locs) for eid, locs in iteritems(dups))
    return errors


def check_markup(container):
    errors = []
    for name, mt in iteritems(container.mime_map):
        if mt in OEB_DOCS:
            lines = []
            root = container.parsed(name)
            for body in root.xpath('//*[local-name()="body"]'):
                if body.text and body.text.strip():
                    lines.append(body.sourceline)
                for child in body.iterchildren('*'):
                    if child.tail and child.tail.strip():
                        lines.append(child.sourceline)
            if lines:
                errors.append(BareTextInBody(name, lines))
    return errors
