#!/usr/pkg/bin/python3.13
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>


import os
from copy import copy
from collections import namedtuple
from datetime import datetime, time
from functools import partial
from threading import Lock

from calibre.constants import config_dir
from calibre.db.categories import Tag
from calibre.ebooks.metadata.sources.identify import urls_from_identifiers
from calibre.utils.date import isoformat, UNDEFINED_DATE, local_tz
from calibre.utils.config import tweaks
from calibre.utils.formatter import EvalFormatter
from calibre.utils.file_type_icons import EXT_MAP
from calibre.utils.icu import collation_order_for_partitioning
from calibre.utils.localization import calibre_langcode_to_name
from calibre.library.comments import comments_to_html, markdown
from calibre.library.field_metadata import category_icon_map
from polyglot.builtins import iteritems, itervalues
from polyglot.urllib import quote

IGNORED_FIELDS = frozenset('cover ondevice path marked au_map'.split())


def encode_datetime(dateval):
    if dateval is None:
        return "None"
    if not isinstance(dateval, datetime):
        dateval = datetime.combine(dateval, time())
    if hasattr(dateval, 'tzinfo') and dateval.tzinfo is None:
        dateval = dateval.replace(tzinfo=local_tz)
    if dateval <= UNDEFINED_DATE:
        return None
    return isoformat(dateval)


empty_val = ((), '', {})
passthrough_comment_types = {'long-text', 'short-text'}


def add_field(field, db, book_id, ans, field_metadata):
    datatype = field_metadata.get('datatype')
    if datatype is not None:
        val = db._field_for(field, book_id)
        if val is not None and val not in empty_val:
            if datatype == 'datetime':
                val = encode_datetime(val)
                if val is None:
                    return
            elif datatype == 'comments' or field == 'comments':
                ctype = field_metadata.get('display', {}).get('interpret_as', 'html')
                if ctype == 'markdown':
                    ans[field + '#markdown#'] = val
                    val = markdown(val)
                elif ctype not in passthrough_comment_types:
                    val = comments_to_html(val)
            elif datatype == 'composite' and field_metadata['display'].get('contains_html'):
                val = comments_to_html(val)
            ans[field] = val


def book_as_json(db, book_id):
    db = db.new_api
    with db.safe_read_lock:
        fmts = db._formats(book_id, verify_formats=False)
        ans = []
        fm = {}
        for fmt in fmts:
            m = db.format_metadata(book_id, fmt)
            if m and m.get('size', 0) > 0:
                ans.append(fmt)
                fm[fmt] = m['size']
        ans = {'formats': ans, 'format_sizes': fm}
        if not ans['formats'] and not db.has_id(book_id):
            return None
        fm = db.field_metadata
        for field in fm.all_field_keys():
            if field not in IGNORED_FIELDS:
                add_field(field, db, book_id, ans, fm[field])
        ids = ans.get('identifiers')
        if ids:
            ans['urls_from_identifiers'] = urls_from_identifiers(ids)
        langs = ans.get('languages')
        if langs:
            ans['lang_names'] = {l:calibre_langcode_to_name(l) for l in langs}
    return ans


_include_fields = frozenset(Tag.__slots__) - frozenset({
    'state', 'is_editable', 'is_searchable', 'original_name', 'use_sort_as_name', 'is_hierarchical'
})


def category_as_json(items, category, display_name, count, tooltip=None, parent=None,
        is_editable=True, is_gst=False, is_hierarchical=False, is_searchable=True,
        is_user_category=False, is_first_letter=False):
    ans = {'category': category, 'name': display_name, 'is_category':True, 'count':count}
    if tooltip:
        ans['tooltip'] = tooltip
    if parent:
        ans['parent'] = parent
    if is_editable:
        ans['is_editable'] = True
    if is_gst:
        ans['is_gst'] = True
    if is_hierarchical:
        ans['is_hierarchical'] = is_hierarchical
    if is_searchable:
        ans['is_searchable'] = True
    if is_user_category:
        ans['is_user_category'] = True
    if is_first_letter:
        ans['is_first_letter'] = True
    item_id = 'c' + str(len(items))
    items[item_id] = ans
    return item_id


def category_item_as_json(x, clear_rating=False):
    ans = {}
    for k in _include_fields:
        val = getattr(x, k)
        if val is not None:
            if k == 'original_categories':
                val = tuple(val)
            ans[k] = val.copy() if isinstance(val, set) else val
    s = ans.get('sort')
    if x.use_sort_as_name:
        ans['name'] = s
    if x.original_name != ans['name']:
        ans['original_name'] = x.original_name
    if x.use_sort_as_name or not s or s == ans['name']:
        ans.pop('sort', None)
    if clear_rating:
        del ans['avg_rating']
    return ans


CategoriesSettings = namedtuple(
    'CategoriesSettings', 'dont_collapse collapse_model collapse_at sort_by'
    ' template using_hierarchy grouped_search_terms hidden_categories hide_empty_categories')


class GroupedSearchTerms:

    __slots__ = ('keys', 'vals', 'hash')

    def __init__(self, src):
        self.keys = frozenset(src)
        self.hash = hash(self.keys)
        # We dont need to store values since this is used as part of a key for
        # a cache and if the values have changed the cache will be invalidated
        # for other reasons anyway (last_modified() will have changed on the
        # db)

    def __contains__(self, val):
        return val in self.keys

    def __hash__(self):
        return self.hash

    def __eq__(self, other):
        try:
            return self.keys == other.keys
        except AttributeError:
            return False


_icon_map = None
_icon_map_lock = Lock()


def icon_map():
    global _icon_map
    with _icon_map_lock:
        if _icon_map is None:
            from calibre.gui2 import gprefs
            _icon_map = category_icon_map.copy()
            custom_icons = gprefs.get('tags_browser_category_icons', {})
            for k, v in iteritems(custom_icons):
                if os.access(os.path.join(config_dir, 'tb_icons', v), os.R_OK):
                    _icon_map[k] = '_' + quote(v)
            _icon_map['file_type_icons'] = {
                k:'mimetypes/%s.png' % v for k, v in iteritems(EXT_MAP)
            }
        return _icon_map


def categories_settings(query, db, gst_container=GroupedSearchTerms):
    dont_collapse = frozenset(query.get('dont_collapse', '').split(','))
    partition_method = query.get('partition_method', 'first letter')
    if partition_method not in {'first letter', 'disable', 'partition'}:
        partition_method = 'first letter'
    try:
        collapse_at = max(0, int(float(query.get('collapse_at', 25))))
    except Exception:
        collapse_at = 25
    sort_by = query.get('sort_tags_by', 'name')
    if sort_by not in {'name', 'popularity', 'rating'}:
        sort_by = 'name'
    collapse_model = partition_method if collapse_at else 'disable'
    template = None
    if collapse_model != 'disable':
        if sort_by != 'name':
            collapse_model = 'partition'
        template = tweaks['categories_collapsed_%s_template' % sort_by]
    using_hierarchy = frozenset(db.pref('categories_using_hierarchy', []))
    hidden_categories = frozenset(db.pref('tag_browser_hidden_categories', set()))
    return CategoriesSettings(
        dont_collapse, collapse_model, collapse_at, sort_by, template,
        using_hierarchy, gst_container(db.pref('grouped_search_terms', {})),
        hidden_categories, query.get('hide_empty_categories') == 'yes')


def create_toplevel_tree(category_data, items, field_metadata, opts):
    # Create the basic tree, containing all top level categories , user
    # categories and grouped search terms
    last_category_node, category_node_map, root = None, {}, {'id':None, 'children':[]}
    node_id_map = {}
    category_nodes, recount_nodes = [], []
    order = tweaks['tag_browser_category_order']
    defvalue = order.get('*', 100)
    categories = [category for category in field_metadata if category in category_data]
    scats = sorted(categories, key=lambda x: order.get(x, defvalue))

    for category in scats:
        is_user_category = category.startswith('@')
        is_gst, tooltip = (is_user_category and category[1:] in opts.grouped_search_terms), ''
        cdata = category_data[category]
        if is_gst:
            tooltip = _('The grouped search term name is "{0}"').format(category)
        elif category != 'news':
            cust_desc = ''
            fm = field_metadata[category]
            if fm['is_custom']:
                cust_desc = fm['display'].get('description', '')
                if cust_desc:
                    cust_desc = '\n' + _('Description:') + ' ' + cust_desc
            tooltip = _('The lookup/search name is "{0}"{1}').format(category, cust_desc)

        if is_user_category:
            path_parts = category.split('.')
            path = ''
            last_category_node = None
            current_root = root
            for i, p in enumerate(path_parts):
                path += p
                if path not in category_node_map:
                    last_category_node = category_as_json(
                        items, path, (p[1:] if i == 0 else p), len(cdata),
                        parent=last_category_node, tooltip=tooltip,
                        is_gst=is_gst, is_editable=((not is_gst) and (i == (len(path_parts)-1))),
                        is_hierarchical=False if is_gst else 5, is_user_category=True
                    )
                    node_id_map[last_category_node] = category_node_map[path] = node = {'id':last_category_node, 'children':[]}
                    category_nodes.append(last_category_node)
                    recount_nodes.append(node)
                    current_root['children'].append(node)
                    current_root = node
                else:
                    current_root = category_node_map[path]
                    last_category_node = current_root['id']
                path += '.'
        else:
            last_category_node = category_as_json(
                items, category, field_metadata[category]['name'], len(cdata),
                tooltip=tooltip
            )
            category_node_map[category] = node_id_map[last_category_node] = node = {'id':last_category_node, 'children':[]}
            root['children'].append(node)
            category_nodes.append(last_category_node)
            recount_nodes.append(node)

    return root, node_id_map, category_nodes, recount_nodes


def build_first_letter_list(category_items):
    # Build a list of 'equal' first letters by noticing changes
    # in ICU's 'ordinal' for the first letter. In this case, the
    # first letter can actually be more than one letter long.
    cl_list = [None] * len(category_items)
    last_ordnum = 0
    last_c = ' '
    for idx, tag in enumerate(category_items):
        if not tag.sort:
            c = ' '
        else:
            c = icu_upper(tag.sort)
        ordnum, ordlen = collation_order_for_partitioning(c)
        if last_ordnum != ordnum:
            last_c = c[0:ordlen]
            last_ordnum = ordnum
        cl_list[idx] = last_c
    return cl_list


categories_with_ratings = {'authors', 'series', 'publisher', 'tags'}


def get_name_components(name):
    components = list(filter(None, [t.strip() for t in name.split('.')]))
    if not components or '.'.join(components) != name:
        components = [name]
    return components


def collapse_partition(collapse_nodes, items, category_node, idx, tag, opts, top_level_component,
    cat_len, category_is_hierarchical, category_items, eval_formatter, is_gst,
    last_idx, node_parent):
    # Only partition at the top level. This means that we must not do a break
    # until the outermost component changes.
    if idx >= last_idx + opts.collapse_at and not tag.original_name.startswith(top_level_component+'.'):
        last = idx + opts.collapse_at - 1 if cat_len > idx + opts.collapse_at else cat_len - 1
        if category_is_hierarchical:
            ct = copy(category_items[last])
            components = get_name_components(ct.original_name)
            ct.sort = ct.name = components[0]
            # Do the first node after the last node so that the components
            # array contains the right values to be used later
            ct2 = copy(tag)
            components = get_name_components(ct2.original_name)
            ct2.sort = ct2.name = components[0]
            format_data = {'last': ct, 'first':ct2}
        else:
            format_data = {'first': tag, 'last': category_items[last]}

        name = eval_formatter.safe_format(opts.template, format_data, '##TAG_VIEW##', None)
        if not name.startswith('##TAG_VIEW##'):
            # Formatter succeeded
            node_id = category_as_json(
                items, items[category_node['id']]['category'], name, 0,
                parent=category_node['id'], is_editable=False, is_gst=is_gst,
                is_hierarchical=category_is_hierarchical, is_searchable=False)
            node_parent = {'id':node_id, 'children':[]}
            collapse_nodes.append(node_parent)
            category_node['children'].append(node_parent)
        last_idx = idx  # remember where we last partitioned
    return last_idx, node_parent


def collapse_first_letter(collapse_nodes, items, category_node, cl_list, idx, is_gst, category_is_hierarchical, collapse_letter, node_parent):
    cl = cl_list[idx]
    if cl != collapse_letter:
        collapse_letter = cl
        node_id = category_as_json(
            items, items[category_node['id']]['category'], collapse_letter, 0,
            parent=category_node['id'], is_editable=False, is_gst=is_gst,
            is_hierarchical=category_is_hierarchical, is_first_letter=True)
        node_parent = {'id':node_id, 'children':[]}
        category_node['children'].append(node_parent)
        collapse_nodes.append(node_parent)
    return collapse_letter, node_parent


def process_category_node(
        category_node, items, category_data, eval_formatter, field_metadata,
        opts, tag_map, hierarchical_tags, node_to_tag_map, collapse_nodes,
        intermediate_nodes, hierarchical_items):
    category = items[category_node['id']]['category']
    category_items = category_data[category]
    cat_len = len(category_items)
    if cat_len <= 0:
        return
    collapse_letter = None
    is_gst = items[category_node['id']].get('is_gst', False)
    collapse_model = 'disable' if category in opts.dont_collapse else opts.collapse_model
    fm = field_metadata[category]
    category_child_map = {}
    is_user_category = fm['kind'] == 'user' and not is_gst
    top_level_component = 'z' + category_items[0].original_name
    last_idx = -opts.collapse_at
    category_is_hierarchical = (
        category in opts.using_hierarchy and opts.sort_by == 'name' and
        category not in {'authors', 'publisher', 'news', 'formats', 'rating'}
    )
    clear_rating = category not in categories_with_ratings and not fm['is_custom'] and not fm['kind'] == 'user'
    collapsible = collapse_model != 'disable' and cat_len > opts.collapse_at
    partitioned = collapse_model == 'partition'
    cl_list = build_first_letter_list(category_items) if collapsible and collapse_model == 'first letter' else ()
    node_parent = category_node

    def create_tag_node(tag, parent):
        # User categories contain references to items in other categories, so
        # reflect that in the node structure as well.
        node_data = tag_map.get(id(tag), None)
        if node_data is None:
            node_id = 'n%d' % len(tag_map)
            node_data = items[node_id] = category_item_as_json(tag, clear_rating=clear_rating)
            tag_map[id(tag)] = (node_id, node_data)
            node_to_tag_map[node_id] = tag
        else:
            node_id, node_data = node_data
        node = {'id':node_id, 'children':[]}
        parent['children'].append(node)
        return node, node_data

    for idx, tag in enumerate(category_items):

        if collapsible:
            if partitioned:
                last_idx, node_parent = collapse_partition(
                collapse_nodes, items, category_node, idx, tag, opts, top_level_component,
                cat_len, category_is_hierarchical, category_items,
                eval_formatter, is_gst, last_idx, node_parent)
            else:  # by 'first letter'
                collapse_letter, node_parent = collapse_first_letter(
                    collapse_nodes, items, category_node, cl_list, idx, is_gst, category_is_hierarchical, collapse_letter, node_parent)
        else:
            node_parent = category_node

        tag_is_hierarchical = id(tag) in hierarchical_tags
        components = get_name_components(tag.original_name) if category_is_hierarchical or tag_is_hierarchical else (tag.original_name,)

        if not tag_is_hierarchical and (
                is_user_category or not category_is_hierarchical or len(components) == 1 or
                (fm['is_custom'] and fm['display'].get('is_names', False))
        ):  # A non-hierarchical leaf item in a non-hierarchical category
            node, item = create_tag_node(tag, node_parent)
            category_child_map[item['name'], item['category']] = node
            intermediate_nodes[tag.category, tag.original_name] = node
        else:
            orig_node_parent = node_parent
            for i, component in enumerate(components):
                if i == 0:
                    child_map = category_child_map
                else:
                    child_map = {}
                    for sibling in node_parent['children']:
                        item = items[sibling['id']]
                        if not item.get('is_category', False):
                            child_map[item['name'], item['category']] = sibling
                cm_key = component, tag.category
                if cm_key in child_map:
                    node_parent = child_map[cm_key]
                    node_id = node_parent['id']
                    item = items[node_id]
                    item['is_hierarchical'] = 3 if tag.category == 'search' else 5
                    if tag.id_set is not None:
                        item['id_set'] |= tag.id_set
                    hierarchical_items.add(node_id)
                    hierarchical_tags.add(id(node_to_tag_map[node_parent['id']]))
                else:
                    if i < len(components) - 1:  # Non-leaf node
                        original_name = '.'.join(components[:i+1])
                        inode = intermediate_nodes.get((tag.category, original_name), None)
                        if inode is None:
                            t = copy(tag)
                            t.original_name, t.count = original_name, 0
                            t.is_editable, t.is_searchable = False, category == 'search'
                            node_parent, item = create_tag_node(t, node_parent)
                            hierarchical_tags.add(id(t))
                            intermediate_nodes[tag.category, original_name] = node_parent
                        else:
                            item = items[inode['id']]
                            ch = node_parent['children']
                            node_parent = {'id':inode['id'], 'children':[]}
                            ch.append(node_parent)
                    else:
                        node_parent, item = create_tag_node(tag, node_parent)
                        if not is_user_category:
                            item['original_name'] = tag.name
                        intermediate_nodes[tag.category, tag.original_name] = node_parent
                    item['name'] = component
                    item['is_hierarchical'] = 3 if tag.category == 'search' else 5
                    hierarchical_tags.add(id(tag))
                    child_map[cm_key] = node_parent
                items[node_parent['id']]['id_set'] |= tag.id_set
            node_parent = orig_node_parent


def iternode_descendants(node):
    for child in node['children']:
        yield child
        yield from iternode_descendants(child)


def fillout_tree(root, items, node_id_map, category_nodes, category_data, field_metadata, opts, book_rating_map):
    eval_formatter = EvalFormatter()
    tag_map, hierarchical_tags, node_to_tag_map = {}, set(), {}
    first, later, collapse_nodes, intermediate_nodes, hierarchical_items = [], [], [], {}, set()
    # User categories have to be processed after normal categories as they can
    # reference hierarchical nodes that were created only during processing of
    # normal categories
    for category_node_id in category_nodes:
        cnode = items[category_node_id]
        coll = later if cnode.get('is_user_category', False) else first
        coll.append(node_id_map[category_node_id])

    for coll in (first, later):
        for cnode in coll:
            process_category_node(
                cnode, items, category_data, eval_formatter, field_metadata,
                opts, tag_map, hierarchical_tags, node_to_tag_map,
                collapse_nodes, intermediate_nodes, hierarchical_items)

    # Do not store id_set in the tag items as it is a lot of data, with not
    # much use. Instead only update the ratings and counts based on id_set
    for item_id in hierarchical_items:
        item = items[item_id]
        total = count = 0
        for book_id in item['id_set']:
            rating = book_rating_map.get(book_id, 0)
            if rating:
                total += rating/2.0
                count += 1
        item['avg_rating'] = float(total)/count if count else 0

    for item_id, item in itervalues(tag_map):
        id_len = len(item.pop('id_set', ()))
        if id_len:
            item['count'] = id_len

    for node in collapse_nodes:
        item = items[node['id']]
        item['count'] = sum(1 for _ in iternode_descendants(node))


def render_categories(opts, db, category_data):
    items = {}
    with db.safe_read_lock:
        root, node_id_map, category_nodes, recount_nodes = create_toplevel_tree(category_data, items, db.field_metadata, opts)
        fillout_tree(root, items, node_id_map, category_nodes, category_data, db.field_metadata, opts, db.fields['rating'].book_value_map)
    for node in recount_nodes:
        item = items[node['id']]
        item['count'] = sum(1 for x in iternode_descendants(node) if not items[x['id']].get('is_category', False))
    if opts.hidden_categories:
        # We have to remove hidden categories after all processing is done as
        # items from a hidden category could be in a user category
        root['children'] = list(filter((lambda child:items[child['id']]['category'] not in opts.hidden_categories), root['children']))
    if opts.hide_empty_categories:
        root['children'] = list(filter((lambda child:items[child['id']]['count'] > 0), root['children']))
    return {'root':root, 'item_map': items}


def categories_as_json(ctx, rd, db, opts, vl):
    return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts), vl=vl)

# Test tag browser {{{


def dump_categories_tree(data):
    root, items = data['root'], data['item_map']
    ans, indent = [], '  '

    def dump_node(node, level=0):
        item = items[node['id']]
        rating = item.get('avg_rating', None) or 0
        if rating:
            rating = ',rating=%.1f' % rating
        try:
            ans.append(indent*level + item['name'] + ' [count={}{}]'.format(item['count'], rating or ''))
        except KeyError:
            print(item)
            raise
        for child in node['children']:
            dump_node(child, level+1)
        if level == 0:
            ans.append('')
    [dump_node(c) for c in root['children']]
    return '\n'.join(ans)


def dump_tags_model(m):
    from qt.core import QModelIndex, Qt
    ans, indent = [], '  '

    def dump_node(index, level=-1):
        if level > -1:
            ans.append(indent*level + index.data(Qt.ItemDataRole.UserRole).dump_data())
        for i in range(m.rowCount(index)):
            dump_node(m.index(i, 0, index), level + 1)
        if level == 0:
            ans.append('')
    dump_node(QModelIndex())
    return '\n'.join(ans)


def test_tag_browser(library_path=None):
    ' Compare output of server and GUI tag browsers '
    from calibre.library import db
    olddb = db(library_path)
    db = olddb.new_api
    opts = categories_settings({}, db)
    # opts = opts._replace(hidden_categories={'publisher'})
    opts = opts._replace(hide_empty_categories=True)
    category_data = db.get_categories(sort=opts.sort_by, first_letter_sort=opts.collapse_model == 'first letter')
    data = render_categories(opts, db, category_data)
    srv_data = dump_categories_tree(data)
    from calibre.gui2 import Application, gprefs
    from calibre.gui2.tag_browser.model import TagsModel
    prefs = {
        'tags_browser_category_icons':gprefs['tags_browser_category_icons'],
        'tags_browser_collapse_at':opts.collapse_at,
        'tags_browser_partition_method': opts.collapse_model,
        'tag_browser_dont_collapse': opts.dont_collapse,
        'tag_browser_hide_empty_categories': opts.hide_empty_categories,
    }
    app = Application([])
    m = TagsModel(None, prefs)
    m.set_database(olddb, opts.hidden_categories)
    m_data = dump_tags_model(m)
    if m_data == srv_data:
        print('No differences found in the two Tag Browser implementations')
        raise SystemExit(0)
    from calibre.gui2.tweak_book.diff.main import Diff
    d = Diff(show_as_window=True)
    d.string_diff(m_data, srv_data, left_name='GUI', right_name='server')
    d.exec()
    del app
# }}}
