#!/usr/bin/env python
"""
 Created by howie.hu.
"""

import re
import aiohttp
import async_timeout

from bs4 import BeautifulSoup
from aiocache.serializers import PickleSerializer, JsonSerializer

from urllib.parse import urlparse, parse_qs, urljoin

from soulbook.database.mongodb import MotorBase
from soulbook.fetcher.decorators import cached
from soulbook.fetcher.function import target_fetch, get_time, get_html_by_requests, \
    get_random_user_agent
from soulbook.fetcher.extract_novels import extract_pre_next_chapter
from soulbook.config import RULES, LATEST_RULES, LOGGER

list_class_content = ("content", "content_read", "txt")
list_id_content = ("chaptercontent", "content", "txt")


@cached(ttl=300, key_from_attr='url', serializer=PickleSerializer(), namespace="main")
async def cache_owllook_novels_content(url, chapter_url, netloc):
    headers = {
        'user-agent': await get_random_user_agent()
    }
    html = await target_fetch(headers=headers, url=url)
    if not html:
        html = get_html_by_requests(url=url, headers=headers)
    if html:
        soup = BeautifulSoup(html, 'html5lib')
        # 章节内容
        content = None
        if RULES.get(netloc):
            selector = RULES[netloc].content_selector
            if selector.get('id', None):
                content = soup.find_all(id_=selector['id'])
            elif selector.get('class', None):
                content = soup.find_all(class_=selector['class'])
            else:
                content = soup.find_all(selector.get('tag'))
        if not content:
            # print("novels_content: body")
            # content = soup.find_all("body")

            # for i in list_class_content:
            #     content = soup.find_all(class_=i)
            #     if content:
            #         break
            # else:
            #     for i in list_id_content:
            #         content = soup.find_all(id=i)
            #         if content:
            #             break

            for i in list_id_content:
                content = soup.find_all(id=i)
                if content:
                    break
            else:
                for i in list_class_content:
                    content = soup.find_all(class_=i)
                    if content:
                        break

        if not content:
            # 直接提取整个版块块
            content = soup.find_all("body")
        if content:
            # 提取出真正的章节标题
            title_reg = r'(第?\s*[一二两三四五六七八九十○零百千万亿0-9１２３４５６７８９０]{1,6}\s*[章回卷节折篇幕集]\s*.*?)[_,-]'
            title = soup.title.string
            extract_title = re.findall(title_reg, title, re.I)
            if extract_title:
                title = extract_title[0]
            else:
                try:
                    title = soup.select('h1')[0].get_text()
                except Exception as e:
                    print(e)
            if not title:
                title = soup.title.string
            # if "_" in title:
            #     title = title.split('_')[0]
            # elif "-" in title:
            #     title = title.split('-')[0]
            next_chapter = extract_pre_next_chapter(url=url, chapter_url=chapter_url,
                                                    html=str(soup))
            content = [str(i) for i in content]
            data = {
                'content': str(''.join(content)),
                'next_chapter': next_chapter,
                'title': title
            }
        else:
            data = None
        return data
    return None


ListIdTab = ("box_con", 'box_chap', 'layout layout-col1', 'listmain', "content", "list-chapterAll")
ListClassTab = (
    "box_con", 'box_chap', 'layout layout-col1', 'listmain', 'book chapterlist', 'book_list',
    'TabCss', 'inner', 'mulu', 'bg')


@cached(ttl=300, key_from_attr='url', serializer=PickleSerializer(), namespace="main")
async def cache_owllook_novels_chapter(url, netloc):
    headers = {
        'user-agent': await get_random_user_agent()
    }
    html = await target_fetch(headers=headers, url=url)
    if not html:
        html = get_html_by_requests(url=url, headers=headers)
    if html:
        soup = BeautifulSoup(html, 'html5lib')
        # 提取出的内容
        content = None
        # 存在解析规则
        if RULES.get(netloc):
            # 匹配解析规则
            selector = RULES[netloc].chapter_selector
            if selector.get('id', None):
                content = soup.find_all(id=selector['id'])
            elif selector.get('class', None):
                content = soup.find_all(class_=selector['class'])
            else:
                content = soup.find_all(selector.get('tag'))
        else:
            data = str(html)
            for i in ListIdTab:
                if 'id="%s"' % i in data:
                    content = soup.find_all(id=i)
                    break
            for i in ListClassTab:
                if 'class="%s"' % i in data:
                    content = soup.find_all(class_=i)
                    break
        if not content:
            # 直接提取整个版块块
            content = soup.find_all("body")
        if content:
            # 防止章节被display:none
            return str(content).replace('style', '') if content else None
    return None


@cached(ttl=10800, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking")
async def cache_owllook_search_ranking():
    motor_db = MotorBase().get_db()
    keyword_cursor = motor_db.search_records.find(
        {'count': {'$gte': 50}},
        {'keyword': 1, 'count': 1, '_id': 0}
    ).sort('count', -1).limit(35)
    result = []
    index = 1
    async for document in keyword_cursor:
        result.append({'keyword': document['keyword'], 'count': document['count'], 'index': index})
        index += 1
    return result


@cached(ttl=3600, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking")
async def cache_others_search_ranking(spider='qidian', novel_type='全部类别'):
    motor_db = MotorBase().get_db()
    item_data = await motor_db.novels_ranking.find_one({'spider': spider, 'type': novel_type},
                                                       {'data': 1, '_id': 0})
    return item_data


async def get_the_latest_chapter(chapter_url, timeout=15):
    try:
        with async_timeout.timeout(timeout):
            url = parse_qs(urlparse(chapter_url).query).get('url', '')
            novels_name = parse_qs(urlparse(chapter_url).query).get('novels_name', '')
            data = None
            if url and novels_name:
                url = url[0]
                novels_name = novels_name[0]
                netloc = urlparse(url).netloc
                if netloc in LATEST_RULES.keys():
                    headers = {
                        'user-agent': await get_random_user_agent()
                    }
                    try:
                        html = await target_fetch(url=url, headers=headers, timeout=timeout)
                        if html is None:
                            html = get_html_by_requests(url=url, headers=headers, timeout=timeout)
                    except TypeError:
                        html = get_html_by_requests(url=url, headers=headers, timeout=timeout)
                    except Exception as e:
                        LOGGER.exception(e)
                        return None
                    try:
                        soup = BeautifulSoup(html, 'html5lib')
                    except Exception as e:
                        LOGGER.exception(e)
                        return None
                    latest_chapter_name, latest_chapter_url = None, None
                    if LATEST_RULES[netloc].plan:
                        meta_value = LATEST_RULES[netloc].meta_value
                        latest_chapter_name = soup.select(
                            'meta[property="{0}"]'.format(
                                meta_value["latest_chapter_name"])) or soup.select(
                            'meta[name="{0}"]'.format(meta_value["latest_chapter_name"]))

                        latest_chapter_name = latest_chapter_name[0].get('content',
                                                                         None) if latest_chapter_name else None
                        latest_chapter_url = soup.select(
                            'meta[property="{0}"]'.format(
                                meta_value["latest_chapter_url"])) or soup.select(
                            'meta[name="{0}"]'.format(meta_value["latest_chapter_url"]))
                        latest_chapter_url = urljoin(chapter_url,
                                                     latest_chapter_url[0].get('content',
                                                                               None)) if latest_chapter_url else None
                    else:
                        selector = LATEST_RULES[netloc].selector
                        content_url = selector.get('content_url')
                        if selector.get('id', None):
                            latest_chapter_soup = soup.find_all(id=selector['id'])
                        elif selector.get('class', None):
                            latest_chapter_soup = soup.find_all(class_=selector['class'])
                        else:
                            latest_chapter_soup = soup.select(selector.get('tag'))
                        if latest_chapter_soup:
                            if content_url == '1':
                                # TODO
                                pass
                            elif content_url == '0':
                                # TODO
                                pass
                            else:
                                latest_chapter_url = content_url + latest_chapter_soup[0].get(
                                    'href', None)
                            latest_chapter_name = latest_chapter_soup[0].get('title', None)
                    if latest_chapter_name and latest_chapter_url:
                        time_current = get_time()
                        # print(latest_chapter_url)
                        data = {
                            "latest_chapter_name": latest_chapter_name,
                            "latest_chapter_url": latest_chapter_url,
                            "owllook_chapter_url": chapter_url,
                            "owllook_content_url": "/owllook_content?url={latest_chapter_url}&name={name}&chapter_url={chapter_url}&novels_name={novels_name}".format(
                                latest_chapter_url=latest_chapter_url,
                                name=latest_chapter_name,
                                chapter_url=url,
                                novels_name=novels_name,
                            ),
                        }
                        # 存储最新章节
                        motor_db = MotorBase().get_db()
                        await motor_db.latest_chapter.update_one(
                            {"novels_name": novels_name, 'owllook_chapter_url': chapter_url},
                            {'$set': {'data': data, "finished_at": time_current}}, upsert=True)
            return data
    except Exception as e:
        LOGGER.exception(e)
        return None


async def update_all_books(loop, timeout=15):
    try:
        motor_db = MotorBase().get_db()
        # 获取所有书架链接游标
        books_url_cursor = motor_db.user_message.find({}, {'books_url.book_url': 1, '_id': 0})
        book_urls = []
        already_urls = set()
        async for document in books_url_cursor:
            if document:
                books_url = document['books_url']

                for book_url in books_url:
                    chapter_url = book_url['book_url']
                    if chapter_url not in already_urls:
                        try:
                            await get_the_latest_chapter(chapter_url, timeout)
                        except Exception as e:
                            LOGGER.exception(e)
                        already_urls.add(chapter_url)
                        # 一组书架链接列表数据
                        #         book_urls += [book_url['book_url'] for book_url in books_url]
                        # url_tasks = [get_the_latest_chapter(each_url, loop) for each_url in set(book_urls)]
                        # tasks = [asyncio.ensure_future(i) for i in url_tasks]
                        # try:
                        #     await asyncio.gather(*tasks)
                        # except asyncio.TimeoutError as e:
                        #     pass
    except Exception as e:
        LOGGER.exception(e)
        return False