import urllib.parse
import datetime
import json
import os
import random
import sys
import time
import pickle
import re

from bs4 import BeautifulSoup
from utilities.BaseParser import open_url_engine


def getPricePages(VPROKURL):
    # global rs
    # rs.get(VPROKURL)
    # time.sleep(6)
    # htmlSource = rs.driver.page_source
    # rs.driver.close()
    with open_url_engine(engine_type='selenium', headless=True) as url_engine:
        # soup = BeautifulSoup(htmlSource, "html.parser")
        soup = url_engine.get_soup(VPROKURL)
        time.sleep(4)
        soup = BeautifulSoup(url_engine.browser.get_page_source())
        mmenu = soup.select("a.xf-catalog-categories__link")
        sections = []
        for m in mmenu:
            sections.append(urllib.parse.urljoin(VPROKURL, m['href']))
            print(m['href'])
    return sections


def scrapePrices(VPROKURL, secURL, url_engine):
    # collector for prices
    # rs = RobustSelenium(7)
    products = {}
    page = 1
    psoup = url_engine.get_soup(urllib.parse.urljoin(secURL, f'?page={page}&sort=rate_desc'))
    time.sleep(8)
    while True:
        # rs.get(urllib.parse.urljoin(secURL, f'?page={page}&sort=rate_desc'))
        # htmlp = rs.driver.page_source
        # psoup = BeautifulSoup(htmlp, "html.parser")
        psoup = url_engine.get_soup(urllib.parse.urljoin(secURL, f'?page={page}&sort=rate_desc'))
        time.sleep(1)
        psoup = BeautifulSoup(url_engine.browser.get_page_source())
        cards = psoup.select('.js-catalog-product')
        newcards = {}
        if cards:
            for c in cards:
                url = c.select_one('.xf-product-title > a')['href']
                url = urllib.parse.urljoin(VPROKURL, url[1:])
                pattern = '(https://www.vprok.ru/product/)(.+)(--)([0-9]+)'
                res = list(re.findall(pattern, url)[0])
                url = res[0] + 'product--' + res[-1]
                if url not in products.keys():
                    try:  # 44211
                        newcards[url] = parseCard(c, url)
                    except ValueError as e:
                        with open('44211_psoup.pkl', 'w') as wrong_price_file:  # 44211
                            pickle.dump(psoup, wrong_price_file)  # 44211
                        with open('wrong_url', 'w') as wrong_url_file:  # 44211
                            wrong_url_file.write(url)  # 44211
                        print(f'44211 found in {url}', file=sys.stderr)  # 44211
            if newcards:
                products.update(newcards)
                page += 1
                print(f'page {page} ' + secURL)
            else:
                break
        else:
            break
    # rs.driver.close()
    return products

def parseCard(card, url):
    parsed = {}
    parsed['DateTimeObserved'] = str(datetime.datetime.now())
    parsed['URL'] = url
    # Check for out of stock status
    stockStatus = card.select_one('.xf-product__out')
    if stockStatus:
        parsed['Status'] = 'OutOfStock'
        parsed['CurrentPrice'] = None
        parsed['CrossedPrice'] = None
    else:
        parsed['Status'] = 'InStock'
        currentprice = card.select_one('.xf-product-cost__current')
        parsed['CurrentPrice'] = float(currentprice['data-cost'])
        crossedprice = card.select_one('.xf-product-cost__prev')
        try:
            if crossedprice and float(crossedprice['data-cost']) > 0.0:
                parsed['CrossedPrice'] = float(crossedprice['data-cost'])
        except:
            parsed['CrossedPrice'] = None
        try:
            parsed['ProductName'] = card.select_one('.xf-product-title > a')['title']
        except:
            print('some error was occurred during obtaining productName, url is:\n', url, file=sys.stderr)
            pass

    if parsed['CurrentPrice'] == 44211:  # 44211
        raise ValueError('Odd value in CurrentPrice, 44211')  # 44211
    return parsed

def main():
    """
    Main function.
    """
    # walk original subset of link and find price lists
    # but not every time, but say 5% of the time

    timeexec = datetime.datetime.now().strftime('%Y%m%d%H')
    print(timeexec)

    PRICEDIR = os.path.join(os.path.dirname(__file__), 'json', os.path.basename(__file__).replace('.py', ''))
    PRICEDIR = os.path.join(PRICEDIR, timeexec)
    os.makedirs(PRICEDIR, exist_ok=True)

    DATAFILE = os.path.join(os.path.dirname(__file__), os.path.basename(__file__).replace('.py', '.data'))

    VPROKURL = 'https://www.vprok.ru/catalog'

    r = random.uniform(0, 1)
    if r > 0.2 or not os.path.exists(DATAFILE):
        pricePages = getPricePages(VPROKURL)
        with open(DATAFILE, 'wb') as f:
            pickle.dump(pricePages, f)
    # load price pages 
    with open(DATAFILE, 'rb') as f:
        pricePages = pickle.load(f)
    # scrape price lists
    pricePages = [*set(pricePages)]
    random.shuffle(pricePages)
    url_engine_options = {
        # 'type': 'requests'
        'type': 'selenium'
        , 'max_attempts': 5
        , 'sleep_time_sec': 3
    }
    with open_url_engine(url_engine_options, headless=True) as url_engine:
        for sectionNumber, page in enumerate(pricePages):
            pricePack = scrapePrices(VPROKURL, page, url_engine)
            # and save data
            with open(os.path.join(PRICEDIR, 'sec' + str(sectionNumber) + '.json'), 'w', encoding='utf8') as f:
                json.dump(pricePack, f, indent=1, ensure_ascii=False)


if __name__ == '__main__':
    main()