#!/usr/bin/env python3

import requests
import logging
import xml.etree.ElementTree as ET
import json
import unidecode
import sqlite3
import common
import traceback
import sys

from bs4 import BeautifulSoup
from datetime import datetime
from kafka import KafkaProducer


def get_categories(url):
    page = requests.get(url)
    root = ET.fromstring(page.content)

    categories = {}
    
    for cat_elm in root.findall("./Data/CategoryNavigationViewModel"):
        cat_label = cat_elm.find("Label")
        cat_url = cat_elm.find("Url")
        categories[cat_label.text] = cat_url.text

    return categories


def get_products(config, cat_url, kafka_producer):
    url = config['action']['products_url']
    url = url.replace("__url__", cat_url)
    url = url.replace("__offset__", "0")
    total_url = url.replace("__limit__", "1")
    products_cookie = config['action']['products_cookie']

    get_total = requests.get(total_url, cookies=products_cookie)
    data_json = json.loads(unidecode.unidecode(get_total.content.decode('utf-8', 'ignore')))
    totalCount = data_json['totalCount']

    all_url = url.replace("__limit__", str(totalCount))
    products = requests.get(all_url, cookies=products_cookie)
    products_json = json.loads(unidecode.unidecode(products.content.decode('utf-8', 'ignore')))

    for product_json in products_json['items']:
        try:
            if product_json['type'] != 'product':
                continue
            product = {}
            price = {}
            product['title'] = product_json['title']
            product['url'] = config['action']['url'] + product_json['url']
            product['image_url'] = config['action']['url'] + product_json['imageUrl']
            product['info'] = product_json['subTitle']
            product['sku_code'] = product_json['code']
            for spec in product_json['specifications']:
                if spec['id'] == 'attEANCodeVariant':
                    product['ean_code'] = spec['value']
                if spec['id'] == 'attLongDescription':
                    product['description'] = spec['value']
            product['brand'] = product_json['brandName']
            price['price'] = product_json['price']
            price['promo'] = 0 if product_json['isDeal'] == False else 1
            price['promo_start'] = product_json['dealStartDate']
            price['promo_end'] = product_json['dealEndDate']

            common.add_product(kafka_producer, config['action']['name'], product, price)
        except Exception as err:
            common.dump_failed_product(config['action']['name'], cat_url, product_json, err, traceback.format_exc())


if __name__ == "__main__" :
    config = common.get_config()

    producer = KafkaProducer(bootstrap_servers=[config['config']['kafka_boostrap_servers']])
    common.insert_update_store(producer, {'store': config['action']['name'], 'url': config['action']['url'], 'image_url': config['action']['logo']})
    common.clear_failed_product(config['action']['name'])
    
    categories = get_categories(config['action']['categories_url'])

    for cat in categories:
        soup = common.get_soup_page_no_limit(f"{config['action']['url']}{categories[cat]}")
        subcat = soup.find_all("a", class_="subcategory-cta-list__cta")
        for sc in subcat:
            get_products(config, sc['href'], producer)

    common.update_store_prices(producer, config['action'])