#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
imageboard-dl

usage: imageboard-dl [-h] [-v] [-s] [-x] [-p PATH] [-d DIRECTORY] URL(s)

@sixem
'''

from bs4 import BeautifulSoup
from os.path import expanduser

import re
import os
import json

from requests import exceptions
from urllib.request import urlopen
from urllib.parse import urlparse

import requests
import shutil
import cfscrape
import argparse
import math
import sys
import mimetypes
import hashlib

try:
    import queue
except ImportError:
    import Queue as queue

import time

import threading
from threading import Thread

imageboard_name = None

debugging = False

# Current Version - D/M/Y
version = "1.0.63 - 2019-09-22"

# General values
general_vars = {
    'cfs_timeout': 30,
    'dir_frmt': '{}-{}',
    'connest_frmt': 'Connection established ..',
    'connest_frmt_ip': 'Connection established ({}:{}) ..'
    }

# Request headers
req_headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0',
    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'en-us,en;q=0.5',
    }

# Regex table used to identify known URLs being passed
sites_regex_table = {
    '4archive': '(https?:\/\/(?:www.)?(?:4archive\.org|randomarchive\.com))\/board\/([A-Za-z]{1,})\/thread\/([0-9]{1,})',
    '4chan': '((https?:\/\/)(?:www.)?boards.(?:4chan|4channel).org\/([A-Za-z]{1,10})\/([A-Za-z]{1,10})\/([0-9]{1,}))',
    '4plebs': '((https?:\/\/)(?:www.)?archive.4plebs.org\/([A-Za-z]{1,10})\/([A-Za-z]{1,10})\/([0-9]{1,}))',
    '7chan.org': '((https?:\/\/)(?:www.)?7chan.org\/([A-Za-z0-9]{1,10})\/([A-Za-z]{1,10})\/([0-9]{1,}).html)',
    '8chan': '((https?:\/\/)(?:www.)?8ch.(?:net|pl)\/([A-Za-z]{1,10})\/([A-Za-z]{1,10})\/([0-9]{1,}).html)',
    'arhivach': '((https?:\/\/)(?:www.)?arhivach.(?:[A-Za-z]{1,})\/[A-Za-z]{1,10}\/([0-9]{1,})\/)',
    'dvach': '((https?:\/\/)(?:www.)?2-?ch.(?:hk|pm|re|tf|wf|yt|so)\/([A-Za-z]{1,10})\/([A-Za-z]{1,10})\/([0-9]{1,}).html)',
    'imgur:album': 'https?:\/\/(?:www.)?(?:m.)?imgur.com\/(a|gallery)\/([0-9A-Za-z]{1,})',
    'warosu.org': '(https?:\/\/(?:www.)?warosu.org\/([A-Za-z0-9]{1,})\/thread/([0-9]{1,}))',
    'foolfuuka': 'https?:\/\/(?:www.)?(archiveofsins\.com|archived?\.moe)\/([A-Za-z0-9]{1,})\/thread\/([0-9]{1,})',
    'sankaku': '((?:https?:\/\/)(?:www.)?(?:chan.)?(?:sankakucomplex).com)\/?(?:.*)?',
    }

# Regex table used for identifying HTML code to sites if the default table (above) returns no matches
adaptive_regex_table = {
    'foolfuuka': [
    '<link rel=[\'"]stylesheet[\'"] type=[\'"]text\/css[\'"] href=[\'"](?:.*)\/foolfuuka\/foolz\/(?:.*)\.css[\'"]>',
    '<script src=[\'"](?:.*)\/foolfuuka\/foolz\/foolfuuka-(?:.*)\.js[\'"]><\/script>'
    ],
    '4chan': [
    'href=\"\/\/ii.yuki\.la\/'
    ],
    'vichan': [
    '<body class=[\'"][A-Za-z0-9]{1,}? vichan [A-Za-z0-9-]{1,}? [A-Za-z0-9-]{1,}?[\'"] data-stylesheet=[\'"]default[\'"]>',
    '<a href=[\'"]https?:\/\/([A-Za-z/.]{1,})?vichan\.net([A-Za-z/.]{1,})?[\'"]>vichan<\/a>',
    '<br>vichan Copyright .{1,10} 20[0-9]{2}-20[0-9]{2} vichan-devel<\/p>'
    ],
}

# Possible reasons for a error code (requests)
possible_reasons = {
    403: ['Are you using a VPN / Proxy?'],
    429 : ['Server limits requests. Try setting a sleep interval.']
    }

# Used for site to function convertion
dict_number_converter = {
    '1': 'one',
    '2': 'two',
    '3': 'three',
    '4': 'four',
    '5': 'five',
    '6': 'six',
    '7': 'seven',
    '8': 'eight',
    '9': 'nine',
    '10': 'ten',
    ':': 'x',
    '.': '_'
    }

# Used for returning results
dict_return_codes = {
    'attempt': 1,
    'skip': 2,
    'download': 3,
    'error': 4
    }

valid_extensions = ['bmp', 'gif', 'jpg', 'jpeg', 'mp4', 'png', 'webm', 'mp3']

def report(site, message):
    print('[{}] {}'.format(site, message))

def print_debug(category, message):
    if debugging == True:
        print('\033[93m[debug:{}] {}\033[0m'.format(category, str(message)))

class download():

    # Default save directory (current working directory)
    save_directory = os.getcwd()

    # Use separate save directory or not
    use_sep_dir = False

    # Available downloaders
    download_type = {
        'generic': 1,
        'cloudflare': 2,
        'content-disposition': 3
        }

    def generic(site, url, destination, name, x='generic'):

        '''Generic Downloader'''

        try:
            req = requests.get(url, headers=req_headers,
                               timeout=general_vars['cfs_timeout'])

            print_debug('request', "URL: {} (Status: {}, Content: {})".format(
                url, req.status_code, req.headers['content-type'])
            )

            try:
                size = utils.sizeof_fmt(int(req.headers['content-length']))
            except:
                size = False

            if req.status_code == 200:

                try:
                    open(destination, 'wb').write(req.content)
                except Exception as e:
                    print_debug('error', str(e))

                if os.path.exists(destination):
                    if size == False:
                        return (dict_return_codes['download'], name)
                    else:
                        return (dict_return_codes['download'], '{} ({})'.format(name, size))

                else: return (dict_return_codes['error'], False)
            else:
                return (dict_return_codes['error'], '{}: {}'.format(req.status_code, name))

        except Exception as e:
            print_debug('error', str(e))
            return (dict_return_codes['error'], False)

    def cloudflare(site, url, destination, name, x='cloudflare'):

        '''Cloudflare Downloader'''

        try:

            cfs = cfscrape.create_scraper()
            req = cfs.get(url, headers=req_headers,
                         timeout=general_vars['cfs_timeout'], stream=True)

            print_debug('request', "URL: {} (Status: {}, Content: {})".format(
                url, req.status_code, req.headers['content-type'])
            )

            try:
                size = utils.sizeof_fmt(int(req.headers['content-length']))
            except:
                size = False

            if req.status_code == 200:
                with open(destination, 'wb') as f:

                    req.raw.decode_content = True
                    shutil.copyfileobj(req.raw, f)

                    if os.path.exists(destination):
                        if size == False:
                            return (dict_return_codes['download'], name)
                        else:
                            return (dict_return_codes['download'], '{} ({})'.format(name, size))

                    else: return (dict_return_codes['error'], False)
            else:
                return (dict_return_codes['error'], '{}: {}'.format(req.status_code, name))

        except Exception as e:
            print_debug('error', str(e))
            return (dict_return_codes['error'], False)

    def contentdisposition(site, url, destination, name, x='content-disposition'):

        '''Content-Disposition (Attachment) Downloader'''

        try:
            req = requests.get(url, headers=req_headers,
                               timeout=general_vars['cfs_timeout'])

            print_debug('request', "URL: {} (Status: {}, Content: {})".format(
                url, req.status_code, req.headers['content-type'])
            )

            try:
                size = utils.sizeof_fmt(int(req.headers['content-length']))
            except:
                size = False

            if req.status_code == 200:

                rm = re.search('attachment; filename="(.*)"', req.headers['Content-Disposition'])

                if rm:
                    try:
                        open(destination, 'wb').write(req.content)
                    except Exception as e:
                        print_debug('error', str(e))

                    if os.path.exists(destination):
                        if size == False:
                            return (dict_return_codes['download'], name)
                        else:
                            return (dict_return_codes['download'], '{} ({})'.format(name, size))

                else: return (dict_return_codes['error'], False)

            else:
                return (dict_return_codes['error'],'{}: {}'.format(req.status_code, name))

        except Exception as e:
            print_debug('error', str(e))
            return (dict_return_codes['error'], False)

class que():
    @staticmethod
    def file(site, uniq, url, filename, downloader):

        values = [site, utils.const_df(site.replace(':', '_'), uniq), url,
                   utils.sanitize_filename(filename), downloader]

        for i in range(0, 5):
            scrapers.box[i].append(values[i])

class utils():
    #https://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
    def sizeof_fmt(num, suffix='B'):

        for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:

            if abs(num) < 1024.0:
                return "%3.1f %s%s" % (num, unit, suffix)
            num /= 1024.0

        return "%.1f %s%s" % (num, 'Yi', suffix)

    def request_code_to_string(code):
        status_codes = requests.status_codes._codes[code]
        if len(status_codes) > 0:
            return (requests.status_codes._codes[code][0].capitalize()).replace('_', ' ')
        else:
            return False

    def getbetween(a, s, e):
        return str(a.split(s)[1]).split(')')[0]

    def get_base_url(url):
        parsed_uri = urlparse(url)
        return '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)

    def get_unique_number_from_url(url):
        hash = hashlib.sha224(url.encode('utf-8')).hexdigest()
        return ''.join(c for c in hash if c.isdigit())[:10]

    def get_simplename_from_url(url):
        x = urlparse(url)
        x = '{uri.scheme}://{uri.netloc}/'.format(uri=x)
        x = x.replace('www.', '')
        x = x.split('//')[1]
        x = x.split('.')[0]

        return x

    def validate_image_url(url):
        for x in valid_extensions:
            if url.lower().endswith(x):
                return True
        return False

    def patch_url(x, a):
        if (x.startswith('/')) and (a.endswith('/')):
            a = a[:-1]
        if x.startswith('/'):
            return a + x
        else:
            return x

    def simplifiy_string(str):
        return str.lower().replace(' ', '_')

    def url_to_digits(str, cutoff=0):

        if str is None or str == '':
            return 0

        out_number = ''

        for ele in str:

            if ele.isdigit():
                out_number += ele

        return out_number[cutoff:]

    def clean_url(a):

        if a.startswith('//'):
            a = 'https:%s' % a

        return a

    def format_count(total, current):

        if len(str(current)) is not len(str(total)):
            current = ('0' * (len(str(total)) - len(str(current)))) + str(current)

        return '{}/{}'.format(current, total)

    def get_filename_from_url(a):

        return a.split('/')[len(a.split('/'))-1]

    def const_df(a, b):

        return general_vars['dir_frmt'].format(a, b)

    def sanitize_filename(fn):

        return "".join([c for c in fn if c.isalpha() or c.isdigit() or c==' ' or c=='.' or c=='_']).rstrip()

    def return_code_to_string(ncode):

        mdict = dict_return_codes
        try:
            return mdict.keys()[mdict.values().index(ncode)]
        except:
            return list(mdict.keys())[list(mdict.values()).index(ncode)]

    def get_json(js):

        try:
            return json.loads(js)
        except:
            raise ErrorParsingJson

    def result_to_string(results):

        out = ''

        if len(results) > 0:

            if results.count(dict_return_codes['skip']) > 0:
                out += ('Downloaded {} file(s), {} file(s) already exists'.format(
                    results.count(dict_return_codes['download']),
                    results.count(dict_return_codes['skip'])))

            else: out += ('Downloaded {} file(s)'.format(results.count
                    (dict_return_codes['download'])))

            if results.count(dict_return_codes['error']) > 0:
                out += (' and encountered {} error(s) when downloading'.format(
                    results.count(dict_return_codes['error'])))

            return out
        else:
            return ('No result were found')

    def re_find(regex, string, group_index):

        rm = re.search(regex, string)

        if rm:
            return rm.group(group_index)
        else:
            return False

class scrapers():

    box = [[], [], [], [], []]

    recent_error = None

    @classmethod
    def request(self, url, use_cookies=None):
        if ibdl.sleep > 0:
            time.sleep(ibdl.sleep)

        cfs = cfscrape.create_scraper()

        request = cfs.get(url, headers = req_headers,
            timeout = general_vars['cfs_timeout'], cookies = use_cookies, stream=True)

        try:
            # Gets the IP of the request
            response_ip = (request.raw._connection.sock.getpeername())
        except:
            response_ip = None

        # See if request is good if not raise an error
        if request.status_code is 200:
            return (request.text, request.cookies, response_ip, request)
        else:
            self.recent_error = request.status_code
            raise ErrorRequest(request.status_code)

    @classmethod
    def request_direct(self, url, use_cookies=None):
        if ibdl.sleep > 0:
            time.sleep(ibdl.sleep)

        cfs = cfscrape.create_scraper()

        request = cfs.get(url, headers = req_headers,
            timeout = general_vars['cfs_timeout'], cookies = use_cookies, stream=True)

        return request

    @classmethod
    def establish(self, a, site, v=True, bs=True, rb=True, ra=False):

        try:
            request = self.request(a)

            if rb is True:
                self.box = [[], [], [], [], []]

            if v is True:
                if request[2] is not None:
                    report(site, general_vars['connest_frmt_ip'].format(request[2][0], request[2][1]))
                else:
                    report(site, general_vars['connest_frmt'])

            if bs is True:
                if ra is True:
                    return (BeautifulSoup(request[0], "html.parser"), request)
                else:
                    return BeautifulSoup(request[0], "html.parser")

            else:
                if ra is True:
                    return request
                else:
                    return request[0]
        except ErrorRequest as exception:
            error_code = int(str(exception))
            status_code = utils.request_code_to_string(error_code)

            if status_code is not False:
                report('error', 'Error requesting page (code: {} - {})'.format(error_code, status_code))
            else:
                report('error', 'Error requesting page (code: {})'.format(error_code))
            return False

    @classmethod
    def download_partial(self, box, verbal=False, custom=False):
        if box == False or box == None or len(box) < 1:
            print_debug('info', 'Partial download received invalid item type.')
            return

        items = ibdl.check_list(self.box, cdir=ibdl.dirn)
        # If custom is true, attempt to continue the download count from previous partial downloads.
        if custom != False:
            custom = ibdl.current + len(items[0])

        ibdl.download_images(items, v=verbal, custom_length=custom)

        self.box = []

    @classmethod
    def foolfuuka(self, a, site="foolfuuka", match=None, uniq=None):

        '''FoolFuuka Downloader'''

        p = self.establish(a, site)

        post_link = [[], []]

        posts = p.findAll("a", {"class" : "post_file_filename"})

        try:
            uniq = p.find("article", {"class" : "thread"})['id'].__str__()
        except:
            uniq = utils.sanitize_filename(a)

        for post in posts:

            if post.has_attr('title'):
                filename = post['title']
            else:
                filename = post.contents[0]

            post_link[0].append(str(post['href']))
            post_link[1].append(filename)

        say_redirect = False

        for x, url in enumerate(post_link[0]):
            if ('/redirect/') in url:
                if say_redirect is False:
                    report('info', 'Collecting redirection data ..'); say_redirect = True

                p = self.establish(url, site, bs=False, rb=False, v=False)
                rm = re.search('content="[0-9]{1,}; url=(.*)">', p)

                if rm:
                    que.file(site, uniq, rm.group(1), post_link[1][x], download.download_type['cloudflare'])
            else:
                que.file(site, uniq, url, post_link[1][x], download.download_type['cloudflare'])

        return self.box

    @classmethod
    def warosu_org(self, a, site="warosu.org", match=None, uniq=None):

        '''warosu.org Downloader'''

        p = self.establish(a, site)

        regex = ('File: [0-9]{1,3} [A-Za-z]{1,3}, [0-9]{1,5}x[0-9]{1,5}, (.*..*)')
        op_post = p.find("div", {"class" : "content"}).find("div")
        uniq = str(op_post['id'])
        url = 'https:' + str(op_post.find("img", {"class" : "thumb"}).parent['href'])
        fn_search = re.search(regex, str(op_post.find("span").contents[0]))

        if fn_search:
            filename = fn_search.group(1)
        else:
            filename = utils.get_filename_from_url(url)

        que.file(site, uniq, url, filename, download.download_type['cloudflare'])

        replies = p.findAll("td", {"class" : "reply"})

        for reply in replies:
            try:
                url = 'https:' + str(reply.find("img", {"class" : "thumb"}).parent['href'])
                x = reply.findAll("span")

                for i in x:
                    if ('File: ') in i.contents[0]:
                        fn_search = re.search(regex, i.contents[0])

                        if fn_search:
                            filename = fn_search.group(1)
                        else:
                            filename = utils.get_filename_from_url(url)

                que.file(site, uniq, url, filename, download.download_type['cloudflare'])

            except:
                pass

        return self.box

    @classmethod
    def imgurxalbum(self, a ,site="imgur:album", match=None, uniq=None):

        '''imgur.com Downloader'''

        p = self.establish(a, site, bs=False)

        album_json = utils.get_json(utils.re_find('image[ ]{2,}: ({.*})', p, 1))
        uniq = album_json['hash']

        if album_json['is_album'] == True:

            report('info', 'album \'{}\' has {} image(s) ..'.format(album_json['title'], album_json['num_images']))

            p = self.establish('http://imgur.com/ajaxalbums/getimages/{}/hit.json'.format(uniq), site, bs=False, v=False)
            album_json = utils.get_json(p)

            for image in album_json['data']['images']:
                extension = ''.join([i for i in image['ext'] if not i.isdigit()])
                filename = '{}{}'.format(image['hash'], extension)
                url = 'https://i.imgur.com/{}'.format(filename)

                que.file(site, uniq, url, utils.get_filename_from_url(url), download.download_type['generic'])

        else:
            report('info', 'This is not an album.')

        return self.box

    @classmethod
    def sevenchan_org(self, a ,site="7chan.org", match=None, uniq=None):

        '''7chan.org Downloader'''

        p = self.establish(a, site)

        rgx = '(\(([0-9]{1,}.[0-9]{1,}[A-Z]{1,2}), ?([0-9]{1,}x[0-9]{1,}), ?(.{1,}.[a-zA-Z]{1,4})\))'
        images = p.findAll("p", {"class" : "file_size"})
        uniq = p.find("input", {"name" : "replythread"})['value'].__str__()
        multi_images = p.findAll("img", {"class" : ["multithumb", "multithumbfirst"]})

        for image in images:
            url = image.find("a")['href']
            filename = None
            rm = re.search(rgx, image.contents[2].replace('\n', ''))
            if rm: filename = rm.group(4)
            else: filename = image.find("a").contents[0].__str__().replace('\n', '')

            que.file(site, uniq, url, filename, download.download_type['generic'])

        for m_image in multi_images:
            url = m_image['src'].replace('thumb', 'src').replace('s.', '.')
            filename = None
            rm = re.search(rgx, m_image['title'])

            if rm: filename = rm.group(4)
            else: filename = m_image['title'].__str__()

            que.file(site, uniq, url, filename, download.download_type['generic'])

        return self.box

    @classmethod
    def arhivach(self, a, site="arhivach", match=None, uniq=None):

        '''arhivach.org Downloader'''

        p = self.establish(a, site)

        images = p.findAll("a", {"class" : ["img_filename"]})

        if uniq is None:
            rm = re.search(sites_regex_table['arhivach'], a)

            if rm:
                uniq = rm.group(3)
            else:
                uniq = ''.join(c for c in a if c.isdigit())

        for img in images:
            url = filename = None

            if img['href'][0] == ('#'):
                match = re.search("'(http(s)?:\/\/(.*).(.{2,5})\/(.*).(.{3,4}))'", img['onclick'].__str__())

                if match:
                    url = match.group(1)
                    filename = utils.get_filename_from_url(url)
            else:
                if img['href'].startswith('/a_cimg/'):
                    url = 'https://arhivach.org/' + img['href'].__str__()
                    filename = img.contents[0].__str__()

                if img['href'].startswith('https://') or img['href'].startswith('http://'):
                    url = img['href'].__str__()
                    filename = img.contents[0].__str__()

            if url is not None:

                que.file(site, uniq, url, filename, download.download_type['cloudflare'])

        return self.box

    @classmethod
    def dvach(self, a, site="dvach", match=None, uniq=None):

        '''2ch.(hk, pm, re, tf, wf, yt, so) Downloader'''

        rm = re.search(sites_regex_table['dvach'], a)
        uniq = rm.group(3) + rm.group(5)

        json_url = ('{}2ch.hk/{}/res/{}.json'.format(rm.group(2), rm.group(3), rm.group(5)))

        p = self.establish(json_url, site, bs=False)

        json_data = utils.get_json(p)

        for post in json_data['threads'][0]['posts']:
            for file in post['files']:
                try:
                    filename = file['fullname']
                except:
                    filename = file['path'].split('/')[int(file['path'].count('/'))]

                url = ('{}2ch.hk{}'.format(rm.group(2), file['path']))
                que.file(site, uniq, url, filename, download.download_type['generic'])

        return self.box

    @classmethod
    def fourarchive(self, a, site="4archive", match=None, uniq=None):

        '''4archive.org Downloader'''

        p = self.establish(a, site)

        posts = p.findAll("div", {"class" : ["postContainer", "opContainer"]})
        uniq = p.find("div", {"class" : "thread"})['id'].__str__()

        if match != None:
            if len(match.groups()) == 3:
                uniq = "{}{}".format(match.group(2), match.group(3))

        for post in posts:

            file_info = post.findAll("div", {"class" : ["fileText"]})

            for fi in file_info:
                file_a = fi.find("a")
                filename = file_a.contents[0].__str__()
                if file_a.has_attr('title'): filename = file_a['title'].__str__()
                url = utils.clean_url(fi.find("a")['href'])

                filename = filename.replace("Full size of ", "")

                if url.startswith('/data/'):
                    url = (utils.re_find(sites_regex_table['4archive'], a, 1) + url)

                que.file(site, uniq, url, filename, download.download_type['generic'])

        return self.box

    @classmethod
    def fourchan(self, a, site="4chan", match=None, uniq=None):

        '''4chan.org Downloader'''

        p = self.establish(a, site)

        posts = p.findAll("div", {"class" : ["postContainer", "opContainer"]})
        uniq = p.find("div", {"class" : "thread"})['id'].__str__()

        if match != None:
            if len(match.groups()) == 5:
                uniq = "{}{}".format(match.group(3), match.group(5))

        for post in posts:

            file_info = post.findAll("div", {"class" : ["fileText"]})

            for fi in file_info:
                file_a = fi.find("a")
                filename = file_a.contents[0].__str__()
                if file_a.has_attr('title'): filename = file_a['title'].__str__()
                url = utils.clean_url(fi.find("a")['href'])

                que.file(site, uniq, url, filename, download.download_type['cloudflare'])

        return self.box

    @classmethod
    def eightchan(self, a, site="8chan", match=None, uniq=None):

        '''8chan (Infinitychan) Downloader'''

        p = self.establish(a, site)

        fi = p.findAll("p", {"class" : "fileinfo"})
        uniq = p.find("div", {"class" : "thread"}).find("a", {"class" : "post_anchor"})['id']

        if match != None:
            if len(match.groups()) == 5:
                uniq = "{}{}".format(match.group(3), match.group(5))

        for f in fi:
            filename = f.find("span", {"class" : "unimportant"}).find("span",
                {"class" : "postfilename"}).contents[0].__str__()
            url = f.find("a")['href'].__str__()

            if url.startswith('/'):
                url = utils.get_base_url(a) + url

            que.file(site, uniq, url, filename, download.download_type['generic'])

        return self.box

    @classmethod
    def fourplebs(self, a, site="4plebs", match=None, uniq=None):

        '''4plebs.org Downloader'''

        p = self.establish(a, site)

        image_links = p.findAll("a", {"class" : "thread_image_link"})
        uniq = p.find("article")['data-thread-num']

        for link in image_links:
            url = link['href'].__str__()
            que.file(site, uniq, url, utils.get_filename_from_url(url), download.download_type['generic'])

        return self.box

    @classmethod
    def vichan(self, a, site="vichan", match=None, uniq=None):

        '''vichan Downloader'''

        p = self.establish(a, site, ra=True)

        base_url = utils.get_base_url(a)
        simple_name = utils.get_simplename_from_url(a)

        if len(simple_name) > 1:
            site = simple_name

        thread_id = p[0].find("input", {"name" : "thread", "type" : "hidden"})

        if thread_id:
            uniq = thread_id['value']
        if uniq is None:
            uniq = utils.get_unique_number_from_url(a)

        replies = p[0].findAll("div", {"class" : ["file", "is-pic"]})

        for reply in replies:
            found = False
            files = reply.find("p", {"class" : "fileinfo"})
            if files:
                image_url = files.find("a")
                if image_url.has_attr('href'):
                    if utils.validate_image_url(image_url['href']):
                        url = utils.patch_url(image_url['href'], base_url)
                        try:
                            filename = files.find("span", {"class" : "postfilename"}).contents[0]
                        except:
                            filename = utils.get_filename_from_url(url)
                        que.file(site, uniq, url, filename, download.download_type['cloudflare'])
                        found = True

            if found == False:
                img_files = reply.findAll("img", {"class" : "post-image"})
                for img_file in img_files:
                    if img_file.parent.has_attr('href'):
                        url = utils.patch_url(img_file.parent['href'], base_url)
                        que.file(site, uniq, url, utils.get_filename_from_url(url), download.download_type['cloudflare'])
                        found = True

        return self.box

    @classmethod
    def sankaku_extend_get_next(self, p):
        if p == False:
            return [False, False]

        pagination = p.findAll("div", {"class" : "pagination"})

        for div in pagination:
            if div.has_attr('next-page-url'):
                return [True, div['next-page-url'].__str__()]

        return [False, False]

    @classmethod
    def sankaku_extend_get_posts(self, p):
        if p == False:
            return []

        posts = []

        content = p.find("div", {"class" : "content"})

        thumb_spans = p.findAll("span", {"class" : "thumb"})

        for span in thumb_spans:
            posts.append(span.find('a')['href'].__str__())

        return posts

    @classmethod
    def sankaku_extend_get_full(self, url, site, uniq):
        req = self.request_direct(url)

        p = BeautifulSoup(req.text, "html.parser")

        is_warned = False

        if req.status_code != 200:
            if req.status_code == 429:
                report('error', 'Too many requests ..')
            return

        for c in p.findAll("div", {"id" : "post-content"}):
            images = c.findAll(['video', 'img'])

            for image in images:
                if image.has_attr('src'):
                    image_url = image['src']

                    if image_url.startswith('//'):
                        image_url = 'https:' + image_url

                    rm = re.search("([a-zA-Z0-9_\.]{1,}\.(jpe?g|png|gif|webm|mp4))", image_url)

                    if rm:
                        filename = rm.group(1)
                    else:
                        filename = utils.get_filename_from_url(image_url.split('?e')[0])

                    que.file(site, uniq, image_url, filename, download.download_type['cloudflare'])

    @classmethod
    def sankaku_extend_handler(self, p, url, site, uniq, verbal = False):
        posts = self.sankaku_extend_get_posts(p)

        threads = [None] * len(posts)

        for i, post in enumerate(posts):
            threads[i] = Thread(
                target=self.sankaku_extend_get_full,
                args=(url + post, site, uniq)
            )

            threads[i].start()

            if ibdl.sleep > 0:
                time.sleep(ibdl.sleep)

            while threading.active_count() > ibdl.maximum_threads:
                time.sleep(1)

        for thread in threads:
            thread.join()

        self.download_partial(self.box, verbal, True)

    @classmethod
    def sankaku(self, a, site="sankaku", match=None, uniq=None):

        '''sankaku Downloader'''

        uniq = utils.get_unique_number_from_url(a)

        p = self.establish(a, site)

        if p == False:
            return

        rm = re.search(sites_regex_table['sankaku'], a)

        if rm:
            url = rm.group(1)
        else:
            return self.box

        get_next = self.sankaku_extend_get_next(p)

        report(site, 'Fetching full URLs from page ..')

        self.sankaku_extend_handler(p, url, site, uniq, True)

        while get_next[0] == True:
            p = self.establish(url + get_next[1], site, v=False)

            rm = re.search("page=([0-9]{1,})", get_next[1])

            if rm:
                report(site, 'Fetching next page (Page: ' + rm.group(1) + ') ..')
            else:
                report(site, 'Fetching next page ..')

            self.sankaku_extend_handler(p, url, site, uniq, True)
            get_next = self.sankaku_extend_get_next(p)

        if self.recent_error == 429:
            if self.recent_error in possible_reasons:
                report('info', 'Possible reasons: ' + possible_reasons[self.recent_error][0])
            else:
                report('info', 'Request limit reached. Try setting a sleep interval.')

        return False

class ibdl():

    imageboard_name = destination = thread_lock = None

    _queue = queue.Queue()

    thread_lock = threading.Lock()

    current = 0

    def __init__(self, site, dest, dirn, sepdir, threads = 32, sleep = 0):

        self.current_url = site

        ibdl.dirn = dirn

        ibdl.maximum_threads = int(threads)

        ibdl.sleep = float(sleep)

        ibdl.current = 0

        if dest is not None:
            download.save_directory = dest

        download.use_sep_dir = sepdir

        self.detect_site()

        method = getattr(scrapers, self.get_function(self.imageboard_name))

        self.download_images(self.check_list(method(a=site, match=self.matched), cdir=dirn))

    @classmethod
    def create_destination(self, a):
        if not os.path.exists(a):
            try:
                os.makedirs(a)
            except FileExistsError:
                pass
            except PermissionError:
                raise ErrorCreatingDirectory
            except:
                raise ErrorCreatingDirectory

    @classmethod
    def download(self, site, uniq, url, name=None, dtype=download.download_type['generic'], length=0, v=True):

        '''This will assign a download function to the current item and download it'''

        # If the item doesn't have a name we strip the filename from the URL and try to use that
        if name is None:
            name = (url.split('/')[len(url.split('/'))-1])
        else:
            name = (utils.sanitize_filename(name))

        # Where we are going to save the item
        if download.use_sep_dir == True:
            destination = os.path.join(download.save_directory, uniq, name)
        else:
            destination = os.path.join(download.save_directory, name)

        if not os.path.exists(destination):
            # Create base directory if it doesn't exist
            self.create_destination(os.path.dirname(destination))

            # Assign a download function and download the item
            if dtype == download.download_type['cloudflare']:
                response = download.cloudflare(site, url, destination, name)
            elif dtype == download.download_type['generic']:
                response = download.generic(site, url, destination, name)
            elif dtype == download.download_type['content-disposition']:
                response = download.contentdisposition(site, url, destination, name)
            else:
                response = download.generic(site, url, destination, name)

            if response[0] != 1 and response[0] != 2:
                string_code = utils.return_code_to_string(response[0])
                # Print the current item and total progress
                if response[1] != False:
                    with self.thread_lock:
                        self.current = self.current + 1
                        if v == True:
                            report(string_code, '{} - {}'.format(utils.format_count(length, self.current), response[1]))

            self._queue.put(response)

        else:
            # If the item (destination) already exists we skip it
            self._queue.put((dict_return_codes['skip'], 'Skipped {}'.format(destination)))

    def detect_site(self):

        '''Match the passed URL against the regex table to find what site we are dealing with'''

        # Search through a list of known sites
        for s, r in sites_regex_table.items():
            match = re.search(r, self.current_url)

            if match:
                self.matched = match
                self.imageboard_name = imageboard_name = s
                report(s, self.current_url)

        # If no known site is found try to search the site source for a matching imageboard
        if self.imageboard_name is None:
            unknown_site = scrapers.request(self.current_url)

            for s, r in adaptive_regex_table.items():

                if isinstance(r, list):
                    for x in r:
                        match = re.search(x, unknown_site[0])
                        if match:
                            self.matched = match
                            self.imageboard_name = imageboard_name = s
                            report(s, self.current_url)

                else:

                    match = re.search(r, unknown_site[0])

                    if match:
                        self.matched = match
                        self.imageboard_name = imageboard_name = s
                        report(s, self.current_url)

        # If both searches are unsuccessful raise not supported error
        if self.imageboard_name is None:
            raise ErrorNotSupported

    def get_function(self, site):

        '''Returns the scraper function we are going to use'''

        for s, r in dict_number_converter.items():
            site = str.replace(site, s, r)

        return site

    @classmethod
    def check_list(self, lst, match=None, cdir=None):

        if lst == False or lst == None:
            return

        temp = []
        count = 2

        if cdir is not None:
            for c, q in enumerate(lst[1]):
                lst[1][c] = cdir
        else:
            for c, q in enumerate(lst[1]):
                lst[4][c] = match


        for i, x in enumerate(lst[3]):
            # Check if a item (file) has a duplicate, if so rename it
            if lst[3][i] not in temp:
                temp.append(lst[3][i])
            else:
                # Rename
                file = os.path.splitext(lst[3][i])
                temp.append('{}({}){}'.format(file[0], str(count), file[1]))
                count += 1

        lst[3] = temp

        return lst

    @classmethod
    def download_images(self, items, match = None, v = True, custom_length = False):

        '''Download the items returned from the scraper'''

        if items == False or items == None:
            return

        results = []

        if len(items[0]) > 0 and v == True:
            if items[1][1:] == items[1][:-1]:
                if download.use_sep_dir == True:
                    report('info', os.path.normpath('Downloading to: {}/{}'.format(download.save_directory, items[1][0])))
                else:
                    report('info', os.path.normpath('Downloading to: {}'.format(download.save_directory)))
            else:
                report('info', 'Downloading to: ' + download.save_directory)

        print_debug('info', "Initiating download process for " + str(len(items[0])) + " item(s) ..")

        threads = [None] * len(items[0])

        threads_list = list()

        if custom_length == False:
            items_length = len(items[0])
        else:
            items_length = int(custom_length)

        for i, main in enumerate(items[0]):
            if ibdl.sleep > 0:
                time.sleep(ibdl.sleep)

            threads[i] = Thread(
                target=self.download,
                args=(main, items[1][i], items[2][i], items[3][i], items[4][i], items_length)
            )

            threads[i].start()

            threads_list.append(threads[i])

            while threading.active_count() > ibdl.maximum_threads:
                print_debug('thread' , 'Excessive threads (' + str(threading.active_count()) + '). Waiting ..')
                time.sleep(1)

        while threading.active_count() > 1:
            print_debug('thread' , 'Threads running (Wind down): ' + str(threading.active_count()) + '.')
            time.sleep(2)

        for thread in threads_list:
            thread.join()

        while not self._queue.empty():
            results.append(self._queue.get())


        return_codes = []

        # Print the final results
        for i in range(0, len(results)):
            return_codes.append(results[i][0])

        report('info', utils.result_to_string(return_codes))

class ErrorRequest(Exception):
    '''Raised if the page returns a bad status code'''

class ErrorNotSupported(Exception):
    '''Raised if the url can't be parsed and/or identified'''

class ErrorCreatingDirectory(Exception):
    '''Raised if directory could not be created'''

class ErrorParsingJson(Exception):
    '''Raised if encountering an error when extracting and or parsing a json object'''

def main():
    if not (sys.version_info > (3, 0)):
        report('info', 'This script requires Python 3+'); sys.exit(0)

    parser = argparse.ArgumentParser(description = "Imageboard Downloader [{}]".format(version))

    parser.add_argument('urls', default = [], nargs = '*', help = 'one or more URLs to scrape')
    parser.add_argument('-p', dest = 'path', default = None, help = 'where to save files (default is working directory)', required = False)
    parser.add_argument('-x', dest='x', action='store_true', help = 'disables saving files into separate folders (see -d)', required = False)
    parser.add_argument('-d', dest = 'directory_name',default = None, help = 'name of a separate folder to save files into \
    (this folder will be created inside the path, default is a unique name fetched from the URL)', required = False)
    parser.add_argument('-v', dest='v', action='store_true', help='show current version', required = False)
    parser.add_argument('--debug', dest='debug', action='store_true', help='print debug information', required = False)
    parser.add_argument('-t', dest = 'thread_count', default = 32, help = 'maximum concurrent threads', required = False)
    parser.add_argument('-s', dest = 'sleep', default = 0, help = 'amount of time in seconds to sleep in between requests', required = False)

    # Retrieve arguments
    args = parser.parse_args()

    try:
        # Print current version
        if args.v:
            report('version', version); sys.exit(0)

        # Enable debugging if paramater is passed
        if args.debug:
            global debugging
            debugging = True

            for arg in args.__dict__:
                if args.__dict__[arg] is not None:
                    print_debug('info', "Args => (" + str(arg) + "): " + str(args.__dict__[arg]))

        # Check if URL(s) were passed, if so go through them, if not exit
        if len(args.urls) == 0:
            # Exit
            report('info', 'No URLs were passed - exiting'); sys.exit(0)
        else:
            # Go through URL(s)
            for url in args.urls:
                ibdl(
                    url, args.path, args.directory_name, not args.x, threads=int(args.thread_count), sleep=float(args.sleep)
                )

    # Exceptions
    except ErrorRequest as exception:
        error_code = int(str(exception))
        status_code = utils.request_code_to_string(error_code)

        if status_code is not False:
            report('error', 'Error requesting page (code: {} - {})'.format(error_code, status_code))
        else:
            report('error', 'Error requesting page (code: {})'.format(error_code))

        if error_code in possible_reasons:
            report('info', 'Possible reasons: ' + possible_reasons[error_code][0])

    except ErrorNotSupported:
        report('error', 'Unsupported URL')

    except ErrorCreatingDirectory:
        report('error', 'Error creating directory, make sure you have the required permissions')

    except ErrorParsingJson:
        report('error', 'Error parsing JSON')

if __name__ == '__main__':
    main()
