Processor.App.Downloader.downloader.DownloaderFull

class Processor.App.Downloader.downloader.DownloaderFull(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 3)

__init__(base_url: str = 'https://data.commoncrawl.org/', digest_verification: bool = True, max_retry: int = 5, sleep_step: int = 3)

Methods

__init__([base_url, digest_verification, ...])
aclose(exc_type, exc_val[, exc_tb])
aopen()
download(domain_record)
unwrap(response, domain_record)