PrivacyScanData/archive/new2/bulk-downloader-for-reddit-development/bdfr/resource.py

#!/usr/bin/env python3
# coding=utf-8

import hashlib
import logging
import re
import time
import urllib.parse
from typing import Callable, Optional

import _hashlib
import requests
from praw.models import Submission

from bdfr.exceptions import BulkDownloaderException

logger = logging.getLogger(__name__)


class Resource:
    def __init__(self, source_submission: Submission, url: str, download_function: Callable, extension: str = None):
        self.source_submission = source_submission
        self.content: Optional[bytes] = None
        self.url = url
        self.hash: Optional[_hashlib.HASH] = None
        self.extension = extension
        self.download_function = download_function
        if not self.extension:
            self.extension = self._determine_extension()

    @staticmethod
    def retry_download(url: str) -> Callable:
        return lambda global_params: Resource.http_download(url, global_params)

    def download(self, download_parameters: Optional[dict] = None):
        if download_parameters is None:
            download_parameters = {}
        if not self.content:
            try:
                content = self.download_function(download_parameters)
            except requests.exceptions.ConnectionError as e:
                raise BulkDownloaderException(f'Could not download resource: {e}')
            except BulkDownloaderException:
                raise
            if content:
                self.content = content
        if not self.hash and self.content:
            self.create_hash()

    def create_hash(self):
        self.hash = hashlib.md5(self.content)

    def _determine_extension(self) -> Optional[str]:
        extension_pattern = re.compile(r'.*(\..{3,5})$')
        stripped_url = urllib.parse.urlsplit(self.url).path
        match = re.search(extension_pattern, stripped_url)
        if match:
            return match.group(1)

    @staticmethod
    def http_download(url: str, download_parameters: dict) -> Optional[bytes]:
        headers = download_parameters.get('headers')
        current_wait_time = 60
        if 'max_wait_time' in download_parameters:
            max_wait_time = download_parameters['max_wait_time']
        else:
            max_wait_time = 300
        while True:
            try:
                response = requests.get(url, headers=headers)
                if re.match(r'^2\d{2}', str(response.status_code)) and response.content:
                    return response.content
                elif response.status_code in (408, 429):
                    raise requests.exceptions.ConnectionError(f'Response code {response.status_code}')
                else:
                    raise BulkDownloaderException(
                        f'Unrecoverable error requesting resource: HTTP Code {response.status_code}')
            except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e:
                logger.warning(f'Error occured downloading from {url}, waiting {current_wait_time} seconds: {e}')
                time.sleep(current_wait_time)
                if current_wait_time < max_wait_time:
                    current_wait_time += 60
                else:
                    logger.error(f'Max wait time exceeded for resource at url {url}')
                    raise