PrivacyScanData/S3/new3/File-uploader-to-cloud-main/file_uploader/google_cloud_service.py

import os
from pprint import pprint
from google.cloud import storage
import json

warnings.filterwarnings("ignore")


class Goggle_cloud_operations(Cloud_operations):
    """Goggle_cloud_operations are dervived class from Cloud_operations
    This class need credentials like access_key,secret_access_key, bucket name, region which is manatory,
    client scret key json which should be available inthe local path
    To create client.json file please refer link below
    Creating and managing service account keys:
    https://cloud.google.com/iam/docs/creating-managing-service-account-keys

    Set up your first Google Cloud Project and download Client File (JSON file):

    https://www.dezyre.com/recipes/upload-files-to-google-drive-using-python
    https://d35mpxyw7m7k7g.cloudfront.net/bigdata_1/Get+Authentication+for+Google+Service+API+.pdf
    Manage Files in Google Cloud Storage With Python
    https://hackersandslackers.com/manage-files-in-google-cloud-storage-with-python/

    Storage classes
    https://cloud.google.com/storage/docs/storage-classes
    """

    def __init__(self, bucket, region, client_config_path):
        """Intilizing all the needed credentials during object creation and validating it"""

    self.__bucket = bucket
    self.__region = region
    #         self.__storage_client = storage_client
    self.__client_config_path = client_config_path
    self.validate_login_credentials()
    print("Google cloud operations started")


def set_credentails_with_env(self, client_config_path):
    """environ varialbe GOOGLE_APPLICATION_CREDENTIALS is set, this is one time activity """
    try:
        os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = os.path.join(
            self.__client_config_path)
        print("Google Client config path set success")
        return True
    except:
        raise Exception(
            'Faild to set up environment varialbe for Google cloud storage')


def validate_login_credentials(self):
    """validate_login_credentials validates the given credentials from user,
    This method is from parent class Cloud_operations"""

    self.set_credentails_with_env(self.__client_config_path)
    try:
        self.__storage_client = storage.Client()  # try to validate credentials if success method return true else stop here
        print("Google Client Authentication set success")
    except:
        raise Exception('Authentication failed for Google cloud storage')


def upload_files(self, list_of_files):
    self.count = 0

    print("Sample of file to be uploaded to Google storage are \t {}".format(
        list_of_files))
    bucket = self.__storage_client.get_bucket(self.__bucket)
    for each_file in tqdm(list_of_files):
        try:
            blob_name = os.path.split(each_file)[-1]
            blob = bucket.blob(blob_name)
            blob.upload_from_filename(
                each_file)  # .upload_file from google cloud api package uploads file to google storage
            self.count += 1
        except Exception as e:
            print("Failed to upload file {} \n{}".format(each_file, e))

    print("Total file succefully uploaded:\t {}".format(self.count))
    return True