#!/usr/bin/env python3 import os import json import time import logging from colors import color from boto3 import session from cli_tools import parser from utils import ProgressPercentage from config import (ACCESS_ID,SECRET_KEY,SERVICE_NAME,REGION_NAME,ENDPOINT_URL,BUCKET_NAME) cwd_dir = os.getcwd() logs_dir = os.path.join(cwd_dir, '.logs') failed_dir = os.path.join(cwd_dir, '.failed_uploads') if not os.path.exists(logs_dir): os.makedirs(logs_dir) logging.basicConfig(filename=os.path.join(logs_dir,'logs.log'), filemode='a',format='%(asctime)s %(levelname)s %(message)s', ) if not os.path.exists(failed_dir): os.makedirs(failed_dir) def log_error(error_file): log_file = os.path.join(failed_dir,"%s.txt"%time.strftime("%Y%m%d")) if not os.path.exists(log_file): with open(log_file, 'w'): pass with open(log_file, 'a') as f: f.write(error_file) # Initiate session session = session.Session() def set_client(): client = session.client( service_name=SERVICE_NAME, region_name=REGION_NAME, endpoint_url=ENDPOINT_URL, aws_access_key_id=ACCESS_ID, aws_secret_access_key=SECRET_KEY ) return client def upload_dir(dirpath,bucket_name=None): client = set_client() if not os.path.isdir(dirpath): raise Exception("Not a Valid Directory :%s" % (dirpath)) bucket = BUCKET_NAME if bucket_name: bucket = bucket_name file_count = 0 error_count = 0 print(color("Initializing ...\n",fg="yellow")) for (root, dirs, filenames) in os.walk(top=dirpath, topdown=True): for filename in filenames: filepath = os.path.join(root, filename) try: print(color("%s \nuploading...\n"%filepath,fg='blue')) client.upload_file(filepath, bucket ,os.path.abspath(filepath) ,Callback=ProgressPercentage(filepath)) print(color("Success\n",fg="lime")) file_count += 1 except Exception as e: logging.error('FilePath: %s' % filepath, exc_info=True) error_count += 1 print(color("Error encountered: %s"%e,fg="#ff1a1a")) log_error("%s\n"%filepath) continue print(color("Total number of files uploaded: %d.\n" % file_count,fg="lime")) print(color("Total number of errors encountered: %d." % error_count,fg="#ff1a1a")) if error_count > 1: print(color("Please Check the log file located at .logs in the current working directory",fg='#ff1a1a')) if __name__ == '__main__': args = parser.parse_args() root_dir = args.dir bucket_name = args.bucket upload_dir(root_dir,bucket_name)