def s3_upload(s3,data,target,out_index_num,norr, extension=".index"): keyname = '/'.join([target['prefix'],target['key']+'.'+str(out_index_num)+extension]) s3uri = 's3://'+target['bucket']+'/'+keyname data.seek(0) extra_args = {'StorageClass':'REDUCED_REDUNDANCY'} if norr: extra_args = {} try: # HeadObject to see if exist (needs ListBucket or this returns 403) s3.Object(target['bucket'], keyname).load() logger.info("{} exist. Skipping...".format(s3uri)) except ClientError as ce: # if does not exist, then upload if ce.response["Error"]["Code"] == "404": try: s3.Bucket(target['bucket']).upload_fileobj(data,keyname,ExtraArgs=extra_args) except ClientError as e: logger.error('Unable to upload to {}'.format(s3uri)) logger.debug('Received error: {}'.format(e)) sys.exit(4) logger.info("Uploaded to {}".format(s3uri)) elif ce.response["Error"]["Code"] == "403": logger.error('Permission error loading {}\n Please check your IAM policy.'.format(s3uri)) logger.debug('Received error: {}'.format(ce)) sys.exit(7) else: logger.error('Unknown error loading {}'.format(s3uri)) logger.debug('Received error: {}'.format(ce)) sys.exit(7) return