s3_upload_9.py 1.4 KB

1234567891011121314151617181920212223242526272829
  1. def uploadThread_small(srcfile, prefix_and_key):
  2. print(f'\033[0;32;1m--->Uploading\033[0m {srcfile["Key"]} - small file')
  3. with open(os.path.join(SrcDir, srcfile["Key"]), 'rb') as data:
  4. for retryTime in range(MaxRetry + 1):
  5. try:
  6. pstart_time = time.time()
  7. chunkdata = data.read()
  8. chunkdata_md5 = hashlib.md5(chunkdata)
  9. s3_dest_client.put_object(
  10. Body=chunkdata,
  11. Bucket=DesBucket,
  12. Key=prefix_and_key,
  13. ContentMD5=base64.b64encode(chunkdata_md5.digest()).decode('utf-8'),
  14. StorageClass=StorageClass
  15. )
  16. pload_time = time.time() - pstart_time
  17. pload_bytes = len(chunkdata)
  18. pload_speed = size_to_str(int(pload_bytes / pload_time)) + "/s"
  19. print(f'\033[0;34;1m --->Complete\033[0m {srcfile["Key"]} - small file - {pload_speed}')
  20. break
  21. except Exception as e:
  22. logger.warning(f'Upload small file Fail: {srcfile["Key"]}, '
  23. f'{str(e)}, Attempts: {retryTime}')
  24. if retryTime >= MaxRetry:
  25. logger.error(f'Fail MaxRetry Download/Upload small file: {srcfile["Key"]}')
  26. return "MaxRetry"
  27. else:
  28. time.sleep(5 * retryTime)
  29. return