nastos3.py 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. #!/usr/bin/env python
  2. # -*- coding: utf-8 -*-
  3. import sys
  4. import os
  5. import getopt
  6. import ConfigParser
  7. import traceback
  8. import logging
  9. import subprocess
  10. import hashlib
  11. BUCKET = ''
  12. ACCESS_KEY = ''
  13. SECRET_KEY = ''
  14. ORIGIN_PATH = ''
  15. INPUTFILE = ''
  16. REMOTE_PATH = ''
  17. def main(argv):
  18. try:
  19. opts, arg = getopt.getopt(argv, "i:l:", ["inputfile","logfile"])
  20. except getopt.GetoptError:
  21. logging.error('Input file error')
  22. sys.exit(2)
  23. for opt, arg in opts:
  24. if opt == '-i':
  25. inputFile = arg
  26. if opt == '-l':
  27. LOGFILE = arg
  28. if LOGFILE:
  29. logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%Y/%m/%d %I:%M:%S %p',
  30. filename=LOGFILE, level=logging.INFO)
  31. logging.info('--------------------Init upload session--------------------')
  32. if checkFile(inputFile):
  33. logging.info('Configuration file is good')
  34. setEnvironment()
  35. uploadAFile()
  36. else:
  37. logging.error('File is not good. Check the file')
  38. sys.exit(0)
  39. def getMD5Base64(filename):
  40. hash = hashlib.md5()
  41. with open(filename) as f:
  42. for chunk in iter(lambda: f.read(4096), ""):
  43. hash.update(chunk)
  44. return hash.digest().encode('base64').strip()
  45. def setEnvironment():
  46. global ACCESS_KEY
  47. global SECRET_KEY
  48. logging.info('Setting environment variables')
  49. os.environ["S3_ACCESS_KEY_ID"] = ACCESS_KEY
  50. logging.info('Variable S3_ACCESS_KEY_ID=*******' + str(os.environ.get('S3_ACCESS_KEY_ID'))[-5:])
  51. os.environ["S3_SECRET_ACCESS_KEY"] = SECRET_KEY
  52. logging.info('Variable S3_SECRET_ACCESS_KEY=*******' + str(os.environ.get('S3_SECRET_ACCESS_KEY'))[-5:])
  53. logging.info('Setted environment variables')
  54. def checkFile(inputfile):
  55. global BUCKET
  56. global ACCESS_KEY
  57. global SECRET_KEY
  58. global ORIGIN_PATH
  59. global REMOTE_PATH
  60. checkFileBool = True
  61. config = ConfigParser.ConfigParser()
  62. config.read(inputfile)
  63. BUCKET = config.get('Job_1', 'bucket')
  64. logging.info('Bucket: ' + BUCKET)
  65. ACCESS_KEY = config.get('Job_1', 'access_key')
  66. SECRET_KEY = config.get('Job_1', 'secret_key')
  67. ORIGIN_PATH = config.get('Job_1', 'origin_path')
  68. logging.info('Origin path: ' + ORIGIN_PATH)
  69. REMOTE_PATH = config.get('Job_1', 'remote_path')
  70. if REMOTE_PATH:
  71. logging.info('Remote path: ' + REMOTE_PATH)
  72. if not BUCKET or not ACCESS_KEY or not SECRET_KEY or not ORIGIN_PATH:
  73. checkFileBool = False
  74. return checkFileBool
  75. def uploadAFile():
  76. try:
  77. global BUCKET
  78. global ACCESS_KEY
  79. global SECRET_KEY
  80. global ORIGIN_PATH
  81. global REMOTE_PATH
  82. ORIGIN_PATH = ORIGIN_PATH.decode().encode('utf-8')
  83. logging.info('Initializing upload of files')
  84. for dirName, subdirList, fileList in os.walk(ORIGIN_PATH):
  85. for filename in fileList:
  86. file = os.path.join(dirName, filename)
  87. filemd5 = str(getMD5Base64(file))
  88. fileSize = os.path.getsize(file)
  89. fileObject = file.replace(ORIGIN_PATH,'')
  90. fileObject = fileObject.replace('///', '/')
  91. if REMOTE_PATH:
  92. fileObject = BUCKET + '/' + REMOTE_PATH + fileObject
  93. s3md5 = subprocess.Popen(['/usr/local/modules/usrsbin/s3 head "' + fileObject + '" | grep x-amz-meta-md5'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
  94. out, err = s3md5.communicate()
  95. subir = True
  96. if out:
  97. out = str(out)
  98. out = out.replace('x-amz-meta-md5: ','').replace(' ','').replace('\n','')
  99. if filemd5 == out:
  100. subir = False
  101. else:
  102. subir = True
  103. if err:
  104. err = str(err)
  105. err = err.replace('\n','')
  106. if err.find('HttpErrorNotFound') != -1:
  107. subir = True
  108. else:
  109. subir = False
  110. if subir:
  111. with open(file, 'rb') as data:
  112. logging.info('Uploading: ' + file)
  113. q = subprocess.check_output(['/usr/local/modules/usrsbin/s3 put "' + fileObject +
  114. '" filename="' + file + '" contentLength=' + str(fileSize) + ' md5=' + filemd5 + ' x-amz-meta-md5=' + str(filemd5) + ' | grep status'], stderr=subprocess.STDOUT, shell=True)
  115. #output, err = q.communicate()
  116. q = str(q)
  117. q = q.replace("ERROR: Nondigit in TotalSize parameter:",'').replace('\n','')
  118. logging.info("Upload finished." + q + ". (Status = 0 means it's all OK)")
  119. else:
  120. logging.info('The file ' + file + ' is the same in s3. Skipping')
  121. logging.info('All files uploaded to bucket ' + BUCKET)
  122. except Exception as err:
  123. traceback.print_exception
  124. logging.error(err)
  125. if __name__ == "__main__":
  126. main(sys.argv[1:])