| 1234567891011121314151617181920212223242526272829303132333435 |
- def upload_pending_to_s3(self):
- """Gets the pending filenames from cache and uploads them."""
- pending_key = get_pending_key()
- pending = cache.get(pending_key, [])
- remaining = []
- for i, file_key in enumerate(pending):
- prefixed_file_key = '%s/%s' % (self.prefix, file_key)
- if self.verbosity > 0:
- print ("Uploading %s..." % prefixed_file_key)
- if self.dry_run:
- self.upload_count += 1
- continue
- filename = self.DIRECTORY + '/' + file_key
- failed = True
- try:
- upload_file_to_s3(prefixed_file_key, filename, self.key,
- do_gzip=True, do_expires=True)
- except boto.exception.S3CreateError as e:
- # TODO: retry to create a few times
- print ("Failed to upload: %s" % e)
- except Exception as e:
- print (e)
- raise
- else:
- failed = False
- self.upload_count += 1
- cache.delete(file_key)
- finally:
- if failed:
- remaining.append(file_key)
- self.remaining_count += 1
- if not self.dry_run:
- cache.set(pending_key, remaining)
|
