def upload_pending_to_s3(self): """Gets the pending filenames from cache and uploads them.""" pending_key = get_pending_key() pending = cache.get(pending_key, []) remaining = [] for i, file_key in enumerate(pending): prefixed_file_key = '%s/%s' % (self.prefix, file_key) if self.verbosity > 0: print ("Uploading %s..." % prefixed_file_key) if self.dry_run: self.upload_count += 1 continue filename = self.DIRECTORY + '/' + file_key failed = True try: upload_file_to_s3(prefixed_file_key, filename, self.key, do_gzip=True, do_expires=True) except boto.exception.S3CreateError as e: # TODO: retry to create a few times print ("Failed to upload: %s" % e) except Exception as e: print (e) raise else: failed = False self.upload_count += 1 cache.delete(file_key) finally: if failed: remaining.append(file_key) self.remaining_count += 1 if not self.dry_run: cache.set(pending_key, remaining)