def track_progress_less_iterations(self, server_had_bytes_before_attempt, roll_back_md5=True, debug=0): # At this point we had a re-tryable failure; see if made progress. if self.server_has_bytes > server_had_bytes_before_attempt: self.progress_less_iterations = 0 # If progress, reset counter. else: self.progress_less_iterations += 1 if roll_back_md5: # Rollback any potential hash updates, as we did not # make any progress in this iteration. self.digesters = self.digesters_before_attempt if self.progress_less_iterations > self.num_retries: # Don't retry any longer in the current process. raise ResumableUploadException( 'Too many resumable upload attempts failed without ' 'progress. You might try this upload again later', ResumableTransferDisposition.ABORT_CUR_PROCESS) # Use binary exponential backoff to desynchronize client requests. sleep_time_secs = random.random() * (2**self.progress_less_iterations) if debug >= 1: print('Got retryable failure (%d progress-less in a row).\n' 'Sleeping %3.1f seconds before re-trying' % (self.progress_less_iterations, sleep_time_secs)) time.sleep(sleep_time_secs)