123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267 |
- ## Copyright 2013-2014 Ray Holder
- ##
- ## Licensed under the Apache License, Version 2.0 (the "License");
- ## you may not use this file except in compliance with the License.
- ## You may obtain a copy of the License at
- ##
- ## http://www.apache.org/licenses/LICENSE-2.0
- ##
- ## Unless required by applicable law or agreed to in writing, software
- ## distributed under the License is distributed on an "AS IS" BASIS,
- ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ## See the License for the specific language governing permissions and
- ## limitations under the License.
- import random
- from pip._vendor import six
- import sys
- import time
- import traceback
- # sys.maxint / 2, since Python 3.2 doesn't have a sys.maxint...
- MAX_WAIT = 1073741823
- def retry(*dargs, **dkw):
- """
- Decorator function that instantiates the Retrying object
- @param *dargs: positional arguments passed to Retrying object
- @param **dkw: keyword arguments passed to the Retrying object
- """
- # support both @retry and @retry() as valid syntax
- if len(dargs) == 1 and callable(dargs[0]):
- def wrap_simple(f):
- @six.wraps(f)
- def wrapped_f(*args, **kw):
- return Retrying().call(f, *args, **kw)
- return wrapped_f
- return wrap_simple(dargs[0])
- else:
- def wrap(f):
- @six.wraps(f)
- def wrapped_f(*args, **kw):
- return Retrying(*dargs, **dkw).call(f, *args, **kw)
- return wrapped_f
- return wrap
- class Retrying(object):
- def __init__(self,
- stop=None, wait=None,
- stop_max_attempt_number=None,
- stop_max_delay=None,
- wait_fixed=None,
- wait_random_min=None, wait_random_max=None,
- wait_incrementing_start=None, wait_incrementing_increment=None,
- wait_exponential_multiplier=None, wait_exponential_max=None,
- retry_on_exception=None,
- retry_on_result=None,
- wrap_exception=False,
- stop_func=None,
- wait_func=None,
- wait_jitter_max=None):
- self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number
- self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay
- self._wait_fixed = 1000 if wait_fixed is None else wait_fixed
- self._wait_random_min = 0 if wait_random_min is None else wait_random_min
- self._wait_random_max = 1000 if wait_random_max is None else wait_random_max
- self._wait_incrementing_start = 0 if wait_incrementing_start is None else wait_incrementing_start
- self._wait_incrementing_increment = 100 if wait_incrementing_increment is None else wait_incrementing_increment
- self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier
- self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max
- self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max
- # TODO add chaining of stop behaviors
- # stop behavior
- stop_funcs = []
- if stop_max_attempt_number is not None:
- stop_funcs.append(self.stop_after_attempt)
- if stop_max_delay is not None:
- stop_funcs.append(self.stop_after_delay)
- if stop_func is not None:
- self.stop = stop_func
- elif stop is None:
- self.stop = lambda attempts, delay: any(f(attempts, delay) for f in stop_funcs)
- else:
- self.stop = getattr(self, stop)
- # TODO add chaining of wait behaviors
- # wait behavior
- wait_funcs = [lambda *args, **kwargs: 0]
- if wait_fixed is not None:
- wait_funcs.append(self.fixed_sleep)
- if wait_random_min is not None or wait_random_max is not None:
- wait_funcs.append(self.random_sleep)
- if wait_incrementing_start is not None or wait_incrementing_increment is not None:
- wait_funcs.append(self.incrementing_sleep)
- if wait_exponential_multiplier is not None or wait_exponential_max is not None:
- wait_funcs.append(self.exponential_sleep)
- if wait_func is not None:
- self.wait = wait_func
- elif wait is None:
- self.wait = lambda attempts, delay: max(f(attempts, delay) for f in wait_funcs)
- else:
- self.wait = getattr(self, wait)
- # retry on exception filter
- if retry_on_exception is None:
- self._retry_on_exception = self.always_reject
- else:
- self._retry_on_exception = retry_on_exception
- # TODO simplify retrying by Exception types
- # retry on result filter
- if retry_on_result is None:
- self._retry_on_result = self.never_reject
- else:
- self._retry_on_result = retry_on_result
- self._wrap_exception = wrap_exception
- def stop_after_attempt(self, previous_attempt_number, delay_since_first_attempt_ms):
- """Stop after the previous attempt >= stop_max_attempt_number."""
- return previous_attempt_number >= self._stop_max_attempt_number
- def stop_after_delay(self, previous_attempt_number, delay_since_first_attempt_ms):
- """Stop after the time from the first attempt >= stop_max_delay."""
- return delay_since_first_attempt_ms >= self._stop_max_delay
- def no_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
- """Don't sleep at all before retrying."""
- return 0
- def fixed_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
- """Sleep a fixed amount of time between each retry."""
- return self._wait_fixed
- def random_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
- """Sleep a random amount of time between wait_random_min and wait_random_max"""
- return random.randint(self._wait_random_min, self._wait_random_max)
- def incrementing_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
- """
- Sleep an incremental amount of time after each attempt, starting at
- wait_incrementing_start and incrementing by wait_incrementing_increment
- """
- result = self._wait_incrementing_start + (self._wait_incrementing_increment * (previous_attempt_number - 1))
- if result < 0:
- result = 0
- return result
- def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
- exp = 2 ** previous_attempt_number
- result = self._wait_exponential_multiplier * exp
- if result > self._wait_exponential_max:
- result = self._wait_exponential_max
- if result < 0:
- result = 0
- return result
- def never_reject(self, result):
- return False
- def always_reject(self, result):
- return True
- def should_reject(self, attempt):
- reject = False
- if attempt.has_exception:
- reject |= self._retry_on_exception(attempt.value[1])
- else:
- reject |= self._retry_on_result(attempt.value)
- return reject
- def call(self, fn, *args, **kwargs):
- start_time = int(round(time.time() * 1000))
- attempt_number = 1
- while True:
- try:
- attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
- except:
- tb = sys.exc_info()
- attempt = Attempt(tb, attempt_number, True)
- if not self.should_reject(attempt):
- return attempt.get(self._wrap_exception)
- delay_since_first_attempt_ms = int(round(time.time() * 1000)) - start_time
- if self.stop(attempt_number, delay_since_first_attempt_ms):
- if not self._wrap_exception and attempt.has_exception:
- # get() on an attempt with an exception should cause it to be raised, but raise just in case
- raise attempt.get()
- else:
- raise RetryError(attempt)
- else:
- sleep = self.wait(attempt_number, delay_since_first_attempt_ms)
- if self._wait_jitter_max:
- jitter = random.random() * self._wait_jitter_max
- sleep = sleep + max(0, jitter)
- time.sleep(sleep / 1000.0)
- attempt_number += 1
- class Attempt(object):
- """
- An Attempt encapsulates a call to a target function that may end as a
- normal return value from the function or an Exception depending on what
- occurred during the execution.
- """
- def __init__(self, value, attempt_number, has_exception):
- self.value = value
- self.attempt_number = attempt_number
- self.has_exception = has_exception
- def get(self, wrap_exception=False):
- """
- Return the return value of this Attempt instance or raise an Exception.
- If wrap_exception is true, this Attempt is wrapped inside of a
- RetryError before being raised.
- """
- if self.has_exception:
- if wrap_exception:
- raise RetryError(self)
- else:
- six.reraise(self.value[0], self.value[1], self.value[2])
- else:
- return self.value
- def __repr__(self):
- if self.has_exception:
- return "Attempts: {0}, Error:\n{1}".format(self.attempt_number, "".join(traceback.format_tb(self.value[2])))
- else:
- return "Attempts: {0}, Value: {1}".format(self.attempt_number, self.value)
- class RetryError(Exception):
- """
- A RetryError encapsulates the last Attempt instance right before giving up.
- """
- def __init__(self, last_attempt):
- self.last_attempt = last_attempt
- def __str__(self):
- return "RetryError[{0}]".format(self.last_attempt)
|