From ae7581a90225448cc609e50963a064b2a2087b25 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 9 Sep 2026 11:03:20 -0500 Subject: [PATCH 01/19] feat: Remove permanent failure modes from FDv1 following RETRY spec --- contract-tests/async_service.py | 2 + contract-tests/service.py | 2 + ldclient/async_client.py | 4 +- ldclient/client.py | 9 +- ldclient/impl/aio/concurrency.py | 35 +- ldclient/impl/aio/transport.py | 41 +- ldclient/impl/async_big_segments.py | 2 +- ldclient/impl/big_segments.py | 2 +- ldclient/impl/datasource/async_polling.py | 81 ++-- ldclient/impl/datasource/async_streaming.py | 164 ++++--- ldclient/impl/datasource/datasource_common.py | 33 +- ldclient/impl/datasource/polling.py | 72 +-- ldclient/impl/datasource/streaming.py | 174 +++++--- ldclient/impl/datasystem/async_fdv2.py | 4 +- ldclient/impl/datasystem/fdv1.py | 2 +- ldclient/impl/datasystem/fdv2.py | 4 +- ldclient/impl/events/async_event_processor.py | 6 +- ldclient/impl/events/event_processor.py | 6 +- .../integrations/files/file_data_source.py | 2 +- .../integrations/files/file_data_sourcev2.py | 2 +- ldclient/impl/repeating_task.py | 68 ++- ldclient/impl/retry.py | 370 ++++++++++++++++ ldclient/impl/util.py | 17 +- ldclient/interfaces.py | 7 +- .../impl/datasource/test_async_polling.py | 238 +++++++++- .../impl/datasource/test_async_streaming.py | 342 ++++++++++++++- .../impl/datasource/test_polling_processor.py | 269 +++++++++++- .../testing/impl/datasource/test_streaming.py | 352 ++++++++++++++- ldclient/testing/impl/test_repeating_task.py | 100 ++++- ldclient/testing/impl/test_retry.py | 410 ++++++++++++++++++ ldclient/testing/test_aio.py | 47 +- ldclient/testing/test_ldclient_end_to_end.py | 19 +- ldclient/testing/test_util.py | 32 ++ 33 files changed, 2604 insertions(+), 314 deletions(-) create mode 100644 ldclient/impl/retry.py create mode 100644 ldclient/testing/impl/test_retry.py diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index b3c04d8a..65e9be5c 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -69,6 +69,8 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'migrations', 'persistent-data-store-redis', 'fdv1-fallback', + 'retry-conformance-fdv1-streaming', + 'retry-conformance-fdv1-polling', ] } return aiohttp.web.Response( diff --git a/contract-tests/service.py b/contract-tests/service.py index a8e93674..260fb77b 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -86,6 +86,8 @@ def status(): 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', + 'retry-conformance-fdv1-streaming', + 'retry-conformance-fdv1-polling', ] } return json.dumps(body), 200, {'Content-type': 'application/json'} diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 031faf4d..edf9ef12 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -384,8 +384,8 @@ async def is_initialized(self) -> bool: If this returns false, it means that the client has not yet successfully connected to LaunchDarkly. It might still be in the process of starting up, or it might be attempting to reconnect after an - unsuccessful attempt, or it might have received an unrecoverable error (such as an invalid SDK key) - and given up. + unsuccessful attempt, or it might have received an error that needs to be fixed (such + as an invalid SDK key). This is a coroutine because determining readiness may query a persistent store. """ diff --git a/ldclient/client.py b/ldclient/client.py index 2329defa..3dae913c 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -311,10 +311,11 @@ def is_initialized(self) -> bool: If this returns false, it means the client has not yet obtained any flag data. It might still be starting up, or attempting to reconnect after an unsuccessful attempt, or it might have received - an unrecoverable error (such as an invalid SDK key) and given up. In this state, feature flag - evaluations will return default values -- unless you are using a persistent store integration and - flag data had already been stored by a successfully connected SDK in the past. You can use - :attr:`data_source_status_provider` to get information on errors, or to wait for a successful retry. + an error that needs to be fixed (such as an invalid SDK key). In this state, feature flag + evaluations will return default values -- unless you are using a persistent store integration + and flag data had already been stored by a successfully connected SDK in the past. You can use + :attr:`data_source_status_provider` to get information on errors, or to wait for a + successful retry. :return: true if the client is initialized and has flag data available """ diff --git a/ldclient/impl/aio/concurrency.py b/ldclient/impl/aio/concurrency.py index ed3f7962..c4b6ad70 100644 --- a/ldclient/impl/aio/concurrency.py +++ b/ldclient/impl/aio/concurrency.py @@ -9,11 +9,11 @@ import asyncio import inspect -import time from queue import Empty as QueueEmpty # noqa: F401 (shared timeout exception) from queue import Full as QueueFull # noqa: F401 (shared capacity exception) from typing import Any, Callable, Coroutine, Optional, Set +from ldclient.impl.repeating_task import DelaySource, FixedDelay from ldclient.impl.util import log @@ -189,25 +189,31 @@ async def stop_all(self, timeout: float = 1) -> None: class AsyncRepeatingTask: - """Calls a callback repeatedly at fixed intervals on a background task. + """Calls a callback repeatedly on a background task, waiting whatever its + :class:`~ldclient.impl.repeating_task.DelaySource` gives. Mirrors the semantics of ``ldclient.impl.repeating_task.RepeatingTask``: - the interval is measured from the start of each invocation, exceptions - from the callback are logged, and ``stop()`` prevents any further - invocations but cannot be undone.""" + the wait starts when the callback returns, exceptions from the callback + are logged, and ``stop()`` prevents any further invocations but cannot be + undone.""" - def __init__(self, label: str, interval: float, initial_delay: float, callable: Callable): + def __init__(self, label: str, delays: DelaySource, initial_delay: float, callable: Callable): self.__label = label - self.__interval = interval + self.__delays = delays self.__initial_delay = initial_delay self.__action = callable self.__stop = AsyncEvent() self.__task: Optional[asyncio.Task] = None + @staticmethod + def at_interval(label: str, interval: float, initial_delay: float, callable: Callable) -> 'AsyncRepeatingTask': + """Creates a task that runs at a fixed interval.""" + return AsyncRepeatingTask(label, FixedDelay(interval), initial_delay, callable) + def start(self): - """Starts the background task. Like a thread, the task can only be - started once.""" + """Starts the background task, if it is not running already.""" if self.__task is not None: - raise RuntimeError("tasks can only be started once") + log.info("Task %s has already been started; ignoring" % self.__label) + return self.__task = asyncio.ensure_future(self._run()) try: self.__task.set_name(f"{self.__label}.repeating") @@ -215,7 +221,9 @@ def start(self): pass def stop(self): - """Tells the background task to stop. It cannot be restarted after this.""" + """Tells the background task to stop. + + The stop is permanent. A later ``start()`` does not resume the task.""" self.__stop.set() task = self.__task # When stop() is called from within the action itself, let the loop @@ -237,14 +245,15 @@ async def _run(self): return stopped = self.__stop.is_set() while not stopped: - next_time = time.time() + self.__interval try: result = self.__action() if inspect.isawaitable(result): await result except Exception as e: log.exception("Unexpected exception on worker task: %s" % e) - delay = next_time - time.time() + # The wait starts when the callback returns, so a slow callback + # never shortens it. + delay = self.__delays.next_delay if delay > 0: stopped = await self.__stop.wait(delay) else: diff --git a/ldclient/impl/aio/transport.py b/ldclient/impl/aio/transport.py index e9aa374f..6226c5d2 100644 --- a/ldclient/impl/aio/transport.py +++ b/ldclient/impl/aio/transport.py @@ -118,11 +118,16 @@ def __init__(self, config, session: Optional[aiohttp.ClientSession] = None, prox self._http_options = http_options if http_options is not None else config.http self._proxy = proxy if proxy is not None else (self._http_options.http_proxy or None) - def create(self, url: str, initial_retry_delay: float, query_params=None) -> AsyncSSEClient: - """Builds an SSE client for the given stream URL. Headers, timeouts, - proxy settings, and the retry/backoff policy come from the SDK config. - ``query_params`` is an optional zero-argument callable evaluated on - each (re)connect to produce additional query string parameters.""" + def create(self, url: str, initial_retry_delay: float, query_params=None, sdk_managed_retry: bool = False) -> AsyncSSEClient: + """Builds an SSE client for the given stream URL. Headers, timeouts and + proxy settings come from the SDK config. ``query_params`` is an + optional zero-argument callable evaluated on each (re)connect to + produce additional query string parameters. + + ``sdk_managed_retry`` moves the delay between connection attempts to + the caller. The SSE client then never waits, and + ``initial_retry_delay`` is ignored. When it is false, the SSE client + backs off on its own.""" base_headers = _base_headers(self._config, ASYNC_USER_AGENT) aiohttp_request_options: dict = { "timeout": aiohttp.ClientTimeout( @@ -134,6 +139,24 @@ def create(self, url: str, initial_retry_delay: float, query_params=None) -> Asy proxy = self._proxy or _get_proxy_url(url) if proxy: aiohttp_request_options["proxy"] = proxy + if sdk_managed_retry: + # A zero base delay plus the no-op base strategy holds + # next_retry_delay at zero, so the SSE client never sleeps. + retry_options: dict = { + "initial_retry_delay": 0, + "retry_delay_strategy": RetryDelayStrategy(), + "retry_delay_reset_threshold": 0, + } + else: + retry_options = { + "initial_retry_delay": initial_retry_delay, + "retry_delay_strategy": RetryDelayStrategy.default( + max_delay=MAX_RETRY_DELAY, + backoff_multiplier=2, + jitter_multiplier=JITTER_RATIO, + ), + "retry_delay_reset_threshold": BACKOFF_RESET_INTERVAL, + } return AsyncSSEClient( connect=AsyncConnectStrategy.http( url=url, @@ -143,12 +166,6 @@ def create(self, url: str, initial_retry_delay: float, query_params=None) -> Asy query_params=query_params, ), error_strategy=ErrorStrategy.always_continue(), # we'll make error-handling decisions when we see a Fault - initial_retry_delay=initial_retry_delay, - retry_delay_strategy=RetryDelayStrategy.default( - max_delay=MAX_RETRY_DELAY, - backoff_multiplier=2, - jitter_multiplier=JITTER_RATIO, - ), - retry_delay_reset_threshold=BACKOFF_RESET_INTERVAL, logger=log, + **retry_options, ) diff --git a/ldclient/impl/async_big_segments.py b/ldclient/impl/async_big_segments.py index ce4aecd0..16315e0a 100644 --- a/ldclient/impl/async_big_segments.py +++ b/ldclient/impl/async_big_segments.py @@ -65,7 +65,7 @@ def __init__(self, config: AsyncBigSegmentsConfig): if self.__store: self.__cache = ExpiringDict(max_len=config.context_cache_size, max_age_seconds=config.context_cache_time) - self.__poll_task = AsyncRepeatingTask("ldclient.bigsegment.status-poll", config.status_poll_interval, 0, self.poll_store_and_update_status) + self.__poll_task = AsyncRepeatingTask.at_interval("ldclient.bigsegment.status-poll", config.status_poll_interval, 0, self.poll_store_and_update_status) def start(self): """Starts the status polling task. Separated from __init__ so the manager diff --git a/ldclient/impl/big_segments.py b/ldclient/impl/big_segments.py index cf2dec61..2d5dcf43 100644 --- a/ldclient/impl/big_segments.py +++ b/ldclient/impl/big_segments.py @@ -67,7 +67,7 @@ def __init__(self, config: BigSegmentsConfig): if self.__store: self.__cache = ExpiringDict(max_len=config.context_cache_size, max_age_seconds=config.context_cache_time) - self.__poll_task = RepeatingTask("ldclient.bigsegment.status-poll", config.status_poll_interval, 0, self.poll_store_and_update_status) + self.__poll_task = RepeatingTask.at_interval("ldclient.bigsegment.status-poll", config.status_poll_interval, 0, self.poll_store_and_update_status) self.__poll_task.start() def stop(self): diff --git a/ldclient/impl/datasource/async_polling.py b/ldclient/impl/datasource/async_polling.py index d0530a7c..3a638ed8 100644 --- a/ldclient/impl/datasource/async_polling.py +++ b/ldclient/impl/datasource/async_polling.py @@ -10,10 +10,15 @@ from ldclient.async_config import AsyncConfig from ldclient.impl.aio.concurrency import AsyncEvent, AsyncRepeatingTask from ldclient.impl.datasource.datasource_common import sink_or_store +from ldclient.impl.retry import ( + FailureKind, + RetryState, + classify_http_status, + for_polling +) from ldclient.impl.util import ( UnsuccessfulResponseException, - http_error_message, - is_http_error_recoverable, + http_error_description, log ) from ldclient.interfaces import ( @@ -27,13 +32,22 @@ class AsyncPollingUpdateProcessor(AsyncUpdateProcessor): - def __init__(self, config: AsyncConfig, requester: AsyncFeatureRequester, store: AsyncFeatureStore, ready: AsyncEvent): + """Polls LaunchDarkly for flag data on its own background task. + + The loop reads its wait from the retry state, which ``_fetch_and_store`` + updates, so a failure can push the next poll further out than the poll + interval. See :mod:`ldclient.impl.retry`. + """ + + def __init__(self, config: AsyncConfig, requester: AsyncFeatureRequester, store: AsyncFeatureStore, ready: AsyncEvent, retry_state: Optional[RetryState] = None): self._config = config self._data_source_update_sink = config.data_source_update_sink self._requester = requester self._store = store self._ready = ready - self._task = AsyncRepeatingTask("ldclient.datasource.polling", config.poll_interval, 0, self._fetch_and_store) + self._retry = retry_state or for_polling(config.poll_interval) + # No initial delay: the first poll is immediate. + self._task = AsyncRepeatingTask("ldclient.datasource.polling", self._retry, 0, self._fetch_and_store) def start(self): log.info("Starting AsyncPollingUpdateProcessor with request interval: " + str(self._config.poll_interval)) @@ -43,7 +57,12 @@ def initialized(self): return self._ready.is_set() and self._store.initialized async def stop(self): - self.__stop_with_error_info(None) + log.info("Stopping AsyncPollingUpdateProcessor") + self._task.stop() + + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.OFF, None) + # Wait for the current poll to finish before closing the transport, so we do # not close it while a request is still using it. The close is in a finally # so an owned transport is still released if stop() is cancelled mid-wait. @@ -52,39 +71,41 @@ async def stop(self): finally: await self._requester.close() - def __stop_with_error_info(self, error: Optional[DataSourceErrorInfo]): - log.info("Stopping AsyncPollingUpdateProcessor") - self._task.stop() - - if self._data_source_update_sink is None: - return - - self._data_source_update_sink.update_status(DataSourceState.OFF, error) - - async def _fetch_and_store(self): + async def _fetch_and_store(self) -> None: + """Makes one poll request and records the outcome on the retry state.""" try: all_data = await self._requester.get_all_data() await sink_or_store(self._data_source_update_sink, self._store).init(all_data) + + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.VALID, None) + + # Report the status before signaling readiness, so a caller that + # wakes on readiness cannot still read INITIALIZING. if not self._ready.is_set() and self._store.initialized: log.info("AsyncPollingUpdateProcessor initialized ok") self._ready.set() - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.VALID, None) + self._retry.record_success() + return except UnsuccessfulResponseException as e: + kind = classify_http_status(e.status) error_info = DataSourceErrorInfo(DataSourceErrorKind.ERROR_RESPONSE, e.status, time.time(), str(e)) + description = "Received %s for polling request" % http_error_description(e.status) + level = log.error if kind is FailureKind.UNEXPECTED else log.warning + stacktrace = None + except Exception as e: + # A certificate failure lands here too, and is as normal as the rest. + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) + description = "Error encountered when updating flags: %s" % e + level = log.error + # The exception is passed explicitly: by the time the message is + # logged, the handler has exited and exc_info() is empty. + stacktrace = e - http_error_message_result = http_error_message(e.status, "polling request") - if not is_http_error_recoverable(e.status): - log.error(http_error_message_result) - self._ready.set() # if client is initializing, make it stop waiting; has no effect if already inited - self.__stop_with_error_info(error_info) - else: - log.warning(http_error_message_result) + delay = self._retry.record_failure(kind) + level("%s - will retry in %.1fs" % (description, delay), exc_info=stacktrace) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - except Exception as e: - log.exception('Error: Exception encountered when updating flags. %s' % e) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e))) + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index 002d7b65..f670b111 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -4,9 +4,10 @@ # currently excluded from documentation - see docs/README.md +import asyncio import json import time -from typing import Any, Optional +from typing import Any, Callable, Optional from urllib import parse from ld_eventsource.actions import Event, Fault, Start @@ -16,14 +17,17 @@ from ldclient.impl.aio.transport import AsyncSSEFactory, make_client_session from ldclient.impl.datasource.datasource_common import ( STREAM_ALL_PATH, + StreamClosedError, parse_path, sink_or_store ) -from ldclient.impl.util import ( - http_error_message, - is_http_error_recoverable, - log +from ldclient.impl.retry import ( + FailureKind, + RetryState, + classify_http_status, + for_streaming ) +from ldclient.impl.util import http_error_description, log from ldclient.interfaces import ( AsyncUpdateProcessor, DataSourceErrorInfo, @@ -34,7 +38,13 @@ class AsyncStreamingUpdateProcessor(AsyncUpdateProcessor): - def __init__(self, config, store, ready, diagnostic_accumulator, sse_factory: Optional[AsyncSSEFactory] = None): + """Reads flag data from LaunchDarkly's streaming endpoint on a background task. + + The SDK owns the delay between connection attempts rather than the SSE + client; see :mod:`ldclient.impl.retry`. + """ + + def __init__(self, config, store, ready, diagnostic_accumulator, sse_factory: Optional[AsyncSSEFactory] = None, retry_state: Optional[RetryState] = None): self._uri = config.stream_base_uri + STREAM_ALL_PATH if config.payload_filter_key is not None: self._uri += '?%s' % parse.urlencode({'filter': config.payload_filter_key}) @@ -51,14 +61,18 @@ def __init__(self, config, store, ready, diagnostic_accumulator, sse_factory: Op self._sse_factory = sse_factory self._owned_session = None self._sse: Any = None - self._connection_attempt_start_time = None + self._connection_attempt_start_time: Optional[float] = None self._runner = AsyncTaskRunner() - self._started = False + self._start_requested = False + self._retry = retry_state or for_streaming(config.initial_reconnect_delay) + self._signalled_healthy = False + self._interrupted_by_sdk = False def start(self): - if self._started: - raise RuntimeError("processors can only be started once") - self._started = True + if self._start_requested: + log.info("AsyncStreamingUpdateProcessor has already been started; ignoring") + return + self._start_requested = True self._runner.spawn("ldclient.datasource.streaming", self._run) async def _run(self): @@ -72,7 +86,7 @@ async def _run(self): log.info("Starting AsyncStreamingUpdateProcessor connecting to uri: " + self._uri) self._running = True try: - self._sse = self._sse_factory.create(self._uri, self._config.initial_reconnect_delay) + self._sse = self._sse_factory.create(self._uri, self._config.initial_reconnect_delay, sdk_managed_retry=True) self._connection_attempt_start_time = time.time() async for action in self._sse.all: if isinstance(action, Start): @@ -80,23 +94,29 @@ async def _run(self): # For the initial connect the pre-loop timestamp is already set. if self._connection_attempt_start_time is None: self._connection_attempt_start_time = time.time() + # A fresh stream has not proved itself healthy yet. + self._signalled_healthy = False elif isinstance(action, Event): message_ok = False + message_handled = False try: message_ok = await self._process_message(action) + message_handled = True except json.decoder.JSONDecodeError as e: log.info("Error while handling stream event; will restart stream: %s" % e) - await self._sse.interrupt() + await self._interrupt_stream() - await self._handle_error(e) + if not await self._handle_error(e): + break except Exception as e: log.warning("Error while handling stream event; will restart stream: %s" % e) - await self._sse.interrupt() + await self._interrupt_stream() - if self._data_source_update_sink is not None: - error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) + if not await self._handle_error(e): + break - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) + if message_handled: + self._record_healthy_operation() if message_ok: self._record_stream_init(False) @@ -109,9 +129,17 @@ async def _run(self): log.info("AsyncStreamingUpdateProcessor initialized ok.") self._ready.set() elif isinstance(action, Fault): - # If the SSE client detects the stream has closed, then it will emit a fault with no-error. We can - # ignore this since we want the connection to continue. + # A Fault with no error means the connection closed cleanly. + # If we asked for that close, we have already recorded the + # failure behind it and must not record it twice. Otherwise + # the server closed a connection it normally leaves open, + # which is a connection failure the SDK backs off from. if action.error is None: + if self._interrupted_by_sdk: + self._interrupted_by_sdk = False + continue + if not await self._handle_error(StreamClosedError()): + break continue if not await self._handle_error(action.error): @@ -141,12 +169,10 @@ def _record_stream_init(self, failed: bool): async def stop(self): # Cancel the run task first: otherwise, if stop() is called before _run has executed, the - # loop could run _run at __stop_with_error_info's await and create a fresh SSE connection - # against the session we're closing. Once the runner is stopped, teardown is safe. + # loop could run _run at the teardown await and create a fresh SSE connection against the + # session we're closing. Once the runner is stopped, teardown is safe. await self._runner.stop_all() - await self.__stop_with_error_info(None) - async def __stop_with_error_info(self, error: Optional[DataSourceErrorInfo]): log.info("Stopping AsyncStreamingUpdateProcessor") self._running = False if self._sse: @@ -156,7 +182,28 @@ async def __stop_with_error_info(self, error: Optional[DataSourceErrorInfo]): if self._data_source_update_sink is None: return - self._data_source_update_sink.update_status(DataSourceState.OFF, error) + # OFF means an explicit shutdown. No stream failure produces it. + self._data_source_update_sink.update_status(DataSourceState.OFF, None) + + async def _interrupt_stream(self): + """Drops the stream connection so the next read reconnects. The SSE + client reports the close as a Fault with no error, and the flag tells + the loop that this one is ours and is already accounted for.""" + self._interrupted_by_sdk = True + await self._sse.interrupt() + + def _record_healthy_operation(self): + """Signals healthy operation on the first message of a fresh stream. + + It fires once per stream. A later message on the same stream must not + restart the reset window. The SSE client's own signal is no use here + because it fires when the connection opens, and an open connection that + has sent no data yet does not show the stream is working. + """ + if self._signalled_healthy: + return + self._signalled_healthy = True + self._retry.record_healthy() def initialized(self): return self._running and self._ready.is_set() is True and self._store.initialized is True @@ -198,46 +245,53 @@ async def _process_message(self, msg: Event) -> bool: # Returns true to continue, false to stop async def _handle_error(self, error: Exception) -> bool: + """Records a stream failure, reports it, and waits before the retry. + + Returns True once the wait is over, or False if the processor was + stopped. No failure ever ends the stream by itself. The wait is + interrupted by cancelling the task, which matters because the extended + regime can ask for an hour. + """ if not self._running: return False # don't retry if we've been deliberately stopped - if isinstance(error, json.decoder.JSONDecodeError): - error_info = DataSourceErrorInfo(DataSourceErrorKind.INVALID_DATA, 0, time.time(), str(error)) + self._record_stream_init(True) - log.error("Unexpected error on stream connection: %s, will retry" % error) - self._record_stream_init(True) - self._connection_attempt_start_time = None + level: Callable[..., None] - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) + if isinstance(error, json.decoder.JSONDecodeError): + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.INVALID_DATA, 0, time.time(), str(error)) + description = "Unparseable data on stream connection: %s" % error + level = log.error elif isinstance(error, HTTPStatusError): - self._record_stream_init(True) - self._connection_attempt_start_time = None - + kind = classify_http_status(error.status) error_info = DataSourceErrorInfo(DataSourceErrorKind.ERROR_RESPONSE, error.status, time.time(), str(error)) + description = "Received %s for stream connection" % http_error_description(error.status) + level = log.error if kind is FailureKind.UNEXPECTED else log.warning + elif isinstance(error, StreamClosedError): + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, time.time(), str(error)) + description = "The server closed the stream connection" + level = log.warning + else: + # A certificate failure lands here too, and is as normal as the rest. + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error)) + # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of HTTP client internals + description = "Error on stream connection: %s" % error + level = log.warning - http_error_message_result = http_error_message(error.status, "stream connection") - if not is_http_error_recoverable(error.status): - log.error(http_error_message_result) - self._running = False - self._ready.set() # if client is initializing, make it stop waiting; has no effect if already inited - await self.__stop_with_error_info(error_info) - return False - else: - log.warning(http_error_message_result) + delay = self._retry.record_failure(kind) + level("%s - will retry in %.1fs" % (description, delay)) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - else: - log.warning("Unexpected error on stream connection: %s, will retry" % error) - self._record_stream_init(True) - self._connection_attempt_start_time = None + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error))) - # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of HTTP client internals - self._connection_attempt_start_time = time.time() + self._sse.next_retry_delay - return True + self._connection_attempt_start_time = time.time() + delay + if delay > 0: + await asyncio.sleep(delay) + return self._running # magic methods for "with" statement (used in testing) async def __aenter__(self): diff --git a/ldclient/impl/datasource/datasource_common.py b/ldclient/impl/datasource/datasource_common.py index aa4da878..ad8de2a0 100644 --- a/ldclient/impl/datasource/datasource_common.py +++ b/ldclient/impl/datasource/datasource_common.py @@ -5,10 +5,16 @@ # currently excluded from documentation - see docs/README.md from collections import namedtuple -from typing import Mapping, Optional, Protocol, runtime_checkable +from typing import ( + Mapping, + Optional, + Protocol, + TypeVar, + Union, + runtime_checkable +) from ldclient.impl.util import _LD_ENVID_HEADER -from ldclient.interfaces import DataSourceUpdateSink, FeatureStore from ldclient.versioned_data_kind import FEATURES, SEGMENTS STREAM_ALL_PATH = '/all' @@ -17,7 +23,28 @@ ParsedPath = namedtuple('ParsedPath', ['kind', 'key']) -def sink_or_store(sink: Optional[DataSourceUpdateSink], store: FeatureStore): +class StreamClosedError(Exception): + """The stream connection closed cleanly, and the SDK did not ask for it. + + The service normally leaves the connection open, so a close the SDK did + not ask for is a connection failure. The SDK backs off before it + reconnects, rather than reconnecting at once. + + It is a NORMAL failure, not an UNEXPECTED one. A load balancer draining + during a rolling deploy closes streams cleanly, and putting that in the + extended regime would take a whole fleet out of service for up to an + hour. + """ + + def __init__(self): + super().__init__("the server closed the stream connection") + + +_Sink = TypeVar('_Sink') +_Store = TypeVar('_Store') + + +def sink_or_store(sink: Optional[_Sink], store: _Store) -> Union[_Sink, _Store]: """ The original implementation of the data sources relied on the feature store directly, which we are trying to move away from. Customers who might have diff --git a/ldclient/impl/datasource/polling.py b/ldclient/impl/datasource/polling.py index 171df9eb..504cb7d8 100644 --- a/ldclient/impl/datasource/polling.py +++ b/ldclient/impl/datasource/polling.py @@ -14,10 +14,15 @@ sink_or_store ) from ldclient.impl.repeating_task import RepeatingTask +from ldclient.impl.retry import ( + FailureKind, + RetryState, + classify_http_status, + for_polling +) from ldclient.impl.util import ( UnsuccessfulResponseException, - http_error_message, - is_http_error_recoverable, + http_error_description, log ) from ldclient.interfaces import ( @@ -38,13 +43,21 @@ def get_all_data_with_headers(self) -> Tuple[Any, Optional[Mapping[str, str]]]: class PollingUpdateProcessor(UpdateProcessor): - def __init__(self, config: Config, requester: FeatureRequester, store: FeatureStore, ready: Event): + """Polls LaunchDarkly for flag data on its own worker thread. + + The task reads its wait from the retry state, which ``_poll`` updates, so a + failure can push the next poll further out than the poll interval. See + :mod:`ldclient.impl.retry`. + """ + + def __init__(self, config: Config, requester: FeatureRequester, store: FeatureStore, ready: Event, retry_state: Optional[RetryState] = None): self._config = config self._data_source_update_sink: Optional[DataSourceUpdateSink] = config.data_source_update_sink self._requester = requester self._store = store self._ready = ready - self._task = RepeatingTask("ldclient.datasource.polling", config.poll_interval, 0, self._poll) + self._retry = retry_state or for_polling(config.poll_interval) + self._task = RepeatingTask("ldclient.datasource.polling", self._retry, 0, self._poll) def start(self): log.info("Starting PollingUpdateProcessor with request interval: " + str(self._config.poll_interval)) @@ -54,46 +67,53 @@ def initialized(self): return self._ready.is_set() is True and self._store.initialized is True def stop(self): - self.__stop_with_error_info(None) - - def __stop_with_error_info(self, error: Optional[DataSourceErrorInfo]): log.info("Stopping PollingUpdateProcessor") self._task.stop() if self._data_source_update_sink is None: return - self._data_source_update_sink.update_status(DataSourceState.OFF, error) + self._data_source_update_sink.update_status(DataSourceState.OFF, None) - def _poll(self): + def _poll(self) -> None: + """Makes one poll request and records the outcome on the retry state.""" try: (all_data, headers) = self._get_all_data_with_headers() record_environment_id(self._data_source_update_sink, headers) sink_or_store(self._data_source_update_sink, self._store).init(all_data) + + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.VALID, None) + + # Report the status before signaling readiness, so a caller that + # wakes on readiness cannot still read INITIALIZING. if not self._ready.is_set() and self._store.initialized: log.info("PollingUpdateProcessor initialized ok") self._ready.set() - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.VALID, None) + self._retry.record_success() + return except UnsuccessfulResponseException as e: + kind = classify_http_status(e.status) error_info = DataSourceErrorInfo(DataSourceErrorKind.ERROR_RESPONSE, e.status, time.time(), str(e)) - - http_error_message_result = http_error_message(e.status, "polling request") - if not is_http_error_recoverable(e.status): - log.error(http_error_message_result) - self._ready.set() # if client is initializing, make it stop waiting; has no effect if already inited - self.__stop_with_error_info(error_info) - else: - log.warning(http_error_message_result) - - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) + description = "Received %s for polling request" % http_error_description(e.status) + level = log.error if kind is FailureKind.UNEXPECTED else log.warning + stacktrace = None except Exception as e: - log.exception('Error: Exception encountered when updating flags. %s' % e) - - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e))) + # A certificate failure lands here too, and is as normal as the rest. + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) + description = "Error encountered when updating flags: %s" % e + level = log.error + # The exception is passed explicitly: by the time the message is + # logged, the handler has exited and exc_info() is empty. + stacktrace = e + + delay = self._retry.record_failure(kind) + level("%s - will retry in %.1fs" % (description, delay), exc_info=stacktrace) + + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) def _get_all_data_with_headers(self) -> Tuple[Any, Optional[Mapping[str, str]]]: """ diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index e5496147..e7a1615c 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -1,7 +1,8 @@ import json import time +from threading import Event as ThreadEvent from threading import Thread -from typing import Optional +from typing import Callable, Optional from urllib import parse from ld_eventsource import SSEClient @@ -15,16 +16,19 @@ from ldclient.impl.datasource.datasource_common import ( STREAM_ALL_PATH, + StreamClosedError, parse_path, record_environment_id, sink_or_store ) from ldclient.impl.http import HTTPFactory, _http_factory -from ldclient.impl.util import ( - http_error_message, - is_http_error_recoverable, - log +from ldclient.impl.retry import ( + FailureKind, + RetryState, + classify_http_status, + for_streaming ) +from ldclient.impl.util import http_error_description, log from ldclient.interfaces import ( DataSourceErrorInfo, DataSourceErrorKind, @@ -37,13 +41,15 @@ # stream will keep this from triggering stream_read_timeout = 5 * 60 -MAX_RETRY_DELAY = 30 -BACKOFF_RESET_INTERVAL = 60 -JITTER_RATIO = 0.5 - class StreamingUpdateProcessor(Thread, UpdateProcessor): - def __init__(self, config, store, ready, diagnostic_accumulator): + """Reads flag data from LaunchDarkly's streaming endpoint on its own thread. + + The SDK owns the delay between connection attempts rather than the SSE + client; see :meth:`_create_sse_client` and :mod:`ldclient.impl.retry`. + """ + + def __init__(self, config, store, ready, diagnostic_accumulator, retry_state: Optional[RetryState] = None): Thread.__init__(self, name="ldclient.datasource.streaming") self.daemon = True self._uri = config.stream_base_uri + STREAM_ALL_PATH @@ -55,7 +61,20 @@ def __init__(self, config, store, ready, diagnostic_accumulator): self._running = False self._ready = ready self._diagnostic_accumulator = diagnostic_accumulator - self._connection_attempt_start_time = None + self._connection_attempt_start_time: Optional[float] = None + self._retry = retry_state or for_streaming(config.initial_reconnect_delay) + self._stop_event = ThreadEvent() + self._signalled_healthy = False + self._interrupted_by_sdk = False + # Thread already owns the name "_started", so this flag cannot use it. + self._start_requested = False + + def start(self): + if self._start_requested: + log.info("StreamingUpdateProcessor has already been started; ignoring") + return + self._start_requested = True + Thread.start(self) def run(self): log.info("Starting StreamingUpdateProcessor connecting to uri: " + self._uri) @@ -65,23 +84,29 @@ def run(self): for action in self._sse.all: if isinstance(action, Start): record_environment_id(self._data_source_update_sink, action.headers) + # A fresh stream has not proved itself healthy yet. + self._signalled_healthy = False elif isinstance(action, Event): message_ok = False + message_handled = False try: message_ok = self._process_message(sink_or_store(self._data_source_update_sink, self._store), action) + message_handled = True except json.decoder.JSONDecodeError as e: log.info("Error while handling stream event; will restart stream: %s" % e) - self._sse.interrupt() + self._interrupt_stream() - self._handle_error(e) + if not self._handle_error(e): + break except Exception as e: log.info("Error while handling stream event; will restart stream: %s" % e) - self._sse.interrupt() + self._interrupt_stream() - if self._data_source_update_sink is not None: - error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) + if not self._handle_error(e): + break - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) + if message_handled: + self._record_healthy_operation() if message_ok: self._record_stream_init(False) @@ -94,9 +119,17 @@ def run(self): log.info("StreamingUpdateProcessor initialized ok.") self._ready.set() elif isinstance(action, Fault): - # If the SSE client detects the stream has closed, then it will emit a fault with no-error. We can - # ignore this since we want the connection to continue. + # A Fault with no error means the connection closed cleanly. If + # we asked for that close, we have already recorded the failure + # behind it and must not record it twice. Otherwise the server + # closed a connection it normally leaves open, which is a + # connection failure the SDK backs off from. if action.error is None: + if self._interrupted_by_sdk: + self._interrupted_by_sdk = False + continue + if not self._handle_error(StreamClosedError()): + break continue if not self._handle_error(action.error): @@ -118,25 +151,51 @@ def _create_sse_client(self) -> SSEClient: url=self._uri, headers=http_factory.base_headers, pool=stream_http_factory.create_pool_manager(1, self._uri), urllib3_request_options={"timeout": stream_http_factory.timeout} ), error_strategy=ErrorStrategy.always_continue(), # we'll make error-handling decisions when we see a Fault - initial_retry_delay=self._config.initial_reconnect_delay, - retry_delay_strategy=RetryDelayStrategy.default(max_delay=MAX_RETRY_DELAY, backoff_multiplier=2, jitter_multiplier=JITTER_RATIO), - retry_delay_reset_threshold=BACKOFF_RESET_INTERVAL, + # The SDK owns the retry delay, so the SSE client must never wait. + # A zero base delay plus the no-op base strategy holds + # next_retry_delay at zero, which is what these three arguments + # are for. The SSE client hands us the Fault before it would + # sleep, so we classify the failure and wait ourselves in + # _handle_error. Our wait is interruptible, which matters because + # the extended regime can ask for an hour. + initial_retry_delay=0, + retry_delay_strategy=RetryDelayStrategy(), + retry_delay_reset_threshold=0, logger=log, ) def stop(self): - self.__stop_with_error_info(None) - - def __stop_with_error_info(self, error: Optional[DataSourceErrorInfo]): log.info("Stopping StreamingUpdateProcessor") self._running = False + self._stop_event.set() if self._sse: self._sse.close() if self._data_source_update_sink is None: return - self._data_source_update_sink.update_status(DataSourceState.OFF, error) + # OFF means an explicit shutdown. No stream failure produces it. + self._data_source_update_sink.update_status(DataSourceState.OFF, None) + + def _interrupt_stream(self): + """Drops the stream connection so the next read reconnects. The SSE + client reports the close as a Fault with no error, and the flag tells + the loop that this one is ours and is already accounted for.""" + self._interrupted_by_sdk = True + self._sse.interrupt() + + def _record_healthy_operation(self): + """Signals healthy operation on the first message of a fresh stream. + + It fires once per stream. A later message on the same stream must not + restart the reset window. The SSE client's own signal is no use here + because it fires when the connection opens, and an open connection that + has sent no data yet does not show the stream is working. + """ + if self._signalled_healthy: + return + self._signalled_healthy = True + self._retry.record_healthy() def initialized(self): return self._running and self._ready.is_set() is True and self._store.initialized is True @@ -176,46 +235,49 @@ def _process_message(self, store, msg: Event) -> bool: # Returns true to continue, false to stop def _handle_error(self, error: Exception) -> bool: + """Records a stream failure, reports it, and waits before the retry. + + Returns True once the wait is over, or False if the processor was + stopped. No failure ever ends the stream by itself. + """ if not self._running: return False # don't retry if we've been deliberately stopped - if isinstance(error, json.decoder.JSONDecodeError): - error_info = DataSourceErrorInfo(DataSourceErrorKind.INVALID_DATA, 0, time.time(), str(error)) + self._record_stream_init(True) - log.error("Unexpected error on stream connection: %s, will retry" % error) - self._record_stream_init(True) - self._connection_attempt_start_time = None + level: Callable[..., None] - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) + if isinstance(error, json.decoder.JSONDecodeError): + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.INVALID_DATA, 0, time.time(), str(error)) + description = "Unparseable data on stream connection: %s" % error + level = log.error elif isinstance(error, HTTPStatusError): - self._record_stream_init(True) - self._connection_attempt_start_time = None - + kind = classify_http_status(error.status) error_info = DataSourceErrorInfo(DataSourceErrorKind.ERROR_RESPONSE, error.status, time.time(), str(error)) + description = "Received %s for stream connection" % http_error_description(error.status) + level = log.error if kind is FailureKind.UNEXPECTED else log.warning + elif isinstance(error, StreamClosedError): + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, time.time(), str(error)) + description = "The server closed the stream connection" + level = log.warning + else: + # A certificate failure lands here too, and is as normal as the rest. + kind = FailureKind.NORMAL + error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error)) + # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals + description = "Error on stream connection: %s" % error + level = log.warning - http_error_message_result = http_error_message(error.status, "stream connection") - if not is_http_error_recoverable(error.status): - log.error(http_error_message_result) - self._ready.set() # if client is initializing, make it stop waiting; has no effect if already inited - self.__stop_with_error_info(error_info) - self.stop() - return False - else: - log.warning(http_error_message_result) + delay = self._retry.record_failure(kind) + level("%s - will retry in %.1fs" % (description, delay)) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - else: - log.warning("Unexpected error on stream connection: %s, will retry" % error) - self._record_stream_init(True) - self._connection_attempt_start_time = None + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error))) - # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals - self._connection_attempt_start_time = time.time() + self._sse.next_retry_delay - return True + self._connection_attempt_start_time = time.time() + delay + return not self._stop_event.wait(delay) # magic methods for "with" statement (used in testing) def __enter__(self): diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index c9d0388d..612fe4cb 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -148,7 +148,7 @@ def _update_availability(self, available: bool) -> None: else: log.warning("Detected persistent store unavailability; updates will be cached until it recovers") if self._poller is None: - task_to_start = AsyncRepeatingTask("ldclient.check-availability", 0.5, 0, self._check_availability) + task_to_start = AsyncRepeatingTask.at_interval("ldclient.check-availability", 0.5, 0, self._check_availability) self._poller = task_to_start self._status_sink(DataStoreStatus(available, True)) @@ -545,7 +545,7 @@ async def _consume_synchronizer_results( :return: the ConditionDirective describing how to proceed """ action_queue: AsyncQueue = AsyncQueue() - timer = AsyncRepeatingTask( + timer = AsyncRepeatingTask.at_interval( label="AsyncFDv2-sync-cond-timer", interval=10, initial_delay=10, diff --git a/ldclient/impl/datasystem/fdv1.py b/ldclient/impl/datasystem/fdv1.py index 38655415..09ca6939 100644 --- a/ldclient/impl/datasystem/fdv1.py +++ b/ldclient/impl/datasystem/fdv1.py @@ -101,7 +101,7 @@ def __update_availability(self, available: bool): return log.warn("Detected persistent store unavailability; updates will be cached until it recovers") - task = RepeatingTask("ldclient.check-availability", 0.5, 0, self.__check_availability) + task = RepeatingTask.at_interval("ldclient.check-availability", 0.5, 0, self.__check_availability) with self.__lock.write(): self.__poller = task diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index 3455eab1..75f8d28c 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -129,7 +129,7 @@ def __update_availability(self, available: bool): poller_to_stop = self.__poller self.__poller = None elif self.__poller is None: - task_to_start = RepeatingTask("ldclient.check-availability", 0.5, 0, self.__check_availability) + task_to_start = RepeatingTask.at_interval("ldclient.check-availability", 0.5, 0, self.__check_availability) self.__poller = task_to_start if available: @@ -536,7 +536,7 @@ def _consume_synchronizer_results( :return: the ConditionDirective describing how to proceed """ action_queue: Queue = Queue() - timer = RepeatingTask( + timer = RepeatingTask.at_interval( label="FDv2-sync-cond-timer", interval=10, initial_delay=10, diff --git a/ldclient/impl/events/async_event_processor.py b/ldclient/impl/events/async_event_processor.py index 1e5b0215..dafd2483 100644 --- a/ldclient/impl/events/async_event_processor.py +++ b/ldclient/impl/events/async_event_processor.py @@ -202,13 +202,13 @@ class DefaultAsyncEventProcessor(AsyncEventProcessor): def __init__(self, config: AsyncConfig, http=None, dispatcher_class=None, diagnostic_accumulator=None): self._inbox = AsyncQueue(config.events_max_pending) self._inbox_full = False - self._flush_timer = AsyncRepeatingTask("ldclient.events.flush", config.flush_interval, config.flush_interval, self.flush) - self._contexts_flush_timer = AsyncRepeatingTask("ldclient.events.context-flush", config.context_keys_flush_interval, config.context_keys_flush_interval, self._flush_contexts) + self._flush_timer = AsyncRepeatingTask.at_interval("ldclient.events.flush", config.flush_interval, config.flush_interval, self.flush) + self._contexts_flush_timer = AsyncRepeatingTask.at_interval("ldclient.events.context-flush", config.context_keys_flush_interval, config.context_keys_flush_interval, self._flush_contexts) self._flush_timer.start() self._contexts_flush_timer.start() self._diagnostic_event_timer: Optional[AsyncRepeatingTask] if diagnostic_accumulator is not None: - self._diagnostic_event_timer = AsyncRepeatingTask("ldclient.events.send-diagnostic", config.diagnostic_recording_interval, config.diagnostic_recording_interval, self._send_diagnostic) + self._diagnostic_event_timer = AsyncRepeatingTask.at_interval("ldclient.events.send-diagnostic", config.diagnostic_recording_interval, config.diagnostic_recording_interval, self._send_diagnostic) self._diagnostic_event_timer.start() else: self._diagnostic_event_timer = None diff --git a/ldclient/impl/events/event_processor.py b/ldclient/impl/events/event_processor.py index 20cf03a4..6581070a 100644 --- a/ldclient/impl/events/event_processor.py +++ b/ldclient/impl/events/event_processor.py @@ -176,12 +176,12 @@ class DefaultEventProcessor(EventProcessor): def __init__(self, config, http=None, dispatcher_class=None, diagnostic_accumulator=None): self._inbox = queue.Queue(config.events_max_pending) self._inbox_full = False - self._flush_timer = RepeatingTask("ldclient.events.flush", config.flush_interval, config.flush_interval, self.flush) - self._contexts_flush_timer = RepeatingTask("ldclient.events.context-flush", config.context_keys_flush_interval, config.context_keys_flush_interval, self._flush_contexts) + self._flush_timer = RepeatingTask.at_interval("ldclient.events.flush", config.flush_interval, config.flush_interval, self.flush) + self._contexts_flush_timer = RepeatingTask.at_interval("ldclient.events.context-flush", config.context_keys_flush_interval, config.context_keys_flush_interval, self._flush_contexts) self._flush_timer.start() self._contexts_flush_timer.start() if diagnostic_accumulator is not None: - self._diagnostic_event_timer = RepeatingTask("ldclient.events.send-diagnostic", config.diagnostic_recording_interval, config.diagnostic_recording_interval, self._send_diagnostic) + self._diagnostic_event_timer = RepeatingTask.at_interval("ldclient.events.send-diagnostic", config.diagnostic_recording_interval, config.diagnostic_recording_interval, self._send_diagnostic) self._diagnostic_event_timer.start() else: self._diagnostic_event_timer = None diff --git a/ldclient/impl/integrations/files/file_data_source.py b/ldclient/impl/integrations/files/file_data_source.py index 0fd0593c..81fb38d3 100644 --- a/ldclient/impl/integrations/files/file_data_source.py +++ b/ldclient/impl/integrations/files/file_data_source.py @@ -183,7 +183,7 @@ def __init__(self, resolved_paths, reloader, interval): self._paths = resolved_paths self._reloader = reloader self._file_times = self._check_file_times() - self._timer = RepeatingTask("ldclient.datasource.file.poll", interval, interval, self._poll) + self._timer = RepeatingTask.at_interval("ldclient.datasource.file.poll", interval, interval, self._poll) self._timer.start() def stop(self): diff --git a/ldclient/impl/integrations/files/file_data_sourcev2.py b/ldclient/impl/integrations/files/file_data_sourcev2.py index 5442b81e..032fc3fb 100644 --- a/ldclient/impl/integrations/files/file_data_sourcev2.py +++ b/ldclient/impl/integrations/files/file_data_sourcev2.py @@ -398,7 +398,7 @@ def __init__(self, resolved_paths, on_change_callback, interval): self._paths = resolved_paths self._on_change = on_change_callback self._file_times = self._check_file_times() - self._timer = RepeatingTask( + self._timer = RepeatingTask.at_interval( "ldclient.datasource.filev2.poll", interval, interval, self._poll ) self._timer.start() diff --git a/ldclient/impl/repeating_task.py b/ldclient/impl/repeating_task.py index 2d65de87..1b58c481 100644 --- a/ldclient/impl/repeating_task.py +++ b/ldclient/impl/repeating_task.py @@ -1,39 +1,84 @@ -import time from threading import Event, Thread -from typing import Callable +from typing import Any, Callable, Protocol from ldclient.impl.util import log +class DelaySource(Protocol): + """Supplies the wait before a repeating task's next invocation.""" + + @property + def next_delay(self) -> float: + """The seconds to wait before the next invocation.""" + ... + + +class FixedDelay(DelaySource): + """A :class:`DelaySource` that always gives the same wait.""" + + def __init__(self, seconds: float): + self.__seconds = seconds + + @property + def next_delay(self) -> float: + return self.__seconds + + class RepeatingTask: """ - A generic mechanism for calling a callback repeatedly at fixed intervals on a worker thread. + A generic mechanism for calling a callback repeatedly on a worker thread. + + The wait between invocations comes from a :class:`DelaySource`, which the + task reads after each one. Use :meth:`at_interval` for the common case of + a fixed interval. """ - def __init__(self, label, interval: float, initial_delay: float, callable: Callable): + def __init__(self, label: str, delays: DelaySource, initial_delay: float, callable: Callable[[], Any]): """ Creates the task, but does not start the worker thread yet. - :param interval: maximum time in seconds between invocations of the callback + :param label: names the worker thread, and appears in log messages + :param delays: supplies the wait after each invocation returns :param initial_delay: time in seconds to wait before the first invocation - :param callable: the function to execute repeatedly + :param callable: the function to execute repeatedly. Anything it + returns is ignored. """ - self.__interval = interval + self.__label = label + self.__delays = delays self.__initial_delay = initial_delay self.__action = callable self.__stop = Event() + self.__started = False self.__thread = Thread(target=self._run, name=f"{label}.repeating") self.__thread.daemon = True + @staticmethod + def at_interval(label: str, interval: float, initial_delay: float, callable: Callable[[], Any]) -> 'RepeatingTask': + """ + Creates a task that runs at a fixed interval. + + :param interval: time in seconds to wait after each invocation returns + """ + return RepeatingTask(label, FixedDelay(interval), initial_delay, callable) + def start(self): """ - Starts the worker thread. + Starts the worker thread, if it is not running already. + + Starting a task twice logs and does nothing, rather than raising, so a + caller that is safe to call more than once stays safe. """ + if self.__started: + log.info("Task %s has already been started; ignoring" % self.__label) + return + self.__started = True self.__thread.start() def stop(self): """ - Tells the worker thread to stop. It cannot be restarted after this. + Tells the worker thread to stop. + + The stop is permanent. A later :meth:`start` does not resume the task. """ self.__stop.set() @@ -43,10 +88,11 @@ def _run(self): return stopped = self.__stop.is_set() while not stopped: - next_time = time.time() + self.__interval try: self.__action() except Exception as e: log.exception("Unexpected exception on worker thread: %s" % e) - delay = next_time - time.time() + # The wait starts when the callback returns, so a slow callback + # never shortens it. + delay = self.__delays.next_delay stopped = self.__stop.wait(delay) if delay > 0 else self.__stop.is_set() diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py new file mode 100644 index 00000000..557cdd24 --- /dev/null +++ b/ldclient/impl/retry.py @@ -0,0 +1,370 @@ +""" +Computes how long to wait before a failed operation is tried again. + +Each failure falls into one of two classes. A ``NORMAL`` failure is one the +service is expected to recover from soon, so the wait stays short. An +``UNEXPECTED`` failure points to a problem that a person has to fix, such as a +rejected SDK key, so the wait becomes much longer. Only an HTTP status can be +``UNEXPECTED``; every network and TLS failure is ``NORMAL``. Neither class ever +tells the caller to give up. There is always a next attempt. + +The wait doubles after each failure, up to a ceiling. A random jitter is then +subtracted, so that many callers do not all try again at the same moment. The +wait never falls below the caller's operating cadence. + +:class:`RetryState` holds the state for one caller. Use :func:`for_streaming` +or :func:`for_polling` to build one with the right parameters and reset policy. +""" + +# currently excluded from documentation - see docs/README.md + +import random +import time +from enum import Enum +from typing import Optional, Protocol + +from ldclient.impl.util import log + +# The delay bounds of the extended regime, in seconds. A component enters the +# extended regime after an unexpected failure. +EXTENDED_INITIAL_DELAY = 5 * 60 +EXTENDED_MAX_DELAY = 60 * 60 + +# The delay bounds of the normal regime for streaming, in seconds. The initial +# delay is configurable as ``initial_reconnect_delay``. +STREAMING_MAX_DELAY = 30 + +# The documented default for ``initial_reconnect_delay``, in seconds. It stands +# in for a configured value of zero or less, which would reconnect with no wait +# at all. +DEFAULT_INITIAL_RECONNECT_DELAY = 1 + +# How long streaming must operate without a failure before its retry state +# resets, in seconds. +STREAMING_RESET_INTERVAL = 60 + +# How many polls in a row must succeed before polling's retry state resets. +POLLING_RESET_SUCCESSES = 2 + +# HTTP statuses in the 4xx range that are still normal failures. Every other +# 4xx is unexpected. +_NORMAL_4XX_STATUSES = frozenset([400, 408, 429]) + +# An upper bound on the backoff exponent, so a long outage cannot overflow the +# delay computation. Any real ceiling is reached long before this. +_MAX_BACKOFF_EXPONENT = 30 + + +class FailureKind(Enum): + """How a failure is classified, which decides how long the next wait is.""" + + NORMAL = 'normal' + """A failure the service is expected to recover from without help.""" + + UNEXPECTED = 'unexpected' + """A failure that suggests a problem a person has to fix. The component + keeps retrying, but much less often.""" + + +def classify_http_status(status: int) -> FailureKind: + """ + Classifies an HTTP status. + + ``400``, ``408`` and ``429`` are normal, as is any ``5xx``. Every other + ``4xx`` -- including ``401`` and ``403`` -- is unexpected. + """ + if 400 <= status < 500 and status not in _NORMAL_4XX_STATUSES: + return FailureKind.UNEXPECTED + return FailureKind.NORMAL + + +class ResetPolicy(Protocol): + """Decides when a component has operated well enough for long enough that + its retry state should reset. This is the only behavioral difference + between streaming and polling.""" + + def note_healthy(self) -> None: + """Records that the component is operating normally.""" + ... + + def note_failure(self) -> None: + """Records a failure, which ends any healthy stretch in progress.""" + ... + + def is_satisfied(self) -> bool: + """Reports whether the reset condition is met.""" + ... + + +class AfterHealthyFor(ResetPolicy): + """Resets once the component has operated without failing for + ``seconds``. This is the streaming policy.""" + + def __init__(self, seconds: float): + self._seconds = seconds + self._healthy_since: Optional[float] = None + + def note_healthy(self) -> None: + if self._healthy_since is None: + self._healthy_since = time.time() + + def note_failure(self) -> None: + self._healthy_since = None + + def is_satisfied(self) -> bool: + if self._healthy_since is None: + return False + return time.time() - self._healthy_since >= self._seconds + + @property + def healthy_since(self) -> Optional[float]: + """When the current healthy stretch began, or None if the component is + not currently healthy.""" + return self._healthy_since + + +class AfterConsecutiveSuccesses(ResetPolicy): + """Resets once ``count`` operations in a row have succeeded. This is the + polling policy.""" + + def __init__(self, count: int): + self._count = count + self._successes = 0 + + def note_healthy(self) -> None: + self._successes += 1 + + def note_failure(self) -> None: + self._successes = 0 + + def is_satisfied(self) -> bool: + return self._successes >= self._count + + @property + def successes(self) -> int: + """How many operations have succeeded in a row.""" + return self._successes + + +class RetryState: + """ + Tracks how long a data source should wait before its next attempt. + + A failure moves the state on and returns the wait. The delay for attempt + ``n`` is ``min(min_delay * 2 ** (n - 1), max_delay)``, less a random + jitter of up to half of it, and never less than the operating cadence. + + An unexpected failure moves the state to the extended regime, which raises + both delay bounds. The bounds stay raised until the reset condition is met, + so a normal failure that follows cannot lower them. + + Three things happen on success, and they are deliberately separate: + + * :meth:`record_success` returns the operating cadence. A backoff wait + applies to a retry, not to every operation, so one success is enough to + go back to the normal cadence even while the retry state is still raised. + * :meth:`record_healthy` feeds the reset policy. + * :meth:`maybe_reset` clears the retry state, but only once the reset + policy is satisfied, which may need more than one success. + + Conflating the first two is a real bug in another SDK: after an outage its + first successful poll still waited twenty minutes or more, even though it + already held fresh data. + """ + + def __init__( + self, + initial_delay: float, + normal_ceiling: float, + extended_initial_delay: float, + extended_ceiling: float, + reset_policy: ResetPolicy, + operating_cadence: float = 0, + ): + """ + :param initial_delay: the delay before the first retry, in seconds + :param normal_ceiling: the longest normal-regime delay, in seconds + :param extended_initial_delay: the delay before the first retry in the + extended regime, in seconds + :param extended_ceiling: the longest extended-regime delay, in seconds + :param reset_policy: decides when the retry state resets + :param operating_cadence: the rate the component normally operates at, + in seconds; no wait is ever shorter than this. Zero disables the + floor, which is what streaming wants. + """ + self._initial_delay = initial_delay + self._normal_ceiling = normal_ceiling + self._extended_initial_delay = extended_initial_delay + self._extended_ceiling = extended_ceiling + self._reset_policy = reset_policy + self._operating_cadence = operating_cadence + + self._n = 0 + self._extended = False + self._min_delay = initial_delay + self._max_delay = max(normal_ceiling, initial_delay) + self._attempts = 0 + # Read before any outcome is recorded, this is the ordinary interval. + self._next_delay = operating_cadence if operating_cadence > 0 else initial_delay + + @property + def next_delay(self) -> float: + """The wait before the next attempt, in seconds, as the last recorded + outcome decided it.""" + return self._next_delay + + @property + def attempts(self) -> int: + """How many failures this state has seen. For logging only.""" + return self._attempts + + @property + def min_delay(self) -> float: + """The delay the current regime starts from, in seconds.""" + return self._min_delay + + @property + def max_delay(self) -> float: + """The longest delay the current regime allows, in seconds.""" + return self._max_delay + + @property + def operating_cadence(self) -> float: + """The rate the component normally operates at, in seconds.""" + return self._operating_cadence + + @property + def in_extended_regime(self) -> bool: + """Whether an unexpected failure has moved this state to the extended + delay bounds.""" + return self._extended + + def record_failure(self, kind: FailureKind, wait_override: Optional[float] = None) -> float: + """ + Records a failed attempt and returns how long to wait before the next + one, in seconds. + + The state moves on before the wait is computed, so the wait always + reflects the failure just recorded. + + :param kind: how the failure was classified + :param wait_override: a wait the server asked for, which replaces the + computed one. LaunchDarkly does not send one on these endpoints, + so this is an unused seam. + """ + self.maybe_reset() + self._attempts += 1 + self._reset_policy.note_failure() + + if kind is FailureKind.UNEXPECTED and not self._extended: + # Moving to the extended regime raises both bounds and starts the + # attempt count over. Only the move does this: a later unexpected + # failure keeps counting up, so the delay is not pinned to the + # extended initial delay. + self._extended = True + self._min_delay = self._extended_initial_delay + self._max_delay = max(self._extended_ceiling, self._min_delay) + self._n = 1 + else: + self._n += 1 + + self._next_delay = self._compute_wait(wait_override) + return self._next_delay + + def record_success(self) -> float: + """ + Records a successful operation and returns how long to wait before the + next one, in seconds. + + The answer is the operating cadence, even when the retry state is still + raised, because a backoff wait applies to a retry and not to every + operation. This does not clear the retry state; :meth:`maybe_reset` + does that once the reset policy is satisfied. + """ + self.record_healthy() + self._next_delay = self._operating_cadence + return self._next_delay + + def record_healthy(self) -> None: + """ + Records that the component is operating normally, and resets the retry + state if that is now enough. + + Streaming calls this once per stream, on the first message of a fresh + stream. Polling calls it through :meth:`record_success`. + """ + self._reset_policy.note_healthy() + self.maybe_reset() + + def maybe_reset(self) -> bool: + """ + Clears the retry state if the reset policy is satisfied, returning the + delay bounds to the normal regime. + + Returns True if it cleared anything. This runs on its own before every + failure, so a caller does not have to call it. + """ + if not self._reset_policy.is_satisfied(): + return False + if self._n == 0 and not self._extended: + return False + self._n = 0 + self._extended = False + self._min_delay = self._initial_delay + self._max_delay = max(self._normal_ceiling, self._initial_delay) + return True + + def _compute_wait(self, wait_override: Optional[float]) -> float: + if wait_override is not None: + return max(wait_override, self._operating_cadence) + exponent = min(max(self._n - 1, 0), _MAX_BACKOFF_EXPONENT) + delay = min(self._min_delay * (2**exponent), self._max_delay) + jitter = random.random() * delay / 2 + return max(delay - jitter, self._operating_cadence) + + +def for_streaming(initial_reconnect_delay: float) -> RetryState: + """ + Builds the retry state for a streaming data source. + + Streaming has no operating cadence, so there is no floor on the wait. It + is healthy from the first message of a fresh stream, and resets after a + minute of that. + + A configured delay of zero or less would reconnect with no wait, so the + documented default stands in for it. ``Config`` does not check this value, + though it does clamp ``poll_interval``. + """ + if initial_reconnect_delay <= 0: + log.warning( + "initial_reconnect_delay must be greater than zero; using the default of %ss" + % DEFAULT_INITIAL_RECONNECT_DELAY + ) + initial_reconnect_delay = DEFAULT_INITIAL_RECONNECT_DELAY + return RetryState( + initial_delay=initial_reconnect_delay, + normal_ceiling=STREAMING_MAX_DELAY, + extended_initial_delay=EXTENDED_INITIAL_DELAY, + extended_ceiling=EXTENDED_MAX_DELAY, + reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), + ) + + +def for_polling(poll_interval: float) -> RetryState: + """ + Builds the retry state for a polling data source. + + The poll interval is polling's operating cadence, so no wait is ever + shorter than it. In the normal regime the delay bounds are the poll + interval itself, which means a normal failure simply polls again on + schedule. Polling is healthy on any successful poll, and resets after two + in a row. + """ + return RetryState( + initial_delay=poll_interval, + normal_ceiling=poll_interval, + extended_initial_delay=max(EXTENDED_INITIAL_DELAY, poll_interval), + extended_ceiling=max(EXTENDED_MAX_DELAY, poll_interval), + reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), + operating_cadence=poll_interval, + ) diff --git a/ldclient/impl/util.py b/ldclient/impl/util.py index 69e7186a..63e90624 100644 --- a/ldclient/impl/util.py +++ b/ldclient/impl/util.py @@ -134,9 +134,15 @@ def throw_if_unsuccessful_response(resp): def is_http_error_recoverable(status): + """ + Reports whether a component that treats some statuses as fatal should + keep going. + + Deprecated. Use :func:`ldclient.impl.retry.classify_http_status` instead. + """ if status >= 400 and status < 500: - return status in _RETRYABLE_STATUSES # all other 4xx besides these are unrecoverable - return True # all other errors are recoverable + return status in _RETRYABLE_STATUSES # all other 4xx besides these are treated as fatal + return True def http_error_description(status): @@ -144,6 +150,13 @@ def http_error_description(status): def http_error_message(status, context, retryable_message="will retry"): + """ + Builds the log message for an HTTP failure in a component that stops on + some statuses. + + Deprecated. The FDv1 data sources build their own message instead, so + that it can report the real retry delay. + """ return "Received %s for %s - %s" % (http_error_description(status), context, retryable_message if is_http_error_recoverable(status) else "giving up permanently") diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 2c9c245f..04aed18c 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -1005,16 +1005,15 @@ class DataSourceState(Enum): In streaming mode, this means that the stream connection failed, or had to be dropped due to some other error, and will be retried after a backoff delay. In polling mode, it means that the last poll - request failed, and a new poll request will be made after the configured polling interval. + request failed, and a new poll request will be made after the polling interval, or after a longer + delay if the error is one that needs to be fixed. """ OFF = 'off' """ Indicates that the data source has been permanently shut down. - This could be because it encountered an unrecoverable error (for instance, the LaunchDarkly service - rejected the SDK key; an invalid SDK key will never become valid), or because the SDK client was - explicitly shut down. + This means the SDK client was explicitly shut down, or that its configuration could not be parsed. """ diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index c639efb2..8a87a6dc 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -3,9 +3,13 @@ """ import asyncio +import logging +import ssl from unittest.mock import AsyncMock, MagicMock, patch +import aiohttp import pytest +from aiohttp.client_reqrep import ConnectionKey from ldclient.config import Config from ldclient.impl.aio.transport_types import TransportResponse @@ -13,6 +17,12 @@ AsyncFeatureRequesterImpl ) from ldclient.impl.datasource.async_polling import AsyncPollingUpdateProcessor +from ldclient.impl.retry import ( + POLLING_RESET_SUCCESSES, + AfterConsecutiveSuccesses, + RetryState, + for_polling +) from ldclient.impl.util import UnsuccessfulResponseException from ldclient.interfaces import ( AsyncDataSourceUpdateSink, @@ -20,6 +30,7 @@ DataSourceState ) from ldclient.testing.mock_async_components import MockAsyncFeatureStore +from ldclient.testing.test_util import no_retry_jitter from ldclient.versioned_data_kind import FEATURES, SEGMENTS # Sample data returned by a successful poll @@ -33,7 +44,34 @@ def make_config(**kwargs): return Config('SDK_KEY', **kwargs) -def make_processor(config=None, store=None, ready=None, requester=None): +# aiohttp's connection errors read the connection key when they are turned +# into a string, which the data source does, so a real one is needed here. +_CONNECTION_KEY = ConnectionKey( + host='app.launchdarkly.com', + port=443, + is_ssl=True, + ssl=True, + proxy=None, + proxy_auth=None, + proxy_headers_hash=None, + server_hostname=None, +) + + +def fast_retry_state(delay=0.001): + """A retry state with tiny delays, so a test does not have to wait out the + real extended-regime delay of five minutes.""" + return RetryState( + initial_delay=delay, + normal_ceiling=delay, + extended_initial_delay=delay, + extended_ceiling=delay, + reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), + operating_cadence=delay, + ) + + +def make_processor(config=None, store=None, ready=None, requester=None, retry_state=None): if config is None: config = make_config() if store is None: @@ -48,6 +86,7 @@ def make_processor(config=None, store=None, ready=None, requester=None): requester=requester, store=store, ready=ready, + retry_state=retry_state, ) @@ -174,31 +213,69 @@ async def test_successful_poll_initializes_store_and_sets_ready(self, mock_inter @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) - async def test_unrecoverable_http_error_stops_polling_and_sets_ready(self, mock_interval): + async def test_unexpected_http_error_keeps_polling_and_leaves_ready_unset(self, mock_interval): mock_interval.__get__ = MagicMock(return_value=0) store = MockAsyncFeatureStore() ready = asyncio.Event() config = make_config() - processor = make_processor(config=config, store=store, ready=ready) + processor = make_processor(config=config, store=store, ready=ready, retry_state=fast_retry_state()) mock_requester = AsyncMock(side_effect=UnsuccessfulResponseException(401)) processor._requester.get_all_data = mock_requester processor.start() - await asyncio.wait_for(ready.wait(), timeout=2.0) + await asyncio.sleep(0.1) - assert ready.is_set() + # A rejected SDK key must not falsely unblock initialization, and it + # must not stop the poller. + assert not ready.is_set() assert not processor.initialized() + assert mock_requester.call_count >= 2 - # The polling task must have stopped itself: no further polls occur. - await asyncio.sleep(0.05) - snapshot = mock_requester.call_count + await processor.stop() + + @pytest.mark.asyncio + @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) + async def test_unexpected_http_error_moves_to_the_extended_regime(self, mock_interval): + mock_interval.__get__ = MagicMock(return_value=0) + + retry = fast_retry_state() + processor = make_processor(retry_state=retry) + processor._requester.get_all_data = AsyncMock(side_effect=UnsuccessfulResponseException(401)) + + processor.start() await asyncio.sleep(0.05) - assert mock_requester.call_count == snapshot + + assert retry.in_extended_regime await processor.stop() + @pytest.mark.asyncio + async def test_the_first_success_after_an_outage_polls_at_the_cadence(self): + # RETRY 1.4.8: a backoff wait applies to a retry, not to every + # operation. _poll returns the wait, so this reads it directly rather + # than measuring elapsed time. + store = MockAsyncFeatureStore() + ready = asyncio.Event() + config = make_config() + with no_retry_jitter(): + retry = for_polling(30) + processor = make_processor(config=config, store=store, ready=ready, retry_state=retry) + + processor._requester.get_all_data = AsyncMock(side_effect=UnsuccessfulResponseException(401)) + await processor._fetch_and_store() + assert retry.next_delay == 5 * 60 + + processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) + await processor._fetch_and_store() + assert retry.next_delay == 30 + assert retry.in_extended_regime, "one success restores the cadence but does not reset" + + await processor._fetch_and_store() + assert retry.next_delay == 30 + assert not retry.in_extended_regime, "two successes in a row reset the state" + @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) async def test_recoverable_http_error_continues_polling(self, mock_interval): @@ -291,6 +368,69 @@ async def get_all_data(): await processor.stop() + @pytest.mark.parametrize( + "error", + [ + aiohttp.ClientConnectorCertificateError( + _CONNECTION_KEY, ssl.SSLCertVerificationError("self-signed certificate") + ), + aiohttp.ClientConnectorSSLError(_CONNECTION_KEY, OSError("handshake failed")), + ssl.SSLEOFError("EOF occurred in violation of protocol"), + ConnectionResetError(104, "reset by peer"), + ], + ids=["aiohttp-certificate", "aiohttp-tls", "peer-close-handshake", "reset"], + ) + @pytest.mark.asyncio + async def test_transport_failures_poll_again_at_the_cadence(self, error): + """No transport failure reaches the extended regime, an aiohttp + certificate failure included. Only an HTTP status can do that.""" + retry = for_polling(30) + processor = make_processor(retry_state=retry) + processor._requester.get_all_data = AsyncMock(side_effect=error) + + await processor._fetch_and_store() + assert retry.next_delay == 30 + assert not retry.in_extended_regime + + @pytest.mark.asyncio + async def test_the_log_reports_the_growing_retry_delay(self, caplog): + """The message has to carry the real delay, so someone reading logs can + see the backoff working.""" + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_polling(30) + processor = make_processor(retry_state=retry) + processor._requester.get_all_data = AsyncMock(side_effect=UnsuccessfulResponseException(401)) + + await processor._fetch_and_store() + await processor._fetch_and_store() + + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + "Received HTTP error 401 (invalid SDK key) for polling request - will retry in 300.0s", + "Received HTTP error 401 (invalid SDK key) for polling request - will retry in 600.0s", + ] + # An error a person has to fix is logged at error level, every time. + assert [r.levelno for r in caplog.records] == [logging.ERROR, logging.ERROR] + + @pytest.mark.asyncio + async def test_a_transport_error_reports_a_delay_and_keeps_its_stacktrace(self, caplog): + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_polling(30) + processor = make_processor(retry_state=retry) + processor._requester.get_all_data = AsyncMock(side_effect=ConnectionResetError(104, "reset by peer")) + + await processor._fetch_and_store() + + record = caplog.records[0] + assert record.getMessage() == "Error encountered when updating flags: [Errno 104] reset by peer - will retry in 30.0s" + # The handler has exited by the time this is logged, so the exception + # has to be carried explicitly for the traceback to survive. + assert record.exc_info is not None + @pytest.mark.asyncio async def test_stop_closes_requester(self): processor = make_processor() @@ -357,7 +497,7 @@ async def slow_poll(): @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) - async def test_unrecoverable_error_updates_sink_to_off(self, mock_interval): + async def test_unexpected_error_updates_sink_to_interrupted_never_off(self, mock_interval): mock_interval.__get__ = MagicMock(return_value=0) store = MockAsyncFeatureStore() @@ -367,7 +507,7 @@ async def test_unrecoverable_error_updates_sink_to_off(self, mock_interval): sink = MagicMock(spec=AsyncDataSourceUpdateSink) config._data_source_update_sink = sink - processor = make_processor(config=config, store=store, ready=ready) + processor = make_processor(config=config, store=store, ready=ready, retry_state=fast_retry_state()) processor._data_source_update_sink = sink processor._requester.get_all_data = AsyncMock( @@ -375,15 +515,65 @@ async def test_unrecoverable_error_updates_sink_to_off(self, mock_interval): ) processor.start() - await asyncio.wait_for(ready.wait(), timeout=2.0) + await asyncio.sleep(0.05) - # Verify the sink was told to go OFF - calls = [call for call in sink.update_status.call_args_list if call.args[0] == DataSourceState.OFF] - assert len(calls) >= 1 - error_info = calls[0].args[1] + interrupted = [c for c in sink.update_status.call_args_list if c.args[0] == DataSourceState.INTERRUPTED] + assert len(interrupted) >= 1 + error_info = interrupted[0].args[1] assert error_info.kind == DataSourceErrorKind.ERROR_RESPONSE assert error_info.status_code == 403 + assert not any(c.args[0] == DataSourceState.OFF for c in sink.update_status.call_args_list) + + await processor.stop() + + @pytest.mark.asyncio + @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) + async def test_stop_updates_sink_to_off(self, mock_interval): + mock_interval.__get__ = MagicMock(return_value=0) + + config = make_config() + sink = MagicMock(spec=AsyncDataSourceUpdateSink) + config._data_source_update_sink = sink + + processor = make_processor(config=config) + processor._data_source_update_sink = sink + processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) + + processor.start() + await processor.stop() + + assert any(c.args[0] == DataSourceState.OFF for c in sink.update_status.call_args_list) + + @pytest.mark.asyncio + @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) + async def test_valid_status_is_reported_before_ready_is_set(self, mock_interval): + # Mirrors go-server-sdk#442: a caller that wakes on readiness must not + # still be able to read INITIALIZING. + mock_interval.__get__ = MagicMock(return_value=0) + + from ldclient.impl.datasource.async_status import ( + AsyncDataSourceUpdateSinkImpl + ) + from ldclient.impl.listeners import Listeners + + store = MockAsyncFeatureStore() + ready = asyncio.Event() + observed = [] + listeners = Listeners() + listeners.add(lambda status: observed.append((status.state, ready.is_set()))) + + config = make_config() + config._data_source_update_sink = AsyncDataSourceUpdateSinkImpl(store, listeners, Listeners()) + + processor = make_processor(config=config, store=store, ready=ready) + processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) + + processor.start() + await asyncio.wait_for(ready.wait(), timeout=2.0) + + assert observed[0] == (DataSourceState.VALID, False) + await processor.stop() @pytest.mark.asyncio @@ -426,16 +616,19 @@ async def test_initialized_returns_false_before_first_poll(self): @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) - async def test_second_start_call_raises(self, mock_interval): + async def test_second_start_call_is_a_no_op(self, mock_interval): + # AsyncLDClient.start() is documented as an idempotent no-op, so + # nothing underneath it may raise on a repeat call. mock_interval.__get__ = MagicMock(return_value=0) processor = make_processor() processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) processor.start() - # Like a thread, the polling task can only be started once - with pytest.raises(RuntimeError): - processor.start() + first_task = processor._task + processor.start() + + assert processor._task is first_task await processor.stop() @@ -448,15 +641,14 @@ async def test_stop_closes_transport_when_cancelled_mid_wait(self): requester.close = AsyncMock() processor = make_processor(requester=requester) - # Replace the repeating task so wait_stopped() hangs until we cancel stop(). + # Replace the task's wait so it hangs until we cancel stop(). waiting = asyncio.Event() async def hang(): waiting.set() await asyncio.Event().wait() - processor._task = MagicMock() - processor._task.wait_stopped = hang + processor._task.wait_stopped = hang # type: ignore[method-assign] stop_task = asyncio.create_task(processor.stop()) await asyncio.wait_for(waiting.wait(), timeout=2.0) diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index ffd5a9ea..7b7d3adb 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -7,9 +7,13 @@ import asyncio import json +import logging +import ssl from unittest import mock +import aiohttp import pytest +from aiohttp.client_reqrep import ConnectionKey from ldclient.config import Config from ldclient.impl.datasource import async_streaming @@ -17,9 +21,19 @@ AsyncStreamingUpdateProcessor ) from ldclient.impl.model import ModelEntity +from ldclient.impl.retry import ( + EXTENDED_INITIAL_DELAY, + EXTENDED_MAX_DELAY, + STREAMING_MAX_DELAY, + STREAMING_RESET_INTERVAL, + AfterHealthyFor, + RetryState, + for_streaming +) from ldclient.interfaces import DataSourceErrorKind, DataSourceState from ldclient.testing.builders import FlagBuilder, SegmentBuilder from ldclient.testing.mock_async_components import MockAsyncFeatureStore +from ldclient.testing.test_util import no_retry_jitter from ldclient.versioned_data_kind import FEATURES, SEGMENTS @@ -75,6 +89,45 @@ async def _actions_generator(actions: list): await asyncio.Event().wait() +def _fast_retry_state(delay: float = 0.001) -> RetryState: + """A retry state with tiny delays, so a test does not have to wait out the + real extended-regime delay of five minutes.""" + return RetryState( + initial_delay=delay, + normal_ceiling=delay, + extended_initial_delay=delay, + extended_ceiling=delay, + reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), + ) + + +# aiohttp's connection errors read the connection key when they are turned +# into a string, which the data source does, so a real one is needed here. +_CONNECTION_KEY = ConnectionKey( + host='stream.launchdarkly.com', + port=443, + is_ssl=True, + ssl=True, + proxy=None, + proxy_auth=None, + proxy_headers_hash=None, + server_hostname=None, +) + + +def _zero_delay_retry_state() -> RetryState: + """A retry state whose normal regime waits no time at all, so a test can + drive ``_handle_error`` without a real sleep. The extended bounds stay + real, so a misclassification still shows up in ``max_delay``.""" + return RetryState( + initial_delay=0, + normal_ceiling=STREAMING_MAX_DELAY, + extended_initial_delay=EXTENDED_INITIAL_DELAY, + extended_ceiling=EXTENDED_MAX_DELAY, + reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), + ) + + class _MockSSE: """Stand-in for AsyncSSEClient exposing the surface the processor uses.""" @@ -101,31 +154,33 @@ class _MockSSEFactory: def __init__(self, actions: list): self._actions = actions self.created: list = [] + self.sdk_managed_retry: list = [] - def create(self, url: str, initial_retry_delay: float) -> _MockSSE: + def create(self, url: str, initial_retry_delay: float, sdk_managed_retry: bool = False) -> _MockSSE: sse = _MockSSE(self._actions) self.created.append(sse) + self.sdk_managed_retry.append(sdk_managed_retry) return sse -def _make_processor(actions, config=None, store=None, ready_event=None, diag=None): +def _make_processor(actions, config=None, store=None, ready_event=None, diag=None, retry_state=None): config = config or _make_config() store = store or MockAsyncFeatureStore() ready_event = ready_event or asyncio.Event() factory = _MockSSEFactory(actions) - proc = AsyncStreamingUpdateProcessor(config, store, ready_event, diag, factory) + proc = AsyncStreamingUpdateProcessor(config, store, ready_event, diag, factory, retry_state=retry_state) return proc, store, ready_event, factory async def _run_with_actions(actions: list, config=None, store=None, ready_event=None, - diag=None, extra_ready_timeout=3.0): + diag=None, extra_ready_timeout=3.0, retry_state=None): """Run the processor against a fake SSE action sequence. Starts the processor and waits for the ready event (up to *extra_ready_timeout* seconds), then returns ``(processor, store, ready_event, factory)``. """ - proc, store, ready, factory = _make_processor(actions, config, store, ready_event, diag) + proc, store, ready, factory = _make_processor(actions, config, store, ready_event, diag, retry_state) proc.start() try: await asyncio.wait_for(ready.wait(), timeout=extra_ready_timeout) @@ -223,27 +278,111 @@ async def test_fault_with_error_does_not_set_ready_by_itself(): @pytest.mark.asyncio -async def test_fault_none_error_is_ignored(): - """A Fault with error=None (clean close) should not update status or stop the processor.""" +async def test_server_close_backs_off_and_does_not_stop_the_processor(): + """A Fault with error=None is the server closing a connection it normally + leaves open. The SDK backs off rather than reconnecting at once, but the + processor keeps running.""" flag = FlagBuilder('f1').version(1).build() put_data = _make_put_data(flags={'f1': _item_dict(flag)}) actions = [ _start(), _event('put', put_data), - _fault(error=None), # clean close — should be ignored + _fault(error=None), # clean close by the server ] - proc, store, ready, _ = await _run_with_actions(actions) + retry = _fast_retry_state() + proc, store, ready, factory = _make_processor(actions, retry_state=retry) + proc.start() + await asyncio.wait_for(ready.wait(), timeout=3.0) + await _wait_until(lambda: retry.attempts >= 1) - assert ready.is_set() assert store.initialized + assert not factory.created[0].closed + assert not retry.in_extended_regime + + await proc.stop() + + +@pytest.mark.asyncio +async def test_server_close_reports_a_network_error(): + from ldclient.impl.datasource.async_status import ( + AsyncDataSourceUpdateSinkImpl + ) + from ldclient.impl.listeners import Listeners + + store = MockAsyncFeatureStore() + statuses = [] + listeners = Listeners() + listeners.add(lambda s: statuses.append(s)) + + config = _make_config() + config._data_source_update_sink = AsyncDataSourceUpdateSinkImpl(store, listeners, Listeners()) + + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + actions = [_start(), _event('put', put_data), _fault(error=None)] + + proc, store, ready, _ = _make_processor(actions, config=config, store=store, retry_state=_fast_retry_state()) + proc.start() + await _wait_until(lambda: any(s.state == DataSourceState.INTERRUPTED for s in statuses)) + + interrupted = [s for s in statuses if s.state == DataSourceState.INTERRUPTED] + assert interrupted[0].error is not None + assert interrupted[0].error.kind == DataSourceErrorKind.NETWORK_ERROR await proc.stop() @pytest.mark.asyncio -async def test_unrecoverable_http_error_stops_processor(): - """An unrecoverable HTTP status closes the stream and reports OFF with error info.""" +async def test_repeated_server_closes_stay_on_the_normal_curve(): + """A load balancer draining during a rolling deploy closes streams + cleanly, over and over. That must never reach the extended regime.""" + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + actions = [] + for _ in range(10): + actions += [_start(), _event('put', put_data), _fault(error=None)] + + retry = _fast_retry_state() + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: retry.attempts >= 10, timeout=5.0) + + assert not retry.in_extended_regime + assert retry.max_delay == _fast_retry_state().max_delay + + await proc.stop() + + +@pytest.mark.asyncio +async def test_our_own_interrupt_is_not_counted_as_a_server_close(): + """Bad JSON makes the SDK drop the connection itself. The SSE client then + reports that close as a Fault with no error, and counting it would record + the same failure twice and wait twice.""" + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + actions = [ + _start(), + _event('put', put_data), + _event('patch', 'not valid json'), + _fault(error=None), # the close our own interrupt caused + ] + + retry = _fast_retry_state() + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: retry.attempts >= 1) + await asyncio.sleep(0.1) + + assert retry.attempts == 1 + + await proc.stop() + + +@pytest.mark.asyncio +async def test_unexpected_http_error_keeps_the_processor_running(): + """A rejected SDK key is retried like any other failure. The state never + goes OFF, and initialization is not falsely unblocked.""" from ld_eventsource.errors import HTTPStatusError from ldclient.impl.datasource.async_status import ( @@ -261,15 +400,17 @@ async def test_unrecoverable_http_error_stops_processor(): actions = [_start(), _fault(error=HTTPStatusError(401))] - proc, store, ready, factory = await _run_with_actions(actions, config=config, store=store) + proc, store, ready, factory = await _run_with_actions( + actions, config=config, store=store, extra_ready_timeout=0.2, + retry_state=_fast_retry_state(), + ) - # The unrecoverable error unblocks initialization without initializing the store. - assert ready.is_set() + assert not ready.is_set() assert not proc.initialized() - assert factory.created[0].closed + assert not factory.created[0].closed + assert all(s.state != DataSourceState.OFF for s in statuses) assert any( - s.state == DataSourceState.OFF - and s.error is not None + s.error is not None and s.error.kind == DataSourceErrorKind.ERROR_RESPONSE and s.error.status_code == 401 for s in statuses @@ -278,6 +419,160 @@ async def test_unrecoverable_http_error_stops_processor(): await proc.stop() +@pytest.mark.asyncio +async def test_unexpected_http_error_moves_to_the_extended_regime(): + from ld_eventsource.errors import HTTPStatusError + + actions = [_start(), _fault(error=HTTPStatusError(401))] + retry = _fast_retry_state() + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: retry.in_extended_regime) + + await proc.stop() + + +@pytest.mark.asyncio +async def test_normal_http_error_stays_in_the_normal_regime(): + from ld_eventsource.errors import HTTPStatusError + + actions = [_start(), _fault(error=HTTPStatusError(503))] + retry = _fast_retry_state() + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: retry.attempts >= 1) + + assert not retry.in_extended_regime + + await proc.stop() + + +@pytest.mark.parametrize( + "error", + [ + aiohttp.ClientConnectorCertificateError( + _CONNECTION_KEY, ssl.SSLCertVerificationError("self-signed certificate") + ), + aiohttp.ClientConnectorSSLError(_CONNECTION_KEY, OSError("handshake failed")), + ssl.SSLEOFError("EOF occurred in violation of protocol"), + ConnectionResetError(104, "reset by peer"), + ], + ids=["aiohttp-certificate", "aiohttp-tls", "peer-close-handshake", "reset"], +) +@pytest.mark.asyncio +async def test_transport_failures_stay_in_the_normal_regime(error): + """No transport failure reaches the extended regime, an aiohttp + certificate failure included. Only an HTTP status can do that.""" + retry = _zero_delay_retry_state() + proc, store, ready, _ = _make_processor([], retry_state=retry) + proc._running = True + + # A misclassification would wait five minutes here, so bound the wait + # rather than let the test hang. + assert await asyncio.wait_for(proc._handle_error(error), timeout=2.0) + + assert not retry.in_extended_regime + assert retry.max_delay == STREAMING_MAX_DELAY + + +class _NoSleep: + """Stands in for the ``asyncio`` module inside async_streaming, so the wait + in _handle_error returns at once. ``sleep`` is all that module uses.""" + + def __init__(self): + self.slept: list = [] + + async def sleep(self, seconds): + self.slept.append(seconds) + + +@pytest.mark.asyncio +async def test_the_log_reports_the_growing_retry_delay(caplog): + """The message has to carry the real delay, so someone reading logs can see + the backoff working. The vaguer wording it replaced could not show this.""" + from ld_eventsource.errors import HTTPStatusError + + caplog.set_level(logging.WARNING) + no_sleep = _NoSleep() + + with no_retry_jitter(), mock.patch.object(async_streaming, 'asyncio', no_sleep): + retry = for_streaming(1) + proc, store, ready, _ = _make_processor([], retry_state=retry) + proc._running = True + + await proc._handle_error(HTTPStatusError(401)) + await proc._handle_error(HTTPStatusError(401)) + + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + "Received HTTP error 401 (invalid SDK key) for stream connection - will retry in 300.0s", + "Received HTTP error 401 (invalid SDK key) for stream connection - will retry in 600.0s", + ] + # An error a person has to fix is logged at error level, every time. + assert [r.levelno for r in caplog.records] == [logging.ERROR, logging.ERROR] + # The reported delay is the one actually waited. + assert no_sleep.slept == [5 * 60, 10 * 60] + + +@pytest.mark.asyncio +async def test_the_processor_asks_the_factory_to_leave_the_delay_to_the_sdk(): + proc, store, ready, factory = _make_processor([]) + proc.start() + await _wait_until(lambda: len(factory.created) > 0) + + assert factory.sdk_managed_retry == [True] + + await proc.stop() + + +@pytest.mark.asyncio +async def test_healthy_operation_is_signalled_once_per_stream(): + """The reset window must start at the first message of a stream. Signalling + again on every later message would keep pushing the window out.""" + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + patch_data = _make_patch_data(FEATURES, _item_dict(FlagBuilder('f1').version(2).build())) + actions = [_start(), _event('put', put_data), _event('patch', patch_data)] + + retry = _fast_retry_state() + healthy_at = [] + retry.record_healthy = lambda: healthy_at.append(len(healthy_at)) # type: ignore[method-assign] + + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: len(healthy_at) > 0) + await asyncio.sleep(0.05) + + assert healthy_at == [0] + + await proc.stop() + + +@pytest.mark.asyncio +async def test_a_fresh_stream_signals_healthy_operation_again(): + from ld_eventsource.errors import HTTPStatusError + + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + actions = [ + _start(), + _event('put', put_data), + _fault(error=HTTPStatusError(503)), + _start(), + _event('put', put_data), + ] + + retry = _fast_retry_state() + healthy_count = [] + retry.record_healthy = lambda: healthy_count.append(1) # type: ignore[method-assign] + + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: len(healthy_count) >= 2) + + await proc.stop() + + @pytest.mark.asyncio async def test_stop_closes_sse_and_finishes_task(): flag = FlagBuilder('f1').version(1).build() @@ -295,13 +590,16 @@ async def test_stop_closes_sse_and_finishes_task(): @pytest.mark.asyncio -async def test_second_start_raises(): +async def test_second_start_is_a_no_op(): + """AsyncLDClient.start() is documented as an idempotent no-op, so nothing + underneath it may raise on a repeat call.""" actions = [_start()] - proc, store, ready, _ = _make_processor(actions) + proc, store, ready, factory = _make_processor(actions) proc.start() try: - with pytest.raises(RuntimeError): - proc.start() + proc.start() + await _wait_until(lambda: len(factory.created) > 0) + assert len(factory.created) == 1 finally: await proc.stop() diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index 06e92d89..9d073167 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -1,13 +1,22 @@ +import logging +import ssl import threading import time import mock +import pytest from ldclient.config import Config from ldclient.feature_store import InMemoryFeatureStore from ldclient.impl.datasource.polling import PollingUpdateProcessor from ldclient.impl.datasource.status import DataSourceUpdateSinkImpl from ldclient.impl.listeners import Listeners +from ldclient.impl.retry import ( + POLLING_RESET_SUCCESSES, + AfterConsecutiveSuccesses, + RetryState, + for_polling +) from ldclient.impl.util import UnsuccessfulResponseException from ldclient.interfaces import ( DataSourceErrorKind, @@ -16,7 +25,7 @@ ) from ldclient.testing.builders import * from ldclient.testing.stub_util import MockFeatureRequester, MockResponse -from ldclient.testing.test_util import SpyListener +from ldclient.testing.test_util import SpyListener, no_retry_jitter from ldclient.versioned_data_kind import FEATURES, SEGMENTS pp = None @@ -37,9 +46,22 @@ def teardown_function(): pp.stop() -def setup_processor(config): +def fast_retry_state(delay=0.05): + """A retry state with tiny delays, so a test does not have to wait out the + real extended-regime delay of five minutes.""" + return RetryState( + initial_delay=delay, + normal_ceiling=delay, + extended_initial_delay=delay, + extended_ceiling=delay, + reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), + operating_cadence=delay, + ) + + +def setup_processor(config, retry_state=None): global pp - pp = PollingUpdateProcessor(config, mock_requester, store, ready) + pp = PollingUpdateProcessor(config, mock_requester, store, ready, retry_state=retry_state) pp.start() @@ -77,12 +99,16 @@ def test_general_connection_error_does_not_cause_immediate_failure(ignore_mock): assert mock_requester.request_count >= 2 -def test_http_401_error_causes_immediate_failure(): - verify_unrecoverable_http_error(401) +def test_http_401_error_does_not_stop_polling(): + verify_unexpected_http_error(401) -def test_http_403_error_causes_immediate_failure(): - verify_unrecoverable_http_error(401) +def test_http_403_error_does_not_stop_polling(): + verify_unexpected_http_error(403) + + +def test_http_404_error_does_not_stop_polling(): + verify_unexpected_http_error(404) def test_http_408_error_does_not_cause_immediate_failure(): @@ -102,7 +128,10 @@ def test_http_503_error_does_not_cause_immediate_failure(): @mock.patch('ldclient.config.Config.poll_interval', new_callable=mock.PropertyMock, return_value=0.1) -def verify_unrecoverable_http_error(http_status_code, ignore_mock): +def verify_unexpected_http_error(http_status_code, ignore_mock): + """An error that needs a person to fix it -- a rejected SDK key, say -- is + still retried. It must not stop the poller, must not report OFF, and must + not falsely unblock initialization.""" spy = SpyListener() listeners = Listeners() listeners.add(spy) @@ -111,16 +140,228 @@ def verify_unrecoverable_http_error(http_status_code, ignore_mock): config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) mock_requester.exception = UnsuccessfulResponseException(http_status_code) - setup_processor(config) + setup_processor(config, retry_state=fast_retry_state()) finished = ready.wait(0.5) - assert finished + assert not finished assert not pp.initialized() + assert mock_requester.request_count >= 2 + + assert len(spy.statuses) > 1 + for status in spy.statuses: + assert status.state == DataSourceState.INITIALIZING + assert status.error.kind == DataSourceErrorKind.ERROR_RESPONSE + assert status.error.status_code == http_status_code + + +@mock.patch('ldclient.config.Config.poll_interval', new_callable=mock.PropertyMock, return_value=0.1) +def test_unexpected_http_error_moves_to_the_extended_regime(ignore_mock): + retry = for_polling(0.1) + mock_requester.exception = UnsuccessfulResponseException(401) + setup_processor(Config("SDK_KEY"), retry_state=retry) + + # The extended regime starts at five minutes, so only the first poll runs. + assert not ready.wait(0.4) + assert retry.in_extended_regime assert mock_requester.request_count == 1 - assert len(spy.statuses) == 1 - assert spy.statuses[0].state == DataSourceState.OFF - assert spy.statuses[0].error.kind == DataSourceErrorKind.ERROR_RESPONSE - assert spy.statuses[0].error.status_code == http_status_code + +def test_the_first_success_after_an_outage_polls_at_the_cadence(): + # RETRY 1.4.8: a backoff wait applies to a retry, not to every operation. + # _poll returns the wait, so this reads it directly rather than measuring + # elapsed time. + with no_retry_jitter(): + retry = for_polling(30) + processor = PollingUpdateProcessor(Config("SDK_KEY"), mock_requester, store, ready, retry_state=retry) + + mock_requester.exception = UnsuccessfulResponseException(401) + processor._poll() + assert retry.next_delay == 5 * 60 + + mock_requester.exception = None + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + processor._poll() + assert retry.next_delay == 30 + assert retry.in_extended_regime, "one success restores the cadence but does not reset" + + processor._poll() + assert retry.next_delay == 30 + assert not retry.in_extended_regime, "two successes in a row reset the state" + + +@pytest.mark.parametrize( + "error", + [ + ssl.SSLCertVerificationError("unable to get local issuer certificate"), + ssl.SSLEOFError("EOF occurred in violation of protocol"), + ConnectionResetError(104, "reset by peer"), + ], + ids=["certificate", "peer-close-handshake", "reset"], +) +def test_transport_failures_poll_again_at_the_cadence(error): + """No transport failure reaches the extended regime, a bad certificate + included. Only an HTTP status can do that.""" + retry = for_polling(30) + processor = PollingUpdateProcessor(Config("SDK_KEY"), mock_requester, store, ready, retry_state=retry) + + mock_requester.exception = error + processor._poll() + assert retry.next_delay == 30 + assert not retry.in_extended_regime + + +@mock.patch('ldclient.config.Config.poll_interval', new_callable=mock.PropertyMock, return_value=0.05) +def test_failure_transitions_from_valid(ignore_mock): + """A rejected SDK key after a poll has succeeded reports INTERRUPTED. OFF + is reserved for an explicit shutdown.""" + spy = SpyListener() + listeners = Listeners() + listeners.add(spy) + + config = Config("SDK_KEY") + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + setup_processor(config, retry_state=fast_retry_state()) + assert ready.wait(2) + assert spy.statuses[0].state == DataSourceState.VALID + + mock_requester.exception = UnsuccessfulResponseException(401) + deadline = time.time() + 2 + while spy.statuses[-1].state == DataSourceState.VALID and time.time() < deadline: + time.sleep(0.01) + + assert spy.statuses[-1].state == DataSourceState.INTERRUPTED + assert spy.statuses[-1].error.kind == DataSourceErrorKind.ERROR_RESPONSE + assert spy.statuses[-1].error.status_code == 401 + assert all(s.state != DataSourceState.OFF for s in spy.statuses) + + +def test_second_start_is_a_no_op(): + """A second start() must not raise. Thread.start() would, so the processor + guards it.""" + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + setup_processor(Config("SDK_KEY")) + pp.start() + + assert ready.wait(2) + assert pp.initialized() + + +def test_the_log_reports_the_growing_retry_delay(caplog): + """The message has to carry the real delay, so someone reading logs can see + the backoff working.""" + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_polling(30) + processor = PollingUpdateProcessor(Config("SDK_KEY"), mock_requester, store, ready, retry_state=retry) + mock_requester.exception = UnsuccessfulResponseException(401) + processor._poll() + processor._poll() + + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + "Received HTTP error 401 (invalid SDK key) for polling request - will retry in 300.0s", + "Received HTTP error 401 (invalid SDK key) for polling request - will retry in 600.0s", + ] + # An error a person has to fix is logged at error level, every time. + assert [r.levelno for r in caplog.records] == [logging.ERROR, logging.ERROR] + + +def test_a_normal_failure_logs_the_poll_interval_at_warning_level(caplog): + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_polling(30) + processor = PollingUpdateProcessor(Config("SDK_KEY"), mock_requester, store, ready, retry_state=retry) + mock_requester.exception = UnsuccessfulResponseException(503) + processor._poll() + + record = caplog.records[0] + assert record.getMessage() == "Received HTTP error 503 for polling request - will retry in 30.0s" + assert record.levelno == logging.WARNING + + +def test_a_transport_error_reports_a_delay_and_keeps_its_stacktrace(caplog): + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_polling(30) + processor = PollingUpdateProcessor(Config("SDK_KEY"), mock_requester, store, ready, retry_state=retry) + mock_requester.exception = ConnectionResetError(104, "reset by peer") + processor._poll() + + record = caplog.records[0] + assert record.getMessage() == "Error encountered when updating flags: [Errno 104] reset by peer - will retry in 30.0s" + # The handler has exited by the time this is logged, so the exception has to + # be carried explicitly for the traceback to survive. + assert record.exc_info is not None + + +def _polling_thread(): + """Finds the task's worker thread by name, so a test can prove it exited + without reaching into the task's private state.""" + return next((t for t in threading.enumerate() if t.name == "ldclient.datasource.polling.repeating"), None) + + +def test_an_extended_regime_wait_is_cut_short_by_stop(): + """The reason the wait has to be interruptible at all. A 401 puts the next + poll five minutes out, and shutdown must not sit through it.""" + mock_requester.exception = UnsuccessfulResponseException(401) + retry = for_polling(30) + setup_processor(Config("SDK_KEY"), retry_state=retry) + + # Let the first poll happen, so the task is inside the long wait. + deadline = time.time() + 2 + while mock_requester.request_count < 1 and time.time() < deadline: + time.sleep(0.01) + assert mock_requester.request_count == 1 + assert retry.in_extended_regime, "the wait under test should be minutes long" + + worker = _polling_thread() + assert worker is not None + + started = time.time() + pp.stop() + worker.join(2) + elapsed = time.time() - started + + # Without an interruptible wait this join would time out and the thread + # would still be sitting in a 300-second sleep. + assert not worker.is_alive() + assert elapsed < 1 + + +def test_stop_reports_off(): + spy = SpyListener() + listeners = Listeners() + listeners.add(spy) + + config = Config("SDK_KEY") + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + setup_processor(config) + assert ready.wait(2) + + pp.stop() + + assert spy.statuses[-1].state == DataSourceState.OFF + + +def test_valid_status_is_reported_before_ready_is_set(): + # Mirrors go-server-sdk#442: a caller that wakes on readiness must not + # still be able to read INITIALIZING. + observed = [] + listeners = Listeners() + listeners.add(lambda status: observed.append((status.state, ready.is_set()))) + + config = Config("SDK_KEY") + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + setup_processor(config) + assert ready.wait(2) + + assert observed[0] == (DataSourceState.VALID, False) @mock.patch('ldclient.config.Config.poll_interval', new_callable=mock.PropertyMock, return_value=0.1) diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 98c9d02a..6dd782de 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -1,15 +1,32 @@ +import logging +import ssl import time from threading import Event from typing import List import pytest +from ld_eventsource import SSEClient +from ld_eventsource.actions import Fault +from ld_eventsource.config import ( + ConnectStrategy, + ErrorStrategy, + RetryDelayStrategy +) +from ld_eventsource.errors import HTTPStatusError from ldclient.config import Config from ldclient.feature_store import InMemoryFeatureStore +from ldclient.impl.datasource.datasource_common import StreamClosedError from ldclient.impl.datasource.status import DataSourceUpdateSinkImpl from ldclient.impl.datasource.streaming import StreamingUpdateProcessor from ldclient.impl.events.diagnostics import _DiagnosticAccumulator from ldclient.impl.listeners import Listeners +from ldclient.impl.retry import ( + STREAMING_RESET_INTERVAL, + AfterHealthyFor, + RetryState, + for_streaming +) from ldclient.interfaces import ( DataSourceErrorKind, DataSourceState, @@ -30,12 +47,25 @@ make_put_event, stream_content ) -from ldclient.testing.test_util import SpyListener +from ldclient.testing.test_util import SpyListener, no_retry_jitter from ldclient.version import VERSION from ldclient.versioned_data_kind import FEATURES, SEGMENTS brief_delay = 0.001 + +def fast_retry_state(delay=brief_delay): + """A retry state with tiny delays, so a test does not have to wait out the + real extended-regime delay of five minutes.""" + return RetryState( + initial_delay=delay, + normal_ceiling=delay, + extended_initial_delay=delay, + extended_ceiling=delay, + reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), + ) + + # These long timeouts are necessary because of a problem in the Windows CI environment where HTTP requests to # the test server running at localhost tests are *extremely* slow. It looks like a similar issue to what's # described at https://stackoverflow.com/questions/2617615/slow-python-http-server-on-localhost but we had no @@ -257,7 +287,9 @@ def test_recoverable_http_error(status): @pytest.mark.parametrize("status", [401, 403, 404]) -def test_unrecoverable_http_error(status): +def test_unexpected_http_error_backs_off_a_long_way(status): + """An error that needs a person to fix it does not stop the stream, but the + next attempt is five minutes out, so only one request is made here.""" error_handler = BasicResponse(status) store = InMemoryFeatureStore() ready = Event() @@ -269,11 +301,318 @@ def test_unrecoverable_http_error(status): with StreamingUpdateProcessor(config, store, ready, None) as sp: sp.start() - ready.wait(5) + # Initialization is not falsely unblocked: the caller waits out + # its own start_wait and then finds the client uninitialized. + assert not ready.wait(1) assert not sp.initialized() + assert sp.is_alive() + assert sp._retry.in_extended_regime server.should_have_requests(1) +@pytest.mark.parametrize("status", [401, 403, 404]) +def test_unexpected_http_error_keeps_retrying(status): + """The same failure with the delay compressed: the stream recovers once the + service does, rather than staying down for ever.""" + error_handler = BasicResponse(status) + store = InMemoryFeatureStore() + ready = Event() + with start_server() as server: + with stream_content(make_put_event()) as stream: + error_then_success = SequentialHandler(error_handler, stream) + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + + spy = SpyListener() + listeners = Listeners() + listeners.add(spy) + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + server.for_path('/all', error_then_success) + + with StreamingUpdateProcessor(config, store, ready, None, retry_state=fast_retry_state()) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + server.should_have_requests(2) + + assert all(s.state != DataSourceState.OFF for s in spy.statuses) + assert spy.statuses[0].state == DataSourceState.INITIALIZING + assert spy.statuses[0].error.status_code == status + assert spy.statuses[-1].state == DataSourceState.VALID + + +def test_sse_client_hands_us_the_fault_before_it_waits(): + """Pins the ld_eventsource ordering the SDK relies on. + + The SDK computes and takes the retry delay itself, which only works + because SSEClient yields the Fault to the caller before its next connect + attempt sleeps. A library change that slept first would make this test + time out rather than fail quietly. + """ + with start_server() as server: + server.for_path('/all', BasicResponse(503)) + client = SSEClient( + connect=ConnectStrategy.http(url=server.uri + '/all'), + error_strategy=ErrorStrategy.always_continue(), + initial_retry_delay=30, + retry_delay_strategy=RetryDelayStrategy.default(max_delay=30, backoff_multiplier=2), + retry_delay_reset_threshold=0, + ) + try: + started = time.time() + first = next(iter(client.all)) + elapsed = time.time() - started + finally: + client.close() + + assert isinstance(first, Fault) + assert isinstance(first.error, HTTPStatusError) + # The library has a long delay queued up but has not taken it yet. + assert client.next_retry_delay >= 15 + assert elapsed < 5 + + +def test_the_sdk_configures_the_sse_client_never_to_wait(): + """The SDK owns the delay, so the library's own delay must stay at zero + however long the outage lasts.""" + store = InMemoryFeatureStore() + with start_server() as server: + server.for_path('/all', BasicResponse(503)) + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=30) + sp = StreamingUpdateProcessor(config, store, Event(), None) + client = sp._create_sse_client() + try: + actions = iter(client.all) + first = next(actions) + second = next(actions) + finally: + client.close() + + assert isinstance(first, Fault) + assert isinstance(second, Fault) + assert client.next_retry_delay == 0 + + +def test_server_close_backs_off_and_keeps_the_stream_running(): + """The service normally leaves the connection open, so a clean close is a + connection failure: the SDK reports it and backs off, rather than + reconnecting in a tight loop.""" + store = InMemoryFeatureStore() + ready = Event() + flagv1 = FlagBuilder('flagkey').version(1).build() + flagv2 = FlagBuilder('flagkey').version(2).build() + + with start_server() as server: + with stream_content(make_put_event([flagv1])) as stream1: + with stream_content(make_put_event([flagv2])) as stream2: + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + + spy = SpyListener() + listeners = Listeners() + listeners.add(spy) + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + server.for_path('/all', SequentialHandler(stream1, stream2)) + + retry = fast_retry_state() + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + + stream1.close() + expect_update(store, FEATURES, flagv2) + + assert retry.attempts >= 1 + assert not retry.in_extended_regime + + interrupted = [s for s in spy.statuses if s.state == DataSourceState.INTERRUPTED] + assert len(interrupted) >= 1 + assert interrupted[0].error.kind == DataSourceErrorKind.NETWORK_ERROR + + +def test_server_close_uses_the_normal_delay_curve(): + """A clean close is a NORMAL failure. Classifying it UNEXPECTED would put + a routine load-balancer drain into the extended regime and take a fleet + out of service for up to an hour.""" + store = InMemoryFeatureStore() + config = Config(sdk_key='sdk-key', initial_reconnect_delay=1) + retry = for_streaming(1) + sp = StreamingUpdateProcessor(config, store, Event(), None, retry_state=retry) + sp._running = True + sp._stop_event.set() # so the wait returns at once + + delays = [] + for _ in range(8): + sp._handle_error(StreamClosedError()) + delays.append(retry.max_delay) + + assert not retry.in_extended_regime + assert delays == [30] * 8 + + +def test_our_own_interrupt_is_not_counted_as_a_server_close(): + """Bad JSON makes the SDK drop the connection itself. The SSE client then + reports that close as a Fault with no error, and counting it would record + the same failure twice and wait twice.""" + store = InMemoryFeatureStore() + ready = Event() + + with start_server() as server: + with stream_content(make_put_event()) as valid_stream, stream_content(make_invalid_put_event()) as invalid_stream: + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + + statuses: List[DataSourceStatus] = [] + listeners = Listeners() + + # The stream fixture holds the connection open, so it has to be + # closed for the server to move on to the next handler. This + # mirrors test_invalid_json_triggers_listener. + def listener(s): + if len(statuses) == 0: + invalid_stream.close() + statuses.append(s) + + listeners.add(listener) + + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + server.for_path('/all', SequentialHandler(invalid_stream, valid_stream)) + + retry = fast_retry_state() + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + server.should_have_requests(2) + + # One failure for the bad JSON, not a second for the close it + # caused. + assert retry.attempts == 1 + + +def test_second_start_is_a_no_op(): + """A second start() must not raise. Thread.start() would, so the processor + guards it.""" + store = InMemoryFeatureStore() + ready = Event() + + with start_server() as server: + with stream_content(make_put_event()) as stream: + config = Config(sdk_key='sdk-key', stream_uri=server.uri) + server.for_path('/all', stream) + + with StreamingUpdateProcessor(config, store, ready, None) as sp: + sp.start() + sp.start() + ready.wait(start_wait) + assert sp.initialized() + + +def _handle_errors_without_waiting(retry, errors): + """Drives _handle_error for each error and returns nothing. The stop event + is pre-set so the interruptible wait returns at once.""" + store = InMemoryFeatureStore() + config = Config(sdk_key='sdk-key', initial_reconnect_delay=1) + sp = StreamingUpdateProcessor(config, store, Event(), None, retry_state=retry) + sp._running = True + sp._stop_event.set() + for error in errors: + sp._handle_error(error) + + +def test_the_log_reports_the_growing_retry_delay(caplog): + """The message has to carry the real delay, so someone reading logs can see + the backoff working. The vaguer wording it replaced could not show this.""" + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_streaming(1) + _handle_errors_without_waiting(retry, [HTTPStatusError(401), HTTPStatusError(401)]) + + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + "Received HTTP error 401 (invalid SDK key) for stream connection - will retry in 300.0s", + "Received HTTP error 401 (invalid SDK key) for stream connection - will retry in 600.0s", + ] + # An error a person has to fix is logged at error level, every time. + assert [r.levelno for r in caplog.records] == [logging.ERROR, logging.ERROR] + + +def test_a_normal_failure_logs_a_short_delay_at_warning_level(caplog): + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_streaming(1) + _handle_errors_without_waiting(retry, [HTTPStatusError(503), HTTPStatusError(503)]) + + messages = [r.getMessage() for r in caplog.records] + assert messages == [ + "Received HTTP error 503 for stream connection - will retry in 1.0s", + "Received HTTP error 503 for stream connection - will retry in 2.0s", + ] + assert [r.levelno for r in caplog.records] == [logging.WARNING, logging.WARNING] + + +def test_a_server_close_and_a_transport_error_both_report_a_delay(caplog): + caplog.set_level(logging.WARNING) + + with no_retry_jitter(): + retry = for_streaming(1) + _handle_errors_without_waiting(retry, [StreamClosedError(), ConnectionResetError(104, "reset by peer")]) + + messages = [r.getMessage() for r in caplog.records] + assert messages[0] == "The server closed the stream connection - will retry in 1.0s" + assert messages[1] == "Error on stream connection: [Errno 104] reset by peer - will retry in 2.0s" + + +def test_healthy_operation_is_signalled_once_per_stream(): + """The reset window must start at the first message of a stream. Signalling + again on every later message would keep pushing the window out.""" + store = InMemoryFeatureStore() + ready = Event() + flag = FlagBuilder('flagkey').version(1).build() + + with start_server() as server: + with stream_content(make_put_event([flag]) + make_patch_event(FEATURES, flag)) as stream: + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + server.for_path('/all', stream) + + retry = fast_retry_state() + healthy_count = [] + retry.record_healthy = lambda: healthy_count.append(1) # type: ignore[method-assign] + + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + expect_update(store, FEATURES, flag) + + assert healthy_count == [1] + + +@pytest.mark.parametrize( + "error", + [ + ssl.SSLCertVerificationError("unable to get local issuer certificate"), + ssl.SSLEOFError("EOF occurred in violation of protocol"), + ConnectionResetError(104, "reset by peer"), + ], + ids=["certificate", "peer-close-handshake", "reset"], +) +def test_transport_failures_stay_in_the_normal_regime(error): + """No transport failure reaches the extended regime, a bad certificate + included. Only an HTTP status can do that.""" + store = InMemoryFeatureStore() + config = Config(sdk_key='sdk-key', initial_reconnect_delay=1) + retry = for_streaming(1) + sp = StreamingUpdateProcessor(config, store, Event(), None, retry_state=retry) + sp._running = True + sp._stop_event.set() # so the wait returns at once + + sp._handle_error(error) + + assert not retry.in_extended_regime + assert retry.max_delay == 30 + + def test_http_proxy(monkeypatch): def _stream_processor_proxy_test(server, config, secure): store = InMemoryFeatureStore() @@ -407,6 +746,8 @@ def listener(s): def test_failure_transitions_from_valid(): + """A rejected SDK key after the stream was valid reports INTERRUPTED. OFF + is reserved for an explicit shutdown.""" store = InMemoryFeatureStore() ready = Event() error_handler = BasicResponse(401) @@ -426,14 +767,15 @@ def test_failure_transitions_from_valid(): with StreamingUpdateProcessor(config, store, ready, None) as sp: sp.start() - ready.wait(start_wait) + # The 401 is retried five minutes out, so readiness never fires. + assert not ready.wait(1) server.should_have_requests(1) assert len(spy.statuses) == 2 assert spy.statuses[0].state == DataSourceState.VALID - assert spy.statuses[1].state == DataSourceState.OFF + assert spy.statuses[1].state == DataSourceState.INTERRUPTED assert spy.statuses[1].error.kind == DataSourceErrorKind.ERROR_RESPONSE assert spy.statuses[1].error.status_code == 401 diff --git a/ldclient/testing/impl/test_repeating_task.py b/ldclient/testing/impl/test_repeating_task.py index 7d29cbf3..0fdf89ff 100644 --- a/ldclient/testing/impl/test_repeating_task.py +++ b/ldclient/testing/impl/test_repeating_task.py @@ -1,13 +1,14 @@ +import logging import time from queue import Empty, Queue from threading import Event -from ldclient.impl.repeating_task import RepeatingTask +from ldclient.impl.repeating_task import DelaySource, FixedDelay, RepeatingTask def test_task_does_not_start_when_created(): signal = Event() - task = RepeatingTask("ldclient.testing.set-signal", 0.01, 0, lambda: signal.set()) + task = RepeatingTask.at_interval("ldclient.testing.set-signal", 0.01, 0, lambda: signal.set()) try: signal_was_set = signal.wait(0.1) assert signal_was_set is False @@ -15,9 +16,47 @@ def test_task_does_not_start_when_created(): task.stop() +def test_a_second_start_logs_and_does_not_raise(caplog): + """A raise here can surface out of a caller that is documented as safe to + call more than once, such as AsyncLDClient.start().""" + caplog.set_level(logging.INFO) + queue = Queue() + task = RepeatingTask.at_interval("ldclient.testing.enqueue-time", 0.01, 0, lambda: queue.put(time.time())) + try: + task.start() + thread = task._RepeatingTask__thread + + task.start() + + assert task._RepeatingTask__thread is thread + assert queue.get(True, 1) is not None # still running + finally: + task.stop() + + assert any( + r.getMessage() == "Task ldclient.testing.enqueue-time has already been started; ignoring" + for r in caplog.records + ) + + +def test_a_start_after_stop_does_not_resume_the_task(): + counter = 0 + + def do_task(): + nonlocal counter + counter += 1 + + task = RepeatingTask.at_interval("ldclient.testing.task-runner", 0.01, 0, do_task) + task.stop() + task.start() + time.sleep(0.1) + + assert counter == 0 + + def test_task_executes_until_stopped(): queue = Queue() - task = RepeatingTask("ldclient.testing.enqueue-time", 0.1, 0, lambda: queue.put(time.time())) + task = RepeatingTask.at_interval("ldclient.testing.enqueue-time", 0.1, 0, lambda: queue.put(time.time())) try: last = None task.start() @@ -39,6 +78,59 @@ def test_task_executes_until_stopped(): assert no_more_items is True +class _MutableDelay(DelaySource): + """A delay source a test can move between invocations.""" + + def __init__(self, seconds: float): + self.seconds = seconds + + @property + def next_delay(self) -> float: + return self.seconds + + +def test_fixed_delay_always_gives_the_same_wait(): + delays = FixedDelay(2.5) + assert delays.next_delay == 2.5 + assert delays.next_delay == 2.5 + + +def test_the_task_reads_the_delay_source_after_every_invocation(): + """A value the action decides takes effect on the next wait.""" + reads = Queue() + delays = _MutableDelay(0.01) + + def do_task(): + reads.put(delays.seconds) + delays.seconds = 0.02 # what the next wait must use + + task = RepeatingTask("ldclient.testing.mutable-delay", delays, 0, do_task) + try: + task.start() + assert reads.get(True, 1) == 0.01 + assert reads.get(True, 1) == 0.02 + assert reads.get(True, 1) == 0.02 + finally: + task.stop() + + +def test_whatever_the_action_returns_is_ignored(): + """Guards big-segment polling, whose action returns a status object.""" + calls = Queue() + + def do_task(): + calls.put(time.time()) + return object() # not a number, and not for the task to interpret + + task = RepeatingTask.at_interval("ldclient.testing.returns-a-value", 0.01, 0, do_task) + try: + task.start() + for _ in range(3): + assert calls.get(True, 1) is not None + finally: + task.stop() + + def test_task_can_be_stopped_from_within_the_task(): counter = 0 stopped = Event() @@ -51,7 +143,7 @@ def do_task(): task.stop() stopped.set() - task = RepeatingTask("ldclient.testing.task-runner", 0.01, 0, do_task) + task = RepeatingTask.at_interval("ldclient.testing.task-runner", 0.01, 0, do_task) try: task.start() assert stopped.wait(0.1) is True diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py new file mode 100644 index 00000000..91858ca1 --- /dev/null +++ b/ldclient/testing/impl/test_retry.py @@ -0,0 +1,410 @@ +""" +Tests for ldclient.impl.retry. + +Nothing here sleeps. A test that needs to move time on uses ``frozen_clock``, +which replaces the ``time`` module the retry module reads. Jitter is removed +for every test by an autouse fixture, so a delay assertion reads the +undisturbed value; the tests that are about jitter override it. +""" + +import logging +import random +from contextlib import contextmanager +from unittest import mock + +import pytest + +from ldclient.impl import retry +from ldclient.impl.retry import ( + DEFAULT_INITIAL_RECONNECT_DELAY, + EXTENDED_INITIAL_DELAY, + EXTENDED_MAX_DELAY, + POLLING_RESET_SUCCESSES, + STREAMING_MAX_DELAY, + STREAMING_RESET_INTERVAL, + AfterConsecutiveSuccesses, + AfterHealthyFor, + FailureKind, + RetryState, + classify_http_status, + for_polling, + for_streaming +) +from ldclient.testing.test_util import fixed_retry_jitter, no_retry_jitter + +NORMAL = FailureKind.NORMAL +UNEXPECTED = FailureKind.UNEXPECTED + + +class _FrozenClock: + """Stands in for the ``time`` module. Time only moves when a test says so.""" + + def __init__(self, now: float): + self.now = now + + def time(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +@contextmanager +def frozen_clock(now: float = 1000.0): + """Freezes the clock the retry module reads. + + Patching the module's own ``time`` reference keeps the change local to + ``ldclient.impl.retry``; every other module keeps the real clock. + """ + clock = _FrozenClock(now) + with mock.patch.object(retry, 'time', clock): + yield clock + + +# The random draw just below 1, which subtracts as much jitter as the spec +# allows: half the delay. +FULL_JITTER = 0.9999999 + + +@pytest.fixture(autouse=True) +def without_jitter(): + """Removes jitter for every test in this module, so a delay assertion can + read the undisturbed value.""" + with no_retry_jitter(): + yield + + +@contextmanager +def real_jitter(): + """Restores the real random source, for a test that asserts the bounds hold + for any draw rather than for one fixed value.""" + with mock.patch.object(retry, 'random', random): + yield + + +def streaming_state(initial_delay=1): + return for_streaming(initial_delay) + + +def polling_state(poll_interval=30): + return for_polling(poll_interval) + + +class TestClassifyHttpStatus: + @pytest.mark.parametrize("status", [400, 408, 429]) + def test_retryable_4xx_statuses_are_normal(self, status): + assert classify_http_status(status) is NORMAL + + @pytest.mark.parametrize("status", [401, 403, 404, 405, 418, 499]) + def test_other_4xx_statuses_are_unexpected(self, status): + assert classify_http_status(status) is UNEXPECTED + + @pytest.mark.parametrize("status", [500, 502, 503, 504, 599]) + def test_5xx_statuses_are_normal(self, status): + assert classify_http_status(status) is NORMAL + + @pytest.mark.parametrize("status", [200, 204, 301, 399]) + def test_non_error_statuses_are_normal(self, status): + assert classify_http_status(status) is NORMAL + + +class TestStreamingInitialDelayGuard: + """``Config`` does not check ``initial_reconnect_delay``, and a value of + zero would reconnect with no wait at all.""" + + @pytest.mark.parametrize("configured", [0, -1, -0.5]) + def test_a_non_positive_delay_falls_back_to_the_default(self, configured, caplog): + caplog.set_level(logging.WARNING) + + state = for_streaming(configured) + + assert state.min_delay == DEFAULT_INITIAL_RECONNECT_DELAY + assert state.record_failure(NORMAL) == DEFAULT_INITIAL_RECONNECT_DELAY + assert caplog.records[0].getMessage() == ( + "initial_reconnect_delay must be greater than zero; using the default of 1s" + ) + + @pytest.mark.parametrize("configured", [0.001, 0.5, 1, 5, 45]) + def test_a_positive_delay_is_left_alone(self, configured, caplog): + caplog.set_level(logging.WARNING) + + state = for_streaming(configured) + + assert state.min_delay == configured + assert state.record_failure(NORMAL) == configured + assert caplog.records == [] + + +class TestStreamingDelayTable: + def test_normal_regime_doubles_up_to_the_ceiling(self): + state = streaming_state(initial_delay=1) + delays = [state.record_failure(NORMAL) for _ in range(8)] + assert delays == [1, 2, 4, 8, 16, 30, 30, 30] + + def test_extended_regime_doubles_up_to_the_ceiling(self): + state = streaming_state(initial_delay=1) + delays = [state.record_failure(UNEXPECTED)] + delays += [state.record_failure(NORMAL) for _ in range(5)] + assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] + + def test_a_configured_initial_delay_raises_the_ceiling_with_it(self): + # RETRY 1.5.4 as amended: maxDelay must not fall below initialDelay. + state = streaming_state(initial_delay=45) + assert state.max_delay == 45 + assert state.record_failure(NORMAL) == 45 + + def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): + # RETRY 1.5.5: a normal failure after an unexpected one must not lower + # the bounds back to the normal regime. + state = streaming_state(initial_delay=1) + state.record_failure(UNEXPECTED) + assert state.in_extended_regime + assert state.max_delay == EXTENDED_MAX_DELAY + + state.record_failure(NORMAL) + assert state.in_extended_regime + assert state.max_delay == EXTENDED_MAX_DELAY + assert state.min_delay == EXTENDED_INITIAL_DELAY + + def test_a_second_unexpected_failure_keeps_counting_up(self): + # Restarting the count on every unexpected failure would pin the delay + # at the extended initial delay for ever. + state = streaming_state(initial_delay=1) + assert state.record_failure(UNEXPECTED) == 5 * 60 + assert state.record_failure(UNEXPECTED) == 10 * 60 + assert state.record_failure(UNEXPECTED) == 20 * 60 + + def test_the_streaming_defaults_match_the_spec(self): + state = streaming_state(initial_delay=1) + assert state.max_delay == STREAMING_MAX_DELAY + assert state.operating_cadence == 0 + assert STREAMING_RESET_INTERVAL == 60 + + +class TestJitter: + def test_jitter_never_removes_more_than_half_the_delay(self): + with fixed_retry_jitter(FULL_JITTER): + state = streaming_state(initial_delay=8) + delay = state.record_failure(NORMAL) + assert 4 <= delay < 8 + + def test_no_jitter_leaves_the_delay_alone(self): + state = streaming_state(initial_delay=8) + assert state.record_failure(NORMAL) == 8 + + def test_every_delay_stays_within_the_jitter_bounds(self): + # The real random source, so the bound has to hold for any draw rather + # than for one seeded sequence. + with real_jitter(): + state = streaming_state(initial_delay=1) + for base in [1, 2, 4, 8, 16, 30, 30, 30]: + delay = state.record_failure(NORMAL) + assert base / 2 <= delay <= base + + +class TestStreamingReset: + def test_a_minute_of_healthy_operation_resets_the_state(self): + # RETRY 1.8.2. The whole minute passes instantly. + with frozen_clock() as clock: + state = streaming_state(initial_delay=1) + state.record_failure(UNEXPECTED) + state.record_failure(NORMAL) + + state.record_healthy() + assert state.in_extended_regime, "the window has not elapsed yet" + + clock.advance(STREAMING_RESET_INTERVAL) + assert state.maybe_reset() + assert not state.in_extended_regime + assert state.max_delay == STREAMING_MAX_DELAY + assert state.record_failure(NORMAL) == 1 + + def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): + with frozen_clock() as clock: + state = streaming_state(initial_delay=1) + state.record_failure(NORMAL) + state.record_failure(NORMAL) + + state.record_healthy() + clock.advance(STREAMING_RESET_INTERVAL) + + # The state resets before this failure is counted, so the delay is + # the first-retry delay again rather than the fourth. + assert state.record_failure(NORMAL) == 1 + + def test_a_short_healthy_stretch_does_not_reset(self): + with frozen_clock() as clock: + state = streaming_state(initial_delay=1) + state.record_failure(NORMAL) + + state.record_healthy() + clock.advance(STREAMING_RESET_INTERVAL - 1) + assert state.record_failure(NORMAL) == 2 + + def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(self): + # Every transport failure is normal, so no amount of flapping reaches + # the extended regime. Each cycle is a healthy stretch shorter than the + # reset window, so the delay climbs, but only to the normal ceiling. + with frozen_clock() as clock: + state = streaming_state(initial_delay=1) + delays = [] + for _ in range(20): + state.record_healthy() + clock.advance(5) + delays.append(state.record_failure(NORMAL)) + clock.advance(1) + + assert not state.in_extended_regime + assert max(delays) == STREAMING_MAX_DELAY + assert state.max_delay == STREAMING_MAX_DELAY + + +class TestPollingCadence: + def test_a_normal_failure_polls_again_on_schedule(self): + state = polling_state(poll_interval=30) + assert [state.record_failure(NORMAL) for _ in range(4)] == [30, 30, 30, 30] + + def test_the_extended_regime_doubles_up_to_an_hour(self): + state = polling_state(poll_interval=30) + delays = [state.record_failure(UNEXPECTED)] + delays += [state.record_failure(NORMAL) for _ in range(5)] + assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] + + def test_the_wait_never_falls_below_the_poll_interval(self): + # RETRY 1.4.9. Full jitter would otherwise halve the delay. + with fixed_retry_jitter(FULL_JITTER): + state = polling_state(poll_interval=30) + assert state.record_failure(NORMAL) == 30 + assert state.record_failure(UNEXPECTED) >= 30 + + def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): + state = polling_state(poll_interval=2 * 60 * 60) + assert state.record_failure(UNEXPECTED) == 2 * 60 * 60 + assert state.max_delay == 2 * 60 * 60 + + def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): + # RETRY 1.4.8. Conflating this with the reset is the bug another SDK + # shipped: its first successful poll after an outage still waited + # twenty minutes or more. + state = polling_state(poll_interval=30) + state.record_failure(UNEXPECTED) + state.record_failure(NORMAL) + assert state.record_failure(NORMAL) == 20 * 60 + + assert state.record_success() == 30 + assert state.in_extended_regime, "one success does not reset the state" + + def test_two_successes_in_a_row_reset_the_state(self): + # RETRY 1.8.2 with the polling reset policy. + state = polling_state(poll_interval=30) + state.record_failure(UNEXPECTED) + + state.record_success() + assert state.in_extended_regime + + state.record_success() + assert not state.in_extended_regime + assert state.record_failure(NORMAL) == 30 + + def test_a_failure_between_two_successes_clears_the_first(self): + state = polling_state(poll_interval=30) + state.record_failure(UNEXPECTED) + state.record_success() + state.record_failure(NORMAL) + state.record_success() + assert state.in_extended_regime + + state.record_success() + assert not state.in_extended_regime + + def test_the_polling_defaults_match_the_spec(self): + state = polling_state(poll_interval=30) + assert state.operating_cadence == 30 + assert state.min_delay == 30 + assert state.max_delay == 30 + assert POLLING_RESET_SUCCESSES == 2 + + +class TestWaitOverride: + def test_an_override_replaces_the_computed_wait(self): + state = streaming_state(initial_delay=1) + state.record_failure(NORMAL) + assert state.record_failure(NORMAL, wait_override=7) == 7 + + def test_an_override_still_respects_the_cadence(self): + state = polling_state(poll_interval=30) + assert state.record_failure(NORMAL, wait_override=1) == 30 + + +class TestAttemptCount: + def test_attempts_counts_every_failure(self): + state = streaming_state(initial_delay=1) + for _ in range(5): + state.record_failure(NORMAL) + assert state.attempts == 5 + + def test_a_reset_does_not_clear_the_attempt_count(self): + # The count is for logging, so it should keep counting across a reset. + with frozen_clock() as clock: + state = streaming_state(initial_delay=1) + state.record_failure(NORMAL) + state.record_healthy() + clock.advance(STREAMING_RESET_INTERVAL) + state.maybe_reset() + assert state.attempts == 1 + + +class TestResetPolicies: + def test_healthy_for_tracks_the_start_of_the_stretch(self): + with frozen_clock() as clock: + policy = AfterHealthyFor(60) + assert not policy.is_satisfied() + + policy.note_healthy() + started = policy.healthy_since + + # A later signal must not push the start of the stretch out. + clock.advance(40) + policy.note_healthy() + assert policy.healthy_since == started + + clock.advance(20) + assert policy.is_satisfied() + + def test_healthy_for_is_cleared_by_a_failure(self): + with frozen_clock() as clock: + policy = AfterHealthyFor(60) + policy.note_healthy() + policy.note_failure() + assert policy.healthy_since is None + + clock.advance(900) + assert not policy.is_satisfied() + + def test_consecutive_successes_counts_up(self): + policy = AfterConsecutiveSuccesses(2) + policy.note_healthy() + assert not policy.is_satisfied() + policy.note_healthy() + assert policy.is_satisfied() + + def test_consecutive_successes_is_cleared_by_a_failure(self): + policy = AfterConsecutiveSuccesses(2) + policy.note_healthy() + policy.note_failure() + assert policy.successes == 0 + assert not policy.is_satisfied() + + +class TestLongOutage: + def test_a_long_outage_cannot_overflow_the_delay(self): + state = RetryState( + initial_delay=1, + normal_ceiling=30, + extended_initial_delay=EXTENDED_INITIAL_DELAY, + extended_ceiling=EXTENDED_MAX_DELAY, + reset_policy=AfterHealthyFor(60), + ) + for _ in range(5000): + delay = state.record_failure(NORMAL) + assert delay == 30 diff --git a/ldclient/testing/test_aio.py b/ldclient/testing/test_aio.py index 85174285..94e6d80c 100644 --- a/ldclient/testing/test_aio.py +++ b/ldclient/testing/test_aio.py @@ -6,6 +6,7 @@ """ import asyncio +import logging import subprocess import sys import threading @@ -127,7 +128,7 @@ async def test_async_fires_repeatedly_then_stops(self): async def action(): counts['n'] += 1 - task = aio.AsyncRepeatingTask("test.repeating", 0.01, 0, action) + task = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0, action) task.start() await _async_wait_until(lambda: counts['n'] >= 3) task.stop() @@ -143,7 +144,7 @@ async def test_async_initial_delay_respected(self): async def action(): counts['n'] += 1 - task = aio.AsyncRepeatingTask("test.repeating", 0.01, 0.1, action) + task = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0.1, action) task.start() await asyncio.sleep(0.03) assert counts['n'] == 0 @@ -157,7 +158,7 @@ async def action(): counts['n'] += 1 raise RuntimeError("boom") - task = aio.AsyncRepeatingTask("test.repeating", 0.01, 0, action) + task = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0, action) task.start() await _async_wait_until(lambda: counts['n'] >= 2) task.stop() @@ -171,22 +172,50 @@ async def action(): counts['n'] += 1 holder['task'].stop() - holder['task'] = aio.AsyncRepeatingTask("test.repeating", 0.01, 0, action) + holder['task'] = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0, action) holder['task'].start() await asyncio.sleep(0.1) assert counts['n'] == 1 @pytest.mark.asyncio - async def test_async_second_start_raises(self): + async def test_async_second_start_logs_and_does_not_raise(self, caplog): + """Mirrors the sync primitive. A raise here can surface out of a caller + that is documented as safe to call more than once.""" + caplog.set_level(logging.INFO) + counts = {'n': 0} + async def action(): - pass + counts['n'] += 1 + + task = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0, action) + task.start() + handle = task._AsyncRepeatingTask__task - task = aio.AsyncRepeatingTask("test.repeating", 0.01, 0, action) task.start() - with pytest.raises(RuntimeError): - task.start() + + assert task._AsyncRepeatingTask__task is handle + await _async_wait_until(lambda: counts['n'] >= 1) task.stop() + assert any( + r.getMessage() == "Task test.repeating has already been started; ignoring" + for r in caplog.records + ) + + @pytest.mark.asyncio + async def test_async_start_after_stop_does_not_resume_the_task(self): + counts = {'n': 0} + + async def action(): + counts['n'] += 1 + + task = aio.AsyncRepeatingTask.at_interval("test.repeating", 0.01, 0, action) + task.stop() + task.start() + await asyncio.sleep(0.05) + + assert counts['n'] == 0 + class TestBoundedTaskSet: @pytest.mark.asyncio diff --git a/ldclient/testing/test_ldclient_end_to_end.py b/ldclient/testing/test_ldclient_end_to_end.py index 8e608d14..ce523a53 100644 --- a/ldclient/testing/test_ldclient_end_to_end.py +++ b/ldclient/testing/test_ldclient_end_to_end.py @@ -1,5 +1,6 @@ import json import sys +import time import pytest @@ -53,12 +54,18 @@ def test_client_starts_in_streaming_mode(): assert r.headers['Authorization'] == sdk_key -def test_client_fails_to_start_in_streaming_mode_with_401_error(): +def test_client_does_not_initialize_in_streaming_mode_with_401_error(): + """A rejected SDK key no longer fails fast. The constructor waits out the + full start_wait and returns uninitialized, while the SDK keeps retrying in + the background.""" with start_server() as stream_server: stream_server.for_path('/all', BasicResponse(401)) config = Config(sdk_key=sdk_key, stream_uri=stream_server.uri, send_events=False) - with LDClient(config=config) as client: + started = time.time() + with LDClient(config=config, start_wait=0.5) as client: + elapsed = time.time() - started + assert elapsed >= 0.5 assert not client.is_initialized() assert client.variation(always_true_flag['key'], user, False) is False @@ -91,12 +98,16 @@ def test_client_starts_in_polling_mode(): assert r.headers['Authorization'] == sdk_key -def test_client_fails_to_start_in_polling_mode_with_401_error(): +def test_client_does_not_initialize_in_polling_mode_with_401_error(): + """As with streaming, a rejected SDK key no longer fails fast.""" with start_server() as poll_server: poll_server.for_path('/sdk/latest-all', BasicResponse(401)) config = Config(sdk_key=sdk_key, base_uri=poll_server.uri, stream=False, send_events=False) - with LDClient(config=config) as client: + started = time.time() + with LDClient(config=config, start_wait=0.5) as client: + elapsed = time.time() - started + assert elapsed >= 0.5 assert not client.is_initialized() assert client.variation(always_true_flag['key'], user, False) is False diff --git a/ldclient/testing/test_util.py b/ldclient/testing/test_util.py index fbfac15a..8f40ab48 100644 --- a/ldclient/testing/test_util.py +++ b/ldclient/testing/test_util.py @@ -1,7 +1,10 @@ import os +from contextlib import contextmanager +from unittest import mock import pytest +from ldclient.impl import retry from ldclient.impl.util import redact_password skip_database_tests = os.environ.get('LD_SKIP_DATABASE_TESTS') == '1' @@ -25,6 +28,35 @@ def test_can_redact_password(password_redaction_tests): assert redact_password(input) == expected +class _FixedRandom: + """Stands in for the ``random`` module, always drawing the same value.""" + + def __init__(self, value: float): + self.value = value + + def random(self) -> float: + return self.value + + +@contextmanager +def fixed_retry_jitter(fraction: float): + """Fixes the jitter that :mod:`ldclient.impl.retry` subtracts from a delay. + + ``0`` subtracts none, so a test can assert an exact delay. A value just + below ``1`` subtracts as much as the spec allows, which is half. + + Patching the retry module's own ``random`` reference keeps the change + local to that module; every other module keeps the real source. + """ + with mock.patch.object(retry, 'random', _FixedRandom(fraction)): + yield + + +def no_retry_jitter(): + """Removes the retry jitter, so a test can assert an exact delay.""" + return fixed_retry_jitter(0.0) + + class SpyListener: def __init__(self): self._statuses = [] From fa8d44ba428b7f0aa47a6b9dba9d6b93ce4de881 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 11 Sep 2026 09:04:58 -0500 Subject: [PATCH 02/19] cleanup and simplify based on feedback --- ldclient/impl/datasource/async_streaming.py | 24 +---- ldclient/impl/datasource/streaming.py | 27 +----- ldclient/impl/retry.py | 74 +++++---------- .../impl/datasource/test_async_streaming.py | 66 ++++++++----- .../testing/impl/datasource/test_streaming.py | 93 ++++++++++++------- ldclient/testing/impl/test_retry.py | 81 ++++++++++++++-- ldclient/testing/test_util.py | 34 +++++++ 7 files changed, 239 insertions(+), 160 deletions(-) diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index f670b111..f5afc506 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -63,16 +63,15 @@ def __init__(self, config, store, ready, diagnostic_accumulator, sse_factory: Op self._sse: Any = None self._connection_attempt_start_time: Optional[float] = None self._runner = AsyncTaskRunner() - self._start_requested = False + self._started = False self._retry = retry_state or for_streaming(config.initial_reconnect_delay) - self._signalled_healthy = False self._interrupted_by_sdk = False def start(self): - if self._start_requested: + if self._started: log.info("AsyncStreamingUpdateProcessor has already been started; ignoring") return - self._start_requested = True + self._started = True self._runner.spawn("ldclient.datasource.streaming", self._run) async def _run(self): @@ -94,8 +93,6 @@ async def _run(self): # For the initial connect the pre-loop timestamp is already set. if self._connection_attempt_start_time is None: self._connection_attempt_start_time = time.time() - # A fresh stream has not proved itself healthy yet. - self._signalled_healthy = False elif isinstance(action, Event): message_ok = False message_handled = False @@ -116,7 +113,7 @@ async def _run(self): break if message_handled: - self._record_healthy_operation() + self._retry.record_success() if message_ok: self._record_stream_init(False) @@ -192,19 +189,6 @@ async def _interrupt_stream(self): self._interrupted_by_sdk = True await self._sse.interrupt() - def _record_healthy_operation(self): - """Signals healthy operation on the first message of a fresh stream. - - It fires once per stream. A later message on the same stream must not - restart the reset window. The SSE client's own signal is no use here - because it fires when the connection opens, and an open connection that - has sent no data yet does not show the stream is working. - """ - if self._signalled_healthy: - return - self._signalled_healthy = True - self._retry.record_healthy() - def initialized(self): return self._running and self._ready.is_set() is True and self._store.initialized is True diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index e7a1615c..935d3823 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -64,17 +64,7 @@ def __init__(self, config, store, ready, diagnostic_accumulator, retry_state: Op self._connection_attempt_start_time: Optional[float] = None self._retry = retry_state or for_streaming(config.initial_reconnect_delay) self._stop_event = ThreadEvent() - self._signalled_healthy = False self._interrupted_by_sdk = False - # Thread already owns the name "_started", so this flag cannot use it. - self._start_requested = False - - def start(self): - if self._start_requested: - log.info("StreamingUpdateProcessor has already been started; ignoring") - return - self._start_requested = True - Thread.start(self) def run(self): log.info("Starting StreamingUpdateProcessor connecting to uri: " + self._uri) @@ -84,8 +74,6 @@ def run(self): for action in self._sse.all: if isinstance(action, Start): record_environment_id(self._data_source_update_sink, action.headers) - # A fresh stream has not proved itself healthy yet. - self._signalled_healthy = False elif isinstance(action, Event): message_ok = False message_handled = False @@ -106,7 +94,7 @@ def run(self): break if message_handled: - self._record_healthy_operation() + self._retry.record_success() if message_ok: self._record_stream_init(False) @@ -184,19 +172,6 @@ def _interrupt_stream(self): self._interrupted_by_sdk = True self._sse.interrupt() - def _record_healthy_operation(self): - """Signals healthy operation on the first message of a fresh stream. - - It fires once per stream. A later message on the same stream must not - restart the reset window. The SSE client's own signal is no use here - because it fires when the connection opens, and an open connection that - has sent no data yet does not show the stream is working. - """ - if self._signalled_healthy: - return - self._signalled_healthy = True - self._retry.record_healthy() - def initialized(self): return self._running and self._ready.is_set() is True and self._store.initialized is True diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 557cdd24..ccba064a 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -101,12 +101,14 @@ class AfterHealthyFor(ResetPolicy): ``seconds``. This is the streaming policy.""" def __init__(self, seconds: float): - self._seconds = seconds + self._healthy_seconds = seconds self._healthy_since: Optional[float] = None def note_healthy(self) -> None: + """Records the monotonic time the component became healthy. Calling + this again while it is still healthy does not move that time.""" if self._healthy_since is None: - self._healthy_since = time.time() + self._healthy_since = time.monotonic() def note_failure(self) -> None: self._healthy_since = None @@ -114,7 +116,7 @@ def note_failure(self) -> None: def is_satisfied(self) -> bool: if self._healthy_since is None: return False - return time.time() - self._healthy_since >= self._seconds + return time.monotonic() - self._healthy_since >= self._healthy_seconds @property def healthy_since(self) -> Optional[float]: @@ -157,19 +159,6 @@ class RetryState: An unexpected failure moves the state to the extended regime, which raises both delay bounds. The bounds stay raised until the reset condition is met, so a normal failure that follows cannot lower them. - - Three things happen on success, and they are deliberately separate: - - * :meth:`record_success` returns the operating cadence. A backoff wait - applies to a retry, not to every operation, so one success is enough to - go back to the normal cadence even while the retry state is still raised. - * :meth:`record_healthy` feeds the reset policy. - * :meth:`maybe_reset` clears the retry state, but only once the reset - policy is satisfied, which may need more than one success. - - Conflating the first two is a real bug in another SDK: after an outage its - first successful poll still waited twenty minutes or more, even though it - already held fresh data. """ def __init__( @@ -252,7 +241,8 @@ def record_failure(self, kind: FailureKind, wait_override: Optional[float] = Non computed one. LaunchDarkly does not send one on these endpoints, so this is an unused seam. """ - self.maybe_reset() + # Only a time-based policy needs this: nothing runs while a stream is healthy. + self._reset_if_due() self._attempts += 1 self._reset_policy.note_failure() @@ -271,48 +261,28 @@ def record_failure(self, kind: FailureKind, wait_override: Optional[float] = Non self._next_delay = self._compute_wait(wait_override) return self._next_delay - def record_success(self) -> float: - """ - Records a successful operation and returns how long to wait before the - next one, in seconds. - - The answer is the operating cadence, even when the retry state is still - raised, because a backoff wait applies to a retry and not to every - operation. This does not clear the retry state; :meth:`maybe_reset` - does that once the reset policy is satisfied. - """ - self.record_healthy() - self._next_delay = self._operating_cadence - return self._next_delay - - def record_healthy(self) -> None: + def record_success(self) -> None: """ - Records that the component is operating normally, and resets the retry - state if that is now enough. + Records a successful operation, and resets the retry state if that is + now enough. - Streaming calls this once per stream, on the first message of a fresh - stream. Polling calls it through :meth:`record_success`. + The wait before the next operation becomes the operating cadence, even + when the retry state is still raised, because a backoff wait applies to + a retry and not to every operation. """ self._reset_policy.note_healthy() - self.maybe_reset() - - def maybe_reset(self) -> bool: - """ - Clears the retry state if the reset policy is satisfied, returning the - delay bounds to the normal regime. + self._reset_if_due() + self._next_delay = self._operating_cadence - Returns True if it cleared anything. This runs on its own before every - failure, so a caller does not have to call it. - """ + def _reset_if_due(self) -> None: + """Clears the retry state when the reset policy is satisfied, returning + the delay bounds to the normal regime.""" if not self._reset_policy.is_satisfied(): - return False - if self._n == 0 and not self._extended: - return False + return self._n = 0 self._extended = False self._min_delay = self._initial_delay self._max_delay = max(self._normal_ceiling, self._initial_delay) - return True def _compute_wait(self, wait_override: Optional[float]) -> float: if wait_override is not None: @@ -334,6 +304,8 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: A configured delay of zero or less would reconnect with no wait, so the documented default stands in for it. ``Config`` does not check this value, though it does clamp ``poll_interval``. + + The extended regime never starts below the configured delay. """ if initial_reconnect_delay <= 0: log.warning( @@ -344,7 +316,7 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: return RetryState( initial_delay=initial_reconnect_delay, normal_ceiling=STREAMING_MAX_DELAY, - extended_initial_delay=EXTENDED_INITIAL_DELAY, + extended_initial_delay=max(EXTENDED_INITIAL_DELAY, initial_reconnect_delay), extended_ceiling=EXTENDED_MAX_DELAY, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), ) @@ -364,7 +336,7 @@ def for_polling(poll_interval: float) -> RetryState: initial_delay=poll_interval, normal_ceiling=poll_interval, extended_initial_delay=max(EXTENDED_INITIAL_DELAY, poll_interval), - extended_ceiling=max(EXTENDED_MAX_DELAY, poll_interval), + extended_ceiling=EXTENDED_MAX_DELAY, reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), operating_cadence=poll_interval, ) diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index 7b7d3adb..a1ef0122 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -33,7 +33,11 @@ from ldclient.interfaces import DataSourceErrorKind, DataSourceState from ldclient.testing.builders import FlagBuilder, SegmentBuilder from ldclient.testing.mock_async_components import MockAsyncFeatureStore -from ldclient.testing.test_util import no_retry_jitter +from ldclient.testing.test_util import ( + no_retry_jitter, + record_healthy_windows, + ticking_clock +) from ldclient.versioned_data_kind import FEATURES, SEGMENTS @@ -89,6 +93,18 @@ async def _actions_generator(actions: list): await asyncio.Event().wait() +def _retry_state_with(policy: AfterHealthyFor) -> RetryState: + """A retry state with tiny delays and a caller-supplied reset policy, so a + test can watch the window.""" + return RetryState( + initial_delay=0.001, + normal_ceiling=0.001, + extended_initial_delay=0.001, + extended_ceiling=0.001, + reset_policy=policy, + ) + + def _fast_retry_state(delay: float = 0.001) -> RetryState: """A retry state with tiny delays, so a test does not have to wait out the real extended-regime delay of five minutes.""" @@ -526,30 +542,33 @@ async def test_the_processor_asks_the_factory_to_leave_the_delay_to_the_sdk(): @pytest.mark.asyncio -async def test_healthy_operation_is_signalled_once_per_stream(): - """The reset window must start at the first message of a stream. Signalling - again on every later message would keep pushing the window out.""" +async def test_several_messages_on_one_stream_do_not_extend_the_reset_window(): + """The window starts at the first message and stays there, however many + more arrive on the same stream.""" flag = FlagBuilder('f1').version(1).build() put_data = _make_put_data(flags={'f1': _item_dict(flag)}) patch_data = _make_patch_data(FEATURES, _item_dict(FlagBuilder('f1').version(2).build())) actions = [_start(), _event('put', put_data), _event('patch', patch_data)] - retry = _fast_retry_state() - healthy_at = [] - retry.record_healthy = lambda: healthy_at.append(len(healthy_at)) # type: ignore[method-assign] - - proc, store, ready, _ = _make_processor(actions, retry_state=retry) - proc.start() - await _wait_until(lambda: len(healthy_at) > 0) - await asyncio.sleep(0.05) + policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) + retry = _retry_state_with(policy) + # The clock moves on every read, so a window that had been restarted reads + # back as a different time. + windows = record_healthy_windows(policy) - assert healthy_at == [0] + with ticking_clock(): + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + await _wait_until(lambda: len(windows) >= 2) + await proc.stop() - await proc.stop() + assert len(set(windows)) == 1, "the window moved between messages" @pytest.mark.asyncio -async def test_a_fresh_stream_signals_healthy_operation_again(): +async def test_a_fresh_stream_starts_a_new_reset_window(): + """A stream teardown clears the window through record_failure, so the next + stream measures its own stretch rather than inheriting the old one.""" from ld_eventsource.errors import HTTPStatusError flag = FlagBuilder('f1').version(1).build() @@ -562,15 +581,18 @@ async def test_a_fresh_stream_signals_healthy_operation_again(): _event('put', put_data), ] - retry = _fast_retry_state() - healthy_count = [] - retry.record_healthy = lambda: healthy_count.append(1) # type: ignore[method-assign] + policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) + retry = _retry_state_with(policy) + windows = record_healthy_windows(policy) - proc, store, ready, _ = _make_processor(actions, retry_state=retry) - proc.start() - await _wait_until(lambda: len(healthy_count) >= 2) + with ticking_clock(): + proc, store, ready, _ = _make_processor(actions, retry_state=retry) + proc.start() + # One put per stream, so two signals in all. + await _wait_until(lambda: len(windows) >= 2) + await proc.stop() - await proc.stop() + assert len(set(windows)) == 2, "the second stream reused the first window" @pytest.mark.asyncio diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 6dd782de..39f69875 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -47,7 +47,12 @@ make_put_event, stream_content ) -from ldclient.testing.test_util import SpyListener, no_retry_jitter +from ldclient.testing.test_util import ( + SpyListener, + no_retry_jitter, + record_healthy_windows, + ticking_clock +) from ldclient.version import VERSION from ldclient.versioned_data_kind import FEATURES, SEGMENTS @@ -488,24 +493,6 @@ def listener(s): assert retry.attempts == 1 -def test_second_start_is_a_no_op(): - """A second start() must not raise. Thread.start() would, so the processor - guards it.""" - store = InMemoryFeatureStore() - ready = Event() - - with start_server() as server: - with stream_content(make_put_event()) as stream: - config = Config(sdk_key='sdk-key', stream_uri=server.uri) - server.for_path('/all', stream) - - with StreamingUpdateProcessor(config, store, ready, None) as sp: - sp.start() - sp.start() - ready.wait(start_wait) - assert sp.initialized() - - def _handle_errors_without_waiting(retry, errors): """Drives _handle_error for each error and returns nothing. The stop event is pre-set so the interruptible wait returns at once.""" @@ -563,9 +550,9 @@ def test_a_server_close_and_a_transport_error_both_report_a_delay(caplog): assert messages[1] == "Error on stream connection: [Errno 104] reset by peer - will retry in 2.0s" -def test_healthy_operation_is_signalled_once_per_stream(): - """The reset window must start at the first message of a stream. Signalling - again on every later message would keep pushing the window out.""" +def test_several_messages_on_one_stream_do_not_extend_the_reset_window(): + """The window starts at the first message and stays there, however many + more arrive on the same stream.""" store = InMemoryFeatureStore() ready = Event() flag = FlagBuilder('flagkey').version(1).build() @@ -575,17 +562,61 @@ def test_healthy_operation_is_signalled_once_per_stream(): config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) server.for_path('/all', stream) - retry = fast_retry_state() - healthy_count = [] - retry.record_healthy = lambda: healthy_count.append(1) # type: ignore[method-assign] + policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) + retry = RetryState( + initial_delay=brief_delay, + normal_ceiling=brief_delay, + extended_initial_delay=brief_delay, + extended_ceiling=brief_delay, + reset_policy=policy, + ) + # The clock moves on every read, so a window that had been + # restarted reads back as a different time. + windows = record_healthy_windows(policy) + with ticking_clock(): + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + expect_update(store, FEATURES, flag) - with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: - sp.start() - ready.wait(start_wait) - assert sp.initialized() - expect_update(store, FEATURES, flag) + assert len(windows) >= 2, "both messages should have signalled" + assert len(set(windows)) == 1, "the window moved between messages" + + +def test_a_fresh_stream_starts_a_new_reset_window(): + """A stream teardown clears the window through record_failure, so the next + stream measures its own stretch rather than inheriting the old one.""" + store = InMemoryFeatureStore() + ready = Event() + flagv1 = FlagBuilder('flagkey').version(1).build() + flagv2 = FlagBuilder('flagkey').version(2).build() + + with start_server() as server: + with stream_content(make_put_event([flagv1])) as stream1: + with stream_content(make_put_event([flagv2])) as stream2: + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + server.for_path('/all', SequentialHandler(stream1, stream2)) - assert healthy_count == [1] + policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) + retry = RetryState( + initial_delay=brief_delay, + normal_ceiling=brief_delay, + extended_initial_delay=brief_delay, + extended_ceiling=brief_delay, + reset_policy=policy, + ) + windows = record_healthy_windows(policy) + with ticking_clock(): + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp.start() + ready.wait(start_wait) + assert sp.initialized() + + stream1.close() + expect_update(store, FEATURES, flagv2) + + assert len(set(windows)) == 2, "the second stream reused the first window" @pytest.mark.parametrize( diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 91858ca1..d82f2569 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -42,7 +42,7 @@ class _FrozenClock: def __init__(self, now: float): self.now = now - def time(self) -> float: + def monotonic(self) -> float: return self.now def advance(self, seconds: float) -> None: @@ -135,6 +135,38 @@ def test_a_positive_delay_is_left_alone(self, configured, caplog): assert caplog.records == [] +class TestStreamingExtendedDelayFloor: + """RETRY 1.5.4.1: a delay that applies after an unexpected failure must not + be below the component's initial delay.""" + + @pytest.mark.parametrize( + "configured,expected", + [(1, 300), (30, 300), (300, 300), (600, 600), (3600, 3600), (0, 300), (-5, 300)], + ) + def test_the_extended_delay_never_starts_below_the_configured_delay(self, configured, expected): + assert for_streaming(configured).record_failure(UNEXPECTED) == expected + + @pytest.mark.parametrize("configured", [1, 30, 300, 600, 3600]) + def test_an_unexpected_failure_never_waits_less_than_a_normal_one(self, configured): + normal = for_streaming(configured).record_failure(NORMAL) + unexpected = for_streaming(configured).record_failure(UNEXPECTED) + assert unexpected >= normal + + @pytest.mark.parametrize( + "configured,ladder", + [ + (1, [300, 600, 1200, 2400, 3600]), + (600, [600, 1200, 2400, 3600, 3600]), + ], + ids=["default", "clamped"], + ) + def test_the_extended_ladder_still_doubles_to_the_ceiling(self, configured, ladder): + state = for_streaming(configured) + delays = [state.record_failure(UNEXPECTED)] + delays += [state.record_failure(NORMAL) for _ in range(4)] + assert delays == ladder + + class TestStreamingDelayTable: def test_normal_regime_doubles_up_to_the_ceiling(self): state = streaming_state(initial_delay=1) @@ -210,11 +242,11 @@ def test_a_minute_of_healthy_operation_resets_the_state(self): state.record_failure(UNEXPECTED) state.record_failure(NORMAL) - state.record_healthy() + state.record_success() assert state.in_extended_regime, "the window has not elapsed yet" clock.advance(STREAMING_RESET_INTERVAL) - assert state.maybe_reset() + state.record_success() assert not state.in_extended_regime assert state.max_delay == STREAMING_MAX_DELAY assert state.record_failure(NORMAL) == 1 @@ -225,7 +257,7 @@ def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): state.record_failure(NORMAL) state.record_failure(NORMAL) - state.record_healthy() + state.record_success() clock.advance(STREAMING_RESET_INTERVAL) # The state resets before this failure is counted, so the delay is @@ -237,7 +269,7 @@ def test_a_short_healthy_stretch_does_not_reset(self): state = streaming_state(initial_delay=1) state.record_failure(NORMAL) - state.record_healthy() + state.record_success() clock.advance(STREAMING_RESET_INTERVAL - 1) assert state.record_failure(NORMAL) == 2 @@ -249,7 +281,7 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se state = streaming_state(initial_delay=1) delays = [] for _ in range(20): - state.record_healthy() + state.record_success() clock.advance(5) delays.append(state.record_failure(NORMAL)) clock.advance(1) @@ -278,9 +310,12 @@ def test_the_wait_never_falls_below_the_poll_interval(self): assert state.record_failure(UNEXPECTED) >= 30 def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): + # The ceiling is lifted by record_failure clamping it against the + # initial delay, not by for_polling clamping the ceiling itself. state = polling_state(poll_interval=2 * 60 * 60) assert state.record_failure(UNEXPECTED) == 2 * 60 * 60 assert state.max_delay == 2 * 60 * 60 + assert state.min_delay == 2 * 60 * 60 def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): # RETRY 1.4.8. Conflating this with the reset is the bug another SDK @@ -291,7 +326,8 @@ def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): state.record_failure(NORMAL) assert state.record_failure(NORMAL) == 20 * 60 - assert state.record_success() == 30 + state.record_success() + assert state.next_delay == 30 assert state.in_extended_regime, "one success does not reset the state" def test_two_successes_in_a_row_reset_the_state(self): @@ -304,6 +340,7 @@ def test_two_successes_in_a_row_reset_the_state(self): state.record_success() assert not state.in_extended_regime + assert state.next_delay == 30 assert state.record_failure(NORMAL) == 30 def test_a_failure_between_two_successes_clears_the_first(self): @@ -348,10 +385,14 @@ def test_a_reset_does_not_clear_the_attempt_count(self): with frozen_clock() as clock: state = streaming_state(initial_delay=1) state.record_failure(NORMAL) - state.record_healthy() + state.record_success() clock.advance(STREAMING_RESET_INTERVAL) - state.maybe_reset() - assert state.attempts == 1 + state.record_success() + + # The reset shows in the delay dropping back to the first-retry + # value, while the count carries on. + assert state.record_failure(NORMAL) == 1 + assert state.attempts == 2 class TestResetPolicies: @@ -371,6 +412,26 @@ def test_healthy_for_tracks_the_start_of_the_stretch(self): clock.advance(20) assert policy.is_satisfied() + def test_many_healthy_signals_do_not_move_the_window(self): + """Streaming signals on every message, so an unconditional assignment + here would push its reset out for ever -- Go's SDK-2845.""" + with frozen_clock() as clock: + policy = AfterHealthyFor(60) + policy.note_healthy() + first = policy.healthy_since + + for _ in range(59): + clock.advance(1) + policy.note_healthy() + + assert policy.healthy_since == first + assert not policy.is_satisfied() + + # The threshold lands 60s after the first signal, not the last. + clock.advance(1) + policy.note_healthy() + assert policy.is_satisfied() + def test_healthy_for_is_cleared_by_a_failure(self): with frozen_clock() as clock: policy = AfterHealthyFor(60) diff --git a/ldclient/testing/test_util.py b/ldclient/testing/test_util.py index 8f40ab48..2036d097 100644 --- a/ldclient/testing/test_util.py +++ b/ldclient/testing/test_util.py @@ -38,6 +38,40 @@ def random(self) -> float: return self.value +class _TickingClock: + """Stands in for the ``time`` module, moving on with every read.""" + + def __init__(self, start: float, step: float): + self.now = start + self.step = step + + def monotonic(self) -> float: + self.now += self.step + return self.now + + +def record_healthy_windows(policy) -> list: + """Records the window each ``note_healthy`` call leaves in place, so a test + can tell one window from the next.""" + windows: list = [] + note = policy.note_healthy + + def wrapper(): + note() + windows.append(policy.healthy_since) + + policy.note_healthy = wrapper # type: ignore[method-assign] + return windows + + +@contextmanager +def ticking_clock(start: float = 1000.0, step: float = 1.0): + """Gives :mod:`ldclient.impl.retry` a clock that moves on every read, so a + timestamp it stored can be told apart from one it stored later.""" + with mock.patch.object(retry, 'time', _TickingClock(start, step)): + yield + + @contextmanager def fixed_retry_jitter(fraction: float): """Fixes the jitter that :mod:`ldclient.impl.retry` subtracts from a delay. From 9413155e480ee832eeae88d166815827304a2a11 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 14 Sep 2026 08:37:03 -0500 Subject: [PATCH 03/19] reset attempt count on reset --- ldclient/impl/aio/concurrency.py | 4 +-- ldclient/impl/delay.py | 28 ++++++++++++++++++++ ldclient/impl/repeating_task.py | 26 +++--------------- ldclient/impl/retry.py | 16 +++++------ ldclient/testing/impl/test_delay.py | 7 +++++ ldclient/testing/impl/test_repeating_task.py | 9 ++----- ldclient/testing/impl/test_retry.py | 25 +++++++---------- 7 files changed, 58 insertions(+), 57 deletions(-) create mode 100644 ldclient/impl/delay.py create mode 100644 ldclient/testing/impl/test_delay.py diff --git a/ldclient/impl/aio/concurrency.py b/ldclient/impl/aio/concurrency.py index c4b6ad70..62ba70d9 100644 --- a/ldclient/impl/aio/concurrency.py +++ b/ldclient/impl/aio/concurrency.py @@ -13,7 +13,7 @@ from queue import Full as QueueFull # noqa: F401 (shared capacity exception) from typing import Any, Callable, Coroutine, Optional, Set -from ldclient.impl.repeating_task import DelaySource, FixedDelay +from ldclient.impl.delay import DelaySource, FixedDelay from ldclient.impl.util import log @@ -190,7 +190,7 @@ async def stop_all(self, timeout: float = 1) -> None: class AsyncRepeatingTask: """Calls a callback repeatedly on a background task, waiting whatever its - :class:`~ldclient.impl.repeating_task.DelaySource` gives. + :class:`~ldclient.impl.delay.DelaySource` gives. Mirrors the semantics of ``ldclient.impl.repeating_task.RepeatingTask``: the wait starts when the callback returns, exceptions from the callback are logged, and ``stop()`` prevents any further invocations but cannot be diff --git a/ldclient/impl/delay.py b/ldclient/impl/delay.py new file mode 100644 index 00000000..b0ebb395 --- /dev/null +++ b/ldclient/impl/delay.py @@ -0,0 +1,28 @@ +""" +The wait a repeating task takes between invocations. Both schedulers read it, +so it belongs to neither. +""" + +# currently excluded from documentation - see docs/README.md + +from typing import Protocol + + +class DelaySource(Protocol): + """Supplies the wait before a repeating task's next invocation.""" + + @property + def next_delay(self) -> float: + """The seconds to wait before the next invocation.""" + ... + + +class FixedDelay(DelaySource): + """A :class:`DelaySource` that always gives the same wait.""" + + def __init__(self, seconds: float): + self.__seconds = seconds + + @property + def next_delay(self) -> float: + return self.__seconds diff --git a/ldclient/impl/repeating_task.py b/ldclient/impl/repeating_task.py index 1b58c481..d2e4abe1 100644 --- a/ldclient/impl/repeating_task.py +++ b/ldclient/impl/repeating_task.py @@ -1,34 +1,16 @@ from threading import Event, Thread -from typing import Any, Callable, Protocol +from typing import Any, Callable +from ldclient.impl.delay import DelaySource, FixedDelay from ldclient.impl.util import log -class DelaySource(Protocol): - """Supplies the wait before a repeating task's next invocation.""" - - @property - def next_delay(self) -> float: - """The seconds to wait before the next invocation.""" - ... - - -class FixedDelay(DelaySource): - """A :class:`DelaySource` that always gives the same wait.""" - - def __init__(self, seconds: float): - self.__seconds = seconds - - @property - def next_delay(self) -> float: - return self.__seconds - - class RepeatingTask: """ A generic mechanism for calling a callback repeatedly on a worker thread. - The wait between invocations comes from a :class:`DelaySource`, which the + The wait between invocations comes from a + :class:`~ldclient.impl.delay.DelaySource`, which the task reads after each one. Use :meth:`at_interval` for the common case of a fixed interval. """ diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index ccba064a..c6ca30aa 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -204,7 +204,7 @@ def next_delay(self) -> float: @property def attempts(self) -> int: - """How many failures this state has seen. For logging only.""" + """How many failures since the last reset. For logging only.""" return self._attempts @property @@ -228,7 +228,7 @@ def in_extended_regime(self) -> bool: delay bounds.""" return self._extended - def record_failure(self, kind: FailureKind, wait_override: Optional[float] = None) -> float: + def record_failure(self, kind: FailureKind) -> float: """ Records a failed attempt and returns how long to wait before the next one, in seconds. @@ -237,9 +237,6 @@ def record_failure(self, kind: FailureKind, wait_override: Optional[float] = Non reflects the failure just recorded. :param kind: how the failure was classified - :param wait_override: a wait the server asked for, which replaces the - computed one. LaunchDarkly does not send one on these endpoints, - so this is an unused seam. """ # Only a time-based policy needs this: nothing runs while a stream is healthy. self._reset_if_due() @@ -248,7 +245,7 @@ def record_failure(self, kind: FailureKind, wait_override: Optional[float] = Non if kind is FailureKind.UNEXPECTED and not self._extended: # Moving to the extended regime raises both bounds and starts the - # attempt count over. Only the move does this: a later unexpected + # delay sequence over. Only the move does this: a later unexpected # failure keeps counting up, so the delay is not pinned to the # extended initial delay. self._extended = True @@ -258,7 +255,7 @@ def record_failure(self, kind: FailureKind, wait_override: Optional[float] = Non else: self._n += 1 - self._next_delay = self._compute_wait(wait_override) + self._next_delay = self._compute_wait() return self._next_delay def record_success(self) -> None: @@ -280,13 +277,12 @@ def _reset_if_due(self) -> None: if not self._reset_policy.is_satisfied(): return self._n = 0 + self._attempts = 0 self._extended = False self._min_delay = self._initial_delay self._max_delay = max(self._normal_ceiling, self._initial_delay) - def _compute_wait(self, wait_override: Optional[float]) -> float: - if wait_override is not None: - return max(wait_override, self._operating_cadence) + def _compute_wait(self) -> float: exponent = min(max(self._n - 1, 0), _MAX_BACKOFF_EXPONENT) delay = min(self._min_delay * (2**exponent), self._max_delay) jitter = random.random() * delay / 2 diff --git a/ldclient/testing/impl/test_delay.py b/ldclient/testing/impl/test_delay.py new file mode 100644 index 00000000..f6684809 --- /dev/null +++ b/ldclient/testing/impl/test_delay.py @@ -0,0 +1,7 @@ +from ldclient.impl.delay import FixedDelay + + +def test_fixed_delay_always_gives_the_same_wait(): + delays = FixedDelay(2.5) + assert delays.next_delay == 2.5 + assert delays.next_delay == 2.5 diff --git a/ldclient/testing/impl/test_repeating_task.py b/ldclient/testing/impl/test_repeating_task.py index 0fdf89ff..5ab6f184 100644 --- a/ldclient/testing/impl/test_repeating_task.py +++ b/ldclient/testing/impl/test_repeating_task.py @@ -3,7 +3,8 @@ from queue import Empty, Queue from threading import Event -from ldclient.impl.repeating_task import DelaySource, FixedDelay, RepeatingTask +from ldclient.impl.delay import DelaySource +from ldclient.impl.repeating_task import RepeatingTask def test_task_does_not_start_when_created(): @@ -89,12 +90,6 @@ def next_delay(self) -> float: return self.seconds -def test_fixed_delay_always_gives_the_same_wait(): - delays = FixedDelay(2.5) - assert delays.next_delay == 2.5 - assert delays.next_delay == 2.5 - - def test_the_task_reads_the_delay_source_after_every_invocation(): """A value the action decides takes effect on the next wait.""" reads = Queue() diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index d82f2569..91823690 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -362,17 +362,6 @@ def test_the_polling_defaults_match_the_spec(self): assert POLLING_RESET_SUCCESSES == 2 -class TestWaitOverride: - def test_an_override_replaces_the_computed_wait(self): - state = streaming_state(initial_delay=1) - state.record_failure(NORMAL) - assert state.record_failure(NORMAL, wait_override=7) == 7 - - def test_an_override_still_respects_the_cadence(self): - state = polling_state(poll_interval=30) - assert state.record_failure(NORMAL, wait_override=1) == 30 - - class TestAttemptCount: def test_attempts_counts_every_failure(self): state = streaming_state(initial_delay=1) @@ -380,19 +369,23 @@ def test_attempts_counts_every_failure(self): state.record_failure(NORMAL) assert state.attempts == 5 - def test_a_reset_does_not_clear_the_attempt_count(self): - # The count is for logging, so it should keep counting across a reset. + def test_a_reset_starts_the_attempt_count_over(self): + # The streaming spec resets both counters: "set attempt to 1, set n + # to 1". with frozen_clock() as clock: state = streaming_state(initial_delay=1) state.record_failure(NORMAL) + state.record_failure(NORMAL) + assert state.attempts == 2 + state.record_success() clock.advance(STREAMING_RESET_INTERVAL) state.record_success() - # The reset shows in the delay dropping back to the first-retry - # value, while the count carries on. + # The delay drops back to the first-retry value, and the count + # starts over with it. assert state.record_failure(NORMAL) == 1 - assert state.attempts == 2 + assert state.attempts == 1 class TestResetPolicies: From 30c2f083aebdbbd1af70119efbaf9b129ab12fff Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 14 Sep 2026 13:49:13 -0500 Subject: [PATCH 04/19] refactor: Report the next delay through the property, not a return value --- ldclient/impl/datasource/async_polling.py | 3 +- ldclient/impl/datasource/async_streaming.py | 3 +- ldclient/impl/datasource/polling.py | 3 +- ldclient/impl/datasource/streaming.py | 3 +- ldclient/impl/retry.py | 17 +++-- ldclient/testing/impl/test_retry.py | 69 ++++++++++++--------- 6 files changed, 54 insertions(+), 44 deletions(-) diff --git a/ldclient/impl/datasource/async_polling.py b/ldclient/impl/datasource/async_polling.py index 3a638ed8..d2099836 100644 --- a/ldclient/impl/datasource/async_polling.py +++ b/ldclient/impl/datasource/async_polling.py @@ -104,7 +104,8 @@ async def _fetch_and_store(self) -> None: # logged, the handler has exited and exc_info() is empty. stacktrace = e - delay = self._retry.record_failure(kind) + self._retry.record_failure(kind) + delay = self._retry.next_delay level("%s - will retry in %.1fs" % (description, delay), exc_info=stacktrace) if self._data_source_update_sink is not None: diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index f5afc506..be9bd7f9 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -266,7 +266,8 @@ async def _handle_error(self, error: Exception) -> bool: description = "Error on stream connection: %s" % error level = log.warning - delay = self._retry.record_failure(kind) + self._retry.record_failure(kind) + delay = self._retry.next_delay level("%s - will retry in %.1fs" % (description, delay)) if self._data_source_update_sink is not None: diff --git a/ldclient/impl/datasource/polling.py b/ldclient/impl/datasource/polling.py index 504cb7d8..d15ea5c4 100644 --- a/ldclient/impl/datasource/polling.py +++ b/ldclient/impl/datasource/polling.py @@ -109,7 +109,8 @@ def _poll(self) -> None: # logged, the handler has exited and exc_info() is empty. stacktrace = e - delay = self._retry.record_failure(kind) + self._retry.record_failure(kind) + delay = self._retry.next_delay level("%s - will retry in %.1fs" % (description, delay), exc_info=stacktrace) if self._data_source_update_sink is not None: diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index 935d3823..ab060e43 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -245,7 +245,8 @@ def _handle_error(self, error: Exception) -> bool: description = "Error on stream connection: %s" % error level = log.warning - delay = self._retry.record_failure(kind) + self._retry.record_failure(kind) + delay = self._retry.next_delay level("%s - will retry in %.1fs" % (description, delay)) if self._data_source_update_sink is not None: diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index c6ca30aa..41d4220b 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -152,9 +152,10 @@ class RetryState: """ Tracks how long a data source should wait before its next attempt. - A failure moves the state on and returns the wait. The delay for attempt - ``n`` is ``min(min_delay * 2 ** (n - 1), max_delay)``, less a random - jitter of up to half of it, and never less than the operating cadence. + A failure moves the state on and decides the next wait, which + :attr:`next_delay` reports. The delay for attempt ``n`` is + ``min(min_delay * 2 ** (n - 1), max_delay)``, less a random jitter of up to + half of it, and never less than the operating cadence. An unexpected failure moves the state to the extended regime, which raises both delay bounds. The bounds stay raised until the reset condition is met, @@ -228,13 +229,12 @@ def in_extended_regime(self) -> bool: delay bounds.""" return self._extended - def record_failure(self, kind: FailureKind) -> float: + def record_failure(self, kind: FailureKind) -> None: """ - Records a failed attempt and returns how long to wait before the next - one, in seconds. + Records a failed attempt, and decides the wait before the next one. - The state moves on before the wait is computed, so the wait always - reflects the failure just recorded. + The state moves on before the wait is computed, so :attr:`next_delay` + always reflects the failure just recorded. :param kind: how the failure was classified """ @@ -256,7 +256,6 @@ def record_failure(self, kind: FailureKind) -> float: self._n += 1 self._next_delay = self._compute_wait() - return self._next_delay def record_success(self) -> None: """ diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 91823690..f94b10e6 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -82,6 +82,13 @@ def real_jitter(): yield +def failure_delay(state, kind=NORMAL) -> float: + """Records a failure and reads back the wait it decided, which is the value + a data source reads.""" + state.record_failure(kind) + return state.next_delay + + def streaming_state(initial_delay=1): return for_streaming(initial_delay) @@ -119,7 +126,7 @@ def test_a_non_positive_delay_falls_back_to_the_default(self, configured, caplog state = for_streaming(configured) assert state.min_delay == DEFAULT_INITIAL_RECONNECT_DELAY - assert state.record_failure(NORMAL) == DEFAULT_INITIAL_RECONNECT_DELAY + assert failure_delay(state, NORMAL) == DEFAULT_INITIAL_RECONNECT_DELAY assert caplog.records[0].getMessage() == ( "initial_reconnect_delay must be greater than zero; using the default of 1s" ) @@ -131,7 +138,7 @@ def test_a_positive_delay_is_left_alone(self, configured, caplog): state = for_streaming(configured) assert state.min_delay == configured - assert state.record_failure(NORMAL) == configured + assert failure_delay(state, NORMAL) == configured assert caplog.records == [] @@ -144,12 +151,12 @@ class TestStreamingExtendedDelayFloor: [(1, 300), (30, 300), (300, 300), (600, 600), (3600, 3600), (0, 300), (-5, 300)], ) def test_the_extended_delay_never_starts_below_the_configured_delay(self, configured, expected): - assert for_streaming(configured).record_failure(UNEXPECTED) == expected + assert failure_delay(for_streaming(configured), UNEXPECTED) == expected @pytest.mark.parametrize("configured", [1, 30, 300, 600, 3600]) def test_an_unexpected_failure_never_waits_less_than_a_normal_one(self, configured): - normal = for_streaming(configured).record_failure(NORMAL) - unexpected = for_streaming(configured).record_failure(UNEXPECTED) + normal = failure_delay(for_streaming(configured), NORMAL) + unexpected = failure_delay(for_streaming(configured), UNEXPECTED) assert unexpected >= normal @pytest.mark.parametrize( @@ -162,28 +169,28 @@ def test_an_unexpected_failure_never_waits_less_than_a_normal_one(self, configur ) def test_the_extended_ladder_still_doubles_to_the_ceiling(self, configured, ladder): state = for_streaming(configured) - delays = [state.record_failure(UNEXPECTED)] - delays += [state.record_failure(NORMAL) for _ in range(4)] + delays = [failure_delay(state, UNEXPECTED)] + delays += [failure_delay(state, NORMAL) for _ in range(4)] assert delays == ladder class TestStreamingDelayTable: def test_normal_regime_doubles_up_to_the_ceiling(self): state = streaming_state(initial_delay=1) - delays = [state.record_failure(NORMAL) for _ in range(8)] + delays = [failure_delay(state, NORMAL) for _ in range(8)] assert delays == [1, 2, 4, 8, 16, 30, 30, 30] def test_extended_regime_doubles_up_to_the_ceiling(self): state = streaming_state(initial_delay=1) - delays = [state.record_failure(UNEXPECTED)] - delays += [state.record_failure(NORMAL) for _ in range(5)] + delays = [failure_delay(state, UNEXPECTED)] + delays += [failure_delay(state, NORMAL) for _ in range(5)] assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] def test_a_configured_initial_delay_raises_the_ceiling_with_it(self): # RETRY 1.5.4 as amended: maxDelay must not fall below initialDelay. state = streaming_state(initial_delay=45) assert state.max_delay == 45 - assert state.record_failure(NORMAL) == 45 + assert failure_delay(state, NORMAL) == 45 def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): # RETRY 1.5.5: a normal failure after an unexpected one must not lower @@ -202,9 +209,9 @@ def test_a_second_unexpected_failure_keeps_counting_up(self): # Restarting the count on every unexpected failure would pin the delay # at the extended initial delay for ever. state = streaming_state(initial_delay=1) - assert state.record_failure(UNEXPECTED) == 5 * 60 - assert state.record_failure(UNEXPECTED) == 10 * 60 - assert state.record_failure(UNEXPECTED) == 20 * 60 + assert failure_delay(state, UNEXPECTED) == 5 * 60 + assert failure_delay(state, UNEXPECTED) == 10 * 60 + assert failure_delay(state, UNEXPECTED) == 20 * 60 def test_the_streaming_defaults_match_the_spec(self): state = streaming_state(initial_delay=1) @@ -217,12 +224,12 @@ class TestJitter: def test_jitter_never_removes_more_than_half_the_delay(self): with fixed_retry_jitter(FULL_JITTER): state = streaming_state(initial_delay=8) - delay = state.record_failure(NORMAL) + delay = failure_delay(state, NORMAL) assert 4 <= delay < 8 def test_no_jitter_leaves_the_delay_alone(self): state = streaming_state(initial_delay=8) - assert state.record_failure(NORMAL) == 8 + assert failure_delay(state, NORMAL) == 8 def test_every_delay_stays_within_the_jitter_bounds(self): # The real random source, so the bound has to hold for any draw rather @@ -230,7 +237,7 @@ def test_every_delay_stays_within_the_jitter_bounds(self): with real_jitter(): state = streaming_state(initial_delay=1) for base in [1, 2, 4, 8, 16, 30, 30, 30]: - delay = state.record_failure(NORMAL) + delay = failure_delay(state, NORMAL) assert base / 2 <= delay <= base @@ -249,7 +256,7 @@ def test_a_minute_of_healthy_operation_resets_the_state(self): state.record_success() assert not state.in_extended_regime assert state.max_delay == STREAMING_MAX_DELAY - assert state.record_failure(NORMAL) == 1 + assert failure_delay(state, NORMAL) == 1 def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): with frozen_clock() as clock: @@ -262,7 +269,7 @@ def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): # The state resets before this failure is counted, so the delay is # the first-retry delay again rather than the fourth. - assert state.record_failure(NORMAL) == 1 + assert failure_delay(state, NORMAL) == 1 def test_a_short_healthy_stretch_does_not_reset(self): with frozen_clock() as clock: @@ -271,7 +278,7 @@ def test_a_short_healthy_stretch_does_not_reset(self): state.record_success() clock.advance(STREAMING_RESET_INTERVAL - 1) - assert state.record_failure(NORMAL) == 2 + assert failure_delay(state, NORMAL) == 2 def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(self): # Every transport failure is normal, so no amount of flapping reaches @@ -283,7 +290,7 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se for _ in range(20): state.record_success() clock.advance(5) - delays.append(state.record_failure(NORMAL)) + delays.append(failure_delay(state, NORMAL)) clock.advance(1) assert not state.in_extended_regime @@ -294,26 +301,26 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se class TestPollingCadence: def test_a_normal_failure_polls_again_on_schedule(self): state = polling_state(poll_interval=30) - assert [state.record_failure(NORMAL) for _ in range(4)] == [30, 30, 30, 30] + assert [failure_delay(state, NORMAL) for _ in range(4)] == [30, 30, 30, 30] def test_the_extended_regime_doubles_up_to_an_hour(self): state = polling_state(poll_interval=30) - delays = [state.record_failure(UNEXPECTED)] - delays += [state.record_failure(NORMAL) for _ in range(5)] + delays = [failure_delay(state, UNEXPECTED)] + delays += [failure_delay(state, NORMAL) for _ in range(5)] assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] def test_the_wait_never_falls_below_the_poll_interval(self): # RETRY 1.4.9. Full jitter would otherwise halve the delay. with fixed_retry_jitter(FULL_JITTER): state = polling_state(poll_interval=30) - assert state.record_failure(NORMAL) == 30 - assert state.record_failure(UNEXPECTED) >= 30 + assert failure_delay(state, NORMAL) == 30 + assert failure_delay(state, UNEXPECTED) >= 30 def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): # The ceiling is lifted by record_failure clamping it against the # initial delay, not by for_polling clamping the ceiling itself. state = polling_state(poll_interval=2 * 60 * 60) - assert state.record_failure(UNEXPECTED) == 2 * 60 * 60 + assert failure_delay(state, UNEXPECTED) == 2 * 60 * 60 assert state.max_delay == 2 * 60 * 60 assert state.min_delay == 2 * 60 * 60 @@ -324,7 +331,7 @@ def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): state = polling_state(poll_interval=30) state.record_failure(UNEXPECTED) state.record_failure(NORMAL) - assert state.record_failure(NORMAL) == 20 * 60 + assert failure_delay(state, NORMAL) == 20 * 60 state.record_success() assert state.next_delay == 30 @@ -341,7 +348,7 @@ def test_two_successes_in_a_row_reset_the_state(self): state.record_success() assert not state.in_extended_regime assert state.next_delay == 30 - assert state.record_failure(NORMAL) == 30 + assert failure_delay(state, NORMAL) == 30 def test_a_failure_between_two_successes_clears_the_first(self): state = polling_state(poll_interval=30) @@ -384,7 +391,7 @@ def test_a_reset_starts_the_attempt_count_over(self): # The delay drops back to the first-retry value, and the count # starts over with it. - assert state.record_failure(NORMAL) == 1 + assert failure_delay(state, NORMAL) == 1 assert state.attempts == 1 @@ -460,5 +467,5 @@ def test_a_long_outage_cannot_overflow_the_delay(self): reset_policy=AfterHealthyFor(60), ) for _ in range(5000): - delay = state.record_failure(NORMAL) + delay = failure_delay(state, NORMAL) assert delay == 30 From 23474a7c1359c8133ac4e7b99ee197d539690f87 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Mon, 14 Sep 2026 14:04:55 -0500 Subject: [PATCH 05/19] chore: Remove spec citations and passthrough test helpers --- .../impl/datasource/test_async_polling.py | 6 +- .../impl/datasource/test_polling_processor.py | 4 +- ldclient/testing/impl/test_retry.py | 84 ++++++++----------- ldclient/testing/test_util.py | 2 +- 4 files changed, 43 insertions(+), 53 deletions(-) diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index 8a87a6dc..b7f1f5b5 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -253,9 +253,9 @@ async def test_unexpected_http_error_moves_to_the_extended_regime(self, mock_int @pytest.mark.asyncio async def test_the_first_success_after_an_outage_polls_at_the_cadence(self): - # RETRY 1.4.8: a backoff wait applies to a retry, not to every - # operation. _poll returns the wait, so this reads it directly rather - # than measuring elapsed time. + # A backoff wait applies to a retry, not to every operation. The + # retry state carries the wait, so this reads it there rather than + # measuring elapsed time. store = MockAsyncFeatureStore() ready = asyncio.Event() config = make_config() diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index 9d073167..606fcd96 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -166,8 +166,8 @@ def test_unexpected_http_error_moves_to_the_extended_regime(ignore_mock): def test_the_first_success_after_an_outage_polls_at_the_cadence(): - # RETRY 1.4.8: a backoff wait applies to a retry, not to every operation. - # _poll returns the wait, so this reads it directly rather than measuring + # A backoff wait applies to a retry, not to every operation. The retry + # state carries the wait, so this reads it there rather than measuring # elapsed time. with no_retry_jitter(): retry = for_polling(30) diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index f94b10e6..5e58a5c0 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -61,8 +61,8 @@ def frozen_clock(now: float = 1000.0): yield clock -# The random draw just below 1, which subtracts as much jitter as the spec -# allows: half the delay. +# The random draw just below 1, which subtracts the most jitter possible: +# half the delay. FULL_JITTER = 0.9999999 @@ -89,14 +89,6 @@ def failure_delay(state, kind=NORMAL) -> float: return state.next_delay -def streaming_state(initial_delay=1): - return for_streaming(initial_delay) - - -def polling_state(poll_interval=30): - return for_polling(poll_interval) - - class TestClassifyHttpStatus: @pytest.mark.parametrize("status", [400, 408, 429]) def test_retryable_4xx_statuses_are_normal(self, status): @@ -143,8 +135,8 @@ def test_a_positive_delay_is_left_alone(self, configured, caplog): class TestStreamingExtendedDelayFloor: - """RETRY 1.5.4.1: a delay that applies after an unexpected failure must not - be below the component's initial delay.""" + """A delay that applies after an unexpected failure must not be below the + component's initial delay.""" @pytest.mark.parametrize( "configured,expected", @@ -176,26 +168,26 @@ def test_the_extended_ladder_still_doubles_to_the_ceiling(self, configured, ladd class TestStreamingDelayTable: def test_normal_regime_doubles_up_to_the_ceiling(self): - state = streaming_state(initial_delay=1) + state = for_streaming(1) delays = [failure_delay(state, NORMAL) for _ in range(8)] assert delays == [1, 2, 4, 8, 16, 30, 30, 30] def test_extended_regime_doubles_up_to_the_ceiling(self): - state = streaming_state(initial_delay=1) + state = for_streaming(1) delays = [failure_delay(state, UNEXPECTED)] delays += [failure_delay(state, NORMAL) for _ in range(5)] assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] def test_a_configured_initial_delay_raises_the_ceiling_with_it(self): - # RETRY 1.5.4 as amended: maxDelay must not fall below initialDelay. - state = streaming_state(initial_delay=45) + # The ceiling must not fall below the initial delay. + state = for_streaming(45) assert state.max_delay == 45 assert failure_delay(state, NORMAL) == 45 def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): - # RETRY 1.5.5: a normal failure after an unexpected one must not lower - # the bounds back to the normal regime. - state = streaming_state(initial_delay=1) + # A normal failure after an unexpected one must not lower the bounds + # back to the normal regime. + state = for_streaming(1) state.record_failure(UNEXPECTED) assert state.in_extended_regime assert state.max_delay == EXTENDED_MAX_DELAY @@ -208,13 +200,13 @@ def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): def test_a_second_unexpected_failure_keeps_counting_up(self): # Restarting the count on every unexpected failure would pin the delay # at the extended initial delay for ever. - state = streaming_state(initial_delay=1) + state = for_streaming(1) assert failure_delay(state, UNEXPECTED) == 5 * 60 assert failure_delay(state, UNEXPECTED) == 10 * 60 assert failure_delay(state, UNEXPECTED) == 20 * 60 def test_the_streaming_defaults_match_the_spec(self): - state = streaming_state(initial_delay=1) + state = for_streaming(1) assert state.max_delay == STREAMING_MAX_DELAY assert state.operating_cadence == 0 assert STREAMING_RESET_INTERVAL == 60 @@ -223,19 +215,19 @@ def test_the_streaming_defaults_match_the_spec(self): class TestJitter: def test_jitter_never_removes_more_than_half_the_delay(self): with fixed_retry_jitter(FULL_JITTER): - state = streaming_state(initial_delay=8) + state = for_streaming(8) delay = failure_delay(state, NORMAL) assert 4 <= delay < 8 def test_no_jitter_leaves_the_delay_alone(self): - state = streaming_state(initial_delay=8) + state = for_streaming(8) assert failure_delay(state, NORMAL) == 8 def test_every_delay_stays_within_the_jitter_bounds(self): # The real random source, so the bound has to hold for any draw rather # than for one seeded sequence. with real_jitter(): - state = streaming_state(initial_delay=1) + state = for_streaming(1) for base in [1, 2, 4, 8, 16, 30, 30, 30]: delay = failure_delay(state, NORMAL) assert base / 2 <= delay <= base @@ -243,9 +235,9 @@ def test_every_delay_stays_within_the_jitter_bounds(self): class TestStreamingReset: def test_a_minute_of_healthy_operation_resets_the_state(self): - # RETRY 1.8.2. The whole minute passes instantly. + # The whole minute passes instantly. with frozen_clock() as clock: - state = streaming_state(initial_delay=1) + state = for_streaming(1) state.record_failure(UNEXPECTED) state.record_failure(NORMAL) @@ -260,7 +252,7 @@ def test_a_minute_of_healthy_operation_resets_the_state(self): def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): with frozen_clock() as clock: - state = streaming_state(initial_delay=1) + state = for_streaming(1) state.record_failure(NORMAL) state.record_failure(NORMAL) @@ -273,7 +265,7 @@ def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): def test_a_short_healthy_stretch_does_not_reset(self): with frozen_clock() as clock: - state = streaming_state(initial_delay=1) + state = for_streaming(1) state.record_failure(NORMAL) state.record_success() @@ -285,7 +277,7 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se # the extended regime. Each cycle is a healthy stretch shorter than the # reset window, so the delay climbs, but only to the normal ceiling. with frozen_clock() as clock: - state = streaming_state(initial_delay=1) + state = for_streaming(1) delays = [] for _ in range(20): state.record_success() @@ -300,35 +292,35 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se class TestPollingCadence: def test_a_normal_failure_polls_again_on_schedule(self): - state = polling_state(poll_interval=30) + state = for_polling(30) assert [failure_delay(state, NORMAL) for _ in range(4)] == [30, 30, 30, 30] def test_the_extended_regime_doubles_up_to_an_hour(self): - state = polling_state(poll_interval=30) + state = for_polling(30) delays = [failure_delay(state, UNEXPECTED)] delays += [failure_delay(state, NORMAL) for _ in range(5)] assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] def test_the_wait_never_falls_below_the_poll_interval(self): - # RETRY 1.4.9. Full jitter would otherwise halve the delay. + # Full jitter would otherwise halve the delay. with fixed_retry_jitter(FULL_JITTER): - state = polling_state(poll_interval=30) + state = for_polling(30) assert failure_delay(state, NORMAL) == 30 assert failure_delay(state, UNEXPECTED) >= 30 def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): # The ceiling is lifted by record_failure clamping it against the # initial delay, not by for_polling clamping the ceiling itself. - state = polling_state(poll_interval=2 * 60 * 60) + state = for_polling(2 * 60 * 60) assert failure_delay(state, UNEXPECTED) == 2 * 60 * 60 assert state.max_delay == 2 * 60 * 60 assert state.min_delay == 2 * 60 * 60 def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): - # RETRY 1.4.8. Conflating this with the reset is the bug another SDK - # shipped: its first successful poll after an outage still waited - # twenty minutes or more. - state = polling_state(poll_interval=30) + # A backoff wait applies to a retry, not to every operation. + # Conflating this with the reset would leave the first successful poll + # after an outage still waiting twenty minutes. + state = for_polling(30) state.record_failure(UNEXPECTED) state.record_failure(NORMAL) assert failure_delay(state, NORMAL) == 20 * 60 @@ -338,8 +330,7 @@ def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): assert state.in_extended_regime, "one success does not reset the state" def test_two_successes_in_a_row_reset_the_state(self): - # RETRY 1.8.2 with the polling reset policy. - state = polling_state(poll_interval=30) + state = for_polling(30) state.record_failure(UNEXPECTED) state.record_success() @@ -351,7 +342,7 @@ def test_two_successes_in_a_row_reset_the_state(self): assert failure_delay(state, NORMAL) == 30 def test_a_failure_between_two_successes_clears_the_first(self): - state = polling_state(poll_interval=30) + state = for_polling(30) state.record_failure(UNEXPECTED) state.record_success() state.record_failure(NORMAL) @@ -362,7 +353,7 @@ def test_a_failure_between_two_successes_clears_the_first(self): assert not state.in_extended_regime def test_the_polling_defaults_match_the_spec(self): - state = polling_state(poll_interval=30) + state = for_polling(30) assert state.operating_cadence == 30 assert state.min_delay == 30 assert state.max_delay == 30 @@ -371,16 +362,15 @@ def test_the_polling_defaults_match_the_spec(self): class TestAttemptCount: def test_attempts_counts_every_failure(self): - state = streaming_state(initial_delay=1) + state = for_streaming(1) for _ in range(5): state.record_failure(NORMAL) assert state.attempts == 5 def test_a_reset_starts_the_attempt_count_over(self): - # The streaming spec resets both counters: "set attempt to 1, set n - # to 1". + # A reset clears both counters, so the next failure is attempt 1. with frozen_clock() as clock: - state = streaming_state(initial_delay=1) + state = for_streaming(1) state.record_failure(NORMAL) state.record_failure(NORMAL) assert state.attempts == 2 @@ -414,7 +404,7 @@ def test_healthy_for_tracks_the_start_of_the_stretch(self): def test_many_healthy_signals_do_not_move_the_window(self): """Streaming signals on every message, so an unconditional assignment - here would push its reset out for ever -- Go's SDK-2845.""" + here would push its reset out for ever.""" with frozen_clock() as clock: policy = AfterHealthyFor(60) policy.note_healthy() diff --git a/ldclient/testing/test_util.py b/ldclient/testing/test_util.py index 2036d097..e2169cc4 100644 --- a/ldclient/testing/test_util.py +++ b/ldclient/testing/test_util.py @@ -77,7 +77,7 @@ def fixed_retry_jitter(fraction: float): """Fixes the jitter that :mod:`ldclient.impl.retry` subtracts from a delay. ``0`` subtracts none, so a test can assert an exact delay. A value just - below ``1`` subtracts as much as the spec allows, which is half. + below ``1`` subtracts as much as possible, which is half. Patching the retry module's own ``random`` reference keeps the change local to that module; every other module keeps the real source. From fbd8f85e51b708ecb24e2691e3154d97f6b2e492 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 15 Sep 2026 13:17:55 -0500 Subject: [PATCH 06/19] fix: Address review findings on input guards, next delay and test coverage --- ldclient/impl/aio/concurrency.py | 1 + ldclient/impl/retry.py | 53 ++++++++---- .../impl/datasource/test_async_polling.py | 4 +- ldclient/testing/impl/test_repeating_task.py | 59 +++++++++++--- ldclient/testing/impl/test_retry.py | 81 +++++++++++++++++-- ldclient/testing/test_aio.py | 42 ++++++++++ 6 files changed, 201 insertions(+), 39 deletions(-) diff --git a/ldclient/impl/aio/concurrency.py b/ldclient/impl/aio/concurrency.py index 62ba70d9..ab1f6220 100644 --- a/ldclient/impl/aio/concurrency.py +++ b/ldclient/impl/aio/concurrency.py @@ -215,6 +215,7 @@ def start(self): log.info("Task %s has already been started; ignoring" % self.__label) return self.__task = asyncio.ensure_future(self._run()) + self.__task.add_done_callback(_log_task_exception) try: self.__task.set_name(f"{self.__label}.repeating") except AttributeError: diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 41d4220b..006aa467 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -18,6 +18,7 @@ # currently excluded from documentation - see docs/README.md +import math import random import time from enum import Enum @@ -34,10 +35,10 @@ # delay is configurable as ``initial_reconnect_delay``. STREAMING_MAX_DELAY = 30 -# The documented default for ``initial_reconnect_delay``, in seconds. It stands -# in for a configured value of zero or less, which would reconnect with no wait -# at all. +# The documented defaults, in seconds. Each stands in for a configured value +# that is not a positive, finite number. DEFAULT_INITIAL_RECONNECT_DELAY = 1 +DEFAULT_POLL_INTERVAL = 30 # How long streaming must operate without a failure before its retry state # resets, in seconds. @@ -195,7 +196,7 @@ def __init__( self._max_delay = max(normal_ceiling, initial_delay) self._attempts = 0 # Read before any outcome is recorded, this is the ordinary interval. - self._next_delay = operating_cadence if operating_cadence > 0 else initial_delay + self._next_delay = self._wait_between_operations() @property def next_delay(self) -> float: @@ -262,13 +263,18 @@ def record_success(self) -> None: Records a successful operation, and resets the retry state if that is now enough. - The wait before the next operation becomes the operating cadence, even - when the retry state is still raised, because a backoff wait applies to - a retry and not to every operation. + The wait before the next operation goes back to the ordinary interval, + even when the retry state is still raised, because a backoff wait + applies to a retry and not to every operation. """ self._reset_policy.note_healthy() self._reset_if_due() - self._next_delay = self._operating_cadence + self._next_delay = self._wait_between_operations() + + def _wait_between_operations(self) -> float: + """The wait when nothing is being retried: the operating cadence, or + the initial delay for a component that has no cadence.""" + return self._operating_cadence if self._operating_cadence > 0 else self._initial_delay def _reset_if_due(self) -> None: """Clears the retry state when the reset policy is satisfied, returning @@ -288,6 +294,19 @@ def _compute_wait(self) -> float: return max(delay - jitter, self._operating_cadence) +def _positive_finite(value: float, default: float, name: str) -> float: + """Returns ``value`` if it is a positive, finite number of seconds, and the + default otherwise. A non-finite value would make the jitter arithmetic + produce a NaN delay, and a non-positive one would retry with no wait.""" + if value > 0 and math.isfinite(value): + return value + log.warning( + "%s must be a positive, finite number of seconds; using the default of %ss" + % (name, default) + ) + return default + + def for_streaming(initial_reconnect_delay: float) -> RetryState: """ Builds the retry state for a streaming data source. @@ -296,18 +315,14 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: is healthy from the first message of a fresh stream, and resets after a minute of that. - A configured delay of zero or less would reconnect with no wait, so the - documented default stands in for it. ``Config`` does not check this value, - though it does clamp ``poll_interval``. + ``Config`` does not check the configured delay, so the documented default + stands in for anything that is not a positive, finite number. The extended regime never starts below the configured delay. """ - if initial_reconnect_delay <= 0: - log.warning( - "initial_reconnect_delay must be greater than zero; using the default of %ss" - % DEFAULT_INITIAL_RECONNECT_DELAY - ) - initial_reconnect_delay = DEFAULT_INITIAL_RECONNECT_DELAY + initial_reconnect_delay = _positive_finite( + initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay' + ) return RetryState( initial_delay=initial_reconnect_delay, normal_ceiling=STREAMING_MAX_DELAY, @@ -326,7 +341,11 @@ def for_polling(poll_interval: float) -> RetryState: interval itself, which means a normal failure simply polls again on schedule. Polling is healthy on any successful poll, and resets after two in a row. + + ``Config`` clamps the poll interval, but the documented default stands in + for anything that reaches here and is not a positive, finite number. """ + poll_interval = _positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval') return RetryState( initial_delay=poll_interval, normal_ceiling=poll_interval, diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index b7f1f5b5..79812547 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -284,7 +284,7 @@ async def test_recoverable_http_error_continues_polling(self, mock_interval): store = MockAsyncFeatureStore() ready = asyncio.Event() config = make_config() - processor = make_processor(config=config, store=store, ready=ready) + processor = make_processor(config=config, store=store, ready=ready, retry_state=fast_retry_state()) call_count = 0 @@ -314,7 +314,7 @@ async def test_general_exception_does_not_stop_polling(self, mock_interval): store = MockAsyncFeatureStore() ready = asyncio.Event() config = make_config() - processor = make_processor(config=config, store=store, ready=ready) + processor = make_processor(config=config, store=store, ready=ready, retry_state=fast_retry_state()) call_count = 0 diff --git a/ldclient/testing/impl/test_repeating_task.py b/ldclient/testing/impl/test_repeating_task.py index 5ab6f184..979652c4 100644 --- a/ldclient/testing/impl/test_repeating_task.py +++ b/ldclient/testing/impl/test_repeating_task.py @@ -79,35 +79,70 @@ def test_task_executes_until_stopped(): assert no_more_items is True -class _MutableDelay(DelaySource): - """A delay source a test can move between invocations.""" +class _RecordingDelay(DelaySource): + """A delay source that records each read, so a test can see when the task + asks for a wait rather than only what the action saw.""" - def __init__(self, seconds: float): + def __init__(self, seconds: float, events: list): self.seconds = seconds + self._events = events @property def next_delay(self) -> float: + self._events.append(('read', self.seconds)) return self.seconds def test_the_task_reads_the_delay_source_after_every_invocation(): - """A value the action decides takes effect on the next wait.""" - reads = Queue() - delays = _MutableDelay(0.01) + """One read per invocation, after it. A task that read the source once up + front would show a read before the first invocation, and would never see + the value the action set.""" + events: list = [] + delays = _RecordingDelay(0.01, events) def do_task(): - reads.put(delays.seconds) - delays.seconds = 0.02 # what the next wait must use + events.append('invoke') + delays.seconds = 0.02 - task = RepeatingTask("ldclient.testing.mutable-delay", delays, 0, do_task) + task = RepeatingTask("ldclient.testing.recording-delay", delays, 0, do_task) try: task.start() - assert reads.get(True, 1) == 0.01 - assert reads.get(True, 1) == 0.02 - assert reads.get(True, 1) == 0.02 + deadline = time.time() + 2 + while events.count('invoke') < 3 and time.time() < deadline: + time.sleep(0.005) finally: task.stop() + # Reads and invocations alternate, starting with an invocation, and every + # read sees 0.02 -- the initial 0.01 is never read. + assert events[:5] == ['invoke', ('read', 0.02), 'invoke', ('read', 0.02), 'invoke'] + + +def test_the_interval_starts_when_the_callback_returns(): + """A slow callback must not shorten its own wait: one invocation to the + next is the interval plus however long the callback took.""" + work = 0.15 + interval = 0.15 + starts = Queue() + + def do_task(): + starts.put(time.time()) + time.sleep(work) + + task = RepeatingTask.at_interval("ldclient.testing.slow-callback", interval, 0, do_task) + try: + first = None + task.start() + first = starts.get(True, 2) + second = starts.get(True, 2) + finally: + task.stop() + + # Measuring the interval from the start of the callback would give about + # `interval`; measuring from its return gives interval + work. The 10% + # slack is for scheduling noise, and leaves the two regimes far apart. + assert (second - first) >= (interval + work) * 0.9 + def test_whatever_the_action_returns_is_ignored(): """Guards big-segment polling, whose action returns a status object.""" diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 5e58a5c0..ca1d67dc 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -8,6 +8,7 @@ """ import logging +import math import random from contextlib import contextmanager from unittest import mock @@ -17,6 +18,7 @@ from ldclient.impl import retry from ldclient.impl.retry import ( DEFAULT_INITIAL_RECONNECT_DELAY, + DEFAULT_POLL_INTERVAL, EXTENDED_INITIAL_DELAY, EXTENDED_MAX_DELAY, POLLING_RESET_SUCCESSES, @@ -107,24 +109,51 @@ def test_non_error_statuses_are_normal(self, status): assert classify_http_status(status) is NORMAL -class TestStreamingInitialDelayGuard: - """``Config`` does not check ``initial_reconnect_delay``, and a value of - zero would reconnect with no wait at all.""" +class TestFactoryInputGuards: + """``Config`` does not check ``initial_reconnect_delay`` at all, and only + clamps ``poll_interval``. A non-positive value would retry with no wait; a + non-finite one makes the jitter arithmetic produce NaN.""" - @pytest.mark.parametrize("configured", [0, -1, -0.5]) - def test_a_non_positive_delay_falls_back_to_the_default(self, configured, caplog): + @pytest.mark.parametrize( + "configured", + [0, -1, -0.5, float('inf'), float('-inf'), float('nan')], + ids=["zero", "negative", "negative-fraction", "inf", "-inf", "nan"], + ) + def test_streaming_falls_back_to_the_default(self, configured, caplog): caplog.set_level(logging.WARNING) state = for_streaming(configured) + delay = failure_delay(state, NORMAL) assert state.min_delay == DEFAULT_INITIAL_RECONNECT_DELAY - assert failure_delay(state, NORMAL) == DEFAULT_INITIAL_RECONNECT_DELAY + assert delay == DEFAULT_INITIAL_RECONNECT_DELAY + assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( - "initial_reconnect_delay must be greater than zero; using the default of 1s" + "initial_reconnect_delay must be a positive, finite number of seconds; " + "using the default of 1s" + ) + + @pytest.mark.parametrize( + "configured", + [0, -5, float('inf'), float('-inf'), float('nan')], + ids=["zero", "negative", "inf", "-inf", "nan"], + ) + def test_polling_falls_back_to_the_default(self, configured, caplog): + caplog.set_level(logging.WARNING) + + state = for_polling(configured) + delay = failure_delay(state, NORMAL) + + assert state.operating_cadence == DEFAULT_POLL_INTERVAL + assert delay == DEFAULT_POLL_INTERVAL + assert math.isfinite(delay) and delay > 0 + assert caplog.records[0].getMessage() == ( + "poll_interval must be a positive, finite number of seconds; " + "using the default of 30s" ) @pytest.mark.parametrize("configured", [0.001, 0.5, 1, 5, 45]) - def test_a_positive_delay_is_left_alone(self, configured, caplog): + def test_a_positive_streaming_delay_is_left_alone(self, configured, caplog): caplog.set_level(logging.WARNING) state = for_streaming(configured) @@ -133,6 +162,16 @@ def test_a_positive_delay_is_left_alone(self, configured, caplog): assert failure_delay(state, NORMAL) == configured assert caplog.records == [] + @pytest.mark.parametrize("configured", [0.001, 1, 30, 300, 2 * 60 * 60]) + def test_a_positive_poll_interval_is_left_alone(self, configured, caplog): + caplog.set_level(logging.WARNING) + + state = for_polling(configured) + + assert state.operating_cadence == configured + assert failure_delay(state, NORMAL) == configured + assert caplog.records == [] + class TestStreamingExtendedDelayFloor: """A delay that applies after an unexpected failure must not be below the @@ -233,6 +272,32 @@ def test_every_delay_stays_within_the_jitter_bounds(self): assert base / 2 <= delay <= base +class TestWaitBetweenOperations: + def test_a_streaming_success_does_not_schedule_a_zero_wait(self): + """Streaming has no cadence, so a success falls back to the initial + delay. Zero would tell a scheduler to run again immediately.""" + state = for_streaming(1) + state.record_failure(NORMAL) + state.record_success() + + assert state.next_delay == 1 + + def test_a_polling_success_schedules_the_cadence(self): + state = for_polling(30) + state.record_failure(NORMAL) + state.record_success() + + assert state.next_delay == 30 + + def test_a_fresh_state_and_a_success_agree(self): + """The constructor and record_success share one expression, so the two + cannot drift apart.""" + for state in (for_streaming(5), for_polling(45)): + fresh = state.next_delay + state.record_success() + assert state.next_delay == fresh + + class TestStreamingReset: def test_a_minute_of_healthy_operation_resets_the_state(self): # The whole minute passes instantly. diff --git a/ldclient/testing/test_aio.py b/ldclient/testing/test_aio.py index 94e6d80c..4bc2668d 100644 --- a/ldclient/testing/test_aio.py +++ b/ldclient/testing/test_aio.py @@ -177,6 +177,48 @@ async def action(): await asyncio.sleep(0.1) assert counts['n'] == 1 + @pytest.mark.asyncio + async def test_async_a_task_that_dies_is_logged(self, caplog): + """Without a done callback the held reference suppresses asyncio's own + warning, so a dead loop would be entirely silent.""" + caplog.set_level(logging.ERROR) + + class Exploding: + @property + def next_delay(self): + raise RuntimeError("delay source is broken") + + async def action(): + pass + + task = aio.AsyncRepeatingTask("test.repeating", Exploding(), 0, action) + task.start() + await _async_wait_until(lambda: caplog.records, timeout=2) + task.stop() + + assert "Unhandled exception in background task" in caplog.records[0].getMessage() + + @pytest.mark.asyncio + async def test_async_interval_starts_when_the_callback_returns(self): + """A slow callback must not shorten its own wait: one invocation to the + next is the interval plus however long the callback took.""" + work = 0.15 + interval = 0.15 + starts: list = [] + + async def action(): + starts.append(time.time()) + await asyncio.sleep(work) + + task = aio.AsyncRepeatingTask.at_interval("test.repeating", interval, 0, action) + task.start() + await _async_wait_until(lambda: len(starts) >= 2, timeout=3) + task.stop() + + # Measuring the interval from the start of the callback would give + # about `interval`; measuring from its return gives interval + work. + assert (starts[1] - starts[0]) >= (interval + work) * 0.9 + @pytest.mark.asyncio async def test_async_second_start_logs_and_does_not_raise(self, caplog): """Mirrors the sync primitive. A raise here can surface out of a caller From da7271313d0a111da2c1bd924b26f4f3cdfd5be8 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 16 Sep 2026 12:09:35 -0500 Subject: [PATCH 07/19] addressing feedback --- ldclient/impl/retry.py | 52 +++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 006aa467..297792ea 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -26,20 +26,17 @@ from ldclient.impl.util import log +# The documented defaults, in seconds. Each stands in for a configured value +# that is not a positive, finite number. +DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY = 1 +DEFAULT_STREAMING_MAX_DELAY = 30 +DEFAULT_POLL_INTERVAL = 30 + # The delay bounds of the extended regime, in seconds. A component enters the # extended regime after an unexpected failure. EXTENDED_INITIAL_DELAY = 5 * 60 EXTENDED_MAX_DELAY = 60 * 60 -# The delay bounds of the normal regime for streaming, in seconds. The initial -# delay is configurable as ``initial_reconnect_delay``. -STREAMING_MAX_DELAY = 30 - -# The documented defaults, in seconds. Each stands in for a configured value -# that is not a positive, finite number. -DEFAULT_INITIAL_RECONNECT_DELAY = 1 -DEFAULT_POLL_INTERVAL = 30 - # How long streaming must operate without a failure before its retry state # resets, in seconds. STREAMING_RESET_INTERVAL = 60 @@ -165,7 +162,7 @@ class RetryState: def __init__( self, - initial_delay: float, + normal_initial_delay: float, normal_ceiling: float, extended_initial_delay: float, extended_ceiling: float, @@ -173,7 +170,7 @@ def __init__( operating_cadence: float = 0, ): """ - :param initial_delay: the delay before the first retry, in seconds + :param normal_initial_delay: the delay before the first retry in the normal regime, in seconds :param normal_ceiling: the longest normal-regime delay, in seconds :param extended_initial_delay: the delay before the first retry in the extended regime, in seconds @@ -183,7 +180,7 @@ def __init__( in seconds; no wait is ever shorter than this. Zero disables the floor, which is what streaming wants. """ - self._initial_delay = initial_delay + self._normal_initial_delay = normal_initial_delay self._normal_ceiling = normal_ceiling self._extended_initial_delay = extended_initial_delay self._extended_ceiling = extended_ceiling @@ -192,8 +189,8 @@ def __init__( self._n = 0 self._extended = False - self._min_delay = initial_delay - self._max_delay = max(normal_ceiling, initial_delay) + self._min_delay = self._normal_initial_delay + self._max_delay = max(self._normal_ceiling, self._normal_initial_delay) self._attempts = 0 # Read before any outcome is recorded, this is the ordinary interval. self._next_delay = self._wait_between_operations() @@ -256,7 +253,11 @@ def record_failure(self, kind: FailureKind) -> None: else: self._n += 1 - self._next_delay = self._compute_wait() + exponent = min(max(self._n - 1, 0), _MAX_BACKOFF_EXPONENT) + delay = min(self._min_delay * (2**exponent), self._max_delay) + jitter = random.random() * delay / 2 + + self._next_delay = max(delay - jitter, self._operating_cadence) def record_success(self) -> None: """ @@ -274,7 +275,7 @@ def record_success(self) -> None: def _wait_between_operations(self) -> float: """The wait when nothing is being retried: the operating cadence, or the initial delay for a component that has no cadence.""" - return self._operating_cadence if self._operating_cadence > 0 else self._initial_delay + return self._operating_cadence if self._operating_cadence >= 0 else self._normal_initial_delay def _reset_if_due(self) -> None: """Clears the retry state when the reset policy is satisfied, returning @@ -284,14 +285,8 @@ def _reset_if_due(self) -> None: self._n = 0 self._attempts = 0 self._extended = False - self._min_delay = self._initial_delay - self._max_delay = max(self._normal_ceiling, self._initial_delay) - - def _compute_wait(self) -> float: - exponent = min(max(self._n - 1, 0), _MAX_BACKOFF_EXPONENT) - delay = min(self._min_delay * (2**exponent), self._max_delay) - jitter = random.random() * delay / 2 - return max(delay - jitter, self._operating_cadence) + self._min_delay = self._normal_initial_delay + self._max_delay = max(self._normal_ceiling, self._normal_initial_delay) def _positive_finite(value: float, default: float, name: str) -> float: @@ -321,14 +316,15 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: The extended regime never starts below the configured delay. """ initial_reconnect_delay = _positive_finite( - initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay' + initial_reconnect_delay, DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay' ) return RetryState( - initial_delay=initial_reconnect_delay, - normal_ceiling=STREAMING_MAX_DELAY, + normal_initial_delay=initial_reconnect_delay, + normal_ceiling=DEFAULT_STREAMING_MAX_DELAY, extended_initial_delay=max(EXTENDED_INITIAL_DELAY, initial_reconnect_delay), extended_ceiling=EXTENDED_MAX_DELAY, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), + operating_cadence=0 ) @@ -347,7 +343,7 @@ def for_polling(poll_interval: float) -> RetryState: """ poll_interval = _positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval') return RetryState( - initial_delay=poll_interval, + normal_initial_delay=poll_interval, normal_ceiling=poll_interval, extended_initial_delay=max(EXTENDED_INITIAL_DELAY, poll_interval), extended_ceiling=EXTENDED_MAX_DELAY, From 619b21ef825e321b1b54cf98c7a27a16045318eb Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 16 Sep 2026 14:54:46 -0500 Subject: [PATCH 08/19] fix: Privatize retry state internals and remove the streaming delay floor Streaming's operating cadence is zero rather than absent, so a healthy stream schedules no wait and a retry delay has no floor. Polling is the only data source that reads next_delay as a DelaySource, and its cadence floor keeps that from reaching zero. Make the seven observational properties private. No data source reads them; they are test instrumentation, and as public API they invite callers to attach logic to unsynchronized state. --- ldclient/impl/retry.py | 65 +++-------- .../impl/datasource/test_async_polling.py | 10 +- .../impl/datasource/test_async_streaming.py | 34 +++--- .../impl/datasource/test_polling_processor.py | 12 +- .../testing/impl/datasource/test_streaming.py | 22 ++-- ldclient/testing/impl/test_retry.py | 109 ++++++++++-------- ldclient/testing/test_util.py | 2 +- 7 files changed, 117 insertions(+), 137 deletions(-) diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 297792ea..ef8095fa 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -116,12 +116,6 @@ def is_satisfied(self) -> bool: return False return time.monotonic() - self._healthy_since >= self._healthy_seconds - @property - def healthy_since(self) -> Optional[float]: - """When the current healthy stretch began, or None if the component is - not currently healthy.""" - return self._healthy_since - class AfterConsecutiveSuccesses(ResetPolicy): """Resets once ``count`` operations in a row have succeeded. This is the @@ -140,11 +134,6 @@ def note_failure(self) -> None: def is_satisfied(self) -> bool: return self._successes >= self._count - @property - def successes(self) -> int: - """How many operations have succeeded in a row.""" - return self._successes - class RetryState: """ @@ -170,15 +159,16 @@ def __init__( operating_cadence: float = 0, ): """ - :param normal_initial_delay: the delay before the first retry in the normal regime, in seconds + :param normal_initial_delay: the delay before the first retry in the + normal regime, in seconds :param normal_ceiling: the longest normal-regime delay, in seconds :param extended_initial_delay: the delay before the first retry in the extended regime, in seconds :param extended_ceiling: the longest extended-regime delay, in seconds :param reset_policy: decides when the retry state resets - :param operating_cadence: the rate the component normally operates at, - in seconds; no wait is ever shorter than this. Zero disables the - floor, which is what streaming wants. + :param operating_cadence: the wait between healthy operations, in + seconds; no wait is ever shorter than this. Zero for a component + that operates continuously. """ self._normal_initial_delay = normal_initial_delay self._normal_ceiling = normal_ceiling @@ -193,40 +183,14 @@ def __init__( self._max_delay = max(self._normal_ceiling, self._normal_initial_delay) self._attempts = 0 # Read before any outcome is recorded, this is the ordinary interval. - self._next_delay = self._wait_between_operations() + self._next_delay = self._operating_cadence @property def next_delay(self) -> float: - """The wait before the next attempt, in seconds, as the last recorded + """The wait before the next operation, in seconds, as the last recorded outcome decided it.""" return self._next_delay - @property - def attempts(self) -> int: - """How many failures since the last reset. For logging only.""" - return self._attempts - - @property - def min_delay(self) -> float: - """The delay the current regime starts from, in seconds.""" - return self._min_delay - - @property - def max_delay(self) -> float: - """The longest delay the current regime allows, in seconds.""" - return self._max_delay - - @property - def operating_cadence(self) -> float: - """The rate the component normally operates at, in seconds.""" - return self._operating_cadence - - @property - def in_extended_regime(self) -> bool: - """Whether an unexpected failure has moved this state to the extended - delay bounds.""" - return self._extended - def record_failure(self, kind: FailureKind) -> None: """ Records a failed attempt, and decides the wait before the next one. @@ -270,12 +234,7 @@ def record_success(self) -> None: """ self._reset_policy.note_healthy() self._reset_if_due() - self._next_delay = self._wait_between_operations() - - def _wait_between_operations(self) -> float: - """The wait when nothing is being retried: the operating cadence, or - the initial delay for a component that has no cadence.""" - return self._operating_cadence if self._operating_cadence >= 0 else self._normal_initial_delay + self._next_delay = self._operating_cadence def _reset_if_due(self) -> None: """Clears the retry state when the reset policy is satisfied, returning @@ -306,9 +265,11 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: """ Builds the retry state for a streaming data source. - Streaming has no operating cadence, so there is no floor on the wait. It - is healthy from the first message of a fresh stream, and resets after a - minute of that. + Streaming's operating cadence is zero, so there is no delay during + healthy operation. Stream failures use either the normal or extended + initial delay to determine their backoff wait. A stream returns to + healthy operation after establishing a successful connection with no + failures during the ``STREAMING_RESET_INTERVAL``. ``Config`` does not check the configured delay, so the documented default stands in for anything that is not a positive, finite number. diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index 79812547..f1921962 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -62,7 +62,7 @@ def fast_retry_state(delay=0.001): """A retry state with tiny delays, so a test does not have to wait out the real extended-regime delay of five minutes.""" return RetryState( - initial_delay=delay, + normal_initial_delay=delay, normal_ceiling=delay, extended_initial_delay=delay, extended_ceiling=delay, @@ -247,7 +247,7 @@ async def test_unexpected_http_error_moves_to_the_extended_regime(self, mock_int processor.start() await asyncio.sleep(0.05) - assert retry.in_extended_regime + assert retry._extended await processor.stop() @@ -270,11 +270,11 @@ async def test_the_first_success_after_an_outage_polls_at_the_cadence(self): processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) await processor._fetch_and_store() assert retry.next_delay == 30 - assert retry.in_extended_regime, "one success restores the cadence but does not reset" + assert retry._extended, "one success restores the cadence but does not reset" await processor._fetch_and_store() assert retry.next_delay == 30 - assert not retry.in_extended_regime, "two successes in a row reset the state" + assert not retry._extended, "two successes in a row reset the state" @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) @@ -390,7 +390,7 @@ async def test_transport_failures_poll_again_at_the_cadence(self, error): await processor._fetch_and_store() assert retry.next_delay == 30 - assert not retry.in_extended_regime + assert not retry._extended @pytest.mark.asyncio async def test_the_log_reports_the_growing_retry_delay(self, caplog): diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index a1ef0122..73484229 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -22,9 +22,9 @@ ) from ldclient.impl.model import ModelEntity from ldclient.impl.retry import ( + DEFAULT_STREAMING_MAX_DELAY, EXTENDED_INITIAL_DELAY, EXTENDED_MAX_DELAY, - STREAMING_MAX_DELAY, STREAMING_RESET_INTERVAL, AfterHealthyFor, RetryState, @@ -97,7 +97,7 @@ def _retry_state_with(policy: AfterHealthyFor) -> RetryState: """A retry state with tiny delays and a caller-supplied reset policy, so a test can watch the window.""" return RetryState( - initial_delay=0.001, + normal_initial_delay=0.001, normal_ceiling=0.001, extended_initial_delay=0.001, extended_ceiling=0.001, @@ -109,7 +109,7 @@ def _fast_retry_state(delay: float = 0.001) -> RetryState: """A retry state with tiny delays, so a test does not have to wait out the real extended-regime delay of five minutes.""" return RetryState( - initial_delay=delay, + normal_initial_delay=delay, normal_ceiling=delay, extended_initial_delay=delay, extended_ceiling=delay, @@ -136,8 +136,8 @@ def _zero_delay_retry_state() -> RetryState: drive ``_handle_error`` without a real sleep. The extended bounds stay real, so a misclassification still shows up in ``max_delay``.""" return RetryState( - initial_delay=0, - normal_ceiling=STREAMING_MAX_DELAY, + normal_initial_delay=0, + normal_ceiling=DEFAULT_STREAMING_MAX_DELAY, extended_initial_delay=EXTENDED_INITIAL_DELAY, extended_ceiling=EXTENDED_MAX_DELAY, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), @@ -310,11 +310,11 @@ async def test_server_close_backs_off_and_does_not_stop_the_processor(): proc, store, ready, factory = _make_processor(actions, retry_state=retry) proc.start() await asyncio.wait_for(ready.wait(), timeout=3.0) - await _wait_until(lambda: retry.attempts >= 1) + await _wait_until(lambda: retry._attempts >= 1) assert store.initialized assert not factory.created[0].closed - assert not retry.in_extended_regime + assert not retry._extended await proc.stop() @@ -362,10 +362,10 @@ async def test_repeated_server_closes_stay_on_the_normal_curve(): retry = _fast_retry_state() proc, store, ready, _ = _make_processor(actions, retry_state=retry) proc.start() - await _wait_until(lambda: retry.attempts >= 10, timeout=5.0) + await _wait_until(lambda: retry._attempts >= 10, timeout=5.0) - assert not retry.in_extended_regime - assert retry.max_delay == _fast_retry_state().max_delay + assert not retry._extended + assert retry._max_delay == _fast_retry_state()._max_delay await proc.stop() @@ -387,10 +387,10 @@ async def test_our_own_interrupt_is_not_counted_as_a_server_close(): retry = _fast_retry_state() proc, store, ready, _ = _make_processor(actions, retry_state=retry) proc.start() - await _wait_until(lambda: retry.attempts >= 1) + await _wait_until(lambda: retry._attempts >= 1) await asyncio.sleep(0.1) - assert retry.attempts == 1 + assert retry._attempts == 1 await proc.stop() @@ -443,7 +443,7 @@ async def test_unexpected_http_error_moves_to_the_extended_regime(): retry = _fast_retry_state() proc, store, ready, _ = _make_processor(actions, retry_state=retry) proc.start() - await _wait_until(lambda: retry.in_extended_regime) + await _wait_until(lambda: retry._extended) await proc.stop() @@ -456,9 +456,9 @@ async def test_normal_http_error_stays_in_the_normal_regime(): retry = _fast_retry_state() proc, store, ready, _ = _make_processor(actions, retry_state=retry) proc.start() - await _wait_until(lambda: retry.attempts >= 1) + await _wait_until(lambda: retry._attempts >= 1) - assert not retry.in_extended_regime + assert not retry._extended await proc.stop() @@ -487,8 +487,8 @@ async def test_transport_failures_stay_in_the_normal_regime(error): # rather than let the test hang. assert await asyncio.wait_for(proc._handle_error(error), timeout=2.0) - assert not retry.in_extended_regime - assert retry.max_delay == STREAMING_MAX_DELAY + assert not retry._extended + assert retry._max_delay == DEFAULT_STREAMING_MAX_DELAY class _NoSleep: diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index 606fcd96..82ddc650 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -50,7 +50,7 @@ def fast_retry_state(delay=0.05): """A retry state with tiny delays, so a test does not have to wait out the real extended-regime delay of five minutes.""" return RetryState( - initial_delay=delay, + normal_initial_delay=delay, normal_ceiling=delay, extended_initial_delay=delay, extended_ceiling=delay, @@ -161,7 +161,7 @@ def test_unexpected_http_error_moves_to_the_extended_regime(ignore_mock): # The extended regime starts at five minutes, so only the first poll runs. assert not ready.wait(0.4) - assert retry.in_extended_regime + assert retry._extended assert mock_requester.request_count == 1 @@ -181,11 +181,11 @@ def test_the_first_success_after_an_outage_polls_at_the_cadence(): mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} processor._poll() assert retry.next_delay == 30 - assert retry.in_extended_regime, "one success restores the cadence but does not reset" + assert retry._extended, "one success restores the cadence but does not reset" processor._poll() assert retry.next_delay == 30 - assert not retry.in_extended_regime, "two successes in a row reset the state" + assert not retry._extended, "two successes in a row reset the state" @pytest.mark.parametrize( @@ -206,7 +206,7 @@ def test_transport_failures_poll_again_at_the_cadence(error): mock_requester.exception = error processor._poll() assert retry.next_delay == 30 - assert not retry.in_extended_regime + assert not retry._extended @mock.patch('ldclient.config.Config.poll_interval', new_callable=mock.PropertyMock, return_value=0.05) @@ -316,7 +316,7 @@ def test_an_extended_regime_wait_is_cut_short_by_stop(): while mock_requester.request_count < 1 and time.time() < deadline: time.sleep(0.01) assert mock_requester.request_count == 1 - assert retry.in_extended_regime, "the wait under test should be minutes long" + assert retry._extended, "the wait under test should be minutes long" worker = _polling_thread() assert worker is not None diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 39f69875..4c8f7e56 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -63,7 +63,7 @@ def fast_retry_state(delay=brief_delay): """A retry state with tiny delays, so a test does not have to wait out the real extended-regime delay of five minutes.""" return RetryState( - initial_delay=delay, + normal_initial_delay=delay, normal_ceiling=delay, extended_initial_delay=delay, extended_ceiling=delay, @@ -311,7 +311,7 @@ def test_unexpected_http_error_backs_off_a_long_way(status): assert not ready.wait(1) assert not sp.initialized() assert sp.is_alive() - assert sp._retry.in_extended_regime + assert sp._retry._extended server.should_have_requests(1) @@ -426,8 +426,8 @@ def test_server_close_backs_off_and_keeps_the_stream_running(): stream1.close() expect_update(store, FEATURES, flagv2) - assert retry.attempts >= 1 - assert not retry.in_extended_regime + assert retry._attempts >= 1 + assert not retry._extended interrupted = [s for s in spy.statuses if s.state == DataSourceState.INTERRUPTED] assert len(interrupted) >= 1 @@ -448,9 +448,9 @@ def test_server_close_uses_the_normal_delay_curve(): delays = [] for _ in range(8): sp._handle_error(StreamClosedError()) - delays.append(retry.max_delay) + delays.append(retry._max_delay) - assert not retry.in_extended_regime + assert not retry._extended assert delays == [30] * 8 @@ -490,7 +490,7 @@ def listener(s): # One failure for the bad JSON, not a second for the close it # caused. - assert retry.attempts == 1 + assert retry._attempts == 1 def _handle_errors_without_waiting(retry, errors): @@ -564,7 +564,7 @@ def test_several_messages_on_one_stream_do_not_extend_the_reset_window(): policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) retry = RetryState( - initial_delay=brief_delay, + normal_initial_delay=brief_delay, normal_ceiling=brief_delay, extended_initial_delay=brief_delay, extended_ceiling=brief_delay, @@ -600,7 +600,7 @@ def test_a_fresh_stream_starts_a_new_reset_window(): policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) retry = RetryState( - initial_delay=brief_delay, + normal_initial_delay=brief_delay, normal_ceiling=brief_delay, extended_initial_delay=brief_delay, extended_ceiling=brief_delay, @@ -640,8 +640,8 @@ def test_transport_failures_stay_in_the_normal_regime(error): sp._handle_error(error) - assert not retry.in_extended_regime - assert retry.max_delay == 30 + assert not retry._extended + assert retry._max_delay == 30 def test_http_proxy(monkeypatch): diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index ca1d67dc..9d4bc1b0 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -17,12 +17,12 @@ from ldclient.impl import retry from ldclient.impl.retry import ( - DEFAULT_INITIAL_RECONNECT_DELAY, DEFAULT_POLL_INTERVAL, + DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY, + DEFAULT_STREAMING_MAX_DELAY, EXTENDED_INITIAL_DELAY, EXTENDED_MAX_DELAY, POLLING_RESET_SUCCESSES, - STREAMING_MAX_DELAY, STREAMING_RESET_INTERVAL, AfterConsecutiveSuccesses, AfterHealthyFor, @@ -125,8 +125,8 @@ def test_streaming_falls_back_to_the_default(self, configured, caplog): state = for_streaming(configured) delay = failure_delay(state, NORMAL) - assert state.min_delay == DEFAULT_INITIAL_RECONNECT_DELAY - assert delay == DEFAULT_INITIAL_RECONNECT_DELAY + assert state._min_delay == DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY + assert delay == DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( "initial_reconnect_delay must be a positive, finite number of seconds; " @@ -144,7 +144,7 @@ def test_polling_falls_back_to_the_default(self, configured, caplog): state = for_polling(configured) delay = failure_delay(state, NORMAL) - assert state.operating_cadence == DEFAULT_POLL_INTERVAL + assert state._operating_cadence == DEFAULT_POLL_INTERVAL assert delay == DEFAULT_POLL_INTERVAL assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( @@ -158,7 +158,7 @@ def test_a_positive_streaming_delay_is_left_alone(self, configured, caplog): state = for_streaming(configured) - assert state.min_delay == configured + assert state._min_delay == configured assert failure_delay(state, NORMAL) == configured assert caplog.records == [] @@ -168,7 +168,7 @@ def test_a_positive_poll_interval_is_left_alone(self, configured, caplog): state = for_polling(configured) - assert state.operating_cadence == configured + assert state._operating_cadence == configured assert failure_delay(state, NORMAL) == configured assert caplog.records == [] @@ -220,7 +220,7 @@ def test_extended_regime_doubles_up_to_the_ceiling(self): def test_a_configured_initial_delay_raises_the_ceiling_with_it(self): # The ceiling must not fall below the initial delay. state = for_streaming(45) - assert state.max_delay == 45 + assert state._max_delay == 45 assert failure_delay(state, NORMAL) == 45 def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): @@ -228,13 +228,13 @@ def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): # back to the normal regime. state = for_streaming(1) state.record_failure(UNEXPECTED) - assert state.in_extended_regime - assert state.max_delay == EXTENDED_MAX_DELAY + assert state._extended + assert state._max_delay == EXTENDED_MAX_DELAY state.record_failure(NORMAL) - assert state.in_extended_regime - assert state.max_delay == EXTENDED_MAX_DELAY - assert state.min_delay == EXTENDED_INITIAL_DELAY + assert state._extended + assert state._max_delay == EXTENDED_MAX_DELAY + assert state._min_delay == EXTENDED_INITIAL_DELAY def test_a_second_unexpected_failure_keeps_counting_up(self): # Restarting the count on every unexpected failure would pin the delay @@ -246,8 +246,8 @@ def test_a_second_unexpected_failure_keeps_counting_up(self): def test_the_streaming_defaults_match_the_spec(self): state = for_streaming(1) - assert state.max_delay == STREAMING_MAX_DELAY - assert state.operating_cadence == 0 + assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY + assert state._operating_cadence == 0 assert STREAMING_RESET_INTERVAL == 60 @@ -273,14 +273,33 @@ def test_every_delay_stays_within_the_jitter_bounds(self): class TestWaitBetweenOperations: - def test_a_streaming_success_does_not_schedule_a_zero_wait(self): - """Streaming has no cadence, so a success falls back to the initial - delay. Zero would tell a scheduler to run again immediately.""" + def test_a_streaming_success_schedules_no_wait(self): + """Streaming's cadence is zero, and a success schedules the cadence.""" state = for_streaming(1) state.record_failure(NORMAL) state.record_success() - assert state.next_delay == 1 + assert state.next_delay == 0 + + def test_a_zero_cadence_puts_no_floor_under_a_retry(self): + """The floor is the cadence, so streaming's jitter is free to take a + retry below the configured delay.""" + with fixed_retry_jitter(FULL_JITTER): + state = for_streaming(1) + assert failure_delay(state, NORMAL) == pytest.approx(0.5) + + def test_only_streaming_can_yield_a_zero_wait(self): + """Polling is the only data source that reads next_delay as a + DelaySource, so a zero there would busy-loop its scheduler. Its + cadence floor rules that out for every outcome.""" + state = for_polling(30) + assert state.next_delay == 30 + with real_jitter(): + for kind in (NORMAL, UNEXPECTED, NORMAL, UNEXPECTED): + state.record_failure(kind) + assert state.next_delay >= 30 + state.record_success() + assert state.next_delay == 30 def test_a_polling_success_schedules_the_cadence(self): state = for_polling(30) @@ -307,12 +326,12 @@ def test_a_minute_of_healthy_operation_resets_the_state(self): state.record_failure(NORMAL) state.record_success() - assert state.in_extended_regime, "the window has not elapsed yet" + assert state._extended, "the window has not elapsed yet" clock.advance(STREAMING_RESET_INTERVAL) state.record_success() - assert not state.in_extended_regime - assert state.max_delay == STREAMING_MAX_DELAY + assert not state._extended + assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY assert failure_delay(state, NORMAL) == 1 def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): @@ -350,9 +369,9 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se delays.append(failure_delay(state, NORMAL)) clock.advance(1) - assert not state.in_extended_regime - assert max(delays) == STREAMING_MAX_DELAY - assert state.max_delay == STREAMING_MAX_DELAY + assert not state._extended + assert max(delays) == DEFAULT_STREAMING_MAX_DELAY + assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY class TestPollingCadence: @@ -378,8 +397,8 @@ def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): # initial delay, not by for_polling clamping the ceiling itself. state = for_polling(2 * 60 * 60) assert failure_delay(state, UNEXPECTED) == 2 * 60 * 60 - assert state.max_delay == 2 * 60 * 60 - assert state.min_delay == 2 * 60 * 60 + assert state._max_delay == 2 * 60 * 60 + assert state._min_delay == 2 * 60 * 60 def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): # A backoff wait applies to a retry, not to every operation. @@ -392,17 +411,17 @@ def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): state.record_success() assert state.next_delay == 30 - assert state.in_extended_regime, "one success does not reset the state" + assert state._extended, "one success does not reset the state" def test_two_successes_in_a_row_reset_the_state(self): state = for_polling(30) state.record_failure(UNEXPECTED) state.record_success() - assert state.in_extended_regime + assert state._extended state.record_success() - assert not state.in_extended_regime + assert not state._extended assert state.next_delay == 30 assert failure_delay(state, NORMAL) == 30 @@ -412,16 +431,16 @@ def test_a_failure_between_two_successes_clears_the_first(self): state.record_success() state.record_failure(NORMAL) state.record_success() - assert state.in_extended_regime + assert state._extended state.record_success() - assert not state.in_extended_regime + assert not state._extended def test_the_polling_defaults_match_the_spec(self): state = for_polling(30) - assert state.operating_cadence == 30 - assert state.min_delay == 30 - assert state.max_delay == 30 + assert state._operating_cadence == 30 + assert state._min_delay == 30 + assert state._max_delay == 30 assert POLLING_RESET_SUCCESSES == 2 @@ -430,7 +449,7 @@ def test_attempts_counts_every_failure(self): state = for_streaming(1) for _ in range(5): state.record_failure(NORMAL) - assert state.attempts == 5 + assert state._attempts == 5 def test_a_reset_starts_the_attempt_count_over(self): # A reset clears both counters, so the next failure is attempt 1. @@ -438,7 +457,7 @@ def test_a_reset_starts_the_attempt_count_over(self): state = for_streaming(1) state.record_failure(NORMAL) state.record_failure(NORMAL) - assert state.attempts == 2 + assert state._attempts == 2 state.record_success() clock.advance(STREAMING_RESET_INTERVAL) @@ -447,7 +466,7 @@ def test_a_reset_starts_the_attempt_count_over(self): # The delay drops back to the first-retry value, and the count # starts over with it. assert failure_delay(state, NORMAL) == 1 - assert state.attempts == 1 + assert state._attempts == 1 class TestResetPolicies: @@ -457,12 +476,12 @@ def test_healthy_for_tracks_the_start_of_the_stretch(self): assert not policy.is_satisfied() policy.note_healthy() - started = policy.healthy_since + started = policy._healthy_since # A later signal must not push the start of the stretch out. clock.advance(40) policy.note_healthy() - assert policy.healthy_since == started + assert policy._healthy_since == started clock.advance(20) assert policy.is_satisfied() @@ -473,13 +492,13 @@ def test_many_healthy_signals_do_not_move_the_window(self): with frozen_clock() as clock: policy = AfterHealthyFor(60) policy.note_healthy() - first = policy.healthy_since + first = policy._healthy_since for _ in range(59): clock.advance(1) policy.note_healthy() - assert policy.healthy_since == first + assert policy._healthy_since == first assert not policy.is_satisfied() # The threshold lands 60s after the first signal, not the last. @@ -492,7 +511,7 @@ def test_healthy_for_is_cleared_by_a_failure(self): policy = AfterHealthyFor(60) policy.note_healthy() policy.note_failure() - assert policy.healthy_since is None + assert policy._healthy_since is None clock.advance(900) assert not policy.is_satisfied() @@ -508,14 +527,14 @@ def test_consecutive_successes_is_cleared_by_a_failure(self): policy = AfterConsecutiveSuccesses(2) policy.note_healthy() policy.note_failure() - assert policy.successes == 0 + assert policy._successes == 0 assert not policy.is_satisfied() class TestLongOutage: def test_a_long_outage_cannot_overflow_the_delay(self): state = RetryState( - initial_delay=1, + normal_initial_delay=1, normal_ceiling=30, extended_initial_delay=EXTENDED_INITIAL_DELAY, extended_ceiling=EXTENDED_MAX_DELAY, diff --git a/ldclient/testing/test_util.py b/ldclient/testing/test_util.py index e2169cc4..e52b1c77 100644 --- a/ldclient/testing/test_util.py +++ b/ldclient/testing/test_util.py @@ -58,7 +58,7 @@ def record_healthy_windows(policy) -> list: def wrapper(): note() - windows.append(policy.healthy_since) + windows.append(policy._healthy_since) policy.note_healthy = wrapper # type: ignore[method-assign] return windows From cd3dd63ee83a227d40ddb03927c8f01ea9ca24cb Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 16 Sep 2026 16:30:30 -0500 Subject: [PATCH 09/19] fix: Validate the configured data source intervals in Config Config accepted any initial_reconnect_delay and clamped poll_interval with max(), which let NaN and inf through because every comparison against NaN is false. A NaN interval reaches Event.wait() and the delay arithmetic downstream. Both are now validated, warn, and fall back to the documented default; the poll interval keeps its 30s minimum on top. Move the check to impl/util.py as validate_positive_finite, next to the validators Config already imports, so config and retry share it without either importing the other. The retry factories keep their own call: a RetryState can be built without going through Config. Also name the delay bounds after the spec's ceiling vocabulary. --- ldclient/async_config.py | 16 +++-- ldclient/config.py | 19 ++++-- ldclient/impl/retry.py | 63 ++++++++----------- ldclient/impl/util.py | 20 ++++++ .../impl/datasource/test_async_polling.py | 4 +- .../impl/datasource/test_async_streaming.py | 18 +++--- .../impl/datasource/test_polling_processor.py | 4 +- .../testing/impl/datasource/test_streaming.py | 12 ++-- ldclient/testing/impl/test_retry.py | 30 ++++----- 9 files changed, 108 insertions(+), 78 deletions(-) diff --git a/ldclient/async_config.py b/ldclient/async_config.py index b46b362e..47e7c283 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -15,6 +15,8 @@ from ldclient.config import ( DEFAULT_BASE_URI, DEFAULT_EVENTS_URI, + DEFAULT_INITIAL_RECONNECT_DELAY, + DEFAULT_POLL_INTERVAL, DEFAULT_STREAM_URI, GET_LATEST_FEATURES_PATH, STREAM_FLAGS_PATH, @@ -28,6 +30,7 @@ from ldclient.impl.util import ( log, validate_application_info, + validate_positive_finite, validate_sdk_key_format ) from ldclient.interfaces import ( @@ -149,11 +152,11 @@ def __init__( flush_interval: float = 5, stream_uri: str = DEFAULT_STREAM_URI, stream: bool = True, - initial_reconnect_delay: float = 1, + initial_reconnect_delay: float = DEFAULT_INITIAL_RECONNECT_DELAY, defaults: dict = {}, send_events: Optional[bool] = None, update_processor_class: Optional[Callable[['AsyncConfig', AsyncFeatureStore, AsyncEvent], AsyncUpdateProcessor]] = None, - poll_interval: float = 30, + poll_interval: float = DEFAULT_POLL_INTERVAL, use_ldd: bool = False, feature_store: Optional[AsyncFeatureStore] = None, feature_requester_class=None, @@ -255,8 +258,13 @@ def __init__( self.__stream_uri = stream_uri.rstrip('/') self.__update_processor_class = update_processor_class self.__stream = stream - self.__initial_reconnect_delay = initial_reconnect_delay - self.__poll_interval = max(poll_interval, 30.0) + self.__initial_reconnect_delay = validate_positive_finite( + initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log + ) + self.__poll_interval = max( + validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log), + DEFAULT_POLL_INTERVAL, + ) self.__use_ldd = use_ldd self.__feature_store = AsyncInMemoryFeatureStore() if not feature_store else feature_store self.__event_processor_class = event_processor_class diff --git a/ldclient/config.py b/ldclient/config.py index d7ac76a2..c8d7d861 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -15,6 +15,7 @@ from ldclient.impl.util import ( log, validate_application_info, + validate_positive_finite, validate_sdk_key_format ) from ldclient.interfaces import ( @@ -36,6 +37,11 @@ DEFAULT_EVENTS_URI = 'https://events.launchdarkly.com' DEFAULT_STREAM_URI = 'https://stream.launchdarkly.com' +# Defaults, in seconds, for the two configurable data source intervals. The +# poll interval is also its own minimum. +DEFAULT_INITIAL_RECONNECT_DELAY = 1 +DEFAULT_POLL_INTERVAL = 30 + class BigSegmentsConfig: """Configuration options related to Big Segments. @@ -295,11 +301,11 @@ def __init__( flush_interval: float = 5, stream_uri: str = DEFAULT_STREAM_URI, stream: bool = True, - initial_reconnect_delay: float = 1, + initial_reconnect_delay: float = DEFAULT_INITIAL_RECONNECT_DELAY, defaults: dict = {}, send_events: Optional[bool] = None, update_processor_class: Optional[Callable[['Config', FeatureStore, Event], UpdateProcessor]] = None, - poll_interval: float = 30, + poll_interval: float = DEFAULT_POLL_INTERVAL, use_ldd: bool = False, feature_store: Optional[FeatureStore] = None, feature_requester_class=None, @@ -401,8 +407,13 @@ def __init__( self.__stream_uri = stream_uri.rstrip('/') self.__update_processor_class = update_processor_class self.__stream = stream - self.__initial_reconnect_delay = initial_reconnect_delay - self.__poll_interval = max(poll_interval, 30.0) + self.__initial_reconnect_delay = validate_positive_finite( + initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log + ) + self.__poll_interval = max( + validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log), + DEFAULT_POLL_INTERVAL, + ) self.__use_ldd = use_ldd self.__feature_store = InMemoryFeatureStore() if not feature_store else feature_store self.__event_processor_class = event_processor_class diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index ef8095fa..e87038ef 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -18,24 +18,24 @@ # currently excluded from documentation - see docs/README.md -import math import random import time from enum import Enum from typing import Optional, Protocol -from ldclient.impl.util import log +from ldclient.config import ( + DEFAULT_INITIAL_RECONNECT_DELAY, + DEFAULT_POLL_INTERVAL +) +from ldclient.impl.util import log, validate_positive_finite -# The documented defaults, in seconds. Each stands in for a configured value -# that is not a positive, finite number. -DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY = 1 -DEFAULT_STREAMING_MAX_DELAY = 30 -DEFAULT_POLL_INTERVAL = 30 +# The delay ceiling of the normal regime for streaming, in seconds. +NORMAL_STREAMING_CEILING_DELAY = 30 # The delay bounds of the extended regime, in seconds. A component enters the # extended regime after an unexpected failure. EXTENDED_INITIAL_DELAY = 5 * 60 -EXTENDED_MAX_DELAY = 60 * 60 +EXTENDED_CEILING_DELAY = 60 * 60 # How long streaming must operate without a failure before its retry state # resets, in seconds. @@ -152,35 +152,37 @@ class RetryState: def __init__( self, normal_initial_delay: float, - normal_ceiling: float, + normal_ceiling_delay: float, extended_initial_delay: float, - extended_ceiling: float, + extended_ceiling_delay: float, reset_policy: ResetPolicy, operating_cadence: float = 0, ): """ :param normal_initial_delay: the delay before the first retry in the normal regime, in seconds - :param normal_ceiling: the longest normal-regime delay, in seconds + :param normal_ceiling_delay: the longest normal-regime delay, in + seconds :param extended_initial_delay: the delay before the first retry in the extended regime, in seconds - :param extended_ceiling: the longest extended-regime delay, in seconds + :param extended_ceiling_delay: the longest extended-regime delay, in + seconds :param reset_policy: decides when the retry state resets :param operating_cadence: the wait between healthy operations, in seconds; no wait is ever shorter than this. Zero for a component that operates continuously. """ self._normal_initial_delay = normal_initial_delay - self._normal_ceiling = normal_ceiling + self._normal_ceiling_delay = normal_ceiling_delay self._extended_initial_delay = extended_initial_delay - self._extended_ceiling = extended_ceiling + self._extended_ceiling_delay = extended_ceiling_delay self._reset_policy = reset_policy self._operating_cadence = operating_cadence self._n = 0 self._extended = False self._min_delay = self._normal_initial_delay - self._max_delay = max(self._normal_ceiling, self._normal_initial_delay) + self._max_delay = max(self._normal_ceiling_delay, self._normal_initial_delay) self._attempts = 0 # Read before any outcome is recorded, this is the ordinary interval. self._next_delay = self._operating_cadence @@ -212,7 +214,7 @@ def record_failure(self, kind: FailureKind) -> None: # extended initial delay. self._extended = True self._min_delay = self._extended_initial_delay - self._max_delay = max(self._extended_ceiling, self._min_delay) + self._max_delay = max(self._extended_ceiling_delay, self._min_delay) self._n = 1 else: self._n += 1 @@ -245,20 +247,7 @@ def _reset_if_due(self) -> None: self._attempts = 0 self._extended = False self._min_delay = self._normal_initial_delay - self._max_delay = max(self._normal_ceiling, self._normal_initial_delay) - - -def _positive_finite(value: float, default: float, name: str) -> float: - """Returns ``value`` if it is a positive, finite number of seconds, and the - default otherwise. A non-finite value would make the jitter arithmetic - produce a NaN delay, and a non-positive one would retry with no wait.""" - if value > 0 and math.isfinite(value): - return value - log.warning( - "%s must be a positive, finite number of seconds; using the default of %ss" - % (name, default) - ) - return default + self._max_delay = max(self._normal_ceiling_delay, self._normal_initial_delay) def for_streaming(initial_reconnect_delay: float) -> RetryState: @@ -276,14 +265,14 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: The extended regime never starts below the configured delay. """ - initial_reconnect_delay = _positive_finite( - initial_reconnect_delay, DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay' + initial_reconnect_delay = validate_positive_finite( + initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log ) return RetryState( normal_initial_delay=initial_reconnect_delay, - normal_ceiling=DEFAULT_STREAMING_MAX_DELAY, + normal_ceiling_delay=NORMAL_STREAMING_CEILING_DELAY, extended_initial_delay=max(EXTENDED_INITIAL_DELAY, initial_reconnect_delay), - extended_ceiling=EXTENDED_MAX_DELAY, + extended_ceiling_delay=EXTENDED_CEILING_DELAY, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), operating_cadence=0 ) @@ -302,12 +291,12 @@ def for_polling(poll_interval: float) -> RetryState: ``Config`` clamps the poll interval, but the documented default stands in for anything that reaches here and is not a positive, finite number. """ - poll_interval = _positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval') + poll_interval = validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log) return RetryState( normal_initial_delay=poll_interval, - normal_ceiling=poll_interval, + normal_ceiling_delay=poll_interval, extended_initial_delay=max(EXTENDED_INITIAL_DELAY, poll_interval), - extended_ceiling=EXTENDED_MAX_DELAY, + extended_ceiling_delay=EXTENDED_CEILING_DELAY, reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), operating_cadence=poll_interval, ) diff --git a/ldclient/impl/util.py b/ldclient/impl/util.py index 63e90624..fcc9da03 100644 --- a/ldclient/impl/util.py +++ b/ldclient/impl/util.py @@ -1,4 +1,5 @@ import logging +import math import re import sys import time @@ -60,6 +61,25 @@ def validate_application_value(value: Any, name: str, logger: logging.Logger) -> return value +def validate_positive_finite(value: float, default: float, name: str, logger: logging.Logger) -> float: + """ + Validates that a number of seconds is positive and finite. + + A non-finite value makes later arithmetic produce NaN, and a non-positive + one makes a wait no wait at all. + + :param value: the number of seconds to validate + :param default: the value to use when ``value`` is not usable + :param name: the option name, for the warning message + :param logger: the logger to use for logging warnings + :return: ``value``, or ``default`` if ``value`` is not positive and finite + """ + if value > 0 and math.isfinite(value): + return value + logger.warning("%s must be a positive, finite number of seconds; using the default of %ss" % (name, default)) + return default + + def validate_sdk_key_format(sdk_key: str, logger: logging.Logger) -> str: """ Validates that an SDK key does not contain invalid characters and is not too long for our systems. diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index 3406d66a..990c292e 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -63,9 +63,9 @@ def fast_retry_state(delay=0.001): real extended-regime delay of five minutes.""" return RetryState( normal_initial_delay=delay, - normal_ceiling=delay, + normal_ceiling_delay=delay, extended_initial_delay=delay, - extended_ceiling=delay, + extended_ceiling_delay=delay, reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), operating_cadence=delay, ) diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index 73484229..3ddea6a4 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -22,9 +22,9 @@ ) from ldclient.impl.model import ModelEntity from ldclient.impl.retry import ( - DEFAULT_STREAMING_MAX_DELAY, + EXTENDED_CEILING_DELAY, EXTENDED_INITIAL_DELAY, - EXTENDED_MAX_DELAY, + NORMAL_STREAMING_CEILING_DELAY, STREAMING_RESET_INTERVAL, AfterHealthyFor, RetryState, @@ -98,9 +98,9 @@ def _retry_state_with(policy: AfterHealthyFor) -> RetryState: test can watch the window.""" return RetryState( normal_initial_delay=0.001, - normal_ceiling=0.001, + normal_ceiling_delay=0.001, extended_initial_delay=0.001, - extended_ceiling=0.001, + extended_ceiling_delay=0.001, reset_policy=policy, ) @@ -110,9 +110,9 @@ def _fast_retry_state(delay: float = 0.001) -> RetryState: real extended-regime delay of five minutes.""" return RetryState( normal_initial_delay=delay, - normal_ceiling=delay, + normal_ceiling_delay=delay, extended_initial_delay=delay, - extended_ceiling=delay, + extended_ceiling_delay=delay, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), ) @@ -137,9 +137,9 @@ def _zero_delay_retry_state() -> RetryState: real, so a misclassification still shows up in ``max_delay``.""" return RetryState( normal_initial_delay=0, - normal_ceiling=DEFAULT_STREAMING_MAX_DELAY, + normal_ceiling_delay=NORMAL_STREAMING_CEILING_DELAY, extended_initial_delay=EXTENDED_INITIAL_DELAY, - extended_ceiling=EXTENDED_MAX_DELAY, + extended_ceiling_delay=EXTENDED_CEILING_DELAY, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), ) @@ -488,7 +488,7 @@ async def test_transport_failures_stay_in_the_normal_regime(error): assert await asyncio.wait_for(proc._handle_error(error), timeout=2.0) assert not retry._extended - assert retry._max_delay == DEFAULT_STREAMING_MAX_DELAY + assert retry._max_delay == NORMAL_STREAMING_CEILING_DELAY class _NoSleep: diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index 82ddc650..bc8d9337 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -51,9 +51,9 @@ def fast_retry_state(delay=0.05): real extended-regime delay of five minutes.""" return RetryState( normal_initial_delay=delay, - normal_ceiling=delay, + normal_ceiling_delay=delay, extended_initial_delay=delay, - extended_ceiling=delay, + extended_ceiling_delay=delay, reset_policy=AfterConsecutiveSuccesses(POLLING_RESET_SUCCESSES), operating_cadence=delay, ) diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 4c8f7e56..88570360 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -64,9 +64,9 @@ def fast_retry_state(delay=brief_delay): real extended-regime delay of five minutes.""" return RetryState( normal_initial_delay=delay, - normal_ceiling=delay, + normal_ceiling_delay=delay, extended_initial_delay=delay, - extended_ceiling=delay, + extended_ceiling_delay=delay, reset_policy=AfterHealthyFor(STREAMING_RESET_INTERVAL), ) @@ -565,9 +565,9 @@ def test_several_messages_on_one_stream_do_not_extend_the_reset_window(): policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) retry = RetryState( normal_initial_delay=brief_delay, - normal_ceiling=brief_delay, + normal_ceiling_delay=brief_delay, extended_initial_delay=brief_delay, - extended_ceiling=brief_delay, + extended_ceiling_delay=brief_delay, reset_policy=policy, ) # The clock moves on every read, so a window that had been @@ -601,9 +601,9 @@ def test_a_fresh_stream_starts_a_new_reset_window(): policy = AfterHealthyFor(STREAMING_RESET_INTERVAL) retry = RetryState( normal_initial_delay=brief_delay, - normal_ceiling=brief_delay, + normal_ceiling_delay=brief_delay, extended_initial_delay=brief_delay, - extended_ceiling=brief_delay, + extended_ceiling_delay=brief_delay, reset_policy=policy, ) windows = record_healthy_windows(policy) diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 9d4bc1b0..8340f522 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -15,13 +15,15 @@ import pytest +from ldclient.config import ( + DEFAULT_INITIAL_RECONNECT_DELAY, + DEFAULT_POLL_INTERVAL +) from ldclient.impl import retry from ldclient.impl.retry import ( - DEFAULT_POLL_INTERVAL, - DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY, - DEFAULT_STREAMING_MAX_DELAY, + EXTENDED_CEILING_DELAY, EXTENDED_INITIAL_DELAY, - EXTENDED_MAX_DELAY, + NORMAL_STREAMING_CEILING_DELAY, POLLING_RESET_SUCCESSES, STREAMING_RESET_INTERVAL, AfterConsecutiveSuccesses, @@ -125,8 +127,8 @@ def test_streaming_falls_back_to_the_default(self, configured, caplog): state = for_streaming(configured) delay = failure_delay(state, NORMAL) - assert state._min_delay == DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY - assert delay == DEFAULT_STREAMING_INITIAL_RECONNECT_DELAY + assert state._min_delay == DEFAULT_INITIAL_RECONNECT_DELAY + assert delay == DEFAULT_INITIAL_RECONNECT_DELAY assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( "initial_reconnect_delay must be a positive, finite number of seconds; " @@ -229,11 +231,11 @@ def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): state = for_streaming(1) state.record_failure(UNEXPECTED) assert state._extended - assert state._max_delay == EXTENDED_MAX_DELAY + assert state._max_delay == EXTENDED_CEILING_DELAY state.record_failure(NORMAL) assert state._extended - assert state._max_delay == EXTENDED_MAX_DELAY + assert state._max_delay == EXTENDED_CEILING_DELAY assert state._min_delay == EXTENDED_INITIAL_DELAY def test_a_second_unexpected_failure_keeps_counting_up(self): @@ -246,7 +248,7 @@ def test_a_second_unexpected_failure_keeps_counting_up(self): def test_the_streaming_defaults_match_the_spec(self): state = for_streaming(1) - assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY + assert state._max_delay == NORMAL_STREAMING_CEILING_DELAY assert state._operating_cadence == 0 assert STREAMING_RESET_INTERVAL == 60 @@ -331,7 +333,7 @@ def test_a_minute_of_healthy_operation_resets_the_state(self): clock.advance(STREAMING_RESET_INTERVAL) state.record_success() assert not state._extended - assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY + assert state._max_delay == NORMAL_STREAMING_CEILING_DELAY assert failure_delay(state, NORMAL) == 1 def test_a_reset_also_happens_on_the_failure_that_ends_a_healthy_stretch(self): @@ -370,8 +372,8 @@ def test_a_fast_flapping_connection_does_not_ratchet_into_the_extended_regime(se clock.advance(1) assert not state._extended - assert max(delays) == DEFAULT_STREAMING_MAX_DELAY - assert state._max_delay == DEFAULT_STREAMING_MAX_DELAY + assert max(delays) == NORMAL_STREAMING_CEILING_DELAY + assert state._max_delay == NORMAL_STREAMING_CEILING_DELAY class TestPollingCadence: @@ -535,9 +537,9 @@ class TestLongOutage: def test_a_long_outage_cannot_overflow_the_delay(self): state = RetryState( normal_initial_delay=1, - normal_ceiling=30, + normal_ceiling_delay=30, extended_initial_delay=EXTENDED_INITIAL_DELAY, - extended_ceiling=EXTENDED_MAX_DELAY, + extended_ceiling_delay=EXTENDED_CEILING_DELAY, reset_policy=AfterHealthyFor(60), ) for _ in range(5000): From 1d0cf93e2abf59533b2d6956b4295d83d68f2bc7 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 16 Sep 2026 16:45:04 -0500 Subject: [PATCH 10/19] refactor: Give the spec's attempts name to the counter that drives the delay The exponent driver was _n and a second counter held the name attempts, which is what Requirement 1.4.1 calls the exponent driver. A reviewer reading self.attempts against the spec was reading the wrong field. The second counter is gone. It had no reader outside tests, not even a logger, and every test that used it recorded only normal failures -- where the two counters are equal by construction. --- ldclient/impl/retry.py | 17 +++++++---------- ldclient/testing/impl/test_retry.py | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index e87038ef..74089487 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -140,9 +140,9 @@ class RetryState: Tracks how long a data source should wait before its next attempt. A failure moves the state on and decides the next wait, which - :attr:`next_delay` reports. The delay for attempt ``n`` is - ``min(min_delay * 2 ** (n - 1), max_delay)``, less a random jitter of up to - half of it, and never less than the operating cadence. + :attr:`next_delay` reports. The delay is + ``min(min_delay * 2 ** (attempts - 1), max_delay)``, less a random jitter + of up to half of it, and never less than the operating cadence. An unexpected failure moves the state to the extended regime, which raises both delay bounds. The bounds stay raised until the reset condition is met, @@ -179,11 +179,10 @@ def __init__( self._reset_policy = reset_policy self._operating_cadence = operating_cadence - self._n = 0 + self._attempts = 0 self._extended = False self._min_delay = self._normal_initial_delay self._max_delay = max(self._normal_ceiling_delay, self._normal_initial_delay) - self._attempts = 0 # Read before any outcome is recorded, this is the ordinary interval. self._next_delay = self._operating_cadence @@ -204,7 +203,6 @@ def record_failure(self, kind: FailureKind) -> None: """ # Only a time-based policy needs this: nothing runs while a stream is healthy. self._reset_if_due() - self._attempts += 1 self._reset_policy.note_failure() if kind is FailureKind.UNEXPECTED and not self._extended: @@ -215,11 +213,11 @@ def record_failure(self, kind: FailureKind) -> None: self._extended = True self._min_delay = self._extended_initial_delay self._max_delay = max(self._extended_ceiling_delay, self._min_delay) - self._n = 1 + self._attempts = 1 else: - self._n += 1 + self._attempts += 1 - exponent = min(max(self._n - 1, 0), _MAX_BACKOFF_EXPONENT) + exponent = min(max(self._attempts - 1, 0), _MAX_BACKOFF_EXPONENT) delay = min(self._min_delay * (2**exponent), self._max_delay) jitter = random.random() * delay / 2 @@ -243,7 +241,6 @@ def _reset_if_due(self) -> None: the delay bounds to the normal regime.""" if not self._reset_policy.is_satisfied(): return - self._n = 0 self._attempts = 0 self._extended = False self._min_delay = self._normal_initial_delay diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 8340f522..bc409419 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -454,7 +454,7 @@ def test_attempts_counts_every_failure(self): assert state._attempts == 5 def test_a_reset_starts_the_attempt_count_over(self): - # A reset clears both counters, so the next failure is attempt 1. + # A reset clears the counter, so the next failure is attempt 1. with frozen_clock() as clock: state = for_streaming(1) state.record_failure(NORMAL) From 1b50827f98104b37a18cb3f72705004222d3dfb8 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 17 Sep 2026 10:32:47 -0500 Subject: [PATCH 11/19] docs: Correct what Config checks in the retry factory docstrings Config now validates both intervals, so the factories' guards are no longer the only check. They still matter -- a RetryState can be built without going through Config -- but the docstrings described the old state, where Config ignored initial_reconnect_delay and only clamped poll_interval. --- ldclient/impl/retry.py | 8 ++++---- ldclient/testing/impl/test_retry.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 74089487..38475ac6 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -257,8 +257,8 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: healthy operation after establishing a successful connection with no failures during the ``STREAMING_RESET_INTERVAL``. - ``Config`` does not check the configured delay, so the documented default - stands in for anything that is not a positive, finite number. + ``Config`` validates the configured delay, so this guard only catches a + state built without it. The extended regime never starts below the configured delay. """ @@ -285,8 +285,8 @@ def for_polling(poll_interval: float) -> RetryState: schedule. Polling is healthy on any successful poll, and resets after two in a row. - ``Config`` clamps the poll interval, but the documented default stands in - for anything that reaches here and is not a positive, finite number. + ``Config`` validates and clamps the poll interval, so this guard only + catches a state built without it. """ poll_interval = validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log) return RetryState( diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index bc409419..65114d65 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -112,9 +112,9 @@ def test_non_error_statuses_are_normal(self, status): class TestFactoryInputGuards: - """``Config`` does not check ``initial_reconnect_delay`` at all, and only - clamps ``poll_interval``. A non-positive value would retry with no wait; a - non-finite one makes the jitter arithmetic produce NaN.""" + """A ``RetryState`` can be built without going through ``Config``, so the + factories guard their own input. A non-positive value would retry with no + wait; a non-finite one makes the jitter arithmetic produce NaN.""" @pytest.mark.parametrize( "configured", From e1f29df170a706595e18abce5c7ad415c37f458f Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 17 Sep 2026 15:41:14 -0500 Subject: [PATCH 12/19] fix: Validate data source intervals in the retry state and bound the wait --- ldclient/async_config.py | 10 +-- ldclient/config.py | 10 +-- ldclient/impl/aio/transport.py | 3 +- ldclient/impl/datasource/async_streaming.py | 7 +-- ldclient/impl/datasource/streaming.py | 18 ++---- ldclient/impl/repeating_task.py | 6 +- ldclient/impl/retry.py | 51 ++++++++------- ldclient/impl/util.py | 34 ++-------- ldclient/interfaces.py | 3 +- .../impl/datasource/test_async_polling.py | 32 +++++++++- .../impl/datasource/test_async_streaming.py | 39 +++++++++++- .../impl/datasource/test_polling_processor.py | 42 +++++++++++-- .../testing/impl/datasource/test_streaming.py | 47 +++++++++++--- ldclient/testing/impl/test_retry.py | 62 +++++++++++++------ ldclient/testing/test_config.py | 42 ++++++++++--- 15 files changed, 271 insertions(+), 135 deletions(-) diff --git a/ldclient/async_config.py b/ldclient/async_config.py index 47e7c283..0fc97dc1 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -30,7 +30,6 @@ from ldclient.impl.util import ( log, validate_application_info, - validate_positive_finite, validate_sdk_key_format ) from ldclient.interfaces import ( @@ -258,13 +257,8 @@ def __init__( self.__stream_uri = stream_uri.rstrip('/') self.__update_processor_class = update_processor_class self.__stream = stream - self.__initial_reconnect_delay = validate_positive_finite( - initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log - ) - self.__poll_interval = max( - validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log), - DEFAULT_POLL_INTERVAL, - ) + self.__initial_reconnect_delay = initial_reconnect_delay + self.__poll_interval = max(poll_interval, DEFAULT_POLL_INTERVAL) self.__use_ldd = use_ldd self.__feature_store = AsyncInMemoryFeatureStore() if not feature_store else feature_store self.__event_processor_class = event_processor_class diff --git a/ldclient/config.py b/ldclient/config.py index c8d7d861..2e9c8e96 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -15,7 +15,6 @@ from ldclient.impl.util import ( log, validate_application_info, - validate_positive_finite, validate_sdk_key_format ) from ldclient.interfaces import ( @@ -407,13 +406,8 @@ def __init__( self.__stream_uri = stream_uri.rstrip('/') self.__update_processor_class = update_processor_class self.__stream = stream - self.__initial_reconnect_delay = validate_positive_finite( - initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log - ) - self.__poll_interval = max( - validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log), - DEFAULT_POLL_INTERVAL, - ) + self.__initial_reconnect_delay = initial_reconnect_delay + self.__poll_interval = max(poll_interval, DEFAULT_POLL_INTERVAL) self.__use_ldd = use_ldd self.__feature_store = InMemoryFeatureStore() if not feature_store else feature_store self.__event_processor_class = event_processor_class diff --git a/ldclient/impl/aio/transport.py b/ldclient/impl/aio/transport.py index 6226c5d2..4da6e0c2 100644 --- a/ldclient/impl/aio/transport.py +++ b/ldclient/impl/aio/transport.py @@ -140,8 +140,7 @@ def create(self, url: str, initial_retry_delay: float, query_params=None, sdk_ma if proxy: aiohttp_request_options["proxy"] = proxy if sdk_managed_retry: - # A zero base delay plus the no-op base strategy holds - # next_retry_delay at zero, so the SSE client never sleeps. + # The SSE client's retry is disabled; the SDK owns the delay. retry_options: dict = { "initial_retry_delay": 0, "retry_delay_strategy": RetryDelayStrategy(), diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index be9bd7f9..06061259 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -126,11 +126,8 @@ async def _run(self): log.info("AsyncStreamingUpdateProcessor initialized ok.") self._ready.set() elif isinstance(action, Fault): - # A Fault with no error means the connection closed cleanly. - # If we asked for that close, we have already recorded the - # failure behind it and must not record it twice. Otherwise - # the server closed a connection it normally leaves open, - # which is a connection failure the SDK backs off from. + # A Fault with no error is a clean close. An interrupt the + # SDK asked for is not a failure. if action.error is None: if self._interrupted_by_sdk: self._interrupted_by_sdk = False diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index ab060e43..39f2c349 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -1,5 +1,6 @@ import json import time +from threading import TIMEOUT_MAX from threading import Event as ThreadEvent from threading import Thread from typing import Callable, Optional @@ -107,11 +108,8 @@ def run(self): log.info("StreamingUpdateProcessor initialized ok.") self._ready.set() elif isinstance(action, Fault): - # A Fault with no error means the connection closed cleanly. If - # we asked for that close, we have already recorded the failure - # behind it and must not record it twice. Otherwise the server - # closed a connection it normally leaves open, which is a - # connection failure the SDK backs off from. + # A Fault with no error is a clean close. An interrupt the SDK + # asked for is not a failure. if action.error is None: if self._interrupted_by_sdk: self._interrupted_by_sdk = False @@ -139,13 +137,7 @@ def _create_sse_client(self) -> SSEClient: url=self._uri, headers=http_factory.base_headers, pool=stream_http_factory.create_pool_manager(1, self._uri), urllib3_request_options={"timeout": stream_http_factory.timeout} ), error_strategy=ErrorStrategy.always_continue(), # we'll make error-handling decisions when we see a Fault - # The SDK owns the retry delay, so the SSE client must never wait. - # A zero base delay plus the no-op base strategy holds - # next_retry_delay at zero, which is what these three arguments - # are for. The SSE client hands us the Fault before it would - # sleep, so we classify the failure and wait ourselves in - # _handle_error. Our wait is interruptible, which matters because - # the extended regime can ask for an hour. + # The SSE client's retry is disabled; the SDK owns the delay. initial_retry_delay=0, retry_delay_strategy=RetryDelayStrategy(), retry_delay_reset_threshold=0, @@ -253,7 +245,7 @@ def _handle_error(self, error: Exception) -> bool: self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) self._connection_attempt_start_time = time.time() + delay - return not self._stop_event.wait(delay) + return not self._stop_event.wait(min(delay, TIMEOUT_MAX)) # magic methods for "with" statement (used in testing) def __enter__(self): diff --git a/ldclient/impl/repeating_task.py b/ldclient/impl/repeating_task.py index d2e4abe1..c8c1fd62 100644 --- a/ldclient/impl/repeating_task.py +++ b/ldclient/impl/repeating_task.py @@ -1,4 +1,4 @@ -from threading import Event, Thread +from threading import TIMEOUT_MAX, Event, Thread from typing import Any, Callable from ldclient.impl.delay import DelaySource, FixedDelay @@ -66,7 +66,7 @@ def stop(self): def _run(self): if self.__initial_delay > 0: - if self.__stop.wait(self.__initial_delay): + if self.__stop.wait(min(self.__initial_delay, TIMEOUT_MAX)): return stopped = self.__stop.is_set() while not stopped: @@ -77,4 +77,4 @@ def _run(self): # The wait starts when the callback returns, so a slow callback # never shortens it. delay = self.__delays.next_delay - stopped = self.__stop.wait(delay) if delay > 0 else self.__stop.is_set() + stopped = self.__stop.wait(min(delay, TIMEOUT_MAX)) if delay > 0 else self.__stop.is_set() diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 38475ac6..5cfc7f74 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -18,6 +18,7 @@ # currently excluded from documentation - see docs/README.md +import math import random import time from enum import Enum @@ -27,7 +28,7 @@ DEFAULT_INITIAL_RECONNECT_DELAY, DEFAULT_POLL_INTERVAL ) -from ldclient.impl.util import log, validate_positive_finite +from ldclient.impl.util import log # The delay ceiling of the normal regime for streaming, in seconds. NORMAL_STREAMING_CEILING_DELAY = 30 @@ -53,6 +54,23 @@ _MAX_BACKOFF_EXPONENT = 30 +def _usable_delay(value: float, default: float, name: str, ceiling: float = math.inf) -> float: + """ + Returns the delay to use, clamped to the ceiling. A value that is + not a positive, finite number of seconds is replaced by the default. + + :param value: the configured number of seconds + :param default: the value to use when ``value`` is not usable + :param name: the option name, for the warning message + :param ceiling: the longest delay allowed + """ + + if value > 0 and math.isfinite(value): + return min(value, ceiling) + log.warning("%s must be a positive, finite number of seconds; using the default of %ss" % (name, default)) + return default + + class FailureKind(Enum): """How a failure is classified, which decides how long the next wait is.""" @@ -251,19 +269,12 @@ def for_streaming(initial_reconnect_delay: float) -> RetryState: """ Builds the retry state for a streaming data source. - Streaming's operating cadence is zero, so there is no delay during - healthy operation. Stream failures use either the normal or extended - initial delay to determine their backoff wait. A stream returns to - healthy operation after establishing a successful connection with no - failures during the ``STREAMING_RESET_INTERVAL``. - - ``Config`` validates the configured delay, so this guard only catches a - state built without it. - - The extended regime never starts below the configured delay. + Streaming's cadence is zero, so a healthy stream never waits. An invalid + delay value is replaced by the documented default; one longer than a + ceiling raises that bound rather than being cut down to it. """ - initial_reconnect_delay = validate_positive_finite( - initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay', log + initial_reconnect_delay = _usable_delay( + initial_reconnect_delay, DEFAULT_INITIAL_RECONNECT_DELAY, 'initial_reconnect_delay' ) return RetryState( normal_initial_delay=initial_reconnect_delay, @@ -279,16 +290,12 @@ def for_polling(poll_interval: float) -> RetryState: """ Builds the retry state for a polling data source. - The poll interval is polling's operating cadence, so no wait is ever - shorter than it. In the normal regime the delay bounds are the poll - interval itself, which means a normal failure simply polls again on - schedule. Polling is healthy on any successful poll, and resets after two - in a row. - - ``Config`` validates and clamps the poll interval, so this guard only - catches a state built without it. + The poll interval is polling's cadence and its normal ceiling, so a normal + failure waits the interval rather than backing off past it. An invalid + interval is replaced by the documented default. No wait is ever shorter + than the interval, so the cadence wins over the extended ceiling. """ - poll_interval = validate_positive_finite(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval', log) + poll_interval = _usable_delay(poll_interval, DEFAULT_POLL_INTERVAL, 'poll_interval') return RetryState( normal_initial_delay=poll_interval, normal_ceiling_delay=poll_interval, diff --git a/ldclient/impl/util.py b/ldclient/impl/util.py index fcc9da03..b7d1713c 100644 --- a/ldclient/impl/util.py +++ b/ldclient/impl/util.py @@ -1,5 +1,4 @@ import logging -import math import re import sys import time @@ -61,25 +60,6 @@ def validate_application_value(value: Any, name: str, logger: logging.Logger) -> return value -def validate_positive_finite(value: float, default: float, name: str, logger: logging.Logger) -> float: - """ - Validates that a number of seconds is positive and finite. - - A non-finite value makes later arithmetic produce NaN, and a non-positive - one makes a wait no wait at all. - - :param value: the number of seconds to validate - :param default: the value to use when ``value`` is not usable - :param name: the option name, for the warning message - :param logger: the logger to use for logging warnings - :return: ``value``, or ``default`` if ``value`` is not positive and finite - """ - if value > 0 and math.isfinite(value): - return value - logger.warning("%s must be a positive, finite number of seconds; using the default of %ss" % (name, default)) - return default - - def validate_sdk_key_format(sdk_key: str, logger: logging.Logger) -> str: """ Validates that an SDK key does not contain invalid characters and is not too long for our systems. @@ -155,14 +135,11 @@ def throw_if_unsuccessful_response(resp): def is_http_error_recoverable(status): """ - Reports whether a component that treats some statuses as fatal should - keep going. - Deprecated. Use :func:`ldclient.impl.retry.classify_http_status` instead. """ if status >= 400 and status < 500: - return status in _RETRYABLE_STATUSES # all other 4xx besides these are treated as fatal - return True + return status in _RETRYABLE_STATUSES # all other 4xx besides these are unrecoverable + return True # all other errors are recoverable def http_error_description(status): @@ -171,11 +148,8 @@ def http_error_description(status): def http_error_message(status, context, retryable_message="will retry"): """ - Builds the log message for an HTTP failure in a component that stops on - some statuses. - - Deprecated. The FDv1 data sources build their own message instead, so - that it can report the real retry delay. + Deprecated. The FDv1 data sources build their own message, so that it can + report the real retry delay. """ return "Received %s for %s - %s" % (http_error_description(status), context, retryable_message if is_http_error_recoverable(status) else "giving up permanently") diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 04aed18c..df8339b3 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -1013,7 +1013,8 @@ class DataSourceState(Enum): """ Indicates that the data source has been permanently shut down. - This means the SDK client was explicitly shut down, or that its configuration could not be parsed. + This could be because the SDK client was explicitly shut down, because its configuration could not + be parsed, or because the data source encountered a condition it will not retry. """ diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index 990c292e..78a6f125 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -5,6 +5,7 @@ import asyncio import logging import ssl +import time from unittest.mock import AsyncMock, MagicMock, patch import aiohttp @@ -58,9 +59,12 @@ def make_config(**kwargs): ) +ONE_HOUR = 60 * 60 + + def fast_retry_state(delay=0.001): - """A retry state with tiny delays, so a test does not have to wait out the - real extended-regime delay of five minutes.""" + """A retry state whose every delay is ``delay``: small enough to skip the + real extended-regime wait, or large enough to prove a stop interrupts one.""" return RetryState( normal_initial_delay=delay, normal_ceiling_delay=delay, @@ -469,6 +473,30 @@ async def close(): assert order == ['poll_done', 'transport_closed'] + @pytest.mark.asyncio + async def test_an_extended_regime_wait_is_cut_short_by_stop(self): + """Shutdown must not sit through an hour-long backoff. The in-flight-poll + case is test_stop_cancels_polling_task_cleanly; this one stops while the + task is waiting between polls.""" + retry = fast_retry_state(ONE_HOUR) + processor = make_processor(retry_state=retry) + processor._requester.get_all_data = AsyncMock( + side_effect=UnsuccessfulResponseException(401) + ) + + processor.start() + # Confirm the wait under test really is long before measuring the stop. + deadline = time.time() + 2 + while retry.next_delay <= 60 and time.time() < deadline: + await asyncio.sleep(0.01) + assert retry.next_delay > 60, "the wait under test should be minutes long" + + started = time.time() + await processor.stop() + elapsed = time.time() - started + + assert elapsed < 2, "stop() took %.2fs" % elapsed + @pytest.mark.asyncio async def test_stop_cancels_polling_task_cleanly(self): store = MockAsyncFeatureStore() diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index 3ddea6a4..05420468 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -9,6 +9,7 @@ import json import logging import ssl +import time from unittest import mock import aiohttp @@ -105,9 +106,12 @@ def _retry_state_with(policy: AfterHealthyFor) -> RetryState: ) +ONE_HOUR = 60 * 60 + + def _fast_retry_state(delay: float = 0.001) -> RetryState: - """A retry state with tiny delays, so a test does not have to wait out the - real extended-regime delay of five minutes.""" + """A retry state whose every delay is ``delay``: small enough to skip the + real extended-regime wait, or large enough to prove a stop interrupts one.""" return RetryState( normal_initial_delay=delay, normal_ceiling_delay=delay, @@ -595,6 +599,37 @@ async def test_a_fresh_stream_starts_a_new_reset_window(): assert len(set(windows)) == 2, "the second stream reused the first window" +@pytest.mark.asyncio +async def test_an_extended_regime_wait_is_cut_short_by_stop(): + """Shutdown must not sit through an hour-long backoff. The healthy-stop case + is test_stop_closes_sse_and_finishes_task; this one stops mid-wait.""" + from ld_eventsource.errors import HTTPStatusError + + retry = _fast_retry_state(ONE_HOUR) + proc, store, ready, _ = _make_processor( + [_start(), _fault(error=HTTPStatusError(401))], retry_state=retry + ) + proc.start() + # Confirm the wait under test really is long before measuring the stop. + await _wait_until(lambda: retry.next_delay > 60) + + started = time.time() + await proc.stop() + elapsed = time.time() - started + + assert elapsed < 2, "stop() took %.2fs" % elapsed + leaked = [t for t in proc._runner._tasks if not t.done()] + assert leaked == [], "stop() returned with the task still running: %r" % leaked + + +@pytest.mark.asyncio +async def test_stop_before_start_and_stop_twice_are_safe(): + proc, store, ready, _ = _make_processor([]) + + await proc.stop() + await proc.stop() + + @pytest.mark.asyncio async def test_stop_closes_sse_and_finishes_task(): flag = FlagBuilder('f1').version(1).build() diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index bc8d9337..3aef8794 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -25,6 +25,7 @@ ) from ldclient.testing.builders import * from ldclient.testing.stub_util import MockFeatureRequester, MockResponse +from ldclient.testing.sync_util import wait_until from ldclient.testing.test_util import SpyListener, no_retry_jitter from ldclient.versioned_data_kind import FEATURES, SEGMENTS @@ -46,9 +47,12 @@ def teardown_function(): pp.stop() +ONE_HOUR = 60 * 60 + + def fast_retry_state(delay=0.05): - """A retry state with tiny delays, so a test does not have to wait out the - real extended-regime delay of five minutes.""" + """A retry state whose every delay is ``delay``: small enough to skip the + real extended-regime wait, or large enough to prove a stop interrupts one.""" return RetryState( normal_initial_delay=delay, normal_ceiling_delay=delay, @@ -160,8 +164,8 @@ def test_unexpected_http_error_moves_to_the_extended_regime(ignore_mock): setup_processor(Config("SDK_KEY"), retry_state=retry) # The extended regime starts at five minutes, so only the first poll runs. - assert not ready.wait(0.4) - assert retry._extended + wait_until(lambda: retry.next_delay > 0.1) + assert not ready.wait(0.1) assert mock_requester.request_count == 1 @@ -332,6 +336,36 @@ def test_an_extended_regime_wait_is_cut_short_by_stop(): assert elapsed < 1 +def test_an_absurd_poll_interval_does_not_kill_the_worker_thread(): + """``Event.wait`` raises above ``threading.TIMEOUT_MAX``, and that raise is + outside the try block around the poll, so the thread used to die while the + SDK still reported itself healthy.""" + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + setup_processor(Config("SDK_KEY", poll_interval=1e10)) + + assert ready.wait(2) + worker = _polling_thread() + assert worker is not None + + # A thread that raised on the wait exits as soon as the first poll returns. + worker.join(0.3) + assert worker.is_alive() + + +def test_stop_twice_and_stop_before_start_are_safe(): + """Neither a stop before the first poll nor a second stop should raise, + including while an hour-long wait is pending.""" + mock_requester.exception = UnsuccessfulResponseException(401) + processor = PollingUpdateProcessor( + Config("SDK_KEY"), mock_requester, store, ready, retry_state=fast_retry_state(ONE_HOUR) + ) + + processor.stop() + processor.stop() + processor.start() + processor.stop() + + def test_stop_reports_off(): spy = SpyListener() listeners = Listeners() diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 88570360..86bb1d5c 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -22,6 +22,7 @@ from ldclient.impl.events.diagnostics import _DiagnosticAccumulator from ldclient.impl.listeners import Listeners from ldclient.impl.retry import ( + NORMAL_STREAMING_CEILING_DELAY, STREAMING_RESET_INTERVAL, AfterHealthyFor, RetryState, @@ -47,6 +48,7 @@ make_put_event, stream_content ) +from ldclient.testing.sync_util import wait_until from ldclient.testing.test_util import ( SpyListener, no_retry_jitter, @@ -57,11 +59,12 @@ from ldclient.versioned_data_kind import FEATURES, SEGMENTS brief_delay = 0.001 +ONE_HOUR = 60 * 60 def fast_retry_state(delay=brief_delay): - """A retry state with tiny delays, so a test does not have to wait out the - real extended-regime delay of five minutes.""" + """A retry state whose every delay is ``delay``: small enough to skip the + real extended-regime wait, or large enough to prove a stop interrupts one.""" return RetryState( normal_initial_delay=delay, normal_ceiling_delay=delay, @@ -306,12 +309,15 @@ def test_unexpected_http_error_backs_off_a_long_way(status): with StreamingUpdateProcessor(config, store, ready, None) as sp: sp.start() - # Initialization is not falsely unblocked: the caller waits out - # its own start_wait and then finds the client uninitialized. - assert not ready.wait(1) + server.wait_until_request_received() + # The next attempt is past the normal ceiling, so the failure + # has been recorded and the extended regime is in use. + wait_until(lambda: sp._retry.next_delay > NORMAL_STREAMING_CEILING_DELAY) + + # Initialization is not falsely unblocked. + assert not ready.wait(0.1) assert not sp.initialized() assert sp.is_alive() - assert sp._retry._extended server.should_have_requests(1) @@ -550,6 +556,30 @@ def test_a_server_close_and_a_transport_error_both_report_a_delay(caplog): assert messages[1] == "Error on stream connection: [Errno 104] reset by peer - will retry in 2.0s" +def test_an_extended_regime_wait_is_cut_short_by_stop(): + """The reason the wait has to be interruptible at all. Shutdown must not + sit through an hour-long backoff.""" + store = InMemoryFeatureStore() + with start_server() as server: + config = Config(sdk_key='sdk-key', stream_uri=server.uri) + server.for_path('/all', BasicResponse(401)) + retry = fast_retry_state(ONE_HOUR) + + with StreamingUpdateProcessor(config, store, Event(), None, retry_state=retry) as sp: + sp.start() + server.wait_until_request_received() + # Confirm the wait under test really is long before measuring the stop. + wait_until(lambda: retry.next_delay > 60) + + started = time.time() + sp.stop() + sp.join(5) + elapsed = time.time() - started + + assert not sp.is_alive() + assert elapsed < 2, "stop() took %.2fs" % elapsed + + def test_several_messages_on_one_stream_do_not_extend_the_reset_window(): """The window starts at the first message and stays there, however many more arrive on the same stream.""" @@ -798,8 +828,11 @@ def test_failure_transitions_from_valid(): with StreamingUpdateProcessor(config, store, ready, None) as sp: sp.start() + server.wait_until_request_received() + wait_until(lambda: len(spy.statuses) == 2) + # The 401 is retried five minutes out, so readiness never fires. - assert not ready.wait(1) + assert not ready.wait(0.1) server.should_have_requests(1) assert len(spy.statuses) == 2 diff --git a/ldclient/testing/impl/test_retry.py b/ldclient/testing/impl/test_retry.py index 65114d65..138f5182 100644 --- a/ldclient/testing/impl/test_retry.py +++ b/ldclient/testing/impl/test_retry.py @@ -112,9 +112,9 @@ def test_non_error_statuses_are_normal(self, status): class TestFactoryInputGuards: - """A ``RetryState`` can be built without going through ``Config``, so the - factories guard their own input. A non-positive value would retry with no - wait; a non-finite one makes the jitter arithmetic produce NaN.""" + """``Config`` reports these options as configured, so the factories are the + only guard. A non-positive value would retry with no wait; a non-finite one + makes the jitter arithmetic produce NaN.""" @pytest.mark.parametrize( "configured", @@ -131,8 +131,7 @@ def test_streaming_falls_back_to_the_default(self, configured, caplog): assert delay == DEFAULT_INITIAL_RECONNECT_DELAY assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( - "initial_reconnect_delay must be a positive, finite number of seconds; " - "using the default of 1s" + "initial_reconnect_delay must be a positive, finite number of seconds; using the default of 1s" ) @pytest.mark.parametrize( @@ -150,11 +149,10 @@ def test_polling_falls_back_to_the_default(self, configured, caplog): assert delay == DEFAULT_POLL_INTERVAL assert math.isfinite(delay) and delay > 0 assert caplog.records[0].getMessage() == ( - "poll_interval must be a positive, finite number of seconds; " - "using the default of 30s" + "poll_interval must be a positive, finite number of seconds; using the default of 30s" ) - @pytest.mark.parametrize("configured", [0.001, 0.5, 1, 5, 45]) + @pytest.mark.parametrize("configured", [0.001, 0.5, 1, 5, 30]) def test_a_positive_streaming_delay_is_left_alone(self, configured, caplog): caplog.set_level(logging.WARNING) @@ -175,6 +173,24 @@ def test_a_positive_poll_interval_is_left_alone(self, configured, caplog): assert caplog.records == [] +class TestStreamingCeilings: + """A configured reconnect delay longer than a regime's ceiling raises that + bound. The configured value wins over our default, rather than being cut + down to it.""" + + @pytest.mark.parametrize("configured", [600, 7200, 86400]) + def test_a_delay_past_the_normal_ceiling_raises_the_bound(self, configured): + assert failure_delay(for_streaming(configured), NORMAL) == configured + + @pytest.mark.parametrize("configured", [7200, 86400]) + def test_a_delay_past_the_extended_ceiling_raises_the_bound(self, configured): + assert failure_delay(for_streaming(configured), UNEXPECTED) == configured + + @pytest.mark.parametrize("configured", [0.5, 1, 30]) + def test_a_delay_within_the_normal_ceiling_is_untouched(self, configured): + assert failure_delay(for_streaming(configured), NORMAL) == configured + + class TestStreamingExtendedDelayFloor: """A delay that applies after an unexpected failure must not be below the component's initial delay.""" @@ -219,11 +235,11 @@ def test_extended_regime_doubles_up_to_the_ceiling(self): delays += [failure_delay(state, NORMAL) for _ in range(5)] assert delays == [5 * 60, 10 * 60, 20 * 60, 40 * 60, 60 * 60, 60 * 60] - def test_a_configured_initial_delay_raises_the_ceiling_with_it(self): - # The ceiling must not fall below the initial delay. - state = for_streaming(45) - assert state._max_delay == 45 - assert failure_delay(state, NORMAL) == 45 + @pytest.mark.parametrize("configured", [1, 30, 45, 600]) + def test_the_ceiling_is_never_below_what_was_configured(self, configured): + # A configured delay longer than the normal ceiling raises the bound, so + # the first wait is never shorter than what the caller asked for. + assert for_streaming(configured)._max_delay == max(NORMAL_STREAMING_CEILING_DELAY, configured) def test_the_ceiling_is_sticky_once_the_extended_regime_starts(self): # A normal failure after an unexpected one must not lower the bounds @@ -395,12 +411,20 @@ def test_the_wait_never_falls_below_the_poll_interval(self): assert failure_delay(state, UNEXPECTED) >= 30 def test_a_poll_interval_longer_than_the_extended_bounds_wins(self): - # The ceiling is lifted by record_failure clamping it against the - # initial delay, not by for_polling clamping the ceiling itself. - state = for_polling(2 * 60 * 60) - assert failure_delay(state, UNEXPECTED) == 2 * 60 * 60 - assert state._max_delay == 2 * 60 * 60 - assert state._min_delay == 2 * 60 * 60 + """Unlike the streaming delay, the poll interval is not clamped to the + extended ceiling. Nothing may poll faster than the configured interval, + so the cadence wins where the two conflict.""" + two_hours = 2 * 60 * 60 + assert two_hours > EXTENDED_CEILING_DELAY + + state = for_polling(two_hours) + + assert failure_delay(state, UNEXPECTED) == two_hours + assert state._max_delay == two_hours + assert state._min_delay == two_hours + assert failure_delay(state, NORMAL) == two_hours + state.record_success() + assert state.next_delay == two_hours def test_one_success_restores_the_cadence_while_the_state_is_still_raised(self): # A backoff wait applies to a retry, not to every operation. diff --git a/ldclient/testing/test_config.py b/ldclient/testing/test_config.py index 3b18c3cf..9595eb0d 100644 --- a/ldclient/testing/test_config.py +++ b/ldclient/testing/test_config.py @@ -1,6 +1,12 @@ import pytest -from ldclient.config import Config +from ldclient.async_config import AsyncConfig +from ldclient.config import DEFAULT_POLL_INTERVAL, Config + +# Both classes handle these options identically, so every case runs against +# both rather than being duplicated and left to drift. +CONFIG_CLASSES = [Config, AsyncConfig] +CONFIG_IDS = ["Config", "AsyncConfig"] def test_copy_config(): @@ -38,14 +44,32 @@ def test_with_wrapper_information_defaults_the_version(): assert wrapped.wrapper_version is None -def test_can_set_valid_poll_interval(): - config = Config(sdk_key="SDK_KEY", poll_interval=31) - assert config.poll_interval == 31 - - -def test_minimum_poll_interval_is_enforced(): - config = Config(sdk_key="SDK_KEY", poll_interval=29) - assert config.poll_interval == 30 +@pytest.mark.parametrize("config_class", CONFIG_CLASSES, ids=CONFIG_IDS) +@pytest.mark.parametrize( + "configured,expected", + [ + (5, DEFAULT_POLL_INTERVAL), + (29, DEFAULT_POLL_INTERVAL), + (30, 30), + (31, 31), + (60, 60), + ], + ids=["below-the-minimum", "just-below", "at-the-minimum", "just-above", "above"], +) +def test_a_poll_interval_below_the_minimum_is_raised_to_it(config_class, configured, expected): + config = config_class(sdk_key="SDK_KEY", poll_interval=configured) + + assert config.poll_interval == expected + + +@pytest.mark.parametrize("config_class", CONFIG_CLASSES, ids=CONFIG_IDS) +@pytest.mark.parametrize("configured", [0.001, 0.5, 1, 30, 600], ids=["tiny", "fraction", "default", "thirty", "long"]) +def test_a_configured_initial_reconnect_delay_is_reported_as_given(config_class, configured): + """This option has no minimum, so a sub-second value survives as given. + The spec ceilings are applied by ``for_streaming``, not here.""" + config = config_class(sdk_key="SDK_KEY", initial_reconnect_delay=configured) + + assert config.initial_reconnect_delay == configured def test_can_set_valid_diagnostic_interval(): From 0047a65c45eb9ca62355857bcf0213d09bb8ce46 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 18 Sep 2026 16:54:12 -0500 Subject: [PATCH 13/19] refactor: Drop the unused ceiling parameter from the delay validator --- ldclient/impl/retry.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ldclient/impl/retry.py b/ldclient/impl/retry.py index 5cfc7f74..edfd1b85 100644 --- a/ldclient/impl/retry.py +++ b/ldclient/impl/retry.py @@ -54,19 +54,18 @@ _MAX_BACKOFF_EXPONENT = 30 -def _usable_delay(value: float, default: float, name: str, ceiling: float = math.inf) -> float: +def _usable_delay(value: float, default: float, name: str) -> float: """ - Returns the delay to use, clamped to the ceiling. A value that is - not a positive, finite number of seconds is replaced by the default. + Returns the delay to use. A value that is not a positive, finite number of + seconds is replaced by the default. :param value: the configured number of seconds :param default: the value to use when ``value`` is not usable :param name: the option name, for the warning message - :param ceiling: the longest delay allowed """ if value > 0 and math.isfinite(value): - return min(value, ceiling) + return value log.warning("%s must be a positive, finite number of seconds; using the default of %ss" % (name, default)) return default From 30bb8ecc32e57a694be4fdcbafd7265f3bc6bbb3 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 18 Sep 2026 17:05:36 -0500 Subject: [PATCH 14/19] fix: Measure the start_wait tests against a monotonic clock and a bound --- ldclient/testing/test_ldclient_end_to_end.py | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ldclient/testing/test_ldclient_end_to_end.py b/ldclient/testing/test_ldclient_end_to_end.py index ce523a53..f5b5b9b0 100644 --- a/ldclient/testing/test_ldclient_end_to_end.py +++ b/ldclient/testing/test_ldclient_end_to_end.py @@ -62,10 +62,14 @@ def test_client_does_not_initialize_in_streaming_mode_with_401_error(): stream_server.for_path('/all', BasicResponse(401)) config = Config(sdk_key=sdk_key, stream_uri=stream_server.uri, send_events=False) - started = time.time() - with LDClient(config=config, start_wait=0.5) as client: - elapsed = time.time() - started - assert elapsed >= 0.5 + start_wait = 0.5 + started = time.monotonic() + with LDClient(config=config, start_wait=start_wait) as client: + elapsed = time.monotonic() - started + # A bound rather than the exact start_wait: Event.wait can return a + # fraction early against a separate clock. Failing fast took + # milliseconds, so this still catches it. + assert elapsed >= start_wait / 2 assert not client.is_initialized() assert client.variation(always_true_flag['key'], user, False) is False @@ -104,10 +108,12 @@ def test_client_does_not_initialize_in_polling_mode_with_401_error(): poll_server.for_path('/sdk/latest-all', BasicResponse(401)) config = Config(sdk_key=sdk_key, base_uri=poll_server.uri, stream=False, send_events=False) - started = time.time() - with LDClient(config=config, start_wait=0.5) as client: - elapsed = time.time() - started - assert elapsed >= 0.5 + start_wait = 0.5 + started = time.monotonic() + with LDClient(config=config, start_wait=start_wait) as client: + elapsed = time.monotonic() - started + # See the streaming case above: a bound, not the exact start_wait. + assert elapsed >= start_wait / 2 assert not client.is_initialized() assert client.variation(always_true_flag['key'], user, False) is False From c0000b60d450ec0c264c8ae2753117db718699e0 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Tue, 22 Sep 2026 12:47:52 -0500 Subject: [PATCH 15/19] fix: Clear a stale interrupt flag and time the attempt after the wait Addresses review feedback on #519. A new stream connection now clears _interrupted_by_sdk. interrupt() is a no-op when the connection has already gone, so no Fault arrives to clear the flag and it could swallow the next genuine server close, losing one failure and skipping one backoff. Both streaming sources get a test that fails without the change. _connection_attempt_start_time is now read after the retry wait instead of predicted before it, so a clock change during the wait cannot skew the stream-init latency we report. Also drops the certificate-classification comment from all four data sources, says why the bare RetryDelayStrategy is passed to both SSE clients, and rewords AsyncPollingUpdateProcessor.stop() so it is clear that the wait after OFF only drains a poll already in flight. --- ldclient/impl/aio/transport.py | 4 ++- ldclient/impl/datasource/async_polling.py | 8 +++--- ldclient/impl/datasource/async_streaming.py | 10 +++++-- ldclient/impl/datasource/polling.py | 1 - ldclient/impl/datasource/streaming.py | 16 ++++++++--- .../impl/datasource/test_async_streaming.py | 24 ++++++++++++++++ .../testing/impl/datasource/test_streaming.py | 28 +++++++++++++++++++ 7 files changed, 79 insertions(+), 12 deletions(-) diff --git a/ldclient/impl/aio/transport.py b/ldclient/impl/aio/transport.py index 4da6e0c2..a83765fe 100644 --- a/ldclient/impl/aio/transport.py +++ b/ldclient/impl/aio/transport.py @@ -140,7 +140,9 @@ def create(self, url: str, initial_retry_delay: float, query_params=None, sdk_ma if proxy: aiohttp_request_options["proxy"] = proxy if sdk_managed_retry: - # The SSE client's retry is disabled; the SDK owns the delay. + # The SSE client's retry is disabled; the SDK owns the delay. The base + # strategy returns the delay unchanged, so the wait is always zero; + # omitting it would select the library's own backoff. retry_options: dict = { "initial_retry_delay": 0, "retry_delay_strategy": RetryDelayStrategy(), diff --git a/ldclient/impl/datasource/async_polling.py b/ldclient/impl/datasource/async_polling.py index d2099836..1197b8a0 100644 --- a/ldclient/impl/datasource/async_polling.py +++ b/ldclient/impl/datasource/async_polling.py @@ -63,9 +63,10 @@ async def stop(self): if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.OFF, None) - # Wait for the current poll to finish before closing the transport, so we do - # not close it while a request is still using it. The close is in a finally - # so an owned transport is still released if stop() is cancelled mid-wait. + # OFF is reported first, so a listener sees the shutdown at once. The wait + # that follows only drains a poll already in flight, so the transport is + # not closed while that request still uses it. The close is in a finally, + # so an owned transport is released even if stop() is cancelled mid-wait. try: await self._task.wait_stopped() finally: @@ -95,7 +96,6 @@ async def _fetch_and_store(self) -> None: level = log.error if kind is FailureKind.UNEXPECTED else log.warning stacktrace = None except Exception as e: - # A certificate failure lands here too, and is as normal as the rest. kind = FailureKind.NORMAL error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) description = "Error encountered when updating flags: %s" % e diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index 06061259..da98536e 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -89,6 +89,10 @@ async def _run(self): self._connection_attempt_start_time = time.time() async for action in self._sse.all: if isinstance(action, Start): + # interrupt() is a no-op when the connection has already gone, so + # clear a stale flag here rather than swallow the next real close. + self._interrupted_by_sdk = False + # On reconnect after an error the timer was cleared; reset it here. # For the initial connect the pre-loop timestamp is already set. if self._connection_attempt_start_time is None: @@ -256,7 +260,6 @@ async def _handle_error(self, error: Exception) -> bool: description = "The server closed the stream connection" level = log.warning else: - # A certificate failure lands here too, and is as normal as the rest. kind = FailureKind.NORMAL error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error)) # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of HTTP client internals @@ -270,9 +273,12 @@ async def _handle_error(self, error: Exception) -> bool: if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - self._connection_attempt_start_time = time.time() + delay if delay > 0: await asyncio.sleep(delay) + + # Read after the wait, so a clock change during it cannot skew the + # stream-init latency we report. + self._connection_attempt_start_time = time.time() return self._running # magic methods for "with" statement (used in testing) diff --git a/ldclient/impl/datasource/polling.py b/ldclient/impl/datasource/polling.py index d15ea5c4..2eacc6ad 100644 --- a/ldclient/impl/datasource/polling.py +++ b/ldclient/impl/datasource/polling.py @@ -100,7 +100,6 @@ def _poll(self) -> None: level = log.error if kind is FailureKind.UNEXPECTED else log.warning stacktrace = None except Exception as e: - # A certificate failure lands here too, and is as normal as the rest. kind = FailureKind.NORMAL error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(e)) description = "Error encountered when updating flags: %s" % e diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index 39f2c349..5a177d4e 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -74,6 +74,9 @@ def run(self): self._connection_attempt_start_time = time.time() for action in self._sse.all: if isinstance(action, Start): + # interrupt() is a no-op when the connection has already gone, so + # clear a stale flag here rather than swallow the next real close. + self._interrupted_by_sdk = False record_environment_id(self._data_source_update_sink, action.headers) elif isinstance(action, Event): message_ok = False @@ -137,7 +140,9 @@ def _create_sse_client(self) -> SSEClient: url=self._uri, headers=http_factory.base_headers, pool=stream_http_factory.create_pool_manager(1, self._uri), urllib3_request_options={"timeout": stream_http_factory.timeout} ), error_strategy=ErrorStrategy.always_continue(), # we'll make error-handling decisions when we see a Fault - # The SSE client's retry is disabled; the SDK owns the delay. + # The SSE client's retry is disabled; the SDK owns the delay. The base + # strategy returns the delay unchanged, so the wait is always zero; + # omitting it would select the library's own backoff. initial_retry_delay=0, retry_delay_strategy=RetryDelayStrategy(), retry_delay_reset_threshold=0, @@ -230,7 +235,6 @@ def _handle_error(self, error: Exception) -> bool: description = "The server closed the stream connection" level = log.warning else: - # A certificate failure lands here too, and is as normal as the rest. kind = FailureKind.NORMAL error_info = DataSourceErrorInfo(DataSourceErrorKind.UNKNOWN, 0, time.time(), str(error)) # no stacktrace here because, for a typical connection error, it'll just be a lengthy tour of urllib3 internals @@ -244,8 +248,12 @@ def _handle_error(self, error: Exception) -> bool: if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.INTERRUPTED, error_info) - self._connection_attempt_start_time = time.time() + delay - return not self._stop_event.wait(min(delay, TIMEOUT_MAX)) + interrupted = self._stop_event.wait(min(delay, TIMEOUT_MAX)) + + # Read after the wait, so a clock change during it cannot skew the + # stream-init latency we report. + self._connection_attempt_start_time = time.time() + return not interrupted # magic methods for "with" statement (used in testing) def __enter__(self): diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index 05420468..4357ba6f 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -399,6 +399,30 @@ async def test_our_own_interrupt_is_not_counted_as_a_server_close(): await proc.stop() +@pytest.mark.asyncio +async def test_a_leaked_interrupt_flag_does_not_swallow_a_server_close(): + """interrupt() is a no-op when the connection has already gone, so no + Fault arrives to clear the flag. A new connection must clear it, or the + next genuine close is recorded as ours and the backoff is skipped.""" + put_data = _make_put_data() + actions = [ + _start(), + _event('put', put_data), + _fault(error=None), # a close the SDK did not ask for + ] + + retry = _fast_retry_state() + proc, _, _, _ = _make_processor(actions, retry_state=retry) + proc._interrupted_by_sdk = True + proc.start() + await _wait_until(lambda: retry._attempts >= 1) + await asyncio.sleep(0.1) + + assert retry._attempts == 1 + + await proc.stop() + + @pytest.mark.asyncio async def test_unexpected_http_error_keeps_the_processor_running(): """A rejected SDK key is retried like any other failure. The state never diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index 86bb1d5c..c249fc3c 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -499,6 +499,34 @@ def listener(s): assert retry._attempts == 1 +def test_a_leaked_interrupt_flag_does_not_swallow_a_server_close(): + """interrupt() is a no-op when the connection has already gone, so no + Fault arrives to clear the flag. A new connection must clear it, or the + next genuine close is recorded as ours and the backoff is skipped.""" + store = InMemoryFeatureStore() + ready = Event() + flagv1 = FlagBuilder('flagkey').version(1).build() + flagv2 = FlagBuilder('flagkey').version(2).build() + + with start_server() as server: + with stream_content(make_put_event([flagv1])) as stream1: + with stream_content(make_put_event([flagv2])) as stream2: + config = Config(sdk_key='sdk-key', stream_uri=server.uri, initial_reconnect_delay=brief_delay) + server.for_path('/all', SequentialHandler(stream1, stream2)) + + retry = fast_retry_state() + with StreamingUpdateProcessor(config, store, ready, None, retry_state=retry) as sp: + sp._interrupted_by_sdk = True + sp.start() + ready.wait(start_wait) + assert sp.initialized() + + stream1.close() + expect_update(store, FEATURES, flagv2) + + assert retry._attempts == 1 + + def _handle_errors_without_waiting(retry, errors): """Drives _handle_error for each error and returns nothing. The stop event is pre-set so the interruptible wait returns at once.""" From e67c29bb8e16d9988b42f51c46ca06d43af54f9a Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 23 Sep 2026 13:11:38 -0500 Subject: [PATCH 16/19] fix: Trim two comments on the data source shutdown and retry wiring Addresses review feedback on #519. AsyncPollingUpdateProcessor.stop() now carries one line saying why it waits before closing the transport. The previous wording read as if the processor kept working after OFF, when it is shutting down. The comment on the bare RetryDelayStrategy keeps only the fact a reader needs: the strategy must be passed, or the SSE client picks its own backoff. --- ldclient/impl/aio/transport.py | 3 +-- ldclient/impl/datasource/async_polling.py | 5 +---- ldclient/impl/datasource/streaming.py | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ldclient/impl/aio/transport.py b/ldclient/impl/aio/transport.py index a83765fe..947d13d3 100644 --- a/ldclient/impl/aio/transport.py +++ b/ldclient/impl/aio/transport.py @@ -141,8 +141,7 @@ def create(self, url: str, initial_retry_delay: float, query_params=None, sdk_ma aiohttp_request_options["proxy"] = proxy if sdk_managed_retry: # The SSE client's retry is disabled; the SDK owns the delay. The base - # strategy returns the delay unchanged, so the wait is always zero; - # omitting it would select the library's own backoff. + # strategy must be passed: omitting it selects the library's backoff. retry_options: dict = { "initial_retry_delay": 0, "retry_delay_strategy": RetryDelayStrategy(), diff --git a/ldclient/impl/datasource/async_polling.py b/ldclient/impl/datasource/async_polling.py index 1197b8a0..91f24915 100644 --- a/ldclient/impl/datasource/async_polling.py +++ b/ldclient/impl/datasource/async_polling.py @@ -63,10 +63,7 @@ async def stop(self): if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.OFF, None) - # OFF is reported first, so a listener sees the shutdown at once. The wait - # that follows only drains a poll already in flight, so the transport is - # not closed while that request still uses it. The close is in a finally, - # so an owned transport is released even if stop() is cancelled mid-wait. + # Do not close the transport while an in-flight request still uses it. try: await self._task.wait_stopped() finally: diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index 5a177d4e..e79c874b 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -141,8 +141,7 @@ def _create_sse_client(self) -> SSEClient: ), error_strategy=ErrorStrategy.always_continue(), # we'll make error-handling decisions when we see a Fault # The SSE client's retry is disabled; the SDK owns the delay. The base - # strategy returns the delay unchanged, so the wait is always zero; - # omitting it would select the library's own backoff. + # strategy must be passed: omitting it selects the library's backoff. initial_retry_delay=0, retry_delay_strategy=RetryDelayStrategy(), retry_delay_reset_threshold=0, From 496830fb8863b54ec61d303a128c921a81df3f5a Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 23 Sep 2026 14:46:38 -0500 Subject: [PATCH 17/19] fix: Latch OFF so no data source status follows a shutdown Addresses review feedback on #519. Both FDv1 data source update sinks now ignore every status after OFF. A poll or stream connection that was still in flight when stop() ran could report VALID or INTERRUPTED afterwards, so a listener saw the data source come back from a shutdown it will never come back from. The latch lives in the sink rather than in each data source because a store write that fails reports INTERRUPTED from __monitor_store_update, which no data source can guard. Both sinks are built once per client and FDv1 never switches data sources, so OFF is terminal for their whole lifetime. StreamingUpdateProcessor declares _sse, so a stop() before the first run no longer raises AttributeError, and run() gives up if a stop landed before the client existed. Without that check stop() had nothing to close and the run went on to read a connection nobody was left to close. The action loop is also wrapped in try/finally, so a raise the loop does not catch can no longer leak the connection pool. The async source already had both, which is why only the sync one changes here. Both streaming sources now report OFF before teardown rather than after. OFF answers "will more data arrive?", not "is every socket closed?", so a slow close must not hold back the status that tells a waiter to give up. The DataSourceState.OFF docstring says what the state now guarantees. --- ldclient/impl/datasource/async_status.py | 5 + ldclient/impl/datasource/async_streaming.py | 23 ++-- ldclient/impl/datasource/status.py | 5 + ldclient/impl/datasource/streaming.py | 122 ++++++++++-------- ldclient/interfaces.py | 6 +- .../impl/datasource/test_async_polling.py | 24 ++++ .../impl/datasource/test_async_status.py | 32 +++++ .../impl/datasource/test_async_streaming.py | 35 +++++ .../impl/datasource/test_polling_processor.py | 17 +++ .../testing/impl/datasource/test_streaming.py | 69 +++++++++- ldclient/testing/impl/test_data_sink.py | 41 +++++- 11 files changed, 310 insertions(+), 69 deletions(-) diff --git a/ldclient/impl/datasource/async_status.py b/ldclient/impl/datasource/async_status.py index 1bebc281..29bccdb0 100644 --- a/ldclient/impl/datasource/async_status.py +++ b/ldclient/impl/datasource/async_status.py @@ -72,6 +72,11 @@ def update_status(self, new_state: DataSourceState, new_error: Optional[DataSour old_status = self.__status + # OFF is terminal. A poll or stream connection that was still in + # flight when the data source stopped must not report after it. + if old_status.state == DataSourceState.OFF: + return + if new_state == DataSourceState.INTERRUPTED and old_status.state == DataSourceState.INITIALIZING: new_state = DataSourceState.INITIALIZING diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index da98536e..62da5c0f 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -166,23 +166,24 @@ def _record_stream_init(self, failed: bool): self._diagnostic_accumulator.record_stream_init(current_time, elapsed if elapsed >= 0 else 0, failed) async def stop(self): - # Cancel the run task first: otherwise, if stop() is called before _run has executed, the - # loop could run _run at the teardown await and create a fresh SSE connection against the - # session we're closing. Once the runner is stopped, teardown is safe. - await self._runner.stop_all() - log.info("Stopping AsyncStreamingUpdateProcessor") self._running = False + + # OFF means an explicit shutdown. No stream failure produces it. It is + # reported before teardown, so a slow close cannot hold back the status + # that tells a waiter to give up. The sink drops anything after OFF. + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.OFF, None) + + # Cancel the run task before the teardown awaits: otherwise, if stop() is called before + # _run has executed, the loop could run _run at the teardown await and create a fresh SSE + # connection against the session we're closing. Once the runner is stopped, teardown is safe. + await self._runner.stop_all() + if self._sse: await self._sse.close() await self._close_owned_session() - if self._data_source_update_sink is None: - return - - # OFF means an explicit shutdown. No stream failure produces it. - self._data_source_update_sink.update_status(DataSourceState.OFF, None) - async def _interrupt_stream(self): """Drops the stream connection so the next read reconnects. The SSE client reports the close as a Fault with no error, and the flag tells diff --git a/ldclient/impl/datasource/status.py b/ldclient/impl/datasource/status.py index c4d046d7..cec2ed24 100644 --- a/ldclient/impl/datasource/status.py +++ b/ldclient/impl/datasource/status.py @@ -80,6 +80,11 @@ def update_status(self, new_state: DataSourceState, new_error: Optional[DataSour with self.__lock.write(): old_status = self.__status + # OFF is terminal. A poll or stream connection that was still in + # flight when the data source stopped must not report after it. + if old_status.state == DataSourceState.OFF: + return + if new_state == DataSourceState.INTERRUPTED and old_status.state == DataSourceState.INITIALIZING: new_state = DataSourceState.INITIALIZING diff --git a/ldclient/impl/datasource/streaming.py b/ldclient/impl/datasource/streaming.py index e79c874b..2e48e918 100644 --- a/ldclient/impl/datasource/streaming.py +++ b/ldclient/impl/datasource/streaming.py @@ -64,6 +64,7 @@ def __init__(self, config, store, ready, diagnostic_accumulator, retry_state: Op self._diagnostic_accumulator = diagnostic_accumulator self._connection_attempt_start_time: Optional[float] = None self._retry = retry_state or for_streaming(config.initial_reconnect_delay) + self._sse: Optional[SSEClient] = None self._stop_event = ThreadEvent() self._interrupted_by_sdk = False @@ -71,59 +72,69 @@ def run(self): log.info("Starting StreamingUpdateProcessor connecting to uri: " + self._uri) self._running = True self._sse = self._create_sse_client() - self._connection_attempt_start_time = time.time() - for action in self._sse.all: - if isinstance(action, Start): - # interrupt() is a no-op when the connection has already gone, so - # clear a stale flag here rather than swallow the next real close. - self._interrupted_by_sdk = False - record_environment_id(self._data_source_update_sink, action.headers) - elif isinstance(action, Event): - message_ok = False - message_handled = False - try: - message_ok = self._process_message(sink_or_store(self._data_source_update_sink, self._store), action) - message_handled = True - except json.decoder.JSONDecodeError as e: - log.info("Error while handling stream event; will restart stream: %s" % e) - self._interrupt_stream() - - if not self._handle_error(e): - break - except Exception as e: - log.info("Error while handling stream event; will restart stream: %s" % e) - self._interrupt_stream() - if not self._handle_error(e): - break + # stop() may have run before the client existed, in which case it had + # nothing to close. Never read a connection nobody is left to close. + if self._stop_event.is_set(): + self._sse.close() + return - if message_handled: - self._retry.record_success() - - if message_ok: - self._record_stream_init(False) - self._connection_attempt_start_time = None - - if self._data_source_update_sink is not None: - self._data_source_update_sink.update_status(DataSourceState.VALID, None) - - if not self._ready.is_set(): - log.info("StreamingUpdateProcessor initialized ok.") - self._ready.set() - elif isinstance(action, Fault): - # A Fault with no error is a clean close. An interrupt the SDK - # asked for is not a failure. - if action.error is None: - if self._interrupted_by_sdk: - self._interrupted_by_sdk = False + self._connection_attempt_start_time = time.time() + try: + for action in self._sse.all: + if isinstance(action, Start): + # interrupt() is a no-op when the connection has already gone, so + # clear a stale flag here rather than swallow the next real close. + self._interrupted_by_sdk = False + record_environment_id(self._data_source_update_sink, action.headers) + elif isinstance(action, Event): + message_ok = False + message_handled = False + try: + message_ok = self._process_message(sink_or_store(self._data_source_update_sink, self._store), action) + message_handled = True + except json.decoder.JSONDecodeError as e: + log.info("Error while handling stream event; will restart stream: %s" % e) + self._interrupt_stream() + + if not self._handle_error(e): + break + except Exception as e: + log.info("Error while handling stream event; will restart stream: %s" % e) + self._interrupt_stream() + + if not self._handle_error(e): + break + + if message_handled: + self._retry.record_success() + + if message_ok: + self._record_stream_init(False) + self._connection_attempt_start_time = None + + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.VALID, None) + + if not self._ready.is_set(): + log.info("StreamingUpdateProcessor initialized ok.") + self._ready.set() + elif isinstance(action, Fault): + # A Fault with no error is a clean close. An interrupt the SDK + # asked for is not a failure. + if action.error is None: + if self._interrupted_by_sdk: + self._interrupted_by_sdk = False + continue + if not self._handle_error(StreamClosedError()): + break continue - if not self._handle_error(StreamClosedError()): - break - continue - if not self._handle_error(action.error): - break - self._sse.close() + if not self._handle_error(action.error): + break + finally: + # A raise inside the loop must not leak the connection pool. + self._sse.close() def _record_stream_init(self, failed: bool): if self._diagnostic_accumulator and self._connection_attempt_start_time: @@ -152,14 +163,15 @@ def stop(self): log.info("Stopping StreamingUpdateProcessor") self._running = False self._stop_event.set() - if self._sse: - self._sse.close() - if self._data_source_update_sink is None: - return + # OFF means an explicit shutdown. No stream failure produces it. It is + # reported before teardown, so a slow close cannot hold back the status + # that tells a waiter to give up. The sink drops anything after OFF. + if self._data_source_update_sink is not None: + self._data_source_update_sink.update_status(DataSourceState.OFF, None) - # OFF means an explicit shutdown. No stream failure produces it. - self._data_source_update_sink.update_status(DataSourceState.OFF, None) + if self._sse: + self._sse.close() def _interrupt_stream(self): """Drops the stream connection so the next read reconnects. The SSE diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index df8339b3..de403354 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -1011,10 +1011,14 @@ class DataSourceState(Enum): OFF = 'off' """ - Indicates that the data source has been permanently shut down. + Indicates that the data source is permanently shut down. This could be because the SDK client was explicitly shut down, because its configuration could not be parsed, or because the data source encountered a condition it will not retry. + + No further state or data follows. A request or connection that was still in flight when the data + source stopped is not reported, so this state is final for the lifetime of the data source. It is + reported when the shutdown begins rather than when the last connection closes. """ diff --git a/ldclient/testing/impl/datasource/test_async_polling.py b/ldclient/testing/impl/datasource/test_async_polling.py index 78a6f125..8d497ae0 100644 --- a/ldclient/testing/impl/datasource/test_async_polling.py +++ b/ldclient/testing/impl/datasource/test_async_polling.py @@ -573,6 +573,30 @@ async def test_stop_updates_sink_to_off(self, mock_interval): assert any(c.args[0] == DataSourceState.OFF for c in sink.update_status.call_args_list) + @pytest.mark.asyncio + async def test_a_poll_finishing_after_stop_reports_nothing(self): + """The poll still in flight when stop() ran must not report after OFF.""" + from ldclient.impl.datasource.async_status import ( + AsyncDataSourceUpdateSinkImpl + ) + from ldclient.impl.listeners import Listeners + + store = MockAsyncFeatureStore() + observed = [] + listeners = Listeners() + listeners.add(lambda status: observed.append(status.state)) + + config = make_config() + config._data_source_update_sink = AsyncDataSourceUpdateSinkImpl(store, listeners, Listeners()) + + processor = make_processor(config=config, store=store) + processor._requester.get_all_data = AsyncMock(return_value=SAMPLE_DATA) + + await processor.stop() + await processor._fetch_and_store() + + assert observed == [DataSourceState.OFF] + @pytest.mark.asyncio @patch('ldclient.config.Config.poll_interval', new_callable=MagicMock) async def test_valid_status_is_reported_before_ready_is_set(self, mock_interval): diff --git a/ldclient/testing/impl/datasource/test_async_status.py b/ldclient/testing/impl/datasource/test_async_status.py index 6790a922..25b8e635 100644 --- a/ldclient/testing/impl/datasource/test_async_status.py +++ b/ldclient/testing/impl/datasource/test_async_status.py @@ -333,3 +333,35 @@ async def test_upsert_records_store_error_on_failure(): assert len(status_capture.statuses) == 1 assert status_capture.statuses[0].error.kind == DataSourceErrorKind.STORE_ERROR + + +@pytest.mark.asyncio +async def test_update_status_off_is_terminal(): + sink, status_listeners, _ = make_sink() + status_capture = StatusCapture() + status_listeners.add(status_capture) + + sink.update_status(DataSourceState.VALID, None) + sink.update_status(DataSourceState.OFF, None) + + # A poll or stream connection still in flight when the data source stopped. + sink.update_status(DataSourceState.VALID, None) + sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, 1000, 'late')) + + assert sink.status.state == DataSourceState.OFF + assert sink.status.error is None + assert [status.state for status in status_capture.statuses] == [DataSourceState.VALID, DataSourceState.OFF] + + +@pytest.mark.asyncio +async def test_store_error_after_off_reports_nothing(): + sink, status_listeners, _ = make_sink(_FailingStore()) + status_capture = StatusCapture() + status_listeners.add(status_capture) + sink.update_status(DataSourceState.OFF, None) + + with pytest.raises(RuntimeError): + await sink.upsert(FEATURES, make_flag('flag-a').to_json_dict()) + + assert sink.status.state == DataSourceState.OFF + assert [status.state for status in status_capture.statuses] == [DataSourceState.OFF] diff --git a/ldclient/testing/impl/datasource/test_async_streaming.py b/ldclient/testing/impl/datasource/test_async_streaming.py index 4357ba6f..0c8865c3 100644 --- a/ldclient/testing/impl/datasource/test_async_streaming.py +++ b/ldclient/testing/impl/datasource/test_async_streaming.py @@ -873,3 +873,38 @@ async def test_diagnostics_recorded_on_successful_init(): assert recorded[0]['failed'] is False await proc.stop() + + +@pytest.mark.asyncio +async def test_off_is_reported_before_teardown(): + """A slow close must not hold back the status that tells a waiter to give + up, so OFF goes out before the connection and session are torn down.""" + order = [] + + class _OrderingSink: + async def init(self, all_data): + pass + + def update_status(self, new_state, new_error): + order.append(new_state) + + config = _make_config() + config._data_source_update_sink = _OrderingSink() + + flag = FlagBuilder('f1').version(1).build() + put_data = _make_put_data(flags={'f1': _item_dict(flag)}) + proc, _, _, factory = await _run_with_actions([_start(), _event('put', put_data)], config=config) + + sse = factory.created[0] + real_close = sse.close + + async def close(): + order.append('closed') + await real_close() + + sse.close = close + + await proc.stop() + + assert DataSourceState.OFF in order + assert order.index(DataSourceState.OFF) < order.index('closed') diff --git a/ldclient/testing/impl/datasource/test_polling_processor.py b/ldclient/testing/impl/datasource/test_polling_processor.py index 3aef8794..2ae603ff 100644 --- a/ldclient/testing/impl/datasource/test_polling_processor.py +++ b/ldclient/testing/impl/datasource/test_polling_processor.py @@ -382,6 +382,23 @@ def test_stop_reports_off(): assert spy.statuses[-1].state == DataSourceState.OFF +def test_a_poll_finishing_after_stop_reports_nothing(): + """The poll still in flight when stop() ran must not report after OFF.""" + spy = SpyListener() + listeners = Listeners() + listeners.add(spy) + + config = Config("SDK_KEY") + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + mock_requester.all_data = {FEATURES: {}, SEGMENTS: {}} + processor = PollingUpdateProcessor(config, mock_requester, store, ready) + + processor.stop() + processor._poll() + + assert [status.state for status in spy.statuses] == [DataSourceState.OFF] + + def test_valid_status_is_reported_before_ready_is_set(): # Mirrors go-server-sdk#442: a caller that wakes on readiness must not # still be able to read INITIALIZING. diff --git a/ldclient/testing/impl/datasource/test_streaming.py b/ldclient/testing/impl/datasource/test_streaming.py index c249fc3c..307af278 100644 --- a/ldclient/testing/impl/datasource/test_streaming.py +++ b/ldclient/testing/impl/datasource/test_streaming.py @@ -1,7 +1,7 @@ import logging import ssl import time -from threading import Event +from threading import Event, Thread from typing import List import pytest @@ -527,6 +527,73 @@ def test_a_leaked_interrupt_flag_does_not_swallow_a_server_close(): assert retry._attempts == 1 +def test_stop_before_start_does_not_raise(): + """stop() can land before run() has built the SSE client.""" + config = Config(sdk_key='sdk-key', stream_uri='http://localhost') + sp = StreamingUpdateProcessor(config, InMemoryFeatureStore(), Event(), None) + sp.stop() + + +def test_a_stop_before_the_connection_exists_still_ends_the_run(): + """stop() has nothing to close when run() has not built the client yet, so + the run itself must not go on to read a connection nobody will close.""" + store = InMemoryFeatureStore() + ready = Event() + + with start_server() as server: + with stream_content(make_put_event()) as stream: + server.for_path('/all', stream) + config = Config(sdk_key='sdk-key', stream_uri=server.uri) + sp = StreamingUpdateProcessor(config, store, ready, None) + + sp.stop() + thread = Thread(target=sp.run, daemon=True) + thread.start() + thread.join(update_wait) + + assert not thread.is_alive() + assert not ready.is_set() + assert not store.initialized + + +def test_a_raise_inside_the_loop_closes_the_connection(): + """A raise the loop does not catch must not leak the connection pool.""" + store = InMemoryFeatureStore() + closes: List[int] = [] + + with start_server() as server: + with stream_content(make_put_event()) as stream: + server.for_path('/all', stream) + config = Config(sdk_key='sdk-key', stream_uri=server.uri) + sp = StreamingUpdateProcessor(config, store, Event(), None) + + # KeyboardInterrupt is not an Exception, so the loop cannot catch it. + def explode(*args, **kwargs): + raise KeyboardInterrupt() + + sp._process_message = explode # type: ignore[method-assign] + + real_create = sp._create_sse_client + + def create_and_watch_close(): + client = real_create() + real_close = client.close + + def close(): + closes.append(1) + real_close() + + client.close = close # type: ignore[method-assign] + return client + + sp._create_sse_client = create_and_watch_close # type: ignore[method-assign] + + with pytest.raises(KeyboardInterrupt): + sp.run() + + assert closes == [1] + + def _handle_errors_without_waiting(retry, errors): """Drives _handle_error for each error and returns nothing. The stop event is pre-set so the interruptible wait returns at once.""" diff --git a/ldclient/testing/impl/test_data_sink.py b/ldclient/testing/impl/test_data_sink.py index d905db78..fcdf00fd 100644 --- a/ldclient/testing/impl/test_data_sink.py +++ b/ldclient/testing/impl/test_data_sink.py @@ -1,3 +1,4 @@ +import time from typing import Callable, Dict import mock @@ -6,7 +7,11 @@ from ldclient.feature_store import InMemoryFeatureStore from ldclient.impl.datasource.status import DataSourceUpdateSinkImpl from ldclient.impl.listeners import Listeners -from ldclient.interfaces import DataSourceErrorKind, DataSourceState +from ldclient.interfaces import ( + DataSourceErrorInfo, + DataSourceErrorKind, + DataSourceState +) from ldclient.testing.builders import ( FlagBuilder, FlagRuleBuilder, @@ -82,6 +87,40 @@ def test_interrupting_initializing_stays_initializing(): assert sink.status.error is None +def test_off_is_terminal(): + spy = SpyListener() + status_listener = Listeners() + status_listener.add(spy) + + sink = DataSourceUpdateSinkImpl(InMemoryFeatureStore(), status_listener, Listeners()) + sink.update_status(DataSourceState.VALID, None) + sink.update_status(DataSourceState.OFF, None) + + # A poll or stream connection still in flight when the data source stopped. + sink.update_status(DataSourceState.VALID, None) + sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, time.time(), 'late')) + + assert sink.status.state == DataSourceState.OFF + assert sink.status.error is None + assert [status.state for status in spy.statuses] == [DataSourceState.VALID, DataSourceState.OFF] + + +@mock.patch('ldclient.feature_store.InMemoryFeatureStore.init', side_effect=[Exception('cannot init')]) +def test_store_error_after_off_reports_nothing(mock_init, prereq_data): + spy = SpyListener() + status_listener = Listeners() + status_listener.add(spy) + + sink = DataSourceUpdateSinkImpl(InMemoryFeatureStore(), status_listener, Listeners()) + sink.update_status(DataSourceState.OFF, None) + + with pytest.raises(Exception): + sink.init(prereq_data) + + assert sink.status.state == DataSourceState.OFF + assert [status.state for status in spy.statuses] == [DataSourceState.OFF] + + def test_listener_is_only_triggered_for_state_changes(): spy = SpyListener() status_listener = Listeners() From 7001f39070630b05b7cedcaa750a45d04599ec2d Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 24 Sep 2026 11:04:57 -0500 Subject: [PATCH 18/19] fix: Shorten the teardown comment and say why two statuses follow OFF Addresses TUI review feedback on 496830f. --- ldclient/impl/datasource/async_streaming.py | 5 ++--- ldclient/testing/impl/test_data_sink.py | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index 62da5c0f..421cba3e 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -175,9 +175,8 @@ async def stop(self): if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.OFF, None) - # Cancel the run task before the teardown awaits: otherwise, if stop() is called before - # _run has executed, the loop could run _run at the teardown await and create a fresh SSE - # connection against the session we're closing. Once the runner is stopped, teardown is safe. + # Cancel the run task before tearing down the rest: otherwise _run could + # start a fresh SSE connection against the session we are closing. await self._runner.stop_all() if self._sse: diff --git a/ldclient/testing/impl/test_data_sink.py b/ldclient/testing/impl/test_data_sink.py index fcdf00fd..bfd3deb2 100644 --- a/ldclient/testing/impl/test_data_sink.py +++ b/ldclient/testing/impl/test_data_sink.py @@ -97,6 +97,8 @@ def test_off_is_terminal(): sink.update_status(DataSourceState.OFF, None) # A poll or stream connection still in flight when the data source stopped. + # Two branches, not two of the states: a plain state change, and one carrying + # an error, which is what survives the same-state dedup. sink.update_status(DataSourceState.VALID, None) sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, time.time(), 'late')) From 816df7a88a32766161115777042faa00aaf2679a Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 24 Sep 2026 11:36:00 -0500 Subject: [PATCH 19/19] fix: Reword the teardown and latch test comments Addresses TUI review feedback on 7001f39. --- ldclient/impl/datasource/async_streaming.py | 4 ++-- ldclient/testing/impl/test_data_sink.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ldclient/impl/datasource/async_streaming.py b/ldclient/impl/datasource/async_streaming.py index 421cba3e..397697d5 100644 --- a/ldclient/impl/datasource/async_streaming.py +++ b/ldclient/impl/datasource/async_streaming.py @@ -175,8 +175,8 @@ async def stop(self): if self._data_source_update_sink is not None: self._data_source_update_sink.update_status(DataSourceState.OFF, None) - # Cancel the run task before tearing down the rest: otherwise _run could - # start a fresh SSE connection against the session we are closing. + # Cancel the run task before tearing down the rest, preventing _run from + # starting a fresh SSE connection against the session we are closing. await self._runner.stop_all() if self._sse: diff --git a/ldclient/testing/impl/test_data_sink.py b/ldclient/testing/impl/test_data_sink.py index bfd3deb2..1ec5d49e 100644 --- a/ldclient/testing/impl/test_data_sink.py +++ b/ldclient/testing/impl/test_data_sink.py @@ -97,8 +97,8 @@ def test_off_is_terminal(): sink.update_status(DataSourceState.OFF, None) # A poll or stream connection still in flight when the data source stopped. - # Two branches, not two of the states: a plain state change, and one carrying - # an error, which is what survives the same-state dedup. + # Test both a plain state change and one carrying an error, so neither can + # get through. sink.update_status(DataSourceState.VALID, None) sink.update_status(DataSourceState.INTERRUPTED, DataSourceErrorInfo(DataSourceErrorKind.NETWORK_ERROR, 0, time.time(), 'late'))