From c7be6d50783ec66254f33bcaf149570306f87f30 Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 1/8] fix: use a versioned User-Agent for all Overpass requests Build the User-Agent from the package version and repo URL and send it with both the interpreter request and the /api/status request, which also moves to https. Without it the status endpoint returns HTTP 406. --- osmnet/__init__.py | 5 +++-- osmnet/load.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/osmnet/__init__.py b/osmnet/__init__.py index 90eefdc..ba0d200 100644 --- a/osmnet/__init__.py +++ b/osmnet/__init__.py @@ -1,5 +1,6 @@ -from .load import * - __version__ = "0.1.7" version = __version__ + +# load.py reads __version__ for the User-Agent, so define it first +from .load import * # noqa: E402 diff --git a/osmnet/load.py b/osmnet/load.py index a01574a..3e1ddf4 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -23,8 +23,14 @@ import geopandas as gpd from osmnet import config +from osmnet import __version__ from osmnet.utils import log, great_circle_dist as gcd +# Overpass rejects requests without a descriptive User-Agent (HTTP 406) +USER_AGENT = 'osmnet/{} (+https://github.com/UDST/osmnet)'.format( + __version__) +OVERPASS_STATUS_URL = 'https://overpass-api.de/api/status' + def osm_filter(network_type): """ @@ -231,7 +237,7 @@ def overpass_request(data, pause_duration=None, timeout=180, start_time = time.time() log('Posting to {} with timeout={}, "{}"'.format(url, timeout, data)) headers = { - "User-Agent": f"osmnet" + "User-Agent": USER_AGENT } response = requests.post( url, @@ -297,13 +303,15 @@ def get_pause_duration(recursive_delay=5, default_duration=10): pause_duration : int """ try: - response = requests.get('http://overpass-api.de/api/status') + response = requests.get(OVERPASS_STATUS_URL, + headers={'User-Agent': USER_AGENT}, + timeout=30) status = response.text.split('\n')[3] status_first_token = status.split(' ')[0] except Exception: # if status endpoint cannot be reached or output parsed, log error # and return default duration - log('Unable to query http://overpass-api.de/api/status', + log('Unable to query {}'.format(OVERPASS_STATUS_URL), level=lg.ERROR) return default_duration From e09409aa2dac5f6c87bed2d55acd028585abacb9 Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 2/8] fix: parse Overpass status by content and cap retries The /api/status layout gained "Rate limit" and "Announced endpoint" lines, so reading line index 3 no longer finds the slot line. Find it by content instead. overpass_request() recursed forever on 429/504 and get_pause_duration() recursed forever while the server was busy. Both now loop with a max_retries keyword (default 5). overpass_request() raises OverpassRetryError when retries run out; get_pause_duration() falls back to default_duration. --- osmnet/load.py | 243 +++++++++++++++++++++++++++++++------------------ 1 file changed, 156 insertions(+), 87 deletions(-) diff --git a/osmnet/load.py b/osmnet/load.py index 3e1ddf4..3329d14 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -32,6 +32,14 @@ OVERPASS_STATUS_URL = 'https://overpass-api.de/api/status' +class OverpassRetryError(Exception): + """ + Raised when the Overpass API keeps returning 429/504 after the + maximum number of retries. + """ + pass + + def osm_filter(network_type): """ Create a filter to query Overpass API for the specified OSM network type. @@ -208,7 +216,7 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, def overpass_request(data, pause_duration=None, timeout=180, - error_pause_duration=None): + error_pause_duration=None, max_retries=5): """ Send a request to the Overpass API via HTTP POST and return the JSON response @@ -225,6 +233,9 @@ def overpass_request(data, pause_duration=None, timeout=180, the timeout interval for the requests library error_pause_duration : int how long to pause in seconds before re-trying requests if error + max_retries : int + how many times to re-try the request after a 429 or 504 response + before raising OverpassRetryError Returns ------- @@ -233,59 +244,118 @@ def overpass_request(data, pause_duration=None, timeout=180, # define the Overpass API URL, then construct a GET-style URL url = 'https://www.overpass-api.de/api/interpreter' - - start_time = time.time() - log('Posting to {} with timeout={}, "{}"'.format(url, timeout, data)) headers = { "User-Agent": USER_AGENT } - response = requests.post( - url, - data=data, - headers=headers, - timeout=timeout, - ) - - # get the response size and the domain, log result - size_kb = len(response.content) / 1000. - domain = re.findall(r'(?s)//(.*?)/', url)[0] - log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds' - .format(size_kb, domain, time.time()-start_time)) - - try: - response_json = response.json() - if 'remark' in response_json: - log('Server remark: "{}"'.format(response_json['remark'], - level=lg.WARNING)) - - except Exception: - # 429 = 'too many requests' and 504 = 'gateway timeout' from server - # overload. handle these errors by recursively - # calling overpass_request until a valid response is achieved - if response.status_code in [429, 504]: - # pause for error_pause_duration seconds before re-trying request - if error_pause_duration is None: - error_pause_duration = get_pause_duration() - log('Server at {} returned status code {} and no JSON data. ' - 'Re-trying request in {:.2f} seconds.' - .format(domain, response.status_code, error_pause_duration), - level=lg.WARNING) - time.sleep(error_pause_duration) - response_json = overpass_request(data=data, - pause_duration=pause_duration, - timeout=timeout) - - # else, this was an unhandled status_code, throw an exception - else: - log('Server at {} returned status code {} and no JSON data' - .format(domain, response.status_code), level=lg.ERROR) - raise Exception('Server returned no JSON data.\n{} {}\n{}' - .format(response, response.reason, response.text)) - - return response_json - - -def get_pause_duration(recursive_delay=5, default_duration=10): + + # 429 = 'too many requests' and 504 = 'gateway timeout' from server + # overload. handle these errors by re-trying the request, at most + # max_retries times + for attempt in range(max_retries + 1): + start_time = time.time() + log('Posting to {} with timeout={}, "{}"'.format(url, timeout, data)) + response = requests.post( + url, + data=data, + headers=headers, + timeout=timeout, + ) + + # get the response size and the domain, log result + size_kb = len(response.content) / 1000. + domain = re.findall(r'(?s)//(.*?)/', url)[0] + log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds' + .format(size_kb, domain, time.time()-start_time)) + + try: + response_json = response.json() + if 'remark' in response_json: + log('Server remark: "{}"'.format(response_json['remark'], + level=lg.WARNING)) + return response_json + + except Exception: + # else, this was an unhandled status_code, throw an exception + if response.status_code not in [429, 504]: + log('Server at {} returned status code {} and no JSON data' + .format(domain, response.status_code), level=lg.ERROR) + raise Exception('Server returned no JSON data.\n{} {}\n{}' + .format(response, response.reason, + response.text)) + + if attempt == max_retries: + break + + # pause for error_pause_duration seconds before re-trying request + pause = error_pause_duration + if pause is None: + pause = get_pause_duration() + log('Server at {} returned status code {} and no JSON data. ' + 'Re-trying request in {:.2f} seconds (retry {} of {}).' + .format(domain, response.status_code, pause, attempt + 1, + max_retries), + level=lg.WARNING) + time.sleep(pause) + + log('Server at {} returned status code {} after {} retries; giving up' + .format(domain, response.status_code, max_retries), level=lg.ERROR) + raise OverpassRetryError( + 'Overpass API at {} still returned status code {} after {} ' + 'retries.'.format(domain, response.status_code, max_retries)) + + +def _parse_overpass_status(text): + """ + Parse the text of the Overpass API /api/status page. + + The relevant line is found by its content, not its position, because the + page layout has changed over time (e.g. the 'Rate limit' and + 'Announced endpoint' lines). + + Parameters + ---------- + text : str + body of the /api/status response + + Returns + ------- + status : tuple + ('available', n_slots), ('slot', seconds until the next slot is + free), ('busy', None) if all slots are taken by running queries, or + (None, None) if the text is not recognized + """ + lines = [line.strip() for line in text.splitlines()] + + # "2 slots available now." + for line in lines: + match = re.match(r'^(\d+)\s+slots?\s+available', line) + if match: + return 'available', int(match.group(1)) + + # "Slot available after: 2026-10-04T12:20:01Z, in 107 seconds." + # there is one such line per busy slot; use the earliest + waits = [] + for line in lines: + if not line.startswith('Slot'): + continue + match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)', line) + if match: + utc_time = date_parser.parse(match.group(1)) + now = dt.datetime.now(dt.timezone.utc) + waits.append(math.ceil((utc_time - now).total_seconds())) + if waits: + return 'slot', max(min(waits), 1) + + # no free slot and no release time: "Currently running queries ..." + for line in lines: + if line.startswith('Currently'): + return 'busy', None + + return None, None + + +def get_pause_duration(recursive_delay=5, default_duration=10, + max_retries=5): """ Check the Overpass API status endpoint to determine how long to wait until next slot is available. @@ -293,56 +363,55 @@ def get_pause_duration(recursive_delay=5, default_duration=10): Parameters ---------- recursive_delay : int - how long to wait between recursive calls if server is currently + how long to wait between status checks if server is currently running a query default_duration : int if fatal error, function falls back on returning this value + max_retries : int + how many times to re-check the status while the server is busy + before falling back on default_duration Returns ------- pause_duration : int """ - try: - response = requests.get(OVERPASS_STATUS_URL, - headers={'User-Agent': USER_AGENT}, - timeout=30) - status = response.text.split('\n')[3] - status_first_token = status.split(' ')[0] - except Exception: - # if status endpoint cannot be reached or output parsed, log error - # and return default duration - log('Unable to query {}'.format(OVERPASS_STATUS_URL), - level=lg.ERROR) - return default_duration - - try: - # if first token is numeric, it indicates the number of slots - # available - no wait required - available_slots = int(status_first_token) - pause_duration = 0 - except Exception: - # if first token is 'Slot', it tells you when your slot will be free - if status_first_token == 'Slot': - utc_time_str = status.split(' ')[3] - utc_time = date_parser.parse(utc_time_str).replace(tzinfo=None) - pause_duration = math.ceil( - (utc_time - dt.datetime.utcnow()).total_seconds()) - pause_duration = max(pause_duration, 1) - - # if first token is 'Currently', it is currently running a query so - # check back in recursive_delay seconds - elif status_first_token == 'Currently': - time.sleep(recursive_delay) - pause_duration = get_pause_duration() + for attempt in range(max_retries + 1): + try: + response = requests.get(OVERPASS_STATUS_URL, + headers={'User-Agent': USER_AGENT}, + timeout=30) + response.raise_for_status() + kind, value = _parse_overpass_status(response.text) + except Exception: + # if status endpoint cannot be reached or output parsed, log + # error and return default duration + log('Unable to query {}'.format(OVERPASS_STATUS_URL), + level=lg.ERROR) + return default_duration + + # slots available - no wait required + if kind == 'available': + return 0 - else: + # the status tells you when your slot will be free + if kind == 'slot': + return value + + if kind is None: # any other status is unrecognized - log an error and return # default duration - log('Unrecognized server status: "{}"'.format(status), + log('Unrecognized server status: "{}"'.format(response.text), level=lg.ERROR) return default_duration - return pause_duration + # the server is currently running queries, so check back in + # recursive_delay seconds + if attempt < max_retries: + time.sleep(recursive_delay) + + log('Overpass server still busy after {} status checks; pausing {} ' + 'seconds'.format(max_retries, default_duration), level=lg.WARNING) + return default_duration def consolidate_subdivide_geometry(geometry, max_query_area_size): From 5a0fd83fec17fa061d2439897ba57ae246a24e70 Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 3/8] test: add offline tests for Overpass status parsing and retries Mock requests so the tests need no network. The status page text captured from overpass-api.de on 2026-10-04 is kept as a fixture. --- osmnet/tests/test_overpass.py | 237 ++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 osmnet/tests/test_overpass.py diff --git a/osmnet/tests/test_overpass.py b/osmnet/tests/test_overpass.py new file mode 100644 index 0000000..af8df12 --- /dev/null +++ b/osmnet/tests/test_overpass.py @@ -0,0 +1,237 @@ +import datetime as dt + +import pytest + +import osmnet +import osmnet.load as load + +# /api/status as served by overpass-api.de on 2026-10-04 (verbatim) +STATUS_LIVE_2026_10_04 = ( + 'Connected as: 843828827\n' + 'Current time: 2026-10-04T12:18:14Z\n' + 'Announced endpoint: gall.openstreetmap.de/\n' + 'Rate limit: 2\n' + '2 slots available now.\n' + 'Currently running queries (pid, space limit, time limit, start time):\n' +) + +# older layout, where the slot line was at index 3 +STATUS_OLD_LAYOUT = ( + 'Connected as: 1234567890\n' + 'Current time: 2020-07-13T18:00:00Z\n' + 'Rate limit: 2\n' + '1 slots available now.\n' + 'Currently running queries (pid, space limit, time limit, start time):\n' +) + +STATUS_SLOT_TEMPLATE = ( + 'Connected as: 843828827\n' + 'Current time: 2026-10-04T12:18:14Z\n' + 'Announced endpoint: gall.openstreetmap.de/\n' + 'Rate limit: 2\n' + 'Slot available after: {late}, in 120 seconds.\n' + 'Slot available after: {early}, in 30 seconds.\n' + 'Currently running queries (pid, space limit, time limit, start time):\n' +) + +STATUS_BUSY = ( + 'Connected as: 843828827\n' + 'Current time: 2026-10-04T12:18:14Z\n' + 'Announced endpoint: gall.openstreetmap.de/\n' + 'Rate limit: 2\n' + 'Currently running queries (pid, space limit, time limit, start time):\n' + '12345\t536870912\t180\t2026-10-04T12:17:50Z\n' + '12346\t536870912\t180\t2026-10-04T12:18:01Z\n' +) + +STATUS_406_HTML = ( + '\n' + '\n' + '406 Not Acceptable\n' + '\n' + '

Not Acceptable

\n' + '\n' +) + + +def _utc_in(seconds): + t = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=seconds) + return t.strftime('%Y-%m-%dT%H:%M:%SZ') + + +class FakeResponse(object): + def __init__(self, status_code=200, text='', json_data=None): + self.status_code = status_code + self.text = text + self.content = text.encode('utf-8') + self.reason = 'reason' + self._json_data = json_data + + def json(self): + if self._json_data is None: + raise ValueError('no JSON') + return self._json_data + + def raise_for_status(self): + if self.status_code >= 400: + raise Exception('HTTP {}'.format(self.status_code)) + + +@pytest.fixture +def no_sleep(monkeypatch): + sleeps = [] + monkeypatch.setattr(load.time, 'sleep', lambda s: sleeps.append(s)) + return sleeps + + +def test_user_agent(): + assert load.USER_AGENT == ( + 'osmnet/{} (+https://github.com/UDST/osmnet)' + .format(osmnet.__version__)) + + +def test_parse_status_live_layout(): + assert load._parse_overpass_status(STATUS_LIVE_2026_10_04) == \ + ('available', 2) + + +def test_parse_status_old_layout(): + assert load._parse_overpass_status(STATUS_OLD_LAYOUT) == ('available', 1) + + +def test_parse_status_slot_available_after(): + text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(120), early=_utc_in(30)) + kind, wait = load._parse_overpass_status(text) + assert kind == 'slot' + assert 25 <= wait <= 31 + + +def test_parse_status_slot_in_past(): + text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(-5), early=_utc_in(-10)) + assert load._parse_overpass_status(text) == ('slot', 1) + + +def test_parse_status_currently_running(): + assert load._parse_overpass_status(STATUS_BUSY) == ('busy', None) + + +def test_parse_status_unrecognized(): + assert load._parse_overpass_status(STATUS_406_HTML) == (None, None) + assert load._parse_overpass_status('') == (None, None) + + +def test_get_pause_duration_sends_user_agent(monkeypatch, no_sleep): + calls = [] + + def fake_get(url, **kwargs): + calls.append((url, kwargs)) + return FakeResponse(text=STATUS_LIVE_2026_10_04) + + monkeypatch.setattr(load.requests, 'get', fake_get) + assert load.get_pause_duration() == 0 + url, kwargs = calls[0] + assert url == 'https://overpass-api.de/api/status' + assert kwargs['headers']['User-Agent'] == load.USER_AGENT + + +def test_get_pause_duration_slot(monkeypatch, no_sleep): + text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(120), early=_utc_in(30)) + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse(text=text)) + assert 25 <= load.get_pause_duration() <= 31 + + +def test_get_pause_duration_406_falls_back(monkeypatch, no_sleep): + monkeypatch.setattr( + load.requests, 'get', + lambda url, **kw: FakeResponse(status_code=406, text=STATUS_406_HTML)) + assert load.get_pause_duration(default_duration=10) == 10 + + +def test_get_pause_duration_busy_then_free(monkeypatch, no_sleep): + pages = [STATUS_BUSY, STATUS_BUSY, STATUS_LIVE_2026_10_04] + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse(text=pages.pop(0))) + assert load.get_pause_duration(recursive_delay=5) == 0 + assert no_sleep == [5, 5] + + +def test_get_pause_duration_busy_is_bounded(monkeypatch, no_sleep): + calls = [] + + def fake_get(url, **kwargs): + calls.append(url) + return FakeResponse(text=STATUS_BUSY) + + monkeypatch.setattr(load.requests, 'get', fake_get) + assert load.get_pause_duration(recursive_delay=5, default_duration=10, + max_retries=3) == 10 + assert len(calls) == 4 + assert no_sleep == [5, 5, 5] + + +def test_overpass_request_sends_user_agent(monkeypatch, no_sleep): + calls = [] + + def fake_post(url, **kwargs): + calls.append(kwargs) + return FakeResponse(text='{}', json_data={'elements': []}) + + monkeypatch.setattr(load.requests, 'post', fake_post) + assert load.overpass_request(data={'data': 'q'}) == {'elements': []} + assert calls[0]['headers']['User-Agent'] == load.USER_AGENT + + +def test_overpass_request_retries_then_succeeds(monkeypatch, no_sleep): + responses = [FakeResponse(status_code=429), FakeResponse(status_code=504), + FakeResponse(text='{}', json_data={'elements': [1]})] + monkeypatch.setattr(load.requests, 'post', + lambda url, **kw: responses.pop(0)) + result = load.overpass_request(data={'data': 'q'}, + error_pause_duration=2) + assert result == {'elements': [1]} + assert no_sleep == [2, 2] + + +def test_overpass_request_retry_exhaustion(monkeypatch, no_sleep): + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + return FakeResponse(status_code=429) + + monkeypatch.setattr(load.requests, 'post', fake_post) + with pytest.raises(load.OverpassRetryError, match='429 after 3 retries'): + load.overpass_request(data={'data': 'q'}, error_pause_duration=1, + max_retries=3) + assert len(calls) == 4 + assert no_sleep == [1, 1, 1] + + +def test_overpass_request_default_max_retries(monkeypatch, no_sleep): + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + return FakeResponse(status_code=504) + + monkeypatch.setattr(load.requests, 'post', fake_post) + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse( + text=STATUS_LIVE_2026_10_04)) + with pytest.raises(load.OverpassRetryError): + load.overpass_request(data={'data': 'q'}) + assert len(calls) == 6 + + +def test_overpass_request_other_status_raises(monkeypatch, no_sleep): + calls = [] + + def fake_post(url, **kwargs): + calls.append(url) + return FakeResponse(status_code=400, text='bad query') + + monkeypatch.setattr(load.requests, 'post', fake_post) + with pytest.raises(Exception, match='Server returned no JSON data'): + load.overpass_request(data={'data': 'q'}) + assert len(calls) == 1 From 0e324580f6d241f578a4f209e32a078f1725b2f8 Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 4/8] test: skip live Overpass tests when OSMNET_SKIP_NETWORK_TESTS=1 --- osmnet/tests/test_load.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/osmnet/tests/test_load.py b/osmnet/tests/test_load.py index 4708f2b..01d2e30 100644 --- a/osmnet/tests/test_load.py +++ b/osmnet/tests/test_load.py @@ -1,3 +1,5 @@ +import os + import numpy.testing as npt import pytest from shapely.geometry import Polygon, MultiPolygon @@ -6,6 +8,12 @@ import osmnet.load as load +# tests that query the live Overpass API; set OSMNET_SKIP_NETWORK_TESTS=1 to +# skip them (as CI does) +requires_network = pytest.mark.skipif( + os.environ.get('OSMNET_SKIP_NETWORK_TESTS') == '1', + reason='queries the live Overpass API') + @pytest.fixture(scope='module') def bbox1(): @@ -84,6 +92,7 @@ def dataframes2(query_data2): return load.parse_network_osm_query(query_data2) +@requires_network def test_make_osm_query(query_data1): assert isinstance(query_data1, dict) assert len(query_data1['elements']) == 27 @@ -153,6 +162,7 @@ def test_process_way(): assert waynodes == expected_waynodes +@requires_network def test_parse_network_osm_query(dataframes1): nodes, ways, waynodes = dataframes1 @@ -161,6 +171,7 @@ def test_parse_network_osm_query(dataframes1): assert len(waynodes.index.unique()) == 4 +@requires_network def test_parse_network_osm_query_raises(): query_template = '[out:json][timeout:{timeout}]{maxsize};(way["highway"]' \ '{filters}({lat_min:.8f},{lng_max:.8f},{lat_max:.8f},' \ @@ -174,6 +185,7 @@ def test_parse_network_osm_query_raises(): load.parse_network_osm_query(data) +@requires_network def test_overpass_request_raises(bbox5): lat_min, lng_max, lat_max, lng_min = bbox5 query_template = '[out:json][timeout:{timeout}]{maxsize};(way["highway"]' \ @@ -187,6 +199,7 @@ def test_overpass_request_raises(bbox5): load.overpass_request(data={'data': query_str}) +@requires_network def test_get_pause_duration(): error_pause_duration = load.get_pause_duration(recursive_delay=5, default_duration=10) @@ -203,6 +216,7 @@ def test_quadrat_cut_geometry(simple_polygon): assert len(multipolygon.geoms) == 4 +@requires_network def test_ways_in_bbox(bbox1, dataframes1): lat_min, lng_max, lat_max, lng_min = bbox1 nodes, ways, waynodes = load.ways_in_bbox(lat_min=lat_min, lng_min=lng_min, @@ -215,6 +229,7 @@ def test_ways_in_bbox(bbox1, dataframes1): waynodes.equals(exp_waynodes) +@requires_network @pytest.mark.parametrize( 'network_type, noset', [('walk', {'motorway', 'motorway_link'}), @@ -229,6 +244,7 @@ def test_ways_in_bbox_walk_network(bbox3, network_type, noset): assert way['highway'] not in noset +@requires_network def test_intersection_nodes1(dataframes1): _, _, waynodes = dataframes1 intersections = load.intersection_nodes(waynodes) @@ -236,6 +252,7 @@ def test_intersection_nodes1(dataframes1): assert intersections == {53041093} +@requires_network def test_intersection_nodes2(dataframes2): _, _, waynodes = dataframes2 intersections = load.intersection_nodes(waynodes) @@ -245,6 +262,7 @@ def test_intersection_nodes2(dataframes2): 4279441432} +@requires_network def test_node_pairs_two_way(dataframes2): nodes, ways, waynodes = dataframes2 pairs = load.node_pairs(nodes, ways, waynodes) @@ -261,6 +279,7 @@ def test_node_pairs_two_way(dataframes2): npt.assert_allclose(pair.distance, 100.575284) +@requires_network def test_node_pairs_one_way(dataframes2): nodes, ways, waynodes = dataframes2 pairs = load.node_pairs(nodes, ways, waynodes, two_way=False) @@ -278,6 +297,7 @@ def test_node_pairs_one_way(dataframes2): npt.assert_allclose(pair.distance, 100.575284) +@requires_network def test_column_names(bbox4): nodes, edges = load.network_from_bbox( @@ -293,6 +313,7 @@ def test_column_names(bbox4): assert col in edges.columns +@requires_network def test_custom_query_pass(bbox5): nodes, edges = load.network_from_bbox( bbox=bbox5, custom_osm_filter='["highway"="service"]' From fa11a0c248dd23173d0d890bd1fbd0f6a892f877 Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 5/8] ci: add GitHub Actions workflow for Python 3.10-3.13 Runs the unit tests that do not need the network on pushes and pull requests to dev. --- .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4bb21bf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: [dev] + pull_request: + branches: [dev] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13'] + env: + # tests that query the live Overpass API are skipped + OSMNET_SKIP_NETWORK_TESTS: '1' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -e . pytest + pip list + - name: Test + run: pytest -v osmnet From 7fed665641878156289d092d50dd0cb9e5e518fb Mon Sep 17 00:00:00 2001 From: Paul Waddell Date: Sun, 4 Oct 2026 08:26:21 -0400 Subject: [PATCH 6/8] docs: add unreleased changelog entry --- CHANGELOG.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 45ecf0e..770c616 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,12 @@ +Unreleased +========== + +* sends a User-Agent header with Overpass API requests and uses https, fixing HTTP 406 errors (#33, thanks to Rhys Williams) +* uses a descriptive User-Agent, ``osmnet/ (+https://github.com/UDST/osmnet)``, for both the interpreter and status requests +* ``get_pause_duration()`` now queries the status page over https with the User-Agent and finds the slot line by content, so it tolerates the current page layout +* retries are bounded: ``overpass_request()`` gets a ``max_retries`` argument (default 5) and raises ``OverpassRetryError`` when 429/504 responses persist; ``get_pause_duration()`` gets a ``max_retries`` argument and falls back to ``default_duration`` while the server stays busy +* adds a GitHub Actions workflow running the offline unit tests on Python 3.10-3.13 + v0.1.7 ====== From 622d9df54c616cb5ef723a2c5ba34e376eb93b88 Mon Sep 17 00:00:00 2001 From: Samuel Maurer Date: Wed, 14 Oct 2026 11:33:12 -0700 Subject: [PATCH 7/8] fix: back off between Overpass retries, use one host, run CI on main Now that the status request succeeds, get_pause_duration() returns 0 whenever slots are free, including while the server is overloaded and returning 504s, so all retries fired back to back. Unless error_pause_duration is given, wait at least 5 seconds, doubling with each retry up to 60, or longer if the status page asks for it. Also send the interpreter request to overpass-api.de, the host the status request uses; log server remarks at WARNING (the level was passed to str.format instead of log); note that pause_duration is unused; and point CI at main, which becomes the integration branch. --- .github/workflows/ci.yml | 4 +-- CHANGELOG.rst | 3 ++ osmnet/load.py | 27 +++++++++++------ osmnet/tests/test_overpass.py | 56 +++++++++++++++++++++++++++++++++-- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bb21bf..63cb760 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [dev] + branches: [main] pull_request: - branches: [dev] + branches: [main] jobs: test: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 770c616..bb05bce 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ Unreleased * uses a descriptive User-Agent, ``osmnet/ (+https://github.com/UDST/osmnet)``, for both the interpreter and status requests * ``get_pause_duration()`` now queries the status page over https with the User-Agent and finds the slot line by content, so it tolerates the current page layout * retries are bounded: ``overpass_request()`` gets a ``max_retries`` argument (default 5) and raises ``OverpassRetryError`` when 429/504 responses persist; ``get_pause_duration()`` gets a ``max_retries`` argument and falls back to ``default_duration`` while the server stays busy +* unless ``error_pause_duration`` is given, retries wait at least 5 seconds, doubling up to 60, even when the status page reports free slots, so 504s from an overloaded server are not retried back to back +* the interpreter request goes to ``overpass-api.de``, the same host as the status request +* server remarks in an Overpass response are logged as warnings, as intended * adds a GitHub Actions workflow running the offline unit tests on Python 3.10-3.13 v0.1.7 diff --git a/osmnet/load.py b/osmnet/load.py index 3329d14..4c40d7a 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -29,8 +29,14 @@ # Overpass rejects requests without a descriptive User-Agent (HTTP 406) USER_AGENT = 'osmnet/{} (+https://github.com/UDST/osmnet)'.format( __version__) +OVERPASS_URL = 'https://overpass-api.de/api/interpreter' OVERPASS_STATUS_URL = 'https://overpass-api.de/api/status' +# minimum pause in seconds before re-trying after a 429 or 504; doubles with +# each retry up to the maximum +RETRY_MIN_PAUSE = 5 +RETRY_MAX_PAUSE = 60 + class OverpassRetryError(Exception): """ @@ -226,13 +232,14 @@ def overpass_request(data, pause_duration=None, timeout=180, data : dict or OrderedDict key-value pairs of parameters to post to Overpass API pause_duration : int - how long to pause in seconds before requests, if None, will query - Overpass API status endpoint - to find when next slot is available + not used; kept for backward compatibility timeout : int the timeout interval for the requests library error_pause_duration : int - how long to pause in seconds before re-trying requests if error + how long to pause in seconds before re-trying requests if error. if + None, the pause is the wait reported by the Overpass API status + endpoint, but at least RETRY_MIN_PAUSE seconds, doubling with each + retry up to RETRY_MAX_PAUSE max_retries : int how many times to re-try the request after a 429 or 504 response before raising OverpassRetryError @@ -242,8 +249,7 @@ def overpass_request(data, pause_duration=None, timeout=180, response_json : dict """ - # define the Overpass API URL, then construct a GET-style URL - url = 'https://www.overpass-api.de/api/interpreter' + url = OVERPASS_URL headers = { "User-Agent": USER_AGENT } @@ -270,8 +276,8 @@ def overpass_request(data, pause_duration=None, timeout=180, try: response_json = response.json() if 'remark' in response_json: - log('Server remark: "{}"'.format(response_json['remark'], - level=lg.WARNING)) + log('Server remark: "{}"'.format(response_json['remark']), + level=lg.WARNING) return response_json except Exception: @@ -289,7 +295,10 @@ def overpass_request(data, pause_duration=None, timeout=180, # pause for error_pause_duration seconds before re-trying request pause = error_pause_duration if pause is None: - pause = get_pause_duration() + # the status endpoint reports free slots even while the server + # is overloaded and returning 504s, so back off regardless + min_pause = min(RETRY_MIN_PAUSE * 2 ** attempt, RETRY_MAX_PAUSE) + pause = max(get_pause_duration(), min_pause) log('Server at {} returned status code {} and no JSON data. ' 'Re-trying request in {:.2f} seconds (retry {} of {}).' .format(domain, response.status_code, pause, attempt + 1, diff --git a/osmnet/tests/test_overpass.py b/osmnet/tests/test_overpass.py index af8df12..1ba239a 100644 --- a/osmnet/tests/test_overpass.py +++ b/osmnet/tests/test_overpass.py @@ -174,12 +174,26 @@ def test_overpass_request_sends_user_agent(monkeypatch, no_sleep): calls = [] def fake_post(url, **kwargs): - calls.append(kwargs) + calls.append((url, kwargs)) return FakeResponse(text='{}', json_data={'elements': []}) monkeypatch.setattr(load.requests, 'post', fake_post) assert load.overpass_request(data={'data': 'q'}) == {'elements': []} - assert calls[0]['headers']['User-Agent'] == load.USER_AGENT + url, kwargs = calls[0] + assert url == 'https://overpass-api.de/api/interpreter' + assert kwargs['headers']['User-Agent'] == load.USER_AGENT + + +def test_overpass_request_logs_remark_as_warning(monkeypatch, caplog): + remark = 'runtime error: Query timed out' + monkeypatch.setattr( + load.requests, 'post', + lambda url, **kw: FakeResponse( + text='{}', json_data={'elements': [], 'remark': remark})) + with caplog.at_level('WARNING', logger='osmnet'): + load.overpass_request(data={'data': 'q'}) + assert any(remark in r.getMessage() and r.levelname == 'WARNING' + for r in caplog.records) def test_overpass_request_retries_then_succeeds(monkeypatch, no_sleep): @@ -208,6 +222,44 @@ def fake_post(url, **kwargs): assert no_sleep == [1, 1, 1] +def test_overpass_request_504_backs_off_when_slots_free(monkeypatch, + no_sleep): + # the status page says slots are free, but the server is overloaded + responses = [FakeResponse(status_code=504) for _ in range(3)] + responses.append(FakeResponse(text='{}', json_data={'elements': [1]})) + monkeypatch.setattr(load.requests, 'post', + lambda url, **kw: responses.pop(0)) + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse( + text=STATUS_LIVE_2026_10_04)) + assert load.overpass_request(data={'data': 'q'}) == {'elements': [1]} + assert no_sleep == [5, 10, 20] + + +def test_overpass_request_backoff_is_capped(monkeypatch, no_sleep): + monkeypatch.setattr(load.requests, 'post', + lambda url, **kw: FakeResponse(status_code=504)) + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse( + text=STATUS_LIVE_2026_10_04)) + with pytest.raises(load.OverpassRetryError): + load.overpass_request(data={'data': 'q'}, max_retries=6) + assert no_sleep == [5, 10, 20, 40, 60, 60] + + +def test_overpass_request_uses_longer_status_wait(monkeypatch, no_sleep): + text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(120), early=_utc_in(30)) + responses = [FakeResponse(status_code=429), + FakeResponse(text='{}', json_data={'elements': [1]})] + monkeypatch.setattr(load.requests, 'post', + lambda url, **kw: responses.pop(0)) + monkeypatch.setattr(load.requests, 'get', + lambda url, **kw: FakeResponse(text=text)) + assert load.overpass_request(data={'data': 'q'}) == {'elements': [1]} + assert len(no_sleep) == 1 + assert 25 <= no_sleep[0] <= 31 + + def test_overpass_request_default_max_retries(monkeypatch, no_sleep): calls = [] From aa8d3f3c6b2381eabfb5e6153fc2efb6317767fe Mon Sep 17 00:00:00 2001 From: Samuel Maurer Date: Wed, 14 Oct 2026 12:32:05 -0700 Subject: [PATCH 8/8] fix: address review of the Overpass retry changes - osm_net_download(), ways_in_bbox() and network_from_bbox() take max_retries and pass it on to overpass_request(). One request that runs out of retries fails the whole download, so callers with large bounding boxes need a way to allow more. - Read the wait for the next slot from the status line's "in N seconds", which does not depend on the local clock, and fall back on the timestamp (parsed with strptime). This drops the dateutil import, which was never a declared dependency. - Treat "0 slots available" as no free slot. - Count status checks correctly in the give-up log message, and log each busy status check, so the time spent polling shows in the log. - Raise ValueError for a negative max_retries instead of failing with UnboundLocalError, and work out the host once per request. --- CHANGELOG.rst | 2 + osmnet/load.py | 59 +++++++++++++++++++++------ osmnet/tests/test_overpass.py | 75 +++++++++++++++++++++++++++++++---- 3 files changed, 116 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb05bce..9fe9ce0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ Unreleased * ``get_pause_duration()`` now queries the status page over https with the User-Agent and finds the slot line by content, so it tolerates the current page layout * retries are bounded: ``overpass_request()`` gets a ``max_retries`` argument (default 5) and raises ``OverpassRetryError`` when 429/504 responses persist; ``get_pause_duration()`` gets a ``max_retries`` argument and falls back to ``default_duration`` while the server stays busy * unless ``error_pause_duration`` is given, retries wait at least 5 seconds, doubling up to 60, even when the status page reports free slots, so 504s from an overloaded server are not retried back to back +* ``osm_net_download()``, ``ways_in_bbox()`` and ``network_from_bbox()`` accept ``max_retries`` and pass it on to each Overpass request, so large downloads can allow more retries +* the wait for the next free slot is read from the status page's "in N seconds" rather than computed from its timestamp and the local clock; ``python-dateutil`` is no longer imported * the interpreter request goes to ``overpass-api.de``, the same host as the status request * server remarks in an Overpass response are logged as warnings, as intended * adds a GitHub Actions workflow running the offline unit tests on Python 3.10-3.13 diff --git a/osmnet/load.py b/osmnet/load.py index 4c40d7a..d388547 100644 --- a/osmnet/load.py +++ b/osmnet/load.py @@ -18,7 +18,6 @@ import numpy as np from shapely.geometry import LineString, Polygon, MultiPolygon from shapely.ops import unary_union -from dateutil import parser as date_parser import datetime as dt import geopandas as gpd @@ -90,7 +89,7 @@ def osm_filter(network_type): def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, network_type='walk', timeout=180, memory=None, max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): + custom_osm_filter=None, max_retries=5): """ Download OSM ways and nodes within a bounding box from the Overpass API. @@ -123,6 +122,12 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' + max_retries : int, optional + how many times to re-try each Overpass request after a 429 or 504 + response before raising OverpassRetryError. Retries wait at least + RETRY_MIN_PAUSE seconds, doubling up to RETRY_MAX_PAUSE, so the + default of 5 gives up after about two and a half minutes. Raise it + for large bounding boxes that need many requests Returns ------- @@ -182,7 +187,8 @@ def osm_net_download(lat_min=None, lng_min=None, lat_max=None, lng_max=None, filters=request_filter, timeout=timeout, maxsize=maxsize) response_json = overpass_request(data={'data': query_str}, - timeout=timeout) + timeout=timeout, + max_retries=max_retries) response_jsons_list.append(response_json) @@ -249,7 +255,11 @@ def overpass_request(data, pause_duration=None, timeout=180, response_json : dict """ + if max_retries < 0: + raise ValueError('max_retries must be 0 or greater') + url = OVERPASS_URL + domain = re.findall(r'(?s)//(.*?)/', url)[0] headers = { "User-Agent": USER_AGENT } @@ -267,9 +277,8 @@ def overpass_request(data, pause_duration=None, timeout=180, timeout=timeout, ) - # get the response size and the domain, log result + # get the response size, log result size_kb = len(response.content) / 1000. - domain = re.findall(r'(?s)//(.*?)/', url)[0] log('Downloaded {:,.1f}KB from {} in {:,.2f} seconds' .format(size_kb, domain, time.time()-start_time)) @@ -338,18 +347,26 @@ def _parse_overpass_status(text): # "2 slots available now." for line in lines: match = re.match(r'^(\d+)\s+slots?\s+available', line) - if match: + if match and int(match.group(1)) > 0: return 'available', int(match.group(1)) # "Slot available after: 2026-10-04T12:20:01Z, in 107 seconds." - # there is one such line per busy slot; use the earliest + # there is one such line per busy slot; use the earliest. prefer the + # server's own count of seconds, which does not depend on the local + # clock, and fall back on the timestamp waits = [] for line in lines: if not line.startswith('Slot'): continue + match = re.search(r'in (-?\d+) seconds?', line) + if match: + waits.append(int(match.group(1))) + continue match = re.search(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)', line) if match: - utc_time = date_parser.parse(match.group(1)) + utc_time = dt.datetime.strptime( + match.group(1), '%Y-%m-%dT%H:%M:%SZ').replace( + tzinfo=dt.timezone.utc) now = dt.datetime.now(dt.timezone.utc) waits.append(math.ceil((utc_time - now).total_seconds())) if waits: @@ -416,10 +433,13 @@ def get_pause_duration(recursive_delay=5, default_duration=10, # the server is currently running queries, so check back in # recursive_delay seconds if attempt < max_retries: + log('Overpass server is running queries; checking its status ' + 'again in {} seconds'.format(recursive_delay)) time.sleep(recursive_delay) log('Overpass server still busy after {} status checks; pausing {} ' - 'seconds'.format(max_retries, default_duration), level=lg.WARNING) + 'seconds'.format(max_retries + 1, default_duration), + level=lg.WARNING) return default_duration @@ -700,7 +720,7 @@ def parse_network_osm_query(data): def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, timeout=180, memory=None, max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): + custom_osm_filter=None, max_retries=5): """ Get DataFrames of OSM data in a bounding box. @@ -733,6 +753,12 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' + max_retries : int, optional + how many times to re-try each Overpass request after a 429 or 504 + response before raising OverpassRetryError. Retries wait at least + RETRY_MIN_PAUSE seconds, doubling up to RETRY_MAX_PAUSE, so the + default of 5 gives up after about two and a half minutes. Raise it + for large bounding boxes that need many requests Returns ------- @@ -744,7 +770,8 @@ def ways_in_bbox(lat_min, lng_min, lat_max, lng_max, network_type, lng_max=lng_max, network_type=network_type, timeout=timeout, memory=memory, max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter)) + custom_osm_filter=custom_osm_filter, + max_retries=max_retries)) def intersection_nodes(waynodes): @@ -857,7 +884,7 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, bbox=None, network_type='walk', two_way=True, timeout=180, memory=None, max_query_area_size=50*1000*50*1000, - custom_osm_filter=None): + custom_osm_filter=None, max_retries=5): """ Make a graph network from a bounding lat/lon box composed of nodes and edges for use in Pandana street network accessibility calculations. @@ -909,6 +936,12 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, follow Overpass API schema. For example to request highway ways that are service roads use: '["highway"="service"]' + max_retries : int, optional + how many times to re-try each Overpass request after a 429 or 504 + response before raising OverpassRetryError. Retries wait at least + RETRY_MIN_PAUSE seconds, doubling up to RETRY_MAX_PAUSE, so the + default of 5 gives up after about two and a half minutes. Raise it + for large bounding boxes that need many requests Returns ------- @@ -940,7 +973,7 @@ def network_from_bbox(lat_min=None, lng_min=None, lat_max=None, lng_max=None, lat_min=lat_min, lng_min=lng_min, lat_max=lat_max, lng_max=lng_max, network_type=network_type, timeout=timeout, memory=memory, max_query_area_size=max_query_area_size, - custom_osm_filter=custom_osm_filter) + custom_osm_filter=custom_osm_filter, max_retries=max_retries) log('Returning OSM data with {:,} nodes and {:,} ways...' .format(len(nodes), len(ways))) diff --git a/osmnet/tests/test_overpass.py b/osmnet/tests/test_overpass.py index 1ba239a..0a725db 100644 --- a/osmnet/tests/test_overpass.py +++ b/osmnet/tests/test_overpass.py @@ -101,16 +101,42 @@ def test_parse_status_old_layout(): def test_parse_status_slot_available_after(): text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(120), early=_utc_in(30)) + assert load._parse_overpass_status(text) == ('slot', 30) + + +def test_parse_status_slot_ignores_local_clock(): + # timestamps an hour off, as with a skewed local clock: the server's + # "in N seconds" still gives the wait + text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(3720), + early=_utc_in(3630)) + assert load._parse_overpass_status(text) == ('slot', 30) + + +def test_parse_status_slot_timestamp_only(): + text = ('Rate limit: 2\n' + 'Slot available after: {}.\n' + 'Slot available after: {}.\n').format(_utc_in(120), _utc_in(30)) kind, wait = load._parse_overpass_status(text) assert kind == 'slot' assert 25 <= wait <= 31 def test_parse_status_slot_in_past(): - text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(-5), early=_utc_in(-10)) + text = ('Rate limit: 2\n' + 'Slot available after: {}, in 0 seconds.\n' + 'Slot available after: {}.\n').format(_utc_in(-5), _utc_in(-10)) assert load._parse_overpass_status(text) == ('slot', 1) +def test_parse_status_zero_slots_available(): + text = ('Rate limit: 2\n' + '0 slots available now.\n' + 'Slot available after: {}, in 30 seconds.\n' + 'Currently running queries (pid, space limit, time limit, ' + 'start time):\n').format(_utc_in(30)) + assert load._parse_overpass_status(text) == ('slot', 30) + + def test_parse_status_currently_running(): assert load._parse_overpass_status(STATUS_BUSY) == ('busy', None) @@ -138,7 +164,7 @@ def test_get_pause_duration_slot(monkeypatch, no_sleep): text = STATUS_SLOT_TEMPLATE.format(late=_utc_in(120), early=_utc_in(30)) monkeypatch.setattr(load.requests, 'get', lambda url, **kw: FakeResponse(text=text)) - assert 25 <= load.get_pause_duration() <= 31 + assert load.get_pause_duration() == 30 def test_get_pause_duration_406_falls_back(monkeypatch, no_sleep): @@ -156,7 +182,7 @@ def test_get_pause_duration_busy_then_free(monkeypatch, no_sleep): assert no_sleep == [5, 5] -def test_get_pause_duration_busy_is_bounded(monkeypatch, no_sleep): +def test_get_pause_duration_busy_is_bounded(monkeypatch, no_sleep, caplog): calls = [] def fake_get(url, **kwargs): @@ -164,10 +190,14 @@ def fake_get(url, **kwargs): return FakeResponse(text=STATUS_BUSY) monkeypatch.setattr(load.requests, 'get', fake_get) - assert load.get_pause_duration(recursive_delay=5, default_duration=10, - max_retries=3) == 10 + with caplog.at_level('WARNING', logger='osmnet'): + assert load.get_pause_duration(recursive_delay=5, + default_duration=10, + max_retries=3) == 10 assert len(calls) == 4 assert no_sleep == [5, 5, 5] + assert any('after 4 status checks' in r.getMessage() + for r in caplog.records) def test_overpass_request_sends_user_agent(monkeypatch, no_sleep): @@ -256,8 +286,7 @@ def test_overpass_request_uses_longer_status_wait(monkeypatch, no_sleep): monkeypatch.setattr(load.requests, 'get', lambda url, **kw: FakeResponse(text=text)) assert load.overpass_request(data={'data': 'q'}) == {'elements': [1]} - assert len(no_sleep) == 1 - assert 25 <= no_sleep[0] <= 31 + assert no_sleep == [30] def test_overpass_request_default_max_retries(monkeypatch, no_sleep): @@ -287,3 +316,35 @@ def fake_post(url, **kwargs): with pytest.raises(Exception, match='Server returned no JSON data'): load.overpass_request(data={'data': 'q'}) assert len(calls) == 1 + + +def test_overpass_request_negative_max_retries(): + with pytest.raises(ValueError, match='max_retries'): + load.overpass_request(data={'data': 'q'}, max_retries=-1) + + +def test_osm_net_download_passes_max_retries(monkeypatch): + calls = [] + + def fake_overpass_request(data, **kwargs): + calls.append(kwargs) + return {'elements': [{'type': 'node', 'id': 1}]} + + monkeypatch.setattr(load, 'overpass_request', fake_overpass_request) + load.osm_net_download(lat_min=37.8, lng_min=-122.26, lat_max=37.81, + lng_max=-122.27, max_retries=9) + assert calls and all(kw['max_retries'] == 9 for kw in calls) + + +def test_network_from_bbox_passes_max_retries(monkeypatch): + calls = [] + + def fake_osm_net_download(**kwargs): + calls.append(kwargs) + raise RuntimeError('stop here') + + monkeypatch.setattr(load, 'osm_net_download', fake_osm_net_download) + with pytest.raises(RuntimeError, match='stop here'): + load.network_from_bbox(bbox=(-122.27, 37.8, -122.26, 37.81), + max_retries=9) + assert calls[0]['max_retries'] == 9