diff --git a/misc/bazel/internal/BUILD.bazel b/misc/bazel/internal/BUILD.bazel index d9663e7f0c01..30a41b72c652 100644 --- a/misc/bazel/internal/BUILD.bazel +++ b/misc/bazel/internal/BUILD.bazel @@ -1 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_test") + exports_files(["install.py"]) + +py_test( + name = "test_git_lfs_probe", + size = "small", + srcs = ["test_git_lfs_probe.py"], + data = ["git_lfs_probe.py"], +) diff --git a/misc/bazel/internal/git_lfs_probe.py b/misc/bazel/internal/git_lfs_probe.py index 57f0fd5b8bb0..9af08f4f1700 100755 --- a/misc/bazel/internal/git_lfs_probe.py +++ b/misc/bazel/internal/git_lfs_probe.py @@ -6,8 +6,13 @@ * "local", if the source file is not an LFS pointer * the sha256 hash, a space character and a transient download link obtained via the LFS protocol otherwise If --hash-only is provided, the transient URL will not be fetched and printed + +Batch requests make up to three attempts per endpoint for transient failures, with +exponential backoff. Retry-After is honored unless it exceeds 30 seconds, in +which case the next endpoint is tried instead of retrying early. """ import dataclasses +import email.utils import sys import pathlib import subprocess @@ -21,7 +26,9 @@ import re import base64 from dataclasses import dataclass +from http.client import IncompleteRead import argparse +import time def options(): p = argparse.ArgumentParser(description=__doc__) @@ -31,6 +38,8 @@ def options(): TIMEOUT = 20 +MAX_ATTEMPTS = 3 +MAX_RETRY_DELAY = 30 def warn(message: str) -> None: print(f"WARNING: {message}", file=sys.stderr) @@ -153,6 +162,57 @@ def get_endpoints() -> typing.Iterable[Endpoint]: yield endpoint +def retry_delay(error: Exception, attempt: int) -> typing.Optional[float]: + delay = 2 ** attempt + if isinstance(error, urllib.error.HTTPError): + if error.code not in (408, 429, 500, 502, 503, 504): + return None + retry_after = error.headers.get("Retry-After") if error.headers else None + if retry_after: + retry_after = retry_after.strip() + try: + if retry_after.isdigit(): + seconds = int(retry_after) + else: + seconds = email.utils.parsedate_to_datetime(retry_after).timestamp() - time.time() + delay = max(delay, seconds) + except (TypeError, ValueError, OverflowError): + # Invalid Retry-After values fall back to exponential backoff. + pass + else: + reason = error.reason if isinstance(error, urllib.error.URLError) else error + if not isinstance(reason, (TimeoutError, ConnectionError, IncompleteRead)): + return None + return delay if delay <= MAX_RETRY_DELAY else None + + +def describe_error(error: Exception) -> str: + # Exception messages and response bodies can contain credentials or signed URLs. + if isinstance(error, urllib.error.HTTPError): + return f"HTTP {error.code}" + if isinstance(error, urllib.error.URLError): + return f"URLError ({type(error.reason).__name__})" + return type(error).__name__ + + +def request_batch(req, endpoint_name): + for attempt in range(MAX_ATTEMPTS): + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + return json.load(resp) + except (urllib.error.URLError, TimeoutError, ConnectionError, IncompleteRead) as error: + delay = retry_delay(error, attempt) + if isinstance(error, urllib.error.HTTPError): + error.close() + if delay is None or attempt == MAX_ATTEMPTS - 1: + raise + warn( + f"encountered {describe_error(error)} at endpoint {endpoint_name}, " + f"retrying in {delay:g}s (attempt {attempt + 2}/{MAX_ATTEMPTS})" + ) + time.sleep(delay) + + # see https://github.com/git-lfs/git-lfs/blob/310d1b4a7d01e8d9d884447df4635c7a9c7642c2/docs/api/basic-transfers.md def get_locations(objects): ret = ["local" for _ in objects] @@ -177,19 +237,18 @@ def get_locations(objects): data=json.dumps(data).encode("ascii"), ) try: - with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: - data = json.load(resp) - assert len(data["objects"]) == len( + response = request_batch(req, endpoint.name) + assert len(response["objects"]) == len( indexes - ), f"received {len(data)} objects, expected {len(indexes)}" - for i, resp in zip(indexes, data["objects"]): + ), f"received {len(response['objects'])} objects, expected {len(indexes)}" + for i, resp in zip(indexes, response["objects"]): ret[i] = f'{resp["oid"]} {resp["actions"]["download"]["href"]}' return ret - except urllib.error.URLError as e: - warn(f"encountered {type(e).__name__} {e}, ignoring endpoint {endpoint.name}") + except (urllib.error.URLError, TimeoutError, ConnectionError, IncompleteRead) as e: + warn(f"encountered {describe_error(e)}, ignoring endpoint {endpoint.name}") continue except KeyError: - warn(f"encountered malformed response, ignoring endpoint {endpoint.name}:\n{json.dumps(data, indent=2)}") + warn(f"encountered malformed response, ignoring endpoint {endpoint.name}") continue raise NoEndpointsFound diff --git a/misc/bazel/internal/test_git_lfs_probe.py b/misc/bazel/internal/test_git_lfs_probe.py new file mode 100644 index 000000000000..dd28d2d7c7a1 --- /dev/null +++ b/misc/bazel/internal/test_git_lfs_probe.py @@ -0,0 +1,209 @@ +import contextlib +import http.client +import io +import json +import os +import pathlib +import runpy +import ssl +import subprocess +import tempfile +import unittest +import urllib.error +from unittest import mock + + +SCRIPT = pathlib.Path(__file__).with_name("git_lfs_probe.py") +ENDPOINT = "https://lfs.example.test/repo/info/lfs" +BACKUP = "https://backup.example.test/repo/info/lfs" +OID = "a" * 64 +DOWNLOAD = "https://downloads.example.test/content?signature=test-signature" + + +def response(): + return io.BytesIO(json.dumps({ + "objects": [{"oid": OID, "actions": {"download": {"href": DOWNLOAD}}}], + }).encode()) + + +def truncated_response(): + sock = mock.Mock() + sock.makefile.return_value = io.BytesIO( + b'HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{"objects": [' + ) + resp = http.client.HTTPResponse(sock) + resp.begin() + return resp + + +def http_error(code, retry_after=None): + headers = {} if retry_after is None else {"Retry-After": retry_after} + return urllib.error.HTTPError( + ENDPOINT + "/objects/batch", code, "test-secret", headers, io.BytesIO() + ) + + +class BatchRequestTest(unittest.TestCase): + def setUp(self): + self.stack = contextlib.ExitStack() + self.addCleanup(self.stack.close) + root = pathlib.Path(self.stack.enter_context(tempfile.TemporaryDirectory())) + pointer = root / "pointer" + pointer.write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{OID}\nsize 42\n" + ) + local = root / "local" + local.write_text("not an LFS pointer\n") + self.stack.enter_context(mock.patch("sys.argv", [str(SCRIPT), str(local), str(pointer)])) + self.stack.enter_context(mock.patch.dict(os.environ, {"GITHUB_TOKEN": ""})) + self.stack.enter_context(mock.patch( + "subprocess.check_output", + side_effect=[ + str(root), + f"Endpoint={ENDPOINT} (auth=none)\nEndpoint (backup)={BACKUP} (auth=none)\n", + ], + )) + self.stack.enter_context(mock.patch( + "subprocess.run", + return_value=subprocess.CompletedProcess( + [], 0, stdout="Authorization: Basic test-secret" + ), + )) + self.urlopen = self.stack.enter_context(mock.patch("urllib.request.urlopen")) + self.sleep = self.stack.enter_context(mock.patch("time.sleep")) + self.stack.enter_context(mock.patch("time.time", return_value=1445412480)) + self.stdout = self.stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + self.stderr = self.stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + + def run_probe(self, outcomes): + self.urlopen.side_effect = outcomes + runpy.run_path(str(SCRIPT), run_name="__main__") + + def assert_requests(self, endpoints): + self.assertEqual(self.urlopen.call_count, len(endpoints)) + for call, endpoint in zip(self.urlopen.call_args_list, endpoints): + request = call.args[0] + self.assertEqual(request.full_url, endpoint + "/objects/batch") + self.assertEqual(request.get_method(), "POST") + self.assertEqual(request.get_header("Authorization"), "Basic test-secret") + self.assertEqual(request.get_header("Content-type"), "application/vnd.git-lfs+json") + self.assertEqual(json.loads(request.data), { + "operation": "download", + "transfers": ["basic"], + "objects": [{"oid": OID, "size": 42}], + "hash_algo": "sha256", + }) + self.assertNotIn("test-secret", self.stderr.getvalue()) + self.assertNotIn("test-signature", self.stderr.getvalue()) + + def test_disconnect_and_read_timeout_retry_same_request(self): + partial_response = mock.MagicMock() + partial_response.__enter__.return_value = partial_response + partial_response.read.side_effect = TimeoutError("test-secret") + self.run_probe([ + http.client.RemoteDisconnected("test-secret"), + partial_response, + response(), + ]) + self.assert_requests([ENDPOINT] * 3) + self.assertEqual(self.sleep.call_args_list, [mock.call(1), mock.call(2)]) + partial_response.__exit__.assert_called_once() + self.assertEqual(self.stdout.getvalue(), f"local\n{OID} {DOWNLOAD}\n") + + def test_truncated_body_retries_same_request(self): + self.run_probe([truncated_response(), response()]) + self.assert_requests([ENDPOINT] * 2) + self.sleep.assert_called_once_with(1) + self.assertEqual(self.stdout.getvalue(), f"local\n{OID} {DOWNLOAD}\n") + + def test_wrapped_timeout_retries_before_fallback(self): + self.run_probe([ + urllib.error.URLError(TimeoutError("test-secret")), + response(), + ]) + self.assert_requests([ENDPOINT] * 2) + self.sleep.assert_called_once_with(1) + + def test_rate_limit_retry_after(self): + cases = [ + ("7", 7), + ("30", 30), + ("Wed, 21 Oct 2015 07:28:09 GMT", 9), + ("Wed, 21 Oct 2015 07:27:00 GMT", 1), + ("invalid", 1), + (None, 1), + ] + for header, delay in cases: + with self.subTest(header=header): + self.urlopen.reset_mock() + self.sleep.reset_mock() + with mock.patch("subprocess.check_output", side_effect=[ + str(SCRIPT.parent), + f"Endpoint={ENDPOINT} (auth=none)\n", + ]): + error = http_error(429, header) + body = error.fp + self.run_probe([error, response()]) + self.assert_requests([ENDPOINT] * 2) + self.sleep.assert_called_once_with(delay) + self.assertTrue(body.closed) + + def test_long_retry_after_skips_endpoint_without_sleeping(self): + self.run_probe([http_error(429, "31"), response()]) + self.assert_requests([ENDPOINT, BACKUP]) + self.sleep.assert_not_called() + self.assertIn("429", self.stderr.getvalue()) + + def test_transient_errors_exhaust_then_fall_back(self): + for failures in [ + [http_error(503), http_error(502), http_error(504)], + [truncated_response() for _ in range(3)], + ]: + with self.subTest(response_type=type(failures[0]).__name__): + self.urlopen.reset_mock() + self.sleep.reset_mock() + with mock.patch("subprocess.check_output", side_effect=[ + str(SCRIPT.parent), + f"Endpoint={ENDPOINT} (auth=none)\nEndpoint (backup)={BACKUP} (auth=none)\n", + ]): + self.run_probe(failures + [response()]) + self.assert_requests([ENDPOINT] * 3 + [BACKUP]) + self.assertEqual(self.sleep.call_args_list, [mock.call(1), mock.call(2)]) + + def test_all_endpoints_exhausted_reports_failure(self): + with self.assertRaises(SystemExit) as raised: + self.run_probe([TimeoutError("test-secret") for _ in range(6)]) + self.assertEqual(raised.exception.code, 1) + self.assert_requests([ENDPOINT] * 3 + [BACKUP] * 3) + self.assertEqual( + self.sleep.call_args_list, + [mock.call(1), mock.call(2), mock.call(1), mock.call(2)], + ) + self.assertIn("ERROR: no valid endpoints found", self.stderr.getvalue()) + self.assertEqual(self.stdout.getvalue(), "") + + def test_permanent_errors_are_not_retried(self): + with self.assertRaises(SystemExit) as raised: + self.run_probe([ + http_error(403), + urllib.error.URLError(ssl.SSLCertVerificationError("test-secret")), + ]) + self.assertEqual(raised.exception.code, 1) + self.assert_requests([ENDPOINT, BACKUP]) + self.sleep.assert_not_called() + self.assertIn("403", self.stderr.getvalue()) + self.assertIn("SSLCertVerificationError", self.stderr.getvalue()) + + def test_malformed_response_fallback_preserves_batch_payload(self): + malformed = io.BytesIO(json.dumps({ + "objects": [{"oid": OID, "actions": {"upload": {"href": DOWNLOAD}}}], + }).encode()) + self.run_probe([malformed, response()]) + self.assert_requests([ENDPOINT, BACKUP]) + self.sleep.assert_not_called() + self.assertEqual(self.stdout.getvalue(), f"local\n{OID} {DOWNLOAD}\n") + + +if __name__ == "__main__": + unittest.main()