| 1 | package edu.ucsb.cs.scaffold.services; | |
| 2 | ||
| 3 | import edu.ucsb.cs.scaffold.utilities.Sleep; | |
| 4 | import java.util.concurrent.atomic.AtomicLong; | |
| 5 | import java.util.function.LongSupplier; | |
| 6 | import java.util.function.Supplier; | |
| 7 | import lombok.extern.slf4j.Slf4j; | |
| 8 | import org.springframework.web.client.HttpClientErrorException; | |
| 9 | import org.springframework.web.client.HttpServerErrorException; | |
| 10 | ||
| 11 | /** | |
| 12 | * Wraps calls to an external REST API (GitHub, PrairieLearn) with three defenses: | |
| 13 | * | |
| 14 | * <ul> | |
| 15 | * <li><b>Pacing:</b> consecutive calls are kept at least {@code paceMs} apart, to stay under rate | |
| 16 | * limits in the first place. The pacer only sleeps the <i>remaining</i> gap since the | |
| 17 | * previous call, so an occasional interactive call is not delayed at all — only tight loops | |
| 18 | * (like a repo sync) are slowed. | |
| 19 | * <li><b>Backoff-retries on 5xx:</b> transient server errors (e.g. GitHub's 502 "Unicorn" page) | |
| 20 | * are retried up to {@code retryMax} times, sleeping {@code retryInitialSleepSeconds} and | |
| 21 | * doubling each time (8, 16, 32...). Only then does the call fail — with a one-line {@link | |
| 22 | * ApiUnavailableException} instead of the provider's HTML error page. | |
| 23 | * <li><b>Rate-limit reaction:</b> a 429, or a 403 whose body mentions "rate limit", counts as a | |
| 24 | * retryable event and additionally doubles the pace <i>permanently</i> (for the life of the | |
| 25 | * service instance), with a log suggestion to raise the pace's configuration variable. | |
| 26 | * </ul> | |
| 27 | * | |
| 28 | * <p>All other client errors (404 for a missing path, 401/403 for a bad token) are rethrown | |
| 29 | * untouched — callers depend on their semantics. | |
| 30 | */ | |
| 31 | @Slf4j | |
| 32 | public class ApiRetryHelper { | |
| 33 | ||
| 34 | private final String apiName; | |
| 35 | private final String paceVariableName; | |
| 36 | private final long retryInitialSleepSeconds; | |
| 37 | private final int retryMax; | |
| 38 | private final AtomicLong paceMs; | |
| 39 | private final AtomicLong lastCallMs = new AtomicLong(0); | |
| 40 | private final LongSupplier nowMs; | |
| 41 | ||
| 42 | /** Thrown when the API is still failing after all retries; carries a clean one-line message. */ | |
| 43 | public static class ApiUnavailableException extends RuntimeException { | |
| 44 | public ApiUnavailableException(String message) { | |
| 45 | super(message); | |
| 46 | } | |
| 47 | } | |
| 48 | ||
| 49 | public ApiRetryHelper( | |
| 50 | String apiName, | |
| 51 | String paceVariableName, | |
| 52 | long retryInitialSleepSeconds, | |
| 53 | int retryMax, | |
| 54 | long paceInitialMs) { | |
| 55 | this( | |
| 56 | apiName, | |
| 57 | paceVariableName, | |
| 58 | retryInitialSleepSeconds, | |
| 59 | retryMax, | |
| 60 | paceInitialMs, | |
| 61 | System::currentTimeMillis); | |
| 62 | } | |
| 63 | ||
| 64 | // Visible for tests: nowMs lets the pacing arithmetic run against a controlled clock. | |
| 65 | ApiRetryHelper( | |
| 66 | String apiName, | |
| 67 | String paceVariableName, | |
| 68 | long retryInitialSleepSeconds, | |
| 69 | int retryMax, | |
| 70 | long paceInitialMs, | |
| 71 | LongSupplier nowMs) { | |
| 72 | this.apiName = apiName; | |
| 73 | this.paceVariableName = paceVariableName; | |
| 74 | this.retryInitialSleepSeconds = retryInitialSleepSeconds; | |
| 75 | this.retryMax = retryMax; | |
| 76 | this.paceMs = new AtomicLong(paceInitialMs); | |
| 77 | this.nowMs = nowMs; | |
| 78 | } | |
| 79 | ||
| 80 | /** | |
| 81 | * Runs {@code call}, pacing it relative to the previous call and retrying per the class contract. | |
| 82 | * {@code description} identifies the request in log messages, e.g. {@code "GET | |
| 83 | * .../contents/questions"}. | |
| 84 | */ | |
| 85 | public <T> T execute(String description, Supplier<T> call) { | |
| 86 | long sleepSeconds = retryInitialSleepSeconds; | |
| 87 | int retriesUsed = 0; | |
| 88 | while (true) { | |
| 89 |
1
1. execute : removed call to edu/ucsb/cs/scaffold/services/ApiRetryHelper::pace → KILLED |
pace(); |
| 90 | try { | |
| 91 |
1
1. execute : replaced return value with null for edu/ucsb/cs/scaffold/services/ApiRetryHelper::execute → KILLED |
return call.get(); |
| 92 | } catch (HttpServerErrorException e) { | |
| 93 |
2
1. execute : negated conditional → KILLED 2. execute : changed conditional boundary → KILLED |
if (retriesUsed >= retryMax) { |
| 94 | throw new ApiUnavailableException( | |
| 95 | "%s API returned %d (%s) for %s; giving up after %d attempts" | |
| 96 | .formatted( | |
| 97 | apiName, | |
| 98 | e.getStatusCode().value(), | |
| 99 |
1
1. execute : Replaced integer addition with subtraction → KILLED |
e.getStatusText(), |
| 100 | description, | |
| 101 | retriesUsed + 1)); | |
| 102 | } | |
| 103 | log.info( | |
| 104 | "{} API returned {} for {}; sleeping {} seconds before retrying", | |
| 105 | apiName, | |
| 106 | e.getStatusCode().value(), | |
| 107 | description, | |
| 108 | sleepSeconds); | |
| 109 |
2
1. execute : Replaced long multiplication with division → KILLED 2. execute : removed call to edu/ucsb/cs/scaffold/utilities/Sleep::sleepQuietly → KILLED |
Sleep.sleepQuietly(sleepSeconds * 1000); |
| 110 |
1
1. execute : Replaced long multiplication with division → KILLED |
sleepSeconds *= 2; |
| 111 |
1
1. execute : Changed increment from 1 to -1 → TIMED_OUT |
retriesUsed++; |
| 112 | } catch (HttpClientErrorException e) { | |
| 113 |
1
1. execute : negated conditional → KILLED |
if (!isRateLimited(e)) { |
| 114 | throw e; | |
| 115 | } | |
| 116 |
2
1. lambda$execute$0 : Replaced long multiplication with division → KILLED 2. lambda$execute$0 : replaced long return with 0 for edu/ucsb/cs/scaffold/services/ApiRetryHelper::lambda$execute$0 → KILLED |
long newPace = paceMs.updateAndGet(pace -> pace * 2); |
| 117 | log.warn( | |
| 118 | "{} API rate limit hit for {}; inter-call pace doubled to {} ms — consider increasing {}", | |
| 119 | apiName, | |
| 120 | description, | |
| 121 | newPace, | |
| 122 | paceVariableName); | |
| 123 |
2
1. execute : changed conditional boundary → KILLED 2. execute : negated conditional → KILLED |
if (retriesUsed >= retryMax) { |
| 124 |
1
1. execute : Replaced integer addition with subtraction → KILLED |
throw new ApiUnavailableException( |
| 125 | "%s API rate limit still in effect for %s; giving up after %d attempts (consider increasing %s)" | |
| 126 | .formatted(apiName, description, retriesUsed + 1, paceVariableName)); | |
| 127 | } | |
| 128 |
2
1. execute : removed call to edu/ucsb/cs/scaffold/utilities/Sleep::sleepQuietly → KILLED 2. execute : Replaced long multiplication with division → KILLED |
Sleep.sleepQuietly(sleepSeconds * 1000); |
| 129 |
1
1. execute : Replaced long multiplication with division → KILLED |
sleepSeconds *= 2; |
| 130 |
1
1. execute : Changed increment from 1 to -1 → TIMED_OUT |
retriesUsed++; |
| 131 | } | |
| 132 | } | |
| 133 | } | |
| 134 | ||
| 135 | /** A 429, or a 403 whose body mentions "rate limit" (GitHub's secondary-limit signature). */ | |
| 136 | private static boolean isRateLimited(HttpClientErrorException e) { | |
| 137 |
1
1. isRateLimited : negated conditional → KILLED |
if (e.getStatusCode().value() == 429) { |
| 138 |
1
1. isRateLimited : replaced boolean return with false for edu/ucsb/cs/scaffold/services/ApiRetryHelper::isRateLimited → KILLED |
return true; |
| 139 | } | |
| 140 |
2
1. isRateLimited : replaced boolean return with true for edu/ucsb/cs/scaffold/services/ApiRetryHelper::isRateLimited → KILLED 2. isRateLimited : negated conditional → KILLED |
return e.getStatusCode().value() == 403 |
| 141 |
1
1. isRateLimited : negated conditional → KILLED |
&& e.getResponseBodyAsString().toLowerCase().contains("rate limit"); |
| 142 | } | |
| 143 | ||
| 144 | /** Sleeps just long enough that consecutive calls are at least paceMs apart. */ | |
| 145 | private void pace() { | |
| 146 | long now = nowMs.getAsLong(); | |
| 147 | long previous = lastCallMs.getAndSet(now); | |
| 148 |
2
1. pace : Replaced long subtraction with addition → TIMED_OUT 2. pace : Replaced long addition with subtraction → KILLED |
long remaining = previous + paceMs.get() - now; |
| 149 |
2
1. pace : negated conditional → KILLED 2. pace : changed conditional boundary → KILLED |
if (remaining > 0) { |
| 150 |
1
1. pace : removed call to edu/ucsb/cs/scaffold/utilities/Sleep::sleepQuietly → KILLED |
Sleep.sleepQuietly(remaining); |
| 151 |
1
1. pace : removed call to java/util/concurrent/atomic/AtomicLong::set → KILLED |
lastCallMs.set(nowMs.getAsLong()); |
| 152 | } | |
| 153 | } | |
| 154 | } | |
Mutations | ||
| 89 |
1.1 |
|
| 91 |
1.1 |
|
| 93 |
1.1 2.2 |
|
| 99 |
1.1 |
|
| 109 |
1.1 2.2 |
|
| 110 |
1.1 |
|
| 111 |
1.1 |
|
| 113 |
1.1 |
|
| 116 |
1.1 2.2 |
|
| 123 |
1.1 2.2 |
|
| 124 |
1.1 |
|
| 128 |
1.1 2.2 |
|
| 129 |
1.1 |
|
| 130 |
1.1 |
|
| 137 |
1.1 |
|
| 138 |
1.1 |
|
| 140 |
1.1 2.2 |
|
| 141 |
1.1 |
|
| 148 |
1.1 2.2 |
|
| 149 |
1.1 2.2 |
|
| 150 |
1.1 |
|
| 151 |
1.1 |