LinkChecker: Support several types of rate limiting

Author: valery1707

src/main/kotlin/name/valery1707/problem/LinkChecker.kt

@@ -2,11 +2,13 @@ package name.valery1707.problem
 
 import java.net.URI
 import java.net.http.HttpClient
+import java.net.http.HttpHeaders
 import java.net.http.HttpRequest
 import java.net.http.HttpResponse
 import java.nio.file.Path
 import java.time.Duration
 import java.time.Instant
+import java.time.format.DateTimeFormatter
 import java.time.temporal.ChronoField.NANO_OF_SECOND
 import kotlin.io.path.ExperimentalPathApi
 import kotlin.io.path.PathWalkOption
@@ -115,19 +117,7 @@ class LinkChecker(private val root: Path) {
                     //Rate limiting: wait and retry
                     in HTTP_RATE_LIMIT -> {
                         val now = Instant.now()
-                        val await = response.headers()
-
-                            // todo Extract to method
-                            // https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#checking-your-rate-limit-status
-                            .map()["x-ratelimit-reset"]
-                            ?.asSequence()
-                            ?.map(String::toLong)?.map(Instant::ofEpochSecond)
-                            ?.map { Duration.between(now.with(NANO_OF_SECOND, 0), it) }
-                            ?.map(Duration::toMillis)
-                            ?.filter { it >= 0 }
-                            ?.firstOrNull()
-
-                            ?: 500
+                        val await = response.headers().rateLimitAwait(now) ?: 500
 
                         logger.debug("Await: $await ms")
                         Thread.sleep(await)
@@ -143,6 +133,40 @@ class LinkChecker(private val root: Path) {
         }
 
         private val HTTP_REDIRECT = setOf(301, 302, 307, 308)
-        private val HTTP_RATE_LIMIT = setOf(403)
+        private val HTTP_RATE_LIMIT = setOf(403, 429)
+
+        private fun HttpHeaders.rateLimitAwait(now: Instant): Long? {
+            val map = map()
+            return HTTP_RATE_LIMIT_EXTRACTORS
+                .flatMap { map[it.key]?.asSequence()?.map { v -> it.value(v.trim(), now) } ?: emptySequence() }
+                .filterNotNull()
+                .firstOrNull { it >= 0 }
+        }
+
+        private val HTTP_RATE_LIMIT_EXTRACTORS: Map<String, (String, Instant) -> Long?> = mapOf(
+            // https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#checking-your-rate-limit-status
+            "x-ratelimit-reset" to { value, now ->
+                value
+                    .toLong()
+                    .let(Instant::ofEpochSecond)
+                    .let { Duration.between(now.with(NANO_OF_SECOND, 0), it) }
+                    .let(Duration::toMillis)
+            },
+            // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+            "Retry-After" to { value, now ->
+                if (value.isDigit()) value.toLong()
+                else HTTP_DATE_FORMAT
+                    .parse(value, Instant::from)
+                    .let { Duration.between(now.with(NANO_OF_SECOND, 0), it) }
+                    .let(Duration::toMillis)
+            },
+        )
+
+        /**
+         * @see <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date">Specification</a>
+         */
+        internal val HTTP_DATE_FORMAT = DateTimeFormatter.RFC_1123_DATE_TIME
+
+        private fun String.isDigit(): Boolean = this.all { it.isDigit() }
     }
 }

src/test/kotlin/name/valery1707/problem/LinkCheckerTest.kt

@@ -1,5 +1,6 @@
 package name.valery1707.problem
 
+import name.valery1707.problem.LinkChecker.Companion.HTTP_DATE_FORMAT
 import name.valery1707.problem.LinkChecker.Companion.toURI
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.fail
@@ -19,6 +20,7 @@ import java.net.http.HttpResponse
 import java.nio.file.Path
 import java.time.Duration
 import java.time.Instant
+import java.time.ZoneId
 import java.util.*
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.Executor
@@ -61,6 +63,10 @@ internal class LinkCheckerTest {
         fun notFound(): ResponseMeta = { 404 to mapOf() }
         fun redirect(code: Int, target: String): ResponseMeta = { code to mapOf("Location" to target) }
         fun rateLimitGH(awaitMillis: Long): ResponseMeta = { 403 to mapOf("x-ratelimit-reset" to Instant.now().plusMillis(awaitMillis).epochSecond.toString()) }
+        fun rateLimitSpecSec(awaitSec: Int): ResponseMeta = { 429 to mapOf("Retry-After" to awaitSec.toString()) }
+        fun rateLimitSpecDate(awaitMillis: Long): ResponseMeta = {
+            429 to mapOf("Retry-After" to HTTP_DATE_FORMAT.format(Instant.now().plusMillis(awaitMillis).atZone(ZoneId.systemDefault())))
+        }
 
         //Check links via: curl --silent -X GET --head 'URL'
         val client = MockedHttpClient.fromMeta(
@@ -83,6 +89,13 @@ internal class LinkCheckerTest {
                     rateLimitGH(-1500),
                     ok(),
                 ),
+                "https://www.bearer.com/" to listOf(
+                    // Use variant with "delay-seconds"
+                    rateLimitSpecSec(1),
+                    // Use variant with "http-date"
+                    rateLimitSpecDate(100),
+                    ok(),
+                ),
                 "https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt" to listOf(
                     notFound(),
                 ),
@@ -95,7 +108,7 @@ internal class LinkCheckerTest {
             mapOf(
                 "Demo.md:1:25" to "https://ya.ru -> 302 -> https://ya.ru/",
                 "Demo.md:3:14" to "http://schema.org -> 301 -> https://schema.org/",
-                "Demo.md:5:14" to "https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt -> 404",
+                "Demo.md:7:14" to "https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt -> 404",
             ),
         )
     }

src/test/resources/linkChecker/Demo.md

@@ -1,5 +1,7 @@
 Link with name: [named](https://ya.ru).
 Link with name: [named](https://ya.ru/).
 Link inlined http://schema.org.
-Link with rate limiting: https://github.com/androidx/androidx/blob/androidx-main/build.gradle
+Link with rate limiting:
+* https://github.com/androidx/androidx/blob/androidx-main/build.gradle
+* https://www.bearer.com/
 Link absent: https://github.com/androidx/androidx/blob/androidx-main/buildSrc/public/src/main/kotlin/androidx/build/LibraryGroups.kt