Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
11 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex
index 44b66fbdd..5e432b285 100644
--- a/lib/pleroma/http/backoff.ex
+++ b/lib/pleroma/http/backoff.ex
@@ -1,122 +1,153 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Backoff do
@moduledoc """
GET wrapper that stops sending requests to a host after it answers
429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
Based on Akkoma's `Pleroma.HTTP.Backoff`.
"""
alias Pleroma.HTTP
require Logger
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
@cache :http_backoff_cache
@default_seconds 5 * 60
@max_seconds 60 * 60
@max_job_age 2 * 60 * 60
@spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
def get(url, headers \\ [], options \\ []) when is_binary(url) do
host = url |> URI.parse() |> Map.get(:host) |> downcase()
case @cachex.get(@cache, host) do
{:ok, true} ->
{:error, :ratelimit}
_ ->
url
|> HTTP.get(headers, options)
|> check_backoff(host)
end
end
defp downcase(nil), do: nil
defp downcase(host), do: String.downcase(host)
- defp check_backoff({:ok, %{status: status, headers: headers}}, host)
+ defp check_backoff({:ok, %{status: status, headers: headers} = env}, host)
when status in [429, 503] do
+ host = response_host(env, host)
seconds = backoff_seconds(headers, DateTime.utc_now())
Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
@cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
{:error, :ratelimit}
end
defp check_backoff(response, _host), do: response
+ # After a redirect, the 429/503 came from the last host, not the requested one.
+ defp response_host(%{url: url}, host) when is_binary(url) do
+ URI.parse(url).host |> downcase() || host
+ end
+
+ defp response_host(_env, host), do: host
+
@doc """
Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the
default backoff, so the job's attempts aren't all spent while the host is
backed off. Jobs inserted more than #{@max_job_age} seconds ago get the
error back and fail as usual.
"""
@spec snooze_ratelimited(any(), Oban.Job.t()) :: any()
def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do
if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do
{:snooze, @default_seconds}
else
error
end
end
def snooze_ratelimited(result, _job), do: result
@doc """
Seconds to back off for, from the first valid `Retry-After` or
`X-RateLimit-Reset` header. Missing, invalid or past values give the
default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
"""
@spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
def backoff_seconds(headers, now) do
[
parse_retry_after(header(headers, "retry-after"), now),
- parse_timestamp(header(headers, "x-ratelimit-reset"), now)
+ parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now)
]
|> Enum.find(@default_seconds, &is_integer/1)
|> min(@max_seconds)
end
defp header(headers, name) do
Enum.find_value(headers, fn
{key, value} when is_binary(key) and is_binary(value) ->
if String.downcase(key) == name, do: String.trim(value)
_ ->
nil
end)
end
# Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
defp parse_retry_after(nil, _now), do: nil
defp parse_retry_after(value, now) do
case Integer.parse(value) do
{seconds, ""} when seconds > 0 -> seconds
{_, ""} -> nil
_ -> parse_timestamp(value, now)
end
end
+ # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or
+ # Unix seconds) or as seconds until the reset.
+ @unix_timestamp_min 1_000_000_000
+
+ defp parse_ratelimit_reset(nil, _now), do: nil
+
+ defp parse_ratelimit_reset(value, now) do
+ case Integer.parse(value) do
+ {timestamp, ""} when timestamp >= @unix_timestamp_min ->
+ timestamp - DateTime.to_unix(now)
+
+ {seconds, ""} ->
+ seconds
+
+ _ ->
+ parse_timestamp(value, now)
+ end
+ |> positive()
+ end
+
+ defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds
+ defp positive(_), do: nil
+
defp parse_timestamp(nil, _now), do: nil
defp parse_timestamp(value, now) do
with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
seconds when seconds > 0 <- DateTime.diff(datetime, now) do
seconds
else
_ -> nil
end
end
defp parse_datetime(value) do
with {:error, _} <- Timex.parse(value, "{RFC1123}"),
{:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
{:ok, datetime}
end
end
end
diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs
index f893eef2e..b650391dd 100644
--- a/test/pleroma/http/backoff_test.exs
+++ b/test/pleroma/http/backoff_test.exs
@@ -1,163 +1,195 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Based on Akkoma's test/pleroma/http/backoff_test.exs
defmodule Pleroma.HTTP.BackoffTest do
use Pleroma.DataCase, async: false
alias Pleroma.HTTP.Backoff
@cache :http_backoff_cache
@url "https://ratelimited.example.com/users/alice"
@host "ratelimited.example.com"
defp mock_response(status, headers \\ []) do
test_pid = self()
Tesla.Mock.mock(fn %{method: :get} ->
send(test_pid, :request)
%Tesla.Env{status: status, headers: headers, body: ""}
end)
end
defp ttl_seconds do
{:ok, ttl} = Cachex.ttl(@cache, @host)
div(ttl, 1000)
end
defp assert_ttl_about(expected) do
ttl = ttl_seconds()
assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}"
end
describe "get/3" do
test "returns the response when the host is not backed off" do
mock_response(200)
assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url)
assert {:ok, nil} = Cachex.get(@cache, @host)
end
test "does not send a request while the host is backed off" do
Cachex.put(@cache, @host, true)
mock_response(200)
assert {:error, :ratelimit} = Backoff.get(@url)
refute_received :request
end
for status <- [429, 503] do
test "backs off from the host after a #{status} response" do
mock_response(unquote(status))
assert {:error, :ratelimit} = Backoff.get(@url)
assert_received :request
assert {:ok, true} = Cachex.get(@cache, @host)
assert {:error, :ratelimit} = Backoff.get(@url <> "/other")
refute_received :request
end
end
test "uses a default backoff without headers" do
mock_response(429)
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(300)
end
test "treats host names case-insensitively" do
mock_response(429)
assert {:error, :ratelimit} = Backoff.get("https://RateLimited.Example.com/users/alice")
assert_received :request
assert {:error, :ratelimit} = Backoff.get(@url)
refute_received :request
end
test "only backs off from the host that answered" do
mock_response(429)
assert {:error, :ratelimit} = Backoff.get(@url)
mock_response(200)
assert {:ok, _} = Backoff.get("https://other.example.com/users/bob")
end
+
+ test "backs off from the host that answered after a redirect" do
+ test_pid = self()
+
+ Tesla.Mock.mock(fn
+ %{url: "https://redirector.example.com/users/alice"} = env ->
+ %{env | status: 302, headers: [{"location", @url}], body: ""}
+
+ %{url: @url} = env ->
+ send(test_pid, :request)
+ %{env | status: 429, headers: [], body: ""}
+ end)
+
+ assert {:error, :ratelimit} = Backoff.get("https://redirector.example.com/users/alice")
+ assert {:ok, true} = Cachex.get(@cache, @host)
+ assert {:ok, nil} = Cachex.get(@cache, "redirector.example.com")
+ end
end
describe "backoff headers" do
test "parses Retry-After in seconds" do
mock_response(429, [{"retry-after", "600"}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
test "parses Retry-After as an HTTP-date" do
date =
DateTime.utc_now()
|> DateTime.add(600)
|> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
mock_response(503, [{"retry-after", date}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
test "parses Retry-After as an ISO 8601 timestamp" do
date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
mock_response(429, [{"retry-after", date}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do
date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
mock_response(429, [{"x-ratelimit-reset", date}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
+ test "parses X-RateLimit-Reset as a Unix timestamp" do
+ reset = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_unix() |> to_string()
+ mock_response(429, [{"x-ratelimit-reset", reset}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset in seconds" do
+ mock_response(429, [{"x-ratelimit-reset", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
test "matches header names case-insensitively" do
mock_response(429, [{"Retry-After", "600"}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
test "prefers Retry-After over X-RateLimit-Reset" do
date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601()
mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
test "falls back to the next header when one is invalid" do
date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(600)
end
for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do
test "ignores the invalid or past Retry-After value #{inspect(value)}" do
mock_response(429, [{"retry-after", unquote(value)}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(300)
end
end
test "caps absurd values at one hour" do
mock_response(429, [{"retry-after", "99999999999"}])
assert {:error, :ratelimit} = Backoff.get(@url)
assert_ttl_about(3600)
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Tue, Nov 17, 6:48 PM (13 h, 48 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1839184
Default Alt Text
(11 KB)

Event Timeline