Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85948848
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
92 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/changelog.d/http-backoff.add b/changelog.d/http-backoff.add
new file mode 100644
index 000000000..5c9eb988b
--- /dev/null
+++ b/changelog.d/http-backoff.add
@@ -0,0 +1 @@
+Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour)
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index aebad6984..080dd1065 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -1,155 +1,155 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Pleroma do
@apps [
:restarter,
:ecto,
:ecto_sql,
:postgrex,
:db_connection,
:cachex,
:flake_id,
:swoosh,
:timex,
:fast_html,
:oban,
:logger_backends
]
- @cachex_children ["object", "user", "scrubber", "web_resp"]
+ @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"]
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Pleroma.Config.Holder.save_default()
Pleroma.Config.Oban.warn()
Pleroma.Application.limiters_setup()
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)
apps =
if adapter == Tesla.Adapter.Gun do
[:gun | @apps]
else
[:hackney | @apps]
end
Enum.each(apps, &Application.ensure_all_started/1)
oban_config = [
crontab: [],
repo: Pleroma.Repo,
log: false,
queues: [],
plugins: []
]
children =
[
Pleroma.Repo,
Pleroma.Emoji,
{Pleroma.Config.TransferTask, false},
Pleroma.Web.Endpoint,
{Oban, oban_config},
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
http_children(adapter) ++
search_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
Supervisor.start_link(children ++ cachex_children,
strategy: :one_for_one,
name: Pleroma.Supervisor
)
if Pleroma.Config.get(:env) not in [:test, :benchmark] do
pleroma_rebooted?()
end
end
defp pleroma_rebooted? do
if Restarter.Pleroma.rebooted?() do
:ok
else
Process.sleep(10)
pleroma_rebooted?()
end
end
def load_pleroma do
Application.load(:pleroma)
end
def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
Keyword.get(options, opt) || shell_prompt(prompt, defval, defname)
end
def shell_prompt(prompt, defval \\ nil, defname \\ nil) do
prompt_message = "#{prompt} [#{defname || defval}] "
input =
if mix_shell?(),
do: Mix.shell().prompt(prompt_message),
else: :io.get_line(prompt_message)
case input do
"\n" ->
case defval do
nil ->
shell_prompt(prompt, defval, defname)
defval ->
defval
end
input ->
String.trim(input)
end
end
def shell_info(message) do
if mix_shell?(),
do: Mix.shell().info(message),
else: IO.puts(message)
end
def shell_error(message) do
if mix_shell?(),
do: Mix.shell().error(message),
else: IO.puts(:stderr, message)
end
@doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)"
def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0)
def escape_sh_path(path) do
~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
end
defp http_children(Tesla.Adapter.Gun) do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
defp http_children(_), do: []
defp search_children do
case Pleroma.Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index adc054f3b..2c8eede1c 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -1,355 +1,356 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Application do
use Application
import Cachex.Spec
alias Pleroma.Config
require Logger
@name Mix.Project.config()[:name]
@version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url]
def name, do: @name
def version, do: @version
def named_version, do: @name <> " " <> @version
def repository, do: @repository
def user_agent do
if Process.whereis(Pleroma.Web.Endpoint) do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.Endpoint.url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
custom ->
custom
end
else
# fallback, if endpoint is not started yet
"Pleroma Data Loader"
end
end
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
# Scrubbers are compiled at runtime and therefore will cause a conflict
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
configure_logger()
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
if Config.get(:env) != :test do
Pleroma.ApplicationRequirements.verify!()
end
load_custom_modules()
Pleroma.Docs.JSON.compile()
limiters_setup()
adapter = Application.get_env(:tesla, :adapter)
if match?({Tesla.Adapter.Finch, _}, adapter) do
Logger.info("Starting Finch")
use_castore_for_otp_cacerts()
Finch.start_link(name: MyFinch)
end
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
# :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18
if Version.compare(System.version(), "1.18.0") == :lt do
Code.compiler_options(warnings_as_errors: false)
end
# Define workers and child supervisors to be supervised
children =
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo
] ++
search_children() ++
[
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
] ++
cachex_children() ++
http_children(adapter) ++
[
Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
{Oban, Config.get(Oban)},
Pleroma.Web.Endpoint
] ++
task_children() ++
streamer_registry() ++
background_migrators() ++
shout_child(shout_enabled?()) ++
[Pleroma.Gopher.Server] ++
[Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
# If we have a lot of caches, default max_restarts can cause test
# resets to fail.
# Go for the default 3 unless we're in test
max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts]
opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts]
Supervisor.start_link(children, opts)
end
def configure_logger do
Config.get([:logger, :backends], [])
|> Enum.each(fn backend ->
case backend_to_logger(backend) do
{:ok, logger} ->
add_logger(logger)
{:error, :console_backend} ->
Logger.warning(":console is no longer considered a backend and is enabled by default")
end
end)
end
defp add_logger(backend) do
case LoggerBackends.add(backend) do
{:ok, _} ->
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
{:error, :already_present} ->
:ok
{:error, reason} ->
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
end
end
defp backend_to_logger(:console), do: {:error, :console_backend}
defp backend_to_logger({:ex_syslogger = backend, name}) do
Logger.warning(
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
)
{:ok, {ExSyslogger, name}}
end
defp backend_to_logger(backend), do: {:ok, backend}
def load_custom_modules do
dir = Config.get([:modules, :runtime_dir])
if dir && File.exists?(dir) do
dir
|> Pleroma.Utils.compile_dir()
|> case do
{:error, _errors, _warnings} ->
raise "Invalid custom modules"
{:ok, modules, _warnings} ->
if Application.get_env(:pleroma, __MODULE__)[:load_custom_modules] do
Enum.each(modules, fn mod ->
Logger.info("Custom module loaded: #{inspect(mod)}")
end)
end
:ok
end
end
end
defp cachex_children do
[
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("scrubber_management", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500),
build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+ build_cachex("http_backoff", limit: 10_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
defp search_children do
case Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
defp emoji_packs_expiration,
do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
defp idempotency_expiration,
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp chat_message_id_idempotency_key_expiration,
do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
defp seconds_valid_interval,
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
@spec build_cachex(String.t(), keyword()) :: map()
def build_cachex(type, opts),
do: %{
id: String.to_atom("cachex_" <> type),
start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
type: :worker
}
defp shout_enabled?, do: Config.get([:shout, :enabled])
defp streamer_registry do
if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do
[
{Registry,
[
name: Pleroma.Web.Streamer.registry(),
keys: :duplicate,
partitions: System.schedulers_online()
]}
]
else
[]
end
end
defp background_migrators do
if Application.get_env(:pleroma, __MODULE__)[:background_migrators] do
[
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
else
[]
end
end
defp shout_child(true) do
[
Pleroma.Web.ShoutChannel.ShoutChannelState,
{Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
]
end
defp shout_child(_), do: []
defp task_children do
children = [
%{
id: :web_push_init,
start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
restart: :temporary
}
]
if Application.get_env(:pleroma, __MODULE__)[:internal_fetch] do
children ++
[
%{
id: :internal_fetch_init,
start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
restart: :temporary
}
]
else
children
end
end
# start hackney and gun pools in tests
defp http_children(adapter) do
if Application.get_env(:pleroma, __MODULE__)[:test_http_pools] do
http_children_hackney() ++ http_children_gun()
else
cond do
match?(Tesla.Adapter.Hackney, adapter) -> http_children_hackney()
match?(Tesla.Adapter.Gun, adapter) -> http_children_gun()
true -> []
end
end
end
# Mint trusts the OTP CA store, which comes from the OS. Make it the CAStore
# bundle that the other adapters trust. OTP 24 has no such store and Mint
# uses CAStore there anyway.
defp use_castore_for_otp_cacerts do
if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_load, 1) do
:ok = :public_key.cacerts_load(CAStore.file_path())
end
end
defp http_children_hackney do
pools = [:federation, :media]
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload | pools]
else
pools
end
for pool <- pools do
options = Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp http_children_gun do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
@spec limiters_setup() :: :ok
def limiters_setup do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Search,
Pleroma.Webhook.Notify
]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])
max_running = Keyword.get(mod_config, :max_running, 5)
max_waiting = Keyword.get(mod_config, :max_waiting, 5)
ConcurrentLimiter.new(module, max_running, max_waiting)
end)
end
end
diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex
new file mode 100644
index 000000000..0d2b99742
--- /dev/null
+++ b/lib/pleroma/http/backoff.ex
@@ -0,0 +1,101 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Backoff do
+ @moduledoc """
+ GET wrapper that stops sending requests to a host after it answers
+ 429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
+
+ Based on Akkoma's `Pleroma.HTTP.Backoff`.
+ """
+
+ alias Pleroma.HTTP
+
+ require Logger
+
+ @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+ @cache :http_backoff_cache
+
+ @default_seconds 5 * 60
+ @max_seconds 60 * 60
+
+ @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
+ def get(url, headers \\ [], options \\ []) when is_binary(url) do
+ host = URI.parse(url).host
+
+ case @cachex.get(@cache, host) do
+ {:ok, true} ->
+ {:error, :ratelimit}
+
+ _ ->
+ url
+ |> HTTP.get(headers, options)
+ |> check_backoff(host)
+ end
+ end
+
+ defp check_backoff({:ok, %{status: status, headers: headers}}, host)
+ when status in [429, 503] do
+ seconds = backoff_seconds(headers, DateTime.utc_now())
+ Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
+ @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
+ {:error, :ratelimit}
+ end
+
+ defp check_backoff(response, _host), do: response
+
+ @doc """
+ Seconds to back off for, from the first valid `Retry-After` or
+ `X-RateLimit-Reset` header. Missing, invalid or past values give the
+ default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
+ """
+ @spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
+ def backoff_seconds(headers, now) do
+ [
+ parse_retry_after(header(headers, "retry-after"), now),
+ parse_timestamp(header(headers, "x-ratelimit-reset"), now)
+ ]
+ |> Enum.find(@default_seconds, &is_integer/1)
+ |> min(@max_seconds)
+ end
+
+ defp header(headers, name) do
+ Enum.find_value(headers, fn
+ {key, value} when is_binary(key) and is_binary(value) ->
+ if String.downcase(key) == name, do: String.trim(value)
+
+ _ ->
+ nil
+ end)
+ end
+
+ # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
+ defp parse_retry_after(nil, _now), do: nil
+
+ defp parse_retry_after(value, now) do
+ case Integer.parse(value) do
+ {seconds, ""} when seconds > 0 -> seconds
+ {_, ""} -> nil
+ _ -> parse_timestamp(value, now)
+ end
+ end
+
+ defp parse_timestamp(nil, _now), do: nil
+
+ defp parse_timestamp(value, now) do
+ with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
+ seconds when seconds > 0 <- DateTime.diff(datetime, now) do
+ seconds
+ else
+ _ -> nil
+ end
+ end
+
+ defp parse_datetime(value) do
+ with {:error, _} <- Timex.parse(value, "{RFC1123}"),
+ {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
+ {:ok, datetime}
+ end
+ end
+end
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 706ee317c..3313583cc 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -1,278 +1,278 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
alias Pleroma.Signature
alias Pleroma.Web.ActivityPub.InternalFetchActor
alias Pleroma.Web.ActivityPub.MRF
alias Pleroma.Web.ActivityPub.ObjectValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.Federator
require Logger
require Pleroma.Constants
@spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
defp reinject_object(%Object{data: %{}} = object, new_data) do
Logger.debug("Reinjecting object #{new_data["id"]}")
with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}),
{:ok, new_data} <- MRF.filter(new_data),
{:ok, new_object, _} <-
Object.Updater.do_update_and_invalidate_cache(
object,
new_data,
_touch_changeset? = true
) do
{:ok, new_object}
else
e ->
Logger.error("Error while processing object: #{inspect(e)}")
{:error, e}
end
end
defp reinject_object(_, new_data) do
with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do
{:ok, object}
else
e -> e
end
end
# The stored id is canonical, so the refetch must not redirect, and the
# remote must not swap in another object or author.
# Based on Akkoma 9061d148b and fee57eb37.
def refetch_object(%Object{data: %{"id" => id} = data} = object) do
with {:local, false} <- {:local, Object.local?(object)},
{:ok, new_data} <- fetch_and_contain(id, true),
{:id, true} <- {:id, new_data["id"] == id},
{:actor, true} <- {:actor, actor_of(new_data) == actor_of(data)},
{:ok, object} <- reinject_object(object, new_data) do
{:ok, object}
else
{:local, true} -> {:ok, object}
e -> {:error, e}
end
end
defp actor_of(data), do: CommonFixes.fix_actor(data)["actor"]
@typep fetcher_errors ::
:error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier
# Note: will create a Create activity, which we need internally at the moment.
@spec fetch_object_from_id(String.t(), list()) ::
{:ok, Object.t()} | {fetcher_errors(), any()} | Pipeline.errors()
def fetch_object_from_id(id, options \\ []) do
with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
{_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
{_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
{_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
params <- prepare_activity_params(data),
{_, :ok} <- {:containment, Containment.contain_origin(id, params)},
{_, {:ok, activity}} <-
{:transmogrifier, Transmogrifier.handle_incoming(params, options)},
{_, _data, %Object{} = object} <-
{:object, data, Object.normalize(activity, fetch: false)} do
{:ok, object}
else
{:normalize, object = %Object{}} ->
{:ok, object}
{:fetch_object, %Object{} = object} ->
{:ok, object}
{:object, data, nil} ->
reinject_object(%Object{}, data)
e ->
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(e)}")
e
end
end
defp prepare_activity_params(data) do
%{
"type" => "Create",
# Should we seriously keep this attributedTo thing?
"actor" => data["actor"] || data["attributedTo"],
"object" => data
}
|> Maps.put_if_present("to", data["to"])
|> Maps.put_if_present("cc", data["cc"])
|> Maps.put_if_present("bto", data["bto"])
|> Maps.put_if_present("bcc", data["bcc"])
end
defp make_signature(id, date) do
uri = URI.parse(id)
signature =
InternalFetchActor.get_actor()
|> Signature.sign(%{
"(request-target)": "get #{uri.path}",
host: uri.host,
date: date
})
{"signature", signature}
end
defp sign_fetch(headers, id, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
[make_signature(id, date) | headers]
else
headers
end
end
defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
[{"date", date} | headers]
else
headers
end
end
def fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(%{"id" => id}),
do: fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(id) when is_binary(id),
do: fetch_and_contain(id, false)
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
# The fetched document's id must be the URL it came from. Display URLs
# (e.g. Mastodon's /@user/123) serve the object under another id, so
# if the id differs we refetch once from that id, which has to be on the
# same host, must not redirect, and must then match exactly.
# Based on Akkoma 8684964c5 and 940792f8b.
defp fetch_and_contain(id, canonical?) do
Logger.debug("Fetching object #{id} via AP")
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{_, true} <- {:mrf, MRF.id_filter(id)},
{_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
{:ok, final_url, body} <- get_object(id),
{_, true} <- {:redirect, not canonical? or same_url?(final_url, id)},
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data),
{_, _, :ok} <- {:id, data, Containment.contain_id_to_fetch(final_url, data)} do
{:ok, data}
else
{:id, %{"id" => canonical_id}, :error} when is_binary(canonical_id) and not canonical? ->
fetch_and_contain(canonical_id, true)
{:id, _, :error} ->
{:error, :id_mismatch}
{:redirect, false} ->
{:error, :canonical_id_redirect}
{:scheme, _} ->
{:error, "Unsupported URI scheme"}
{:local_fetch, _} ->
{:error, "Trying to fetch local resource"}
{:error, e} ->
{:error, e}
{:mrf, false} ->
{:error, {:reject, "Filtered by id"}}
e ->
{:error, e}
end
end
defp same_url?(url, other), do: Containment.contain_id_to_fetch(url, %{"id" => other}) == :ok
defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, original_url) do
{:cross_domain_redirect, final_host != URI.parse(original_url).host}
end
defp get_object(id) do
date = Pleroma.Signature.signed_date()
headers =
[{"accept", "application/activity+json"}]
|> maybe_date_fetch(date)
|> sign_fetch(id, date)
- case HTTP.get(id, headers) do
+ case HTTP.Backoff.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 ->
remote_host = if final_url, do: URI.parse(final_url).host, else: nil
with {:cross_domain_redirect, false} <- check_crossdomain_redirect(remote_host, id),
{_, content_type} <- List.keyfind(headers, "content-type", 0),
{:ok, _media_type} <- verify_content_type(content_type) do
{:ok, if(final_url in [nil, ""], do: id, else: final_url), body}
else
{:cross_domain_redirect, true} ->
{:error, {:cross_domain_redirect, true}}
error ->
error
end
# Handle the case where URL is not in the response (older HTTP library versions)
{:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case verify_content_type(content_type) do
{:ok, _} -> {:ok, id, body}
error -> error
end
_ ->
{:error, {:content_type, nil}}
end
{:ok, %{status: code}} when code in [401, 403] ->
{:error, :forbidden}
{:ok, %{status: code}} when code in [404, 410] ->
{:error, :not_found}
{:error, e} ->
{:error, e}
e ->
{:error, e}
end
end
defp safe_json_decode(nil), do: {:ok, nil}
defp safe_json_decode(json), do: Jason.decode(json)
defp verify_content_type(content_type) do
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, :activity_json}
{:ok, "application", "ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, :ld_json}
_ ->
{:error, {:content_type, content_type}}
end
end
end
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 62a2715f5..cb312405c 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -1,287 +1,295 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.WebFinger do
alias Pleroma.HTTP
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Publisher
alias Pleroma.Web.Endpoint
alias Pleroma.Web.XML
alias Pleroma.XmlBuilder
require Jason
require Logger
def host_meta do
base_url = Endpoint.url()
{
:XRD,
%{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"},
{
:Link,
%{
rel: "lrdd",
type: "application/xrd+xml",
template: "#{base_url}/.well-known/webfinger?resource={uri}"
}
}
}
|> XmlBuilder.to_doc()
end
def webfinger(resource, fmt) when fmt in ["XML", "JSON"] do
host = Pleroma.Web.Endpoint.host()
regex =
if webfinger_domain = Pleroma.Config.get([__MODULE__, :domain]) do
~r/(acct:)?(?<username>[a-z0-9A-Z_\.-]+)@(#{host}|#{webfinger_domain})$/
else
~r/(acct:)?(?<username>[a-z0-9A-Z_\.-]+)@#{host}$/
end
with %{"username" => username} <- Regex.named_captures(regex, resource),
%User{} = user <- User.get_cached_by_nickname(username) do
{:ok, represent_user(user, fmt)}
else
_e ->
with %User{} = user <- User.get_cached_by_ap_id(resource) do
{:ok, represent_user(user, fmt)}
else
_e ->
{:error, "Couldn't find user"}
end
end
end
defp gather_links(%User{} = user) do
[
%{
"rel" => "http://webfinger.net/rel/profile-page",
"type" => "text/html",
"href" => user.ap_id
}
] ++ Publisher.gather_webfinger_links(user)
end
defp gather_aliases(%User{} = user) do
[user.ap_id | user.also_known_as]
end
def represent_user(user, "JSON") do
%{
"subject" => "acct:#{user.nickname}@#{host()}",
"aliases" => gather_aliases(user),
"links" => gather_links(user)
}
end
def represent_user(user, "XML") do
aliases =
user
|> gather_aliases()
|> Enum.map(&{:Alias, &1})
links =
gather_links(user)
|> Enum.map(fn link -> {:Link, link} end)
{
:XRD,
%{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"},
[
{:Subject, "acct:#{user.nickname}@#{host()}"}
] ++ aliases ++ links
}
|> XmlBuilder.to_doc()
end
def host do
Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host()
end
defp webfinger_from_xml(body) do
with {:ok, doc} <- XML.parse_document(body) do
subject = XML.string_from_xpath("//Subject", doc)
subscribe_address =
~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template}
|> XML.string_from_xpath(doc)
ap_id =
~s{//Link[@rel="self" and @type="application/activity+json"]/@href}
|> XML.string_from_xpath(doc)
data = %{
"subject" => subject,
"subscribe_address" => subscribe_address,
"ap_id" => ap_id
}
{:ok, data}
end
end
defp webfinger_from_json(body) do
with {:ok, doc} <- Jason.decode(body) do
data =
Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data ->
case {link["type"], link["rel"]} do
{"application/activity+json", "self"} ->
Map.put(data, "ap_id", link["href"])
{"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} ->
Map.put(data, "ap_id", link["href"])
{nil, "http://ostatus.org/schema/1.0/subscribe"} ->
Map.put(data, "subscribe_address", link["template"])
_ ->
Logger.debug("Unhandled type: #{inspect(link["type"])}")
data
end
end)
{:ok, data}
end
end
def get_template_from_xml(body) do
xpath = "//Link[@rel='lrdd']/@template"
with {:ok, doc} <- XML.parse_document(body),
template when template != nil <- XML.string_from_xpath(xpath, doc) do
{:ok, template}
end
end
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def find_lrdd_template(domain) do
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
- {:commit, fetch_lrdd_template(domain)}
+ case fetch_lrdd_template(domain) do
+ # retry once the backoff expires instead of caching the miss
+ {:error, :ratelimit} = error -> {:ignore, error}
+ result -> {:commit, result}
+ end
end)
rescue
e -> {:error, "Cachex error: #{inspect(e)}"}
end
defp fetch_lrdd_template(domain) do
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
meta_url = "https://#{domain}/.well-known/host-meta"
- with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do
+ with {:ok, %{status: status, body: body}} when status in 200..299 <-
+ HTTP.Backoff.get(meta_url) do
get_template_from_xml(body)
else
+ {:error, :ratelimit} = error ->
+ error
+
error ->
Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}")
{:error, :lrdd_not_found}
end
end
defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
case find_lrdd_template(domain) do
{:ok, template} ->
String.replace(template, "{uri}", encoded_account)
_ ->
"https://#{domain}/.well-known/webfinger?resource=#{encoded_account}"
end
end
defp get_address_from_domain(domain, account) when is_binary(domain) do
encoded_account = URI.encode("acct:#{account}")
get_address_from_domain(domain, encoded_account)
end
defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
@spec finger(String.t()) :: {:ok, map()} | {:error, any()}
def finger(account), do: do_finger(account, true)
defp do_finger(account, follow_redirects) do
account = String.trim_leading(account, "@")
domain =
with [_name, domain] <- String.split(account, "@") do
domain
else
_e ->
URI.parse(account).host
end
with address when is_binary(address) <- get_address_from_domain(domain, account),
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
- HTTP.get(
+ HTTP.Backoff.get(
address,
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", subtype, _} when subtype in ~w(xrd+xml xml) ->
webfinger_from_xml(body)
{:ok, "application", subtype, _} when subtype in ~w(jrd+json json) ->
webfinger_from_json(body)
_ ->
{:error, {:content_type, content_type}}
end
_ ->
{:error, {:content_type, nil}}
end
|> case do
{:ok, data} ->
if follow_redirects do
validate_webfinger(address, data)
else
{:ok, data}
end
error ->
error
end
else
error ->
Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
error
end
end
defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
with [_name, acct_host] <- String.split(acct, "@"),
{_, resolved_url} <- {:address, get_address_from_domain(acct_host, subject)},
{_, true} <- {:url_match, resolved_webfinger_matches?(request_url, resolved_url, data)} do
{:ok, data}
else
_ -> {:error, {:webfinger_invalid, request_url, data}}
end
end
defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
defp resolved_webfinger_matches?(request_url, resolved_url, _data)
when request_url == resolved_url do
true
end
defp resolved_webfinger_matches?(
_request_url,
_resolved_url,
%{"subject" => "acct:" <> acct} = data
) do
with {:ok, %{"subject" => "acct:" <> new_acct} = new_data} <- do_finger(acct, false),
true <- acct == new_acct,
true <- webfinger_data_matches?(data, new_data) do
true
else
_ -> false
end
end
defp webfinger_data_matches?(%{"ap_id" => ap_id}, %{"ap_id" => ap_id}) when ap_id != "" do
true
end
defp webfinger_data_matches?(_data, _new_data), do: false
end
diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs
new file mode 100644
index 000000000..48c8514bd
--- /dev/null
+++ b/test/pleroma/http/backoff_test.exs
@@ -0,0 +1,154 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Based on Akkoma's test/pleroma/http/backoff_test.exs
+defmodule Pleroma.HTTP.BackoffTest do
+ use Pleroma.DataCase, async: false
+
+ alias Pleroma.HTTP.Backoff
+
+ @cache :http_backoff_cache
+ @url "https://ratelimited.example.com/users/alice"
+ @host "ratelimited.example.com"
+
+ defp mock_response(status, headers \\ []) do
+ test_pid = self()
+
+ Tesla.Mock.mock(fn %{method: :get} ->
+ send(test_pid, :request)
+ %Tesla.Env{status: status, headers: headers, body: ""}
+ end)
+ end
+
+ defp ttl_seconds do
+ {:ok, ttl} = Cachex.ttl(@cache, @host)
+ div(ttl, 1000)
+ end
+
+ defp assert_ttl_about(expected) do
+ ttl = ttl_seconds()
+ assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}"
+ end
+
+ describe "get/3" do
+ test "returns the response when the host is not backed off" do
+ mock_response(200)
+
+ assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url)
+ assert {:ok, nil} = Cachex.get(@cache, @host)
+ end
+
+ test "does not send a request while the host is backed off" do
+ Cachex.put(@cache, @host, true)
+ mock_response(200)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ refute_received :request
+ end
+
+ for status <- [429, 503] do
+ test "backs off from the host after a #{status} response" do
+ mock_response(unquote(status))
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_received :request
+ assert {:ok, true} = Cachex.get(@cache, @host)
+
+ assert {:error, :ratelimit} = Backoff.get(@url <> "/other")
+ refute_received :request
+ end
+ end
+
+ test "uses a default backoff without headers" do
+ mock_response(429)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+
+ test "only backs off from the host that answered" do
+ mock_response(429)
+ assert {:error, :ratelimit} = Backoff.get(@url)
+
+ mock_response(200)
+ assert {:ok, _} = Backoff.get("https://other.example.com/users/bob")
+ end
+ end
+
+ describe "backoff headers" do
+ test "parses Retry-After in seconds" do
+ mock_response(429, [{"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an HTTP-date" do
+ date =
+ DateTime.utc_now()
+ |> DateTime.add(600)
+ |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+ mock_response(503, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "matches header names case-insensitively" do
+ mock_response(429, [{"Retry-After", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "prefers Retry-After over X-RateLimit-Reset" do
+ date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "falls back to the next header when one is invalid" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do
+ test "ignores the invalid or past Retry-After value #{inspect(value)}" do
+ mock_response(429, [{"retry-after", unquote(value)}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+ end
+
+ test "caps absurd values at one hour" do
+ mock_response(429, [{"retry-after", "99999999999"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(3600)
+ end
+ end
+end
diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs
index 61a6fb517..80b172a36 100644
--- a/test/pleroma/object/fetcher_test.exs
+++ b/test/pleroma/object/fetcher_test.exs
@@ -1,1034 +1,1055 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.FetcherTest do
use Pleroma.DataCase
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.Web.ActivityPub.ObjectValidator
require Pleroma.Constants
import Mock
import Pleroma.Factory
import Tesla.Mock
setup do
mock(fn
%{method: :get, url: "https://mastodon.example.org/users/userisgone"} ->
%Tesla.Env{status: 410}
%{method: :get, url: "https://mastodon.example.org/users/userisgone404"} ->
%Tesla.Env{status: 404}
%{
method: :get,
url:
"https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/json"}],
body: File.read!("test/fixtures/spoofed-object.json")
}
env ->
apply(HttpRequestMock, :request, [env])
end)
:ok
end
describe "error cases" do
setup do
mock(fn
%{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} ->
%Tesla.Env{
status: 200,
# Pleroma redirects /notice/:id to the object id
url: "https://social.sakamoto.gq/objects/f20f2497-66d9-4a52-a2e1-1be2a39c32c1",
body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://social.sakamoto.gq/users/eal"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/fetch_mocks/eal.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://busshi.moe/users/tuxcrafting/statuses/104410921027210069"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://busshi.moe/users/tuxcrafting"} ->
%Tesla.Env{
status: 500
}
%{
method: :get,
url: "https://stereophonic.space/objects/02997b83-3ea7-4b63-94af-ef3aa2d4ed17"
} ->
%Tesla.Env{
status: 500
}
end)
:ok
end
test "it works when fetching the OP actor errors out" do
# Here we simulate a case where the author of the OP can't be read
assert {:ok, _} =
Fetcher.fetch_object_from_id(
"https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"
)
end
end
+ describe "rate limiting" do
+ test "a backed-off host gets no further requests" do
+ test_pid = self()
+
+ mock(fn %{method: :get, url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+
+ first = "https://ratelimited.example.com/objects/1"
+ second = "https://ratelimited.example.com/objects/2"
+
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first)
+ assert_received {:request, ^first}
+
+ assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second)
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second)
+ refute_received {:request, _}
+ end
+ end
+
describe "max thread distance restriction" do
@ap_id "http://mastodon.example.org/@admin/99541947525187367"
setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
test "it returns thread depth exceeded error if thread depth is exceeded" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:allowed_depth, false} = Fetcher.fetch_object_from_id(@ap_id, depth: 1)
end
test "it fetches object if max thread depth is restricted to 0 and depth is not specified" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id)
end
test "it fetches object if requested depth does not exceed max thread depth" do
clear_config([:instance, :federation_incoming_replies_max_depth], 10)
assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id, depth: 10)
end
end
describe "actor origin containment" do
test "it rejects objects with a bogus origin" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json")
end
test "it rejects objects when attributedTo is wrong (variant 1)" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json")
end
test "it rejects objects when attributedTo is wrong (variant 2)" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json")
end
end
describe "fetching an object" do
test "it fetches an object" do
{:ok, object} =
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert _activity = Activity.get_create_by_object_ap_id(object.data["id"])
{:ok, object_again} =
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert [attachment] = object.data["attachment"]
assert is_list(attachment["url"])
assert object == object_again
end
test "Return MRF reason when fetched status is rejected by one" do
clear_config([:mrf_keyword, :reject], ["yeah"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
assert {:transmogrifier, {:reject, "[KeywordPolicy] Matches with rejected keyword"}} ==
Fetcher.fetch_object_from_id(
"http://mastodon.example.org/@admin/99541947525187367"
)
end
test "it does not fetch a spoofed object uploaded on an instance as an attachment" do
assert {:fetch, {:error, {:content_type, "application/json"}}} =
Fetcher.fetch_object_from_id(
"https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
)
end
test "it does not fetch from local instance" do
local_url = Pleroma.Web.Endpoint.url() <> "/objects/local_resource"
assert {:fetch, {:error, "Trying to fetch local resource"}} =
Fetcher.fetch_object_from_id(local_url)
end
test "it validates content-type headers according to ActivityPub spec" do
# Setup a mock for an object with invalid content-type
mock(fn
%{method: :get, url: "https://example.com/objects/invalid-content-type"} ->
%Tesla.Env{
status: 200,
# Not a valid AP content-type
headers: [{"content-type", "application/json"}],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/invalid-content-type",
"type" => "Note",
"content" => "This has an invalid content type",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
assert {:fetch, {:error, {:content_type, "application/json"}}} =
Fetcher.fetch_object_from_id("https://example.com/objects/invalid-content-type")
end
test "it accepts objects with application/ld+json and ActivityStreams profile" do
# Setup a mock for an object with ld+json content-type and AS profile
mock(fn
%{method: :get, url: "https://example.com/objects/valid-ld-json"} ->
%Tesla.Env{
status: 200,
headers: [
{"content-type",
"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""}
],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/valid-ld-json",
"type" => "Note",
"content" => "This has a valid ld+json content type",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
# This should pass if content-type validation works correctly
assert {:ok, object} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://example.com/objects/valid-ld-json"
)
assert object["content"] == "This has a valid ld+json content type"
end
test "it rejects objects with no content-type header" do
# Setup a mock for an object with no content-type header
mock(fn
%{method: :get, url: "https://example.com/objects/no-content-type"} ->
%Tesla.Env{
status: 200,
# No content-type header
headers: [],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/no-content-type",
"type" => "Note",
"content" => "This has no content type header",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
# We want to test that the request fails with a missing content-type error
# but the actual error is {:fetch, {:error, nil}} - we'll check for this format
result = Fetcher.fetch_object_from_id("https://example.com/objects/no-content-type")
assert {:fetch, {:error, nil}} = result
end
end
describe "implementation quirks" do
test "it can fetch plume articles" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
)
assert object
end
test "it can fetch peertube videos" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
)
assert object
end
test "it can fetch Mobilizon events" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"
)
assert object
end
test "it can fetch wedistribute articles" do
{:ok, object} =
Fetcher.fetch_object_from_id("https://wedistribute.org/wp-json/pterotype/v1/object/85810")
assert object
end
test "all objects with fake directions are rejected by the object fetcher" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://info.pleroma.site/activity4.json"
)
end
test "handle HTTP 410 Gone response" do
assert {:error, :not_found} ==
Fetcher.fetch_and_contain_remote_object_from_id(
"https://mastodon.example.org/users/userisgone"
)
end
test "handle HTTP 404 response" do
assert {:error, :not_found} ==
Fetcher.fetch_and_contain_remote_object_from_id(
"https://mastodon.example.org/users/userisgone404"
)
end
test "it can fetch pleroma polls with attachments" do
{:ok, object} =
Fetcher.fetch_object_from_id("https://patch.cx/objects/tesla_mock/poll_attachment")
assert object
end
end
describe "pruning" do
test "it can refetch pruned objects" do
object_id = "http://mastodon.example.org/@admin/99541947525187367"
{:ok, object} = Fetcher.fetch_object_from_id(object_id)
assert object
{:ok, _object} = Object.prune(object)
refute Object.get_by_ap_id(object_id)
{:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
assert object.data["id"] == object_two.data["id"]
assert object.id != object_two.id
end
end
describe "signed fetches" do
setup do: clear_config([:activitypub, :sign_object_fetches])
test_with_mock "it signs fetches when configured to do so",
Pleroma.Signature,
[:passthrough],
[] do
clear_config([:activitypub, :sign_object_fetches], true)
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert called(Pleroma.Signature.sign(:_, :_))
end
test_with_mock "it doesn't sign fetches when not configured to do so",
Pleroma.Signature,
[:passthrough],
[] do
clear_config([:activitypub, :sign_object_fetches], false)
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
refute called(Pleroma.Signature.sign(:_, :_))
end
end
describe "refetching" do
setup do
insert(:user, ap_id: "https://mastodon.social/users/emelie")
object1 = %{
"id" => "https://mastodon.social/1",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 1",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:20Z"
}
{:ok, local_object1, _} = ObjectValidator.validate(object1, [])
object2 = %{
"id" => "https://mastodon.social/2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 2",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:25Z",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:21Z"
}
],
"totalItems" => 1
}
}
{:ok, local_object2, _} = ObjectValidator.validate(object2, [])
mock(fn
%{
method: :get,
url: "https://mastodon.social/1"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z"))
}
%{
method: :get,
url: "https://mastodon.social/2"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z"))
}
%{
method: :get,
url: "https://mastodon.social/users/emelie/collections/featured"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://mastodon.social/users/emelie/collections/featured",
"type" => "OrderedCollection",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"orderedItems" => [],
"totalItems" => 0
})
}
env ->
apply(HttpRequestMock, :request, [env])
end)
%{object1: local_object1, object2: local_object2}
end
test "it rejects a refetched object whose actor changed", %{object1: object1} do
insert(:user, ap_id: "https://mastodon.social/users/someone_else")
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
object1
|> Map.merge(%{
"actor" => "https://mastodon.social/users/someone_else",
"attributedTo" => "https://mastodon.social/users/someone_else",
"content" => "hijacked"
})
|> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
assert Object.get_by_ap_id("https://mastodon.social/1").data["content"] == "test 1"
end
test "it rejects a refetched object whose author has no id", %{object1: object1} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
object1
|> Map.delete("actor")
|> Map.put("attributedTo", %{"type" => "Person"})
|> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it rejects a refetched object whose id changed", %{object1: object1} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
url: "https://mastodon.social/2",
headers: [{"content-type", "application/activity+json"}],
body: object1 |> Map.put("id", "https://mastodon.social/2") |> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it rejects a refetched object whose id differs only by a trailing slash", %{
object1: object1
} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: object1 |> Map.put("id", "https://mastodon.social/1/") |> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z"
}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} =
refetched.data
end
test "it uses formerRepresentations from remote if possible", %{object2: object2} do
{:ok, o} = Object.create(object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} =
refetched.data
end
test "it replaces formerRepresentations with the one from remote", %{object2: object2} do
full_object2 =
object2
|> Map.merge(%{
"content" => "mew mew #def",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 2"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 2",
"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}
} = refetched.data
end
test "it adds to formerRepresentations if the remote does not have one and the object has changed",
%{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"content" => "mew mew #def",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 1"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 1",
"formerRepresentations" => %{
"orderedItems" => [
%{"content" => "mew mew #def"},
%{"content" => "mew mew 1"}
],
"totalItems" => 2
}
} = refetched.data
end
test "it keeps the history intact if only updated time has changed",
%{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"updated" => "2023-05-08 23:43:47Z",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 1"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 1",
"formerRepresentations" => %{
"orderedItems" => [
%{"content" => "mew mew 1"}
],
"totalItems" => 1
}
} = refetched.data
end
test "it goes through ObjectValidator and MRF", %{object2: object2} do
with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough],
filter: fn
%{"type" => "Note"} = object ->
{:ok, Map.put(object, "content", "MRFd content")}
arg ->
passthrough([arg])
end do
{:ok, o} = Object.create(object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"content" => "MRFd content"} = refetched.data
end
end
end
describe "cross-domain redirect handling" do
setup do
mock(fn
# Cross-domain redirect with original domain in id
%{method: :get, url: "https://original.test/objects/123"} ->
%Tesla.Env{
status: 200,
url: "https://media.test/objects/123",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/123",
"type" => "Note",
"content" => "This is redirected content",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Cross-domain redirect with final domain in id
%{method: :get, url: "https://original.test/objects/final-domain-id"} ->
%Tesla.Env{
status: 200,
url: "https://media.test/objects/final-domain-id",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://media.test/objects/final-domain-id",
"type" => "Note",
"content" => "This has final domain in id",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Same-domain redirect to the object id
%{method: :get, url: "https://original.test/@actor/redirect"} ->
%Tesla.Env{
status: 200,
url: "https://original.test/objects/redirect-target",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/redirect-target",
"type" => "Note",
"content" => "This has a same-domain redirect",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Same-domain redirect to a document claiming the original URL as id
%{method: :get, url: "https://original.test/objects/same-domain-redirect"} ->
%Tesla.Env{
status: 200,
url: "https://original.test/objects/different-path",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/same-domain-redirect",
"type" => "Note",
"content" => "This has a same-domain redirect",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Test case with missing url field in response (common in tests)
%{method: :get, url: "https://original.test/objects/missing-url"} ->
%Tesla.Env{
status: 200,
# No url field
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/missing-url",
"type" => "Note",
"content" => "This has no URL field in response",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
end)
:ok
end
test "it rejects objects from cross-domain redirects with original domain in id" do
assert {:error, {:cross_domain_redirect, true}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/123"
)
end
test "it rejects objects from cross-domain redirects with final domain in id" do
assert {:error, {:cross_domain_redirect, true}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/final-domain-id"
)
end
test "it accepts objects with same-domain redirects" do
assert {:ok, data} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/@actor/redirect"
)
assert data["content"] == "This has a same-domain redirect"
end
test "it rejects redirect targets that claim the original URL as id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/same-domain-redirect"
)
end
test "it handles responses without URL field (common in tests)" do
assert {:ok, data} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/missing-url"
)
assert data["content"] == "This has no URL field in response"
end
end
describe "id containment" do
setup do
note = fn id, content ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => id,
"type" => "Note",
"content" => content,
"actor" => "https://host.test/users/actor",
"attributedTo" => "https://host.test/users/actor"
})
}
end
mock(fn
# display URL serving the canonical object
%{method: :get, url: "https://host.test/@actor/1"} ->
note.("https://host.test/objects/1", "display")
%{method: :get, url: "https://host.test/objects/1"} ->
note.("https://host.test/objects/1", "canonical")
# uploaded file claiming to be another object on the same host
%{method: :get, url: "https://host.test/media/evil.json"} ->
note.("https://host.test/objects/2", "spoofed")
%{method: :get, url: "https://host.test/objects/2"} ->
note.("https://host.test/objects/2", "real")
# canonical id does not serve itself
%{method: :get, url: "https://host.test/media/loop.json"} ->
note.("https://host.test/objects/3", "loop")
%{method: :get, url: "https://host.test/objects/3"} ->
note.("https://host.test/objects/4", "other")
# canonical id redirects
%{method: :get, url: "https://host.test/media/redirect.json"} ->
note.("https://host.test/objects/5", "redirect")
%{method: :get, url: "https://host.test/objects/5"} ->
%{note.("https://host.test/objects/5", "moved") | url: "https://host.test/media/x"}
%{method: :get, url: "https://host.test/objects/slash"} ->
note.("https://host.test/objects/slash/", "slash")
%{method: :get, url: "https://host.test/objects/no-id"} ->
note.(nil, "no id")
end)
:ok
end
test "it refetches display URLs from the canonical id" do
assert {:ok, %{"id" => "https://host.test/objects/1", "content" => "canonical"}} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/@actor/1")
end
test "it does not trust a document claiming another id on the same host" do
assert {:ok, %{"id" => "https://host.test/objects/2", "content" => "real"}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/evil.json"
)
end
test "it refetches at most once" do
assert {:error, :id_mismatch} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/loop.json"
)
end
test "it does not follow redirects when refetching from the canonical id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/redirect.json"
)
end
test "it ignores a trailing slash" do
assert {:ok, %{"content" => "slash"}} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/objects/slash")
end
test "it accepts ids that only differ by URL encoding" do
clear_config(:test_url_encoding, true)
mock(fn %{method: :get, url: url} ->
# the encoder sends uppercase escapes
assert url in [
"https://host.test/users/%E3%81%82",
"https://host.test/notes/%E3%81%82"
]
id =
if url =~ "users",
do: "https://host.test/users/あ",
else: "https://host.test/notes/%e3%81%82"
%Tesla.Env{
status: 200,
url: url,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(%{"id" => id, "type" => "Note"})
}
end)
assert {:ok, _} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/users/あ")
assert {:ok, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/notes/%e3%81%82"
)
end
test "it rejects objects whose author has no id" do
mock(fn %{method: :get, url: "https://host.test/objects/anon" = url} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => url,
"type" => "Note",
"content" => "anon",
"attributedTo" => %{"type" => "Person", "name" => "nobody"}
})
}
end)
assert {:containment, :error} =
Fetcher.fetch_object_from_id("https://host.test/objects/anon")
end
test "it rejects documents without an id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/objects/no-id")
end
end
describe "fetch with history" do
setup do
object2 = %{
"id" => "https://mastodon.social/2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 2",
"bcc" => [],
"bto" => [],
"cc" => ["https://mastodon.social/users/emelie/followers"],
"to" => [],
"summary" => "",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => ["https://mastodon.social/users/emelie/followers"],
"to" => [],
"summary" => ""
}
],
"totalItems" => 1
}
}
mock(fn
%{
method: :get,
url: "https://mastodon.social/2"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object2)
}
%{
method: :get,
url: "https://mastodon.social/users/emelie/collections/featured"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://mastodon.social/users/emelie/collections/featured",
"type" => "OrderedCollection",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"orderedItems" => [],
"totalItems" => 0
})
}
env ->
apply(HttpRequestMock, :request, [env])
end)
%{object2: object2}
end
test "it gets history", %{object2: object2} do
{:ok, object} = Fetcher.fetch_object_from_id(object2["id"])
assert %{
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [%{}]
}
} = object.data
end
end
end
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index da10abdd2..fc99bee41 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -1,308 +1,333 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.WebFingerTest do
use Pleroma.DataCase, async: false
alias Pleroma.Web.WebFinger
import Pleroma.Factory
import Tesla.Mock
setup do
Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache)
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
describe "host meta" do
test "returns a link to the xml lrdd" do
host_info = WebFinger.host_meta()
assert String.contains?(host_info, Pleroma.Web.Endpoint.url())
end
end
describe "incoming webfinger request" do
test "works for fqns" do
user = insert(:user)
{:ok, result} =
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", "XML")
assert is_binary(result)
end
test "works for ap_ids" do
user = insert(:user)
{:ok, result} = WebFinger.webfinger(user.ap_id, "XML")
assert is_binary(result)
end
end
test "requires exact match for Endpoint host or WebFinger domain" do
clear_config([Pleroma.Web.WebFinger, :domain], "pleroma.dev")
user = insert(:user)
assert {:error, "Couldn't find user"} ==
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}xxxx", "JSON")
assert {:error, "Couldn't find user"} ==
WebFinger.webfinger("#{user.nickname}@pleroma.devxxxx", "JSON")
assert {:ok, _} =
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", "JSON")
assert {:ok, _} =
WebFinger.webfinger("#{user.nickname}@pleroma.dev", "JSON")
end
describe "fingering" do
test "returns error for nonsensical input" do
assert {:error, _} = WebFinger.finger("bliblablu")
assert {:error, _} = WebFinger.finger("pleroma.social")
end
test "returns error when there is no content-type header" do
Tesla.Mock.mock(fn
%{url: "https://social.heldscal.la/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/social.heldscal.la_host_meta")
}}
%{
url:
"https://social.heldscal.la/.well-known/webfinger?resource=acct:invalid_content@social.heldscal.la"
} ->
{:ok, %Tesla.Env{status: 200, body: ""}}
end)
user = "invalid_content@social.heldscal.la"
assert {:error, {:content_type, nil}} = WebFinger.finger(user)
end
test "returns error when fails parse xml or json" do
user = "invalid_content@social.heldscal.la"
assert {:error, %Jason.DecodeError{}} = WebFinger.finger(user)
end
test "returns the ActivityPub actor URI for an ActivityPub user" do
user = "framasoft@framatube.org"
{:ok, _data} = WebFinger.finger(user)
end
test "it works for AP-only user" do
user = "kpherox@mstdn.jp"
{:ok, data} = WebFinger.finger(user)
assert data["magic_key"] == nil
assert data["salmon"] == nil
assert data["topic"] == nil
assert data["subject"] == "acct:kPherox@mstdn.jp"
assert data["ap_id"] == "https://mstdn.jp/users/kPherox"
assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}"
end
test "it gets the xrd endpoint" do
{:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la")
assert template == "https://social.heldscal.la/.well-known/webfinger?resource={uri}"
end
test "it gets the xrd endpoint for hubzilla" do
{:ok, template} = WebFinger.find_lrdd_template("macgirvin.com")
assert template == "https://macgirvin.com/xrd/?uri={uri}"
end
test "it gets the xrd endpoint for statusnet" do
{:ok, template} = WebFinger.find_lrdd_template("status.alpicola.com")
assert template == "https://status.alpicola.com/main/xrd?uri={uri}"
end
test "it works with idna domains as nickname" do
nickname = "lain@" <> to_string(:idna.encode("zetsubou.みんな"))
{:ok, _data} = WebFinger.finger(nickname)
end
test "it works with idna domains as link" do
ap_id = "https://" <> to_string(:idna.encode("zetsubou.みんな")) <> "/users/lain"
{:ok, _data} = WebFinger.finger(ap_id)
end
test "respects json content-type" do
Tesla.Mock.mock(fn
%{
url:
"https://mastodon.social/.well-known/webfinger?resource=acct:emelie@mastodon.social"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json"),
headers: [{"content-type", "application/jrd+json"}]
}}
%{url: "https://mastodon.social/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mastodon.social_host_meta")
}}
end)
{:ok, _data} = WebFinger.finger("emelie@mastodon.social")
end
test "respects xml content-type" do
Tesla.Mock.mock(fn
%{
url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.xml"),
headers: [{"content-type", "application/xrd+xml"}]
}}
%{url: "https://pawoo.net/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
}}
end)
{:ok, _data} = WebFinger.finger("pekorino@pawoo.net")
end
test "refuses to process XML remote entities" do
Tesla.Mock.mock(fn
%{
url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/xml_external_entities.xml"),
headers: [{"content-type", "application/xrd+xml"}]
}}
%{url: "https://pawoo.net/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
}}
end)
assert :error = WebFinger.finger("pekorino@pawoo.net")
end
test "prevents spoofing" do
Tesla.Mock.mock(fn
%{
url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"),
headers: [{"content-type", "application/jrd+json"}]
}}
%{url: "https://gleasonator.com/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta")
}}
%{url: "https://whitehouse.gov/.well-known/webfinger?resource=acct:trump@whitehouse.gov"} ->
{:ok, %Tesla.Env{status: 404}}
end)
{:error, _data} = WebFinger.finger("alex@gleasonator.com")
end
test "prevents forgeries" do
Tesla.Mock.mock(fn
%{
url:
"https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"
} ->
fake_webfinger =
File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!()
Tesla.Mock.json(fake_webfinger)
%{url: url}
when url in [
"https://poa.st/.well-known/webfinger?resource=acct:graf@poa.st",
"https://fba.ryona.agency/.well-known/host-meta"
] ->
{:ok, %Tesla.Env{status: 404}}
end)
assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency")
end
test "prevents forgeries even when the spoofed subject exists on the target domain" do
Tesla.Mock.mock(fn
%{url: url}
when url in [
"https://attacker.example/.well-known/host-meta",
"https://victim.example/.well-known/host-meta"
] ->
{:ok, %Tesla.Env{status: 404}}
%{
url:
"https://attacker.example/.well-known/webfinger?resource=acct:alice@attacker.example"
} ->
Tesla.Mock.json(%{
"subject" => "acct:alice@victim.example",
"links" => [
%{
"rel" => "self",
"type" => "application/activity+json",
"href" => "https://attacker.example/users/alice"
}
]
})
%{url: "https://victim.example/.well-known/webfinger?resource=acct:alice@victim.example"} ->
Tesla.Mock.json(%{
"subject" => "acct:alice@victim.example",
"links" => [
%{
"rel" => "self",
"type" => "application/activity+json",
"href" => "https://victim.example/users/alice"
}
]
})
end)
assert {:error, _} = WebFinger.finger("alice@attacker.example")
end
test "works for correctly set up split-domain instances implementing host-meta redirect" do
{:ok, _data} = WebFinger.finger("a@pleroma.example")
{:ok, _data} = WebFinger.finger("a@sub.pleroma.example")
end
test "works for correctly set up split-domain instances without host-meta redirect" do
{:ok, _data} = WebFinger.finger("a@mastodon.example")
{:ok, _data} = WebFinger.finger("a@sub.mastodon.example")
end
end
+
+ describe "rate limiting" do
+ setup do
+ Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy)
+ test_pid = self()
+
+ Tesla.Mock.mock_global(fn %{url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+ end
+
+ test "backs off from a host that rate limits host-meta" do
+ assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example")
+ assert_received {:request, "https://ratelimited.example/.well-known/host-meta"}
+ refute_received {:request, _}
+
+ # a rate-limited host-meta lookup must not be cached as "no LRDD template"
+ assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example")
+
+ assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example")
+ assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example")
+ refute_received {:request, _}
+ end
+ end
end
diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs
index a3b900a9b..9945ca143 100644
--- a/test/pleroma/workers/remote_fetcher_worker_test.exs
+++ b/test/pleroma/workers/remote_fetcher_worker_test.exs
@@ -1,84 +1,98 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
use Pleroma.DataCase, async: false
use Oban.Testing, repo: Pleroma.Repo
alias Pleroma.Workers.RemoteFetcherWorker
@deleted_object_one "https://deleted-404.example.com/"
@deleted_object_two "https://deleted-410.example.com/"
@unauthorized_object "https://unauthorized.example.com/"
@depth_object "https://depth.example.com/"
@content_type_object "https://bad_content_type.example.com/"
+ @ratelimited_object "https://ratelimited.example.com/"
describe "RemoteFetcherWorker" do
setup do
Tesla.Mock.mock(fn
%{method: :get, url: @deleted_object_one} ->
%Tesla.Env{
status: 404
}
%{method: :get, url: @deleted_object_two} ->
%Tesla.Env{
status: 410
}
%{method: :get, url: @unauthorized_object} ->
%Tesla.Env{
status: 403
}
%{method: :get, url: @depth_object} ->
%Tesla.Env{
status: 200
}
+ %{method: :get, url: @ratelimited_object} ->
+ %Tesla.Env{
+ status: 429
+ }
+
%{method: :get, url: @content_type_object} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/json"}],
body: File.read!("test/fixtures/spoofed-object.json")
}
end)
end
test "does not retry jobs for a deleted object" do
[
%{"op" => "fetch_remote", "id" => @deleted_object_one},
%{"op" => "fetch_remote", "id" => @deleted_object_two}
]
|> Enum.each(fn job -> assert {:cancel, _} = perform_job(RemoteFetcherWorker, job) end)
end
test "does not retry jobs for an unauthorized object" do
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @unauthorized_object
})
end
test "does not retry jobs for an an object that exceeded depth" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @depth_object,
"depth" => 1
})
end
+ test "retries jobs for a rate-limited host" do
+ assert {:error, :ratelimit} =
+ perform_job(RemoteFetcherWorker, %{
+ "op" => "fetch_remote",
+ "id" => @ratelimited_object
+ })
+ end
+
test "does not retry jobs for when object returns wrong content type" do
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @content_type_object
})
end
end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Tue, Nov 17, 1:49 PM (8 h, 39 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1839576
Default Alt Text
(92 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline