Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
92 KB
Referenced Files
None
Subscribers
None
diff --git a/changelog.d/http-backoff.add b/changelog.d/http-backoff.add
new file mode 100644
index 000000000..5c9eb988b
--- /dev/null
+++ b/changelog.d/http-backoff.add
@@ -0,0 +1 @@
+Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour)
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index aebad6984..080dd1065 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -1,155 +1,155 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Pleroma do
@apps [
:restarter,
:ecto,
:ecto_sql,
:postgrex,
:db_connection,
:cachex,
:flake_id,
:swoosh,
:timex,
:fast_html,
:oban,
:logger_backends
]
- @cachex_children ["object", "user", "scrubber", "web_resp"]
+ @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"]
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Pleroma.Config.Holder.save_default()
Pleroma.Config.Oban.warn()
Pleroma.Application.limiters_setup()
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)
apps =
if adapter == Tesla.Adapter.Gun do
[:gun | @apps]
else
[:hackney | @apps]
end
Enum.each(apps, &Application.ensure_all_started/1)
oban_config = [
crontab: [],
repo: Pleroma.Repo,
log: false,
queues: [],
plugins: []
]
children =
[
Pleroma.Repo,
Pleroma.Emoji,
{Pleroma.Config.TransferTask, false},
Pleroma.Web.Endpoint,
{Oban, oban_config},
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
http_children(adapter) ++
search_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
Supervisor.start_link(children ++ cachex_children,
strategy: :one_for_one,
name: Pleroma.Supervisor
)
if Pleroma.Config.get(:env) not in [:test, :benchmark] do
pleroma_rebooted?()
end
end
defp pleroma_rebooted? do
if Restarter.Pleroma.rebooted?() do
:ok
else
Process.sleep(10)
pleroma_rebooted?()
end
end
def load_pleroma do
Application.load(:pleroma)
end
def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
Keyword.get(options, opt) || shell_prompt(prompt, defval, defname)
end
def shell_prompt(prompt, defval \\ nil, defname \\ nil) do
prompt_message = "#{prompt} [#{defname || defval}] "
input =
if mix_shell?(),
do: Mix.shell().prompt(prompt_message),
else: :io.get_line(prompt_message)
case input do
"\n" ->
case defval do
nil ->
shell_prompt(prompt, defval, defname)
defval ->
defval
end
input ->
String.trim(input)
end
end
def shell_info(message) do
if mix_shell?(),
do: Mix.shell().info(message),
else: IO.puts(message)
end
def shell_error(message) do
if mix_shell?(),
do: Mix.shell().error(message),
else: IO.puts(:stderr, message)
end
@doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)"
def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0)
def escape_sh_path(path) do
~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
end
defp http_children(Tesla.Adapter.Gun) do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
defp http_children(_), do: []
defp search_children do
case Pleroma.Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index adc054f3b..2c8eede1c 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -1,355 +1,356 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Application do
use Application
import Cachex.Spec
alias Pleroma.Config
require Logger
@name Mix.Project.config()[:name]
@version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url]
def name, do: @name
def version, do: @version
def named_version, do: @name <> " " <> @version
def repository, do: @repository
def user_agent do
if Process.whereis(Pleroma.Web.Endpoint) do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.Endpoint.url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
custom ->
custom
end
else
# fallback, if endpoint is not started yet
"Pleroma Data Loader"
end
end
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
# Scrubbers are compiled at runtime and therefore will cause a conflict
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
configure_logger()
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
if Config.get(:env) != :test do
Pleroma.ApplicationRequirements.verify!()
end
load_custom_modules()
Pleroma.Docs.JSON.compile()
limiters_setup()
adapter = Application.get_env(:tesla, :adapter)
if match?({Tesla.Adapter.Finch, _}, adapter) do
Logger.info("Starting Finch")
use_castore_for_otp_cacerts()
Finch.start_link(name: MyFinch)
end
# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
# due to protocol consolidation warnings
# :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18
if Version.compare(System.version(), "1.18.0") == :lt do
Code.compiler_options(warnings_as_errors: false)
end
# Define workers and child supervisors to be supervised
children =
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo
] ++
search_children() ++
[
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
] ++
cachex_children() ++
http_children(adapter) ++
[
Pleroma.Stats,
Pleroma.JobQueueMonitor,
{Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
{Oban, Config.get(Oban)},
Pleroma.Web.Endpoint
] ++
task_children() ++
streamer_registry() ++
background_migrators() ++
shout_child(shout_enabled?()) ++
[Pleroma.Gopher.Server] ++
[Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
# If we have a lot of caches, default max_restarts can cause test
# resets to fail.
# Go for the default 3 unless we're in test
max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts]
opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts]
Supervisor.start_link(children, opts)
end
def configure_logger do
Config.get([:logger, :backends], [])
|> Enum.each(fn backend ->
case backend_to_logger(backend) do
{:ok, logger} ->
add_logger(logger)
{:error, :console_backend} ->
Logger.warning(":console is no longer considered a backend and is enabled by default")
end
end)
end
defp add_logger(backend) do
case LoggerBackends.add(backend) do
{:ok, _} ->
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
{:error, :already_present} ->
:ok
{:error, reason} ->
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
end
end
defp backend_to_logger(:console), do: {:error, :console_backend}
defp backend_to_logger({:ex_syslogger = backend, name}) do
Logger.warning(
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
)
{:ok, {ExSyslogger, name}}
end
defp backend_to_logger(backend), do: {:ok, backend}
def load_custom_modules do
dir = Config.get([:modules, :runtime_dir])
if dir && File.exists?(dir) do
dir
|> Pleroma.Utils.compile_dir()
|> case do
{:error, _errors, _warnings} ->
raise "Invalid custom modules"
{:ok, modules, _warnings} ->
if Application.get_env(:pleroma, __MODULE__)[:load_custom_modules] do
Enum.each(modules, fn mod ->
Logger.info("Custom module loaded: #{inspect(mod)}")
end)
end
:ok
end
end
end
defp cachex_children do
[
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
build_cachex("scrubber_management", limit: 2500),
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500),
build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000
),
build_cachex("rel_me", limit: 2500),
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+ build_cachex("http_backoff", limit: 10_000),
build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
]
end
defp search_children do
case Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
defp emoji_packs_expiration,
do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
defp idempotency_expiration,
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp chat_message_id_idempotency_key_expiration,
do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
defp seconds_valid_interval,
do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
@spec build_cachex(String.t(), keyword()) :: map()
def build_cachex(type, opts),
do: %{
id: String.to_atom("cachex_" <> type),
start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
type: :worker
}
defp shout_enabled?, do: Config.get([:shout, :enabled])
defp streamer_registry do
if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do
[
{Registry,
[
name: Pleroma.Web.Streamer.registry(),
keys: :duplicate,
partitions: System.schedulers_online()
]}
]
else
[]
end
end
defp background_migrators do
if Application.get_env(:pleroma, __MODULE__)[:background_migrators] do
[
Pleroma.Migrators.HashtagsTableMigrator,
Pleroma.Migrators.ContextObjectsDeletionMigrator
]
else
[]
end
end
defp shout_child(true) do
[
Pleroma.Web.ShoutChannel.ShoutChannelState,
{Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
]
end
defp shout_child(_), do: []
defp task_children do
children = [
%{
id: :web_push_init,
start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
restart: :temporary
}
]
if Application.get_env(:pleroma, __MODULE__)[:internal_fetch] do
children ++
[
%{
id: :internal_fetch_init,
start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
restart: :temporary
}
]
else
children
end
end
# start hackney and gun pools in tests
defp http_children(adapter) do
if Application.get_env(:pleroma, __MODULE__)[:test_http_pools] do
http_children_hackney() ++ http_children_gun()
else
cond do
match?(Tesla.Adapter.Hackney, adapter) -> http_children_hackney()
match?(Tesla.Adapter.Gun, adapter) -> http_children_gun()
true -> []
end
end
end
# Mint trusts the OTP CA store, which comes from the OS. Make it the CAStore
# bundle that the other adapters trust. OTP 24 has no such store and Mint
# uses CAStore there anyway.
defp use_castore_for_otp_cacerts do
if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_load, 1) do
:ok = :public_key.cacerts_load(CAStore.file_path())
end
end
defp http_children_hackney do
pools = [:federation, :media]
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload | pools]
else
pools
end
for pool <- pools do
options = Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp http_children_gun do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
@spec limiters_setup() :: :ok
def limiters_setup do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Search,
Pleroma.Webhook.Notify
]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])
max_running = Keyword.get(mod_config, :max_running, 5)
max_waiting = Keyword.get(mod_config, :max_waiting, 5)
ConcurrentLimiter.new(module, max_running, max_waiting)
end)
end
end
diff --git a/lib/pleroma/http/backoff.ex b/lib/pleroma/http/backoff.ex
new file mode 100644
index 000000000..0d2b99742
--- /dev/null
+++ b/lib/pleroma/http/backoff.ex
@@ -0,0 +1,101 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Backoff do
+ @moduledoc """
+ GET wrapper that stops sending requests to a host after it answers
+ 429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
+
+ Based on Akkoma's `Pleroma.HTTP.Backoff`.
+ """
+
+ alias Pleroma.HTTP
+
+ require Logger
+
+ @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+ @cache :http_backoff_cache
+
+ @default_seconds 5 * 60
+ @max_seconds 60 * 60
+
+ @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
+ def get(url, headers \\ [], options \\ []) when is_binary(url) do
+ host = URI.parse(url).host
+
+ case @cachex.get(@cache, host) do
+ {:ok, true} ->
+ {:error, :ratelimit}
+
+ _ ->
+ url
+ |> HTTP.get(headers, options)
+ |> check_backoff(host)
+ end
+ end
+
+ defp check_backoff({:ok, %{status: status, headers: headers}}, host)
+ when status in [429, 503] do
+ seconds = backoff_seconds(headers, DateTime.utc_now())
+ Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
+ @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
+ {:error, :ratelimit}
+ end
+
+ defp check_backoff(response, _host), do: response
+
+ @doc """
+ Seconds to back off for, from the first valid `Retry-After` or
+ `X-RateLimit-Reset` header. Missing, invalid or past values give the
+ default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
+ """
+ @spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
+ def backoff_seconds(headers, now) do
+ [
+ parse_retry_after(header(headers, "retry-after"), now),
+ parse_timestamp(header(headers, "x-ratelimit-reset"), now)
+ ]
+ |> Enum.find(@default_seconds, &is_integer/1)
+ |> min(@max_seconds)
+ end
+
+ defp header(headers, name) do
+ Enum.find_value(headers, fn
+ {key, value} when is_binary(key) and is_binary(value) ->
+ if String.downcase(key) == name, do: String.trim(value)
+
+ _ ->
+ nil
+ end)
+ end
+
+ # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
+ defp parse_retry_after(nil, _now), do: nil
+
+ defp parse_retry_after(value, now) do
+ case Integer.parse(value) do
+ {seconds, ""} when seconds > 0 -> seconds
+ {_, ""} -> nil
+ _ -> parse_timestamp(value, now)
+ end
+ end
+
+ defp parse_timestamp(nil, _now), do: nil
+
+ defp parse_timestamp(value, now) do
+ with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
+ seconds when seconds > 0 <- DateTime.diff(datetime, now) do
+ seconds
+ else
+ _ -> nil
+ end
+ end
+
+ defp parse_datetime(value) do
+ with {:error, _} <- Timex.parse(value, "{RFC1123}"),
+ {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
+ {:ok, datetime}
+ end
+ end
+end
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 706ee317c..3313583cc 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -1,278 +1,278 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
alias Pleroma.Signature
alias Pleroma.Web.ActivityPub.InternalFetchActor
alias Pleroma.Web.ActivityPub.MRF
alias Pleroma.Web.ActivityPub.ObjectValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.Federator
require Logger
require Pleroma.Constants
@spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
defp reinject_object(%Object{data: %{}} = object, new_data) do
Logger.debug("Reinjecting object #{new_data["id"]}")
with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}),
{:ok, new_data} <- MRF.filter(new_data),
{:ok, new_object, _} <-
Object.Updater.do_update_and_invalidate_cache(
object,
new_data,
_touch_changeset? = true
) do
{:ok, new_object}
else
e ->
Logger.error("Error while processing object: #{inspect(e)}")
{:error, e}
end
end
defp reinject_object(_, new_data) do
with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do
{:ok, object}
else
e -> e
end
end
# The stored id is canonical, so the refetch must not redirect, and the
# remote must not swap in another object or author.
# Based on Akkoma 9061d148b and fee57eb37.
def refetch_object(%Object{data: %{"id" => id} = data} = object) do
with {:local, false} <- {:local, Object.local?(object)},
{:ok, new_data} <- fetch_and_contain(id, true),
{:id, true} <- {:id, new_data["id"] == id},
{:actor, true} <- {:actor, actor_of(new_data) == actor_of(data)},
{:ok, object} <- reinject_object(object, new_data) do
{:ok, object}
else
{:local, true} -> {:ok, object}
e -> {:error, e}
end
end
defp actor_of(data), do: CommonFixes.fix_actor(data)["actor"]
@typep fetcher_errors ::
:error | :reject | :allowed_depth | :fetch | :containment | :transmogrifier
# Note: will create a Create activity, which we need internally at the moment.
@spec fetch_object_from_id(String.t(), list()) ::
{:ok, Object.t()} | {fetcher_errors(), any()} | Pipeline.errors()
def fetch_object_from_id(id, options \\ []) do
with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
{_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
{_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
{_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
params <- prepare_activity_params(data),
{_, :ok} <- {:containment, Containment.contain_origin(id, params)},
{_, {:ok, activity}} <-
{:transmogrifier, Transmogrifier.handle_incoming(params, options)},
{_, _data, %Object{} = object} <-
{:object, data, Object.normalize(activity, fetch: false)} do
{:ok, object}
else
{:normalize, object = %Object{}} ->
{:ok, object}
{:fetch_object, %Object{} = object} ->
{:ok, object}
{:object, data, nil} ->
reinject_object(%Object{}, data)
e ->
Logger.metadata(object: id)
Logger.error("Object rejected while fetching #{id} #{inspect(e)}")
e
end
end
defp prepare_activity_params(data) do
%{
"type" => "Create",
# Should we seriously keep this attributedTo thing?
"actor" => data["actor"] || data["attributedTo"],
"object" => data
}
|> Maps.put_if_present("to", data["to"])
|> Maps.put_if_present("cc", data["cc"])
|> Maps.put_if_present("bto", data["bto"])
|> Maps.put_if_present("bcc", data["bcc"])
end
defp make_signature(id, date) do
uri = URI.parse(id)
signature =
InternalFetchActor.get_actor()
|> Signature.sign(%{
"(request-target)": "get #{uri.path}",
host: uri.host,
date: date
})
{"signature", signature}
end
defp sign_fetch(headers, id, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
[make_signature(id, date) | headers]
else
headers
end
end
defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
[{"date", date} | headers]
else
headers
end
end
def fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(%{"id" => id}),
do: fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(id) when is_binary(id),
do: fetch_and_contain(id, false)
def fetch_and_contain_remote_object_from_id(_id),
do: {:error, "id must be a string"}
# The fetched document's id must be the URL it came from. Display URLs
# (e.g. Mastodon's /@user/123) serve the object under another id, so
# if the id differs we refetch once from that id, which has to be on the
# same host, must not redirect, and must then match exactly.
# Based on Akkoma 8684964c5 and 940792f8b.
defp fetch_and_contain(id, canonical?) do
Logger.debug("Fetching object #{id} via AP")
with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
{_, true} <- {:mrf, MRF.id_filter(id)},
{_, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
{:ok, final_url, body} <- get_object(id),
{_, true} <- {:redirect, not canonical? or same_url?(final_url, id)},
{:ok, data} <- safe_json_decode(body),
:ok <- Containment.contain_origin_from_id(id, data),
{_, _, :ok} <- {:id, data, Containment.contain_id_to_fetch(final_url, data)} do
{:ok, data}
else
{:id, %{"id" => canonical_id}, :error} when is_binary(canonical_id) and not canonical? ->
fetch_and_contain(canonical_id, true)
{:id, _, :error} ->
{:error, :id_mismatch}
{:redirect, false} ->
{:error, :canonical_id_redirect}
{:scheme, _} ->
{:error, "Unsupported URI scheme"}
{:local_fetch, _} ->
{:error, "Trying to fetch local resource"}
{:error, e} ->
{:error, e}
{:mrf, false} ->
{:error, {:reject, "Filtered by id"}}
e ->
{:error, e}
end
end
defp same_url?(url, other), do: Containment.contain_id_to_fetch(url, %{"id" => other}) == :ok
defp check_crossdomain_redirect(final_host, _original_url) when is_nil(final_host) do
{:cross_domain_redirect, false}
end
defp check_crossdomain_redirect(final_host, original_url) do
{:cross_domain_redirect, final_host != URI.parse(original_url).host}
end
defp get_object(id) do
date = Pleroma.Signature.signed_date()
headers =
[{"accept", "application/activity+json"}]
|> maybe_date_fetch(date)
|> sign_fetch(id, date)
- case HTTP.get(id, headers) do
+ case HTTP.Backoff.get(id, headers) do
{:ok, %{body: body, status: code, headers: headers, url: final_url}}
when code in 200..299 ->
remote_host = if final_url, do: URI.parse(final_url).host, else: nil
with {:cross_domain_redirect, false} <- check_crossdomain_redirect(remote_host, id),
{_, content_type} <- List.keyfind(headers, "content-type", 0),
{:ok, _media_type} <- verify_content_type(content_type) do
{:ok, if(final_url in [nil, ""], do: id, else: final_url), body}
else
{:cross_domain_redirect, true} ->
{:error, {:cross_domain_redirect, true}}
error ->
error
end
# Handle the case where URL is not in the response (older HTTP library versions)
{:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case verify_content_type(content_type) do
{:ok, _} -> {:ok, id, body}
error -> error
end
_ ->
{:error, {:content_type, nil}}
end
{:ok, %{status: code}} when code in [401, 403] ->
{:error, :forbidden}
{:ok, %{status: code}} when code in [404, 410] ->
{:error, :not_found}
{:error, e} ->
{:error, e}
e ->
{:error, e}
end
end
defp safe_json_decode(nil), do: {:ok, nil}
defp safe_json_decode(json), do: Jason.decode(json)
defp verify_content_type(content_type) do
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", "activity+json", _} ->
{:ok, :activity_json}
{:ok, "application", "ld+json", %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
{:ok, :ld_json}
_ ->
{:error, {:content_type, content_type}}
end
end
end
diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex
index 62a2715f5..cb312405c 100644
--- a/lib/pleroma/web/web_finger.ex
+++ b/lib/pleroma/web/web_finger.ex
@@ -1,287 +1,295 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.WebFinger do
alias Pleroma.HTTP
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Publisher
alias Pleroma.Web.Endpoint
alias Pleroma.Web.XML
alias Pleroma.XmlBuilder
require Jason
require Logger
def host_meta do
base_url = Endpoint.url()
{
:XRD,
%{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"},
{
:Link,
%{
rel: "lrdd",
type: "application/xrd+xml",
template: "#{base_url}/.well-known/webfinger?resource={uri}"
}
}
}
|> XmlBuilder.to_doc()
end
def webfinger(resource, fmt) when fmt in ["XML", "JSON"] do
host = Pleroma.Web.Endpoint.host()
regex =
if webfinger_domain = Pleroma.Config.get([__MODULE__, :domain]) do
~r/(acct:)?(?<username>[a-z0-9A-Z_\.-]+)@(#{host}|#{webfinger_domain})$/
else
~r/(acct:)?(?<username>[a-z0-9A-Z_\.-]+)@#{host}$/
end
with %{"username" => username} <- Regex.named_captures(regex, resource),
%User{} = user <- User.get_cached_by_nickname(username) do
{:ok, represent_user(user, fmt)}
else
_e ->
with %User{} = user <- User.get_cached_by_ap_id(resource) do
{:ok, represent_user(user, fmt)}
else
_e ->
{:error, "Couldn't find user"}
end
end
end
defp gather_links(%User{} = user) do
[
%{
"rel" => "http://webfinger.net/rel/profile-page",
"type" => "text/html",
"href" => user.ap_id
}
] ++ Publisher.gather_webfinger_links(user)
end
defp gather_aliases(%User{} = user) do
[user.ap_id | user.also_known_as]
end
def represent_user(user, "JSON") do
%{
"subject" => "acct:#{user.nickname}@#{host()}",
"aliases" => gather_aliases(user),
"links" => gather_links(user)
}
end
def represent_user(user, "XML") do
aliases =
user
|> gather_aliases()
|> Enum.map(&{:Alias, &1})
links =
gather_links(user)
|> Enum.map(fn link -> {:Link, link} end)
{
:XRD,
%{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"},
[
{:Subject, "acct:#{user.nickname}@#{host()}"}
] ++ aliases ++ links
}
|> XmlBuilder.to_doc()
end
def host do
Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host()
end
defp webfinger_from_xml(body) do
with {:ok, doc} <- XML.parse_document(body) do
subject = XML.string_from_xpath("//Subject", doc)
subscribe_address =
~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template}
|> XML.string_from_xpath(doc)
ap_id =
~s{//Link[@rel="self" and @type="application/activity+json"]/@href}
|> XML.string_from_xpath(doc)
data = %{
"subject" => subject,
"subscribe_address" => subscribe_address,
"ap_id" => ap_id
}
{:ok, data}
end
end
defp webfinger_from_json(body) do
with {:ok, doc} <- Jason.decode(body) do
data =
Enum.reduce(doc["links"], %{"subject" => doc["subject"]}, fn link, data ->
case {link["type"], link["rel"]} do
{"application/activity+json", "self"} ->
Map.put(data, "ap_id", link["href"])
{"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} ->
Map.put(data, "ap_id", link["href"])
{nil, "http://ostatus.org/schema/1.0/subscribe"} ->
Map.put(data, "subscribe_address", link["template"])
_ ->
Logger.debug("Unhandled type: #{inspect(link["type"])}")
data
end
end)
{:ok, data}
end
end
def get_template_from_xml(body) do
xpath = "//Link[@rel='lrdd']/@template"
with {:ok, doc} <- XML.parse_document(body),
template when template != nil <- XML.string_from_xpath(xpath, doc) do
{:ok, template}
end
end
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def find_lrdd_template(domain) do
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
- {:commit, fetch_lrdd_template(domain)}
+ case fetch_lrdd_template(domain) do
+ # retry once the backoff expires instead of caching the miss
+ {:error, :ratelimit} = error -> {:ignore, error}
+ result -> {:commit, result}
+ end
end)
rescue
e -> {:error, "Cachex error: #{inspect(e)}"}
end
defp fetch_lrdd_template(domain) do
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
meta_url = "https://#{domain}/.well-known/host-meta"
- with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do
+ with {:ok, %{status: status, body: body}} when status in 200..299 <-
+ HTTP.Backoff.get(meta_url) do
get_template_from_xml(body)
else
+ {:error, :ratelimit} = error ->
+ error
+
error ->
Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}")
{:error, :lrdd_not_found}
end
end
defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
case find_lrdd_template(domain) do
{:ok, template} ->
String.replace(template, "{uri}", encoded_account)
_ ->
"https://#{domain}/.well-known/webfinger?resource=#{encoded_account}"
end
end
defp get_address_from_domain(domain, account) when is_binary(domain) do
encoded_account = URI.encode("acct:#{account}")
get_address_from_domain(domain, encoded_account)
end
defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
@spec finger(String.t()) :: {:ok, map()} | {:error, any()}
def finger(account), do: do_finger(account, true)
defp do_finger(account, follow_redirects) do
account = String.trim_leading(account, "@")
domain =
with [_name, domain] <- String.split(account, "@") do
domain
else
_e ->
URI.parse(account).host
end
with address when is_binary(address) <- get_address_from_domain(domain, account),
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
- HTTP.get(
+ HTTP.Backoff.get(
address,
[{"accept", "application/xrd+xml,application/jrd+json"}]
) do
case List.keyfind(headers, "content-type", 0) do
{_, content_type} ->
case Plug.Conn.Utils.media_type(content_type) do
{:ok, "application", subtype, _} when subtype in ~w(xrd+xml xml) ->
webfinger_from_xml(body)
{:ok, "application", subtype, _} when subtype in ~w(jrd+json json) ->
webfinger_from_json(body)
_ ->
{:error, {:content_type, content_type}}
end
_ ->
{:error, {:content_type, nil}}
end
|> case do
{:ok, data} ->
if follow_redirects do
validate_webfinger(address, data)
else
{:ok, data}
end
error ->
error
end
else
error ->
Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
error
end
end
defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
with [_name, acct_host] <- String.split(acct, "@"),
{_, resolved_url} <- {:address, get_address_from_domain(acct_host, subject)},
{_, true} <- {:url_match, resolved_webfinger_matches?(request_url, resolved_url, data)} do
{:ok, data}
else
_ -> {:error, {:webfinger_invalid, request_url, data}}
end
end
defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
defp resolved_webfinger_matches?(request_url, resolved_url, _data)
when request_url == resolved_url do
true
end
defp resolved_webfinger_matches?(
_request_url,
_resolved_url,
%{"subject" => "acct:" <> acct} = data
) do
with {:ok, %{"subject" => "acct:" <> new_acct} = new_data} <- do_finger(acct, false),
true <- acct == new_acct,
true <- webfinger_data_matches?(data, new_data) do
true
else
_ -> false
end
end
defp webfinger_data_matches?(%{"ap_id" => ap_id}, %{"ap_id" => ap_id}) when ap_id != "" do
true
end
defp webfinger_data_matches?(_data, _new_data), do: false
end
diff --git a/test/pleroma/http/backoff_test.exs b/test/pleroma/http/backoff_test.exs
new file mode 100644
index 000000000..48c8514bd
--- /dev/null
+++ b/test/pleroma/http/backoff_test.exs
@@ -0,0 +1,154 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Based on Akkoma's test/pleroma/http/backoff_test.exs
+defmodule Pleroma.HTTP.BackoffTest do
+ use Pleroma.DataCase, async: false
+
+ alias Pleroma.HTTP.Backoff
+
+ @cache :http_backoff_cache
+ @url "https://ratelimited.example.com/users/alice"
+ @host "ratelimited.example.com"
+
+ defp mock_response(status, headers \\ []) do
+ test_pid = self()
+
+ Tesla.Mock.mock(fn %{method: :get} ->
+ send(test_pid, :request)
+ %Tesla.Env{status: status, headers: headers, body: ""}
+ end)
+ end
+
+ defp ttl_seconds do
+ {:ok, ttl} = Cachex.ttl(@cache, @host)
+ div(ttl, 1000)
+ end
+
+ defp assert_ttl_about(expected) do
+ ttl = ttl_seconds()
+ assert ttl in (expected - 15)..expected//1, "expected a TTL near #{expected}, got #{ttl}"
+ end
+
+ describe "get/3" do
+ test "returns the response when the host is not backed off" do
+ mock_response(200)
+
+ assert {:ok, %Tesla.Env{status: 200}} = Backoff.get(@url)
+ assert {:ok, nil} = Cachex.get(@cache, @host)
+ end
+
+ test "does not send a request while the host is backed off" do
+ Cachex.put(@cache, @host, true)
+ mock_response(200)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ refute_received :request
+ end
+
+ for status <- [429, 503] do
+ test "backs off from the host after a #{status} response" do
+ mock_response(unquote(status))
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_received :request
+ assert {:ok, true} = Cachex.get(@cache, @host)
+
+ assert {:error, :ratelimit} = Backoff.get(@url <> "/other")
+ refute_received :request
+ end
+ end
+
+ test "uses a default backoff without headers" do
+ mock_response(429)
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+
+ test "only backs off from the host that answered" do
+ mock_response(429)
+ assert {:error, :ratelimit} = Backoff.get(@url)
+
+ mock_response(200)
+ assert {:ok, _} = Backoff.get("https://other.example.com/users/bob")
+ end
+ end
+
+ describe "backoff headers" do
+ test "parses Retry-After in seconds" do
+ mock_response(429, [{"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an HTTP-date" do
+ date =
+ DateTime.utc_now()
+ |> DateTime.add(600)
+ |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+ mock_response(503, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses Retry-After as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "parses X-RateLimit-Reset as an ISO 8601 timestamp" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "matches header names case-insensitively" do
+ mock_response(429, [{"Retry-After", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "prefers Retry-After over X-RateLimit-Reset" do
+ date = DateTime.utc_now() |> DateTime.add(1200) |> DateTime.to_iso8601()
+ mock_response(429, [{"x-ratelimit-reset", date}, {"retry-after", "600"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ test "falls back to the next header when one is invalid" do
+ date = DateTime.utc_now() |> DateTime.add(600) |> DateTime.to_iso8601()
+ mock_response(429, [{"retry-after", "soon"}, {"x-ratelimit-reset", date}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(600)
+ end
+
+ for value <- ["soon", "-5", "0", "12.5", "", "Wed, 21 Oct 2015 07:28:00 GMT"] do
+ test "ignores the invalid or past Retry-After value #{inspect(value)}" do
+ mock_response(429, [{"retry-after", unquote(value)}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(300)
+ end
+ end
+
+ test "caps absurd values at one hour" do
+ mock_response(429, [{"retry-after", "99999999999"}])
+
+ assert {:error, :ratelimit} = Backoff.get(@url)
+ assert_ttl_about(3600)
+ end
+ end
+end
diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs
index 61a6fb517..80b172a36 100644
--- a/test/pleroma/object/fetcher_test.exs
+++ b/test/pleroma/object/fetcher_test.exs
@@ -1,1034 +1,1055 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Object.FetcherTest do
use Pleroma.DataCase
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.Web.ActivityPub.ObjectValidator
require Pleroma.Constants
import Mock
import Pleroma.Factory
import Tesla.Mock
setup do
mock(fn
%{method: :get, url: "https://mastodon.example.org/users/userisgone"} ->
%Tesla.Env{status: 410}
%{method: :get, url: "https://mastodon.example.org/users/userisgone404"} ->
%Tesla.Env{status: 404}
%{
method: :get,
url:
"https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/json"}],
body: File.read!("test/fixtures/spoofed-object.json")
}
env ->
apply(HttpRequestMock, :request, [env])
end)
:ok
end
describe "error cases" do
setup do
mock(fn
%{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} ->
%Tesla.Env{
status: 200,
# Pleroma redirects /notice/:id to the object id
url: "https://social.sakamoto.gq/objects/f20f2497-66d9-4a52-a2e1-1be2a39c32c1",
body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://social.sakamoto.gq/users/eal"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/fetch_mocks/eal.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://busshi.moe/users/tuxcrafting/statuses/104410921027210069"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json"),
headers: HttpRequestMock.activitypub_object_headers()
}
%{method: :get, url: "https://busshi.moe/users/tuxcrafting"} ->
%Tesla.Env{
status: 500
}
%{
method: :get,
url: "https://stereophonic.space/objects/02997b83-3ea7-4b63-94af-ef3aa2d4ed17"
} ->
%Tesla.Env{
status: 500
}
end)
:ok
end
test "it works when fetching the OP actor errors out" do
# Here we simulate a case where the author of the OP can't be read
assert {:ok, _} =
Fetcher.fetch_object_from_id(
"https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"
)
end
end
+ describe "rate limiting" do
+ test "a backed-off host gets no further requests" do
+ test_pid = self()
+
+ mock(fn %{method: :get, url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+
+ first = "https://ratelimited.example.com/objects/1"
+ second = "https://ratelimited.example.com/objects/2"
+
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(first)
+ assert_received {:request, ^first}
+
+ assert {:error, :ratelimit} = Fetcher.fetch_and_contain_remote_object_from_id(second)
+ assert {:fetch, {:error, :ratelimit}} = Fetcher.fetch_object_from_id(second)
+ refute_received {:request, _}
+ end
+ end
+
describe "max thread distance restriction" do
@ap_id "http://mastodon.example.org/@admin/99541947525187367"
setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
test "it returns thread depth exceeded error if thread depth is exceeded" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:allowed_depth, false} = Fetcher.fetch_object_from_id(@ap_id, depth: 1)
end
test "it fetches object if max thread depth is restricted to 0 and depth is not specified" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id)
end
test "it fetches object if requested depth does not exceed max thread depth" do
clear_config([:instance, :federation_incoming_replies_max_depth], 10)
assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id, depth: 10)
end
end
describe "actor origin containment" do
test "it rejects objects with a bogus origin" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json")
end
test "it rejects objects when attributedTo is wrong (variant 1)" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json")
end
test "it rejects objects when attributedTo is wrong (variant 2)" do
{:containment, :error} =
Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json")
end
end
describe "fetching an object" do
test "it fetches an object" do
{:ok, object} =
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert _activity = Activity.get_create_by_object_ap_id(object.data["id"])
{:ok, object_again} =
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert [attachment] = object.data["attachment"]
assert is_list(attachment["url"])
assert object == object_again
end
test "Return MRF reason when fetched status is rejected by one" do
clear_config([:mrf_keyword, :reject], ["yeah"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
assert {:transmogrifier, {:reject, "[KeywordPolicy] Matches with rejected keyword"}} ==
Fetcher.fetch_object_from_id(
"http://mastodon.example.org/@admin/99541947525187367"
)
end
test "it does not fetch a spoofed object uploaded on an instance as an attachment" do
assert {:fetch, {:error, {:content_type, "application/json"}}} =
Fetcher.fetch_object_from_id(
"https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json"
)
end
test "it does not fetch from local instance" do
local_url = Pleroma.Web.Endpoint.url() <> "/objects/local_resource"
assert {:fetch, {:error, "Trying to fetch local resource"}} =
Fetcher.fetch_object_from_id(local_url)
end
test "it validates content-type headers according to ActivityPub spec" do
# Setup a mock for an object with invalid content-type
mock(fn
%{method: :get, url: "https://example.com/objects/invalid-content-type"} ->
%Tesla.Env{
status: 200,
# Not a valid AP content-type
headers: [{"content-type", "application/json"}],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/invalid-content-type",
"type" => "Note",
"content" => "This has an invalid content type",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
assert {:fetch, {:error, {:content_type, "application/json"}}} =
Fetcher.fetch_object_from_id("https://example.com/objects/invalid-content-type")
end
test "it accepts objects with application/ld+json and ActivityStreams profile" do
# Setup a mock for an object with ld+json content-type and AS profile
mock(fn
%{method: :get, url: "https://example.com/objects/valid-ld-json"} ->
%Tesla.Env{
status: 200,
headers: [
{"content-type",
"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""}
],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/valid-ld-json",
"type" => "Note",
"content" => "This has a valid ld+json content type",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
# This should pass if content-type validation works correctly
assert {:ok, object} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://example.com/objects/valid-ld-json"
)
assert object["content"] == "This has a valid ld+json content type"
end
test "it rejects objects with no content-type header" do
# Setup a mock for an object with no content-type header
mock(fn
%{method: :get, url: "https://example.com/objects/no-content-type"} ->
%Tesla.Env{
status: 200,
# No content-type header
headers: [],
body:
Jason.encode!(%{
"id" => "https://example.com/objects/no-content-type",
"type" => "Note",
"content" => "This has no content type header",
"actor" => "https://example.com/users/actor",
"attributedTo" => "https://example.com/users/actor"
})
}
end)
# We want to test that the request fails with a missing content-type error
# but the actual error is {:fetch, {:error, nil}} - we'll check for this format
result = Fetcher.fetch_object_from_id("https://example.com/objects/no-content-type")
assert {:fetch, {:error, nil}} = result
end
end
describe "implementation quirks" do
test "it can fetch plume articles" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
)
assert object
end
test "it can fetch peertube videos" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
)
assert object
end
test "it can fetch Mobilizon events" do
{:ok, object} =
Fetcher.fetch_object_from_id(
"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"
)
assert object
end
test "it can fetch wedistribute articles" do
{:ok, object} =
Fetcher.fetch_object_from_id("https://wedistribute.org/wp-json/pterotype/v1/object/85810")
assert object
end
test "all objects with fake directions are rejected by the object fetcher" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://info.pleroma.site/activity4.json"
)
end
test "handle HTTP 410 Gone response" do
assert {:error, :not_found} ==
Fetcher.fetch_and_contain_remote_object_from_id(
"https://mastodon.example.org/users/userisgone"
)
end
test "handle HTTP 404 response" do
assert {:error, :not_found} ==
Fetcher.fetch_and_contain_remote_object_from_id(
"https://mastodon.example.org/users/userisgone404"
)
end
test "it can fetch pleroma polls with attachments" do
{:ok, object} =
Fetcher.fetch_object_from_id("https://patch.cx/objects/tesla_mock/poll_attachment")
assert object
end
end
describe "pruning" do
test "it can refetch pruned objects" do
object_id = "http://mastodon.example.org/@admin/99541947525187367"
{:ok, object} = Fetcher.fetch_object_from_id(object_id)
assert object
{:ok, _object} = Object.prune(object)
refute Object.get_by_ap_id(object_id)
{:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
assert object.data["id"] == object_two.data["id"]
assert object.id != object_two.id
end
end
describe "signed fetches" do
setup do: clear_config([:activitypub, :sign_object_fetches])
test_with_mock "it signs fetches when configured to do so",
Pleroma.Signature,
[:passthrough],
[] do
clear_config([:activitypub, :sign_object_fetches], true)
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert called(Pleroma.Signature.sign(:_, :_))
end
test_with_mock "it doesn't sign fetches when not configured to do so",
Pleroma.Signature,
[:passthrough],
[] do
clear_config([:activitypub, :sign_object_fetches], false)
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
refute called(Pleroma.Signature.sign(:_, :_))
end
end
describe "refetching" do
setup do
insert(:user, ap_id: "https://mastodon.social/users/emelie")
object1 = %{
"id" => "https://mastodon.social/1",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 1",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:20Z"
}
{:ok, local_object1, _} = ObjectValidator.validate(object1, [])
object2 = %{
"id" => "https://mastodon.social/2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 2",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:25Z",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z",
"updated" => "2023-05-09 23:43:21Z"
}
],
"totalItems" => 1
}
}
{:ok, local_object2, _} = ObjectValidator.validate(object2, [])
mock(fn
%{
method: :get,
url: "https://mastodon.social/1"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object1 |> Map.put("updated", "2023-05-09 23:44:20Z"))
}
%{
method: :get,
url: "https://mastodon.social/2"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object2 |> Map.put("updated", "2023-05-09 23:44:20Z"))
}
%{
method: :get,
url: "https://mastodon.social/users/emelie/collections/featured"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://mastodon.social/users/emelie/collections/featured",
"type" => "OrderedCollection",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"orderedItems" => [],
"totalItems" => 0
})
}
env ->
apply(HttpRequestMock, :request, [env])
end)
%{object1: local_object1, object2: local_object2}
end
test "it rejects a refetched object whose actor changed", %{object1: object1} do
insert(:user, ap_id: "https://mastodon.social/users/someone_else")
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
object1
|> Map.merge(%{
"actor" => "https://mastodon.social/users/someone_else",
"attributedTo" => "https://mastodon.social/users/someone_else",
"content" => "hijacked"
})
|> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
assert Object.get_by_ap_id("https://mastodon.social/1").data["content"] == "test 1"
end
test "it rejects a refetched object whose author has no id", %{object1: object1} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
object1
|> Map.delete("actor")
|> Map.put("attributedTo", %{"type" => "Person"})
|> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it rejects a refetched object whose id changed", %{object1: object1} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
url: "https://mastodon.social/2",
headers: [{"content-type", "application/activity+json"}],
body: object1 |> Map.put("id", "https://mastodon.social/2") |> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it rejects a refetched object whose id differs only by a trailing slash", %{
object1: object1
} do
{:ok, o} = Object.create(object1)
mock(fn %{method: :get, url: "https://mastodon.social/1"} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: object1 |> Map.put("id", "https://mastodon.social/1/") |> Jason.encode!()
}
end)
assert {:error, _} = Fetcher.refetch_object(o)
end
test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => [],
"to" => [Pleroma.Constants.as_public()],
"summary" => "",
"published" => "2023-05-08 23:43:20Z"
}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} =
refetched.data
end
test "it uses formerRepresentations from remote if possible", %{object2: object2} do
{:ok, o} = Object.create(object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} =
refetched.data
end
test "it replaces formerRepresentations with the one from remote", %{object2: object2} do
full_object2 =
object2
|> Map.merge(%{
"content" => "mew mew #def",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 2"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 2",
"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}
} = refetched.data
end
test "it adds to formerRepresentations if the remote does not have one and the object has changed",
%{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"content" => "mew mew #def",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 1"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 1",
"formerRepresentations" => %{
"orderedItems" => [
%{"content" => "mew mew #def"},
%{"content" => "mew mew 1"}
],
"totalItems" => 2
}
} = refetched.data
end
test "it keeps the history intact if only updated time has changed",
%{object1: object1} do
full_object1 =
object1
|> Map.merge(%{
"updated" => "2023-05-08 23:43:47Z",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{"type" => "Note", "content" => "mew mew 1"}
],
"totalItems" => 1
}
})
{:ok, o} = Object.create(full_object1)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{
"content" => "test 1",
"formerRepresentations" => %{
"orderedItems" => [
%{"content" => "mew mew 1"}
],
"totalItems" => 1
}
} = refetched.data
end
test "it goes through ObjectValidator and MRF", %{object2: object2} do
with_mock Pleroma.Web.ActivityPub.MRF, [:passthrough],
filter: fn
%{"type" => "Note"} = object ->
{:ok, Map.put(object, "content", "MRFd content")}
arg ->
passthrough([arg])
end do
{:ok, o} = Object.create(object2)
assert {:ok, refetched} = Fetcher.refetch_object(o)
assert %{"content" => "MRFd content"} = refetched.data
end
end
end
describe "cross-domain redirect handling" do
setup do
mock(fn
# Cross-domain redirect with original domain in id
%{method: :get, url: "https://original.test/objects/123"} ->
%Tesla.Env{
status: 200,
url: "https://media.test/objects/123",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/123",
"type" => "Note",
"content" => "This is redirected content",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Cross-domain redirect with final domain in id
%{method: :get, url: "https://original.test/objects/final-domain-id"} ->
%Tesla.Env{
status: 200,
url: "https://media.test/objects/final-domain-id",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://media.test/objects/final-domain-id",
"type" => "Note",
"content" => "This has final domain in id",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Same-domain redirect to the object id
%{method: :get, url: "https://original.test/@actor/redirect"} ->
%Tesla.Env{
status: 200,
url: "https://original.test/objects/redirect-target",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/redirect-target",
"type" => "Note",
"content" => "This has a same-domain redirect",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Same-domain redirect to a document claiming the original URL as id
%{method: :get, url: "https://original.test/objects/same-domain-redirect"} ->
%Tesla.Env{
status: 200,
url: "https://original.test/objects/different-path",
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/same-domain-redirect",
"type" => "Note",
"content" => "This has a same-domain redirect",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
# Test case with missing url field in response (common in tests)
%{method: :get, url: "https://original.test/objects/missing-url"} ->
%Tesla.Env{
status: 200,
# No url field
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://original.test/objects/missing-url",
"type" => "Note",
"content" => "This has no URL field in response",
"actor" => "https://original.test/users/actor",
"attributedTo" => "https://original.test/users/actor"
})
}
end)
:ok
end
test "it rejects objects from cross-domain redirects with original domain in id" do
assert {:error, {:cross_domain_redirect, true}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/123"
)
end
test "it rejects objects from cross-domain redirects with final domain in id" do
assert {:error, {:cross_domain_redirect, true}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/final-domain-id"
)
end
test "it accepts objects with same-domain redirects" do
assert {:ok, data} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/@actor/redirect"
)
assert data["content"] == "This has a same-domain redirect"
end
test "it rejects redirect targets that claim the original URL as id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/same-domain-redirect"
)
end
test "it handles responses without URL field (common in tests)" do
assert {:ok, data} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://original.test/objects/missing-url"
)
assert data["content"] == "This has no URL field in response"
end
end
describe "id containment" do
setup do
note = fn id, content ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => id,
"type" => "Note",
"content" => content,
"actor" => "https://host.test/users/actor",
"attributedTo" => "https://host.test/users/actor"
})
}
end
mock(fn
# display URL serving the canonical object
%{method: :get, url: "https://host.test/@actor/1"} ->
note.("https://host.test/objects/1", "display")
%{method: :get, url: "https://host.test/objects/1"} ->
note.("https://host.test/objects/1", "canonical")
# uploaded file claiming to be another object on the same host
%{method: :get, url: "https://host.test/media/evil.json"} ->
note.("https://host.test/objects/2", "spoofed")
%{method: :get, url: "https://host.test/objects/2"} ->
note.("https://host.test/objects/2", "real")
# canonical id does not serve itself
%{method: :get, url: "https://host.test/media/loop.json"} ->
note.("https://host.test/objects/3", "loop")
%{method: :get, url: "https://host.test/objects/3"} ->
note.("https://host.test/objects/4", "other")
# canonical id redirects
%{method: :get, url: "https://host.test/media/redirect.json"} ->
note.("https://host.test/objects/5", "redirect")
%{method: :get, url: "https://host.test/objects/5"} ->
%{note.("https://host.test/objects/5", "moved") | url: "https://host.test/media/x"}
%{method: :get, url: "https://host.test/objects/slash"} ->
note.("https://host.test/objects/slash/", "slash")
%{method: :get, url: "https://host.test/objects/no-id"} ->
note.(nil, "no id")
end)
:ok
end
test "it refetches display URLs from the canonical id" do
assert {:ok, %{"id" => "https://host.test/objects/1", "content" => "canonical"}} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/@actor/1")
end
test "it does not trust a document claiming another id on the same host" do
assert {:ok, %{"id" => "https://host.test/objects/2", "content" => "real"}} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/evil.json"
)
end
test "it refetches at most once" do
assert {:error, :id_mismatch} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/loop.json"
)
end
test "it does not follow redirects when refetching from the canonical id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/media/redirect.json"
)
end
test "it ignores a trailing slash" do
assert {:ok, %{"content" => "slash"}} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/objects/slash")
end
test "it accepts ids that only differ by URL encoding" do
clear_config(:test_url_encoding, true)
mock(fn %{method: :get, url: url} ->
# the encoder sends uppercase escapes
assert url in [
"https://host.test/users/%E3%81%82",
"https://host.test/notes/%E3%81%82"
]
id =
if url =~ "users",
do: "https://host.test/users/あ",
else: "https://host.test/notes/%e3%81%82"
%Tesla.Env{
status: 200,
url: url,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(%{"id" => id, "type" => "Note"})
}
end)
assert {:ok, _} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/users/あ")
assert {:ok, _} =
Fetcher.fetch_and_contain_remote_object_from_id(
"https://host.test/notes/%e3%81%82"
)
end
test "it rejects objects whose author has no id" do
mock(fn %{method: :get, url: "https://host.test/objects/anon" = url} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => url,
"type" => "Note",
"content" => "anon",
"attributedTo" => %{"type" => "Person", "name" => "nobody"}
})
}
end)
assert {:containment, :error} =
Fetcher.fetch_object_from_id("https://host.test/objects/anon")
end
test "it rejects documents without an id" do
assert {:error, _} =
Fetcher.fetch_and_contain_remote_object_from_id("https://host.test/objects/no-id")
end
end
describe "fetch with history" do
setup do
object2 = %{
"id" => "https://mastodon.social/2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"type" => "Note",
"content" => "test 2",
"bcc" => [],
"bto" => [],
"cc" => ["https://mastodon.social/users/emelie/followers"],
"to" => [],
"summary" => "",
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [
%{
"type" => "Note",
"content" => "orig 2",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"bcc" => [],
"bto" => [],
"cc" => ["https://mastodon.social/users/emelie/followers"],
"to" => [],
"summary" => ""
}
],
"totalItems" => 1
}
}
mock(fn
%{
method: :get,
url: "https://mastodon.social/2"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body: Jason.encode!(object2)
}
%{
method: :get,
url: "https://mastodon.social/users/emelie/collections/featured"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/activity+json"}],
body:
Jason.encode!(%{
"id" => "https://mastodon.social/users/emelie/collections/featured",
"type" => "OrderedCollection",
"actor" => "https://mastodon.social/users/emelie",
"attributedTo" => "https://mastodon.social/users/emelie",
"orderedItems" => [],
"totalItems" => 0
})
}
env ->
apply(HttpRequestMock, :request, [env])
end)
%{object2: object2}
end
test "it gets history", %{object2: object2} do
{:ok, object} = Fetcher.fetch_object_from_id(object2["id"])
assert %{
"formerRepresentations" => %{
"type" => "OrderedCollection",
"orderedItems" => [%{}]
}
} = object.data
end
end
end
diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs
index da10abdd2..fc99bee41 100644
--- a/test/pleroma/web/web_finger_test.exs
+++ b/test/pleroma/web/web_finger_test.exs
@@ -1,308 +1,333 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.WebFingerTest do
use Pleroma.DataCase, async: false
alias Pleroma.Web.WebFinger
import Pleroma.Factory
import Tesla.Mock
setup do
Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache)
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
describe "host meta" do
test "returns a link to the xml lrdd" do
host_info = WebFinger.host_meta()
assert String.contains?(host_info, Pleroma.Web.Endpoint.url())
end
end
describe "incoming webfinger request" do
test "works for fqns" do
user = insert(:user)
{:ok, result} =
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", "XML")
assert is_binary(result)
end
test "works for ap_ids" do
user = insert(:user)
{:ok, result} = WebFinger.webfinger(user.ap_id, "XML")
assert is_binary(result)
end
end
test "requires exact match for Endpoint host or WebFinger domain" do
clear_config([Pleroma.Web.WebFinger, :domain], "pleroma.dev")
user = insert(:user)
assert {:error, "Couldn't find user"} ==
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}xxxx", "JSON")
assert {:error, "Couldn't find user"} ==
WebFinger.webfinger("#{user.nickname}@pleroma.devxxxx", "JSON")
assert {:ok, _} =
WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", "JSON")
assert {:ok, _} =
WebFinger.webfinger("#{user.nickname}@pleroma.dev", "JSON")
end
describe "fingering" do
test "returns error for nonsensical input" do
assert {:error, _} = WebFinger.finger("bliblablu")
assert {:error, _} = WebFinger.finger("pleroma.social")
end
test "returns error when there is no content-type header" do
Tesla.Mock.mock(fn
%{url: "https://social.heldscal.la/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/social.heldscal.la_host_meta")
}}
%{
url:
"https://social.heldscal.la/.well-known/webfinger?resource=acct:invalid_content@social.heldscal.la"
} ->
{:ok, %Tesla.Env{status: 200, body: ""}}
end)
user = "invalid_content@social.heldscal.la"
assert {:error, {:content_type, nil}} = WebFinger.finger(user)
end
test "returns error when fails parse xml or json" do
user = "invalid_content@social.heldscal.la"
assert {:error, %Jason.DecodeError{}} = WebFinger.finger(user)
end
test "returns the ActivityPub actor URI for an ActivityPub user" do
user = "framasoft@framatube.org"
{:ok, _data} = WebFinger.finger(user)
end
test "it works for AP-only user" do
user = "kpherox@mstdn.jp"
{:ok, data} = WebFinger.finger(user)
assert data["magic_key"] == nil
assert data["salmon"] == nil
assert data["topic"] == nil
assert data["subject"] == "acct:kPherox@mstdn.jp"
assert data["ap_id"] == "https://mstdn.jp/users/kPherox"
assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}"
end
test "it gets the xrd endpoint" do
{:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la")
assert template == "https://social.heldscal.la/.well-known/webfinger?resource={uri}"
end
test "it gets the xrd endpoint for hubzilla" do
{:ok, template} = WebFinger.find_lrdd_template("macgirvin.com")
assert template == "https://macgirvin.com/xrd/?uri={uri}"
end
test "it gets the xrd endpoint for statusnet" do
{:ok, template} = WebFinger.find_lrdd_template("status.alpicola.com")
assert template == "https://status.alpicola.com/main/xrd?uri={uri}"
end
test "it works with idna domains as nickname" do
nickname = "lain@" <> to_string(:idna.encode("zetsubou.みんな"))
{:ok, _data} = WebFinger.finger(nickname)
end
test "it works with idna domains as link" do
ap_id = "https://" <> to_string(:idna.encode("zetsubou.みんな")) <> "/users/lain"
{:ok, _data} = WebFinger.finger(ap_id)
end
test "respects json content-type" do
Tesla.Mock.mock(fn
%{
url:
"https://mastodon.social/.well-known/webfinger?resource=acct:emelie@mastodon.social"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json"),
headers: [{"content-type", "application/jrd+json"}]
}}
%{url: "https://mastodon.social/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/mastodon.social_host_meta")
}}
end)
{:ok, _data} = WebFinger.finger("emelie@mastodon.social")
end
test "respects xml content-type" do
Tesla.Mock.mock(fn
%{
url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.xml"),
headers: [{"content-type", "application/xrd+xml"}]
}}
%{url: "https://pawoo.net/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
}}
end)
{:ok, _data} = WebFinger.finger("pekorino@pawoo.net")
end
test "refuses to process XML remote entities" do
Tesla.Mock.mock(fn
%{
url: "https://pawoo.net/.well-known/webfinger?resource=acct:pekorino@pawoo.net"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/xml_external_entities.xml"),
headers: [{"content-type", "application/xrd+xml"}]
}}
%{url: "https://pawoo.net/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/pawoo.net_host_meta")
}}
end)
assert :error = WebFinger.finger("pekorino@pawoo.net")
end
test "prevents spoofing" do
Tesla.Mock.mock(fn
%{
url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com"
} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"),
headers: [{"content-type", "application/jrd+json"}]
}}
%{url: "https://gleasonator.com/.well-known/host-meta"} ->
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta")
}}
%{url: "https://whitehouse.gov/.well-known/webfinger?resource=acct:trump@whitehouse.gov"} ->
{:ok, %Tesla.Env{status: 404}}
end)
{:error, _data} = WebFinger.finger("alex@gleasonator.com")
end
test "prevents forgeries" do
Tesla.Mock.mock(fn
%{
url:
"https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"
} ->
fake_webfinger =
File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!()
Tesla.Mock.json(fake_webfinger)
%{url: url}
when url in [
"https://poa.st/.well-known/webfinger?resource=acct:graf@poa.st",
"https://fba.ryona.agency/.well-known/host-meta"
] ->
{:ok, %Tesla.Env{status: 404}}
end)
assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency")
end
test "prevents forgeries even when the spoofed subject exists on the target domain" do
Tesla.Mock.mock(fn
%{url: url}
when url in [
"https://attacker.example/.well-known/host-meta",
"https://victim.example/.well-known/host-meta"
] ->
{:ok, %Tesla.Env{status: 404}}
%{
url:
"https://attacker.example/.well-known/webfinger?resource=acct:alice@attacker.example"
} ->
Tesla.Mock.json(%{
"subject" => "acct:alice@victim.example",
"links" => [
%{
"rel" => "self",
"type" => "application/activity+json",
"href" => "https://attacker.example/users/alice"
}
]
})
%{url: "https://victim.example/.well-known/webfinger?resource=acct:alice@victim.example"} ->
Tesla.Mock.json(%{
"subject" => "acct:alice@victim.example",
"links" => [
%{
"rel" => "self",
"type" => "application/activity+json",
"href" => "https://victim.example/users/alice"
}
]
})
end)
assert {:error, _} = WebFinger.finger("alice@attacker.example")
end
test "works for correctly set up split-domain instances implementing host-meta redirect" do
{:ok, _data} = WebFinger.finger("a@pleroma.example")
{:ok, _data} = WebFinger.finger("a@sub.pleroma.example")
end
test "works for correctly set up split-domain instances without host-meta redirect" do
{:ok, _data} = WebFinger.finger("a@mastodon.example")
{:ok, _data} = WebFinger.finger("a@sub.mastodon.example")
end
end
+
+ describe "rate limiting" do
+ setup do
+ Mox.stub_with(Pleroma.CachexMock, Pleroma.CachexProxy)
+ test_pid = self()
+
+ Tesla.Mock.mock_global(fn %{url: url} ->
+ send(test_pid, {:request, url})
+ %Tesla.Env{status: 429, headers: [{"retry-after", "600"}]}
+ end)
+ end
+
+ test "backs off from a host that rate limits host-meta" do
+ assert {:error, :ratelimit} = WebFinger.finger("alice@ratelimited.example")
+ assert_received {:request, "https://ratelimited.example/.well-known/host-meta"}
+ refute_received {:request, _}
+
+ # a rate-limited host-meta lookup must not be cached as "no LRDD template"
+ assert {:ok, nil} = Cachex.get(:host_meta_cache, "ratelimited.example")
+
+ assert {:error, :ratelimit} = WebFinger.finger("bob@ratelimited.example")
+ assert {:error, :ratelimit} = WebFinger.find_lrdd_template("ratelimited.example")
+ refute_received {:request, _}
+ end
+ end
end
diff --git a/test/pleroma/workers/remote_fetcher_worker_test.exs b/test/pleroma/workers/remote_fetcher_worker_test.exs
index a3b900a9b..9945ca143 100644
--- a/test/pleroma/workers/remote_fetcher_worker_test.exs
+++ b/test/pleroma/workers/remote_fetcher_worker_test.exs
@@ -1,84 +1,98 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.RemoteFetcherWorkerTest do
use Pleroma.DataCase, async: false
use Oban.Testing, repo: Pleroma.Repo
alias Pleroma.Workers.RemoteFetcherWorker
@deleted_object_one "https://deleted-404.example.com/"
@deleted_object_two "https://deleted-410.example.com/"
@unauthorized_object "https://unauthorized.example.com/"
@depth_object "https://depth.example.com/"
@content_type_object "https://bad_content_type.example.com/"
+ @ratelimited_object "https://ratelimited.example.com/"
describe "RemoteFetcherWorker" do
setup do
Tesla.Mock.mock(fn
%{method: :get, url: @deleted_object_one} ->
%Tesla.Env{
status: 404
}
%{method: :get, url: @deleted_object_two} ->
%Tesla.Env{
status: 410
}
%{method: :get, url: @unauthorized_object} ->
%Tesla.Env{
status: 403
}
%{method: :get, url: @depth_object} ->
%Tesla.Env{
status: 200
}
+ %{method: :get, url: @ratelimited_object} ->
+ %Tesla.Env{
+ status: 429
+ }
+
%{method: :get, url: @content_type_object} ->
%Tesla.Env{
status: 200,
headers: [{"content-type", "application/json"}],
body: File.read!("test/fixtures/spoofed-object.json")
}
end)
end
test "does not retry jobs for a deleted object" do
[
%{"op" => "fetch_remote", "id" => @deleted_object_one},
%{"op" => "fetch_remote", "id" => @deleted_object_two}
]
|> Enum.each(fn job -> assert {:cancel, _} = perform_job(RemoteFetcherWorker, job) end)
end
test "does not retry jobs for an unauthorized object" do
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @unauthorized_object
})
end
test "does not retry jobs for an an object that exceeded depth" do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @depth_object,
"depth" => 1
})
end
+ test "retries jobs for a rate-limited host" do
+ assert {:error, :ratelimit} =
+ perform_job(RemoteFetcherWorker, %{
+ "op" => "fetch_remote",
+ "id" => @ratelimited_object
+ })
+ end
+
test "does not retry jobs for when object returns wrong content type" do
assert {:cancel, _} =
perform_job(RemoteFetcherWorker, %{
"op" => "fetch_remote",
"id" => @content_type_object
})
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Tue, Nov 17, 1:49 PM (8 h, 39 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1839576
Default Alt Text
(92 KB)

Event Timeline