Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
22 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index eedad7675..707200737 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -1,82 +1,83 @@
defmodule Pleroma.Application do
use Application
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec
import Cachex.Spec
# Define workers and child supervisors to be supervised
children =
[
# Start the Ecto repository
supervisor(Pleroma.Repo, []),
worker(Pleroma.Emoji, []),
# Start the endpoint when the application starts
supervisor(Pleroma.Web.Endpoint, []),
# Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
# worker(Pleroma.Worker, [arg1, arg2, arg3]),
worker(
Cachex,
[
:user_cache,
[
default_ttl: 25000,
ttl_interval: 1000,
limit: 2500
]
],
id: :cachex_user
),
worker(
Cachex,
[
:object_cache,
[
default_ttl: 25000,
ttl_interval: 1000,
limit: 2500
]
],
id: :cachex_object
),
worker(
Cachex,
[
:idempotency_cache,
[
expiration:
expiration(
default: :timer.seconds(6 * 60 * 60),
interval: :timer.seconds(60)
),
limit: 2500
]
],
id: :cachex_idem
),
worker(Pleroma.Web.Federator, []),
- worker(Pleroma.Stats, []),
- worker(Pleroma.Gopher.Server, [])
+ worker(Pleroma.Web.Federator.RetryQueue, []),
+ worker(Pleroma.Gopher.Server, []),
+ worker(Pleroma.Stats, [])
] ++
if Mix.env() == :test,
do: [],
else:
[worker(Pleroma.Web.Streamer, [])] ++
if(
!chat_enabled(),
do: [],
else: [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
)
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
Supervisor.start_link(children, opts)
end
defp chat_enabled do
Application.get_env(:pleroma, :chat, []) |> Keyword.get(:enabled)
end
end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 6554fd2ef..000883cc2 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -1,225 +1,222 @@
defmodule Pleroma.Web.Federator do
use GenServer
alias Pleroma.User
alias Pleroma.Activity
alias Pleroma.Web.{WebFinger, Websub}
+ alias Pleroma.Web.Federator.RetryQueue
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Relay
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.OStatus
require Logger
@websub Application.get_env(:pleroma, :websub)
@ostatus Application.get_env(:pleroma, :ostatus)
@httpoison Application.get_env(:pleroma, :httpoison)
@max_jobs 20
def init(args) do
{:ok, args}
end
def start_link do
spawn(fn ->
# 1 minute
Process.sleep(1000 * 60 * 1)
enqueue(:refresh_subscriptions, nil)
end)
GenServer.start_link(
__MODULE__,
%{
in: {:sets.new(), []},
out: {:sets.new(), []}
},
name: __MODULE__
)
end
def handle(:refresh_subscriptions, _) do
Logger.debug("Federator running refresh subscriptions")
Websub.refresh_subscriptions()
spawn(fn ->
# 6 hours
Process.sleep(1000 * 60 * 60 * 6)
enqueue(:refresh_subscriptions, nil)
end)
end
def handle(:request_subscription, websub) do
Logger.debug("Refreshing #{websub.topic}")
with {:ok, websub} <- Websub.request_subscription(websub) do
Logger.debug("Successfully refreshed #{websub.topic}")
else
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
end
end
def handle(:publish, activity) do
Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
{:ok, actor} = WebFinger.ensure_keys_present(actor)
if ActivityPub.is_public?(activity) do
if OStatus.is_representable?(activity) do
Logger.info(fn -> "Sending #{activity.data["id"]} out via WebSub" end)
Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
Logger.info(fn -> "Sending #{activity.data["id"]} out via Salmon" end)
Pleroma.Web.Salmon.publish(actor, activity)
end
if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
Relay.publish(activity)
end
end
Logger.info(fn -> "Sending #{activity.data["id"]} out via AP" end)
Pleroma.Web.ActivityPub.ActivityPub.publish(actor, activity)
end
end
def handle(:verify_websub, websub) do
Logger.debug(fn ->
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
end)
@websub.verify(websub)
end
def handle(:incoming_doc, doc) do
Logger.info("Got document, trying to parse")
@ostatus.handle_incoming(doc)
end
def handle(:incoming_ap_doc, params) do
Logger.info("Handling incoming AP activity")
params = Utils.normalize_params(params)
# NOTE: we use the actor ID to do the containment, this is fine because an
# actor shouldn't be acting on objects outside their own AP server.
with {:ok, _user} <- ap_enabled_actor(params["actor"]),
nil <- Activity.normalize(params["id"]),
:ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
{:ok, activity}
else
%Activity{} ->
Logger.info("Already had #{params["id"]}")
:error
_e ->
# Just drop those for now
Logger.info("Unhandled activity")
Logger.info(Poison.encode!(params, pretty: 2))
:error
end
end
def handle(:publish_single_ap, params) do
- ActivityPub.publish_one(params)
- end
-
- def handle(:publish_single_websub, %{xml: xml, topic: topic, callback: callback, secret: secret}) do
- signature = @websub.sign(secret || "", xml)
- Logger.debug(fn -> "Pushing #{topic} to #{callback}" end)
-
- with {:ok, %{status_code: code}} <-
- @httpoison.post(
- callback,
- xml,
- [
- {"Content-Type", "application/atom+xml"},
- {"X-Hub-Signature", "sha1=#{signature}"}
- ],
- timeout: 10000,
- recv_timeout: 20000,
- hackney: [pool: :default]
- ) do
- Logger.debug(fn -> "Pushed to #{callback}, code #{code}" end)
- else
- e ->
- Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+ case ActivityPub.publish_one(params) do
+ {:ok, _} ->
+ :ok
+
+ {:error, _} ->
+ RetryQueue.enqueue(params, ActivityPub)
+ end
+ end
+
+ def handle(
+ :publish_single_websub,
+ %{xml: xml, topic: topic, callback: callback, secret: secret} = params
+ ) do
+ case Websub.publish_one(params) do
+ {:ok, _} ->
+ :ok
+
+ {:error, _} ->
+ RetryQueue.enqueue(params, Websub)
end
end
def handle(type, _) do
Logger.debug(fn -> "Unknown task: #{type}" end)
{:error, "Don't know what to do with this"}
end
def enqueue(type, payload, priority \\ 1) do
if Pleroma.Config.get([:instance, :federating]) do
if Mix.env() == :test do
handle(type, payload)
else
GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
end
end
end
def maybe_start_job(running_jobs, queue) do
if :sets.size(running_jobs) < @max_jobs && queue != [] do
{{type, payload}, queue} = queue_pop(queue)
{:ok, pid} = Task.start(fn -> handle(type, payload) end)
mref = Process.monitor(pid)
{:sets.add_element(mref, running_jobs), queue}
else
{running_jobs, queue}
end
end
def handle_cast({:enqueue, type, payload, _priority}, state)
when type in [:incoming_doc, :incoming_ap_doc] do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_queue = enqueue_sorted(i_queue, {type, payload}, 1)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast({:enqueue, type, payload, _priority}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
o_queue = enqueue_sorted(o_queue, {type, payload}, 1)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast(m, state) do
IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}")
{:noreply, state}
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_running_jobs = :sets.del_element(ref, i_running_jobs)
o_running_jobs = :sets.del_element(ref, o_running_jobs)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def enqueue_sorted(queue, element, priority) do
[%{item: element, priority: priority} | queue]
|> Enum.sort_by(fn %{priority: priority} -> priority end)
end
def queue_pop([%{item: element} | queue]) do
{element, queue}
end
def ap_enabled_actor(id) do
user = User.get_by_ap_id(id)
if User.ap_enabled?(user) do
{:ok, user}
else
ActivityPub.make_user_from_ap_id(id)
end
end
end
diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex
new file mode 100644
index 000000000..06c094f26
--- /dev/null
+++ b/lib/pleroma/web/federator/retry_queue.ex
@@ -0,0 +1,71 @@
+defmodule Pleroma.Web.Federator.RetryQueue do
+ use GenServer
+ alias Pleroma.Web.{WebFinger, Websub}
+ alias Pleroma.Web.ActivityPub.ActivityPub
+ require Logger
+
+ @websub Application.get_env(:pleroma, :websub)
+ @ostatus Application.get_env(:pleroma, :websub)
+ @httpoison Application.get_env(:pleroma, :websub)
+ @instance Application.get_env(:pleroma, :websub)
+ # initial timeout, 5 min
+ @initial_timeout 30_000
+ @max_retries 5
+
+ def init(args) do
+ {:ok, args}
+ end
+
+ def start_link() do
+ GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+ end
+
+ def enqueue(data, transport, retries \\ 0) do
+ GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
+ end
+
+ def get_retry_params(retries) do
+ if retries > @max_retries do
+ {:drop, "Max retries reached"}
+ else
+ {:retry, growth_function(retries)}
+ end
+ end
+
+ def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_count} = state) do
+ case get_retry_params(retries) do
+ {:retry, timeout} ->
+ Process.send_after(
+ __MODULE__,
+ {:send, data, transport, retries},
+ growth_function(retries)
+ )
+
+ {:noreply, state}
+
+ {:drop, message} ->
+ Logger.debug(message)
+ {:noreply, %{state | dropped: drop_count + 1}}
+ end
+ end
+
+ def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
+ case transport.publish_one(data) do
+ {:ok, _} ->
+ {:noreply, %{state | delivered: delivery_count + 1}}
+
+ {:error, reason} ->
+ enqueue(data, transport, retries)
+ {:noreply, state}
+ end
+ end
+
+ def handle_info(unknown, state) do
+ Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
+ {:noreply, state}
+ end
+
+ defp growth_function(retries) do
+ round(@initial_timeout * :math.pow(retries, 3))
+ end
+end
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index e494811f9..396dcf045 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -1,255 +1,280 @@
defmodule Pleroma.Web.Websub do
alias Ecto.Changeset
alias Pleroma.Repo
alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
alias Pleroma.Web.OStatus.FeedRepresenter
alias Pleroma.Web.{XML, Endpoint, OStatus}
alias Pleroma.Web.Router.Helpers
require Logger
import Ecto.Query
@httpoison Application.get_env(:pleroma, :httpoison)
def verify(subscription, getter \\ &@httpoison.get/3) do
challenge = Base.encode16(:crypto.strong_rand_bytes(8))
lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at)
lease_seconds = lease_seconds |> to_string
params = %{
"hub.challenge": challenge,
"hub.lease_seconds": lease_seconds,
"hub.topic": subscription.topic,
"hub.mode": "subscribe"
}
url = hd(String.split(subscription.callback, "?"))
query = URI.parse(subscription.callback).query || ""
params = Map.merge(params, URI.decode_query(query))
with {:ok, response} <- getter.(url, [], params: params),
^challenge <- response.body do
changeset = Changeset.change(subscription, %{state: "active"})
Repo.update(changeset)
else
e ->
Logger.debug("Couldn't verify subscription")
Logger.debug(inspect(e))
{:error, subscription}
end
end
@supported_activities [
"Create",
"Follow",
"Like",
"Announce",
"Undo",
"Delete"
]
def publish(topic, user, %{data: %{"type" => type}} = activity)
when type in @supported_activities do
# TODO: Only send to still valid subscriptions.
query =
from(
sub in WebsubServerSubscription,
where: sub.topic == ^topic and sub.state == "active",
where: fragment("? > NOW()", sub.valid_until)
)
subscriptions = Repo.all(query)
Enum.each(subscriptions, fn sub ->
response =
user
|> FeedRepresenter.to_simple_form([activity], [user])
|> :xmerl.export_simple(:xmerl_xml)
|> to_string
data = %{
xml: response,
topic: topic,
callback: sub.callback,
secret: sub.secret
}
Pleroma.Web.Federator.enqueue(:publish_single_websub, data)
end)
end
def publish(_, _, _), do: ""
def sign(secret, doc) do
:crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16() |> String.downcase()
end
def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do
with {:ok, topic} <- valid_topic(params, user),
{:ok, lease_time} <- lease_time(params),
secret <- params["hub.secret"],
callback <- params["hub.callback"] do
subscription = get_subscription(topic, callback)
data = %{
state: subscription.state || "requested",
topic: topic,
secret: secret,
callback: callback
}
change = Changeset.change(subscription, data)
websub = Repo.insert_or_update!(change)
change =
Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)})
websub = Repo.update!(change)
Pleroma.Web.Federator.enqueue(:verify_websub, websub)
{:ok, websub}
else
{:error, reason} ->
Logger.debug("Couldn't create subscription")
Logger.debug(inspect(reason))
{:error, reason}
end
end
defp get_subscription(topic, callback) do
Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) ||
%WebsubServerSubscription{}
end
# Temp hack for mastodon.
defp lease_time(%{"hub.lease_seconds" => ""}) do
# three days
{:ok, 60 * 60 * 24 * 3}
end
defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do
{:ok, String.to_integer(lease_seconds)}
end
defp lease_time(_) do
# three days
{:ok, 60 * 60 * 24 * 3}
end
defp valid_topic(%{"hub.topic" => topic}, user) do
if topic == OStatus.feed_path(user) do
{:ok, OStatus.feed_path(user)}
else
{:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"}
end
end
def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
topic = subscribed.info["topic"]
# FIXME: Race condition, use transactions
{:ok, subscription} =
with subscription when not is_nil(subscription) <-
Repo.get_by(WebsubClientSubscription, topic: topic) do
subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq()
change = Ecto.Changeset.change(subscription, %{subscribers: subscribers})
Repo.update(change)
else
_e ->
subscription = %WebsubClientSubscription{
topic: topic,
hub: subscribed.info["hub"],
subscribers: [subscriber.ap_id],
state: "requested",
secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64(),
user: subscribed
}
Repo.insert(subscription)
end
requester.(subscription)
end
def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
with {:ok, response} <- getter.(topic),
status_code when status_code in 200..299 <- response.status_code,
body <- response.body,
doc <- XML.parse_document(body),
uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc),
hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do
name = XML.string_from_xpath("/feed/author[1]/name", doc)
preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
avatar = OStatus.make_avatar_object(doc)
bio = XML.string_from_xpath("/feed/author[1]/summary", doc)
{:ok,
%{
"uri" => uri,
"hub" => hub,
"nickname" => preferredUsername || name,
"name" => displayName || name,
"host" => URI.parse(uri).host,
"avatar" => avatar,
"bio" => bio
}}
else
e ->
{:error, e}
end
end
def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do
data = [
"hub.mode": "subscribe",
"hub.topic": websub.topic,
"hub.secret": websub.secret,
"hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id)
]
# This checks once a second if we are confirmed yet
websub_checker = fn ->
helper = fn helper ->
:timer.sleep(1000)
websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted")
if websub, do: websub, else: helper.(helper)
end
helper.(helper)
end
task = Task.async(websub_checker)
with {:ok, %{status_code: 202}} <-
poster.(websub.hub, {:form, data}, "Content-type": "application/x-www-form-urlencoded"),
{:ok, websub} <- Task.yield(task, timeout) do
{:ok, websub}
else
e ->
Task.shutdown(task)
change = Ecto.Changeset.change(websub, %{state: "rejected"})
{:ok, websub} = Repo.update(change)
Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end)
Logger.debug(fn -> "error: #{inspect(e)}" end)
{:error, websub}
end
end
def refresh_subscriptions(delta \\ 60 * 60 * 24) do
Logger.debug("Refreshing subscriptions")
cut_off = NaiveDateTime.add(NaiveDateTime.utc_now(), delta)
query = from(sub in WebsubClientSubscription, where: sub.valid_until < ^cut_off)
subs = Repo.all(query)
Enum.each(subs, fn sub ->
Pleroma.Web.Federator.enqueue(:request_subscription, sub)
end)
end
+
+ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+ signature = sign(secret || "", xml)
+ Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
+
+ with {:ok, %{status_code: code}} <-
+ @httpoison.post(
+ callback,
+ xml,
+ [
+ {"Content-Type", "application/atom+xml"},
+ {"X-Hub-Signature", "sha1=#{signature}"}
+ ],
+ timeout: 10000,
+ recv_timeout: 20000,
+ hackney: [pool: :default]
+ ) do
+ Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
+ {:ok, code}
+ else
+ e ->
+ Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+ {:error, e}
+ end
+ end
end
diff --git a/test/web/retry_queue_test.exs b/test/web/retry_queue_test.exs
new file mode 100644
index 000000000..ce2964993
--- /dev/null
+++ b/test/web/retry_queue_test.exs
@@ -0,0 +1,31 @@
+defmodule MockActivityPub do
+ def publish_one(ret) do
+ {ret, "success"}
+ end
+end
+
+defmodule Pleroma.ActivityTest do
+ use Pleroma.DataCase
+ alias Pleroma.Web.Federator.RetryQueue
+
+ @small_retry_count 0
+ @hopeless_retry_count 10
+
+ test "failed posts are retried" do
+ {:retry, _timeout} = RetryQueue.get_retry_params(@small_retry_count)
+
+ assert {:noreply, %{delivered: 1}} ==
+ RetryQueue.handle_info({:send, :ok, MockActivityPub, @small_retry_count}, %{
+ delivered: 0
+ })
+ end
+
+ test "posts that have been tried too many times are dropped" do
+ {:drop, _timeout} = RetryQueue.get_retry_params(@hopeless_retry_count)
+
+ assert {:noreply, %{dropped: 1}} ==
+ RetryQueue.handle_cast({:maybe_enqueue, %{}, nil, @hopeless_retry_count}, %{
+ dropped: 0
+ })
+ end
+end

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 19, 3:44 AM (1 d, 20 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1840905
Default Alt Text
(22 KB)

Event Timeline