Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85950955
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
22 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index eedad7675..707200737 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -1,82 +1,83 @@
defmodule Pleroma.Application do
use Application
# See http://elixir-lang.org/docs/stable/elixir/Application.html
# for more information on OTP Applications
def start(_type, _args) do
import Supervisor.Spec
import Cachex.Spec
# Define workers and child supervisors to be supervised
children =
[
# Start the Ecto repository
supervisor(Pleroma.Repo, []),
worker(Pleroma.Emoji, []),
# Start the endpoint when the application starts
supervisor(Pleroma.Web.Endpoint, []),
# Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3)
# worker(Pleroma.Worker, [arg1, arg2, arg3]),
worker(
Cachex,
[
:user_cache,
[
default_ttl: 25000,
ttl_interval: 1000,
limit: 2500
]
],
id: :cachex_user
),
worker(
Cachex,
[
:object_cache,
[
default_ttl: 25000,
ttl_interval: 1000,
limit: 2500
]
],
id: :cachex_object
),
worker(
Cachex,
[
:idempotency_cache,
[
expiration:
expiration(
default: :timer.seconds(6 * 60 * 60),
interval: :timer.seconds(60)
),
limit: 2500
]
],
id: :cachex_idem
),
worker(Pleroma.Web.Federator, []),
- worker(Pleroma.Stats, []),
- worker(Pleroma.Gopher.Server, [])
+ worker(Pleroma.Web.Federator.RetryQueue, []),
+ worker(Pleroma.Gopher.Server, []),
+ worker(Pleroma.Stats, [])
] ++
if Mix.env() == :test,
do: [],
else:
[worker(Pleroma.Web.Streamer, [])] ++
if(
!chat_enabled(),
do: [],
else: [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
)
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
Supervisor.start_link(children, opts)
end
defp chat_enabled do
Application.get_env(:pleroma, :chat, []) |> Keyword.get(:enabled)
end
end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 6554fd2ef..000883cc2 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -1,225 +1,222 @@
defmodule Pleroma.Web.Federator do
use GenServer
alias Pleroma.User
alias Pleroma.Activity
alias Pleroma.Web.{WebFinger, Websub}
+ alias Pleroma.Web.Federator.RetryQueue
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Relay
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.OStatus
require Logger
@websub Application.get_env(:pleroma, :websub)
@ostatus Application.get_env(:pleroma, :ostatus)
@httpoison Application.get_env(:pleroma, :httpoison)
@max_jobs 20
def init(args) do
{:ok, args}
end
def start_link do
spawn(fn ->
# 1 minute
Process.sleep(1000 * 60 * 1)
enqueue(:refresh_subscriptions, nil)
end)
GenServer.start_link(
__MODULE__,
%{
in: {:sets.new(), []},
out: {:sets.new(), []}
},
name: __MODULE__
)
end
def handle(:refresh_subscriptions, _) do
Logger.debug("Federator running refresh subscriptions")
Websub.refresh_subscriptions()
spawn(fn ->
# 6 hours
Process.sleep(1000 * 60 * 60 * 6)
enqueue(:refresh_subscriptions, nil)
end)
end
def handle(:request_subscription, websub) do
Logger.debug("Refreshing #{websub.topic}")
with {:ok, websub} <- Websub.request_subscription(websub) do
Logger.debug("Successfully refreshed #{websub.topic}")
else
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
end
end
def handle(:publish, activity) do
Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
{:ok, actor} = WebFinger.ensure_keys_present(actor)
if ActivityPub.is_public?(activity) do
if OStatus.is_representable?(activity) do
Logger.info(fn -> "Sending #{activity.data["id"]} out via WebSub" end)
Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
Logger.info(fn -> "Sending #{activity.data["id"]} out via Salmon" end)
Pleroma.Web.Salmon.publish(actor, activity)
end
if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
Relay.publish(activity)
end
end
Logger.info(fn -> "Sending #{activity.data["id"]} out via AP" end)
Pleroma.Web.ActivityPub.ActivityPub.publish(actor, activity)
end
end
def handle(:verify_websub, websub) do
Logger.debug(fn ->
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
end)
@websub.verify(websub)
end
def handle(:incoming_doc, doc) do
Logger.info("Got document, trying to parse")
@ostatus.handle_incoming(doc)
end
def handle(:incoming_ap_doc, params) do
Logger.info("Handling incoming AP activity")
params = Utils.normalize_params(params)
# NOTE: we use the actor ID to do the containment, this is fine because an
# actor shouldn't be acting on objects outside their own AP server.
with {:ok, _user} <- ap_enabled_actor(params["actor"]),
nil <- Activity.normalize(params["id"]),
:ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
{:ok, activity}
else
%Activity{} ->
Logger.info("Already had #{params["id"]}")
:error
_e ->
# Just drop those for now
Logger.info("Unhandled activity")
Logger.info(Poison.encode!(params, pretty: 2))
:error
end
end
def handle(:publish_single_ap, params) do
- ActivityPub.publish_one(params)
- end
-
- def handle(:publish_single_websub, %{xml: xml, topic: topic, callback: callback, secret: secret}) do
- signature = @websub.sign(secret || "", xml)
- Logger.debug(fn -> "Pushing #{topic} to #{callback}" end)
-
- with {:ok, %{status_code: code}} <-
- @httpoison.post(
- callback,
- xml,
- [
- {"Content-Type", "application/atom+xml"},
- {"X-Hub-Signature", "sha1=#{signature}"}
- ],
- timeout: 10000,
- recv_timeout: 20000,
- hackney: [pool: :default]
- ) do
- Logger.debug(fn -> "Pushed to #{callback}, code #{code}" end)
- else
- e ->
- Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+ case ActivityPub.publish_one(params) do
+ {:ok, _} ->
+ :ok
+
+ {:error, _} ->
+ RetryQueue.enqueue(params, ActivityPub)
+ end
+ end
+
+ def handle(
+ :publish_single_websub,
+ %{xml: xml, topic: topic, callback: callback, secret: secret} = params
+ ) do
+ case Websub.publish_one(params) do
+ {:ok, _} ->
+ :ok
+
+ {:error, _} ->
+ RetryQueue.enqueue(params, Websub)
end
end
def handle(type, _) do
Logger.debug(fn -> "Unknown task: #{type}" end)
{:error, "Don't know what to do with this"}
end
def enqueue(type, payload, priority \\ 1) do
if Pleroma.Config.get([:instance, :federating]) do
if Mix.env() == :test do
handle(type, payload)
else
GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
end
end
end
def maybe_start_job(running_jobs, queue) do
if :sets.size(running_jobs) < @max_jobs && queue != [] do
{{type, payload}, queue} = queue_pop(queue)
{:ok, pid} = Task.start(fn -> handle(type, payload) end)
mref = Process.monitor(pid)
{:sets.add_element(mref, running_jobs), queue}
else
{running_jobs, queue}
end
end
def handle_cast({:enqueue, type, payload, _priority}, state)
when type in [:incoming_doc, :incoming_ap_doc] do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_queue = enqueue_sorted(i_queue, {type, payload}, 1)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast({:enqueue, type, payload, _priority}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
o_queue = enqueue_sorted(o_queue, {type, payload}, 1)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast(m, state) do
IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}")
{:noreply, state}
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_running_jobs = :sets.del_element(ref, i_running_jobs)
o_running_jobs = :sets.del_element(ref, o_running_jobs)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def enqueue_sorted(queue, element, priority) do
[%{item: element, priority: priority} | queue]
|> Enum.sort_by(fn %{priority: priority} -> priority end)
end
def queue_pop([%{item: element} | queue]) do
{element, queue}
end
def ap_enabled_actor(id) do
user = User.get_by_ap_id(id)
if User.ap_enabled?(user) do
{:ok, user}
else
ActivityPub.make_user_from_ap_id(id)
end
end
end
diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex
new file mode 100644
index 000000000..06c094f26
--- /dev/null
+++ b/lib/pleroma/web/federator/retry_queue.ex
@@ -0,0 +1,71 @@
+defmodule Pleroma.Web.Federator.RetryQueue do
+ use GenServer
+ alias Pleroma.Web.{WebFinger, Websub}
+ alias Pleroma.Web.ActivityPub.ActivityPub
+ require Logger
+
+ @websub Application.get_env(:pleroma, :websub)
+ @ostatus Application.get_env(:pleroma, :websub)
+ @httpoison Application.get_env(:pleroma, :websub)
+ @instance Application.get_env(:pleroma, :websub)
+ # initial timeout, 5 min
+ @initial_timeout 30_000
+ @max_retries 5
+
+ def init(args) do
+ {:ok, args}
+ end
+
+ def start_link() do
+ GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+ end
+
+ def enqueue(data, transport, retries \\ 0) do
+ GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
+ end
+
+ def get_retry_params(retries) do
+ if retries > @max_retries do
+ {:drop, "Max retries reached"}
+ else
+ {:retry, growth_function(retries)}
+ end
+ end
+
+ def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_count} = state) do
+ case get_retry_params(retries) do
+ {:retry, timeout} ->
+ Process.send_after(
+ __MODULE__,
+ {:send, data, transport, retries},
+ growth_function(retries)
+ )
+
+ {:noreply, state}
+
+ {:drop, message} ->
+ Logger.debug(message)
+ {:noreply, %{state | dropped: drop_count + 1}}
+ end
+ end
+
+ def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
+ case transport.publish_one(data) do
+ {:ok, _} ->
+ {:noreply, %{state | delivered: delivery_count + 1}}
+
+ {:error, reason} ->
+ enqueue(data, transport, retries)
+ {:noreply, state}
+ end
+ end
+
+ def handle_info(unknown, state) do
+ Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
+ {:noreply, state}
+ end
+
+ defp growth_function(retries) do
+ round(@initial_timeout * :math.pow(retries, 3))
+ end
+end
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index e494811f9..396dcf045 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -1,255 +1,280 @@
defmodule Pleroma.Web.Websub do
alias Ecto.Changeset
alias Pleroma.Repo
alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
alias Pleroma.Web.OStatus.FeedRepresenter
alias Pleroma.Web.{XML, Endpoint, OStatus}
alias Pleroma.Web.Router.Helpers
require Logger
import Ecto.Query
@httpoison Application.get_env(:pleroma, :httpoison)
def verify(subscription, getter \\ &@httpoison.get/3) do
challenge = Base.encode16(:crypto.strong_rand_bytes(8))
lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at)
lease_seconds = lease_seconds |> to_string
params = %{
"hub.challenge": challenge,
"hub.lease_seconds": lease_seconds,
"hub.topic": subscription.topic,
"hub.mode": "subscribe"
}
url = hd(String.split(subscription.callback, "?"))
query = URI.parse(subscription.callback).query || ""
params = Map.merge(params, URI.decode_query(query))
with {:ok, response} <- getter.(url, [], params: params),
^challenge <- response.body do
changeset = Changeset.change(subscription, %{state: "active"})
Repo.update(changeset)
else
e ->
Logger.debug("Couldn't verify subscription")
Logger.debug(inspect(e))
{:error, subscription}
end
end
@supported_activities [
"Create",
"Follow",
"Like",
"Announce",
"Undo",
"Delete"
]
def publish(topic, user, %{data: %{"type" => type}} = activity)
when type in @supported_activities do
# TODO: Only send to still valid subscriptions.
query =
from(
sub in WebsubServerSubscription,
where: sub.topic == ^topic and sub.state == "active",
where: fragment("? > NOW()", sub.valid_until)
)
subscriptions = Repo.all(query)
Enum.each(subscriptions, fn sub ->
response =
user
|> FeedRepresenter.to_simple_form([activity], [user])
|> :xmerl.export_simple(:xmerl_xml)
|> to_string
data = %{
xml: response,
topic: topic,
callback: sub.callback,
secret: sub.secret
}
Pleroma.Web.Federator.enqueue(:publish_single_websub, data)
end)
end
def publish(_, _, _), do: ""
def sign(secret, doc) do
:crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16() |> String.downcase()
end
def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do
with {:ok, topic} <- valid_topic(params, user),
{:ok, lease_time} <- lease_time(params),
secret <- params["hub.secret"],
callback <- params["hub.callback"] do
subscription = get_subscription(topic, callback)
data = %{
state: subscription.state || "requested",
topic: topic,
secret: secret,
callback: callback
}
change = Changeset.change(subscription, data)
websub = Repo.insert_or_update!(change)
change =
Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)})
websub = Repo.update!(change)
Pleroma.Web.Federator.enqueue(:verify_websub, websub)
{:ok, websub}
else
{:error, reason} ->
Logger.debug("Couldn't create subscription")
Logger.debug(inspect(reason))
{:error, reason}
end
end
defp get_subscription(topic, callback) do
Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) ||
%WebsubServerSubscription{}
end
# Temp hack for mastodon.
defp lease_time(%{"hub.lease_seconds" => ""}) do
# three days
{:ok, 60 * 60 * 24 * 3}
end
defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do
{:ok, String.to_integer(lease_seconds)}
end
defp lease_time(_) do
# three days
{:ok, 60 * 60 * 24 * 3}
end
defp valid_topic(%{"hub.topic" => topic}, user) do
if topic == OStatus.feed_path(user) do
{:ok, OStatus.feed_path(user)}
else
{:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"}
end
end
def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
topic = subscribed.info["topic"]
# FIXME: Race condition, use transactions
{:ok, subscription} =
with subscription when not is_nil(subscription) <-
Repo.get_by(WebsubClientSubscription, topic: topic) do
subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq()
change = Ecto.Changeset.change(subscription, %{subscribers: subscribers})
Repo.update(change)
else
_e ->
subscription = %WebsubClientSubscription{
topic: topic,
hub: subscribed.info["hub"],
subscribers: [subscriber.ap_id],
state: "requested",
secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64(),
user: subscribed
}
Repo.insert(subscription)
end
requester.(subscription)
end
def gather_feed_data(topic, getter \\ &@httpoison.get/1) do
with {:ok, response} <- getter.(topic),
status_code when status_code in 200..299 <- response.status_code,
body <- response.body,
doc <- XML.parse_document(body),
uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc),
hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do
name = XML.string_from_xpath("/feed/author[1]/name", doc)
preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
avatar = OStatus.make_avatar_object(doc)
bio = XML.string_from_xpath("/feed/author[1]/summary", doc)
{:ok,
%{
"uri" => uri,
"hub" => hub,
"nickname" => preferredUsername || name,
"name" => displayName || name,
"host" => URI.parse(uri).host,
"avatar" => avatar,
"bio" => bio
}}
else
e ->
{:error, e}
end
end
def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do
data = [
"hub.mode": "subscribe",
"hub.topic": websub.topic,
"hub.secret": websub.secret,
"hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id)
]
# This checks once a second if we are confirmed yet
websub_checker = fn ->
helper = fn helper ->
:timer.sleep(1000)
websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted")
if websub, do: websub, else: helper.(helper)
end
helper.(helper)
end
task = Task.async(websub_checker)
with {:ok, %{status_code: 202}} <-
poster.(websub.hub, {:form, data}, "Content-type": "application/x-www-form-urlencoded"),
{:ok, websub} <- Task.yield(task, timeout) do
{:ok, websub}
else
e ->
Task.shutdown(task)
change = Ecto.Changeset.change(websub, %{state: "rejected"})
{:ok, websub} = Repo.update(change)
Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end)
Logger.debug(fn -> "error: #{inspect(e)}" end)
{:error, websub}
end
end
def refresh_subscriptions(delta \\ 60 * 60 * 24) do
Logger.debug("Refreshing subscriptions")
cut_off = NaiveDateTime.add(NaiveDateTime.utc_now(), delta)
query = from(sub in WebsubClientSubscription, where: sub.valid_until < ^cut_off)
subs = Repo.all(query)
Enum.each(subs, fn sub ->
Pleroma.Web.Federator.enqueue(:request_subscription, sub)
end)
end
+
+ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+ signature = sign(secret || "", xml)
+ Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
+
+ with {:ok, %{status_code: code}} <-
+ @httpoison.post(
+ callback,
+ xml,
+ [
+ {"Content-Type", "application/atom+xml"},
+ {"X-Hub-Signature", "sha1=#{signature}"}
+ ],
+ timeout: 10000,
+ recv_timeout: 20000,
+ hackney: [pool: :default]
+ ) do
+ Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
+ {:ok, code}
+ else
+ e ->
+ Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+ {:error, e}
+ end
+ end
end
diff --git a/test/web/retry_queue_test.exs b/test/web/retry_queue_test.exs
new file mode 100644
index 000000000..ce2964993
--- /dev/null
+++ b/test/web/retry_queue_test.exs
@@ -0,0 +1,31 @@
+defmodule MockActivityPub do
+ def publish_one(ret) do
+ {ret, "success"}
+ end
+end
+
+defmodule Pleroma.ActivityTest do
+ use Pleroma.DataCase
+ alias Pleroma.Web.Federator.RetryQueue
+
+ @small_retry_count 0
+ @hopeless_retry_count 10
+
+ test "failed posts are retried" do
+ {:retry, _timeout} = RetryQueue.get_retry_params(@small_retry_count)
+
+ assert {:noreply, %{delivered: 1}} ==
+ RetryQueue.handle_info({:send, :ok, MockActivityPub, @small_retry_count}, %{
+ delivered: 0
+ })
+ end
+
+ test "posts that have been tried too many times are dropped" do
+ {:drop, _timeout} = RetryQueue.get_retry_params(@hopeless_retry_count)
+
+ assert {:noreply, %{dropped: 1}} ==
+ RetryQueue.handle_cast({:maybe_enqueue, %{}, nil, @hopeless_retry_count}, %{
+ dropped: 0
+ })
+ end
+end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Nov 19, 3:44 AM (1 d, 20 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1840905
Default Alt Text
(22 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline