Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
21 KB
Referenced Files
None
Subscribers
None
diff --git a/config/config.exs b/config/config.exs
index 90e3a4aec..d30b33197 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -1,257 +1,265 @@
# This file is responsible for configuring your application
# and its dependencies with the aid of the Mix.Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
use Mix.Config
# General application configuration
config :pleroma, ecto_repos: [Pleroma.Repo]
config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
config :pleroma, Pleroma.Captcha,
enabled: false,
seconds_retained: 180,
method: Pleroma.Captcha.Kocaptcha
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
filters: [],
proxy_remote: false,
proxy_opts: []
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
public_endpoint: "https://s3.amazonaws.com"
config :pleroma, Pleroma.Uploaders.MDII,
cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
files: "https://mdii.sakura.ne.jp"
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
config :pleroma, :uri_schemes,
valid_schemes: [
"https",
"http",
"dat",
"dweb",
"gopher",
"ipfs",
"ipns",
"irc",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
"xmpp"
]
# Configures the endpoint
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "localhost"],
http: [
dispatch: [
{:_,
[
{"/api/v1/streaming", Elixir.Pleroma.Web.MastodonAPI.WebsocketHandler, []},
{:_, Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
]}
]
],
protocol: "https",
secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
signing_salt: "CqaoopA2",
render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
pubsub: [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2],
secure_cookie_flag: true
# Configures Elixir's Logger
config :logger, :console,
format: "$time $metadata[$level] $message\n",
metadata: [:request_id]
config :mime, :types, %{
"application/xml" => ["xml"],
"application/xrd+xml" => ["xrd+xml"],
"application/jrd+json" => ["jrd+json"],
"application/activity+json" => ["activity+json"],
"application/ld+json" => ["activity+json"]
}
config :pleroma, :websub, Pleroma.Web.Websub
config :pleroma, :ostatus, Pleroma.Web.OStatus
config :pleroma, :httpoison, Pleroma.HTTP
config :tesla, adapter: Tesla.Adapter.Hackney
# Configures http settings, upstream proxy etc.
config :pleroma, :http, proxy_url: nil
config :pleroma, :instance,
name: "Pleroma",
email: "example@example.com",
description: "A Pleroma instance, an alternative fediverse server",
limit: 5_000,
remote_limit: 100_000,
upload_limit: 16_000_000,
avatar_upload_limit: 2_000_000,
background_upload_limit: 4_000_000,
banner_upload_limit: 4_000_000,
registrations_open: true,
federating: true,
allow_relay: true,
rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
public: true,
quarantined_instances: [],
managed_config: true,
static_dir: "instance/static/",
allowed_post_formats: [
"text/plain",
"text/html",
"text/markdown"
],
finmoji_enabled: true,
mrf_transparency: true
config :pleroma, :markup,
# XXX - unfortunately, inline images must be enabled by default right now, because
# of custom emoji. Issue #275 discusses defanging that somehow.
allow_inline_images: true,
allow_headings: false,
allow_tables: false,
allow_fonts: false,
scrub_policy: [
Pleroma.HTML.Transform.MediaProxy,
Pleroma.HTML.Scrubber.Default
]
config :pleroma, :fe,
theme: "pleroma-dark",
logo: "/static/logo.png",
logo_mask: true,
logo_margin: "0.1em",
background: "/static/aurora_borealis.jpg",
redirect_root_no_login: "/main/all",
redirect_root_login: "/main/friends",
show_instance_panel: true,
scope_options_enabled: false,
formatting_options_enabled: false,
collapse_message_with_subject: false,
hide_post_stats: false,
hide_user_stats: false,
scope_copy: true,
subject_line_behavior: "email",
always_show_subject_input: true
config :pleroma, :activitypub,
accept_blocks: true,
unfollow_blocked: true,
outgoing_blocks: true,
follow_handshake_timeout: 500
config :pleroma, :user, deny_follow_blocked: true
config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default
config :pleroma, :mrf_rejectnonpublic,
allow_followersonly: false,
allow_direct: false
config :pleroma, :mrf_hellthread, threshold: 10
config :pleroma, :mrf_simple,
media_removal: [],
media_nsfw: [],
federated_timeline_removal: [],
reject: [],
accept: []
config :pleroma, :media_proxy,
enabled: false,
# base_url: "https://cache.pleroma.social",
proxy_opts: [
# inline_content_types: [] | false | true,
# http: [:insecure]
]
config :pleroma, :chat, enabled: true
config :ecto, json_library: Jason
config :phoenix, :format_encoders, json: Jason
config :pleroma, :gopher,
enabled: false,
ip: {0, 0, 0, 0},
port: 9999
config :pleroma, :suggestions,
enabled: false,
third_party_engine:
"http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}",
timeout: 300_000,
limit: 23,
web: "https://vinayaka.distsn.org/?{{host}}+{{user}}"
config :pleroma, :http_security,
enabled: true,
sts: false,
sts_max_age: 31_536_000,
ct_max_age: 2_592_000,
referrer_policy: "same-origin"
config :cors_plug,
max_age: 86_400,
methods: ["POST", "PUT", "DELETE", "GET", "PATCH", "OPTIONS"],
expose: [
"Link",
"X-RateLimit-Reset",
"X-RateLimit-Limit",
"X-RateLimit-Remaining",
"X-Request-Id",
"Idempotency-Key"
],
credentials: true,
headers: ["Authorization", "Content-Type", "Idempotency-Key"]
config :pleroma, Pleroma.User,
restricted_nicknames: [
"about",
"~",
"main",
"users",
"settings",
"objects",
"activities",
"web",
"registration",
"friend-requests",
"pleroma",
"api",
"tag",
"notice",
"status",
"user-search",
"ostatus_subscribe",
"oauth",
"push",
"relay",
"inbox",
".well-known",
"nodeinfo",
"auth",
"proxy",
"dev",
"internal"
]
+config :pleroma, Pleroma.Web.Federator, max_jobs: 50
+
+config :pleroma, Pleroma.Web.Federator.RetryQueue,
+ enabled: false,
+ max_jobs: 20,
+ initial_timeout: 30,
+ max_retries: 5
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index d25bfc0c1..f3a0e18b8 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -1,229 +1,228 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Federator do
use GenServer
alias Pleroma.User
alias Pleroma.Activity
alias Pleroma.Web.{WebFinger, Websub}
alias Pleroma.Web.Federator.RetryQueue
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Relay
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.OStatus
require Logger
@websub Application.get_env(:pleroma, :websub)
@ostatus Application.get_env(:pleroma, :ostatus)
- @max_jobs 20
def init(args) do
{:ok, args}
end
def start_link do
spawn(fn ->
# 1 minute
Process.sleep(1000 * 60 * 1)
enqueue(:refresh_subscriptions, nil)
end)
GenServer.start_link(
__MODULE__,
%{
in: {:sets.new(), []},
out: {:sets.new(), []}
},
name: __MODULE__
)
end
def handle(:refresh_subscriptions, _) do
Logger.debug("Federator running refresh subscriptions")
Websub.refresh_subscriptions()
spawn(fn ->
# 6 hours
Process.sleep(1000 * 60 * 60 * 6)
enqueue(:refresh_subscriptions, nil)
end)
end
def handle(:request_subscription, websub) do
Logger.debug("Refreshing #{websub.topic}")
with {:ok, websub} <- Websub.request_subscription(websub) do
Logger.debug("Successfully refreshed #{websub.topic}")
else
_e -> Logger.debug("Couldn't refresh #{websub.topic}")
end
end
def handle(:publish, activity) do
Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)
with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
{:ok, actor} = WebFinger.ensure_keys_present(actor)
if ActivityPub.is_public?(activity) do
if OStatus.is_representable?(activity) do
Logger.info(fn -> "Sending #{activity.data["id"]} out via WebSub" end)
Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
Logger.info(fn -> "Sending #{activity.data["id"]} out via Salmon" end)
Pleroma.Web.Salmon.publish(actor, activity)
end
if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
Relay.publish(activity)
end
end
Logger.info(fn -> "Sending #{activity.data["id"]} out via AP" end)
Pleroma.Web.ActivityPub.ActivityPub.publish(actor, activity)
end
end
def handle(:verify_websub, websub) do
Logger.debug(fn ->
"Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
end)
@websub.verify(websub)
end
def handle(:incoming_doc, doc) do
Logger.info("Got document, trying to parse")
@ostatus.handle_incoming(doc)
end
def handle(:incoming_ap_doc, params) do
Logger.info("Handling incoming AP activity")
params = Utils.normalize_params(params)
# NOTE: we use the actor ID to do the containment, this is fine because an
# actor shouldn't be acting on objects outside their own AP server.
with {:ok, _user} <- ap_enabled_actor(params["actor"]),
nil <- Activity.normalize(params["id"]),
:ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
{:ok, activity}
else
%Activity{} ->
Logger.info("Already had #{params["id"]}")
:error
_e ->
# Just drop those for now
Logger.info("Unhandled activity")
Logger.info(Poison.encode!(params, pretty: 2))
:error
end
end
def handle(:publish_single_ap, params) do
case ActivityPub.publish_one(params) do
{:ok, _} ->
:ok
{:error, _} ->
RetryQueue.enqueue(params, ActivityPub)
end
end
def handle(
:publish_single_websub,
%{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params
) do
case Websub.publish_one(params) do
{:ok, _} ->
:ok
{:error, _} ->
RetryQueue.enqueue(params, Websub)
end
end
def handle(type, _) do
Logger.debug(fn -> "Unknown task: #{type}" end)
{:error, "Don't know what to do with this"}
end
if Mix.env() == :test do
def enqueue(type, payload, _priority \\ 1) do
if Pleroma.Config.get([:instance, :federating]) do
handle(type, payload)
end
end
else
def enqueue(type, payload, priority \\ 1) do
if Pleroma.Config.get([:instance, :federating]) do
GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
end
end
end
def maybe_start_job(running_jobs, queue) do
- if :sets.size(running_jobs) < @max_jobs && queue != [] do
+ if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do
{{type, payload}, queue} = queue_pop(queue)
{:ok, pid} = Task.start(fn -> handle(type, payload) end)
mref = Process.monitor(pid)
{:sets.add_element(mref, running_jobs), queue}
else
{running_jobs, queue}
end
end
def handle_cast({:enqueue, type, payload, _priority}, state)
when type in [:incoming_doc, :incoming_ap_doc] do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_queue = enqueue_sorted(i_queue, {type, payload}, 1)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast({:enqueue, type, payload, _priority}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
o_queue = enqueue_sorted(o_queue, {type, payload}, 1)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def handle_cast(m, state) do
IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}")
{:noreply, state}
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
%{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
i_running_jobs = :sets.del_element(ref, i_running_jobs)
o_running_jobs = :sets.del_element(ref, o_running_jobs)
{i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
{o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
{:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
end
def enqueue_sorted(queue, element, priority) do
[%{item: element, priority: priority} | queue]
|> Enum.sort_by(fn %{priority: priority} -> priority end)
end
def queue_pop([%{item: element} | queue]) do
{element, queue}
end
def ap_enabled_actor(id) do
user = User.get_by_ap_id(id)
if User.ap_enabled?(user) do
{:ok, user}
else
ActivityPub.make_user_from_ap_id(id)
end
end
end
diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex
index c57bbb77c..230a2c939 100644
--- a/lib/pleroma/web/federator/retry_queue.ex
+++ b/lib/pleroma/web/federator/retry_queue.ex
@@ -1,243 +1,238 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Federator.RetryQueue do
use GenServer
require Logger
- # seconds
- @initial_timeout 30
- @max_retries 5
-
- @max_jobs 20
-
def init(args) do
queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected])
{:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}}
end
def start_link() do
enabled =
- if Mix.env() == :test, do: true, else: Pleroma.Config.get([:retry_queue, :enabled], false)
+ if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false)
if enabled do
Logger.info("Starting retry queue")
linkres =
GenServer.start_link(
__MODULE__,
%{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil},
name: __MODULE__
)
maybe_kickoff_timer()
linkres
else
Logger.info("Retry queue disabled")
:ignore
end
end
def enqueue(data, transport, retries \\ 0) do
GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
end
def get_stats() do
GenServer.call(__MODULE__, :get_stats)
end
def reset_stats() do
GenServer.call(__MODULE__, :reset_stats)
end
def get_retry_params(retries) do
- if retries > @max_retries do
+ if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do
{:drop, "Max retries reached"}
else
{:retry, growth_function(retries)}
end
end
def get_retry_timer_interval() do
Pleroma.Config.get([:retry_queue, :interval], 1000)
end
defp ets_count_expires(table, current_time) do
:ets.select_count(
table,
[
{
{:"$1", :"$2"},
[{:"=<", :"$1", {:const, current_time}}],
[true]
}
]
)
end
defp ets_pop_n_expired(table, current_time, desired) do
{popped, _continuation} =
:ets.select(
table,
[
{
{:"$1", :"$2"},
[{:"=<", :"$1", {:const, current_time}}],
[:"$_"]
}
],
desired
)
popped
|> List.foldl(true, fn e, acc ->
:ets.delete_object(table, e)
acc
end)
popped
end
def maybe_start_job(running_jobs, queue_table) do
# we don't want to hit the ets or the DateTime more times than we have to
# could optimize slightly further by not using the count, and instead grabbing
# up to N objects early...
current_time = DateTime.to_unix(DateTime.utc_now())
n_running_jobs = :sets.size(running_jobs)
- if n_running_jobs < @max_jobs do
+ if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do
n_ready_jobs = ets_count_expires(queue_table, current_time)
if n_ready_jobs > 0 do
# figure out how many we could start
- available_job_slots = @max_jobs - n_running_jobs
+ available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs
start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
else
running_jobs
end
else
running_jobs
end
end
defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do
running_jobs
end
defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
when available_job_slots > 0 do
candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots)
candidates
|> List.foldl(running_jobs, fn {_, e}, rj ->
{:ok, pid} = Task.start(fn -> worker(e) end)
mref = Process.monitor(pid)
:sets.add_element(mref, rj)
end)
end
def worker({:send, data, transport, retries}) do
case transport.publish_one(data) do
{:ok, _} ->
GenServer.cast(__MODULE__, :inc_delivered)
:delivered
{:error, _reason} ->
enqueue(data, transport, retries)
:retry
end
end
def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
{:reply, %{delivered: delivery_count, dropped: drop_count}, state}
end
def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
{:reply, %{delivered: delivery_count, dropped: drop_count},
%{state | delivered: 0, dropped: 0}}
end
def handle_cast(:reset_stats, state) do
{:noreply, %{state | delivered: 0, dropped: 0}}
end
def handle_cast(
{:maybe_enqueue, data, transport, retries},
%{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state
) do
case get_retry_params(retries) do
{:retry, timeout} ->
:ets.insert(queue_table, {timeout, {:send, data, transport, retries}})
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
{:drop, message} ->
Logger.debug(message)
{:noreply, %{state | dropped: drop_count + 1}}
end
end
def handle_cast(:kickoff_timer, state) do
retry_interval = get_retry_timer_interval()
Process.send_after(__MODULE__, :retry_timer_run, retry_interval)
{:noreply, state}
end
def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do
{:noreply, %{state | delivered: delivery_count + 1}}
end
def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do
{:noreply, %{state | dropped: drop_count + 1}}
end
def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
case transport.publish_one(data) do
{:ok, _} ->
{:noreply, %{state | delivered: delivery_count + 1}}
{:error, _reason} ->
enqueue(data, transport, retries)
{:noreply, state}
end
end
def handle_info(
:retry_timer_run,
%{queue_table: queue_table, running_jobs: running_jobs} = state
) do
maybe_kickoff_timer()
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
end
def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
%{running_jobs: running_jobs, queue_table: queue_table} = state
running_jobs = :sets.del_element(ref, running_jobs)
running_jobs = maybe_start_job(running_jobs, queue_table)
{:noreply, %{state | running_jobs: running_jobs}}
end
def handle_info(unknown, state) do
Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
{:noreply, state}
end
if Mix.env() == :test do
defp growth_function(_retries) do
- _shutit = @initial_timeout
+ _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout])
DateTime.to_unix(DateTime.utc_now()) - 1
end
else
defp growth_function(retries) do
- round(@initial_timeout * :math.pow(retries, 3)) + DateTime.to_unix(DateTime.utc_now())
+ round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) +
+ DateTime.to_unix(DateTime.utc_now())
end
end
defp maybe_kickoff_timer() do
GenServer.cast(__MODULE__, :kickoff_timer)
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 1, 12:25 AM (19 h, 16 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1801968
Default Alt Text
(21 KB)

Event Timeline