No OneTemporary
Actions

Size

22 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
	index d9a4a00d7..e83324ad3 100644
	--- a/lib/pleroma/application.ex
	+++ b/lib/pleroma/application.ex
	@@ -1,342 +1,343 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Application do
	use Application

	import Cachex.Spec

	alias Pleroma.Config

	require Logger

	@name Mix.Project.config()[:name]
	@version Mix.Project.config()[:version]
	@repository Mix.Project.config()[:source_url]
	@mix_env Mix.env()

	def name, do: @name
	def version, do: @version
	def named_version, do: @name <> " " <> @version
	def repository, do: @repository

	def user_agent do
	if Process.whereis(Pleroma.Web.Endpoint) do
	case Config.get([:http, :user_agent], :default) do
	:default ->
	info = "#{Pleroma.Web.Endpoint.url()} <#{Config.get([:instance, :email], "")}>"
	named_version() <> "; " <> info

	custom ->
	custom
	end
	else
	# fallback, if endpoint is not started yet
	"Pleroma Data Loader"
	end
	end

	# See http://elixir-lang.org/docs/stable/elixir/Application.html
	# for more information on OTP Applications
	def start(_type, _args) do
	# Scrubbers are compiled at runtime and therefore will cause a conflict
	# every time the application is restarted, so we disable module
	# conflicts at runtime
	Code.compiler_options(ignore_module_conflict: true)
	# Disable warnings_as_errors at runtime, it breaks Phoenix live reload
	# due to protocol consolidation warnings
	Code.compiler_options(warnings_as_errors: false)
	Pleroma.Telemetry.Logger.attach()
	Config.Holder.save_default()
	Pleroma.HTML.compile_scrubbers()
	Pleroma.Config.Oban.warn()
	Config.DeprecationWarnings.warn()
	Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
	Pleroma.ApplicationRequirements.verify!()
	setup_instrumenters()
	load_custom_modules()
	Pleroma.Docs.JSON.compile()
	limiters_setup()

	adapter = Application.get_env(:tesla, :adapter)

	if match?({Tesla.Adapter.Finch, _}, adapter) do
	Logger.info("Starting Finch")
	Finch.start_link(name: MyFinch)
	end

	if adapter == Tesla.Adapter.Gun do
	if version = Pleroma.OTPVersion.version() do
	[major, minor] =
	version
	\|> String.split(".")
	\|> Enum.map(&String.to_integer/1)
	\|> Enum.take(2)

	if (major == 22 and minor < 2) or major < 22 do
	raise "
	!!!OTP VERSION WARNING!!!
	You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
	"
	end
	else
	raise "
	!!!OTP VERSION WARNING!!!
	To support correct handling of unordered certificates chains - OTP version must be > 22.2.
	"
	end
	end

	# Define workers and child supervisors to be supervised
	children =
	cluster_children() ++
	[
	Pleroma.Repo,
	Config.TransferTask,
	Pleroma.Emoji,
	- Pleroma.Web.Plugs.RateLimiter.Supervisor
	+ Pleroma.Web.Plugs.RateLimiter.Supervisor,
	+ {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
	] ++
	cachex_children() ++
	http_children(adapter, @mix_env) ++
	[
	Pleroma.Stats,
	Pleroma.JobQueueMonitor,
	{Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
	{Oban, Config.get(Oban)},
	- Pleroma.Web.Endpoint
	+ Pleroma.Web.Endpoint,
	+ Pleroma.Web.Streamer.Worker
	] ++
	task_children(@mix_env) ++
	dont_run_in_test(@mix_env) ++
	shout_child(shout_enabled?()) ++
	[Pleroma.Gopher.Server]

	# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
	# for other strategies and supported options
	# If we have a lot of caches, default max_restarts can cause test
	# resets to fail.
	# Go for the default 3 unless we're in test
	max_restarts =
	if @mix_env == :test do
	100
	else
	3
	end

	opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts]
	result = Supervisor.start_link(children, opts)

	set_postgres_server_version()

	result
	end

	defp set_postgres_server_version do
	version =
	with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
	{num, _} <- Float.parse(version) do
	num
	else
	e ->
	Logger.warn(
	"Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
	)

	9.6
	end

	:persistent_term.put({Pleroma.Repo, :postgres_version}, version)
	end

	def load_custom_modules do
	dir = Config.get([:modules, :runtime_dir])

	if dir && File.exists?(dir) do
	dir
	\|> Pleroma.Utils.compile_dir()
	\|> case do
	{:error, _errors, _warnings} ->
	raise "Invalid custom modules"

	{:ok, modules, _warnings} ->
	if @mix_env != :test do
	Enum.each(modules, fn mod ->
	Logger.info("Custom module loaded: #{inspect(mod)}")
	end)
	end

	:ok
	end
	end
	end

	defp setup_instrumenters do
	require Prometheus.Registry

	if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
	:ok =
	:telemetry.attach(
	"prometheus-ecto",
	[:pleroma, :repo, :query],
	&Pleroma.Repo.Instrumenter.handle_event/4,
	%{}
	)

	Pleroma.Repo.Instrumenter.setup()
	end

	Pleroma.Web.Endpoint.MetricsExporter.setup()
	Pleroma.Web.Endpoint.PipelineInstrumenter.setup()

	# Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
	# Pleroma.Web.Endpoint.Instrumenter.setup()
	PrometheusPhx.setup()
	end

	defp cluster_children do
	topologies = Application.get_env(:libcluster, :topologies)

	if topologies do
	[{Cluster.Supervisor, [topologies, [name: Pleroma.ClusterSupervisor]]}]
	else
	[]
	end
	end

	defp cachex_children do
	[
	build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
	build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
	build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
	build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
	build_cachex("scrubber", limit: 2500),
	build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
	build_cachex("web_resp", limit: 2500),
	build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
	build_cachex("failed_proxy_url", limit: 2500),
	build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
	build_cachex("chat_message_id_idempotency_key",
	expiration: chat_message_id_idempotency_key_expiration(),
	limit: 500_000
	)
	]
	end

	defp emoji_packs_expiration,
	do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))

	defp idempotency_expiration,
	do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))

	defp chat_message_id_idempotency_key_expiration,
	do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))

	defp seconds_valid_interval,
	do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))

	@spec build_cachex(String.t(), keyword()) :: map()
	def build_cachex(type, opts),
	do: %{
	id: String.to_atom("cachex_" <> type),
	start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
	type: :worker
	}

	defp shout_enabled?, do: Config.get([:shout, :enabled])

	defp dont_run_in_test(env) when env in [:test, :benchmark], do: []

	defp dont_run_in_test(_) do
	[
	{Registry,
	[
	name: Pleroma.Web.Streamer.registry(),
	keys: :duplicate,
	partitions: System.schedulers_online()
	]}
	] ++ background_migrators()
	end

	defp background_migrators do
	[
	Pleroma.Migrators.HashtagsTableMigrator
	]
	end

	defp shout_child(true) do
	[
	- Pleroma.Web.ShoutChannel.ShoutChannelState,
	- {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
	+ Pleroma.Web.ShoutChannel.ShoutChannelState
	]
	end

	defp shout_child(_), do: []

	defp task_children(:test) do
	[
	%{
	id: :web_push_init,
	start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
	restart: :temporary
	}
	]
	end

	defp task_children(_) do
	[
	%{
	id: :web_push_init,
	start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
	restart: :temporary
	},
	%{
	id: :internal_fetch_init,
	start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
	restart: :temporary
	}
	]
	end

	# start hackney and gun pools in tests
	defp http_children(_, :test) do
	http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil)
	end

	defp http_children(Tesla.Adapter.Hackney, _) do
	pools = [:federation, :media]

	pools =
	if Config.get([Pleroma.Upload, :proxy_remote]) do
	[:upload \| pools]
	else
	pools
	end

	for pool <- pools do
	options = Config.get([:hackney_pools, pool])
	:hackney_pool.child_spec(pool, options)
	end
	end

	defp http_children(Tesla.Adapter.Gun, _) do
	Pleroma.Gun.ConnectionPool.children() ++
	[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
	end

	defp http_children(_, _), do: []

	@spec limiters_setup() :: :ok
	def limiters_setup do
	config = Config.get(ConcurrentLimiter, [])

	[Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
	\|> Enum.each(fn module ->
	mod_config = Keyword.get(config, module, [])

	max_running = Keyword.get(mod_config, :max_running, 5)
	max_waiting = Keyword.get(mod_config, :max_waiting, 5)

	ConcurrentLimiter.new(module, max_running, max_waiting)
	end)
	end
	end
	diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
	index ff7f62a1e..e4f61618a 100644
	--- a/lib/pleroma/web/streamer.ex
	+++ b/lib/pleroma/web/streamer.ex
	@@ -1,344 +1,350 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.Streamer do
	require Logger

	alias Pleroma.Activity
	alias Pleroma.Chat.MessageReference
	alias Pleroma.Config
	alias Pleroma.Conversation.Participation
	alias Pleroma.Notification
	alias Pleroma.Object
	alias Pleroma.User
	alias Pleroma.Web.ActivityPub.ActivityPub
	alias Pleroma.Web.ActivityPub.Visibility
	alias Pleroma.Web.CommonAPI
	alias Pleroma.Web.OAuth.Token
	alias Pleroma.Web.Plugs.OAuthScopesPlug
	alias Pleroma.Web.StreamerView

	@mix_env Mix.env()
	@registry Pleroma.Web.StreamerRegistry

	def registry, do: @registry

	@public_streams ["public", "public:local", "public:media", "public:local:media"]
	@user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]

	@doc "Expands and authorizes a stream, and registers the process for streaming."
	@spec get_topic_and_add_socket(
	stream :: String.t(),
	User.t() \| nil,
	Token.t() \| nil,
	Map.t() \| nil
	) ::
	{:ok, topic :: String.t()} \| {:error, :bad_topic} \| {:error, :unauthorized}
	def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do
	with {:ok, topic} <- get_topic(stream, user, oauth_token, params) do
	add_socket(topic, user)
	end
	end

	@doc "Expand and authorizes a stream"
	@spec get_topic(stream :: String.t(), User.t() \| nil, Token.t() \| nil, Map.t()) ::
	{:ok, topic :: String.t()} \| {:error, :bad_topic}
	def get_topic(stream, user, oauth_token, params \\ %{})

	# Allow all public steams.
	def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do
	{:ok, stream}
	end

	# Allow all hashtags streams.
	def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
	{:ok, "hashtag:" <> tag}
	end

	# Allow remote instance streams.
	def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do
	{:ok, "public:remote:" <> instance}
	end

	def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do
	{:ok, "public:remote:media:" <> instance}
	end

	# Expand user streams.
	def get_topic(
	stream,
	%User{id: user_id} = user,
	%Token{user_id: user_id} = oauth_token,
	_params
	)
	when stream in @user_streams do
	# Note: "read" works for all user streams (not mentioning it since it's an ancestor scope)
	required_scopes =
	if stream == "user:notification" do
	["read:notifications"]
	else
	["read:statuses"]
	end

	if OAuthScopesPlug.filter_descendants(required_scopes, oauth_token.scopes) == [] do
	{:error, :unauthorized}
	else
	{:ok, stream <> ":" <> to_string(user.id)}
	end
	end

	def get_topic(stream, _user, _oauth_token, _params) when stream in @user_streams do
	{:error, :unauthorized}
	end

	# List streams.
	def get_topic(
	"list",
	%User{id: user_id} = user,
	%Token{user_id: user_id} = oauth_token,
	%{"list" => id}
	) do
	cond do
	OAuthScopesPlug.filter_descendants(["read", "read:lists"], oauth_token.scopes) == [] ->
	{:error, :unauthorized}

	Pleroma.List.get(id, user) ->
	{:ok, "list:" <> to_string(id)}

	true ->
	{:error, :bad_topic}
	end
	end

	def get_topic("list", _user, _oauth_token, _params) do
	{:error, :unauthorized}
	end

	def get_topic(_stream, _user, _oauth_token, _params) do
	{:error, :bad_topic}
	end

	@doc "Registers the process for streaming. Use `get_topic/3` to get the full authorized topic."
	def add_socket(topic, user) do
	if should_env_send?() do
	auth? = if user, do: true
	Registry.register(@registry, topic, auth?)
	end

	{:ok, topic}
	end

	def remove_socket(topic) do
	if should_env_send?(), do: Registry.unregister(@registry, topic)
	end

	def stream(topics, items) do
	+ if should_env_send?() do
	+ Pleroma.Web.Streamer.Worker.request_stream(topics, items)
	+ end
	+ end
	+
	+ def exec_stream(topics, items) do
	if should_env_send?() do
	for topic <- List.wrap(topics), item <- List.wrap(items) do
	spawn(fn -> do_stream(topic, item) end)
	end
	end
	end

	def filtered_by_user?(user, item, streamed_type \\ :activity)

	def filtered_by_user?(%User{} = user, %Activity{} = item, streamed_type) do
	%{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} =
	User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute])

	recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids)
	recipients = MapSet.new(item.recipients)
	domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks)

	with parent <- Object.normalize(item, fetch: false) \|\| item,
	true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)),
	true <- item.data["type"] != "Announce" \|\| item.actor not in reblog_muted_ap_ids,
	true <-
	!(streamed_type == :activity && item.data["type"] == "Announce" &&
	parent.data["actor"] == user.ap_id),
	true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(parent.data["actor"] not in &1)),
	true <- MapSet.disjoint?(recipients, recipient_blocks),
	%{host: item_host} <- URI.parse(item.actor),
	%{host: parent_host} <- URI.parse(parent.data["actor"]),
	false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host),
	false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host),
	true <- thread_containment(item, user),
	false <- CommonAPI.thread_muted?(user, parent) do
	false
	else
	_ -> true
	end
	end

	def filtered_by_user?(%User{} = user, %Notification{activity: activity}, _) do
	filtered_by_user?(user, activity, :notification)
	end

	defp do_stream("direct", item) do
	recipient_topics =
	User.get_recipients_from_activity(item)
	\|> Enum.map(fn %{id: id} -> "direct:#{id}" end)

	Enum.each(recipient_topics, fn user_topic ->
	Logger.debug("Trying to push direct message to #{user_topic}\n\n")
	push_to_socket(user_topic, item)
	end)
	end

	defp do_stream("follow_relationship", item) do
	text = StreamerView.render("follow_relationships_update.json", item)
	user_topic = "user:#{item.follower.id}"

	Logger.debug("Trying to push follow relationship update to #{user_topic}\n\n")

	Registry.dispatch(@registry, user_topic, fn list ->
	Enum.each(list, fn {pid, _auth} ->
	send(pid, {:text, text})
	end)
	end)
	end

	defp do_stream("participation", participation) do
	user_topic = "direct:#{participation.user_id}"
	Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n")

	push_to_socket(user_topic, participation)
	end

	defp do_stream("list", item) do
	# filter the recipient list if the activity is not public, see #270.
	recipient_lists =
	case Visibility.is_public?(item) do
	true ->
	Pleroma.List.get_lists_from_activity(item)

	_ ->
	Pleroma.List.get_lists_from_activity(item)
	\|> Enum.filter(fn list ->
	owner = User.get_cached_by_id(list.user_id)

	Visibility.visible_for_user?(item, owner)
	end)
	end

	recipient_topics =
	recipient_lists
	\|> Enum.map(fn %{id: id} -> "list:#{id}" end)

	Enum.each(recipient_topics, fn list_topic ->
	Logger.debug("Trying to push message to #{list_topic}\n\n")
	push_to_socket(list_topic, item)
	end)
	end

	defp do_stream(topic, %Notification{} = item)
	when topic in ["user", "user:notification"] do
	Registry.dispatch(@registry, "#{topic}:#{item.user_id}", fn list ->
	Enum.each(list, fn {pid, _auth} ->
	send(pid, {:render_with_user, StreamerView, "notification.json", item})
	end)
	end)
	end

	defp do_stream(topic, {user, %MessageReference{} = cm_ref})
	when topic in ["user", "user:pleroma_chat"] do
	topic = "#{topic}:#{user.id}"

	text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})

	Registry.dispatch(@registry, topic, fn list ->
	Enum.each(list, fn {pid, _auth} ->
	send(pid, {:text, text})
	end)
	end)
	end

	defp do_stream("user", item) do
	Logger.debug("Trying to push to users")

	recipient_topics =
	User.get_recipients_from_activity(item)
	\|> Enum.map(fn %{id: id} -> "user:#{id}" end)

	Enum.each(recipient_topics, fn topic ->
	push_to_socket(topic, item)
	end)
	end

	defp do_stream(topic, item) do
	Logger.debug("Trying to push to #{topic}")
	Logger.debug("Pushing item to #{topic}")
	push_to_socket(topic, item)
	end

	defp push_to_socket(topic, %Participation{} = participation) do
	rendered = StreamerView.render("conversation.json", participation)

	Registry.dispatch(@registry, topic, fn list ->
	Enum.each(list, fn {pid, _} ->
	send(pid, {:text, rendered})
	end)
	end)
	end

	defp push_to_socket(topic, %Activity{
	data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id}
	}) do
	rendered = Jason.encode!(%{event: "delete", payload: to_string(deleted_activity_id)})

	Registry.dispatch(@registry, topic, fn list ->
	Enum.each(list, fn {pid, _} ->
	send(pid, {:text, rendered})
	end)
	end)
	end

	defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop

	defp push_to_socket(topic, item) do
	anon_render = StreamerView.render("update.json", item)

	Registry.dispatch(@registry, topic, fn list ->
	Enum.each(list, fn {pid, auth?} ->
	if auth? do
	send(pid, {:render_with_user, StreamerView, "update.json", item})
	else
	send(pid, {:text, anon_render})
	end
	end)
	end)
	end

	defp thread_containment(_activity, %User{skip_thread_containment: true}), do: true

	defp thread_containment(activity, user) do
	if Config.get([:instance, :skip_thread_containment]) do
	true
	else
	ActivityPub.contain_activity(activity, user)
	end
	end

	# In test environement, only return true if the registry is started.
	# In benchmark environment, returns false.
	# In any other environment, always returns true.
	cond do
	@mix_env == :test ->
	def should_env_send? do
	case Process.whereis(@registry) do
	nil ->
	false

	pid ->
	Process.alive?(pid)
	end
	end

	@mix_env == :benchmark ->
	def should_env_send?, do: false

	true ->
	def should_env_send?, do: true
	end
	end
	diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex
	new file mode 100644
	index 000000000..e8616c7ed
	--- /dev/null
	+++ b/lib/pleroma/web/streamer/worker.ex
	@@ -0,0 +1,36 @@
	+# Pleroma: A lightweight social networking server
	+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	+# SPDX-License-Identifier: AGPL-3.0-only
	+
	+defmodule Pleroma.Web.Streamer.Worker do
	+ use GenServer
	+
	+ @pubsub Pleroma.PubSub
	+ @streamer_topic "streamer"
	+
	+ def start_link(_) do
	+ GenServer.start_link(__MODULE__, [], name: __MODULE__)
	+ end
	+
	+ @impl true
	+ def init(_) do
	+ :ok = Phoenix.PubSub.subscribe(@pubsub, @streamer_topic)
	+ {:ok, []}
	+ end
	+
	+ @impl true
	+ def handle_info({:stream, topics, items}, state) do
	+ Pleroma.Web.Streamer.exec_stream(topics, items)
	+ {:noreply, state}
	+ end
	+
	+ @impl true
	+ def handle_call({:request_stream, topics, items}, _from, state) do
	+ Phoenix.PubSub.broadcast!(@pubsub, @streamer_topic, {:stream, topics, items})
	+ {:reply, :ok, state}
	+ end
	+
	+ def request_stream(topics, items) do
	+ GenServer.call(__MODULE__, {:request_stream, topics, items})
	+ end
	+end

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jul 18, 6:20 PM (1 d, 5 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1695110
Default Alt Text: (22 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions