Page MenuHomePhorge
Files F113410

No OneTemporary
Actions

Size
20 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 7c0b0eead..199d3812f 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -1,348 +1,348 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.Instance do
@moduledoc "Instance."
alias Pleroma.Instances
alias Pleroma.Instances.Instance
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
import Ecto.Query
import Ecto.Changeset
require Logger
schema "instances" do
field(:host, :string)
field(:unreachable_since, :naive_datetime_usec)
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
field(:software_name, :string)
field(:software_version, :string)
field(:software_repository, :string)
end
field(:metadata_updated_at, :utc_datetime)
timestamps()
end
defdelegate host(url_or_host), to: Instances
def changeset(struct, params \\ %{}) do
struct
|> cast(params, __schema__(:fields) -- [:metadata])
|> cast_embed(:metadata, with: &metadata_changeset/2)
|> validate_required([:host])
|> unique_constraint(:host)
end
def metadata_changeset(struct, params \\ %{}) do
struct
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: %{}
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
urls_or_hosts
|> Enum.map(&(&1 && host(&1)))
|> Enum.filter(&(to_string(&1) != ""))
unreachable_since_by_host =
Repo.all(
from(i in Instance,
where: i.host in ^hosts,
select: {i.host, i.unreachable_since}
)
)
|> Map.new(& &1)
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if !unreachable_since ||
NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
{entry, unreachable_since}
end
end
|> Enum.filter(& &1)
|> Map.new(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
!Repo.one(
from(i in Instance,
where:
i.host == ^host(url_or_host) and
i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
select: true
)
)
end
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
host = host(url_or_host)
existing_record = Repo.get_by(Instance, %{host: host})
cond do
is_nil(existing_record) ->
%Instance{}
|> changeset(%{host: host})
|> Repo.insert()
is_nil(existing_record.unreachable_since) ->
{:ok, existing_record}
true ->
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
:telemetry.execute(
[:pleroma, :instance, :reachable],
- %{host: host},
- %{instance: existing_record}
+ %{count: 1},
+ %{host: host, instance: existing_record}
)
result
end
end
def set_reachable(_), do: {:error, nil}
def set_unreachable(url_or_host, unreachable_since \\ nil)
def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now()
host = host(url_or_host)
existing_record = Repo.get_by(Instance, %{host: host})
changes = %{unreachable_since: unreachable_since}
result =
cond do
is_nil(existing_record) ->
%Instance{}
|> changeset(Map.put(changes, :host, host))
|> Repo.insert()
existing_record.unreachable_since &&
NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
{:ok, existing_record}
true ->
existing_record
|> changeset(changes)
|> Repo.update()
end
:telemetry.execute(
[:pleroma, :instance, :unreachable],
- %{host: host},
- %{instance: existing_record}
+ %{count: 1},
+ %{host: host, instance: existing_record}
)
result
end
def set_unreachable(_, _), do: {:error, nil}
def get_consistently_unreachable do
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
from(i in Instance,
where: ^reachability_datetime_threshold > i.unreachable_since,
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
|> Repo.all()
end
defp parse_datetime(datetime) when is_binary(datetime) do
NaiveDateTime.from_iso8601(datetime)
end
defp parse_datetime(datetime), do: datetime
def get_or_update_favicon(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.favicon_updated_at &&
NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
existing_record.favicon
else
favicon = scrape_favicon(instance_uri)
if existing_record do
existing_record
|> changeset(%{favicon: favicon, favicon_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
|> Repo.insert()
end
favicon
end
rescue
e ->
Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
nil
end
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
{:parse,
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
{_, favicon} when is_binary(favicon) <-
{:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
favicon
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
def get_or_update_metadata(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.metadata_updated_at &&
NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
existing_record.metadata
else
metadata = scrape_metadata(instance_uri)
if existing_record do
existing_record
|> changeset(%{metadata: metadata, metadata_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
|> Repo.insert()
end
metadata
end
end
defp get_nodeinfo_uri(well_known) do
links = Map.get(well_known, "links", [])
nodeinfo21 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]
nodeinfo20 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]
cond do
is_binary(nodeinfo21) -> {:ok, nodeinfo21}
is_binary(nodeinfo20) -> {:ok, nodeinfo20}
true -> {:error, :no_links}
end
end
defp scrape_metadata(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: well_known_body}} <-
instance_uri
|> URI.merge("/.well-known/nodeinfo")
|> to_string()
|> Pleroma.HTTP.get([{"accept", "application/json"}]),
{:ok, well_known_json} <- Jason.decode(well_known_body),
{:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
{:ok, %Tesla.Env{body: nodeinfo_body}} <-
Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
{:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
# Can extract more metadata from NodeInfo but need to be careful about it's size,
# can't just dump the entire thing
software = Map.get(nodeinfo, "software", %{})
%{
software_name: software["name"],
software_version: software["version"]
}
|> Maps.put_if_present(:software_repository, software["repository"])
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
@doc """
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
def perform(:delete_instance, host) when is_binary(host) do
User.Query.build(%{nickname: "@#{host}"})
|> Repo.chunk_stream(100, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
User.perform(:delete, user)
end)
end)
|> Stream.run()
end
end
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
index 43959b82a..41f9fcc67 100644
--- a/lib/pleroma/telemetry/logger.ex
+++ b/lib/pleroma/telemetry/logger.ex
@@ -1,358 +1,358 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Telemetry.Logger do
@moduledoc "Transforms Pleroma telemetry events to logs"
require Logger
@events [
[:oban, :job, :exception],
[:pleroma, :activity, :mrf, :filter],
[:pleroma, :activity, :mrf, :pass],
[:pleroma, :activity, :mrf, :reject],
[:pleroma, :activitypub, :inbox],
[:pleroma, :activitypub, :publisher],
[:pleroma, :instance, :reachable],
[:pleroma, :instance, :unreachable],
[:pleroma, :connection_pool, :client, :add],
[:pleroma, :connection_pool, :client, :dead],
[:pleroma, :connection_pool, :provision_failure],
[:pleroma, :connection_pool, :reclaim, :start],
[:pleroma, :connection_pool, :reclaim, :stop],
[:pleroma, :user, :account, :confirm],
[:pleroma, :user, :account, :password_reset],
[:pleroma, :user, :account, :register],
[:pleroma, :user, :o_auth, :failure],
[:pleroma, :user, :o_auth, :revoke],
[:pleroma, :user, :o_auth, :success],
[:pleroma, :user, :create],
[:pleroma, :user, :delete],
[:pleroma, :user, :update],
[:tesla, :request, :exception],
[:tesla, :request, :stop]
]
def setup do
if not Pleroma.Config.get([Pleroma.Telemetry, :phoenix_logs]) do
Pleroma.Telemetry.disable_phoenix_logs()
end
Enum.each(@events, fn event ->
:telemetry.attach({__MODULE__, event}, event, &__MODULE__.handle_event/4, [])
end)
end
# Passing anonymous functions instead of strings to logger is intentional,
# that way strings won't be concatenated if the message is going to be thrown
# out anyway due to higher log level configured
def handle_event([:oban, :job, :exception], _measure, meta, _) do
Logger.error(fn ->
"[Oban] #{meta.worker} error: #{inspect(meta.error)} args: #{inspect(meta.args)}"
end)
end
def handle_event(
[:pleroma, :connection_pool, :reclaim, :start],
_measurements,
%{max_connections: max_connections, reclaim_max: reclaim_max},
_
) do
Logger.debug(fn ->
"Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{reclaim_max} connections"
end)
end
def handle_event(
[:pleroma, :connection_pool, :reclaim, :stop],
%{count: 0},
_measurements,
_
) do
Logger.debug(fn ->
"Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts"
end)
end
def handle_event(
[:pleroma, :connection_pool, :reclaim, :stop],
%{count: reclaimed_count},
_measurements,
_
) do
Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end)
end
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
_measurements,
%{opts: [key | _]},
_
) do
Logger.debug(fn ->
"Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion"
end)
end
def handle_event(
[:pleroma, :connection_pool, :client, :dead],
_measurements,
%{client_pid: client_pid, key: key, reason: reason},
_
) do
Logger.debug(fn ->
"Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}"
end)
end
def handle_event(
[:pleroma, :connection_pool, :client, :add],
_measurements,
%{clients: [_, _ | _] = clients, key: key, protocol: :http},
_
) do
Logger.debug(fn ->
"Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur."
end)
end
def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok
def handle_event(
[:pleroma, :activitypub, :inbox],
_measurements,
%{ap_id: ap_id, type: type},
_
) do
Logger.info(fn ->
"Inbox: received #{type} of #{ap_id}"
end)
end
def handle_event(
[:pleroma, :activity, :mrf, :pass],
_measurements,
%{activity: activity, mrf: mrf},
_
) do
type = activity["type"]
ap_id = activity["id"]
Logger.debug(fn ->
"#{mrf}: passed #{inspect(type)} of #{inspect(ap_id)}"
end)
end
def handle_event(
[:pleroma, :activity, :mrf, :filter],
_measurements,
%{activity: activity, mrf: mrf},
_
) do
type = activity["type"]
ap_id = activity["id"]
Logger.debug(fn ->
"#{mrf}: filtered #{inspect(type)} of #{inspect(ap_id)}"
end)
end
def handle_event(
[:pleroma, :activity, :mrf, :reject],
_measurements,
%{activity: activity, mrf: mrf, reason: reason},
_
) do
type = activity["type"]
ap_id = activity["id"]
Logger.info(fn ->
"#{mrf}: rejected #{inspect(type)} of #{inspect(ap_id)}: #{inspect(reason)}"
end)
end
def handle_event(
[:pleroma, :user, :create],
_measurements,
%{nickname: nickname},
_
) do
Logger.info(fn ->
"User: created #{nickname}"
end)
end
def handle_event(
[:pleroma, :user, :delete],
_measurements,
%{nickname: nickname},
_
) do
Logger.info(fn ->
"User: deleted #{nickname}"
end)
end
def handle_event(
[:pleroma, :user, :disable],
_measurements,
%{nickname: nickname},
_
) do
Logger.info(fn ->
"User: disabled #{nickname}"
end)
end
def handle_event(
[:pleroma, :user, :update],
_measurements,
%{nickname: nickname},
_
) do
Logger.debug(fn ->
"User: updated #{nickname}"
end)
end
def handle_event(
[:pleroma, :instance, :reachable],
- %{host: host},
- %{instance: instance},
+ _measurements,
+ %{host: host, instance: instance},
_
) do
unreachable_since = get_in(instance, [Access.key(:unreachable_since)]) || "UNDEFINED"
Logger.info(fn ->
"Instance: #{host} is set reachable (was unreachable since: #{unreachable_since})"
end)
end
def handle_event(
[:pleroma, :instance, :unreachable],
- %{host: host},
- %{instance: _instance},
+ _measurements,
+ %{host: host, instance: _instance},
_
) do
Logger.info(fn ->
"Instance: #{host} is set unreachable"
end)
end
def handle_event(
[:pleroma, :user, :o_auth, :failure],
_measurements,
%{ip: ip, nickname: name},
_
) do
Logger.error(fn ->
"OAuth: authentication failure for #{name} from #{ip}"
end)
end
def handle_event(
[:pleroma, :user, :o_auth, :revoke],
_measurements,
%{ip: ip, nickname: name},
_
) do
Logger.info(fn ->
"OAuth: token revoked for #{name} from #{ip}"
end)
end
def handle_event(
[:pleroma, :user, :o_auth, :success],
_measurements,
%{ip: ip, nickname: name},
_
) do
Logger.info(fn ->
"OAuth: authentication success for #{name} from #{ip}"
end)
end
def handle_event(
[:pleroma, :user, :account, :confirm],
%{user: user},
_,
_
) do
Logger.info(fn ->
"Account: #{user.nickname} account confirmed"
end)
end
def handle_event(
[:pleroma, :user, :account, :password_reset],
%{for: for},
%{ip: ip},
_
) do
Logger.info(fn ->
"Account: password reset requested for #{for} from #{ip}"
end)
end
def handle_event(
[:pleroma, :user, :account, :register],
%{user: user},
%{ip: ip},
_
) do
Logger.info(fn ->
"Account: #{user.nickname} registered from #{ip}"
end)
end
def handle_event(
[:pleroma, :activitypub, :publisher],
%{count: count},
%{nickname: nickname, type: type},
_
) do
Logger.info(fn ->
"Publisher: delivering #{type} for #{nickname} to #{count} inboxes"
end)
end
def handle_event([:tesla, :request, :exception], _measure, meta, _) do
Logger.error(fn -> "Pleroma.HTTP exception #{inspect(meta.stacktrace)}" end)
end
def handle_event(
[:tesla, :request, :stop],
_measurements,
%{env: %Tesla.Env{method: method, url: url}, error: error} = _meta,
_
) do
Logger.warning(fn ->
"Pleroma.HTTP :#{method} failed for #{url} with error #{inspect(error)}"
end)
end
def handle_event(
[:tesla, :request, :stop],
_measurements,
%{env: %Tesla.Env{method: method, status: status, url: url}} = _meta,
_
) do
cond do
status in 200..299 ->
:ok
true ->
Logger.warning(fn -> "Pleroma.HTTP :#{method} status #{status} for #{url}" end)
end
end
# Catchall
def handle_event(event, measurements, metadata, _) do
Logger.debug(fn ->
"Unhandled telemetry event #{inspect(event)} with measurements #{inspect(measurements)} and metadata #{inspect(metadata)}"
end)
:ok
end
end

File Metadata

Mime Type
text/x-diff
Expires
Mon, Nov 25, 5:29 AM (1 d, 8 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39624
Default Alt Text
(20 KB)

Event Timeline