Page MenuHomePhorge
Files F85806242

No OneTemporary
Actions

Size
21 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a5529ad44..9756c66dc 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -1,220 +1,318 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.Instance do
@moduledoc "Instance."
alias Pleroma.Instances
alias Pleroma.Instances.Instance
+ alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.BackgroundWorker
use Ecto.Schema
import Ecto.Query
import Ecto.Changeset
require Logger
schema "instances" do
field(:host, :string)
field(:unreachable_since, :naive_datetime_usec)
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
+ embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
+ field(:software_name, :string)
+ field(:software_version, :string)
+ field(:software_repository, :string)
+ end
+
+ field(:metadata_updated_at, :utc_datetime)
+
timestamps()
end
defdelegate host(url_or_host), to: Instances
def changeset(struct, params \\ %{}) do
struct
- |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at])
+ |> cast(params, __schema__(:fields) -- [:metadata])
+ |> cast_embed(:metadata, with: &metadata_changeset/2)
|> validate_required([:host])
|> unique_constraint(:host)
end
+ def metadata_changeset(struct, params \\ %{}) do
+ struct
+ |> cast(params, [:software_name, :software_version, :software_repository])
+ end
+
def filter_reachable([]), do: %{}
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
urls_or_hosts
|> Enum.map(&(&1 && host(&1)))
|> Enum.filter(&(to_string(&1) != ""))
unreachable_since_by_host =
Repo.all(
from(i in Instance,
where: i.host in ^hosts,
select: {i.host, i.unreachable_since}
)
)
|> Map.new(& &1)
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if !unreachable_since ||
NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
{entry, unreachable_since}
end
end
|> Enum.filter(& &1)
|> Map.new(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
!Repo.one(
from(i in Instance,
where:
i.host == ^host(url_or_host) and
i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
select: true
)
)
end
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
with host <- host(url_or_host),
%Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
{:ok, _instance} =
existing_record
|> changeset(%{unreachable_since: nil})
|> Repo.update()
end
end
def set_reachable(_), do: {:error, nil}
def set_unreachable(url_or_host, unreachable_since \\ nil)
def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now()
host = host(url_or_host)
existing_record = Repo.get_by(Instance, %{host: host})
changes = %{unreachable_since: unreachable_since}
cond do
is_nil(existing_record) ->
%Instance{}
|> changeset(Map.put(changes, :host, host))
|> Repo.insert()
existing_record.unreachable_since &&
NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
{:ok, existing_record}
true ->
existing_record
|> changeset(changes)
|> Repo.update()
end
end
def set_unreachable(_, _), do: {:error, nil}
def get_consistently_unreachable do
reachability_datetime_threshold = Instances.reachability_datetime_threshold()
from(i in Instance,
where: ^reachability_datetime_threshold > i.unreachable_since,
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
|> Repo.all()
end
defp parse_datetime(datetime) when is_binary(datetime) do
NaiveDateTime.from_iso8601(datetime)
end
defp parse_datetime(datetime), do: datetime
def get_or_update_favicon(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.favicon_updated_at &&
NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
existing_record.favicon
else
favicon = scrape_favicon(instance_uri)
if existing_record do
existing_record
|> changeset(%{favicon: favicon, favicon_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
|> Repo.insert()
end
favicon
end
rescue
e ->
Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
nil
end
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
{:parse,
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
{_, favicon} when is_binary(favicon) <-
{:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
favicon
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warn(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
+ def get_or_update_metadata(%URI{host: host} = instance_uri) do
+ existing_record = Repo.get_by(Instance, %{host: host})
+ now = NaiveDateTime.utc_now()
+
+ if existing_record && existing_record.metadata_updated_at &&
+ NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
+ existing_record.metadata
+ else
+ metadata = scrape_metadata(instance_uri)
+
+ if existing_record do
+ existing_record
+ |> changeset(%{metadata: metadata, metadata_updated_at: now})
+ |> Repo.update()
+ else
+ %Instance{}
+ |> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
+ |> Repo.insert()
+ end
+
+ metadata
+ end
+ end
+
+ defp get_nodeinfo_uri(well_known) do
+ links = Map.get(well_known, "links", [])
+
+ nodeinfo21 =
+ Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]
+
+ nodeinfo20 =
+ Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]
+
+ cond do
+ is_binary(nodeinfo21) -> {:ok, nodeinfo21}
+ is_binary(nodeinfo20) -> {:ok, nodeinfo20}
+ true -> {:error, :no_links}
+ end
+ end
+
+ defp scrape_metadata(%URI{} = instance_uri) do
+ try do
+ with {_, true} <- {:reachable, reachable?(instance_uri.host)},
+ {:ok, %Tesla.Env{body: well_known_body}} <-
+ instance_uri
+ |> URI.merge("/.well-known/nodeinfo")
+ |> to_string()
+ |> Pleroma.HTTP.get([{"accept", "application/json"}]),
+ {:ok, well_known_json} <- Jason.decode(well_known_body),
+ {:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
+ {:ok, %Tesla.Env{body: nodeinfo_body}} <-
+ Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
+ {:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
+ # Can extract more metadata from NodeInfo but need to be careful about it's size,
+ # can't just dump the entire thing
+ software = Map.get(nodeinfo, "software", %{})
+
+ %{
+ software_name: software["name"],
+ software_version: software["version"]
+ }
+ |> Maps.put_if_present(:software_repository, software["repository"])
+ else
+ {:reachable, false} ->
+ Logger.debug(
+ "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
+ )
+
+ nil
+
+ _ ->
+ nil
+ end
+ rescue
+ e ->
+ Logger.warn(
+ "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+ )
+
+ nil
+ end
+ end
+
@doc """
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
def delete_users_and_activities(host) when is_binary(host) do
BackgroundWorker.enqueue("delete_instance", %{"host" => host})
end
def perform(:delete_instance, host) when is_binary(host) do
User.Query.build(%{nickname: "@#{host}"})
|> Repo.chunk_stream(100, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
User.perform(:delete, user)
end)
end)
|> Stream.run()
end
end
diff --git a/priv/repo/migrations/20230306112859_instances_add_metadata.exs b/priv/repo/migrations/20230306112859_instances_add_metadata.exs
new file mode 100644
index 000000000..898f5220e
--- /dev/null
+++ b/priv/repo/migrations/20230306112859_instances_add_metadata.exs
@@ -0,0 +1,14 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.InstancesAddMetadata do
+ use Ecto.Migration
+
+ def change do
+ alter table(:instances) do
+ add(:metadata, :map)
+ add(:metadata_updated_at, :utc_datetime)
+ end
+ end
+end
diff --git a/test/fixtures/mastodon-nodeinfo20.json b/test/fixtures/mastodon-nodeinfo20.json
new file mode 100644
index 000000000..35010fdf0
--- /dev/null
+++ b/test/fixtures/mastodon-nodeinfo20.json
@@ -0,0 +1 @@
+{"version":"2.0","software":{"name":"mastodon","version":"4.1.0"},"protocols":["activitypub"],"services":{"outbound":[],"inbound":[]},"usage":{"users":{"total":971090,"activeMonth":167218,"activeHalfyear":384808},"localPosts":52071541},"openRegistrations":true,"metadata":{}}
\ No newline at end of file
diff --git a/test/fixtures/mastodon-well-known-nodeinfo.json b/test/fixtures/mastodon-well-known-nodeinfo.json
new file mode 100644
index 000000000..237d5462a
--- /dev/null
+++ b/test/fixtures/mastodon-well-known-nodeinfo.json
@@ -0,0 +1 @@
+{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://mastodon.example.org/nodeinfo/2.0"}]}
\ No newline at end of file
diff --git a/test/fixtures/wildebeest-nodeinfo21.json b/test/fixtures/wildebeest-nodeinfo21.json
new file mode 100644
index 000000000..c6af474bf
--- /dev/null
+++ b/test/fixtures/wildebeest-nodeinfo21.json
@@ -0,0 +1 @@
+{"version":"2.1","software":{"name":"wildebeest","version":"0.0.1","repository":"https://github.com/cloudflare/wildebeest"},"protocols":["activitypub"],"usage":{"users":{"total":1,"activeMonth":1,"activeHalfyear":1}},"openRegistrations":false,"metadata":{"upstream":{"name":"mastodon","version":"3.5.1"}}}
\ No newline at end of file
diff --git a/test/fixtures/wildebeest-well-known-nodeinfo.json b/test/fixtures/wildebeest-well-known-nodeinfo.json
new file mode 100644
index 000000000..c7ddb43af
--- /dev/null
+++ b/test/fixtures/wildebeest-well-known-nodeinfo.json
@@ -0,0 +1 @@
+{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://wildebeest.example.org/nodeinfo/2.0"},{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://wildebeest.example.org/nodeinfo/2.1"}]}
\ No newline at end of file
diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs
index 861519bce..a769f9362 100644
--- a/test/pleroma/instances/instance_test.exs
+++ b/test/pleroma/instances/instance_test.exs
@@ -1,192 +1,252 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.InstanceTest do
alias Pleroma.Instances
alias Pleroma.Instances.Instance
alias Pleroma.Repo
alias Pleroma.Tests.ObanHelpers
alias Pleroma.Web.CommonAPI
use Pleroma.DataCase
import ExUnit.CaptureLog
import Pleroma.Factory
setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
describe "set_reachable/1" do
test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
instance = insert(:instance, unreachable_since: unreachable_since)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "does NOT create an Instance record in case of no existing matching record" do
host = "domain.org"
assert nil == Instance.set_reachable(host)
assert [] = Repo.all(Ecto.Query.from(i in Instance))
assert Instance.reachable?(host)
end
end
describe "set_unreachable/1" do
test "creates new record having `unreachable_since` to current time if record does not exist" do
assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path")
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
assert "domain.com" == instance.host
end
test "sets `unreachable_since` of existing record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
refute instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's present" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "set_unreachable/2" do
test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100)
assert {:ok, _} = Instance.set_unreachable(instance.host, past_value)
instance = Repo.get(Instance, instance.id)
assert past_value == instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now())
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "get_or_update_favicon/1" do
test "Scrapes favicon URLs" do
Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
}
end)
assert "https://favicon.example.org/favicon.png" ==
Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
end
test "Returns nil on too long favicon URLs" do
long_favicon_url =
"https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"
Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body:
~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
}
end)
assert capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://long-favicon.example.org/")
)
end) =~
"Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
end
test "Handles not getting a favicon URL properly" do
Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><h1>I wil look down and whisper "GNO.."</h1></head></html>]
}
end)
refute capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://no-favicon.example.org/")
)
end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: "
end
test "Doesn't scrapes unreachable instances" do
instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold())
url = "https://" <> instance.host
assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~
"Instance.scrape_favicon(\"#{url}\") ignored unreachable host"
end
end
+ describe "get_or_update_metadata/1" do
+ test "Scrapes Wildebeest NodeInfo" do
+ Tesla.Mock.mock(fn
+ %{url: "https://wildebeest.example.org/.well-known/nodeinfo"} ->
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/wildebeest-well-known-nodeinfo.json")
+ }
+
+ %{url: "https://wildebeest.example.org/nodeinfo/2.1"} ->
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/wildebeest-nodeinfo21.json")
+ }
+ end)
+
+ expected = %{
+ software_name: "wildebeest",
+ software_repository: "https://github.com/cloudflare/wildebeest",
+ software_version: "0.0.1"
+ }
+
+ assert expected ==
+ Instance.get_or_update_metadata(URI.parse("https://wildebeest.example.org/"))
+
+ expected = %Pleroma.Instances.Instance.Pleroma.Instances.Metadata{
+ software_name: "wildebeest",
+ software_repository: "https://github.com/cloudflare/wildebeest",
+ software_version: "0.0.1"
+ }
+
+ assert expected ==
+ Repo.get_by(Pleroma.Instances.Instance, %{host: "wildebeest.example.org"}).metadata
+ end
+
+ test "Scrapes Mastodon NodeInfo" do
+ Tesla.Mock.mock(fn
+ %{url: "https://mastodon.example.org/.well-known/nodeinfo"} ->
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/mastodon-well-known-nodeinfo.json")
+ }
+
+ %{url: "https://mastodon.example.org/nodeinfo/2.0"} ->
+ %Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/mastodon-nodeinfo20.json")
+ }
+ end)
+
+ expected = %{
+ software_name: "mastodon",
+ software_version: "4.1.0"
+ }
+
+ assert expected ==
+ Instance.get_or_update_metadata(URI.parse("https://mastodon.example.org/"))
+ end
+ end
+
test "delete_users_and_activities/1 deletes remote instance users and activities" do
[mario, luigi, _peach, wario] =
users = [
insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario"),
insert(:user, nickname: "luigi@mushroom.kingdom", name: "Luigi"),
insert(:user, nickname: "peach@mushroom.kingdom", name: "Peach"),
insert(:user, nickname: "wario@greedville.biz", name: "Wario")
]
{:ok, post1} = CommonAPI.post(mario, %{status: "letsa go!"})
{:ok, post2} = CommonAPI.post(luigi, %{status: "itsa me... luigi"})
{:ok, post3} = CommonAPI.post(wario, %{status: "WHA-HA-HA!"})
{:ok, job} = Instance.delete_users_and_activities("mushroom.kingdom")
:ok = ObanHelpers.perform(job)
[mario, luigi, peach, wario] = Repo.reload(users)
refute mario.is_active
refute luigi.is_active
refute peach.is_active
refute peach.name == "Peach"
assert wario.is_active
assert wario.name == "Wario"
assert [nil, nil, %{}] = Repo.reload([post1, post2, post3])
end
end

File Metadata

Mime Type
text/x-diff
Expires
Mon, Oct 12, 5:30 PM (1 d, 18 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1786384
Default Alt Text
(21 KB)

Event Timeline