Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85824636
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
24 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 79fbd538f..52dbba8ad 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -1,44 +1,44 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances do
@moduledoc "Instances context."
alias Pleroma.Instances.Instance
defdelegate filter_reachable(urls_or_hosts), to: Instance
defdelegate reachable?(url_or_host), to: Instance
defdelegate set_reachable(url_or_host), to: Instance
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
defdelegate get_unreachable, to: Instance
def host(url_or_host) when is_binary(url_or_host) do
if url_or_host =~ ~r/^http/i do
URI.parse(url_or_host).host
else
url_or_host
end
end
@doc "Schedules reachability checks for all unreachable instances"
def check_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end)
end
@doc "Deletes all users and activities for unreachable instances"
def delete_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
- Instance.delete_users_and_activities(domain)
+ Instance.delete(domain)
end)
end
end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index cf896ca08..3695e0b75 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -1,309 +1,309 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.Instance do
@moduledoc "Instance."
alias Pleroma.Instances
alias Pleroma.Instances.Instance
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
import Ecto.Query
import Ecto.Changeset
require Logger
schema "instances" do
field(:host, :string)
field(:unreachable_since, :naive_datetime_usec)
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
field(:software_name, :string)
field(:software_version, :string)
field(:software_repository, :string)
end
field(:metadata_updated_at, :utc_datetime)
timestamps()
end
defdelegate host(url_or_host), to: Instances
def changeset(struct, params \\ %{}) do
struct
|> cast(params, __schema__(:fields) -- [:metadata])
|> cast_embed(:metadata, with: &metadata_changeset/2)
|> validate_required([:host])
|> unique_constraint(:host)
end
def metadata_changeset(struct, params \\ %{}) do
struct
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: []
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
urls_or_hosts
|> Enum.map(&(&1 && host(&1)))
|> Enum.filter(&(to_string(&1) != ""))
unreachable_since_by_host =
Repo.all(
from(i in Instance,
where: i.host in ^hosts,
select: {i.host, i.unreachable_since}
)
)
|> Map.new(& &1)
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if is_nil(unreachable_since) do
entry
end
end
|> Enum.filter(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
!Repo.one(
from(i in Instance,
where:
i.host == ^host(url_or_host) and
not is_nil(i.unreachable_since),
select: true
)
)
end
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
host = host(url_or_host)
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
result
end
def set_reachable(_), do: {:error, nil}
def set_unreachable(url_or_host, unreachable_since \\ nil)
def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now()
host = host(url_or_host)
existing_record = Repo.get_by(Instance, %{host: host})
changes = %{unreachable_since: unreachable_since}
cond do
is_nil(existing_record) ->
%Instance{}
|> changeset(Map.put(changes, :host, host))
|> Repo.insert()
existing_record.unreachable_since &&
NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
{:ok, existing_record}
true ->
existing_record
|> changeset(changes)
|> Repo.update()
end
end
def set_unreachable(_, _), do: {:error, nil}
def get_unreachable do
from(i in Instance,
where: not is_nil(i.unreachable_since),
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
|> Repo.all()
end
defp parse_datetime(datetime) when is_binary(datetime) do
NaiveDateTime.from_iso8601(datetime)
end
defp parse_datetime(datetime), do: datetime
def get_or_update_favicon(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.favicon_updated_at &&
NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
existing_record.favicon
else
favicon = scrape_favicon(instance_uri)
if existing_record do
existing_record
|> changeset(%{favicon: favicon, favicon_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
|> Repo.insert()
end
favicon
end
rescue
e ->
Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
nil
end
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
{:parse,
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
{_, favicon} when is_binary(favicon) <-
{:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
favicon
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
def get_or_update_metadata(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.metadata_updated_at &&
NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
existing_record.metadata
else
metadata = scrape_metadata(instance_uri)
if existing_record do
existing_record
|> changeset(%{metadata: metadata, metadata_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
|> Repo.insert()
end
metadata
end
end
defp get_nodeinfo_uri(well_known) do
links = Map.get(well_known, "links", [])
nodeinfo21 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]
nodeinfo20 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]
cond do
is_binary(nodeinfo21) -> {:ok, nodeinfo21}
is_binary(nodeinfo20) -> {:ok, nodeinfo20}
true -> {:error, :no_links}
end
end
defp scrape_metadata(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: well_known_body}} <-
instance_uri
|> URI.merge("/.well-known/nodeinfo")
|> to_string()
|> Pleroma.HTTP.get([{"accept", "application/json"}]),
{:ok, well_known_json} <- Jason.decode(well_known_body),
{:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
{:ok, %Tesla.Env{body: nodeinfo_body}} <-
Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
{:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
# Can extract more metadata from NodeInfo but need to be careful about it's size,
# can't just dump the entire thing
software = Map.get(nodeinfo, "software", %{})
%{
software_name: software["name"],
software_version: software["version"]
}
|> Maps.put_if_present(:software_repository, software["repository"])
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
@doc """
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
- def delete_users_and_activities(host) when is_binary(host) do
+ def delete(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
@doc "Schedules reachability check for instance"
def check_unreachable(domain) when is_binary(domain) do
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end
end
diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex
index 117a72280..40d4d812e 100644
--- a/lib/pleroma/web/admin_api/controllers/instance_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex
@@ -1,63 +1,63 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.InstanceController do
use Pleroma.Web, :controller
import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3]
alias Pleroma.Instances.Instance
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.AdminAPI
alias Pleroma.Web.Plugs.OAuthScopesPlug
require Logger
@default_page_size 50
plug(
OAuthScopesPlug,
%{scopes: ["admin:read:statuses"]}
when action in [:list_statuses]
)
plug(
OAuthScopesPlug,
%{scopes: ["admin:write:accounts", "admin:write:statuses"]}
when action in [:delete]
)
action_fallback(AdminAPI.FallbackController)
def list_statuses(conn, %{"instance" => instance} = params) do
with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
{page, page_size} = page_params(params)
result =
ActivityPub.fetch_statuses(nil, %{
instance: instance,
limit: page_size,
offset: (page - 1) * page_size,
exclude_reblogs: not with_reblogs,
total: true
})
conn
|> put_view(AdminAPI.StatusView)
|> render("index.json", %{total: result[:total], activities: result[:items], as: :activity})
end
def delete(conn, %{"instance" => instance}) do
- with {:ok, _job} <- Instance.delete_users_and_activities(instance) do
+ with {:ok, _job} <- Instance.delete(instance) do
json(conn, instance)
end
end
defp page_params(params) do
{
fetch_integer_param(params, "page", 1),
fetch_integer_param(params, "page_size", @default_page_size)
}
end
end
diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs
index 83e70ac38..bc3e7993e 100644
--- a/test/pleroma/instances/instance_test.exs
+++ b/test/pleroma/instances/instance_test.exs
@@ -1,282 +1,282 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.InstanceTest do
alias Pleroma.Instances.Instance
alias Pleroma.Repo
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.DataCase
import ExUnit.CaptureLog
import Pleroma.Factory
describe "set_reachable/1" do
test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
instance = insert(:instance, unreachable_since: unreachable_since)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "cancels all ReachabilityWorker jobs for the domain" do
domain = "cancelme.example.org"
insert(:instance, host: domain, unreachable_since: NaiveDateTime.utc_now())
# Insert a ReachabilityWorker job for this domain, scheduled 5 minutes in the future
scheduled_at = DateTime.add(DateTime.utc_now(), 300, :second)
{:ok, job} =
Pleroma.Workers.ReachabilityWorker.new(
%{"domain" => domain, "phase" => "phase_1min", "attempt" => 1},
scheduled_at: scheduled_at
)
|> Oban.insert()
# Ensure the job is present
job = Pleroma.Repo.get(Oban.Job, job.id)
assert job
# Call set_reachable, which should delete the job
assert {:ok, _} = Instance.set_reachable(domain)
# Reload the job and assert it is deleted
job = Pleroma.Repo.get(Oban.Job, job.id)
refute job
end
end
describe "set_unreachable/1" do
test "creates new record having `unreachable_since` to current time if record does not exist" do
assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path")
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
assert "domain.com" == instance.host
end
test "sets `unreachable_since` of existing record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
refute instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's present" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "set_unreachable/2" do
test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100)
assert {:ok, _} = Instance.set_unreachable(instance.host, past_value)
instance = Repo.get(Instance, instance.id)
assert past_value == instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now())
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "get_or_update_favicon/1" do
test "Scrapes favicon URLs" do
Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
}
end)
assert "https://favicon.example.org/favicon.png" ==
Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
end
test "Returns nil on too long favicon URLs" do
long_favicon_url =
"https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"
Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body:
~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
}
end)
assert capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://long-favicon.example.org/")
)
end) =~
"Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
end
test "Handles not getting a favicon URL properly" do
Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><h1>I wil look down and whisper "GNO.."</h1></head></html>]
}
end)
refute capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://no-favicon.example.org/")
)
end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: "
end
test "Doesn't scrapes unreachable instances" do
instance =
insert(:instance,
unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.add(-:timer.hours(24))
)
url = "https://" <> instance.host
assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~
"Instance.scrape_favicon(\"#{url}\") ignored unreachable host"
end
end
describe "get_or_update_metadata/1" do
test "Scrapes Wildebeest NodeInfo" do
Tesla.Mock.mock(fn
%{url: "https://wildebeest.example.org/.well-known/nodeinfo"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/wildebeest-well-known-nodeinfo.json")
}
%{url: "https://wildebeest.example.org/nodeinfo/2.1"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/wildebeest-nodeinfo21.json")
}
end)
expected = %{
software_name: "wildebeest",
software_repository: "https://github.com/cloudflare/wildebeest",
software_version: "0.0.1"
}
assert expected ==
Instance.get_or_update_metadata(URI.parse("https://wildebeest.example.org/"))
expected = %Pleroma.Instances.Instance.Pleroma.Instances.Metadata{
software_name: "wildebeest",
software_repository: "https://github.com/cloudflare/wildebeest",
software_version: "0.0.1"
}
assert expected ==
Repo.get_by(Pleroma.Instances.Instance, %{host: "wildebeest.example.org"}).metadata
end
test "Scrapes Mastodon NodeInfo" do
Tesla.Mock.mock(fn
%{url: "https://mastodon.example.org/.well-known/nodeinfo"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/mastodon-well-known-nodeinfo.json")
}
%{url: "https://mastodon.example.org/nodeinfo/2.0"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/mastodon-nodeinfo20.json")
}
end)
expected = %{
software_name: "mastodon",
software_version: "4.1.0"
}
assert expected ==
Instance.get_or_update_metadata(URI.parse("https://mastodon.example.org/"))
end
end
- test "delete_users_and_activities/1 schedules a job to delete the instance and users" do
+ test "delete/1 schedules a job to delete the instance and users" do
insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario")
- {:ok, _job} = Instance.delete_users_and_activities("mushroom.kingdom")
+ {:ok, _job} = Instance.delete("mushroom.kingdom")
assert_enqueued(
worker: Pleroma.Workers.DeleteWorker,
args: %{"op" => "delete_instance", "host" => "mushroom.kingdom"}
)
end
describe "check_unreachable/1" do
test "schedules a ReachabilityWorker job for the given domain" do
domain = "test.example.com"
# Call check_unreachable
assert {:ok, _job} = Instance.check_unreachable(domain)
# Verify that a ReachabilityWorker job was scheduled
jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
assert length(jobs) == 1
[job] = jobs
assert job.args["domain"] == domain
end
test "handles multiple calls for the same domain (uniqueness enforced)" do
domain = "duplicate.example.com"
assert {:ok, _job1} = Instance.check_unreachable(domain)
# Second call for the same domain
assert {:ok, %Oban.Job{conflict?: true}} = Instance.check_unreachable(domain)
# Should only have one job due to uniqueness
jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
assert length(jobs) == 1
[job] = jobs
assert job.args["domain"] == domain
end
end
end
diff --git a/test/pleroma/workers/delete_worker_test.exs b/test/pleroma/workers/delete_worker_test.exs
index b914aaee2..1becd0c03 100644
--- a/test/pleroma/workers/delete_worker_test.exs
+++ b/test/pleroma/workers/delete_worker_test.exs
@@ -1,39 +1,39 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.DeleteWorkerTest do
use Pleroma.DataCase, async: true
use Oban.Testing, repo: Pleroma.Repo
import Pleroma.Factory
alias Pleroma.Instances.Instance
alias Pleroma.Tests.ObanHelpers
alias Pleroma.Workers.DeleteWorker
describe "instance deletion" do
test "creates individual Oban jobs for each user when deleting an instance" do
user1 = insert(:user, nickname: "alice@example.com", name: "Alice")
user2 = insert(:user, nickname: "bob@example.com", name: "Bob")
- {:ok, job} = Instance.delete_users_and_activities("example.com")
+ {:ok, job} = Instance.delete("example.com")
assert_enqueued(
worker: DeleteWorker,
args: %{"op" => "delete_instance", "host" => "example.com"}
)
{:ok, :ok} = ObanHelpers.perform(job)
delete_user_jobs = all_enqueued(worker: DeleteWorker, args: %{"op" => "delete_user"})
assert length(delete_user_jobs) == 2
user_ids = [user1.id, user2.id]
job_user_ids = Enum.map(delete_user_jobs, fn job -> job.args["user_id"] end)
assert Enum.sort(user_ids) == Enum.sort(job_user_ids)
end
end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Oct 29, 5:25 PM (1 d, 12 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1798408
Default Alt Text
(24 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline