Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
24 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 79fbd538f..52dbba8ad 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -1,44 +1,44 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances do
@moduledoc "Instances context."
alias Pleroma.Instances.Instance
defdelegate filter_reachable(urls_or_hosts), to: Instance
defdelegate reachable?(url_or_host), to: Instance
defdelegate set_reachable(url_or_host), to: Instance
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance
defdelegate get_unreachable, to: Instance
def host(url_or_host) when is_binary(url_or_host) do
if url_or_host =~ ~r/^http/i do
URI.parse(url_or_host).host
else
url_or_host
end
end
@doc "Schedules reachability checks for all unreachable instances"
def check_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end)
end
@doc "Deletes all users and activities for unreachable instances"
def delete_all_unreachable do
get_unreachable()
|> Enum.each(fn {domain, _} ->
- Instance.delete_users_and_activities(domain)
+ Instance.delete(domain)
end)
end
end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index cf896ca08..3695e0b75 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -1,309 +1,309 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.Instance do
@moduledoc "Instance."
alias Pleroma.Instances
alias Pleroma.Instances.Instance
alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.Workers.DeleteWorker
use Ecto.Schema
import Ecto.Query
import Ecto.Changeset
require Logger
schema "instances" do
field(:host, :string)
field(:unreachable_since, :naive_datetime_usec)
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
field(:software_name, :string)
field(:software_version, :string)
field(:software_repository, :string)
end
field(:metadata_updated_at, :utc_datetime)
timestamps()
end
defdelegate host(url_or_host), to: Instances
def changeset(struct, params \\ %{}) do
struct
|> cast(params, __schema__(:fields) -- [:metadata])
|> cast_embed(:metadata, with: &metadata_changeset/2)
|> validate_required([:host])
|> unique_constraint(:host)
end
def metadata_changeset(struct, params \\ %{}) do
struct
|> cast(params, [:software_name, :software_version, :software_repository])
end
def filter_reachable([]), do: []
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
hosts =
urls_or_hosts
|> Enum.map(&(&1 && host(&1)))
|> Enum.filter(&(to_string(&1) != ""))
unreachable_since_by_host =
Repo.all(
from(i in Instance,
where: i.host in ^hosts,
select: {i.host, i.unreachable_since}
)
)
|> Map.new(& &1)
for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
host = host(entry)
unreachable_since = unreachable_since_by_host[host]
if is_nil(unreachable_since) do
entry
end
end
|> Enum.filter(& &1)
end
def reachable?(url_or_host) when is_binary(url_or_host) do
!Repo.one(
from(i in Instance,
where:
i.host == ^host(url_or_host) and
not is_nil(i.unreachable_since),
select: true
)
)
end
def reachable?(url_or_host) when is_binary(url_or_host), do: true
def set_reachable(url_or_host) when is_binary(url_or_host) do
host = host(url_or_host)
result =
%Instance{host: host}
|> changeset(%{unreachable_since: nil})
|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
Pleroma.Workers.ReachabilityWorker.delete_jobs_for_host(host)
result
end
def set_reachable(_), do: {:error, nil}
def set_unreachable(url_or_host, unreachable_since \\ nil)
def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now()
host = host(url_or_host)
existing_record = Repo.get_by(Instance, %{host: host})
changes = %{unreachable_since: unreachable_since}
cond do
is_nil(existing_record) ->
%Instance{}
|> changeset(Map.put(changes, :host, host))
|> Repo.insert()
existing_record.unreachable_since &&
NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
{:ok, existing_record}
true ->
existing_record
|> changeset(changes)
|> Repo.update()
end
end
def set_unreachable(_, _), do: {:error, nil}
def get_unreachable do
from(i in Instance,
where: not is_nil(i.unreachable_since),
order_by: i.unreachable_since,
select: {i.host, i.unreachable_since}
)
|> Repo.all()
end
defp parse_datetime(datetime) when is_binary(datetime) do
NaiveDateTime.from_iso8601(datetime)
end
defp parse_datetime(datetime), do: datetime
def get_or_update_favicon(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.favicon_updated_at &&
NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
existing_record.favicon
else
favicon = scrape_favicon(instance_uri)
if existing_record do
existing_record
|> changeset(%{favicon: favicon, favicon_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
|> Repo.insert()
end
favicon
end
rescue
e ->
Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
nil
end
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
{_, [favicon_rel | _]} when is_binary(favicon_rel) <-
{:parse,
html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
{_, favicon} when is_binary(favicon) <-
{:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
favicon
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
def get_or_update_metadata(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.metadata_updated_at &&
NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
existing_record.metadata
else
metadata = scrape_metadata(instance_uri)
if existing_record do
existing_record
|> changeset(%{metadata: metadata, metadata_updated_at: now})
|> Repo.update()
else
%Instance{}
|> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
|> Repo.insert()
end
metadata
end
end
defp get_nodeinfo_uri(well_known) do
links = Map.get(well_known, "links", [])
nodeinfo21 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]
nodeinfo20 =
Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]
cond do
is_binary(nodeinfo21) -> {:ok, nodeinfo21}
is_binary(nodeinfo20) -> {:ok, nodeinfo20}
true -> {:error, :no_links}
end
end
defp scrape_metadata(%URI{} = instance_uri) do
try do
with {_, true} <- {:reachable, reachable?(instance_uri.host)},
{:ok, %Tesla.Env{body: well_known_body}} <-
instance_uri
|> URI.merge("/.well-known/nodeinfo")
|> to_string()
|> Pleroma.HTTP.get([{"accept", "application/json"}]),
{:ok, well_known_json} <- Jason.decode(well_known_body),
{:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
{:ok, %Tesla.Env{body: nodeinfo_body}} <-
Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
{:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
# Can extract more metadata from NodeInfo but need to be careful about it's size,
# can't just dump the entire thing
software = Map.get(nodeinfo, "software", %{})
%{
software_name: software["name"],
software_version: software["version"]
}
|> Maps.put_if_present(:software_repository, software["repository"])
else
{:reachable, false} ->
Logger.debug(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
)
nil
_ ->
nil
end
rescue
e ->
Logger.warning(
"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
)
nil
end
end
@doc """
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
"""
- def delete_users_and_activities(host) when is_binary(host) do
+ def delete(host) when is_binary(host) do
DeleteWorker.new(%{"op" => "delete_instance", "host" => host})
|> Oban.insert()
end
@doc "Schedules reachability check for instance"
def check_unreachable(domain) when is_binary(domain) do
Pleroma.Workers.ReachabilityWorker.new(%{"domain" => domain})
|> Oban.insert()
end
end
diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex
index 117a72280..40d4d812e 100644
--- a/lib/pleroma/web/admin_api/controllers/instance_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex
@@ -1,63 +1,63 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.InstanceController do
use Pleroma.Web, :controller
import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3]
alias Pleroma.Instances.Instance
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.AdminAPI
alias Pleroma.Web.Plugs.OAuthScopesPlug
require Logger
@default_page_size 50
plug(
OAuthScopesPlug,
%{scopes: ["admin:read:statuses"]}
when action in [:list_statuses]
)
plug(
OAuthScopesPlug,
%{scopes: ["admin:write:accounts", "admin:write:statuses"]}
when action in [:delete]
)
action_fallback(AdminAPI.FallbackController)
def list_statuses(conn, %{"instance" => instance} = params) do
with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
{page, page_size} = page_params(params)
result =
ActivityPub.fetch_statuses(nil, %{
instance: instance,
limit: page_size,
offset: (page - 1) * page_size,
exclude_reblogs: not with_reblogs,
total: true
})
conn
|> put_view(AdminAPI.StatusView)
|> render("index.json", %{total: result[:total], activities: result[:items], as: :activity})
end
def delete(conn, %{"instance" => instance}) do
- with {:ok, _job} <- Instance.delete_users_and_activities(instance) do
+ with {:ok, _job} <- Instance.delete(instance) do
json(conn, instance)
end
end
defp page_params(params) do
{
fetch_integer_param(params, "page", 1),
fetch_integer_param(params, "page_size", @default_page_size)
}
end
end
diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs
index 83e70ac38..bc3e7993e 100644
--- a/test/pleroma/instances/instance_test.exs
+++ b/test/pleroma/instances/instance_test.exs
@@ -1,282 +1,282 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Instances.InstanceTest do
alias Pleroma.Instances.Instance
alias Pleroma.Repo
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.DataCase
import ExUnit.CaptureLog
import Pleroma.Factory
describe "set_reachable/1" do
test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
instance = insert(:instance, unreachable_since: unreachable_since)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
assert {:ok, instance} = Instance.set_reachable(instance.host)
refute instance.unreachable_since
end
test "cancels all ReachabilityWorker jobs for the domain" do
domain = "cancelme.example.org"
insert(:instance, host: domain, unreachable_since: NaiveDateTime.utc_now())
# Insert a ReachabilityWorker job for this domain, scheduled 5 minutes in the future
scheduled_at = DateTime.add(DateTime.utc_now(), 300, :second)
{:ok, job} =
Pleroma.Workers.ReachabilityWorker.new(
%{"domain" => domain, "phase" => "phase_1min", "attempt" => 1},
scheduled_at: scheduled_at
)
|> Oban.insert()
# Ensure the job is present
job = Pleroma.Repo.get(Oban.Job, job.id)
assert job
# Call set_reachable, which should delete the job
assert {:ok, _} = Instance.set_reachable(domain)
# Reload the job and assert it is deleted
job = Pleroma.Repo.get(Oban.Job, job.id)
refute job
end
end
describe "set_unreachable/1" do
test "creates new record having `unreachable_since` to current time if record does not exist" do
assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path")
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
assert "domain.com" == instance.host
end
test "sets `unreachable_since` of existing record having nil `unreachable_since`" do
instance = insert(:instance, unreachable_since: nil)
refute instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's present" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host)
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "set_unreachable/2" do
test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100)
assert {:ok, _} = Instance.set_unreachable(instance.host, past_value)
instance = Repo.get(Instance, instance.id)
assert past_value == instance.unreachable_since
end
test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do
instance =
insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))
assert instance.unreachable_since
initial_value = instance.unreachable_since
assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now())
instance = Repo.get(Instance, instance.id)
assert initial_value == instance.unreachable_since
end
end
describe "get_or_update_favicon/1" do
test "Scrapes favicon URLs" do
Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
}
end)
assert "https://favicon.example.org/favicon.png" ==
Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
end
test "Returns nil on too long favicon URLs" do
long_favicon_url =
"https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"
Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body:
~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
}
end)
assert capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://long-favicon.example.org/")
)
end) =~
"Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
end
test "Handles not getting a favicon URL properly" do
Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} ->
%Tesla.Env{
status: 200,
body: ~s[<html><head><h1>I wil look down and whisper "GNO.."</h1></head></html>]
}
end)
refute capture_log(fn ->
assert nil ==
Instance.get_or_update_favicon(
URI.parse("https://no-favicon.example.org/")
)
end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: "
end
test "Doesn't scrapes unreachable instances" do
instance =
insert(:instance,
unreachable_since: NaiveDateTime.utc_now() |> NaiveDateTime.add(-:timer.hours(24))
)
url = "https://" <> instance.host
assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~
"Instance.scrape_favicon(\"#{url}\") ignored unreachable host"
end
end
describe "get_or_update_metadata/1" do
test "Scrapes Wildebeest NodeInfo" do
Tesla.Mock.mock(fn
%{url: "https://wildebeest.example.org/.well-known/nodeinfo"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/wildebeest-well-known-nodeinfo.json")
}
%{url: "https://wildebeest.example.org/nodeinfo/2.1"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/wildebeest-nodeinfo21.json")
}
end)
expected = %{
software_name: "wildebeest",
software_repository: "https://github.com/cloudflare/wildebeest",
software_version: "0.0.1"
}
assert expected ==
Instance.get_or_update_metadata(URI.parse("https://wildebeest.example.org/"))
expected = %Pleroma.Instances.Instance.Pleroma.Instances.Metadata{
software_name: "wildebeest",
software_repository: "https://github.com/cloudflare/wildebeest",
software_version: "0.0.1"
}
assert expected ==
Repo.get_by(Pleroma.Instances.Instance, %{host: "wildebeest.example.org"}).metadata
end
test "Scrapes Mastodon NodeInfo" do
Tesla.Mock.mock(fn
%{url: "https://mastodon.example.org/.well-known/nodeinfo"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/mastodon-well-known-nodeinfo.json")
}
%{url: "https://mastodon.example.org/nodeinfo/2.0"} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/mastodon-nodeinfo20.json")
}
end)
expected = %{
software_name: "mastodon",
software_version: "4.1.0"
}
assert expected ==
Instance.get_or_update_metadata(URI.parse("https://mastodon.example.org/"))
end
end
- test "delete_users_and_activities/1 schedules a job to delete the instance and users" do
+ test "delete/1 schedules a job to delete the instance and users" do
insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario")
- {:ok, _job} = Instance.delete_users_and_activities("mushroom.kingdom")
+ {:ok, _job} = Instance.delete("mushroom.kingdom")
assert_enqueued(
worker: Pleroma.Workers.DeleteWorker,
args: %{"op" => "delete_instance", "host" => "mushroom.kingdom"}
)
end
describe "check_unreachable/1" do
test "schedules a ReachabilityWorker job for the given domain" do
domain = "test.example.com"
# Call check_unreachable
assert {:ok, _job} = Instance.check_unreachable(domain)
# Verify that a ReachabilityWorker job was scheduled
jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
assert length(jobs) == 1
[job] = jobs
assert job.args["domain"] == domain
end
test "handles multiple calls for the same domain (uniqueness enforced)" do
domain = "duplicate.example.com"
assert {:ok, _job1} = Instance.check_unreachable(domain)
# Second call for the same domain
assert {:ok, %Oban.Job{conflict?: true}} = Instance.check_unreachable(domain)
# Should only have one job due to uniqueness
jobs = all_enqueued(worker: Pleroma.Workers.ReachabilityWorker)
assert length(jobs) == 1
[job] = jobs
assert job.args["domain"] == domain
end
end
end
diff --git a/test/pleroma/workers/delete_worker_test.exs b/test/pleroma/workers/delete_worker_test.exs
index b914aaee2..1becd0c03 100644
--- a/test/pleroma/workers/delete_worker_test.exs
+++ b/test/pleroma/workers/delete_worker_test.exs
@@ -1,39 +1,39 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.DeleteWorkerTest do
use Pleroma.DataCase, async: true
use Oban.Testing, repo: Pleroma.Repo
import Pleroma.Factory
alias Pleroma.Instances.Instance
alias Pleroma.Tests.ObanHelpers
alias Pleroma.Workers.DeleteWorker
describe "instance deletion" do
test "creates individual Oban jobs for each user when deleting an instance" do
user1 = insert(:user, nickname: "alice@example.com", name: "Alice")
user2 = insert(:user, nickname: "bob@example.com", name: "Bob")
- {:ok, job} = Instance.delete_users_and_activities("example.com")
+ {:ok, job} = Instance.delete("example.com")
assert_enqueued(
worker: DeleteWorker,
args: %{"op" => "delete_instance", "host" => "example.com"}
)
{:ok, :ok} = ObanHelpers.perform(job)
delete_user_jobs = all_enqueued(worker: DeleteWorker, args: %{"op" => "delete_user"})
assert length(delete_user_jobs) == 2
user_ids = [user1.id, user2.id]
job_user_ids = Enum.map(delete_user_jobs, fn job -> job.args["user_id"] end)
assert Enum.sort(user_ids) == Enum.sort(job_user_ids)
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Thu, Oct 29, 5:25 PM (1 d, 14 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1798408
Default Alt Text
(24 KB)

Event Timeline