No OneTemporary
Actions

Size

19 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/changelog.d/reachability.change b/changelog.d/reachability.change
	new file mode 100644
	index 000000000..06f63272b
	--- /dev/null
	+++ b/changelog.d/reachability.change
	@@ -0,0 +1 @@
	+Reduce the reachability timestamp update to a single upsert query
	diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
	index 9756c66dc..101e5dc88 100644
	--- a/lib/pleroma/instances/instance.ex
	+++ b/lib/pleroma/instances/instance.ex
	@@ -1,318 +1,314 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Instances.Instance do
	@moduledoc "Instance."

	alias Pleroma.Instances
	alias Pleroma.Instances.Instance
	alias Pleroma.Maps
	alias Pleroma.Repo
	alias Pleroma.User
	alias Pleroma.Workers.BackgroundWorker

	use Ecto.Schema

	import Ecto.Query
	import Ecto.Changeset

	require Logger

	schema "instances" do
	field(:host, :string)
	field(:unreachable_since, :naive_datetime_usec)
	field(:favicon, :string)
	field(:favicon_updated_at, :naive_datetime)

	embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
	field(:software_name, :string)
	field(:software_version, :string)
	field(:software_repository, :string)
	end

	field(:metadata_updated_at, :utc_datetime)

	timestamps()
	end

	defdelegate host(url_or_host), to: Instances

	def changeset(struct, params \\ %{}) do
	struct
	\|> cast(params, __schema__(:fields) -- [:metadata])
	\|> cast_embed(:metadata, with: &metadata_changeset/2)
	\|> validate_required([:host])
	\|> unique_constraint(:host)
	end

	def metadata_changeset(struct, params \\ %{}) do
	struct
	\|> cast(params, [:software_name, :software_version, :software_repository])
	end

	def filter_reachable([]), do: %{}

	def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
	hosts =
	urls_or_hosts
	\|> Enum.map(&(&1 && host(&1)))
	\|> Enum.filter(&(to_string(&1) != ""))

	unreachable_since_by_host =
	Repo.all(
	from(i in Instance,
	where: i.host in ^hosts,
	select: {i.host, i.unreachable_since}
	)
	)
	\|> Map.new(& &1)

	reachability_datetime_threshold = Instances.reachability_datetime_threshold()

	for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
	host = host(entry)
	unreachable_since = unreachable_since_by_host[host]

	if !unreachable_since \|\|
	NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
	{entry, unreachable_since}
	end
	end
	\|> Enum.filter(& &1)
	\|> Map.new(& &1)
	end

	def reachable?(url_or_host) when is_binary(url_or_host) do
	!Repo.one(
	from(i in Instance,
	where:
	i.host == ^host(url_or_host) and
	i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
	select: true
	)
	)
	end

	def reachable?(url_or_host) when is_binary(url_or_host), do: true

	def set_reachable(url_or_host) when is_binary(url_or_host) do
	- with host <- host(url_or_host),
	- %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
	- {:ok, _instance} =
	- existing_record
	- \|> changeset(%{unreachable_since: nil})
	- \|> Repo.update()
	- end
	+ %Instance{host: host(url_or_host)}
	+ \|> changeset(%{unreachable_since: nil})
	+ \|> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
	end

	def set_reachable(_), do: {:error, nil}

	def set_unreachable(url_or_host, unreachable_since \\ nil)

	def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
	unreachable_since = parse_datetime(unreachable_since) \|\| NaiveDateTime.utc_now()
	host = host(url_or_host)
	existing_record = Repo.get_by(Instance, %{host: host})

	changes = %{unreachable_since: unreachable_since}

	cond do
	is_nil(existing_record) ->
	%Instance{}
	\|> changeset(Map.put(changes, :host, host))
	\|> Repo.insert()

	existing_record.unreachable_since &&
	NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
	{:ok, existing_record}

	true ->
	existing_record
	\|> changeset(changes)
	\|> Repo.update()
	end
	end

	def set_unreachable(_, _), do: {:error, nil}

	def get_consistently_unreachable do
	reachability_datetime_threshold = Instances.reachability_datetime_threshold()

	from(i in Instance,
	where: ^reachability_datetime_threshold > i.unreachable_since,
	order_by: i.unreachable_since,
	select: {i.host, i.unreachable_since}
	)
	\|> Repo.all()
	end

	defp parse_datetime(datetime) when is_binary(datetime) do
	NaiveDateTime.from_iso8601(datetime)
	end

	defp parse_datetime(datetime), do: datetime

	def get_or_update_favicon(%URI{host: host} = instance_uri) do
	existing_record = Repo.get_by(Instance, %{host: host})
	now = NaiveDateTime.utc_now()

	if existing_record && existing_record.favicon_updated_at &&
	NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
	existing_record.favicon
	else
	favicon = scrape_favicon(instance_uri)

	if existing_record do
	existing_record
	\|> changeset(%{favicon: favicon, favicon_updated_at: now})
	\|> Repo.update()
	else
	%Instance{}
	\|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
	\|> Repo.insert()
	end

	favicon
	end
	rescue
	e ->
	Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
	nil
	end

	defp scrape_favicon(%URI{} = instance_uri) do
	try do
	with {_, true} <- {:reachable, reachable?(instance_uri.host)},
	{:ok, %Tesla.Env{body: html}} <-
	Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
	{_, [favicon_rel \| _]} when is_binary(favicon_rel) <-
	{:parse,
	html \|> Floki.parse_document!() \|> Floki.attribute("link[rel=icon]", "href")},
	{_, favicon} when is_binary(favicon) <-
	{:merge, URI.merge(instance_uri, favicon_rel) \|> to_string()} do
	favicon
	else
	{:reachable, false} ->
	Logger.debug(
	"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
	)

	nil

	_ ->
	nil
	end
	rescue
	e ->
	Logger.warn(
	"Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
	)

	nil
	end
	end

	def get_or_update_metadata(%URI{host: host} = instance_uri) do
	existing_record = Repo.get_by(Instance, %{host: host})
	now = NaiveDateTime.utc_now()

	if existing_record && existing_record.metadata_updated_at &&
	NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
	existing_record.metadata
	else
	metadata = scrape_metadata(instance_uri)

	if existing_record do
	existing_record
	\|> changeset(%{metadata: metadata, metadata_updated_at: now})
	\|> Repo.update()
	else
	%Instance{}
	\|> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
	\|> Repo.insert()
	end

	metadata
	end
	end

	defp get_nodeinfo_uri(well_known) do
	links = Map.get(well_known, "links", [])

	nodeinfo21 =
	Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]

	nodeinfo20 =
	Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]

	cond do
	is_binary(nodeinfo21) -> {:ok, nodeinfo21}
	is_binary(nodeinfo20) -> {:ok, nodeinfo20}
	true -> {:error, :no_links}
	end
	end

	defp scrape_metadata(%URI{} = instance_uri) do
	try do
	with {_, true} <- {:reachable, reachable?(instance_uri.host)},
	{:ok, %Tesla.Env{body: well_known_body}} <-
	instance_uri
	\|> URI.merge("/.well-known/nodeinfo")
	\|> to_string()
	\|> Pleroma.HTTP.get([{"accept", "application/json"}]),
	{:ok, well_known_json} <- Jason.decode(well_known_body),
	{:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
	{:ok, %Tesla.Env{body: nodeinfo_body}} <-
	Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
	{:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
	# Can extract more metadata from NodeInfo but need to be careful about it's size,
	# can't just dump the entire thing
	software = Map.get(nodeinfo, "software", %{})

	%{
	software_name: software["name"],
	software_version: software["version"]
	}
	\|> Maps.put_if_present(:software_repository, software["repository"])
	else
	{:reachable, false} ->
	Logger.debug(
	"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
	)

	nil

	_ ->
	nil
	end
	rescue
	e ->
	Logger.warn(
	"Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
	)

	nil
	end
	end

	@doc """
	Deletes all users from an instance in a background task, thus also deleting
	all of those users' activities and notifications.
	"""
	def delete_users_and_activities(host) when is_binary(host) do
	BackgroundWorker.enqueue("delete_instance", %{"host" => host})
	end

	def perform(:delete_instance, host) when is_binary(host) do
	User.Query.build(%{nickname: "@#{host}"})
	\|> Repo.chunk_stream(100, :batches)
	\|> Stream.each(fn users ->
	users
	\|> Enum.each(fn user ->
	User.perform(:delete, user)
	end)
	end)
	\|> Stream.run()
	end
	end
	diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs
	index a769f9362..6a718be21 100644
	--- a/test/pleroma/instances/instance_test.exs
	+++ b/test/pleroma/instances/instance_test.exs
	@@ -1,252 +1,244 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Instances.InstanceTest do
	alias Pleroma.Instances
	alias Pleroma.Instances.Instance
	alias Pleroma.Repo
	alias Pleroma.Tests.ObanHelpers
	alias Pleroma.Web.CommonAPI

	use Pleroma.DataCase

	import ExUnit.CaptureLog
	import Pleroma.Factory

	setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)

	describe "set_reachable/1" do
	test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do
	unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now())
	instance = insert(:instance, unreachable_since: unreachable_since)

	assert {:ok, instance} = Instance.set_reachable(instance.host)
	refute instance.unreachable_since
	end

	test "keeps nil `unreachable_since` of existing matching Instance record having nil `unreachable_since`" do
	instance = insert(:instance, unreachable_since: nil)

	assert {:ok, instance} = Instance.set_reachable(instance.host)
	refute instance.unreachable_since
	end
	-
	- test "does NOT create an Instance record in case of no existing matching record" do
	- host = "domain.org"
	- assert nil == Instance.set_reachable(host)
	-
	- assert [] = Repo.all(Ecto.Query.from(i in Instance))
	- assert Instance.reachable?(host)
	- end
	end

	describe "set_unreachable/1" do
	test "creates new record having `unreachable_since` to current time if record does not exist" do
	assert {:ok, instance} = Instance.set_unreachable("https://domain.com/path")

	instance = Repo.get(Instance, instance.id)
	assert instance.unreachable_since
	assert "domain.com" == instance.host
	end

	test "sets `unreachable_since` of existing record having nil `unreachable_since`" do
	instance = insert(:instance, unreachable_since: nil)
	refute instance.unreachable_since

	assert {:ok, _} = Instance.set_unreachable(instance.host)

	instance = Repo.get(Instance, instance.id)
	assert instance.unreachable_since
	end

	test "does NOT modify `unreachable_since` value of existing record in case it's present" do
	instance =
	insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))

	assert instance.unreachable_since
	initial_value = instance.unreachable_since

	assert {:ok, _} = Instance.set_unreachable(instance.host)

	instance = Repo.get(Instance, instance.id)
	assert initial_value == instance.unreachable_since
	end
	end

	describe "set_unreachable/2" do
	test "sets `unreachable_since` value of existing record in case it's newer than supplied value" do
	instance =
	insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))

	assert instance.unreachable_since

	past_value = NaiveDateTime.add(NaiveDateTime.utc_now(), -100)
	assert {:ok, _} = Instance.set_unreachable(instance.host, past_value)

	instance = Repo.get(Instance, instance.id)
	assert past_value == instance.unreachable_since
	end

	test "does NOT modify `unreachable_since` value of existing record in case it's equal to or older than supplied value" do
	instance =
	insert(:instance, unreachable_since: NaiveDateTime.add(NaiveDateTime.utc_now(), -10))

	assert instance.unreachable_since
	initial_value = instance.unreachable_since

	assert {:ok, _} = Instance.set_unreachable(instance.host, NaiveDateTime.utc_now())

	instance = Repo.get(Instance, instance.id)
	assert initial_value == instance.unreachable_since
	end
	end

	describe "get_or_update_favicon/1" do
	test "Scrapes favicon URLs" do
	Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
	%Tesla.Env{
	status: 200,
	body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
	}
	end)

	assert "https://favicon.example.org/favicon.png" ==
	Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
	end

	test "Returns nil on too long favicon URLs" do
	long_favicon_url =
	"https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"

	Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
	%Tesla.Env{
	status: 200,
	body:
	~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
	}
	end)

	assert capture_log(fn ->
	assert nil ==
	Instance.get_or_update_favicon(
	URI.parse("https://long-favicon.example.org/")
	)
	end) =~
	"Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
	end

	test "Handles not getting a favicon URL properly" do
	Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} ->
	%Tesla.Env{
	status: 200,
	body: ~s[<html><head><h1>I wil look down and whisper "GNO.."</h1></head></html>]
	}
	end)

	refute capture_log(fn ->
	assert nil ==
	Instance.get_or_update_favicon(
	URI.parse("https://no-favicon.example.org/")
	)
	end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: "
	end

	test "Doesn't scrapes unreachable instances" do
	instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold())
	url = "https://" <> instance.host

	assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~
	"Instance.scrape_favicon(\"#{url}\") ignored unreachable host"
	end
	end

	describe "get_or_update_metadata/1" do
	test "Scrapes Wildebeest NodeInfo" do
	Tesla.Mock.mock(fn
	%{url: "https://wildebeest.example.org/.well-known/nodeinfo"} ->
	%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/wildebeest-well-known-nodeinfo.json")
	}

	%{url: "https://wildebeest.example.org/nodeinfo/2.1"} ->
	%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/wildebeest-nodeinfo21.json")
	}
	end)

	expected = %{
	software_name: "wildebeest",
	software_repository: "https://github.com/cloudflare/wildebeest",
	software_version: "0.0.1"
	}

	assert expected ==
	Instance.get_or_update_metadata(URI.parse("https://wildebeest.example.org/"))

	expected = %Pleroma.Instances.Instance.Pleroma.Instances.Metadata{
	software_name: "wildebeest",
	software_repository: "https://github.com/cloudflare/wildebeest",
	software_version: "0.0.1"
	}

	assert expected ==
	Repo.get_by(Pleroma.Instances.Instance, %{host: "wildebeest.example.org"}).metadata
	end

	test "Scrapes Mastodon NodeInfo" do
	Tesla.Mock.mock(fn
	%{url: "https://mastodon.example.org/.well-known/nodeinfo"} ->
	%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/mastodon-well-known-nodeinfo.json")
	}

	%{url: "https://mastodon.example.org/nodeinfo/2.0"} ->
	%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/mastodon-nodeinfo20.json")
	}
	end)

	expected = %{
	software_name: "mastodon",
	software_version: "4.1.0"
	}

	assert expected ==
	Instance.get_or_update_metadata(URI.parse("https://mastodon.example.org/"))
	end
	end

	test "delete_users_and_activities/1 deletes remote instance users and activities" do
	[mario, luigi, _peach, wario] =
	users = [
	insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario"),
	insert(:user, nickname: "luigi@mushroom.kingdom", name: "Luigi"),
	insert(:user, nickname: "peach@mushroom.kingdom", name: "Peach"),
	insert(:user, nickname: "wario@greedville.biz", name: "Wario")
	]

	{:ok, post1} = CommonAPI.post(mario, %{status: "letsa go!"})
	{:ok, post2} = CommonAPI.post(luigi, %{status: "itsa me... luigi"})
	{:ok, post3} = CommonAPI.post(wario, %{status: "WHA-HA-HA!"})

	{:ok, job} = Instance.delete_users_and_activities("mushroom.kingdom")
	:ok = ObanHelpers.perform(job)

	[mario, luigi, peach, wario] = Repo.reload(users)

	refute mario.is_active
	refute luigi.is_active
	refute peach.is_active
	refute peach.name == "Peach"

	assert wario.is_active
	assert wario.name == "Wario"

	assert [nil, nil, %{}] = Repo.reload([post1, post2, post3])
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Sat, Oct 10, 12:39 PM (1 d, 8 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1785187
Default Alt Text: (19 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions