No OneTemporary
Actions

Size

36 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/changelog.d/prune-hashtag-follow-3376.fix b/changelog.d/prune-hashtag-follow-3376.fix
	new file mode 100644
	index 000000000..cdb4e9a79
	--- /dev/null
	+++ b/changelog.d/prune-hashtag-follow-3376.fix
	@@ -0,0 +1 @@
	+DB prune: Check if user follows hashtag with no objects before deletion
	diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
	index e52b5e0a7..396536827 100644
	--- a/lib/mix/tasks/pleroma/database.ex
	+++ b/lib/mix/tasks/pleroma/database.ex
	@@ -1,387 +1,392 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Mix.Tasks.Pleroma.Database do
	alias Pleroma.Conversation
	alias Pleroma.Maintenance
	alias Pleroma.Object
	alias Pleroma.Repo
	alias Pleroma.User

	require Logger
	require Pleroma.Constants

	import Ecto.Query
	import Mix.Pleroma

	use Mix.Task

	@shortdoc "A collection of database related tasks"
	@moduledoc File.read!("docs/administration/CLI_tasks/database.md")

	def run(["remove_embedded_objects" \| args]) do
	{options, [], []} =
	OptionParser.parse(
	args,
	strict: [
	vacuum: :boolean
	]
	)

	start_pleroma()
	Logger.info("Removing embedded objects")

	Repo.query!(
	"update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
	[],
	timeout: :infinity
	)

	if Keyword.get(options, :vacuum) do
	Maintenance.vacuum("full")
	end
	end

	def run(["bump_all_conversations"]) do
	start_pleroma()
	Conversation.bump_for_all_activities()
	end

	def run(["update_users_following_followers_counts"]) do
	start_pleroma()

	Repo.transaction(
	fn ->
	from(u in User, select: u)
	\|> Repo.stream()
	\|> Stream.each(&User.update_follower_count/1)
	\|> Stream.run()
	end,
	timeout: :infinity
	)
	end

	def run(["prune_objects" \| args]) do
	{options, [], []} =
	OptionParser.parse(
	args,
	strict: [
	vacuum: :boolean,
	keep_threads: :boolean,
	keep_non_public: :boolean,
	prune_orphaned_activities: :boolean
	]
	)

	start_pleroma()

	deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
	time_deadline = NaiveDateTime.utc_now() \|> NaiveDateTime.add(-(deadline * 86_400))

	log_message = "Pruning objects older than #{deadline} days"

	log_message =
	if Keyword.get(options, :keep_non_public) do
	log_message <> ", keeping non public posts"
	else
	log_message
	end

	log_message =
	if Keyword.get(options, :keep_threads) do
	log_message <> ", keeping threads intact"
	else
	log_message
	end

	log_message =
	if Keyword.get(options, :prune_orphaned_activities) do
	log_message <> ", pruning orphaned activities"
	else
	log_message
	end

	log_message =
	if Keyword.get(options, :vacuum) do
	log_message <>
	", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
	else
	log_message
	end

	Logger.info(log_message)

	if Keyword.get(options, :keep_threads) do
	# We want to delete objects from threads where
	# 1. the newest post is still old
	# 2. none of the activities is local
	# 3. none of the activities is bookmarked
	# 4. optionally none of the posts is non-public
	deletable_context =
	if Keyword.get(options, :keep_non_public) do
	Pleroma.Activity
	\|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
	\|> group_by([a], fragment("? ->> 'context'::text", a.data))
	\|> having(
	[a],
	not fragment(
	# Posts (checked on Create Activity) is non-public
	"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
	a.data,
	^Pleroma.Constants.as_public(),
	a.data,
	^Pleroma.Constants.as_public(),
	a.data
	)
	)
	else
	Pleroma.Activity
	\|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
	\|> group_by([a], fragment("? ->> 'context'::text", a.data))
	end
	\|> having([a], max(a.updated_at) < ^time_deadline)
	\|> having([a], not fragment("bool_or(?)", a.local))
	\|> having([_, b], fragment("max(?::text) is null", b.id))
	\|> select([a], fragment("? ->> 'context'::text", a.data))

	Pleroma.Object
	\|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
	else
	if Keyword.get(options, :keep_non_public) do
	Pleroma.Object
	\|> where(
	[o],
	fragment(
	"?->'to' \\? ? OR ?->'cc' \\? ?",
	o.data,
	^Pleroma.Constants.as_public(),
	o.data,
	^Pleroma.Constants.as_public()
	)
	)
	else
	Pleroma.Object
	end
	\|> where([o], o.updated_at < ^time_deadline)
	\|> where(
	[o],
	fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
	)
	end
	\|> Repo.delete_all(timeout: :infinity)

	if !Keyword.get(options, :keep_threads) do
	# Without the --keep-threads option, it's possible that bookmarked
	# objects have been deleted. We remove the corresponding bookmarks.
	"""
	delete from public.bookmarks
	where id in (
	select b.id from public.bookmarks b
	left join public.activities a on b.activity_id = a.id
	left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
	where o.id is null
	)
	"""
	\|> Repo.query([], timeout: :infinity)
	end

	if Keyword.get(options, :prune_orphaned_activities) do
	# Prune activities who link to a single object
	"""
	delete from public.activities
	where id in (
	select a.id from public.activities a
	left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
	left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
	left join public.users u on a.data ->> 'object' = u.ap_id
	where not a.local
	and jsonb_typeof(a."data" -> 'object') = 'string'
	and o.id is null
	and a2.id is null
	and u.id is null
	)
	"""
	\|> Repo.query([], timeout: :infinity)

	# Prune activities who link to an array of objects
	"""
	delete from public.activities
	where id in (
	select a.id from public.activities a
	join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
	left join public.objects o on j.value = o.data ->> 'id'
	left join public.activities a2 on j.value = a2.data ->> 'id'
	left join public.users u on j.value = u.ap_id
	group by a.id
	having max(o.data ->> 'id') is null
	and max(a2.data ->> 'id') is null
	and max(u.ap_id) is null
	)
	"""
	\|> Repo.query([], timeout: :infinity)
	end

	"""
	DELETE FROM hashtags AS ht
	WHERE NOT EXISTS (
	SELECT 1 FROM hashtags_objects hto
	- WHERE ht.id = hto.hashtag_id)
	+ WHERE ht.id = hto.hashtag_id
	+ )
	+ AND NOT EXISTS (
	+ SELECT 1 FROM user_follows_hashtag ufh
	+ WHERE ht.id = ufh.hashtag_id
	+ )
	"""
	\|> Repo.query()

	if Keyword.get(options, :vacuum) do
	Maintenance.vacuum("full")
	end
	end

	def run(["fix_likes_collections"]) do
	start_pleroma()

	from(object in Object,
	where: fragment("(?)->>'likes' is not null", object.data),
	select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
	)
	\|> Pleroma.Repo.chunk_stream(100, :batches)
	\|> Stream.each(fn objects ->
	ids =
	objects
	\|> Enum.filter(fn object -> object.likes \|> Jason.decode!() \|> is_map() end)
	\|> Enum.map(& &1.id)

	Object
	\|> where([object], object.id in ^ids)
	\|> update([object],
	set: [
	data:
	fragment(
	"safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)",
	object.data
	)
	]
	)
	\|> Repo.update_all([], timeout: :infinity)
	end)
	\|> Stream.run()
	end

	def run(["vacuum", args]) do
	start_pleroma()

	Maintenance.vacuum(args)
	end

	def run(["ensure_expiration"]) do
	start_pleroma()
	days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)

	Pleroma.Activity
	\|> join(:inner, [a], o in Object,
	on:
	fragment(
	"(?->>'id') = associated_object_id((?))",
	o.data,
	a.data
	)
	)
	\|> where(local: true)
	\|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
	\|> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
	\|> Pleroma.Repo.chunk_stream(100, :batches)
	\|> Stream.each(fn activities ->
	Enum.each(activities, fn activity ->
	expires_at =
	activity.inserted_at
	\|> DateTime.from_naive!("Etc/UTC")
	\|> Timex.shift(days: days)

	Pleroma.Workers.PurgeExpiredActivity.enqueue(
	%{
	activity_id: activity.id
	},
	scheduled_at: expires_at
	)
	end)
	end)
	\|> Stream.run()
	end

	def run(["set_text_search_config", tsconfig]) do
	start_pleroma()
	%{rows: [[tsc]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SHOW default_text_search_config;")
	shell_info("Current default_text_search_config: #{tsc}")

	%{rows: [[db]]} = Ecto.Adapters.SQL.query!(Pleroma.Repo, "SELECT current_database();")
	shell_info("Update default_text_search_config: #{tsconfig}")

	%{messages: msg} =
	Ecto.Adapters.SQL.query!(
	Pleroma.Repo,
	"ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';"
	)

	# non-exist config will not raise exception but only give >0 messages
	if length(msg) > 0 do
	shell_info("Error: #{inspect(msg, pretty: true)}")
	else
	rum_enabled = Pleroma.Config.get([:database, :rum_enabled])
	shell_info("Recreate index, RUM: #{rum_enabled}")

	# Note SQL below needs to be kept up-to-date with latest GIN or RUM index definition in future
	if rum_enabled do
	Ecto.Adapters.SQL.query!(
	Pleroma.Repo,
	"CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN
	new.fts_content := to_tsvector(new.data->>'content');
	RETURN new;
	END
	$$ LANGUAGE plpgsql",
	[],
	timeout: :infinity
	)

	shell_info("Refresh RUM index")
	Ecto.Adapters.SQL.query!(Pleroma.Repo, "UPDATE objects SET updated_at = NOW();")
	else
	Ecto.Adapters.SQL.query!(Pleroma.Repo, "DROP INDEX IF EXISTS objects_fts;")

	Ecto.Adapters.SQL.query!(
	Pleroma.Repo,
	"CREATE INDEX CONCURRENTLY objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content')); ",
	[],
	timeout: :infinity
	)
	end

	shell_info(~c"Done.")
	end
	end

	# Rolls back a specific migration (leaving subsequent migrations applied).
	# WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility.
	# Based on https://stackoverflow.com/a/53825840
	def run(["rollback", version]) do
	prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?"

	if shell_prompt(prompt, "n") in ~w(Yn Y y) do
	{_, result, _} =
	Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
	version = String.to_integer(version)
	re = ~r/^#{version}_.*\.exs/
	path = Ecto.Migrator.migrations_path(repo)

	with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))},
	{_, [{mod, _} \| _]} <- {:compile, Code.compile_file(Path.join(path, file))},
	{_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do
	{:ok, "Reversed migration: #{file}"}
	else
	{:find, _} -> {:error, "No migration found with version prefix: #{version}"}
	{:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"}
	{:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"}
	end
	end)

	shell_info(inspect(result))
	end
	end
	end
	diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs
	index 19df17b60..5b567325c 100644
	--- a/test/mix/tasks/pleroma/database_test.exs
	+++ b/test/mix/tasks/pleroma/database_test.exs
	@@ -1,660 +1,694 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Mix.Tasks.Pleroma.DatabaseTest do
	use Pleroma.DataCase, async: false
	use Oban.Testing, repo: Pleroma.Repo

	alias Pleroma.Activity
	alias Pleroma.Bookmark
	+ alias Pleroma.Hashtag
	alias Pleroma.Object
	alias Pleroma.Repo
	alias Pleroma.User
	alias Pleroma.Web.CommonAPI

	import Pleroma.Factory

	setup_all do
	Mix.shell(Mix.Shell.Process)

	on_exit(fn ->
	Mix.shell(Mix.Shell.IO)
	end)

	:ok
	end

	describe "running remove_embedded_objects" do
	test "it replaces objects with references" do
	user = insert(:user)
	{:ok, activity} = CommonAPI.post(user, %{status: "test"})
	new_data = Map.put(activity.data, "object", activity.object.data)

	{:ok, activity} =
	activity
	\|> Activity.change(%{data: new_data})
	\|> Repo.update()

	assert is_map(activity.data["object"])

	Mix.Tasks.Pleroma.Database.run(["remove_embedded_objects"])

	activity = Activity.get_by_id_with_object(activity.id)
	assert is_binary(activity.data["object"])
	end
	end

	describe "prune_objects" do
	setup do
	deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1

	old_insert_date =
	Timex.now()
	\|> Timex.shift(days: -deadline)
	\|> Timex.to_naive_datetime()
	\|> NaiveDateTime.truncate(:second)

	%{old_insert_date: old_insert_date}
	end

	test "it prunes old objects from the database", %{old_insert_date: old_insert_date} do
	insert(:note)

	%{id: note_remote_public_id} =
	:note
	\|> insert()
	\|> Ecto.Changeset.change(%{updated_at: old_insert_date})
	\|> Repo.update!()

	note_remote_non_public =
	%{id: note_remote_non_public_id, data: note_remote_non_public_data} =
	:note
	\|> insert()

	note_remote_non_public
	\|> Ecto.Changeset.change(%{
	updated_at: old_insert_date,
	data: note_remote_non_public_data \|> update_in(["to"], fn _ -> [] end)
	})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 3

	Mix.Tasks.Pleroma.Database.run(["prune_objects"])

	assert length(Repo.all(Object)) == 1
	refute Object.get_by_id(note_remote_public_id)
	refute Object.get_by_id(note_remote_non_public_id)
	end

	test "it cleans up bookmarks", %{old_insert_date: old_insert_date} do
	user = insert(:user)
	{:ok, old_object_activity} = CommonAPI.post(user, %{status: "yadayada"})

	Repo.one(Object)
	\|> Ecto.Changeset.change(%{updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, new_object_activity} = CommonAPI.post(user, %{status: "yadayada"})

	{:ok, _} = Bookmark.create(user.id, old_object_activity.id)
	{:ok, _} = Bookmark.create(user.id, new_object_activity.id)

	assert length(Repo.all(Object)) == 2
	assert length(Repo.all(Bookmark)) == 2

	Mix.Tasks.Pleroma.Database.run(["prune_objects"])

	assert length(Repo.all(Object)) == 1
	assert length(Repo.all(Bookmark)) == 1
	refute Bookmark.get(user.id, old_object_activity.id)
	end

	test "with the --keep-non-public option it still keeps non-public posts even if they are not local",
	%{old_insert_date: old_insert_date} do
	insert(:note)

	%{id: note_remote_id} =
	:note
	\|> insert()
	\|> Ecto.Changeset.change(%{updated_at: old_insert_date})
	\|> Repo.update!()

	note_remote_non_public =
	%{data: note_remote_non_public_data} =
	:note
	\|> insert()

	note_remote_non_public
	\|> Ecto.Changeset.change(%{
	updated_at: old_insert_date,
	data: note_remote_non_public_data \|> update_in(["to"], fn _ -> [] end)
	})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 3

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-non-public"])

	assert length(Repo.all(Object)) == 2
	refute Object.get_by_id(note_remote_id)
	end

	test "with the --keep-threads and --keep-non-public option it keeps old threads with non-public replies even if the interaction is not local",
	%{old_insert_date: old_insert_date} do
	# For non-public we only check Create Activities because only these are relevant for threads
	# Flags are always non-public, Announces from relays can be non-public...

	remote_user1 = insert(:user, local: false)
	remote_user2 = insert(:user, local: false)

	# Old remote non-public reply (should be kept)
	{:ok, old_remote_post1_activity} =
	CommonAPI.post(remote_user1, %{status: "some thing", local: false})

	old_remote_post1_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_remote_non_public_reply_activity} =
	CommonAPI.post(remote_user2, %{
	status: "some reply",
	in_reply_to_status_id: old_remote_post1_activity.id
	})

	old_remote_non_public_reply_activity
	\|> Ecto.Changeset.change(%{
	local: false,
	updated_at: old_insert_date,
	data: old_remote_non_public_reply_activity.data \|> update_in(["to"], fn _ -> [] end)
	})
	\|> Repo.update!()

	# Old remote non-public Announce (should be removed)
	{:ok, old_remote_post2_activity = %{data: %{"object" => old_remote_post2_id}}} =
	CommonAPI.post(remote_user1, %{status: "some thing", local: false})

	old_remote_post2_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_remote_non_public_repeat_activity} =
	CommonAPI.repeat(old_remote_post2_activity.id, remote_user2)

	old_remote_non_public_repeat_activity
	\|> Ecto.Changeset.change(%{
	local: false,
	updated_at: old_insert_date,
	data: old_remote_non_public_repeat_activity.data \|> update_in(["to"], fn _ -> [] end)
	})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 3

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads", "--keep-non-public"])

	Repo.all(Pleroma.Activity)
	assert length(Repo.all(Object)) == 2
	refute Object.get_by_ap_id(old_remote_post2_id)
	end

	test "with the --keep-threads option it still keeps non-old threads even with no local interactions" do
	remote_user = insert(:user, local: false)
	remote_user2 = insert(:user, local: false)

	{:ok, remote_post_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	{:ok, remote_post_reply_activity} =
	CommonAPI.post(remote_user2, %{
	status: "some reply",
	in_reply_to_status_id: remote_post_activity.id
	})

	remote_post_activity
	\|> Ecto.Changeset.change(%{local: false})
	\|> Repo.update!()

	remote_post_reply_activity
	\|> Ecto.Changeset.change(%{local: false})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 2

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])

	assert length(Repo.all(Object)) == 2
	end

	test "with the --keep-threads option it deletes old threads with no local interaction", %{
	old_insert_date: old_insert_date
	} do
	remote_user = insert(:user, local: false)
	remote_user2 = insert(:user, local: false)

	{:ok, old_remote_post_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	old_remote_post_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_remote_post_reply_activity} =
	CommonAPI.post(remote_user2, %{
	status: "some reply",
	in_reply_to_status_id: old_remote_post_activity.id
	})

	old_remote_post_reply_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_favourite_activity} =
	CommonAPI.favorite(old_remote_post_activity.id, remote_user2)

	old_favourite_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post_activity.id, remote_user2)

	old_repeat_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 2

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])

	assert length(Repo.all(Object)) == 0
	end

	test "with the --keep-threads option it keeps old threads with local interaction", %{
	old_insert_date: old_insert_date
	} do
	remote_user = insert(:user, local: false)
	local_user = insert(:user, local: true)

	# local reply
	{:ok, old_remote_post1_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	old_remote_post1_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_local_post2_reply_activity} =
	CommonAPI.post(local_user, %{
	status: "some reply",
	in_reply_to_status_id: old_remote_post1_activity.id
	})

	old_local_post2_reply_activity
	\|> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
	\|> Repo.update!()

	# local Like
	{:ok, old_remote_post3_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	old_remote_post3_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_favourite_activity} = CommonAPI.favorite(old_remote_post3_activity.id, local_user)

	old_favourite_activity
	\|> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
	\|> Repo.update!()

	# local Announce
	{:ok, old_remote_post4_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	old_remote_post4_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	{:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post4_activity.id, local_user)

	old_repeat_activity
	\|> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
	\|> Repo.update!()

	assert length(Repo.all(Object)) == 4

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])

	assert length(Repo.all(Object)) == 4
	end

	test "with the --keep-threads option it keeps old threads with bookmarked posts", %{
	old_insert_date: old_insert_date
	} do
	remote_user = insert(:user, local: false)
	local_user = insert(:user, local: true)

	{:ok, old_remote_post_activity} =
	CommonAPI.post(remote_user, %{status: "some thing", local: false})

	old_remote_post_activity
	\|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
	\|> Repo.update!()

	Pleroma.Bookmark.create(local_user.id, old_remote_post_activity.id)

	assert length(Repo.all(Object)) == 1

	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])

	assert length(Repo.all(Object)) == 1
	end

	test "We don't have unexpected tables which may contain objects that are referenced by activities" do
	# We can delete orphaned activities. For that we look for the objects
	# they reference in the 'objects', 'activities', and 'users' table.
	# If someone adds another table with objects (idk, maybe with separate
	# relations, or collections or w/e), then we need to make sure we
	# add logic for that in the 'prune_objects' task so that we don't
	# wrongly delete their corresponding activities.
	# So when someone adds (or removes) a table, this test will fail.
	# Either the table contains objects which can be referenced from the
	# activities table
	# => in that case the prune_objects job should be adapted so we don't
	# delete activities who still have the referenced object.
	# Or it doesn't contain objects which can be referenced from the activities table
	# => in that case you can add/remove the table to/from this (sorted) list.

	assert Repo.query!(
	"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';"
	).rows
	\|> Enum.sort() == [
	["activities"],
	["announcement_read_relationships"],
	["announcements"],
	["apps"],
	["backups"],
	["bookmark_folders"],
	["bookmarks"],
	["chat_message_references"],
	["chats"],
	["config"],
	["conversation_participation_recipient_ships"],
	["conversation_participations"],
	["conversations"],
	["counter_cache"],
	["data_migration_failed_ids"],
	["data_migrations"],
	["deliveries"],
	["filters"],
	["following_relationships"],
	["hashtags"],
	["hashtags_objects"],
	["instances"],
	["lists"],
	["markers"],
	["mfa_tokens"],
	["moderation_log"],
	["notifications"],
	["oauth_authorizations"],
	["oauth_tokens"],
	["oban_jobs"],
	["oban_peers"],
	["objects"],
	["password_reset_tokens"],
	["push_subscriptions"],
	["registrations"],
	["report_notes"],
	["rich_media_card"],
	["rules"],
	["scheduled_activities"],
	["schema_migrations"],
	["thread_mutes"],
	["user_follows_hashtag"],
	# ["user_frontend_setting_profiles"], # not in pleroma
	["user_invite_tokens"],
	["user_notes"],
	["user_relationships"],
	["users"]
	]
	end

	test "it prunes orphaned activities with the --prune-orphaned-activities" do
	# Add a remote activity which references an Object
	%Object{} \|> Map.merge(%{data: %{"id" => "object_for_activity"}}) \|> Repo.insert()

	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{"id" => "remote_activity_with_object", "object" => "object_for_activity"}
	})
	\|> Repo.insert()

	# Add a remote activity which references an activity
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_with_activity",
	"object" => "remote_activity_with_object"
	}
	})
	\|> Repo.insert()

	# Add a remote activity which references an Actor
	%User{} \|> Map.merge(%{ap_id: "actor"}) \|> Repo.insert()

	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{"id" => "remote_activity_with_actor", "object" => "actor"}
	})
	\|> Repo.insert()

	# Add a remote activity without existing referenced object, activity or actor
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_without_existing_referenced_object",
	"object" => "non_existing"
	}
	})
	\|> Repo.insert()

	# Add a local activity without existing referenced object, activity or actor
	%Activity{}
	\|> Map.merge(%{
	local: true,
	data: %{"id" => "local_activity_with_actor", "object" => "non_existing"}
	})
	\|> Repo.insert()

	# The remote activities without existing reference,
	# and only the remote activities without existing reference, are deleted
	# if, and only if, we provide the --prune-orphaned-activities option
	assert length(Repo.all(Activity)) == 5
	Mix.Tasks.Pleroma.Database.run(["prune_objects"])
	assert length(Repo.all(Activity)) == 5
	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
	activities = Repo.all(Activity)

	assert "remote_activity_without_existing_referenced_object" not in Enum.map(
	activities,
	fn a -> a.data["id"] end
	)

	assert length(activities) == 4
	end

	test "it prunes orphaned activities with the --prune-orphaned-activities when the objects are referenced from an array" do
	%Object{} \|> Map.merge(%{data: %{"id" => "existing_object"}}) \|> Repo.insert()
	%User{} \|> Map.merge(%{ap_id: "existing_actor"}) \|> Repo.insert()

	# Multiple objects, one object exists (keep)
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_existing_object",
	"object" => ["non_ existing_object", "existing_object"]
	}
	})
	\|> Repo.insert()

	# Multiple objects, one actor exists (keep)
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_existing_actor",
	"object" => ["non_ existing_object", "existing_actor"]
	}
	})
	\|> Repo.insert()

	# Multiple objects, one activity exists (keep)
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_existing_activity",
	"object" => ["non_ existing_object", "remote_activity_existing_actor"]
	}
	})
	\|> Repo.insert()

	# Multiple objects none exist (prune)
	%Activity{}
	\|> Map.merge(%{
	local: false,
	data: %{
	"id" => "remote_activity_without_existing_referenced_object",
	"object" => ["owo", "whats_this"]
	}
	})
	\|> Repo.insert()

	assert length(Repo.all(Activity)) == 4
	Mix.Tasks.Pleroma.Database.run(["prune_objects"])
	assert length(Repo.all(Activity)) == 4
	Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
	activities = Repo.all(Activity)
	assert length(activities) == 3

	assert "remote_activity_without_existing_referenced_object" not in Enum.map(
	activities,
	fn a -> a.data["id"] end
	)

	assert length(activities) == 3
	end
	+
	+ test "it prunes hashtags with no objects associated", %{old_insert_date: old_insert_date} do
	+ user = insert(:user)
	+
	+ {:ok, hashtag_post_activity} =
	+ CommonAPI.post(user, %{status: "morning #cofe", local: true})
	+
	+ hashtag_post_object = Object.normalize(hashtag_post_activity)
	+
	+ {:ok, hashtag_post2_activity} =
	+ CommonAPI.post(user, %{status: "morning #cawfee", local: true})
	+
	+ hashtag_post2_object = Object.normalize(hashtag_post2_activity)
	+
	+ hashtag_post_object
	+ \|> Ecto.Changeset.change(%{updated_at: old_insert_date})
	+ \|> Repo.update!()
	+
	+ hashtag_post2_object
	+ \|> Ecto.Changeset.change(%{updated_at: old_insert_date})
	+ \|> Repo.update!()
	+
	+ # Test whether hashtags with follow relationships are kept
	+ User.follow_hashtag(user, Hashtag.get_by_name("cofe"))
	+
	+ assert length(Repo.all(Hashtag)) == 2
	+ assert length(Repo.all(Object)) == 2
	+
	+ Mix.Tasks.Pleroma.Database.run(["prune_objects"])
	+ assert length(Repo.all(Hashtag)) == 1
	+ assert length(Repo.all(Object)) == 0
	+ assert Repo.one(Hashtag) \|> Map.fetch!(:name) == "cofe"
	+ end
	end

	describe "running update_users_following_followers_counts" do
	test "following and followers count are updated" do
	[user, user2] = insert_pair(:user)
	{:ok, %User{} = user, _user2} = User.follow(user, user2)

	following = User.following(user)

	assert length(following) == 2
	assert user.follower_count == 0

	{:ok, user} =
	user
	\|> Ecto.Changeset.change(%{follower_count: 3})
	\|> Repo.update()

	assert user.follower_count == 3

	assert {:ok, :ok} ==
	Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"])

	user = User.get_by_id(user.id)

	assert length(User.following(user)) == 2
	assert user.follower_count == 0
	end
	end

	describe "running fix_likes_collections" do
	test "it turns OrderedCollection likes into empty arrays" do
	[user, user2] = insert_pair(:user)

	{:ok, %{id: id, object: object}} = CommonAPI.post(user, %{status: "test"})
	{:ok, %{object: object2}} = CommonAPI.post(user, %{status: "test test"})

	CommonAPI.favorite(id, user2)

	likes = %{
	"first" =>
	"http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
	"id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
	"totalItems" => 3,
	"type" => "OrderedCollection"
	}

	new_data = Map.put(object2.data, "likes", likes)

	object2
	\|> Ecto.Changeset.change(%{data: new_data})
	\|> Repo.update()

	assert length(Object.get_by_id(object.id).data["likes"]) == 1
	assert is_map(Object.get_by_id(object2.id).data["likes"])

	assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])

	assert length(Object.get_by_id(object.id).data["likes"]) == 1
	assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
	end
	end

	describe "ensure_expiration" do
	test "it adds to expiration old statuses" do
	activity1 = insert(:note_activity)

	{:ok, inserted_at, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z")
	activity2 = insert(:note_activity, %{inserted_at: inserted_at})

	%{id: activity_id3} = insert(:note_activity)

	expires_at = DateTime.add(DateTime.utc_now(), 60 * 61)

	Pleroma.Workers.PurgeExpiredActivity.enqueue(
	%{
	activity_id: activity_id3
	},
	scheduled_at: expires_at
	)

	Mix.Tasks.Pleroma.Database.run(["ensure_expiration"])

	assert_enqueued(
	worker: Pleroma.Workers.PurgeExpiredActivity,
	args: %{activity_id: activity1.id},
	scheduled_at:
	activity1.inserted_at
	\|> DateTime.from_naive!("Etc/UTC")
	\|> Timex.shift(days: 365)
	)

	assert_enqueued(
	worker: Pleroma.Workers.PurgeExpiredActivity,
	args: %{activity_id: activity2.id},
	scheduled_at:
	activity2.inserted_at
	\|> DateTime.from_naive!("Etc/UTC")
	\|> Timex.shift(days: 365)
	)

	assert_enqueued(
	worker: Pleroma.Workers.PurgeExpiredActivity,
	args: %{activity_id: activity_id3},
	scheduled_at: expires_at
	)
	end
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Sun, Aug 30, 8:35 PM (1 d, 12 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1738152
Default Alt Text: (36 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions