No OneTemporary
Actions

Size

32 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
	index 7ddb03529..adebebc7a 100644
	--- a/lib/pleroma/application.ex
	+++ b/lib/pleroma/application.ex
	@@ -1,244 +1,244 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Application do
	use Application

	import Cachex.Spec

	alias Pleroma.Config

	require Logger

	@name Mix.Project.config()[:name]
	@version Mix.Project.config()[:version]
	@repository Mix.Project.config()[:source_url]
	@env Mix.env()

	def name, do: @name
	def version, do: @version
	def named_version, do: @name <> " " <> @version
	def repository, do: @repository

	def user_agent do
	case Config.get([:http, :user_agent], :default) do
	:default ->
	info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
	named_version() <> "; " <> info

	custom ->
	custom
	end
	end

	# See http://elixir-lang.org/docs/stable/elixir/Application.html
	# for more information on OTP Applications
	def start(_type, _args) do
	Pleroma.Config.Holder.save_default()
	Pleroma.HTML.compile_scrubbers()
	Config.DeprecationWarnings.warn()
	Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
	Pleroma.Repo.check_migrations_applied!()
	setup_instrumenters()
	load_custom_modules()

	adapter = Application.get_env(:tesla, :adapter)

	if adapter == Tesla.Adapter.Gun do
	if version = Pleroma.OTPVersion.version() do
	[major, minor] =
	version
	\|> String.split(".")
	\|> Enum.map(&String.to_integer/1)
	\|> Enum.take(2)

	if (major == 22 and minor < 2) or major < 22 do
	raise "
	!!!OTP VERSION WARNING!!!
	You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
	"
	end
	else
	raise "
	!!!OTP VERSION WARNING!!!
	To support correct handling of unordered certificates chains - OTP version must be > 22.2.
	"
	end
	end

	# Define workers and child supervisors to be supervised
	children =
	[
	Pleroma.Repo,
	Config.TransferTask,
	Pleroma.Emoji,
	Pleroma.Plugs.RateLimiter.Supervisor
	] ++
	cachex_children() ++
	http_children(adapter, @env) ++
	[
	Pleroma.Stats,
	Pleroma.JobQueueMonitor,
	{Oban, Config.get(Oban)}
	] ++
	task_children(@env) ++
	streamer_child(@env) ++
	chat_child(@env, chat_enabled?()) ++
	[
	Pleroma.Web.Endpoint,
	Pleroma.Gopher.Server
	]

	# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
	# for other strategies and supported options
	opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
	Supervisor.start_link(children, opts)
	end

	def load_custom_modules do
	dir = Config.get([:modules, :runtime_dir])

	if dir && File.exists?(dir) do
	dir
	\|> Pleroma.Utils.compile_dir()
	\|> case do
	{:error, _errors, _warnings} ->
	raise "Invalid custom modules"

	{:ok, modules, _warnings} ->
	if @env != :test do
	Enum.each(modules, fn mod ->
	Logger.info("Custom module loaded: #{inspect(mod)}")
	end)
	end

	:ok
	end
	end
	end

	defp setup_instrumenters do
	require Prometheus.Registry

	if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
	:ok =
	:telemetry.attach(
	"prometheus-ecto",
	[:pleroma, :repo, :query],
	&Pleroma.Repo.Instrumenter.handle_event/4,
	%{}
	)

	Pleroma.Repo.Instrumenter.setup()
	end

	Pleroma.Web.Endpoint.MetricsExporter.setup()
	Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
	Pleroma.Web.Endpoint.Instrumenter.setup()
	end

	defp cachex_children do
	[
	build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
	build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
	build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
	build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
	build_cachex("scrubber", limit: 2500),
	build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
	build_cachex("web_resp", limit: 2500),
	build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
	build_cachex("failed_proxy_url", limit: 2500),
	- build_cachex("deleted_urls", default_ttl: :timer.minutes(60), limit: 2500)
	+ build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
	]
	end

	defp emoji_packs_expiration,
	do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))

	defp idempotency_expiration,
	do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))

	defp seconds_valid_interval,
	do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))

	defp build_cachex(type, opts),
	do: %{
	id: String.to_atom("cachex_" <> type),
	start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]},
	type: :worker
	}

	defp chat_enabled?, do: Config.get([:chat, :enabled])

	defp streamer_child(env) when env in [:test, :benchmark], do: []

	defp streamer_child(_) do
	[
	{Registry,
	[
	name: Pleroma.Web.Streamer.registry(),
	keys: :duplicate,
	partitions: System.schedulers_online()
	]}
	]
	end

	defp chat_child(_env, true) do
	[Pleroma.Web.ChatChannel.ChatChannelState]
	end

	defp chat_child(_, _), do: []

	defp task_children(:test) do
	[
	%{
	id: :web_push_init,
	start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
	restart: :temporary
	}
	]
	end

	defp task_children(_) do
	[
	%{
	id: :web_push_init,
	start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
	restart: :temporary
	},
	%{
	id: :internal_fetch_init,
	start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
	restart: :temporary
	}
	]
	end

	# start hackney and gun pools in tests
	defp http_children(_, :test) do
	hackney_options = Config.get([:hackney_pools, :federation])
	hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
	[hackney_pool, Pleroma.Pool.Supervisor]
	end

	defp http_children(Tesla.Adapter.Hackney, _) do
	pools = [:federation, :media]

	pools =
	if Config.get([Pleroma.Upload, :proxy_remote]) do
	[:upload \| pools]
	else
	pools
	end

	for pool <- pools do
	options = Config.get([:hackney_pools, pool])
	:hackney_pool.child_spec(pool, options)
	end
	end

	defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]

	defp http_children(_, _), do: []
	end
	diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
	index 324f8a7ee..162d20262 100644
	--- a/lib/pleroma/web/media_proxy/invalidation.ex
	+++ b/lib/pleroma/web/media_proxy/invalidation.ex
	@@ -1,36 +1,36 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.MediaProxy.Invalidation do
	@moduledoc false

	@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} \| {:error, String.t()}

	alias Pleroma.Config
	alias Pleroma.Web.MediaProxy

	+ def enabled, do: Config.get([:media_proxy, :invalidation, :enabled])
	+
	@spec purge(list(String.t()) \| String.t()) :: {:ok, list(String.t())} \| {:error, String.t()}
	def purge(urls) do
	prepared_urls = prepare_urls(urls)

	- if Config.get([:media_proxy, :invalidation, :enabled]) do
	- result = do_purge(prepared_urls)
	- MediaProxy.remove_from_deleted_urls(prepared_urls)
	- result
	+ if enabled() do
	+ do_purge(prepared_urls)
	else
	{:ok, prepared_urls}
	end
	end

	defp do_purge(urls) do
	provider = Config.get([:media_proxy, :invalidation, :provider])
	provider.purge(urls, Config.get(provider))
	end

	defp prepare_urls(urls) do
	urls
	\|> List.wrap()
	\|> Enum.map(&MediaProxy.url(&1))
	end
	end
	diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
	index 4a0fec288..b3988b946 100644
	--- a/lib/pleroma/web/media_proxy/media_proxy.ex
	+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
	@@ -1,104 +1,108 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.MediaProxy do
	alias Pleroma.Config
	alias Pleroma.Upload
	alias Pleroma.Web

	@base64_opts [padding: false]

	@spec in_deleted_urls(String.t()) :: boolean()
	def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url), 1)

	def remove_from_deleted_urls(urls) when is_list(urls) do
	- Enum.each(urls, &remove_from_deleted_urls/1)
	+ Cachex.execute!(:deleted_urls_cache, fn cache ->
	+ Enum.each(urls, &Cachex.del(cache, &1))
	+ end)
	end

	def remove_from_deleted_urls(url) when is_binary(url) do
	Cachex.del(:deleted_urls_cache, url)
	end

	def put_in_deleted_urls(urls) when is_list(urls) do
	- Enum.each(urls, &put_in_deleted_urls/1)
	+ Cachex.execute!(:deleted_urls_cache, fn cache ->
	+ Enum.each(urls, &Cachex.put(cache, &1, true))
	+ end)
	end

	def put_in_deleted_urls(url) when is_binary(url) do
	Cachex.put(:deleted_urls_cache, url, true)
	end

	def url(url) when is_nil(url) or url == "", do: nil
	def url("/" <> _ = url), do: url

	def url(url) do
	if disabled?() or local?(url) or whitelisted?(url) do
	url
	else
	encode_url(url)
	end
	end

	defp disabled?, do: !Config.get([:media_proxy, :enabled], false)

	defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())

	defp whitelisted?(url) do
	%{host: domain} = URI.parse(url)

	mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])

	upload_base_url_domain =
	if !is_nil(Config.get([Upload, :base_url])) do
	[URI.parse(Config.get([Upload, :base_url])).host]
	else
	[]
	end

	whitelist = mediaproxy_whitelist ++ upload_base_url_domain

	Enum.any?(whitelist, fn pattern ->
	String.equivalent?(domain, pattern)
	end)
	end

	def encode_url(url) do
	base64 = Base.url_encode64(url, @base64_opts)

	sig64 =
	base64
	\|> signed_url
	\|> Base.url_encode64(@base64_opts)

	build_url(sig64, base64, filename(url))
	end

	def decode_url(sig, url) do
	with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
	signature when signature == sig <- signed_url(url) do
	{:ok, Base.url_decode64!(url, @base64_opts)}
	else
	_ -> {:error, :invalid_signature}
	end
	end

	defp signed_url(url) do
	:crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url)
	end

	def filename(url_or_path) do
	if path = URI.parse(url_or_path).path, do: Path.basename(path)
	end

	def build_url(sig_base64, url_base64, filename \\ nil) do
	[
	Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()),
	"proxy",
	sig_base64,
	url_base64,
	filename
	]
	\|> Enum.filter(& &1)
	\|> Path.join()
	end
	end
	diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
	index 5fb0b9584..816d681dd 100644
	--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
	+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
	@@ -1,115 +1,124 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Workers.AttachmentsCleanupWorker do
	import Ecto.Query

	alias Pleroma.Object
	alias Pleroma.Repo
	+ alias Pleroma.Web.MediaProxy

	use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"

	@impl Oban.Worker
	def perform(
	%{
	"op" => "cleanup_attachments",
	"object" => %{"data" => %{"attachment" => [_ \| _] = attachments, "actor" => actor}}
	},
	_job
	) do
	hrefs =
	Enum.flat_map(attachments, fn attachment ->
	Enum.map(attachment["url"], & &1["href"])
	end)

	uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])

	prefix =
	case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
	nil -> "media"
	_ -> ""
	end

	base_url =
	String.trim_trailing(
	Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
	"/"
	)

	# find all objects for copies of the attachments, name and actor doesn't matter here
	{object_ids, attachment_urls} =
	hrefs
	\|> fetch_objects
	\|> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
	\|> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
	with 1 <- count do
	{ids ++ [id], hrefs ++ [href]}
	else
	_ -> {ids ++ [id], hrefs}
	end
	end)

	- Pleroma.Web.MediaProxy.put_in_deleted_urls(attachment_urls)
	+ lock_attachments(MediaProxy.Invalidation.enabled(), attachment_urls)

	Enum.each(attachment_urls, fn href ->
	href
	\|> String.trim_leading("#{base_url}/#{prefix}")
	\|> uploader.delete_file()
	end)

	Repo.delete_all(from(o in Object, where: o.id in ^object_ids))

	- cache_purge(attachment_urls)
	+ cache_purge(MediaProxy.Invalidation.enabled(), attachment_urls)

	{:ok, :success}
	end

	def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}

	- defp cache_purge(attachment_urls) do
	- Pleroma.Web.MediaProxy.Invalidation.purge(attachment_urls)
	+ defp cache_purge(true, attachment_urls) do
	+ MediaProxy.Invalidation.purge(attachment_urls)
	end

	+ defp cache_purge(_, _), do: :ok
	+
	+ defp lock_attachments(true, attachment_urls) do
	+ MediaProxy.put_in_deleted_urls(attachment_urls)
	+ end
	+
	+ defp lock_attachments(_, _), do: :ok
	+
	# we should delete 1 object for any given attachment, but don't delete
	# files if there are more than 1 object for it
	- def prepare_objects(objects, actor, names) do
	+ defp prepare_objects(objects, actor, names) do
	objects
	\|> Enum.reduce(%{}, fn %{
	id: id,
	data: %{
	"url" => [%{"href" => href}],
	"actor" => obj_actor,
	"name" => name
	}
	},
	acc ->
	Map.update(acc, href, %{id: id, count: 1}, fn val ->
	case obj_actor == actor and name in names do
	true ->
	# set id of the actor's object that will be deleted
	%{val \| id: id, count: val.count + 1}

	false ->
	# another actor's object, just increase count to not delete file
	%{val \| count: val.count + 1}
	end
	end)
	end)
	end

	defp fetch_objects(hrefs) do
	from(o in Object,
	where:
	fragment(
	"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?\| (?)",
	o.data,
	o.data,
	^hrefs
	)
	)
	# The query above can be time consumptive on large instances until we
	# refactor how uploads are stored
	\|> Repo.all(timeout: :infinity)
	end
	end
	diff --git a/test/object_test.exs b/test/object_test.exs
	index 198d3b1cf..085469495 100644
	--- a/test/object_test.exs
	+++ b/test/object_test.exs
	@@ -1,402 +1,404 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.ObjectTest do
	use Pleroma.DataCase
	use Oban.Testing, repo: Pleroma.Repo
	import ExUnit.CaptureLog
	import Pleroma.Factory
	import Tesla.Mock
	alias Pleroma.Activity
	alias Pleroma.Object
	alias Pleroma.Repo
	alias Pleroma.Tests.ObanHelpers
	alias Pleroma.Web.CommonAPI

	setup do
	mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
	:ok
	end

	test "returns an object by it's AP id" do
	object = insert(:note)
	found_object = Object.get_by_ap_id(object.data["id"])

	assert object == found_object
	end

	describe "generic changeset" do
	test "it ensures uniqueness of the id" do
	object = insert(:note)
	cs = Object.change(%Object{}, %{data: %{id: object.data["id"]}})
	assert cs.valid?

	{:error, _result} = Repo.insert(cs)
	end
	end

	describe "deletion function" do
	test "deletes an object" do
	object = insert(:note)
	found_object = Object.get_by_ap_id(object.data["id"])

	assert object == found_object

	Object.delete(found_object)

	found_object = Object.get_by_ap_id(object.data["id"])

	refute object == found_object

	assert found_object.data["type"] == "Tombstone"
	end

	test "ensures cache is cleared for the object" do
	object = insert(:note)
	cached_object = Object.get_cached_by_ap_id(object.data["id"])

	assert object == cached_object

	Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe")

	Object.delete(cached_object)

	{:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}")
	{:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path)

	cached_object = Object.get_cached_by_ap_id(object.data["id"])

	refute object == cached_object

	assert cached_object.data["type"] == "Tombstone"
	end
	end

	describe "delete attachments" do
	setup do: clear_config([Pleroma.Upload])
	setup do: clear_config([:instance, :cleanup_attachments])
	+ setup do: clear_config([:media_proxy])

	test "Disabled via config" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([:instance, :cleanup_attachments], false)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	refute Object.get_by_id(attachment.id) == nil

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
	+ refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end

	test "in subdirectories" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([:instance, :cleanup_attachments], true)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	end

	test "with dedupe enabled" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
	Pleroma.Config.put([:instance, :cleanup_attachments], true)

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	File.mkdir_p!(uploads_dir)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	filename = Path.basename(href)

	assert {:ok, files} = File.ls(uploads_dir)
	assert filename in files

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil
	assert {:ok, files} = File.ls(uploads_dir)
	refute filename in files
	end

	test "with objects that have legacy data.url attribute" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([:instance, :cleanup_attachments], true)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	{:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	end

	test "With custom base_url" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
	Pleroma.Config.put([:instance, :cleanup_attachments], true)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	end
	end

	describe "normalizer" do
	test "fetches unknown objects by default" do
	%Object{} =
	object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367")

	assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
	end

	test "fetches unknown objects when fetch_remote is explicitly true" do
	%Object{} =
	object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true)

	assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
	end

	test "does not fetch unknown objects when fetch_remote is false" do
	assert is_nil(
	Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)
	)
	end
	end

	describe "get_by_id_and_maybe_refetch" do
	setup do
	mock(fn
	%{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
	%Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")}

	env ->
	apply(HttpRequestMock, :request, [env])
	end)

	mock_modified = fn resp ->
	mock(fn
	%{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
	resp

	env ->
	apply(HttpRequestMock, :request, [env])
	end)
	end

	on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end)

	[mock_modified: mock_modified]
	end

	test "refetches if the time since the last refetch is greater than the interval", %{
	mock_modified: mock_modified
	} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
	end

	test "returns the old object if refetch fails", %{mock_modified: mock_modified} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	assert capture_log(fn ->
	mock_modified.(%Tesla.Env{status: 404, body: ""})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
	end) =~
	"[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"
	end

	test "does not refetch if the time since the last refetch is greater than the interval", %{
	mock_modified: mock_modified
	} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
	end

	test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	user = insert(:user)
	activity = Activity.get_create_by_object_ap_id(object.data["id"])
	{:ok, activity} = CommonAPI.favorite(user, activity.id)
	object = Object.get_by_ap_id(activity.data["object"])

	assert object.data["like_count"] == 1

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3

	assert updated_object.data["like_count"] == 1
	end
	end
	end
	diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs
	index edddc385b..3a9fa8c88 100644
	--- a/test/web/media_proxy/invalidation_test.exs
	+++ b/test/web/media_proxy/invalidation_test.exs
	@@ -1,65 +1,65 @@
	defmodule Pleroma.Web.MediaProxy.InvalidationTest do
	use ExUnit.Case
	use Pleroma.Tests.Helpers

	alias Pleroma.Config
	alias Pleroma.Web.MediaProxy.Invalidation

	import ExUnit.CaptureLog
	import Mock
	import Tesla.Mock

	setup do: clear_config([:media_proxy])

	setup do
	on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
	:ok
	end

	describe "Invalidation.Http" do
	test "perform request to clear cache" do
	Config.put([:media_proxy, :enabled], false)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)

	Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
	image_url = "http://example.com/media/example.jpg"
	Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)

	mock(fn
	%{
	method: :purge,
	url: "http://example.com/media/example.jpg",
	headers: [{"x-refresh", 1}]
	} ->
	%Tesla.Env{status: 200}
	end)

	assert capture_log(fn ->
	assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	assert Invalidation.purge([image_url]) == {:ok, [image_url]}
	- refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	end) =~ "Running cache purge: [\"#{image_url}\"]"
	end
	end

	describe "Invalidation.Script" do
	test "run script to clear cache" do
	Config.put([:media_proxy, :enabled], false)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
	Config.put([Invalidation.Script], script_path: "purge-nginx")

	image_url = "http://example.com/media/example.jpg"
	Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)

	with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
	assert capture_log(fn ->
	assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	assert Invalidation.purge([image_url]) == {:ok, [image_url]}
	- refute Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
	end) =~ "Running cache purge: [\"#{image_url}\"]"
	end
	end
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Sat, Oct 10, 9:10 AM (1 d, 2 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1785127
Default Alt Text: (32 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions