No OneTemporary
Actions

Size

20 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
	index fac731c5c..537cb76d6 100644
	--- a/lib/pleroma/web/media_proxy/invalidation.ex
	+++ b/lib/pleroma/web/media_proxy/invalidation.ex
	@@ -1,36 +1,37 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.MediaProxy.Invalidation do
	@moduledoc false

	@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} \| {:error, String.t()}

	alias Pleroma.Config
	alias Pleroma.Web.MediaProxy

	def enabled, do: Config.get([:media_proxy, :invalidation, :enabled])

	@spec purge(list(String.t()) \| String.t()) :: {:ok, list(String.t())} \| {:error, String.t()}
	def purge(urls) do
	prepared_urls = prepare_urls(urls)

	if enabled() do
	do_purge(prepared_urls)
	else
	{:ok, prepared_urls}
	end
	end

	defp do_purge(urls) do
	provider = Config.get([:media_proxy, :invalidation, :provider])
	provider.purge(urls, Config.get(provider))
	end

	def prepare_urls(urls) do
	urls
	\|> List.wrap()
	+ \|> Enum.filter(&MediaProxy.is_url_proxiable?(&1))
	\|> Enum.map(&MediaProxy.url(&1))
	end
	end
	diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
	index 7b8e7e823..e5f1d242e 100644
	--- a/lib/pleroma/web/media_proxy/media_proxy.ex
	+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
	@@ -1,109 +1,121 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.MediaProxy do
	alias Pleroma.Config
	alias Pleroma.Upload
	alias Pleroma.Web
	alias Pleroma.Web.MediaProxy.Invalidation

	@base64_opts [padding: false]

	@spec in_deleted_urls(String.t()) :: boolean()
	def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1)

	def remove_from_deleted_urls(urls) when is_list(urls) do
	Cachex.execute!(:deleted_urls_cache, fn cache ->
	Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
	end)
	end

	def remove_from_deleted_urls(url) when is_binary(url) do
	Cachex.del(:deleted_urls_cache, url(url))
	end

	def put_in_deleted_urls(urls) when is_list(urls) do
	Cachex.execute!(:deleted_urls_cache, fn cache ->
	Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
	end)
	end

	def put_in_deleted_urls(url) when is_binary(url) do
	- Cachex.put(:deleted_urls_cache, url(url), true)
	+ if is_url_proxiable?(url) do
	+ Cachex.put(:deleted_urls_cache, url(url), true)
	+ else
	+ true
	+ end
	end

	def url(url) when is_nil(url) or url == "", do: nil
	def url("/" <> _ = url), do: url

	def url(url) do
	- if disabled?() or local?(url) or whitelisted?(url) do
	+ if disabled?() or not is_url_proxiable?(url) do
	url
	else
	encode_url(url)
	end
	end

	+ def is_url_proxiable?(url) do
	+ if local?(url) or whitelisted?(url) do
	+ false
	+ else
	+ true
	+ end
	+ end
	+
	defp disabled?, do: !Config.get([:media_proxy, :enabled], false)

	defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())

	defp whitelisted?(url) do
	%{host: domain} = URI.parse(url)

	mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])

	upload_base_url_domain =
	if !is_nil(Config.get([Upload, :base_url])) do
	[URI.parse(Config.get([Upload, :base_url])).host]
	else
	[]
	end

	whitelist = mediaproxy_whitelist ++ upload_base_url_domain

	Enum.any?(whitelist, fn pattern ->
	String.equivalent?(domain, pattern)
	end)
	end

	def encode_url(url) do
	base64 = Base.url_encode64(url, @base64_opts)

	sig64 =
	base64
	\|> signed_url
	\|> Base.url_encode64(@base64_opts)

	build_url(sig64, base64, filename(url))
	end

	def decode_url(sig, url) do
	with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
	signature when signature == sig <- signed_url(url) do
	{:ok, Base.url_decode64!(url, @base64_opts)}
	else
	_ -> {:error, :invalid_signature}
	end
	end

	defp signed_url(url) do
	:crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url)
	end

	def filename(url_or_path) do
	if path = URI.parse(url_or_path).path, do: Path.basename(path)
	end

	def build_url(sig_base64, url_base64, filename \\ nil) do
	[
	Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()),
	"proxy",
	sig_base64,
	url_base64,
	filename
	]
	\|> Enum.filter(& &1)
	\|> Path.join()
	end
	end
	diff --git a/test/object_test.exs b/test/object_test.exs
	index ca9f918ef..5cd2d2c80 100644
	--- a/test/object_test.exs
	+++ b/test/object_test.exs
	@@ -1,418 +1,418 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.ObjectTest do
	use Pleroma.DataCase
	use Oban.Testing, repo: Pleroma.Repo
	import ExUnit.CaptureLog
	import Pleroma.Factory
	import Tesla.Mock
	alias Pleroma.Activity
	alias Pleroma.Config
	alias Pleroma.Object
	alias Pleroma.Repo
	alias Pleroma.Tests.ObanHelpers
	alias Pleroma.Web.CommonAPI
	alias Pleroma.Web.MediaProxy.Invalidation

	setup do
	mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
	:ok
	end

	test "returns an object by it's AP id" do
	object = insert(:note)
	found_object = Object.get_by_ap_id(object.data["id"])

	assert object == found_object
	end

	describe "generic changeset" do
	test "it ensures uniqueness of the id" do
	object = insert(:note)
	cs = Object.change(%Object{}, %{data: %{id: object.data["id"]}})
	assert cs.valid?

	{:error, _result} = Repo.insert(cs)
	end
	end

	describe "deletion function" do
	test "deletes an object" do
	object = insert(:note)
	found_object = Object.get_by_ap_id(object.data["id"])

	assert object == found_object

	Object.delete(found_object)

	found_object = Object.get_by_ap_id(object.data["id"])

	refute object == found_object

	assert found_object.data["type"] == "Tombstone"
	end

	test "ensures cache is cleared for the object" do
	object = insert(:note)
	cached_object = Object.get_cached_by_ap_id(object.data["id"])

	assert object == cached_object

	Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe")

	Object.delete(cached_object)

	{:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}")
	{:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path)

	cached_object = Object.get_cached_by_ap_id(object.data["id"])

	refute object == cached_object

	assert cached_object.data["type"] == "Tombstone"
	end
	end

	describe "delete attachments" do
	setup do: clear_config([Pleroma.Upload])
	setup do: clear_config([:instance, :cleanup_attachments])
	setup do: clear_config([:media_proxy])

	test "Disabled via config" do
	Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Pleroma.Config.put([:instance, :cleanup_attachments], false)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	refute Object.get_by_id(attachment.id) == nil

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
	refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end

	test "in subdirectories" do
	Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Config.put([:instance, :cleanup_attachments], true)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	- assert Pleroma.Web.MediaProxy.in_deleted_urls(href)
	+ refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end

	test "with dedupe enabled" do
	Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
	Config.put([:instance, :cleanup_attachments], true)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock)

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	File.mkdir_p!(uploads_dir)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	filename = Path.basename(href)

	assert {:ok, files} = File.ls(uploads_dir)
	assert filename in files

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil
	assert {:ok, files} = File.ls(uploads_dir)
	refute filename in files
	- assert Pleroma.Web.MediaProxy.in_deleted_urls(href)
	+ refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end

	test "with objects that have legacy data.url attribute" do
	Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Config.put([:instance, :cleanup_attachments], true)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	{:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	- assert Pleroma.Web.MediaProxy.in_deleted_urls(href)
	+ refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end

	test "With custom base_url" do
	Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
	Config.put([Pleroma.Upload, :base_url], "https://sub.domain.tld/dir/")
	Config.put([:instance, :cleanup_attachments], true)
	Config.put([:media_proxy, :invalidation, :enabled], true)
	Config.put([:media_proxy, :invalidation, :provider], Invalidation.Mock)

	file = %Plug.Upload{
	content_type: "image/jpg",
	path: Path.absname("test/fixtures/image.jpg"),
	filename: "an_image.jpg"
	}

	user = insert(:user)

	{:ok, %Object{} = attachment} =
	Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)

	%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
	note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})

	uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])

	path = href \|> Path.dirname() \|> Path.basename()

	assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")

	Object.delete(note)

	ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))

	assert Object.get_by_id(note.id).data["deleted"]
	assert Object.get_by_id(attachment.id) == nil

	assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
	- assert Pleroma.Web.MediaProxy.in_deleted_urls(href)
	+ refute Pleroma.Web.MediaProxy.in_deleted_urls(href)
	end
	end

	describe "normalizer" do
	test "fetches unknown objects by default" do
	%Object{} =
	object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367")

	assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
	end

	test "fetches unknown objects when fetch_remote is explicitly true" do
	%Object{} =
	object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true)

	assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
	end

	test "does not fetch unknown objects when fetch_remote is false" do
	assert is_nil(
	Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)
	)
	end
	end

	describe "get_by_id_and_maybe_refetch" do
	setup do
	mock(fn
	%{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
	%Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")}

	env ->
	apply(HttpRequestMock, :request, [env])
	end)

	mock_modified = fn resp ->
	mock(fn
	%{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
	resp

	env ->
	apply(HttpRequestMock, :request, [env])
	end)
	end

	on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end)

	[mock_modified: mock_modified]
	end

	test "refetches if the time since the last refetch is greater than the interval", %{
	mock_modified: mock_modified
	} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
	end

	test "returns the old object if refetch fails", %{mock_modified: mock_modified} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	assert capture_log(fn ->
	mock_modified.(%Tesla.Env{status: 404, body: ""})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
	end) =~
	"[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"
	end

	test "does not refetch if the time since the last refetch is greater than the interval", %{
	mock_modified: mock_modified
	} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
	end

	test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
	%Object{} =
	object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")

	Object.set_cache(object)

	assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
	assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0

	user = insert(:user)
	activity = Activity.get_create_by_object_ap_id(object.data["id"])
	{:ok, activity} = CommonAPI.favorite(user, activity.id)
	object = Object.get_by_ap_id(activity.data["object"])

	assert object.data["like_count"] == 1

	mock_modified.(%Tesla.Env{
	status: 200,
	body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
	})

	updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
	object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
	assert updated_object == object_in_cache
	assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
	assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3

	assert updated_object.data["like_count"] == 1
	end
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Sat, Aug 8, 10:01 PM (1 d, 18 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1724007
Default Alt Text: (20 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions