Page MenuHomePhorge
Files F85649412

No OneTemporary
Actions

Size
12 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index 537cb76d6..fac731c5c 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -1,37 +1,36 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false
@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config
alias Pleroma.Web.MediaProxy
def enabled, do: Config.get([:media_proxy, :invalidation, :enabled])
@spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do
prepared_urls = prepare_urls(urls)
if enabled() do
do_purge(prepared_urls)
else
{:ok, prepared_urls}
end
end
defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider])
provider.purge(urls, Config.get(provider))
end
def prepare_urls(urls) do
urls
|> List.wrap()
- |> Enum.filter(&MediaProxy.is_url_proxiable?(&1))
|> Enum.map(&MediaProxy.url(&1))
end
end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index e5f1d242e..59ca217ab 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -1,121 +1,118 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Config
alias Pleroma.Upload
alias Pleroma.Web
alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
@spec in_deleted_urls(String.t()) :: boolean()
def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1)
def remove_from_deleted_urls(urls) when is_list(urls) do
Cachex.execute!(:deleted_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end)
end
def remove_from_deleted_urls(url) when is_binary(url) do
Cachex.del(:deleted_urls_cache, url(url))
end
def put_in_deleted_urls(urls) when is_list(urls) do
Cachex.execute!(:deleted_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end)
end
def put_in_deleted_urls(url) when is_binary(url) do
- if is_url_proxiable?(url) do
- Cachex.put(:deleted_urls_cache, url(url), true)
- else
- true
- end
+ Cachex.put(:deleted_urls_cache, url(url), true)
end
def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
if disabled?() or not is_url_proxiable?(url) do
url
else
encode_url(url)
end
end
+ @spec is_url_proxiable?(String.t()) :: boolean()
def is_url_proxiable?(url) do
if local?(url) or whitelisted?(url) do
false
else
true
end
end
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
defp whitelisted?(url) do
%{host: domain} = URI.parse(url)
mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])
upload_base_url_domain =
if !is_nil(Config.get([Upload, :base_url])) do
[URI.parse(Config.get([Upload, :base_url])).host]
else
[]
end
whitelist = mediaproxy_whitelist ++ upload_base_url_domain
Enum.any?(whitelist, fn pattern ->
String.equivalent?(domain, pattern)
end)
end
def encode_url(url) do
base64 = Base.url_encode64(url, @base64_opts)
sig64 =
base64
|> signed_url
|> Base.url_encode64(@base64_opts)
build_url(sig64, base64, filename(url))
end
def decode_url(sig, url) do
with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
signature when signature == sig <- signed_url(url) do
{:ok, Base.url_decode64!(url, @base64_opts)}
else
_ -> {:error, :invalid_signature}
end
end
defp signed_url(url) do
:crypto.hmac(:sha, Config.get([Web.Endpoint, :secret_key_base]), url)
end
def filename(url_or_path) do
if path = URI.parse(url_or_path).path, do: Path.basename(path)
end
def build_url(sig_base64, url_base64, filename \\ nil) do
[
Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()),
"proxy",
sig_base64,
url_base64,
filename
]
|> Enum.filter(& &1)
|> Path.join()
end
end
diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex
index 170bca976..4a7b67080 100644
--- a/lib/pleroma/workers/attachments_cleanup_worker.ex
+++ b/lib/pleroma/workers/attachments_cleanup_worker.ex
@@ -1,125 +1,130 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.AttachmentsCleanupWorker do
import Ecto.Query
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.Web.MediaProxy
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
@impl Oban.Worker
def perform(
%{
"op" => "cleanup_attachments",
"object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}
},
_job
) do
hrefs =
Enum.flat_map(attachments, fn attachment ->
Enum.map(attachment["url"], & &1["href"])
end)
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url =
String.trim_trailing(
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
"/"
)
# find all objects for copies of the attachments, name and actor doesn't matter here
{object_ids, attachment_urls, exclude_urls} =
hrefs
|> fetch_objects
|> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
|> Enum.reduce({[], [], []}, fn {href, %{id: id, count: count}},
{ids, hrefs, exclude_urls} ->
with 1 <- count do
{ids ++ [id], hrefs ++ [href], exclude_urls}
else
_ -> {ids ++ [id], hrefs, exclude_urls ++ [href]}
end
end)
lock_attachments(MediaProxy.Invalidation.enabled(), hrefs -- exclude_urls)
Enum.each(attachment_urls, fn href ->
href
|> String.trim_leading("#{base_url}/#{prefix}")
|> uploader.delete_file()
end)
delete_objects(object_ids)
cache_purge(MediaProxy.Invalidation.enabled(), hrefs -- exclude_urls)
{:ok, :success}
end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
defp delete_objects([_ | _] = object_ids) do
Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
end
defp delete_objects(_), do: :ok
defp cache_purge(true, [_ | _] = urls), do: MediaProxy.Invalidation.purge(urls)
defp cache_purge(_, _), do: :ok
- defp lock_attachments(true, urls), do: MediaProxy.put_in_deleted_urls(urls)
+ defp lock_attachments(true, [_ | _] = urls) do
+ urls
+ |> Enum.filter(&MediaProxy.is_url_proxiable?(&1))
+ |> MediaProxy.put_in_deleted_urls()
+ end
+
defp lock_attachments(_, _), do: :ok
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
def prepare_objects(objects, actor, names) do
objects
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
end
def fetch_objects(hrefs) do
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timeout: :infinity)
end
end
diff --git a/test/workers/attachments_cleanup_worker_test.exs b/test/workers/attachments_cleanup_worker_test.exs
new file mode 100644
index 000000000..ec13eedbe
--- /dev/null
+++ b/test/workers/attachments_cleanup_worker_test.exs
@@ -0,0 +1,91 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
+ use Pleroma.DataCase
+
+ alias Pleroma.Config
+ alias Pleroma.Web.MediaProxy
+ alias Pleroma.Workers.AttachmentsCleanupWorker
+
+ import Mock
+ import Pleroma.Factory
+
+ describe "delete attachments" do
+ setup do: clear_config([Pleroma.Upload])
+ setup do: clear_config([:instance, :cleanup_attachments])
+ setup do: clear_config([:media_proxy])
+
+ test "deletes attachment objects and run purge cache" do
+ Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ Config.put([:instance, :cleanup_attachments], true)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Mock)
+
+ file = %Plug.Upload{
+ content_type: "image/jpg",
+ path: Path.absname("test/fixtures/image.jpg"),
+ filename: "an_image.jpg"
+ }
+
+ user = insert(:user)
+
+ {:ok, %Pleroma.Object{} = attachment} =
+ Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
+
+ remote_url = "http://example.com/media/d6661b98ae72e39.jpg"
+ %{data: %{"url" => [%{"href" => local_url}]}} = attachment
+
+ note =
+ insert(:note, %{
+ user: user,
+ data: %{
+ "attachment" => [
+ attachment.data,
+ %{
+ "actor" => user.ap_id,
+ "name" => "v_image.jpg",
+ "type" => "Document",
+ "url" => [
+ %{"href" => remote_url, "mediaType" => "image/jpeg", "type" => "Link"}
+ ]
+ }
+ ]
+ }
+ })
+
+ uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
+
+ path = local_url |> Path.dirname() |> Path.basename()
+
+ assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+
+ with_mocks [
+ {MediaProxy.Invalidation, [:passthrough],
+ [purge: fn [^local_url, ^remote_url] -> :ok end]}
+ ] do
+ assert AttachmentsCleanupWorker.perform(
+ %{"op" => "cleanup_attachments", "object" => %{"data" => note.data}},
+ :job
+ ) == {:ok, :success}
+ end
+
+ refute Pleroma.Object.get_by_id(attachment.id)
+ assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+
+ refute Pleroma.Web.MediaProxy.in_deleted_urls(local_url)
+ assert Pleroma.Web.MediaProxy.in_deleted_urls(remote_url)
+ end
+
+ test "skip execution" do
+ assert AttachmentsCleanupWorker.perform(
+ %{
+ "op" => "cleanup_attachments",
+ "object" => %{}
+ },
+ :job
+ ) == {:ok, :skip}
+ end
+ end
+end

File Metadata

Mime Type
text/x-diff
Expires
Sat, Aug 29, 11:31 AM (1 d, 23 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1737052
Default Alt Text
(12 KB)

Event Timeline