Page MenuHomePhorge
Files F85826058

No OneTemporary
Actions

Authored By
Unknown
Size
9 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex
index 4ec50e132..46b879434 100644
--- a/lib/pleroma/web/rich_media/backfill.ex
+++ b/lib/pleroma/web/rich_media/backfill.ex
@@ -1,101 +1,101 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Backfill do
alias Pleroma.Web.RichMedia.Card
alias Pleroma.Web.RichMedia.Parser
alias Pleroma.Web.RichMedia.Parser.TTL
- alias Pleroma.Workers.RichMediaExpirationWorker
+ alias Pleroma.Workers.RichMediaWorker
require Logger
@backfiller Pleroma.Config.get([__MODULE__, :provider], Pleroma.Web.RichMedia.Backfill.Task)
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
@max_attempts 3
@retry 5_000
def start(%{url: url} = args) when is_binary(url) do
url_hash = Card.url_to_hash(url)
args =
args
|> Map.put(:attempt, 1)
|> Map.put(:url_hash, url_hash)
@backfiller.run(args)
end
def run(%{url: url, url_hash: url_hash, attempt: attempt} = args)
when attempt <= @max_attempts do
case Parser.parse(url) do
{:ok, fields} ->
{:ok, card} = Card.create(url, fields)
maybe_schedule_expiration(url, fields)
if Map.has_key?(args, :activity_id) do
stream_update(args)
end
warm_cache(url_hash, card)
{:error, {:invalid_metadata, fields}} ->
Logger.debug("Rich media incomplete or invalid metadata for #{url}: #{inspect(fields)}")
negative_cache(url_hash)
{:error, :body_too_large} ->
Logger.error("Rich media error for #{url}: :body_too_large")
negative_cache(url_hash)
{:error, {:content_type, type}} ->
Logger.debug("Rich media error for #{url}: :content_type is #{type}")
negative_cache(url_hash)
e ->
Logger.debug("Rich media error for #{url}: #{inspect(e)}")
:timer.sleep(@retry * attempt)
run(%{args | attempt: attempt + 1})
end
end
def run(%{url: url, url_hash: url_hash}) do
Logger.debug("Rich media failure for #{url}")
negative_cache(url_hash, :timer.minutes(15))
end
defp maybe_schedule_expiration(url, fields) do
case TTL.process(fields, url) do
{:ok, ttl} when is_number(ttl) ->
timestamp = DateTime.from_unix!(ttl)
- RichMediaExpirationWorker.new(%{"url" => url}, scheduled_at: timestamp)
+ RichMediaWorker.new(%{"op" => "expire", "url" => url}, scheduled_at: timestamp)
|> Oban.insert()
_ ->
:ok
end
end
defp stream_update(%{activity_id: activity_id}) do
Pleroma.Activity.get_by_id(activity_id)
|> Pleroma.Activity.normalize()
|> Pleroma.Web.ActivityPub.ActivityPub.stream_out()
end
defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
end
defmodule Pleroma.Web.RichMedia.Backfill.Task do
alias Pleroma.Web.RichMedia.Backfill
def run(args) do
Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
name: {:global, {:rich_media, args.url_hash}}
)
end
end
diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_worker.ex
similarity index 71%
rename from lib/pleroma/workers/rich_media_expiration_worker.ex
rename to lib/pleroma/workers/rich_media_worker.ex
index 0b74687cf..968395c64 100644
--- a/lib/pleroma/workers/rich_media_expiration_worker.ex
+++ b/lib/pleroma/workers/rich_media_worker.ex
@@ -1,15 +1,15 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.Workers.RichMediaExpirationWorker do
+defmodule Pleroma.Workers.RichMediaWorker do
alias Pleroma.Web.RichMedia.Card
use Oban.Worker,
queue: :background
@impl Oban.Worker
- def perform(%Job{args: %{"url" => url} = _args}) do
+ def perform(%Job{args: %{"op" => "expire", "url" => url} = _args}) do
Card.delete(url)
end
end
diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
index cc28aa7f3..8fd9e6a5f 100644
--- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
+++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs
@@ -1,105 +1,108 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
use Pleroma.DataCase, async: false
use Oban.Testing, repo: Pleroma.Repo
import Mox
alias Pleroma.UnstubbedConfigMock, as: ConfigMock
alias Pleroma.Web.RichMedia.Card
alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
setup do
ConfigMock
|> stub_with(Pleroma.Test.StaticConfig)
clear_config([:rich_media, :enabled], true)
:ok
end
test "s3 signed url is parsed correct for expiration time" do
url = "https://pleroma.social/amz"
{:ok, timestamp} =
Timex.now()
|> DateTime.truncate(:second)
|> Timex.format("{ISO:Basic:Z}")
# in seconds
valid_till = 30
metadata = construct_metadata(timestamp, valid_till, url)
expire_time =
Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till)
assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
end
test "s3 signed url is parsed and correct ttl is set for rich media" do
url = "https://pleroma.social/amz"
{:ok, timestamp} =
Timex.now()
|> DateTime.truncate(:second)
|> Timex.format("{ISO:Basic:Z}")
# in seconds
valid_till = 30
metadata = construct_metadata(timestamp, valid_till, url)
body = """
<meta name="twitter:card" content="Pleroma" />
<meta name="twitter:site" content="Pleroma" />
<meta name="twitter:title" content="Pleroma" />
<meta name="twitter:description" content="Pleroma" />
<meta name="twitter:image" content="#{Map.get(metadata, "image")}" />
"""
Tesla.Mock.mock(fn
%{
method: :get,
url: ^url
} ->
%Tesla.Env{status: 200, body: body}
%{method: :head} ->
%Tesla.Env{status: 200}
end)
Card.get_or_backfill_by_url(url)
- assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
+ assert_enqueued(
+ worker: Pleroma.Workers.RichMediaWorker,
+ args: %{"op" => "expire", "url" => url}
+ )
[%Oban.Job{scheduled_at: scheduled_at}] = all_enqueued()
timestamp_dt = Timex.parse!(timestamp, "{ISO:Basic:Z}")
assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
end
test "AWS URL for an image without expiration works" do
og_data = %{"image" => "https://amazonaws.com/image.png"}
assert is_nil(AwsSignedUrl.ttl(og_data, ""))
end
defp construct_s3_url(timestamp, valid_till) do
"https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
end
defp construct_metadata(timestamp, valid_till, url) do
%{
"image" => construct_s3_url(timestamp, valid_till),
"site" => "Pleroma",
"title" => "Pleroma",
"description" => "Pleroma",
"url" => url
}
end
end
diff --git a/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs b/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs
index 770968d47..e275ee523 100644
--- a/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs
+++ b/test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs
@@ -1,41 +1,44 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser.TTL.OpengraphTest do
use Pleroma.DataCase
use Oban.Testing, repo: Pleroma.Repo
import Mox
alias Pleroma.UnstubbedConfigMock, as: ConfigMock
alias Pleroma.Web.RichMedia.Card
setup do
ConfigMock
|> stub_with(Pleroma.Test.StaticConfig)
clear_config([:rich_media, :enabled], true)
:ok
end
test "OpenGraph TTL value is honored" do
url = "https://reddit.com/r/somepost"
Tesla.Mock.mock(fn
%{
method: :get,
url: ^url
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/reddit.html")}
%{method: :head} ->
%Tesla.Env{status: 200}
end)
Card.get_or_backfill_by_url(url)
- assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
+ assert_enqueued(
+ worker: Pleroma.Workers.RichMediaWorker,
+ args: %{"op" => "expire", "url" => url}
+ )
end
end

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 30, 4:57 AM (23 h, 13 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1799495
Default Alt Text
(9 KB)

Event Timeline