Page MenuHomePhorge
Files F85827254

No OneTemporary
Actions

Authored By
Unknown
Size
15 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index ca592833f..e9aa2dd03 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -1,115 +1,119 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
+ require Logger
+
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
+ @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
+ @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
- try do
- Cachex.fetch!(:rich_media_cache, url, fn _ ->
- {:commit, parse_url(url)}
- end)
- |> set_ttl_based_on_image(url)
- rescue
- e ->
- {:error, "Cachex error: #{inspect(e)}"}
- end
+ Cachex.fetch!(:rich_media_cache, url, fn _ ->
+ with {:ok, data} <- parse_url(url) do
+ {:commit, {:ok, data}}
+ else
+ error -> {:ignore, error}
+ end
+ end)
+ |> set_ttl_based_on_image(url)
end
end
@doc """
Set the rich media cache based on the expiration time of image.
Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
## Example
defmodule MyModule do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, url) do
image_url = Map.get(data, :image)
# do some parsing in the url and get the ttl of the image
# and return ttl is unix time
parse_ttl_from_url(image_url)
end
end
Define the module in the config
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
+ @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
+ {:ok, map()} | {:error, any()}
def set_ttl_based_on_image({:ok, data}, url) do
with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
- ttl when is_number(ttl) <- get_ttl_from_image(data, url) do
+ {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
{:ok, data}
else
_ ->
{:ok, data}
end
end
+ def set_ttl_based_on_image({:error, _} = error, _) do
+ Logger.error("parsing error: #{inspect(error)}")
+ error
+ end
+
defp get_ttl_from_image(data, url) do
- Pleroma.Config.get([:rich_media, :ttl_setters])
+ [:rich_media, :ttl_setters]
+ |> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
_, error ->
error
end)
end
defp parse_url(url) do
- try do
- {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
-
+ with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
+ {:ok, html} <- Floki.parse_document(html) do
html
- |> parse_html()
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
|> check_parsed_data()
- rescue
- e ->
- {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
end
end
- defp parse_html(html), do: Floki.parse_document!(html)
-
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do
data when data != %{} -> {:halt, data}
_ -> {:cont, acc}
end
end)
end
defp check_parsed_data(%{"title" => title} = data)
when is_binary(title) and title != "" do
{:ok, data}
end
defp check_parsed_data(data) do
{:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
end
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
not match?({:ok, _}, Jason.encode(%{key => val}))
end)
|> Map.new()
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
index 0dc1efdaf..c5aaea2d4 100644
--- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
@@ -1,51 +1,46 @@
defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
@impl Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, _url) do
image = Map.get(data, :image)
if is_aws_signed_url(image) do
image
|> parse_query_params()
|> format_query_params()
|> get_expiration_timestamp()
+ else
+ {:error, "Not aws signed url #{inspect(image)}"}
end
end
- defp is_aws_signed_url(""), do: nil
- defp is_aws_signed_url(nil), do: nil
-
- defp is_aws_signed_url(image) when is_binary(image) do
+ defp is_aws_signed_url(image) when is_binary(image) and image != "" do
%URI{host: host, query: query} = URI.parse(image)
- if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do
- image
- else
- nil
- end
+ String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
end
defp is_aws_signed_url(_), do: nil
defp parse_query_params(image) do
%URI{query: query} = URI.parse(image)
query
end
defp format_query_params(query) do
query
|> String.split(~r/&|=/)
|> Enum.chunk_every(2)
|> Map.new(fn [k, v] -> {k, v} end)
end
defp get_expiration_timestamp(params) when is_map(params) do
{:ok, date} =
params
|> Map.get("X-Amz-Date")
|> Timex.parse("{ISO:Basic:Z}")
- Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
+ {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))}
end
end
diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs
index b30f4400e..a21f3c935 100644
--- a/test/web/rich_media/aws_signed_url_test.exs
+++ b/test/web/rich_media/aws_signed_url_test.exs
@@ -1,82 +1,82 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do
use ExUnit.Case, async: true
test "s3 signed url is parsed correct for expiration time" do
url = "https://pleroma.social/amz"
{:ok, timestamp} =
Timex.now()
|> DateTime.truncate(:second)
|> Timex.format("{ISO:Basic:Z}")
# in seconds
valid_till = 30
metadata = construct_metadata(timestamp, valid_till, url)
expire_time =
Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till)
- assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
+ assert {:ok, expire_time} == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
end
test "s3 signed url is parsed and correct ttl is set for rich media" do
url = "https://pleroma.social/amz"
{:ok, timestamp} =
Timex.now()
|> DateTime.truncate(:second)
|> Timex.format("{ISO:Basic:Z}")
# in seconds
valid_till = 30
metadata = construct_metadata(timestamp, valid_till, url)
body = """
<meta name="twitter:card" content="Pleroma" />
<meta name="twitter:site" content="Pleroma" />
<meta name="twitter:title" content="Pleroma" />
<meta name="twitter:description" content="Pleroma" />
<meta name="twitter:image" content="#{Map.get(metadata, :image)}" />
"""
Tesla.Mock.mock(fn
%{
method: :get,
url: "https://pleroma.social/amz"
} ->
%Tesla.Env{status: 200, body: body}
end)
Cachex.put(:rich_media_cache, url, metadata)
Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url)
{:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
# as there is delay in setting and pulling the data from cache we ignore 1 second
# make it 2 seconds for flakyness
assert_in_delta(valid_till * 1000, cache_ttl, 2000)
end
defp construct_s3_url(timestamp, valid_till) do
"https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{
timestamp
}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
end
defp construct_metadata(timestamp, valid_till, url) do
%{
image: construct_s3_url(timestamp, valid_till),
site: "Pleroma",
title: "Pleroma",
description: "Pleroma",
url: url
}
end
end
diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs
index 420a612c6..1e09cbf84 100644
--- a/test/web/rich_media/parser_test.exs
+++ b/test/web/rich_media/parser_test.exs
@@ -1,137 +1,149 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.ParserTest do
use ExUnit.Case, async: true
+ alias Pleroma.Web.RichMedia.Parser
+
setup do
Tesla.Mock.mock(fn
%{
method: :get,
url: "http://example.com/ogp"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
%{
method: :get,
url: "http://example.com/non-ogp"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}
%{
method: :get,
url: "http://example.com/ogp-missing-title"
} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
}
%{
method: :get,
url: "http://example.com/twitter-card"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}
%{
method: :get,
url: "http://example.com/oembed"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}
%{
method: :get,
url: "http://example.com/oembed.json"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}
%{method: :get, url: "http://example.com/empty"} ->
%Tesla.Env{status: 200, body: "hello"}
+
+ %{method: :get, url: "http://example.com/malformed"} ->
+ %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}
+
+ %{method: :get, url: "http://example.com/error"} ->
+ {:error, :overload}
end)
:ok
end
test "returns error when no metadata present" do
- assert {:error, _} = Pleroma.Web.RichMedia.Parser.parse("http://example.com/empty")
+ assert {:error, _} = Parser.parse("http://example.com/empty")
end
test "doesn't just add a title" do
- assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/non-ogp") ==
+ assert Parser.parse("http://example.com/non-ogp") ==
{:error,
"Found metadata was invalid or incomplete: %{\"url\" => \"http://example.com/non-ogp\"}"}
end
test "parses ogp" do
- assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp") ==
+ assert Parser.parse("http://example.com/ogp") ==
{:ok,
%{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"title" => "The Rock",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
"type" => "video.movie",
"url" => "http://example.com/ogp"
}}
end
test "falls back to <title> when ogp:title is missing" do
- assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp-missing-title") ==
+ assert Parser.parse("http://example.com/ogp-missing-title") ==
{:ok,
%{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"title" => "The Rock (1996)",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
"type" => "video.movie",
"url" => "http://example.com/ogp-missing-title"
}}
end
test "parses twitter card" do
- assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/twitter-card") ==
+ assert Parser.parse("http://example.com/twitter-card") ==
{:ok,
%{
"card" => "summary",
"site" => "@flickr",
"image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
"title" => "Small Island Developing States Photo Submission",
"description" => "View the album on Flickr.",
"url" => "http://example.com/twitter-card"
}}
end
test "parses OEmbed" do
- assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/oembed") ==
+ assert Parser.parse("http://example.com/oembed") ==
{:ok,
%{
"author_name" => "‮‭‬bees‬",
"author_url" => "https://www.flickr.com/photos/bees/",
"cache_age" => 3600,
"flickr_type" => "photo",
"height" => "768",
"html" =>
"<a data-flickr-embed=\"true\" href=\"https://www.flickr.com/photos/bees/2362225867/\" title=\"Bacon Lollys by ‮‭‬bees‬, on Flickr\"><img src=\"https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_b.jpg\" width=\"1024\" height=\"768\" alt=\"Bacon Lollys\"></a><script async src=\"https://embedr.flickr.com/assets/client-code.js\" charset=\"utf-8\"></script>",
"license" => "All Rights Reserved",
"license_id" => 0,
"provider_name" => "Flickr",
"provider_url" => "https://www.flickr.com/",
"thumbnail_height" => 150,
"thumbnail_url" =>
"https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_q.jpg",
"thumbnail_width" => 150,
"title" => "Bacon Lollys",
"type" => "photo",
"url" => "http://example.com/oembed",
"version" => "1.0",
"web_page" => "https://www.flickr.com/photos/bees/2362225867/",
"web_page_short_url" => "https://flic.kr/p/4AK2sc",
"width" => "1024"
}}
end
test "rejects invalid OGP data" do
- assert {:error, _} = Pleroma.Web.RichMedia.Parser.parse("http://example.com/malformed")
+ assert {:error, _} = Parser.parse("http://example.com/malformed")
+ end
+
+ test "returns error if getting page was not successful" do
+ assert {:error, :overload} = Parser.parse("http://example.com/error")
end
end

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 30, 5:05 PM (14 h, 43 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1800332
Default Alt Text
(15 KB)

Event Timeline