Page MenuHomePhorge
Files F85827646

No OneTemporary
Actions

Authored By
Unknown
Size
13 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 66c90682c..0532e51ad 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -1,174 +1,171 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
- alias Pleroma.Web.RichMedia.Parser.Card
alias Pleroma.Web.RichMedia.Parser.Embed
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
with {:ok, data} <- get_cached_or_parse(url),
{:ok, _} <- set_ttl_based_on_image(data, url) do
{:ok, data}
end
end
defp get_cached_or_parse(url) do
case @cachex.fetch(:rich_media_cache, url, fn ->
case parse_url(url) do
{:ok, _} = res ->
{:commit, res}
{:error, reason} = e ->
# Unfortunately we have to log errors here, instead of doing that
# along with ttl setting at the bottom. Otherwise we can get log spam
# if more than one process was waiting for the rich media card
# while it was generated. Ideally we would set ttl here as well,
# so we don't override it number_of_waiters_on_generation
# times, but one, obviously, can't set ttl for not-yet-created entry
# and Cachex doesn't support returning ttl from the fetch callback.
log_error(url, reason)
{:commit, e}
end
end) do
{action, res} when action in [:commit, :ok] ->
case res do
{:ok, _data} = res ->
res
{:error, reason} = e ->
if action == :commit, do: set_error_ttl(url, reason)
e
end
{:error, e} ->
{:error, {:cachex_error, e}}
end
end
defp set_error_ttl(_url, :body_too_large), do: :ok
defp set_error_ttl(_url, {:content_type, _}), do: :ok
# The TTL is not set for the errors above, since they are unlikely to change
# with time
defp set_error_ttl(url, _reason) do
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
@cachex.expire(:rich_media_cache, url, ttl)
:ok
end
defp log_error(url, {:invalid_metadata, data}) do
Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
end
defp log_error(url, reason) do
Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
end
end
@doc """
Set the rich media cache based on the expiration time of image.
Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
## Example
defmodule MyModule do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, url) do
image_url = Map.get(data, :image)
# do some parsing in the url and get the ttl of the image
# and return ttl is unix time
parse_ttl_from_url(image_url)
end
end
Define the module in the config
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
@spec set_ttl_based_on_image(map(), String.t()) ::
{:ok, Integer.t() | :noop} | {:error, :no_key}
def set_ttl_based_on_image(data, url) do
case get_ttl_from_image(data, url) do
{:ok, ttl} when is_number(ttl) ->
ttl = ttl * 1000
case @cachex.expire_at(:rich_media_cache, url, ttl) do
{:ok, true} -> {:ok, ttl}
{:ok, false} -> {:error, :no_key}
end
_ ->
{:ok, :noop}
end
end
defp get_ttl_from_image(data, url) do
[:rich_media, :ttl_setters]
|> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
_, error ->
error
end)
end
def parse_url(url) do
case maybe_fetch_oembed(url) do
{:ok, %Embed{} = embed} -> {:ok, embed}
_ -> fetch_document(url)
end
end
defp maybe_fetch_oembed(url) do
with true <- Pleroma.Config.get([:rich_media, :oembed_providers_enabled]),
{:ok, oembed_url} <- OEmbedProviders.oembed_url(url),
{:ok, %Tesla.Env{body: json}} <-
Pleroma.Web.RichMedia.Helpers.oembed_get(oembed_url),
{:ok, data} <- Jason.decode(json),
- embed <- %Embed{url: url, oembed: data},
- {:ok, %Card{}} <- Card.validate(embed) do
+ embed <- %Embed{url: url, oembed: data} do
{:ok, embed}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp fetch_document(url) do
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html),
- %Embed{} = embed <- parse_embed(html, url),
- {:ok, %Card{}} <- Card.validate(embed) do
+ %Embed{} = embed <- parse_embed(html, url) do
{:ok, embed}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp parse_embed(html, url) do
Enum.reduce(parsers(), %Embed{url: url}, fn parser, acc ->
parser.parse(html, acc)
end)
end
end
diff --git a/test/pleroma/web/rich_media/parser_test.exs b/test/pleroma/web/rich_media/parser_test.exs
index f7d010219..8506953bb 100644
--- a/test/pleroma/web/rich_media/parser_test.exs
+++ b/test/pleroma/web/rich_media/parser_test.exs
@@ -1,184 +1,216 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.ParserTest do
use ExUnit.Case, async: true
alias Pleroma.Web.RichMedia.Parser
alias Pleroma.Web.RichMedia.Parser.Embed
setup do
Tesla.Mock.mock(fn
%{
method: :get,
url: "http://example.com/ogp"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
%{
method: :get,
url: "http://example.com/non-ogp"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}
%{
method: :get,
url: "http://example.com/ogp-missing-title"
} ->
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
}
%{
method: :get,
url: "http://example.com/twitter-card"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}
%{
method: :get,
url: "http://example.com/oembed"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}
%{
method: :get,
url: "http://example.com/oembed.json"
} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}
%{method: :get, url: "http://example.com/empty"} ->
%Tesla.Env{status: 200, body: "hello"}
%{method: :get, url: "http://example.com/malformed"} ->
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}
%{method: :get, url: "http://example.com/error"} ->
{:error, :overload}
%{
method: :head,
url: "http://example.com/huge-page"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
}
%{
method: :head,
url: "http://example.com/pdf-file"
} ->
%Tesla.Env{
status: 200,
headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
}
%{method: :head} ->
%Tesla.Env{status: 404, body: "", headers: []}
end)
:ok
end
- test "returns error when no metadata present" do
- assert {:error, _} = Parser.parse("http://example.com/empty")
+ test "returns empty embed when no metadata present" do
+ expected = %Embed{
+ meta: %{},
+ oembed: nil,
+ title: nil,
+ url: "http://example.com/empty"
+ }
+
+ assert Parser.parse("http://example.com/empty") == {:ok, expected}
end
test "parses ogp" do
url = "http://example.com/ogp"
expected = %Embed{
meta: %{
"og:image" => "http://ia.media-imdb.com/images/rock.jpg",
"og:title" => "The Rock",
"og:description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
"og:type" => "video.movie",
"og:url" => "http://www.imdb.com/title/tt0117500/"
},
oembed: nil,
title: "The Rock (1996)",
url: "http://example.com/ogp"
}
assert Parser.parse(url) == {:ok, expected}
end
test "gets <title> tag" do
url = "http://example.com/ogp-missing-title"
expected = "The Rock (1996)"
assert {:ok, %Embed{title: ^expected}} = Parser.parse(url)
end
test "parses twitter card" do
url = "http://example.com/twitter-card"
expected = %Embed{
meta: %{
"twitter:card" => "summary",
"twitter:description" => "View the album on Flickr.",
"twitter:image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
"twitter:site" => "@flickr",
"twitter:title" => "Small Island Developing States Photo Submission"
},
oembed: nil,
title: nil,
url: "http://example.com/twitter-card"
}
assert Parser.parse(url) == {:ok, expected}
end
test "parses OEmbed" do
url = "http://example.com/oembed"
expected = %Embed{
meta: %{},
oembed: %{
"author_name" => "‮‭‬bees‬",
"author_url" => "https://www.flickr.com/photos/bees/",
"cache_age" => 3600,
"flickr_type" => "photo",
"height" => "768",
"html" =>
"<a data-flickr-embed=\"true\" href=\"https://www.flickr.com/photos/bees/2362225867/\" title=\"Bacon Lollys by ‮‭‬bees‬, on Flickr\"><img src=\"https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_b.jpg\" width=\"1024\" height=\"768\" alt=\"Bacon Lollys\"></a><script async src=\"https://embedr.flickr.com/assets/client-code.js\" charset=\"utf-8\"></script>",
"license" => "All Rights Reserved",
"license_id" => 0,
"provider_name" => "Flickr",
"provider_url" => "https://www.flickr.com/",
"thumbnail_height" => 150,
"thumbnail_url" => "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_q.jpg",
"thumbnail_width" => 150,
"title" => "Bacon Lollys",
"type" => "photo",
"url" => "https://farm4.staticflickr.com/3040/2362225867_4a87ab8baf_b.jpg",
"version" => "1.0",
"web_page" => "https://www.flickr.com/photos/bees/2362225867/",
"web_page_short_url" => "https://flic.kr/p/4AK2sc",
"width" => "1024"
},
url: "http://example.com/oembed"
}
assert Parser.parse(url) == {:ok, expected}
end
- test "rejects invalid OGP data" do
- assert {:error, _} = Parser.parse("http://example.com/malformed")
+ test "cleans corrupted meta data" do
+ expected = %Embed{
+ meta: %{
+ "Keywords" => "Konsument i zakupy",
+ "ROBOTS" => "NOARCHIVE",
+ "fb:app_id" => "515714931781741",
+ "fb:pages" => "288018984602680",
+ "google-site-verification" => "3P4BE3hLw82QWqtseIE60qQcOtrpMxMnCNkcv62pjTA",
+ "news_keywords" => "Konsument i zakupy",
+ "og:image" =>
+ "https://bi.im-g.pl/im/f7/49/17/z24418295FBW,Prace-nad-projektem-chusty-antysmogowej-rozpoczely.jpg",
+ "og:locale" => "pl_PL",
+ "og:site_name" => "wyborcza.biz",
+ "og:type" => "article",
+ "og:url" =>
+ "http://wyborcza.biz/biznes/7,147743,24417936,pomysl-na-biznes-chusta-ktora-chroni-przed-smogiem.html",
+ "twitter:card" => "summary_large_image",
+ "twitter:image" =>
+ "https://bi.im-g.pl/im/f7/49/17/z24418295FBW,Prace-nad-projektem-chusty-antysmogowej-rozpoczely.jpg",
+ "viewport" => "width=device-width, user-scalable=yes"
+ },
+ oembed: nil,
+ title: nil,
+ url: "http://example.com/malformed"
+ }
+
+ assert Parser.parse("http://example.com/malformed") == {:ok, expected}
end
test "returns error if getting page was not successful" do
assert {:error, :overload} = Parser.parse("http://example.com/error")
end
test "does a HEAD request to check if the body is too large" do
assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
end
test "does a HEAD request to check if the body is html" do
assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
end
end

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 30, 10:16 PM (16 h, 57 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1800623
Default Alt Text
(13 KB)

Event Timeline