Page MenuHomePhorge
Files F85829455

No OneTemporary
Actions

Authored By
Unknown
Size
10 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index d7b0cab6f..dee0f61ac 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -1,191 +1,191 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
alias Pleroma.Web.RichMedia.Parser.Card
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
with {:ok, data} <- get_cached_or_parse(url),
{:ok, _} <- set_ttl_based_on_image(data, url) do
{:ok, data}
end
end
defp get_cached_or_parse(url) do
case @cachex.fetch(:rich_media_cache, url, fn ->
case parse_url(url) do
{:ok, _} = res ->
{:commit, res}
{:error, reason} = e ->
# Unfortunately we have to log errors here, instead of doing that
# along with ttl setting at the bottom. Otherwise we can get log spam
# if more than one process was waiting for the rich media card
# while it was generated. Ideally we would set ttl here as well,
# so we don't override it number_of_waiters_on_generation
# times, but one, obviously, can't set ttl for not-yet-created entry
# and Cachex doesn't support returning ttl from the fetch callback.
log_error(url, reason)
{:commit, e}
end
end) do
{action, res} when action in [:commit, :ok] ->
case res do
{:ok, _data} = res ->
res
{:error, reason} = e ->
if action == :commit, do: set_error_ttl(url, reason)
e
end
{:error, e} ->
{:error, {:cachex_error, e}}
end
end
defp set_error_ttl(_url, :body_too_large), do: :ok
defp set_error_ttl(_url, {:content_type, _}), do: :ok
# The TTL is not set for the errors above, since they are unlikely to change
# with time
defp set_error_ttl(url, _reason) do
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
@cachex.expire(:rich_media_cache, url, ttl)
:ok
end
defp log_error(url, {:invalid_metadata, data}) do
Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
end
defp log_error(url, reason) do
Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
end
end
@doc """
Set the rich media cache based on the expiration time of image.
Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
## Example
defmodule MyModule do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, url) do
image_url = Map.get(data, :image)
# do some parsing in the url and get the ttl of the image
# and return ttl is unix time
parse_ttl_from_url(image_url)
end
end
Define the module in the config
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
@spec set_ttl_based_on_image(map(), String.t()) ::
{:ok, Integer.t() | :noop} | {:error, :no_key}
def set_ttl_based_on_image(data, url) do
case get_ttl_from_image(data, url) do
{:ok, ttl} when is_number(ttl) ->
ttl = ttl * 1000
case @cachex.expire_at(:rich_media_cache, url, ttl) do
{:ok, true} -> {:ok, ttl}
{:ok, false} -> {:error, :no_key}
end
_ ->
{:ok, :noop}
end
end
defp get_ttl_from_image(data, url) do
[:rich_media, :ttl_setters]
|> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
_, error ->
error
end)
end
def parse_url(url) do
case maybe_fetch_oembed(url) do
{:ok, %Card{} = card} -> {:ok, card}
_ -> fetch_document(url)
end
end
defp maybe_fetch_oembed(url) do
with {:ok, oembed_url} <- OEmbedProviders.oembed_url(url),
{:ok, %Tesla.Env{body: json}} <-
Pleroma.Web.RichMedia.Helpers.oembed_get(oembed_url),
{:ok, data} <- Jason.decode(json),
%Card{} = card <- Card.from_oembed(data, url) do
{:ok, card}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp fetch_document(url) do
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html) do
html
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
- |> Card.from_meta_tags(url)
+ |> Card.from_discovery(url)
|> check_card()
end
end
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do
data when data != %{} -> {:halt, data}
_ -> {:cont, acc}
end
end)
end
defp check_card(%Card{title: title} = card)
when is_binary(title) and title != "" do
{:ok, card}
end
defp check_card(card) do
{:error, {:invalid_metadata, card}}
end
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
not match?({:ok, _}, Jason.encode(%{key => val}))
end)
|> Map.new()
end
end
diff --git a/lib/pleroma/web/rich_media/parser/card.ex b/lib/pleroma/web/rich_media/parser/card.ex
index 5ea719485..da0acc789 100644
--- a/lib/pleroma/web/rich_media/parser/card.ex
+++ b/lib/pleroma/web/rich_media/parser/card.ex
@@ -1,75 +1,77 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser.Card do
@types ["link", "photo", "video", "rich"]
# https://docs.joinmastodon.org/entities/card/
defstruct url: nil,
title: nil,
description: "",
type: "link",
author_name: "",
author_url: "",
provider_name: "",
provider_url: "",
html: "",
width: 0,
height: 0,
image: nil,
embed_url: "",
blurhash: nil
def from_oembed(%{"type" => type, "title" => title} = oembed, url) when type in @types do
%__MODULE__{
url: url,
title: title,
description: "",
type: type,
author_name: oembed["author_name"],
author_url: oembed["author_url"],
provider_name: oembed["provider_name"],
provider_url: oembed["provider_url"],
html: oembed["html"],
width: oembed["width"],
height: oembed["height"],
image: oembed["thumbnail_url"] |> proxy(),
embed_url: oembed["url"] |> proxy()
}
end
def from_oembed(_oembed, _url), do: nil
- def from_meta_tags(rich_media, page_url) do
+ def from_discovery(%{"type" => "link"} = rich_media, page_url) do
page_url_data = URI.parse(page_url)
page_url_data =
if is_binary(rich_media["url"]) do
URI.merge(page_url_data, URI.parse(rich_media["url"]))
else
page_url_data
end
page_url = page_url_data |> to_string
image_url =
if is_binary(rich_media["image"]) do
URI.merge(page_url_data, URI.parse(rich_media["image"]))
|> to_string
end
%__MODULE__{
type: "link",
provider_name: page_url_data.host,
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url |> proxy(),
title: rich_media["title"] || "",
description: rich_media["description"] || ""
}
end
+ def from_discovery(rich_media, url), do: from_oembed(rich_media, url)
+
defp proxy(url) when is_binary(url), do: Pleroma.Web.MediaProxy.url(url)
defp proxy(_), do: nil
end
diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex
index 09eabec56..8dc378c3c 100644
--- a/lib/pleroma/web/rich_media/parsers/o_embed.ex
+++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex
@@ -1,29 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
def parse(html, _data) do
with elements = [_ | _] <- get_discovery_data(html),
oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
{:ok, oembed_data} <- get_oembed_data(oembed_url) do
oembed_data
else
_e -> %{}
end
end
defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
end
defp get_oembed_url([{"link", attributes, _children} | _]) do
Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
end
defp get_oembed_data(url) do
- with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do
+ with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.oembed_get(url) do
Jason.decode(json)
end
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/twitter_card.ex b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
index 0adf84159..31546819e 100644
--- a/lib/pleroma/web/rich_media/parsers/twitter_card.ex
+++ b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
@@ -1,15 +1,16 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
@spec parse(list(), map()) :: map()
def parse(html, data) do
data
|> MetaTagsParser.parse(html, "og", "property")
|> MetaTagsParser.parse(html, "twitter", "name")
|> MetaTagsParser.parse(html, "twitter", "property")
+ |> Map.put("type", "link")
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sat, Oct 31, 8:13 PM (13 h, 14 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1801756
Default Alt Text
(10 KB)

Event Timeline