Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85951005
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
26 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index dee0f61ac..d628513be 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -1,191 +1,188 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
alias Pleroma.Web.RichMedia.Parser.Card
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
with {:ok, data} <- get_cached_or_parse(url),
{:ok, _} <- set_ttl_based_on_image(data, url) do
{:ok, data}
end
end
defp get_cached_or_parse(url) do
case @cachex.fetch(:rich_media_cache, url, fn ->
case parse_url(url) do
{:ok, _} = res ->
{:commit, res}
{:error, reason} = e ->
# Unfortunately we have to log errors here, instead of doing that
# along with ttl setting at the bottom. Otherwise we can get log spam
# if more than one process was waiting for the rich media card
# while it was generated. Ideally we would set ttl here as well,
# so we don't override it number_of_waiters_on_generation
# times, but one, obviously, can't set ttl for not-yet-created entry
# and Cachex doesn't support returning ttl from the fetch callback.
log_error(url, reason)
{:commit, e}
end
end) do
{action, res} when action in [:commit, :ok] ->
case res do
{:ok, _data} = res ->
res
{:error, reason} = e ->
if action == :commit, do: set_error_ttl(url, reason)
e
end
{:error, e} ->
{:error, {:cachex_error, e}}
end
end
defp set_error_ttl(_url, :body_too_large), do: :ok
defp set_error_ttl(_url, {:content_type, _}), do: :ok
# The TTL is not set for the errors above, since they are unlikely to change
# with time
defp set_error_ttl(url, _reason) do
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
@cachex.expire(:rich_media_cache, url, ttl)
:ok
end
defp log_error(url, {:invalid_metadata, data}) do
Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
end
defp log_error(url, reason) do
Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
end
end
@doc """
Set the rich media cache based on the expiration time of image.
Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
## Example
defmodule MyModule do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, url) do
image_url = Map.get(data, :image)
# do some parsing in the url and get the ttl of the image
# and return ttl is unix time
parse_ttl_from_url(image_url)
end
end
Define the module in the config
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
@spec set_ttl_based_on_image(map(), String.t()) ::
{:ok, Integer.t() | :noop} | {:error, :no_key}
def set_ttl_based_on_image(data, url) do
case get_ttl_from_image(data, url) do
{:ok, ttl} when is_number(ttl) ->
ttl = ttl * 1000
case @cachex.expire_at(:rich_media_cache, url, ttl) do
{:ok, true} -> {:ok, ttl}
{:ok, false} -> {:error, :no_key}
end
_ ->
{:ok, :noop}
end
end
defp get_ttl_from_image(data, url) do
[:rich_media, :ttl_setters]
|> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
_, error ->
error
end)
end
def parse_url(url) do
case maybe_fetch_oembed(url) do
{:ok, %Card{} = card} -> {:ok, card}
_ -> fetch_document(url)
end
end
defp maybe_fetch_oembed(url) do
with {:ok, oembed_url} <- OEmbedProviders.oembed_url(url),
{:ok, %Tesla.Env{body: json}} <-
Pleroma.Web.RichMedia.Helpers.oembed_get(oembed_url),
{:ok, data} <- Jason.decode(json),
%Card{} = card <- Card.from_oembed(data, url) do
{:ok, card}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp fetch_document(url) do
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html) do
html
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
|> Card.from_discovery(url)
|> check_card()
end
end
defp maybe_parse(html) do
- Enum.reduce_while(parsers(), %{}, fn parser, acc ->
- case parser.parse(html, acc) do
- data when data != %{} -> {:halt, data}
- _ -> {:cont, acc}
- end
+ Enum.reduce(parsers(), %{}, fn parser, acc ->
+ parser.parse(html, acc)
end)
end
defp check_card(%Card{title: title} = card)
when is_binary(title) and title != "" do
{:ok, card}
end
defp check_card(card) do
{:error, {:invalid_metadata, card}}
end
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
not match?({:ok, _}, Jason.encode(%{key => val}))
end)
|> Map.new()
end
end
diff --git a/lib/pleroma/web/rich_media/parser/meta_tags.ex b/lib/pleroma/web/rich_media/parser/meta_tags.ex
new file mode 100644
index 000000000..888ac3fc4
--- /dev/null
+++ b/lib/pleroma/web/rich_media/parser/meta_tags.ex
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.MetaTags do
+ @moduledoc """
+ Parses a `Floki.html_tree/0` and returns a map of raw `<meta>` tag values.
+ """
+ @spec parse(html_tree :: Floki.html_tree()) :: map()
+ def parse(html_tree) do
+ html_tree
+ |> Floki.find("meta")
+ |> Enum.reduce(%{}, fn html_node, acc ->
+ case parse_node(html_node) do
+ {:ok, {name, content}} -> Map.put(acc, name, content)
+ _ -> acc
+ end
+ end)
+ end
+
+ defp parse_node({_tag, attrs, _children}) when is_list(attrs) do
+ case Map.new(attrs) do
+ %{"name" => name, "content" => content} -> {:ok, {name, content}}
+ %{"property" => name, "content" => content} -> {:ok, {name, content}}
+ _ -> {:error, :invalid_meta_tag}
+ end
+ end
+
+ defp parse_node(_), do: {:error, :invalid_meta_tag}
+end
diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
index 31c3d1e33..5375037b3 100644
--- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
@@ -1,46 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
- def parse(data, html, prefix, key_name, value_name \\ "content") do
+ def parse(html, prefix, key_name, value_name \\ "content") do
html
|> get_elements(key_name, prefix)
- |> Enum.reduce(data, fn el, acc ->
- attributes = normalize_attributes(el, prefix, key_name, value_name)
-
+ |> Enum.reduce(%{}, fn el, acc ->
+ attributes = normalize_attributes(el, key_name, value_name)
Map.merge(acc, attributes)
end)
- |> maybe_put_title(html)
+ end
+
+ defp get_elements(html, key_names, prefix) when is_list(key_names) do
+ Enum.reduce(key_names, [], fn key_name, acc ->
+ acc ++ Floki.find(html, "meta[#{key_name}^='#{prefix}:']")
+ end)
end
defp get_elements(html, key_name, prefix) do
- html |> Floki.find("meta[#{key_name}^='#{prefix}:']")
+ get_elements(html, [key_name], prefix)
end
- defp normalize_attributes(html_node, prefix, key_name, value_name) do
+ defp normalize_attributes(html_node, key_names, value_name) when is_list(key_names) do
{_tag, attributes, _children} = html_node
+ data = Map.new(attributes)
- data =
- Map.new(attributes, fn {name, value} ->
- {name, String.trim_leading(value, "#{prefix}:")}
- end)
-
- %{data[key_name] => data[value_name]}
+ Enum.reduce(key_names, %{}, fn key_name, acc ->
+ if data[key_name], do: Map.put(acc, data[key_name], data[value_name]), else: acc
+ end)
end
- defp maybe_put_title(%{"title" => _} = meta, _), do: meta
-
- defp maybe_put_title(meta, html) when meta != %{} do
- case get_page_title(html) do
- "" -> meta
- title -> Map.put_new(meta, "title", title)
- end
+ defp normalize_attributes(html_node, key_name, value_name) do
+ normalize_attributes(html_node, [key_name], value_name)
end
- defp maybe_put_title(meta, _), do: meta
-
- defp get_page_title(html) do
+ def get_page_title(html) do
Floki.find(html, "html head title") |> List.first() |> Floki.text()
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex
index 8dc378c3c..4f7cd1260 100644
--- a/lib/pleroma/web/rich_media/parsers/o_embed.ex
+++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex
@@ -1,29 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
- def parse(html, _data) do
+ def parse(html, data) do
with elements = [_ | _] <- get_discovery_data(html),
oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
{:ok, oembed_data} <- get_oembed_data(oembed_url) do
- oembed_data
+ Map.put(data, :oembed, oembed_data)
else
- _e -> %{}
+ _e -> data
end
end
defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
end
defp get_oembed_url([{"link", attributes, _children} | _]) do
Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
end
defp get_oembed_data(url) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.oembed_get(url) do
Jason.decode(json)
end
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/twitter_card.ex b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
index 31546819e..c10b9370e 100644
--- a/lib/pleroma/web/rich_media/parsers/twitter_card.ex
+++ b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
@@ -1,16 +1,15 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
@spec parse(list(), map()) :: map()
def parse(html, data) do
data
- |> MetaTagsParser.parse(html, "og", "property")
- |> MetaTagsParser.parse(html, "twitter", "name")
- |> MetaTagsParser.parse(html, "twitter", "property")
- |> Map.put("type", "link")
+ |> Map.put(:title, MetaTagsParser.get_page_title(html))
+ |> Map.put(:opengraph, MetaTagsParser.parse(html, "og", "property"))
+ |> Map.put(:twitter, MetaTagsParser.parse(html, "twitter", ["name", "property"]))
end
end
diff --git a/test/pleroma/web/rich_media/parser/meta_tags_test.exs b/test/pleroma/web/rich_media/parser/meta_tags_test.exs
new file mode 100644
index 000000000..128c83a95
--- /dev/null
+++ b/test/pleroma/web/rich_media/parser/meta_tags_test.exs
@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.MetaTagsTest do
+ use ExUnit.Case, async: true
+ alias Pleroma.Web.RichMedia.Parser.MetaTags
+
+ test "returns a map of <meta> values" do
+ html =
+ File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
+ |> Floki.parse_document!()
+
+ expected = %{
+ "CG" => "nyregion",
+ "CN" => "experience-tech-and-society",
+ "CT" => "spotlight",
+ "PST" => "News",
+ "PT" => "article",
+ "SCG" => "",
+ "al:android:app_name" => "NYTimes",
+ "al:android:package" => "com.nytimes.android",
+ "al:android:url" => "nytimes://reader/id/100000006583622",
+ "al:ipad:app_name" => "NYTimes",
+ "al:ipad:app_store_id" => "357066198",
+ "al:ipad:url" =>
+ "nytimes://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "al:iphone:app_name" => "NYTimes",
+ "al:iphone:app_store_id" => "284862083",
+ "al:iphone:url" =>
+ "nytimes://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "article:modified" => "2019-08-02T09:30:23.000Z",
+ "article:published" => "2019-08-01T17:15:31.000Z",
+ "article:section" => "New York",
+ "article:tag" => "New York City",
+ "articleid" => "100000006583622",
+ "byl" => "By Joseph Goldstein and Ali Watkins",
+ "description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "fb:app_id" => "9869919170",
+ "image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "msapplication-starturl" => "https://www.nytimes.com",
+ "news_keywords" =>
+ "NYPD,Juvenile delinquency,Facial Recognition,Privacy,Government Surveillance,Police,Civil Rights,NYC",
+ "nyt_uri" => "nyt://article/9da58246-2495-505f-9abd-b5fda8e67b56",
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:type" => "article",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "pdate" => "20190801",
+ "pubp_event_id" => "pubp://event/47a657bafa8a476bb36832f90ee5ac6e",
+ "robots" => "noarchive",
+ "thumbnail" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-thumbStandard.jpg",
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "viewport" => "width=device-width, initial-scale=1, maximum-scale=1"
+ }
+
+ assert MetaTags.parse(html) == expected
+ end
+end
diff --git a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
index 2aacd29a3..909ce636d 100644
--- a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
@@ -1,127 +1,164 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
use ExUnit.Case, async: true
alias Pleroma.Web.RichMedia.Parsers.TwitterCard
test "returns error when html not contains twitter card" do
assert TwitterCard.parse([{"html", [], [{"head", [], []}, {"body", [], []}]}], %{}) == %{}
end
test "parses twitter card with only name attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers3.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "app:id:googleplay" => "com.nytimes.android",
- "app:name:googleplay" => "NYTimes",
- "app:url:googleplay" => "nytimes://reader/id/100000006583622",
- "site" => nil,
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
- "type" => "article",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database."
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:site" => nil
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:type" => "article",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "parses twitter card with only property attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers2.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "card" => "summary_large_image",
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
- "image:alt" => "",
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "type" => "article"
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "og:type" => "article"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "parses twitter card with name & property attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "app:id:googleplay" => "com.nytimes.android",
- "app:name:googleplay" => "NYTimes",
- "app:url:googleplay" => "nytimes://reader/id/100000006583622",
- "card" => "summary_large_image",
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
- "image:alt" => "",
- "site" => nil,
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "type" => "article"
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:site" => nil,
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "og:type" => "article"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "respect only first title tag on the page" do
image_path =
"https://assets.atlasobscura.com/media/W1siZiIsInVwbG9hZHMvYXNzZXRzLzkwYzgyMzI4LThlMDUtNGRiNS05MDg3LTUzMGUxZTM5N2RmMmVkOTM5ZDM4MGM4OTIx" <>
"YTQ5MF9EQVIgZXhodW1hdGlvbiBvZiBNYXJnYXJldCBDb3JiaW4gZ3JhdmUgMTkyNi5qcGciXSxbInAiLCJjb252ZXJ0IiwiIl0sWyJwIiwiY29udmVydCIsIi1xdWFsaXR5IDgxIC1hdXRvLW9" <>
"yaWVudCJdLFsicCIsInRodW1iIiwiNjAweD4iXV0/DAR%20exhumation%20of%20Margaret%20Corbin%20grave%201926.jpg"
html =
File.read!("test/fixtures/margaret-corbin-grave-west-point.html") |> Floki.parse_document!()
assert TwitterCard.parse(html, %{}) ==
%{
"site" => "@atlasobscura",
"title" => "The Missing Grave of Margaret Corbin, Revolutionary War Veteran",
"card" => "summary_large_image",
"image" => image_path,
"description" =>
"She's the only woman veteran honored with a monument at West Point. But where was she buried?",
"site_name" => "Atlas Obscura",
"type" => "article",
"url" => "http://www.atlasobscura.com/articles/margaret-corbin-grave-west-point"
}
end
test "takes first founded title in html head if there is html markup error" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers4.html")
|> Floki.parse_document!()
assert TwitterCard.parse(html, %{}) ==
%{
"site" => nil,
"title" =>
"She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
"app:id:googleplay" => "com.nytimes.android",
"app:name:googleplay" => "NYTimes",
"app:url:googleplay" => "nytimes://reader/id/100000006583622",
"description" =>
"With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
"image" =>
"https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
"type" => "article",
"url" =>
"https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
}
end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Nov 19, 4:43 AM (1 d, 21 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1840944
Default Alt Text
(26 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline