Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
26 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index dee0f61ac..d628513be 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -1,191 +1,188 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
alias Pleroma.Web.RichMedia.Parser.Card
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
with {:ok, data} <- get_cached_or_parse(url),
{:ok, _} <- set_ttl_based_on_image(data, url) do
{:ok, data}
end
end
defp get_cached_or_parse(url) do
case @cachex.fetch(:rich_media_cache, url, fn ->
case parse_url(url) do
{:ok, _} = res ->
{:commit, res}
{:error, reason} = e ->
# Unfortunately we have to log errors here, instead of doing that
# along with ttl setting at the bottom. Otherwise we can get log spam
# if more than one process was waiting for the rich media card
# while it was generated. Ideally we would set ttl here as well,
# so we don't override it number_of_waiters_on_generation
# times, but one, obviously, can't set ttl for not-yet-created entry
# and Cachex doesn't support returning ttl from the fetch callback.
log_error(url, reason)
{:commit, e}
end
end) do
{action, res} when action in [:commit, :ok] ->
case res do
{:ok, _data} = res ->
res
{:error, reason} = e ->
if action == :commit, do: set_error_ttl(url, reason)
e
end
{:error, e} ->
{:error, {:cachex_error, e}}
end
end
defp set_error_ttl(_url, :body_too_large), do: :ok
defp set_error_ttl(_url, {:content_type, _}), do: :ok
# The TTL is not set for the errors above, since they are unlikely to change
# with time
defp set_error_ttl(url, _reason) do
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
@cachex.expire(:rich_media_cache, url, ttl)
:ok
end
defp log_error(url, {:invalid_metadata, data}) do
Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
end
defp log_error(url, reason) do
Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
end
end
@doc """
Set the rich media cache based on the expiration time of image.
Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
## Example
defmodule MyModule do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
def ttl(data, url) do
image_url = Map.get(data, :image)
# do some parsing in the url and get the ttl of the image
# and return ttl is unix time
parse_ttl_from_url(image_url)
end
end
Define the module in the config
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
@spec set_ttl_based_on_image(map(), String.t()) ::
{:ok, Integer.t() | :noop} | {:error, :no_key}
def set_ttl_based_on_image(data, url) do
case get_ttl_from_image(data, url) do
{:ok, ttl} when is_number(ttl) ->
ttl = ttl * 1000
case @cachex.expire_at(:rich_media_cache, url, ttl) do
{:ok, true} -> {:ok, ttl}
{:ok, false} -> {:error, :no_key}
end
_ ->
{:ok, :noop}
end
end
defp get_ttl_from_image(data, url) do
[:rich_media, :ttl_setters]
|> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
_, error ->
error
end)
end
def parse_url(url) do
case maybe_fetch_oembed(url) do
{:ok, %Card{} = card} -> {:ok, card}
_ -> fetch_document(url)
end
end
defp maybe_fetch_oembed(url) do
with {:ok, oembed_url} <- OEmbedProviders.oembed_url(url),
{:ok, %Tesla.Env{body: json}} <-
Pleroma.Web.RichMedia.Helpers.oembed_get(oembed_url),
{:ok, data} <- Jason.decode(json),
%Card{} = card <- Card.from_oembed(data, url) do
{:ok, card}
else
{:error, error} -> {:error, error}
error -> {:error, error}
end
end
defp fetch_document(url) do
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html) do
html
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
|> Card.from_discovery(url)
|> check_card()
end
end
defp maybe_parse(html) do
- Enum.reduce_while(parsers(), %{}, fn parser, acc ->
- case parser.parse(html, acc) do
- data when data != %{} -> {:halt, data}
- _ -> {:cont, acc}
- end
+ Enum.reduce(parsers(), %{}, fn parser, acc ->
+ parser.parse(html, acc)
end)
end
defp check_card(%Card{title: title} = card)
when is_binary(title) and title != "" do
{:ok, card}
end
defp check_card(card) do
{:error, {:invalid_metadata, card}}
end
defp clean_parsed_data(data) do
data
|> Enum.reject(fn {key, val} ->
not match?({:ok, _}, Jason.encode(%{key => val}))
end)
|> Map.new()
end
end
diff --git a/lib/pleroma/web/rich_media/parser/meta_tags.ex b/lib/pleroma/web/rich_media/parser/meta_tags.ex
new file mode 100644
index 000000000..888ac3fc4
--- /dev/null
+++ b/lib/pleroma/web/rich_media/parser/meta_tags.ex
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.MetaTags do
+ @moduledoc """
+ Parses a `Floki.html_tree/0` and returns a map of raw `<meta>` tag values.
+ """
+ @spec parse(html_tree :: Floki.html_tree()) :: map()
+ def parse(html_tree) do
+ html_tree
+ |> Floki.find("meta")
+ |> Enum.reduce(%{}, fn html_node, acc ->
+ case parse_node(html_node) do
+ {:ok, {name, content}} -> Map.put(acc, name, content)
+ _ -> acc
+ end
+ end)
+ end
+
+ defp parse_node({_tag, attrs, _children}) when is_list(attrs) do
+ case Map.new(attrs) do
+ %{"name" => name, "content" => content} -> {:ok, {name, content}}
+ %{"property" => name, "content" => content} -> {:ok, {name, content}}
+ _ -> {:error, :invalid_meta_tag}
+ end
+ end
+
+ defp parse_node(_), do: {:error, :invalid_meta_tag}
+end
diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
index 31c3d1e33..5375037b3 100644
--- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
@@ -1,46 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
- def parse(data, html, prefix, key_name, value_name \\ "content") do
+ def parse(html, prefix, key_name, value_name \\ "content") do
html
|> get_elements(key_name, prefix)
- |> Enum.reduce(data, fn el, acc ->
- attributes = normalize_attributes(el, prefix, key_name, value_name)
-
+ |> Enum.reduce(%{}, fn el, acc ->
+ attributes = normalize_attributes(el, key_name, value_name)
Map.merge(acc, attributes)
end)
- |> maybe_put_title(html)
+ end
+
+ defp get_elements(html, key_names, prefix) when is_list(key_names) do
+ Enum.reduce(key_names, [], fn key_name, acc ->
+ acc ++ Floki.find(html, "meta[#{key_name}^='#{prefix}:']")
+ end)
end
defp get_elements(html, key_name, prefix) do
- html |> Floki.find("meta[#{key_name}^='#{prefix}:']")
+ get_elements(html, [key_name], prefix)
end
- defp normalize_attributes(html_node, prefix, key_name, value_name) do
+ defp normalize_attributes(html_node, key_names, value_name) when is_list(key_names) do
{_tag, attributes, _children} = html_node
+ data = Map.new(attributes)
- data =
- Map.new(attributes, fn {name, value} ->
- {name, String.trim_leading(value, "#{prefix}:")}
- end)
-
- %{data[key_name] => data[value_name]}
+ Enum.reduce(key_names, %{}, fn key_name, acc ->
+ if data[key_name], do: Map.put(acc, data[key_name], data[value_name]), else: acc
+ end)
end
- defp maybe_put_title(%{"title" => _} = meta, _), do: meta
-
- defp maybe_put_title(meta, html) when meta != %{} do
- case get_page_title(html) do
- "" -> meta
- title -> Map.put_new(meta, "title", title)
- end
+ defp normalize_attributes(html_node, key_name, value_name) do
+ normalize_attributes(html_node, [key_name], value_name)
end
- defp maybe_put_title(meta, _), do: meta
-
- defp get_page_title(html) do
+ def get_page_title(html) do
Floki.find(html, "html head title") |> List.first() |> Floki.text()
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex
index 8dc378c3c..4f7cd1260 100644
--- a/lib/pleroma/web/rich_media/parsers/o_embed.ex
+++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex
@@ -1,29 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
- def parse(html, _data) do
+ def parse(html, data) do
with elements = [_ | _] <- get_discovery_data(html),
oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
{:ok, oembed_data} <- get_oembed_data(oembed_url) do
- oembed_data
+ Map.put(data, :oembed, oembed_data)
else
- _e -> %{}
+ _e -> data
end
end
defp get_discovery_data(html) do
html |> Floki.find("link[type='application/json+oembed']")
end
defp get_oembed_url([{"link", attributes, _children} | _]) do
Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
end
defp get_oembed_data(url) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.oembed_get(url) do
Jason.decode(json)
end
end
end
diff --git a/lib/pleroma/web/rich_media/parsers/twitter_card.ex b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
index 31546819e..c10b9370e 100644
--- a/lib/pleroma/web/rich_media/parsers/twitter_card.ex
+++ b/lib/pleroma/web/rich_media/parsers/twitter_card.ex
@@ -1,16 +1,15 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
@spec parse(list(), map()) :: map()
def parse(html, data) do
data
- |> MetaTagsParser.parse(html, "og", "property")
- |> MetaTagsParser.parse(html, "twitter", "name")
- |> MetaTagsParser.parse(html, "twitter", "property")
- |> Map.put("type", "link")
+ |> Map.put(:title, MetaTagsParser.get_page_title(html))
+ |> Map.put(:opengraph, MetaTagsParser.parse(html, "og", "property"))
+ |> Map.put(:twitter, MetaTagsParser.parse(html, "twitter", ["name", "property"]))
end
end
diff --git a/test/pleroma/web/rich_media/parser/meta_tags_test.exs b/test/pleroma/web/rich_media/parser/meta_tags_test.exs
new file mode 100644
index 000000000..128c83a95
--- /dev/null
+++ b/test/pleroma/web/rich_media/parser/meta_tags_test.exs
@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.MetaTagsTest do
+ use ExUnit.Case, async: true
+ alias Pleroma.Web.RichMedia.Parser.MetaTags
+
+ test "returns a map of <meta> values" do
+ html =
+ File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
+ |> Floki.parse_document!()
+
+ expected = %{
+ "CG" => "nyregion",
+ "CN" => "experience-tech-and-society",
+ "CT" => "spotlight",
+ "PST" => "News",
+ "PT" => "article",
+ "SCG" => "",
+ "al:android:app_name" => "NYTimes",
+ "al:android:package" => "com.nytimes.android",
+ "al:android:url" => "nytimes://reader/id/100000006583622",
+ "al:ipad:app_name" => "NYTimes",
+ "al:ipad:app_store_id" => "357066198",
+ "al:ipad:url" =>
+ "nytimes://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "al:iphone:app_name" => "NYTimes",
+ "al:iphone:app_store_id" => "284862083",
+ "al:iphone:url" =>
+ "nytimes://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "article:modified" => "2019-08-02T09:30:23.000Z",
+ "article:published" => "2019-08-01T17:15:31.000Z",
+ "article:section" => "New York",
+ "article:tag" => "New York City",
+ "articleid" => "100000006583622",
+ "byl" => "By Joseph Goldstein and Ali Watkins",
+ "description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "fb:app_id" => "9869919170",
+ "image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "msapplication-starturl" => "https://www.nytimes.com",
+ "news_keywords" =>
+ "NYPD,Juvenile delinquency,Facial Recognition,Privacy,Government Surveillance,Police,Civil Rights,NYC",
+ "nyt_uri" => "nyt://article/9da58246-2495-505f-9abd-b5fda8e67b56",
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:type" => "article",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "pdate" => "20190801",
+ "pubp_event_id" => "pubp://event/47a657bafa8a476bb36832f90ee5ac6e",
+ "robots" => "noarchive",
+ "thumbnail" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-thumbStandard.jpg",
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "viewport" => "width=device-width, initial-scale=1, maximum-scale=1"
+ }
+
+ assert MetaTags.parse(html) == expected
+ end
+end
diff --git a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
index 2aacd29a3..909ce636d 100644
--- a/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/pleroma/web/rich_media/parsers/twitter_card_test.exs
@@ -1,127 +1,164 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
use ExUnit.Case, async: true
alias Pleroma.Web.RichMedia.Parsers.TwitterCard
test "returns error when html not contains twitter card" do
assert TwitterCard.parse([{"html", [], [{"head", [], []}, {"body", [], []}]}], %{}) == %{}
end
test "parses twitter card with only name attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers3.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "app:id:googleplay" => "com.nytimes.android",
- "app:name:googleplay" => "NYTimes",
- "app:url:googleplay" => "nytimes://reader/id/100000006583622",
- "site" => nil,
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
- "type" => "article",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database."
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:site" => nil
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:type" => "article",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "parses twitter card with only property attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers2.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "card" => "summary_large_image",
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
- "image:alt" => "",
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "type" => "article"
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "og:type" => "article"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "parses twitter card with name & property attributes" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
|> Floki.parse_document!()
- assert TwitterCard.parse(html, %{}) ==
- %{
- "app:id:googleplay" => "com.nytimes.android",
- "app:name:googleplay" => "NYTimes",
- "app:url:googleplay" => "nytimes://reader/id/100000006583622",
- "card" => "summary_large_image",
- "description" =>
- "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
- "image" =>
- "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
- "image:alt" => "",
- "site" => nil,
- "title" =>
- "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
- "url" =>
- "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
- "type" => "article"
- }
+ expected = %{
+ title:
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times",
+ twitter: %{
+ "twitter:app:id:googleplay" => "com.nytimes.android",
+ "twitter:app:name:googleplay" => "NYTimes",
+ "twitter:app:url:googleplay" => "nytimes://reader/id/100000006583622",
+ "twitter:card" => "summary_large_image",
+ "twitter:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "twitter:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg",
+ "twitter:image:alt" => "",
+ "twitter:site" => nil,
+ "twitter:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "twitter:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
+ },
+ opengraph: %{
+ "og:description" =>
+ "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
+ "og:image" =>
+ "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
+ "og:title" =>
+ "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
+ "og:url" =>
+ "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html",
+ "og:type" => "article"
+ }
+ }
+
+ assert TwitterCard.parse(html, %{}) == expected
end
test "respect only first title tag on the page" do
image_path =
"https://assets.atlasobscura.com/media/W1siZiIsInVwbG9hZHMvYXNzZXRzLzkwYzgyMzI4LThlMDUtNGRiNS05MDg3LTUzMGUxZTM5N2RmMmVkOTM5ZDM4MGM4OTIx" <>
"YTQ5MF9EQVIgZXhodW1hdGlvbiBvZiBNYXJnYXJldCBDb3JiaW4gZ3JhdmUgMTkyNi5qcGciXSxbInAiLCJjb252ZXJ0IiwiIl0sWyJwIiwiY29udmVydCIsIi1xdWFsaXR5IDgxIC1hdXRvLW9" <>
"yaWVudCJdLFsicCIsInRodW1iIiwiNjAweD4iXV0/DAR%20exhumation%20of%20Margaret%20Corbin%20grave%201926.jpg"
html =
File.read!("test/fixtures/margaret-corbin-grave-west-point.html") |> Floki.parse_document!()
assert TwitterCard.parse(html, %{}) ==
%{
"site" => "@atlasobscura",
"title" => "The Missing Grave of Margaret Corbin, Revolutionary War Veteran",
"card" => "summary_large_image",
"image" => image_path,
"description" =>
"She's the only woman veteran honored with a monument at West Point. But where was she buried?",
"site_name" => "Atlas Obscura",
"type" => "article",
"url" => "http://www.atlasobscura.com/articles/margaret-corbin-grave-west-point"
}
end
test "takes first founded title in html head if there is html markup error" do
html =
File.read!("test/fixtures/nypd-facial-recognition-children-teenagers4.html")
|> Floki.parse_document!()
assert TwitterCard.parse(html, %{}) ==
%{
"site" => nil,
"title" =>
"She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.",
"app:id:googleplay" => "com.nytimes.android",
"app:name:googleplay" => "NYTimes",
"app:url:googleplay" => "nytimes://reader/id/100000006583622",
"description" =>
"With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.",
"image" =>
"https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-facebookJumbo.jpg",
"type" => "article",
"url" =>
"https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html"
}
end
end

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 19, 4:43 AM (1 d, 21 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1840944
Default Alt Text
(26 KB)

Event Timeline