Page MenuHomePhorge
Files F85651396

No OneTemporary
Actions

Size
18 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/fast_html.ex b/lib/fast_html.ex
index ac76c27..58ec125 100644
--- a/lib/fast_html.ex
+++ b/lib/fast_html.ex
@@ -1,102 +1,102 @@
defmodule :fast_html do
@moduledoc """
A module to decode html into a tree structure.
Based on [Alexander Borisov's myhtml](https://github.com/lexborisov/myhtml),
this binding gains the properties of being html-spec compliant and very fast.
## Example
iex> :fast_html.decode("<h1>Hello world</h1>")
- {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}}
Benchmark results (removed Nif calling mode) on various file sizes on a 2,5Ghz Core i7:
Settings:
duration: 1.0 s
## FileSizesBench
[15:28:42] 1/3: github_trending_js.html 341k
[15:28:46] 2/3: w3c_html5.html 131k
[15:28:48] 3/3: wikipedia_hyperlink.html 97k
Finished in 7.52 seconds
## FileSizesBench
benchmark name iterations average time
wikipedia_hyperlink.html 97k 1000 1385.86 µs/op
w3c_html5.html 131k 1000 2179.30 µs/op
github_trending_js.html 341k 500 5686.21 µs/op
"""
@type tag() :: String.t() | atom()
@type attr() :: {String.t(), String.t()}
@type attr_list() :: [] | [attr()]
@type comment_node() :: {:comment, String.t()}
@type comment_node3() :: {:comment, [], String.t()}
@type tree() ::
{tag(), attr_list(), tree()}
| {tag(), attr_list(), nil}
| comment_node()
| comment_node3()
@type format_flag() :: :html_atoms | :nil_self_closing | :comment_tuple3
@doc """
Returns a tree representation from the given html string.
## Examples
iex> :fast_html.decode("<h1>Hello world</h1>")
- {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}}
iex> :fast_html.decode("<span class='hello'>Hi there</span>")
- {"html", [],
+ {:ok, {"html", [],
[{"head", [], []},
- {"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}
+ {"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}}
iex> :fast_html.decode("<body><!-- a comment --!></body>")
- {"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}}
iex> :fast_html.decode("<br>")
- {"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}}
"""
- @spec decode(String.t()) :: tree()
+ @spec decode(String.t()) :: {:ok, tree()} | {:error, String.t() | atom()}
def decode(bin) do
decode(bin, format: [])
end
@doc """
Returns a tree representation from the given html string.
This variant allows you to pass in one or more of the following format flags:
* `:html_atoms` uses atoms for known html tags (faster), binaries for everything else.
* `:nil_self_closing` uses `nil` to designate self-closing tags and void elements.
For example `<br>` is then being represented like `{"br", [], nil}`.
See http://w3c.github.io/html-reference/syntax.html#void-elements for a full list of void elements.
* `:comment_tuple3` uses 3-tuple elements for comments, instead of the default 2-tuple element.
## Examples
iex> :fast_html.decode("<h1>Hello world</h1>", format: [:html_atoms])
- {:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}
+ {:ok, {:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}}
iex> :fast_html.decode("<br>", format: [:nil_self_closing])
- {"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}}
iex> :fast_html.decode("<body><!-- a comment --!></body>", format: [:comment_tuple3])
- {"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}
+ {:ok, {"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}}
iex> html = "<body><!-- a comment --!><unknown /></body>"
iex> :fast_html.decode(html, format: [:html_atoms, :nil_self_closing, :comment_tuple3])
- {:html, [],
+ {:ok, {:html, [],
[{:head, [], []},
- {:body, [], [{:comment, [], " a comment "}, {"unknown", [], nil}]}]}
+ {:body, [], [{:comment, [], " a comment "}, {"unknown", [], nil}]}]}}
"""
- @spec decode(String.t(), format: [format_flag()]) :: tree()
+ @spec decode(String.t(), format: [format_flag()]) ::
+ {:ok, tree()} | {:error, String.t() | atom()}
def decode(bin, format: flags) do
- {:ok, res} = FastHtml.Cnode.call({:decode, bin, flags})
- res
+ FastHtml.Cnode.call({:decode, bin, flags})
end
end
diff --git a/lib/fast_html/application.ex b/lib/fast_html/application.ex
index ba4c0db..192dab4 100644
--- a/lib/fast_html/application.ex
+++ b/lib/fast_html/application.ex
@@ -1,65 +1,68 @@
defmodule FastHtml.Application do
@moduledoc false
use Application
def random_sname, do: :crypto.strong_rand_bytes(4) |> Base.encode16(case: :lower)
def start(_type, _args) do
case maybe_setup_node() do
{:error, message} -> raise message
_ -> :ok
end
- Supervisor.start_link([FastHtml.Cnode], strategy: :one_for_one, name: FastHtml.Supervisor)
+ Supervisor.start_link([{FastHtml.Cnode, Application.get_env(:fast_html, :cnode, [])}],
+ strategy: :one_for_one,
+ name: FastHtml.Supervisor
+ )
end
defp maybe_setup_node() do
with {_, false} <- {:alive, Node.alive?()},
{:ok, epmd_path} <- find_epmd(),
:ok <- start_epmd(epmd_path),
{:ok, _pid} = pid_tuple <- start_node() do
pid_tuple
else
{:alive, _} ->
:ok
{:error, _} = e ->
e
end
end
defp find_epmd() do
case System.find_executable("epmd") do
nil ->
{:error,
"Could not find epmd executable. Please ensure the location it's in is present in your PATH or start epmd manually beforehand"}
executable ->
{:ok, executable}
end
end
defp start_epmd(path) do
case System.cmd(path, ["-daemon"]) do
{_result, 0} -> :ok
{_result, exit_code} -> {:error, "Could not start epmd, exit code: #{exit_code}"}
end
end
defp hostname() do
{:ok, ifaddrs} = :inet.getifaddrs()
ifaddrs
|> Enum.filter(fn {_name, value} -> :loopback in Keyword.get(value, :flags) end)
|> Enum.at(0)
|> elem(1)
|> Keyword.get(:addr)
|> :inet.ntoa()
|> to_string()
end
defp start_node() do
Node.start(:"master_#{random_sname()}@#{hostname()}")
end
end
diff --git a/lib/fast_html/cnode.ex b/lib/fast_html/cnode.ex
index c1faa6e..209d125 100644
--- a/lib/fast_html/cnode.ex
+++ b/lib/fast_html/cnode.ex
@@ -1,137 +1,146 @@
defmodule FastHtml.Cnode do
- @moduledoc false
+ @moduledoc """
+ Manages myhtml c-node.
+
+ ## Configuration
+ ```elixir
+ config :fast_html, :cnode,
+ sname: "myhtml_worker", # Defaults to myhtml_<random bytes>
+ spawn_inactive_timeout: 5000 # Defaults to 10000
+ ```
+ """
@spawn_inactive_timeout 10000
application = Mix.Project.config()[:app]
use GenServer
require Logger
+ @doc false
def start_link(args) do
GenServer.start_link(__MODULE__, args, name: __MODULE__)
end
+ @doc false
def init(args) do
- args =
- if args == [] do
- %{}
- else
- args
- end
-
exec_path = Path.join(:code.priv_dir(unquote(application)), "myhtml_worker")
- sname = Map.get_lazy(args, :sname, &default_sname/0)
- hostname = Map.get_lazy(args, :hostname, &master_hostname/0)
+ sname = Keyword.get_lazy(args, :sname, &default_sname/0)
+ hostname = master_hostname()
addr = :"#{sname}@#{hostname}"
- spawn_inactive_timeout = Map.get(args, :spawn_inactive_timeout, @spawn_inactive_timeout)
+ spawn_inactive_timeout = Keyword.get(args, :spawn_inactive_timeout, @spawn_inactive_timeout)
state = %{
exec_path: exec_path,
sname: sname,
addr: addr,
hostname: hostname,
spawn_inactive_timeout: spawn_inactive_timeout
}
connect_or_spawn_cnode(state)
end
defp default_sname, do: "myhtml_#{FastHtml.Application.random_sname()}"
defp master_sname, do: Node.self() |> to_string |> String.split("@") |> List.first()
defp master_hostname, do: Node.self() |> to_string |> String.split("@") |> List.last()
defp connect_or_spawn_cnode(state) do
case connect_cnode(state) do
{:stop, _} -> spawn_cnode(state)
{:ok, state} -> state
end
end
defp connect_cnode(%{addr: addr} = state) do
if Node.connect(addr) do
Logger.debug("connected to #{addr}")
{:ok, state}
else
Logger.debug("connecting to #{addr} failed")
{:stop, :cnode_connection_fail}
end
end
defp spawn_cnode(%{exec_path: exec_path, sname: sname, hostname: hostname} = state) do
Logger.debug("Spawning #{sname}@#{hostname}")
cookie = :erlang.get_cookie()
port =
Port.open({:spawn_executable, exec_path}, [
:binary,
:exit_status,
:stderr_to_stdout,
line: 4096,
args: [sname, hostname, cookie, master_sname()]
])
pid = Keyword.get(Port.info(port), :os_pid)
state = Map.put(state, :pid, pid)
await_cnode_ready(port, state)
end
defp await_cnode_ready(
port,
%{spawn_inactive_timeout: timeout, addr: addr} = state
) do
ready_line = to_string(addr) <> " ready"
receive do
{^port, {:data, {:eol, ^ready_line}}} ->
connect_cnode(state)
{^port, {:data, {:eol, line}}} ->
Logger.debug("c-node is saying: #{line}")
await_cnode_ready(port, state)
{^port, {:exit_status, exit_status}} ->
Logger.debug("unexpected c-node exit: #{exit_status}")
{:stop, :cnode_unexpected_exit}
message ->
Logger.warn("unhandled message while waiting for cnode to be ready:\n#{inspect(message)}")
await_cnode_ready(port, state)
after
timeout ->
{:stop, :spawn_inactive_timeout}
end
end
+ @doc false
def handle_info({:nodedown, _cnode}, state) do
{:stop, :nodedown, state}
end
+ @doc false
def handle_info(msg, state) do
Logger.warn("unhandled handle_info: #{inspect(msg)}")
{:noreply, state}
end
+ @doc false
def handle_call(:addr, _from, %{addr: addr} = state) do
{:reply, addr, state}
end
+ @doc false
def terminate(_reason, %{pid: pid}) when pid != nil do
System.cmd("kill", ["-9", to_string(pid)])
:normal
end
+ @doc "Call into myhtml cnode"
def call(msg, timeout \\ 10000) do
node = GenServer.call(__MODULE__, :addr)
send({nil, node}, msg)
receive do
{:myhtml_worker, res} -> {:ok, res}
after
timeout -> {:error, :timeout}
end
end
end
diff --git a/test/fast_html_test.exs b/test/fast_html_test.exs
index 304edd0..60f46bc 100644
--- a/test/fast_html_test.exs
+++ b/test/fast_html_test.exs
@@ -1,140 +1,152 @@
defmodule :fast_html_test do
use ExUnit.Case
doctest :fast_html
test "doesn't segfault when <!----> is encountered" do
- assert {"html", _attrs, _children} = :fast_html.decode("<div> <!----> </div>")
+ assert {:ok, {"html", _attrs, _children}} = :fast_html.decode("<div> <!----> </div>")
end
test "builds a tree, formatted like mochiweb by default" do
- assert {"html", [],
- [
- {"head", [], []},
- {"body", [],
- [
- {"br", [], []}
- ]}
- ]} = :fast_html.decode("<br>")
+ assert {:ok,
+ {"html", [],
+ [
+ {"head", [], []},
+ {"body", [],
+ [
+ {"br", [], []}
+ ]}
+ ]}} = :fast_html.decode("<br>")
end
test "builds a tree, html tags as atoms" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {:br, [], []}
- ]}
- ]} = :fast_html.decode("<br>", format: [:html_atoms])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {:br, [], []}
+ ]}
+ ]}} = :fast_html.decode("<br>", format: [:html_atoms])
end
test "builds a tree, nil self closing" do
- assert {"html", [],
- [
- {"head", [], []},
- {"body", [],
- [
- {"br", [], nil},
- {"esi:include", [], nil}
- ]}
- ]} = :fast_html.decode("<br><esi:include />", format: [:nil_self_closing])
+ assert {:ok,
+ {"html", [],
+ [
+ {"head", [], []},
+ {"body", [],
+ [
+ {"br", [], nil},
+ {"esi:include", [], nil}
+ ]}
+ ]}} = :fast_html.decode("<br><esi:include />", format: [:nil_self_closing])
end
test "builds a tree, multiple format options" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {:br, [], nil}
- ]}
- ]} = :fast_html.decode("<br>", format: [:html_atoms, :nil_self_closing])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {:br, [], nil}
+ ]}
+ ]}} = :fast_html.decode("<br>", format: [:html_atoms, :nil_self_closing])
end
test "attributes" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {:span, [{"id", "test"}, {"class", "foo garble"}], []}
- ]}
- ]} =
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {:span, [{"id", "test"}, {"class", "foo garble"}], []}
+ ]}
+ ]}} =
:fast_html.decode(~s'<span id="test" class="foo garble"></span>',
format: [:html_atoms]
)
end
test "single attributes" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {:button, [{"disabled", "disabled"}, {"class", "foo garble"}], []}
- ]}
- ]} =
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {:button, [{"disabled", "disabled"}, {"class", "foo garble"}], []}
+ ]}
+ ]}} =
:fast_html.decode(~s'<button disabled class="foo garble"></span>',
format: [:html_atoms]
)
end
test "text nodes" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- "text node"
- ]}
- ]} = :fast_html.decode(~s'<body>text node</body>', format: [:html_atoms])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ "text node"
+ ]}
+ ]}} = :fast_html.decode(~s'<body>text node</body>', format: [:html_atoms])
end
test "broken input" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {:a, [{"<", "<"}], [" asdf"]}
- ]}
- ]} = :fast_html.decode(~s'<a <> asdf', format: [:html_atoms])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {:a, [{"<", "<"}], [" asdf"]}
+ ]}
+ ]}} = :fast_html.decode(~s'<a <> asdf', format: [:html_atoms])
end
test "namespaced tags" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {"svg:svg", [],
- [
- {"svg:path", [], []},
- {"svg:a", [], []}
- ]}
- ]}
- ]} = :fast_html.decode(~s'<svg><path></path><a></a></svg>', format: [:html_atoms])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {"svg:svg", [],
+ [
+ {"svg:path", [], []},
+ {"svg:a", [], []}
+ ]}
+ ]}
+ ]}} = :fast_html.decode(~s'<svg><path></path><a></a></svg>', format: [:html_atoms])
end
test "custom namespaced tags" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- {"esi:include", [], nil}
- ]}
- ]} = :fast_html.decode(~s'<esi:include />', format: [:html_atoms, :nil_self_closing])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ {"esi:include", [], nil}
+ ]}
+ ]}} =
+ :fast_html.decode(~s'<esi:include />', format: [:html_atoms, :nil_self_closing])
end
test "html comments" do
- assert {:html, [],
- [
- {:head, [], []},
- {:body, [],
- [
- comment: " a comment "
- ]}
- ]} = :fast_html.decode(~s'<body><!-- a comment --></body>', format: [:html_atoms])
+ assert {:ok,
+ {:html, [],
+ [
+ {:head, [], []},
+ {:body, [],
+ [
+ comment: " a comment "
+ ]}
+ ]}} = :fast_html.decode(~s'<body><!-- a comment --></body>', format: [:html_atoms])
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sun, Aug 30, 8:49 PM (1 d, 19 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1738153
Default Alt Text
(18 KB)

Event Timeline