Page MenuHomePhorge
Files F85823331

No OneTemporary
Actions

Authored By
Unknown
Size
14 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/fast_html.ex b/lib/fast_html.ex
index 6c66a6e..ac76c27 100644
--- a/lib/fast_html.ex
+++ b/lib/fast_html.ex
@@ -1,102 +1,102 @@
defmodule :fast_html do
@moduledoc """
A module to decode html into a tree structure.
Based on [Alexander Borisov's myhtml](https://github.com/lexborisov/myhtml),
this binding gains the properties of being html-spec compliant and very fast.
## Example
iex> :fast_html.decode("<h1>Hello world</h1>")
{"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
Benchmark results (removed Nif calling mode) on various file sizes on a 2,5Ghz Core i7:
Settings:
duration: 1.0 s
## FileSizesBench
[15:28:42] 1/3: github_trending_js.html 341k
[15:28:46] 2/3: w3c_html5.html 131k
[15:28:48] 3/3: wikipedia_hyperlink.html 97k
Finished in 7.52 seconds
## FileSizesBench
benchmark name iterations average time
wikipedia_hyperlink.html 97k 1000 1385.86 µs/op
w3c_html5.html 131k 1000 2179.30 µs/op
github_trending_js.html 341k 500 5686.21 µs/op
"""
@type tag() :: String.t() | atom()
@type attr() :: {String.t(), String.t()}
@type attr_list() :: [] | [attr()]
@type comment_node() :: {:comment, String.t()}
@type comment_node3() :: {:comment, [], String.t()}
@type tree() ::
{tag(), attr_list(), tree()}
| {tag(), attr_list(), nil}
| comment_node()
| comment_node3()
@type format_flag() :: :html_atoms | :nil_self_closing | :comment_tuple3
@doc """
Returns a tree representation from the given html string.
## Examples
iex> :fast_html.decode("<h1>Hello world</h1>")
{"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
iex> :fast_html.decode("<span class='hello'>Hi there</span>")
{"html", [],
[{"head", [], []},
{"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}
iex> :fast_html.decode("<body><!-- a comment --!></body>")
{"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}
iex> :fast_html.decode("<br>")
{"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}
"""
@spec decode(String.t()) :: tree()
def decode(bin) do
decode(bin, format: [])
end
@doc """
Returns a tree representation from the given html string.
This variant allows you to pass in one or more of the following format flags:
* `:html_atoms` uses atoms for known html tags (faster), binaries for everything else.
* `:nil_self_closing` uses `nil` to designate self-closing tags and void elements.
For example `<br>` is then being represented like `{"br", [], nil}`.
See http://w3c.github.io/html-reference/syntax.html#void-elements for a full list of void elements.
* `:comment_tuple3` uses 3-tuple elements for comments, instead of the default 2-tuple element.
## Examples
iex> :fast_html.decode("<h1>Hello world</h1>", format: [:html_atoms])
{:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}
iex> :fast_html.decode("<br>", format: [:nil_self_closing])
{"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}
iex> :fast_html.decode("<body><!-- a comment --!></body>", format: [:comment_tuple3])
{"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}
iex> html = "<body><!-- a comment --!><unknown /></body>"
iex> :fast_html.decode(html, format: [:html_atoms, :nil_self_closing, :comment_tuple3])
{:html, [],
[{:head, [], []},
{:body, [], [{:comment, [], " a comment "}, {"unknown", [], nil}]}]}
"""
@spec decode(String.t(), format: [format_flag()]) :: tree()
def decode(bin, format: flags) do
- {:ok, {:myhtml_worker, res}} = Nodex.Cnode.call(FastHtml.Cnode, {:decode, bin, flags})
+ {:ok, res} = FastHtml.Cnode.call({:decode, bin, flags})
res
end
end
diff --git a/lib/fast_html/application.ex b/lib/fast_html/application.ex
index 08243a0..48f6696 100644
--- a/lib/fast_html/application.ex
+++ b/lib/fast_html/application.ex
@@ -1,68 +1,53 @@
defmodule FastHtml.Application do
@moduledoc false
use Application
- application = Mix.Project.config()[:app]
-
- defp random_sname, do: :crypto.strong_rand_bytes(4) |> Base.encode16(case: :lower)
-
- defp sname, do: :"myhtml_#{random_sname()}"
+ def random_sname, do: :crypto.strong_rand_bytes(4) |> Base.encode16(case: :lower)
def start(_type, _args) do
- import Supervisor.Spec
-
case maybe_setup_node() do
{:error, message} -> raise message
_ -> :ok
end
- myhtml_worker = Path.join(:code.priv_dir(unquote(application)), "myhtml_worker")
-
- children = [
- worker(Nodex.Cnode, [
- %{exec_path: myhtml_worker, sname: sname()},
- [name: FastHtml.Cnode]
- ])
- ]
-
- Supervisor.start_link(children, strategy: :one_for_one, name: FastHtml.Supervisor)
+ Supervisor.start_link([FastHtml.Cnode], strategy: :one_for_one, name: FastHtml.Supervisor)
end
defp maybe_setup_node() do
with {_, false} <- {:alive, Node.alive?()},
{:ok, epmd_path} <- find_epmd(),
:ok <- start_epmd(epmd_path),
{:ok, _pid} = pid_tuple <- start_node() do
pid_tuple
else
{:alive, _} ->
:ok
{:error, _} = e ->
e
end
end
defp find_epmd() do
case System.find_executable("epmd") do
nil ->
{:error,
"Could not find epmd executable. Please ensure the location it's in is present in your PATH or start epmd manually beforehand"}
executable ->
{:ok, executable}
end
end
defp start_epmd(path) do
case System.cmd(path, ["-daemon"]) do
{_result, 0} -> :ok
{_result, exit_code} -> {:error, "Could not start epmd, exit code: #{exit_code}"}
end
end
defp start_node() do
Node.start(:"master_#{random_sname()}@127.0.0.1")
end
end
diff --git a/lib/fast_html/cnode.ex b/lib/fast_html/cnode.ex
new file mode 100644
index 0000000..c1faa6e
--- /dev/null
+++ b/lib/fast_html/cnode.ex
@@ -0,0 +1,137 @@
+defmodule FastHtml.Cnode do
+ @moduledoc false
+
+ @spawn_inactive_timeout 10000
+
+ application = Mix.Project.config()[:app]
+
+ use GenServer
+ require Logger
+
+ def start_link(args) do
+ GenServer.start_link(__MODULE__, args, name: __MODULE__)
+ end
+
+ def init(args) do
+ args =
+ if args == [] do
+ %{}
+ else
+ args
+ end
+
+ exec_path = Path.join(:code.priv_dir(unquote(application)), "myhtml_worker")
+
+ sname = Map.get_lazy(args, :sname, &default_sname/0)
+ hostname = Map.get_lazy(args, :hostname, &master_hostname/0)
+ addr = :"#{sname}@#{hostname}"
+
+ spawn_inactive_timeout = Map.get(args, :spawn_inactive_timeout, @spawn_inactive_timeout)
+
+ state = %{
+ exec_path: exec_path,
+ sname: sname,
+ addr: addr,
+ hostname: hostname,
+ spawn_inactive_timeout: spawn_inactive_timeout
+ }
+
+ connect_or_spawn_cnode(state)
+ end
+
+ defp default_sname, do: "myhtml_#{FastHtml.Application.random_sname()}"
+ defp master_sname, do: Node.self() |> to_string |> String.split("@") |> List.first()
+ defp master_hostname, do: Node.self() |> to_string |> String.split("@") |> List.last()
+
+ defp connect_or_spawn_cnode(state) do
+ case connect_cnode(state) do
+ {:stop, _} -> spawn_cnode(state)
+ {:ok, state} -> state
+ end
+ end
+
+ defp connect_cnode(%{addr: addr} = state) do
+ if Node.connect(addr) do
+ Logger.debug("connected to #{addr}")
+ {:ok, state}
+ else
+ Logger.debug("connecting to #{addr} failed")
+ {:stop, :cnode_connection_fail}
+ end
+ end
+
+ defp spawn_cnode(%{exec_path: exec_path, sname: sname, hostname: hostname} = state) do
+ Logger.debug("Spawning #{sname}@#{hostname}")
+
+ cookie = :erlang.get_cookie()
+
+ port =
+ Port.open({:spawn_executable, exec_path}, [
+ :binary,
+ :exit_status,
+ :stderr_to_stdout,
+ line: 4096,
+ args: [sname, hostname, cookie, master_sname()]
+ ])
+
+ pid = Keyword.get(Port.info(port), :os_pid)
+ state = Map.put(state, :pid, pid)
+ await_cnode_ready(port, state)
+ end
+
+ defp await_cnode_ready(
+ port,
+ %{spawn_inactive_timeout: timeout, addr: addr} = state
+ ) do
+ ready_line = to_string(addr) <> " ready"
+
+ receive do
+ {^port, {:data, {:eol, ^ready_line}}} ->
+ connect_cnode(state)
+
+ {^port, {:data, {:eol, line}}} ->
+ Logger.debug("c-node is saying: #{line}")
+ await_cnode_ready(port, state)
+
+ {^port, {:exit_status, exit_status}} ->
+ Logger.debug("unexpected c-node exit: #{exit_status}")
+ {:stop, :cnode_unexpected_exit}
+
+ message ->
+ Logger.warn("unhandled message while waiting for cnode to be ready:\n#{inspect(message)}")
+ await_cnode_ready(port, state)
+ after
+ timeout ->
+ {:stop, :spawn_inactive_timeout}
+ end
+ end
+
+ def handle_info({:nodedown, _cnode}, state) do
+ {:stop, :nodedown, state}
+ end
+
+ def handle_info(msg, state) do
+ Logger.warn("unhandled handle_info: #{inspect(msg)}")
+ {:noreply, state}
+ end
+
+ def handle_call(:addr, _from, %{addr: addr} = state) do
+ {:reply, addr, state}
+ end
+
+ def terminate(_reason, %{pid: pid}) when pid != nil do
+ System.cmd("kill", ["-9", to_string(pid)])
+ :normal
+ end
+
+ def call(msg, timeout \\ 10000) do
+ node = GenServer.call(__MODULE__, :addr)
+ send({nil, node}, msg)
+
+ receive do
+ {:myhtml_worker, res} -> {:ok, res}
+ after
+ timeout -> {:error, :timeout}
+ end
+ end
+end
diff --git a/mix.exs b/mix.exs
index ff8ad8d..27366d5 100644
--- a/mix.exs
+++ b/mix.exs
@@ -1,150 +1,146 @@
defmodule FastHTML.Mixfile do
use Mix.Project
def project do
[
app: :fast_html,
version: "0.2.1",
elixir: "~> 1.5",
deps: deps(),
package: package(),
compilers: [:fast_html_cnode_make] ++ Mix.compilers(),
build_embedded: Mix.env() == :prod,
start_permanent: Mix.env() == :prod,
name: "FastHTML",
description: """
A module to decode HTML into a tree,
porting all properties of the underlying
library myhtml, being fast and correct
in regards to the html spec.
Originally based on Myhtmlex.
""",
docs: docs()
]
end
def package do
[
maintainers: ["Ariadne Conill"],
licenses: ["GNU LGPL"],
links: %{
"GitLab" => "https://git.pleroma.social/pleroma/fast_html",
"Issues" => "https://git.pleroma.social/pleroma/fast_html/issues",
"MyHTML" => "https://github.com/lexborisov/myhtml"
},
files: [
"lib",
"c_src",
"priv/.gitignore",
"test",
"Makefile",
"mix.exs",
"README.md",
"LICENSE"
]
]
end
def application do
[
extra_applications: [:logger],
mod: {FastHtml.Application, []},
# used to detect conflicts with other applications named processes
registered: [FastHtml.Cnode, FastHtml.Supervisor]
]
end
defp deps do
[
# documentation helpers
{:ex_doc, ">= 0.0.0", only: :dev},
# benchmarking helpers
- {:benchee, "~> 1.0", only: :dev},
- # cnode helpers
- {:nodex,
- git: "https://git.pleroma.social/pleroma/nodex",
- ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"}
+ {:benchee, "~> 1.0", only: :dev}
]
end
defp docs do
[
main: "FastHTML"
]
end
end
defmodule Mix.Tasks.Compile.FastHtmlCnodeMake do
@artifacts [
"priv/myhtml_worker"
]
def find_make do
_make_cmd =
System.get_env("MAKE") ||
case :os.type() do
{:unix, :freebsd} -> "gmake"
{:unix, :openbsd} -> "gmake"
{:unix, :netbsd} -> "gmake"
{:unix, :dragonfly} -> "gmake"
_ -> "make"
end
end
defp otp_version do
:erlang.system_info(:otp_release)
|> to_string()
|> String.to_integer()
end
defp otp_22_or_newer? do
otp_version() >= 22
end
def run(_) do
make_cmd = find_make()
exit_code =
if match?({:win32, _}, :os.type()) do
IO.warn("Windows is not yet a target.")
1
else
{result, exit_code} =
System.cmd(
make_cmd,
@artifacts,
stderr_to_stdout: true,
env: [
{"MIX_ENV", to_string(Mix.env())},
{"OTP22_DEF", (otp_22_or_newer?() && "YES") || "NO"}
]
)
IO.binwrite(result)
exit_code
end
if exit_code == 0 do
:ok
else
{:error,
[
%Mix.Task.Compiler.Diagnostic{
compiler_name: "FastHTML Cnode",
message: "Make exited with #{exit_code}",
severity: :error,
file: nil,
position: nil
}
]}
end
end
def clean() do
make_cmd = find_make()
{result, _error_code} = System.cmd(make_cmd, ["clean"], stderr_to_stdout: true)
Mix.shell().info(result)
:ok
end
end
diff --git a/mix.lock b/mix.lock
index 1686bba..7152a93 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,9 +1,8 @@
%{
"benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
- "cnodex": {:git, "https://github.com/Overbryd/cnodex.git", "c1c4cde21295db07f87bb74006ab5f7222720db9", []},
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
"earmark": {:hex, :earmark, "1.2.3", "206eb2e2ac1a794aa5256f3982de7a76bf4579ff91cb28d0e17ea2c9491e46a4", [:mix], [], "hexpm"},
"ex_doc": {:hex, :ex_doc, "0.16.3", "cd2a4cfe5d26e37502d3ec776702c72efa1adfa24ed9ce723bb565f4c30bd31a", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"},
"myhtml": {:git, "https://github.com/lexborisov/myhtml.git", "fe2cf577570666d058a2b7167c26d3384a758e19", [branch: "master"]},
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
}

File Metadata

Mime Type
text/x-diff
Expires
Thu, Oct 29, 4:18 AM (7 h, 20 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1797527
Default Alt Text
(14 KB)

Event Timeline