Page MenuHomePhorge
Files F85651379

No OneTemporary
Actions

Size
13 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/README.md b/README.md
index 16a70f6..e772c36 100644
--- a/README.md
+++ b/README.md
@@ -1,35 +1,25 @@
# FastHTML
A C Node wrapping lexborisov's [myhtml](https://github.com/lexborisov/myhtml).
Primarily used with [FastSanitize](https://git.pleroma.social/pleroma/fast_sanitize).
* Available as a hex package: `{:fast_html, "~> 0.1.0"}`
-* [Documentation](https://hexdocs.pm/fast_html/FastHTML.html)
+* [Documentation](https://hexdocs.pm/fast_html/fast_html.html)
-## Example
+## Benchmarks
- iex> :fast_html.decode("<h1>Hello world</h1>")
- {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
+The following table provides median times it takes to decode a string to a tree for html parsers that can be used from Elixir. Benchmarks were conducted on a machine with `Intel Core i7-3520M @ 2.90GHz` CPU and 16GB of RAM. The `mix fast_html.bench` task can be used for running the benchmark by yourself.
- Benchmark results (removed Nif calling mode) on various file sizes on a 2,5Ghz Core i7:
-
- Settings:
- duration: 1.0 s
-
- ## FileSizesBench
- [15:28:42] 1/3: github_trending_js.html 341k
- [15:28:46] 2/3: w3c_html5.html 131k
- [15:28:48] 3/3: wikipedia_hyperlink.html 97k
-
- Finished in 7.52 seconds
-
- ## FileSizesBench
- benchmark name iterations average time
- wikipedia_hyperlink.html 97k 1000 1385.86 µs/op
- w3c_html5.html 131k 1000 2179.30 µs/op
- github_trending_js.html 341k 500 5686.21 µs/op
+| File/Parser | fast_html (C-Node) | mochiweb_html (erlang) | html5ever (Rust NIF) | Myhtmlex (NIF) |
+|----------------------|--------------------|------------------------|----------------------|----------------|
+| document-large.html | 178.13 ms | 3471.70 ms | 799.20 ms | 402.64 ms |
+| document-medium.html | 2.85 ms | 26.58 ms | 9.06 ms | 3.72 ms |
+| document-small.html | 1.08 ms | 5.45 ms | 2.10 ms | 1.24 ms |
+| fragment-large.html | 1.50 ms | 10.91 ms | 6.03 ms | 1.91 ms |
+| fragment-small.html | 434.64 μs | 83.02 μs | 57.97 μs | 311.39 μs |
+The slowdown on `fragment-small.html` is due to C-Node overhead. Unlike html5ever and Myhtmlex in NIF mode, `fast_html` has the parser process isolated and communicates with it over the network, so even if a fatal crash in the parser happens, it won't bring down the entire VM.
## Contribution / Bug Reports
* Please make sure you do `git submodule update` after a checkout/pull
* The project aims to be fully tested
diff --git a/lib/fast_html.ex b/lib/fast_html.ex
index 387278c..7ed80c9 100644
--- a/lib/fast_html.ex
+++ b/lib/fast_html.ex
@@ -1,77 +1,74 @@
defmodule :fast_html do
@moduledoc """
A module to decode html into a tree structure.
-
- Based on [Alexander Borisov's myhtml](https://github.com/lexborisov/myhtml),
- this binding gains the properties of being html-spec compliant and very fast.
"""
@type tag() :: String.t() | atom()
@type attr() :: {String.t(), String.t()}
@type attr_list() :: [] | [attr()]
@type comment_node() :: {:comment, String.t()}
@type comment_node3() :: {:comment, [], String.t()}
@type tree() ::
{tag(), attr_list(), tree()}
| {tag(), attr_list(), nil}
| comment_node()
| comment_node3()
@type format_flag() :: :html_atoms | :nil_self_closing | :comment_tuple3
@doc """
Returns a tree representation from the given html string.
`opts` is a keyword list of options, the options available:
* `timeout` - Call timeout
* `format` - Format flags for the tree
The following format flags are available:
* `:html_atoms` uses atoms for known html tags (faster), binaries for everything else.
* `:nil_self_closing` uses `nil` to designate self-closing tags and void elements.
For example `<br>` is then being represented like `{"br", [], nil}`.
See http://w3c.github.io/html-reference/syntax.html#void-elements for a full list of void elements.
* `:comment_tuple3` uses 3-tuple elements for comments, instead of the default 2-tuple element.
## Examples
iex> :fast_html.decode("<h1>Hello world</h1>")
{:ok, {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}}
iex> :fast_html.decode("Hello world", timeout: 0)
{:error, :timeout}
iex> :fast_html.decode("<span class='hello'>Hi there</span>")
{:ok, {"html", [],
[{"head", [], []},
{"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}}
iex> :fast_html.decode("<body><!-- a comment --!></body>")
{:ok, {"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}}
iex> :fast_html.decode("<br>")
{:ok, {"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}}
iex> :fast_html.decode("<h1>Hello world</h1>", format: [:html_atoms])
{:ok, {:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}}
iex> :fast_html.decode("<br>", format: [:nil_self_closing])
{:ok, {"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}}
iex> :fast_html.decode("<body><!-- a comment --!></body>", format: [:comment_tuple3])
{:ok, {"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}}
iex> html = "<body><!-- a comment --!><unknown /></body>"
iex> :fast_html.decode(html, format: [:html_atoms, :nil_self_closing, :comment_tuple3])
{:ok, {:html, [],
[{:head, [], []},
{:body, [], [{:comment, [], " a comment "}, {"unknown", [], nil}]}]}}
"""
@spec decode(String.t(), format: [format_flag()]) ::
{:ok, tree()} | {:error, String.t() | atom()}
def decode(bin, opts \\ []) do
flags = Keyword.get(opts, :format, [])
timeout = Keyword.get(opts, :timeout, 10000)
FastHtml.Cnode.call({:decode, bin, flags}, timeout)
end
end
diff --git a/lib/mix/tasks/fast_html/bench.ex b/lib/mix/tasks/fast_html/bench.ex
index 8e2cf8b..98dc778 100644
--- a/lib/mix/tasks/fast_html/bench.ex
+++ b/lib/mix/tasks/fast_html/bench.ex
@@ -1,26 +1,29 @@
defmodule Mix.Tasks.FastHtml.Bench do
@moduledoc "Benchmarking task."
use Mix.Task
@input_dir "lib/mix/tasks/fast_html/html"
def run(_) do
Application.ensure_all_started(:fast_html)
inputs =
Enum.reduce(File.ls!(@input_dir), %{}, fn input_name, acc ->
input = File.read!(Path.join(@input_dir, input_name))
Map.put(acc, input_name, input)
end)
Benchee.run(
%{
- "Decoding" => fn input -> :fast_html.decode(input) end
+ "fast_html" => fn input -> :fast_html.decode(input) end,
+ "myhtmlex nif" => fn input -> Myhtmlex.Nif.decode(input) end,
+ "html5ever nif" => fn input -> Html5ever.parse(input) end,
+ "mochiweb_html" => fn input -> :mochiweb_html.parse(input) end
},
inputs: inputs,
save: [path: "fast_html.bench"],
load: "fast_html.bench"
)
end
end
diff --git a/mix.exs b/mix.exs
index 30067d3..6b65d08 100644
--- a/mix.exs
+++ b/mix.exs
@@ -1,146 +1,150 @@
defmodule FastHtml.Mixfile do
use Mix.Project
def project do
[
app: :fast_html,
version: "0.9.2",
elixir: "~> 1.5",
deps: deps(),
package: package(),
compilers: [:fast_html_cnode_make] ++ Mix.compilers(),
build_embedded: Mix.env() == :prod,
start_permanent: Mix.env() == :prod,
name: "FastHtml",
description: """
A module to decode HTML into a tree,
porting all properties of the underlying
library myhtml, being fast and correct
in regards to the html spec.
Originally based on Myhtmlex.
""",
docs: docs()
]
end
def package do
[
maintainers: ["Ariadne Conill", "rinpatch"],
licenses: ["GNU LGPL"],
links: %{
"GitLab" => "https://git.pleroma.social/pleroma/fast_html",
"Issues" => "https://git.pleroma.social/pleroma/fast_html/issues",
"MyHTML" => "https://github.com/lexborisov/myhtml"
},
files: [
"lib",
"c_src",
"priv/.gitignore",
"test",
"Makefile",
"mix.exs",
"README.md",
"LICENSE"
]
]
end
def application do
[
extra_applications: [:logger],
mod: {FastHtml.Application, []},
# used to detect conflicts with other applications named processes
registered: [FastHtml.Cnode, FastHtml.Supervisor]
]
end
defp deps do
[
# documentation helpers
{:ex_doc, "~> 0.19", only: :dev},
# benchmarking helpers
- {:benchee, "~> 1.0", only: :dev}
+ {:benchee, "~> 1.0", only: :dev},
+ {:myhtmlex, "~> 0.2.0", only: :dev, runtime: false},
+ {:mochiweb, "~> 2.18", only: :dev},
+ {:html5ever, "~> 0.7.0", only: :dev}
]
end
defp docs do
[
- main: "fast_html"
+ main: "readme",
+ extras: ["README.md"]
]
end
end
defmodule Mix.Tasks.Compile.FastHtmlCnodeMake do
@artifacts [
"priv/myhtml_worker"
]
def find_make do
_make_cmd =
System.get_env("MAKE") ||
case :os.type() do
{:unix, :freebsd} -> "gmake"
{:unix, :openbsd} -> "gmake"
{:unix, :netbsd} -> "gmake"
{:unix, :dragonfly} -> "gmake"
_ -> "make"
end
end
defp otp_version do
:erlang.system_info(:otp_release)
|> to_string()
|> String.to_integer()
end
defp otp_22_or_newer? do
otp_version() >= 22
end
def run(_) do
make_cmd = find_make()
exit_code =
if match?({:win32, _}, :os.type()) do
IO.warn("Windows is not yet a target.")
1
else
{result, exit_code} =
System.cmd(
make_cmd,
@artifacts,
stderr_to_stdout: true,
env: [
{"MIX_ENV", to_string(Mix.env())},
{"OTP22_DEF", (otp_22_or_newer?() && "YES") || "NO"}
]
)
IO.binwrite(result)
exit_code
end
if exit_code == 0 do
:ok
else
{:error,
[
%Mix.Task.Compiler.Diagnostic{
compiler_name: "FastHtml Cnode",
message: "Make exited with #{exit_code}",
severity: :error,
file: nil,
position: nil
}
]}
end
end
def clean() do
make_cmd = find_make()
{result, _error_code} = System.cmd(make_cmd, ["clean"], stderr_to_stdout: true)
Mix.shell().info(result)
:ok
end
end
diff --git a/mix.lock b/mix.lock
index ff0803c..432b8c3 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,11 +1,15 @@
%{
"benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
"earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"},
"ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
+ "html5ever": {:hex, :html5ever, "0.7.0", "9f63ec1c783b2dc9f326840fcc993c01e926dbdef4e51ba1bbe5355993c258b4", [:mix], [{:rustler, "~> 0.18.0", [hex: :rustler, repo: "hexpm", optional: false]}], "hexpm"},
"makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
"makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
+ "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
"myhtml": {:git, "https://github.com/lexborisov/myhtml.git", "fe2cf577570666d058a2b7167c26d3384a758e19", [branch: "master"]},
+ "myhtmlex": {:hex, :myhtmlex, "0.2.1", "d6f3eb1826f7cdaa0225a996569da0930d1a334405510845c905ae59295ab226", [:make, :mix], [{:nodex, "~> 0.1.1", [hex: :nodex, repo: "hexpm", optional: false]}], "hexpm"},
"nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"},
- "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
+ "nodex": {:hex, :nodex, "0.1.1", "ed2f7bbe19ea62a43ad4b7ad332eb3f9ca12c64a35a5802a0eb545b93ebe32af", [:mix], [], "hexpm"},
+ "rustler": {:hex, :rustler, "0.18.0", "db4bd0c613d83a1badc31be90ddada6f9821de29e4afd15c53a5da61882e4f2d", [:mix], [], "hexpm"},
}

File Metadata

Mime Type
text/x-diff
Expires
Sun, Aug 30, 7:56 PM (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1738141
Default Alt Text
(13 KB)

Event Timeline