No OneTemporary
Actions

Size

29 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/.gitmodules b/.gitmodules
	new file mode 100644
	index 0000000..d48938b
	--- /dev/null
	+++ b/.gitmodules
	@@ -0,0 +1,3 @@
	+[submodule "c_src/myhtml"]
	+ path = c_src/myhtml
	+ url = https://github.com/lexborisov/myhtml.git
	diff --git a/Makefile b/Makefile
	index 6180bf0..7b5b6a5 100644
	--- a/Makefile
	+++ b/Makefile
	@@ -1,59 +1,63 @@
	MIX = mix
	MYHTMLEX_CFLAGS = -g -O3 -std=c99 -pedantic -Wcomment -Wall
	# we need to compile position independent code
	MYHTMLEX_CFLAGS += -fpic -DPIC
	# myhtmlex is using stpcpy, as defined in gnu string.h
	# MYHTMLEX_CFLAGS += -D_GNU_SOURCE
	# base on the same posix c source as myhtml
	# MYHTMLEX_CFLAGS += -D_POSIX_C_SOURCE=199309
	# turn warnings into errors
	# MYHTMLEX_CFLAGS += -Werror
	# ignore unused variables
	# MYHTMLEX_CFLAGS += -Wno-unused-variable
	# ignore unused parameter warnings
	MYHTMLEX_CFLAGS += -Wno-unused-parameter

	# set erlang include path
	ERLANG_PATH = $(shell erl -eval 'io:format("~s", [lists:concat([code:root_dir(), "/erts-", erlang:system_info(version), "/include"])])' -s init stop -noshell)
	MYHTMLEX_CFLAGS += -I$(ERLANG_PATH)

	# expecting myhtml fetched as a mix dependency
	-MYHTML_PATH = deps/myhtml
	+ifeq ($(wildcard c_src/myhtml),)
	+ MYHTML_PATH = deps/myhtml
	+else
	+ MYHTML_PATH = c_src/myhtml
	+endif
	MYHTML_STATIC = $(MYHTML_PATH)/lib/libmyhtml_static.a
	MYHTMLEX_CFLAGS += -I$(MYHTML_PATH)/include

	# that would be used for a dynamically linked build
	# MYHTMLEX_CFLAGS += -L$(MYHTML_PATH)/lib

	MYHTMLEX_LDFLAGS = -shared

	# platform specific
	UNAME = $(shell uname -s)
	ifeq ($(wilcard Makefile.$(UNAME)),)
	include Makefile.$(UNAME)
	endif

	.PHONY: all myhtmlex

	all: myhtmlex

	myhtmlex: priv/myhtmlex.so
	$(MIX) compile

	deps/myhtml:
	$(MIX) deps.get

	-$(MYHTML_STATIC): deps/myhtml
	+$(MYHTML_STATIC): $(MYHTML_PATH)
	$(MAKE) -C $(MYHTML_PATH) library

	priv/myhtmlex.so: src/myhtmlex.c $(MYHTML_STATIC)
	test -d priv \|\| mkdir priv
	$(CC) $(MYHTMLEX_CFLAGS) $(MYHTMLEX_LDFLAGS) -o $@ $< $(MYHTML_STATIC)

	clean:
	$(MIX) clean
	$(RM) priv/myhtmlex.so

	clean-myhtml:
	$(MAKE) -C $(MYHTML_PATH) clean

	diff --git a/c_src/myhtml b/c_src/myhtml
	new file mode 160000
	index 0000000..fe2cf57
	--- /dev/null
	+++ b/c_src/myhtml
	@@ -0,0 +1 @@
	+Subproject commit fe2cf577570666d058a2b7167c26d3384a758e19
	diff --git a/lib/myhtmlex.ex b/lib/myhtmlex.ex
	index 2369eaa..9c41d7f 100644
	--- a/lib/myhtmlex.ex
	+++ b/lib/myhtmlex.ex
	@@ -1,21 +1,149 @@
	defmodule Myhtmlex do
	+ @moduledoc """
	+ A module to decode html into a tree structure.
	+
	+ Based on [Alexander Borisov's myhtml](https://github.com/lexborisov/myhtml),
	+ this binding gains the properties of being html-spec compliant and very fast.
	+
	+ ## Example
	+
	+ iex> Myhtmlex.decode("<h1>Hello world</h1>")
	+ {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
	+
	+ Benchmark results on various file sizes on a 2,5Ghz Core i7:
	+
	+ Settings:
	+ duration: 1.0 s
	+
	+ ## FileSizesBench
	+ [15:28:42] 1/3: github_trending_js.html 341k
	+ [15:28:46] 2/3: w3c_html5.html 131k
	+ [15:28:48] 3/3: wikipedia_hyperlink.html 97k
	+
	+ Finished in 7.52 seconds
	+
	+ ## FileSizesBench
	+ benchmark name iterations average time
	+ wikipedia_hyperlink.html 97k 1000 1385.86 µs/op
	+ w3c_html5.html 131k 1000 2179.30 µs/op
	+ github_trending_js.html 341k 500 5686.21 µs/op
	+
	+ ## Thoughts
	+
	+ I need to a fast html-parsing library in Erlang/Elixir.
	+ So falling back to c, and to myhtml especially, is a natural move.
	+
	+ But Erlang interoperability is a tricky mine-field.
	+ This increase in parsing speed does not come for free.
	+
	+ The current implementation can be considered a proof-of-concept.
	+ The myhtml code is called as a dirty-nif and executed inside the Erlang-VM.
	+ Thus completely giving up the safety of the Erlang-VM. I am not saying that myhtml is unsafe, but
	+ the slightest Segfault brings down the whole Erlang-VM.
	+ So, I consider this mode of operation unsafe, and not recommended for production use.
	+
	+ The other option, that I have on my roadmap, is to call into a C-Node.
	+ A separate OS-process that receives calls from erlang and returns to the calling process.
	+
	+ Another option is to call into a Port driver.
	+ A separate OS-process that communicates via stdin/stdout.
	+
	+ So to recap, I want a fast and safe html-parsing library for Erlang/Elixir.
	+
	+ Not quite there, yet.
	+ """
	+
	+ @type tag() :: String.t \| atom()
	+ @type attr() :: {String.t, String.t}
	+ @type attr_list() :: [] \| [attr()]
	+ @type comment_node() :: {:comment, String.t}
	+ @type comment_node3() :: {:comment, [], String.t}
	+ @type tree() :: {tag(), attr_list(), tree()}
	+ \| {tag(), attr_list(), nil}
	+ \| comment_node()
	+ \| comment_node3()
	+ @type format_flag() :: :html_atoms \| :nil_self_closing \| :comment_tuple3
	+
	+ @doc """
	+ Returns a tree representation from the given html string.
	+
	+ ## Examples
	+
	+ iex> Myhtmlex.decode("<h1>Hello world</h1>")
	+ {"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}
	+
	+ iex> Myhtmlex.decode("<span class='hello'>Hi there</span>")
	+ {"html", [],
	+ [{"head", [], []},
	+ {"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}
	+
	+ iex> Myhtmlex.decode("<body><!-- a comment --!></body>")
	+ {"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}
	+
	+ iex> Myhtmlex.decode("<br>")
	+ {"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}
	+ """
	+ @spec decode(String.t) :: tree()
	def decode(bin) do
	Myhtmlex.Decoder.decode(bin)
	end

	+ @doc """
	+ Returns a tree representation from the given html string.
	+
	+ This variant allows you to pass in one or more of the following format flags:
	+
	+ * `:html_atoms` uses atoms for known html tags (faster), binaries for everything else.
	+ * `:nil_self_closing` uses `nil` to designate self-closing tags and void elements.
	+ For example `<br>` is then being represented like `{"br", [], nil}`.
	+ See http://w3c.github.io/html-reference/syntax.html#void-elements for a full list of void elements.
	+ * `:comment_tuple3` uses 3-tuple elements for comments, instead of the default 2-tuple element.
	+
	+ ## Examples
	+
	+ iex> Myhtmlex.decode("<h1>Hello world</h1>", format: [:html_atoms])
	+ {:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}
	+
	+ iex> Myhtmlex.decode("<br>", format: [:nil_self_closing])
	+ {"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}
	+
	+ iex> Myhtmlex.decode("<body><!-- a comment --!></body>", format: [:comment_tuple3])
	+ {"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}
	+
	+ iex> html = "<body><!-- a comment --!><unknown /></body>"
	+ iex> Myhtmlex.decode(html, format: [:html_atoms, :nil_self_closing, :comment_tuple3])
	+ {:html, [],
	+ [{:head, [], []},
	+ {:body, [], [{:comment, " a comment "}, {"unknown", [], nil}]}]}
	+
	+ """
	+ @spec decode(String.t, format: [format_flag()]) :: tree()
	def decode(bin, format: flags) do
	Myhtmlex.Decoder.decode(bin, flags)
	end

	+ @doc """
	+ Returns a reference to an internally parsed myhtml_tree_t.
	+ """
	+ @spec open(String.t) :: reference()
	def open(bin) do
	Myhtmlex.Decoder.open(bin)
	end

	- def decode_tree(tree) do
	- Myhtmlex.Decoder.decode_tree(tree)
	+ @doc """
	+ Returns a tree representation from the given reference. See `decode/1` for example output.
	+ """
	+ @spec decode_tree(reference()) :: tree()
	+ def decode_tree(ref) do
	+ Myhtmlex.Decoder.decode_tree(ref)
	end

	- def decode_tree(tree, format: flags) do
	- Myhtmlex.Decoder.decode_tree(tree, flags)
	+ @doc """
	+ Returns a tree representation from the given reference. See `decode/2` for options and example output.
	+ """
	+ @spec decode_tree(reference(), format: [format_flag()]) :: tree()
	+ def decode_tree(ref, format: flags) do
	+ Myhtmlex.Decoder.decode_tree(ref, flags)
	end
	end
	+
	diff --git a/lib/myhtmlex/decoder.ex b/lib/myhtmlex/decoder.ex
	index 2516cf5..ed9d642 100644
	--- a/lib/myhtmlex/decoder.ex
	+++ b/lib/myhtmlex/decoder.ex
	@@ -1,26 +1,27 @@
	defmodule Myhtmlex.Decoder do
	+ @moduledoc false
	@on_load { :init, 0 }

	app = Mix.Project.config[:app]

	def init do
	path = :filename.join(:code.priv_dir(unquote(app)), 'myhtmlex')
	:ok = :erlang.load_nif(path, 0)
	end

	@spec decode(bin :: String.t) :: {atom(), list(), list()}
	def decode(bin)
	def decode(_), do: exit(:nif_library_not_loaded)

	def decode(bin, flags)
	def decode(_, _), do: exit(:nif_library_not_loaded)

	def open(bin)
	def open(_), do: exit(:nif_library_not_loaded)

	def decode_tree(tree)
	def decode_tree(_), do: exit(:nif_library_not_loaded)

	def decode_tree(tree, flags)
	def decode_tree(_, _), do: exit(:nif_library_not_loaded)
	end
	diff --git a/lib/myhtmlex/doc.ex b/lib/myhtmlex/doc.ex
	deleted file mode 100644
	index 2697056..0000000
	--- a/lib/myhtmlex/doc.ex
	+++ /dev/null
	@@ -1,18 +0,0 @@
	-defmodule Myhtmlex.Doc do
	- defstruct ref: nil, source: nil
	-end
	-
	-defimpl Inspect, for: Myhtmlex.Doc do
	- import Inspect.Algebra
	-
	- def inspect(%Myhtmlex.Doc{source: source}, opts) do
	- cut = String.slice(source, 0..60)
	- cut = if String.length(cut) < String.length(source) do
	- "#{cut}..."
	- else
	- cut
	- end
	- concat ["#Myhtmlex.Doc<source: ", to_doc(cut, opts), ">"]
	- end
	-end
	-
	diff --git a/mix.exs b/mix.exs
	index 70a42c6..b7084e7 100644
	--- a/mix.exs
	+++ b/mix.exs
	@@ -1,48 +1,71 @@
	defmodule Mix.Tasks.Compile.Myhtml do
	def run(_) do
	if match? {:win32, _}, :os.type do
	IO.warn "Windows is not yet a target."
	exit(1)
	else
	- File.mkdir_p("priv")
	{result, _error_code} = System.cmd("make", ["priv/myhtmlex.so"], stderr_to_stdout: true)
	IO.binwrite result
	end
	:ok
	end
	end

	defmodule Myhtmlex.Mixfile do
	use Mix.Project

	def project do
	[
	app: :myhtmlex,
	version: "0.1.0",
	elixir: "~> 1.5",
	compilers: [:myhtml, :elixir, :app],
	start_permanent: Mix.env == :prod,
	+ description: "A module to decode HTML into a tree, porting all properties of the underlying library myhtml, being fast and correct in regards to the html spec.",
	+ package: package(),
	deps: deps()
	]
	end

	+ def package do
	+ [
	+ maintainers: ["Lukas Rieder"],
	+ licenses: ["GNU LGPL"],
	+ links: %{
	+ "Github" => "https://github.com/Overbryd/myhtmlex",
	+ "Issues" => "https://github.com/Overbryd/myhtmlex/issues",
	+ "MyHTML" => "https://github.com/lexborisov/myhtml"
	+ },
	+ files: [
	+ "lib",
	+ "src",
	+ "c_src",
	+ "Makefile",
	+ "Makefile.Darwin",
	+ "Makefile.Linux",
	+ "mix.exs",
	+ "README.md",
	+ "LICENSE"
	+ ]
	+ ]
	+ end
	+
	# Run "mix help compile.app" to learn about applications.
	def application do
	[
	extra_applications: [:logger]
	]
	end

	# Run "mix help deps" to learn about dependencies.
	defp deps do
	[
	- # myhtml c library
	- {:myhtml, github: "Overbryd/myhtml", branch: "feat/node-is-void-element", app: false},
	- # {:myhtml, github: "lexborisov/myhtml", tag: "v4.0.2", app: false},
	+ # in dev environment, manage myhtml c library with mix
	+ {:myhtml, github: "lexborisov/myhtml", branch: "master", app: false, only: :dev},
	# documentation helpers
	{:ex_doc, ">= 0.0.0", only: :dev},
	# benchmarking helpers
	{:benchfella, "~> 0.3.0", only: :dev}
	]
	end
	end
	diff --git a/mix.lock b/mix.lock
	index c600c08..ddb538a4 100644
	--- a/mix.lock
	+++ b/mix.lock
	@@ -1,4 +1,4 @@
	%{"benchfella": {:hex, :benchfella, "0.3.5", "b2122c234117b3f91ed7b43b6e915e19e1ab216971154acd0a80ce0e9b8c05f5", [], [], "hexpm"},
	"earmark": {:hex, :earmark, "1.2.3", "206eb2e2ac1a794aa5256f3982de7a76bf4579ff91cb28d0e17ea2c9491e46a4", [], [], "hexpm"},
	"ex_doc": {:hex, :ex_doc, "0.16.3", "cd2a4cfe5d26e37502d3ec776702c72efa1adfa24ed9ce723bb565f4c30bd31a", [], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"},
	- "myhtml": {:git, "https://github.com/Overbryd/myhtml.git", "697d2aad1392ef555fd3d632674d7ad7717d1e6a", [branch: "feat/node-is-void-element"]}}
	+ "myhtml": {:git, "https://github.com/lexborisov/myhtml.git", "fe2cf577570666d058a2b7167c26d3384a758e19", [branch: "master"]}}
	diff --git a/src/myhtmlex.c b/src/myhtmlex.c
	index bc4e070..2fde6b0 100644
	--- a/src/myhtmlex.c
	+++ b/src/myhtmlex.c
	@@ -1,423 +1,428 @@
	#include "myhtmlex.h"

	char*
	lowercase(char* c)
	{
	char* p = c;
	while(*p)
	{
	p = tolower((unsigned char)p);
	p++;
	}
	return c;
	}

	ERL_NIF_TERM
	make_atom(ErlNifEnv* env, const char* name)
	{
	ERL_NIF_TERM ret;
	if(enif_make_existing_atom(env, name, &ret, ERL_NIF_LATIN1)) {
	return ret;
	}
	return enif_make_atom(env, name);
	}

	ERL_NIF_TERM
	nif_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
	ERL_NIF_TERM result;
	myhtmlex_ref_t* ref;

	// fetch nif state
	myhtmlex_state_t* state = (myhtmlex_state_t*) enif_priv_data(env);

	// placeholder for the html binary we want to read from erlang caller
	ErlNifBinary html_bin;
	// read binary into &html_bin from argv[0] (first argument)
	if (!enif_inspect_iolist_as_binary(env, argv[0], &html_bin))
	{
	// blame the user if html_bin is not a binary
	return enif_make_badarg(env);
	}

	ref = enif_alloc_resource(state->myhtml_tree_rt, sizeof(myhtmlex_ref_t));
	ref->tree = myhtml_tree_create();
	myhtml_tree_init(ref->tree, state->myhtml);
	mystatus_t status = myhtml_parse(ref->tree, MyENCODING_UTF_8, (char*) html_bin.data, (size_t) html_bin.size);
	if (status != MyHTML_STATUS_OK)
	{
	// TODO: what is the correct reaction for a not ok state?
	return enif_make_badarg(env);
	}
	ref->root = myhtml_tree_get_document(ref->tree);

	result = enif_make_resource(env, ref);
	return result;
	}

	ERL_NIF_TERM
	nif_decode_tree(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
	ERL_NIF_TERM result;
	myhtmlex_ref_t* ref;
	unsigned char parse_flags = 0;

	// fetch nif state
	myhtmlex_state_t* state = (myhtmlex_state_t*) enif_priv_data(env);

	// fetch reference
	if (!enif_get_resource(env, argv[0], state->myhtml_tree_rt, (void **) &ref))
	{
	return enif_make_badarg(env);
	}
	// we should have received format flags in a list
	if (argc == 2)
	{
	if (!enif_is_list(env, argv[1]))
	{
	// blame the user if second argument is not a list
	return enif_make_badarg(env);
	}
	parse_flags = read_parse_flags(env, &argv[1]);
	}

	// build erlang tree
	result = build_tree(env, ref->tree, myhtml_node_last_child(ref->root), &parse_flags);

	// return tree to erlang
	return result;
	}

	ERL_NIF_TERM
	nif_decode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) {
	ERL_NIF_TERM result;
	unsigned char parse_flags = 0;

	// fetch nif state
	myhtmlex_state_t* state = (myhtmlex_state_t*) enif_priv_data(env);

	// placeholder for the html binary we want to read from erlang caller
	ErlNifBinary html_bin;
	// read binary into &html_bin from argv[0] (first argument)
	if (argc < 1 \|\| !enif_inspect_iolist_as_binary(env, argv[0], &html_bin))
	{
	// blame the user if html_bin is not a binary
	return enif_make_badarg(env);
	}
	// we should have received format flags in a list
	if (argc == 2)
	{
	if (!enif_is_list(env, argv[1]))
	{
	// blame the user if second argument is not a list
	return enif_make_badarg(env);
	}
	parse_flags = read_parse_flags(env, &argv[1]);
	}

	// parse html into tree
	// use parse_single for now, threaded mode is buggy with some files
	mystatus_t status = myhtml_parse(state->tree, MyENCODING_UTF_8, (char*) html_bin.data, (size_t) html_bin.size);
	if (status != MyHTML_STATUS_OK)
	{
	// TODO: what is the correct reaction for a not ok state?
	return enif_make_badarg(env);
	}

	// build erlang tree
	myhtml_tree_node_t *root = myhtml_tree_get_document(state->tree);
	result = build_tree(env, state->tree, myhtml_node_last_child(root), &parse_flags);

	// return tree to erlang
	return result;
	}

	unsigned char
	read_parse_flags(ErlNifEnv* env, const ERL_NIF_TERM* options)
	{
	unsigned char parse_flags = 0;
	ERL_NIF_TERM flag;

	// only look at 2 flags max (more are not implemented yet)
	for (int i = 0; i < 2; i++)
	{
	if (!enif_get_list_cell(env, options, &flag, (ERL_NIF_TERM)options)) break;
	if (!enif_is_atom(env, flag)) return enif_make_badarg(env);
	// set parse flags
	if (enif_compare(flag, ATOM_HTML_ATOMS) == 0)
	{
	parse_flags \|= FLAG_HTML_ATOMS;
	}
	else if (enif_compare(flag, ATOM_NIL_SELF_CLOSING) == 0)
	{
	parse_flags \|= FLAG_NIL_SELF_CLOSING;
	}
	+ else if (enif_compare(flag, ATOM_COMMENT_TUPLE3) == 0)
	+ {
	+ parse_flags \|= FLAG_COMMENT_TUPLE3;
	+ }
	}

	return parse_flags;
	}

	ERL_NIF_TERM
	build_node_children(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* parent, unsigned char* parse_flags)
	{
	if (myhtml_node_is_close_self(parent) && (*parse_flags & FLAG_NIL_SELF_CLOSING))
	{
	return ATOM_NIL;
	}

	myhtml_tree_node_t* child = myhtml_node_last_child(parent);
	if (child == NULL)
	{
	if (myhtml_node_is_void_element(parent) && (*parse_flags & FLAG_NIL_SELF_CLOSING))
	{
	return ATOM_NIL;
	}
	else
	{
	return EMPTY_LIST;
	}
	}

	ERL_NIF_TERM list = enif_make_list(env, 0);

	while (child)
	{
	ERL_NIF_TERM node_tuple = build_tree(env, tree, child, parse_flags);
	list = enif_make_list_cell(env, node_tuple, list);

	// get previous child, building the list from reverse
	child = myhtml_node_prev(child);
	}

	return list;
	}

	ERL_NIF_TERM
	build_node_attrs(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* node)
	{
	myhtml_tree_attr_t* attr = myhtml_node_attribute_last(node);
	if (attr == NULL)
	{
	return EMPTY_LIST;
	}

	ERL_NIF_TERM list = enif_make_list(env, 0);

	while (attr)
	{
	ErlNifBinary name;
	ERL_NIF_TERM name_bin;
	ErlNifBinary value;
	ERL_NIF_TERM value_bin;
	ERL_NIF_TERM attr_tuple;

	size_t attr_name_len;
	const char *attr_name = myhtml_attribute_key(attr, &attr_name_len);
	size_t attr_value_len;
	const char *attr_value = myhtml_attribute_value(attr, &attr_value_len);

	if (attr_value) {
	enif_alloc_binary(attr_value_len, &value);
	memcpy(value.data, attr_value, attr_value_len);
	value_bin = enif_make_binary(env, &value);
	} else {
	enif_alloc_binary(attr_name_len, &value);
	memcpy(value.data, attr_name, attr_name_len);
	value_bin = enif_make_binary(env, &value);
	}
	enif_alloc_binary(attr_name_len, &name);
	memcpy(name.data, attr_name, attr_name_len);
	name_bin = enif_make_binary(env, &name);

	attr_tuple = enif_make_tuple2(env, name_bin, value_bin);
	list = enif_make_list_cell(env, attr_tuple, list);

	// get prev attribute, building the list from reverse
	attr = myhtml_attribute_prev(attr);
	}

	return list;
	}

	ERL_NIF_TERM
	build_tree(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* node, unsigned char* parse_flags)
	{
	ERL_NIF_TERM result;
	myhtml_tag_id_t tag_id = myhtml_node_tag_id(node);
	myhtml_namespace_t tag_ns = myhtml_node_namespace(node);

	if (tag_id == MyHTML_TAG__TEXT)
	{
	ErlNifBinary text;
	size_t text_len;
	const char* node_text = myhtml_node_text(node, &text_len);
	enif_alloc_binary(text_len, &text);
	memcpy(text.data, node_text, text_len);

	result = enif_make_binary(env, &text);
	}
	else if (tag_id == MyHTML_TAG__COMMENT)
	{
	ErlNifBinary comment;
	size_t comment_len;
	const char* node_comment = myhtml_node_text(node, &comment_len);
	enif_alloc_binary(comment_len, &comment);
	memcpy(comment.data, node_comment, comment_len);

	if (*parse_flags & FLAG_COMMENT_TUPLE3)
	{
	result = enif_make_tuple3(env,
	ATOM_COMMENT,
	EMPTY_LIST,
	enif_make_binary(env, &comment)
	);
	}
	else
	{
	result = enif_make_tuple2(env, ATOM_COMMENT, enif_make_binary(env, &comment));
	}
	}
	else
	{
	ERL_NIF_TERM tag;
	ERL_NIF_TERM children;
	ERL_NIF_TERM attrs;

	// get name of tag
	size_t tag_name_len;
	const char *tag_name = myhtml_tag_name_by_id(tree, tag_id, &tag_name_len);
	// get namespace of tag
	size_t tag_ns_len;
	const char *tag_ns_name_ptr = myhtml_namespace_name_by_id(tag_ns, &tag_ns_len);
	char *tag_ns_buffer;
	char buffer [tag_ns_len + tag_name_len + 1];
	char *tag_string = buffer;
	size_t tag_string_len;

	if (tag_ns != MyHTML_NAMESPACE_HTML)
	{
	// tag_ns_name_ptr is unmodifyable, copy it in our tag_ns_buffer to make it modifyable.
	tag_ns_buffer = malloc(tag_ns_len);
	strcpy(tag_ns_buffer, tag_ns_name_ptr);
	// lowercase tag buffer (can be removed, just a nice to have)
	tag_ns_buffer = lowercase(tag_ns_buffer);
	// prepend namespace to tag name, e.g. "svg:path"
	stpcpy(stpcpy(stpcpy(tag_string, tag_ns_buffer), ":"), tag_name);
	tag_string_len = tag_ns_len + tag_name_len + 1; // +1 for colon
	}
	else
	{
	stpcpy(tag_string, tag_name);
	tag_string_len = tag_name_len;
	}

	// put unknown and non-html tags it in a binary
	if (!(*parse_flags & FLAG_HTML_ATOMS) \|\| (tag_id == MyHTML_TAG__UNDEF \|\| tag_id == MyHTML_TAG_LAST_ENTRY \|\| tag_ns != MyHTML_NAMESPACE_HTML))
	{
	ErlNifBinary tag_b;
	enif_alloc_binary(tag_string_len, &tag_b);
	memcpy(tag_b.data, tag_string, tag_string_len);
	tag = enif_make_binary(env, &tag_b);
	}
	else
	{
	tag = make_atom(env, tag_string);
	}

	// attributes
	attrs = build_node_attrs(env, tree, node);

	// add children or nil as a self-closing flag
	children = build_node_children(env, tree, node, parse_flags);

	// free allocated resources
	if (tag_ns != MyHTML_NAMESPACE_HTML)
	{
	free(tag_ns_buffer);
	}

	result = enif_make_tuple3(env,
	tag,
	attrs,
	children
	);
	}

	return result;
	}

	void
	nif_cleanup_myhtmlex_ref(ErlNifEnv* env, void* obj)
	{
	myhtmlex_ref_t* ref = (myhtmlex_ref_t*) obj;
	// release myhtml resources
	myhtml_tree_destroy(ref->tree);
	}

	// Erlang NIF

	static int
	load(ErlNifEnv env, void *priv, ERL_NIF_TERM info)
	{
	myhtmlex_state_t* state = enif_alloc(sizeof(myhtmlex_state_t));
	if (state == NULL)
	{
	return 1;
	}

	state->myhtml_tree_rt = enif_open_resource_type(
	env,
	NULL,
	"myhtmlex_ref_t",
	&nif_cleanup_myhtmlex_ref,
	ERL_NIF_RT_CREATE \| ERL_NIF_RT_TAKEOVER,
	NULL
	);
	ATOM_NIL = make_atom(env, "nil");
	ATOM_COMMENT = make_atom(env, "comment");
	ATOM_HTML_ATOMS = make_atom(env, "html_atoms");
	ATOM_NIL_SELF_CLOSING = make_atom(env, "nil_self_closing");
	+ ATOM_COMMENT_TUPLE3 = make_atom(env, "comment_tuple3");
	EMPTY_LIST = enif_make_list(env, 0);

	// myhtml basic init
	state->myhtml = myhtml_create();
	myhtml_init(state->myhtml, MyHTML_OPTIONS_DEFAULT, 1, 0);
	state->tree = myhtml_tree_create();
	myhtml_tree_init(state->tree, state->myhtml);

	priv = (void) state;
	return 0;
	}

	static int
	reload(ErlNifEnv env, void *priv, ERL_NIF_TERM info)
	{
	return 0;
	}

	static int
	upgrade(ErlNifEnv env, void priv, void *old_priv, ERL_NIF_TERM info)
	{
	return load(env, priv, info);
	}

	static void
	unload(ErlNifEnv env, void priv)
	{
	myhtmlex_state_t* state = (myhtmlex_state_t*) priv;

	myhtml_tree_destroy(state->tree);
	myhtml_destroy(state->myhtml);
	enif_free(priv);
	return;
	}

	static ErlNifFunc funcs[] =
	{
	{"decode", 1, nif_decode, ERL_NIF_DIRTY_JOB_CPU_BOUND},
	{"decode", 2, nif_decode, ERL_NIF_DIRTY_JOB_CPU_BOUND},
	{"open", 1, nif_open, ERL_NIF_DIRTY_JOB_CPU_BOUND},
	{"decode_tree", 1, nif_decode_tree, ERL_NIF_DIRTY_JOB_CPU_BOUND},
	{"decode_tree", 2, nif_decode_tree, ERL_NIF_DIRTY_JOB_CPU_BOUND}
	};

	ERL_NIF_INIT(Elixir.Myhtmlex.Decoder, funcs, &load, &reload, &upgrade, &unload)

	diff --git a/src/myhtmlex.h b/src/myhtmlex.h
	index 83b39b7..b11c2cd 100644
	--- a/src/myhtmlex.h
	+++ b/src/myhtmlex.h
	@@ -1,55 +1,56 @@
	#ifndef MYHTMLEX_H
	#define MYHTMLEX_H

	#include <stdlib.h>
	#include <ctype.h>
	#include <string.h>
	#include "erl_nif.h"
	#include <myhtml/myhtml.h>
	#include <myhtml/mynamespace.h>

	char*
	lowercase(char* c);
	// myhtmlex.c
	ERL_NIF_TERM
	make_atom(ErlNifEnv* env, const char* name);
	ERL_NIF_TERM
	nif_decode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
	ERL_NIF_TERM
	nif_decode_tree(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
	ERL_NIF_TERM
	nif_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
	ERL_NIF_TERM
	build_node_attrs(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* node);
	ERL_NIF_TERM
	build_tree(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* node, unsigned char* flags);
	ERL_NIF_TERM
	build_node_children(ErlNifEnv* env, myhtml_tree_t* tree, myhtml_tree_node_t* node, unsigned char* flags);
	void
	nif_cleanup_myhtml_tree(ErlNifEnv* env, void* obj);
	unsigned char
	read_parse_flags(ErlNifEnv* env, const ERL_NIF_TERM* options);

	// consts
	ERL_NIF_TERM ATOM_NIL;
	ERL_NIF_TERM ATOM_COMMENT;
	ERL_NIF_TERM ATOM_HTML_ATOMS;
	ERL_NIF_TERM ATOM_NIL_SELF_CLOSING;
	+ERL_NIF_TERM ATOM_COMMENT_TUPLE3;
	ERL_NIF_TERM EMPTY_LIST;
	const unsigned char FLAG_HTML_ATOMS = 1 << 0;
	const unsigned char FLAG_NIL_SELF_CLOSING = 1 << 1;
	const unsigned char FLAG_COMMENT_TUPLE3 = 1 << 2;

	typedef struct {
	myhtml_t* myhtml;
	myhtml_tree_t* tree;

	ErlNifResourceType* myhtml_tree_rt;
	} myhtmlex_state_t;

	typedef struct {
	myhtml_tree_t* tree;
	myhtml_tree_node_t *root;
	} myhtmlex_ref_t;

	#endif // included myhtmlex.h
	diff --git a/test/myhtmlex_test.exs b/test/myhtmlex_test.exs
	index 0a693d0..9f3a2f1 100644
	--- a/test/myhtmlex_test.exs
	+++ b/test/myhtmlex_test.exs
	@@ -1,122 +1,132 @@
	defmodule MyhtmlexTest do
	use ExUnit.Case
	doctest Myhtmlex

	test "builds a tree, formatted like mochiweb by default" do
	assert {"html", [], [
	{"head", [], []},
	{"body", [], [
	{"br", [], []}
	]}
	]} = Myhtmlex.decode("<br>")
	end

	test "builds a tree, html tags as atoms" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{:br, [], []}
	]}
	]} = Myhtmlex.decode("<br>", format: [:html_atoms])
	end

	test "builds a tree, nil self closing" do
	assert {"html", [], [
	{"head", [], []},
	{"body", [], [
	{"br", [], nil},
	{"esi:include", [], nil}
	]}
	]} = Myhtmlex.decode("<br><esi:include />", format: [:nil_self_closing])
	end

	test "builds a tree, multiple format options" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{:br, [], nil}
	]}
	]} = Myhtmlex.decode("<br>", format: [:html_atoms, :nil_self_closing])
	end

	test "attributes" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{:span, [{"id", "test"}, {"class", "foo garble"}], []}
	]}
	]} = Myhtmlex.decode(~s'<span id="test" class="foo garble"></span>', format: [:html_atoms])
	end

	test "single attributes" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{:button, [{"disabled", "disabled"}, {"class", "foo garble"}], []}
	]}
	]} = Myhtmlex.decode(~s'<button disabled class="foo garble"></span>', format: [:html_atoms])
	end

	test "text nodes" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	"text node"
	]}
	]} = Myhtmlex.decode(~s'<body>text node</body>', format: [:html_atoms])
	end

	test "broken input" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{:a, [{"<", "<"}], [" asdf"]}
	]}
	]} = Myhtmlex.decode(~s'<a <> asdf', format: [:html_atoms])
	end

	test "open" do
	ref = Myhtmlex.open(~s'<dif class="a"></div><div class="b"></div>')
	assert is_reference(ref)
	end

	test "open and decode_tree" do
	ref = Myhtmlex.open(~s'text node')
	assert is_reference(ref)
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	"text node"
	]}
	]} = Myhtmlex.decode_tree(ref, format: [:html_atoms])
	end

	test "namespaced tags" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{"svg:svg", [], [
	{"svg:path", [], []},
	{"svg:a", [], []}
	]}
	]}
	]} = Myhtmlex.decode(~s'<svg><path></path><a></a></svg>', format: [:html_atoms])
	end

	test "custom namespaced tags" do
	assert {:html, [], [
	{:head, [], []},
	{:body, [], [
	{"esi:include", [], nil}
	]}
	]} = Myhtmlex.decode(~s'<esi:include />', format: [:html_atoms, :nil_self_closing])
	end

	- test "open this nasty github file (works fine in parse single, parse threaded hangs)" do
	+ test "html comments" do
	+ assert {:html, [], [
	+ {:head, [], []},
	+ {:body, [], [
	+ comment: " a comment "
	+ ]}
	+ ]} = Myhtmlex.decode(~s'<body><!-- a comment --></body>', format: [:html_atoms])
	+ end
	+
	+ test "parse a larger file (131K)" do
	html = File.read!("bench/github_trending_js.html")
	ref = Myhtmlex.open(html)
	assert is_reference(ref)
	+ assert is_tuple(Myhtmlex.decode_tree(ref))
	end

	end

File Metadata

Mime Type: text/x-diff
Expires: Sun, Jul 19, 1:32 PM (1 d, 19 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1695286
Default Alt Text: (29 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions