No OneTemporary
Actions

Size

32 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/CHANGELOG.md b/CHANGELOG.md
	index d3d4b1e..a1ee0c9 100644
	--- a/CHANGELOG.md
	+++ b/CHANGELOG.md
	@@ -1,11 +1,17 @@
	# Changelog
	All notable changes to this project will be documented in this file.

	The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

	+## [1.0.1] - 2019-12-11
	+### Added
	+- `:fast_html.decode_fragment`
	+### Fixed
	+- Errors from C-Node not being reported, timing out instead
	+
	## [1.0.0] - 2019-12-02
	### Changed
	- BREAKING: `:fast_html.decode` now returns an array of nodes at the top level, instead of a single node. This was done because it's possible to have more than one root node, for example in (`<!-- a comment --> <html> </html>` both the comment and the `html` tag are root nodes).

	### Fixed
	- Worker going into infinite loop when decoding a document with more than one root node.
	diff --git a/README.md b/README.md
	index 826aa7f..c38a5fb 100644
	--- a/README.md
	+++ b/README.md
	@@ -1,26 +1,35 @@
	# FastHTML

	A C Node wrapping lexborisov's [myhtml](https://github.com/lexborisov/myhtml).
	Primarily used with [FastSanitize](https://git.pleroma.social/pleroma/fast_sanitize).

	* Available as a hex package: `{:fast_html, "~> 0.99"}`
	* [Documentation](https://hexdocs.pm/fast_html/fast_html.html)

	## Benchmarks

	The following table provides median times it takes to decode a string to a tree for html parsers that can be used from Elixir. Benchmarks were conducted on a machine with `Intel Core i7-3520M @ 2.90GHz` CPU and 16GB of RAM. The `mix fast_html.bench` task can be used for running the benchmark by yourself.

	\| File/Parser \| fast_html (C-Node) \| mochiweb_html (erlang) \| html5ever (Rust NIF) \| Myhtmlex (NIF)¹ \|
	\|----------------------\|--------------------\|------------------------\|----------------------\|----------------\|
	\| document-large.html \| 178.13 ms \| 3471.70 ms \| 799.20 ms \| 402.64 ms \|
	\| document-medium.html \| 2.85 ms \| 26.58 ms \| 9.06 ms \| 3.72 ms \|
	\| document-small.html \| 1.08 ms \| 5.45 ms \| 2.10 ms \| 1.24 ms \|
	\| fragment-large.html \| 1.50 ms \| 10.91 ms \| 6.03 ms \| 1.91 ms \|
	\| fragment-small.html² \| 434.64 μs \| 83.02 μs \| 57.97 μs \| 311.39 μs \|

	1. Myhtmlex has a C-Node mode as well, but it wasn't benchmarked here because it segfaults on `document-large.html`
	2. The slowdown on `fragment-small.html` is due to C-Node overhead. Unlike html5ever and Myhtmlex in NIF mode, `fast_html` has the parser process isolated and communicates with it over the network, so even if a fatal crash in the parser happens, it won't bring down the entire VM.
	+
	+## Note about running with [Swarm](https://github.com/bitwalker/swarm)
	+
	+Since the myhtml worker runs as a separate node, Swarm will try to sync with it. Of course it will fail since it's not a real Erlang node. To prevent it from doing that, you can add the following to your configuration:
	+
	+```elixir
	+config :swarm, node_blacklist: [~r/myhtml_.*$/]
	+```
	+
	## Contribution / Bug Reports

	* Please make sure you do `git submodule update` after a checkout/pull
	* The project aims to be fully tested
	diff --git a/c_src/myhtml_worker.c b/c_src/myhtml_worker.c
	index 32c6670..afc176e 100644
	--- a/c_src/myhtml_worker.c
	+++ b/c_src/myhtml_worker.c
	@@ -1,516 +1,570 @@
	#include <stdlib.h>
	#include <stdbool.h>
	#include <stdio.h>
	#include <string.h>
	#include <stdarg.h>
	#include <unistd.h>
	#include <sys/types.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <errno.h>
	#include <ctype.h>

	#ifndef _REENTRANT
	#define _REENTRANT /* For some reason __erl_errno is undefined unless _REENTRANT is defined */
	#endif

	#include "ei.h"
	#ifndef OTP_22_OR_NEWER
	# include "erl_interface.h"
	#endif

	#include <myhtml/myhtml.h>
	#include <myhtml/mynamespace.h>

	#include "tstack.h"

	#ifdef __GNUC__
	# define AFP(x, y) __attribute__((format (printf, x, y)))
	#else
	# define AFP(x, y)
	#endif

	#ifdef __GNUC__
	# define NORETURN __attribute__((noreturn))
	#else
	# define NORETURN
	#endif

	typedef struct _state_t {
	int fd;
	myhtml_t * myhtml;
	ei_cnode ec;
	bool looping;
	ei_x_buff buffer;
	} state_t;

	typedef enum parse_flags_e {
	FLAG_HTML_ATOMS = 1 << 0,
	FLAG_NIL_SELF_CLOSING = 1 << 1,
	FLAG_COMMENT_TUPLE3 = 1 << 2
	} parse_flags_t;

	static void handle_emsg(state_t * state, erlang_msg * emsg);
	static void handle_send(state_t * state, erlang_msg * emsg);
	static void err_term(ei_x_buff * response, const char * error_atom);
	static parse_flags_t decode_parse_flags(state_t * state, int arity);
	-static void decode(state_t * state, ei_x_buff * response, const char * bin_data, size_t bin_size, parse_flags_t parse_flags);
	+static void decode(state_t * state, ei_x_buff * response, const char * bin_data, size_t bin_size, parse_flags_t parse_flags, bool fragment, myhtml_tag_id_t context_tag_id);

	static void build_tree(ei_x_buff * response, myhtml_tree_t * tree, parse_flags_t parse_flags);
	static void prepare_node_attrs(ei_x_buff * response, myhtml_tree_node_t * node);

	static inline char * lowercase(char * c);

	static void panic(const char *fmt, ...) AFP(1, 2);
	static void panic(const char *fmt, ...) {
	char buf[4096];
	va_list va;

	va_start (va, fmt);
	vsnprintf (buf, sizeof buf, fmt, va);
	va_end (va);

	fprintf (stderr, "myhtml worker: error: %s\n", buf);
	exit (EXIT_FAILURE);
	}

	static void usage (void) NORETURN;
	static void usage (void) {
	fputs ("usage: myhtml_worker sname hostname cookie tname\n\n"
	" sname the short name you want this c-node to connect as\n"
	" hostname the hostname\n"
	" cookie the authentication cookie\n"
	" tname the target node short name to connect to\n", stderr);
	exit (EXIT_FAILURE);
	}

	int main(int argc, const char *argv[]) {
	#ifdef OTP_22_OR_NEWER
	// initialize erlang client library
	ei_init ();
	#else
	erl_init (NULL, -1);
	#endif

	if (argc != 5)
	usage ();

	const char *sname = argv[1];
	const char *hostname = argv[2];
	const char *cookie = argv[3];
	const char *tname = argv[4];

	char full_name[1024];
	char target_node[1024];

	snprintf (full_name, sizeof full_name, "%s@%s", sname, hostname);
	snprintf (target_node, sizeof target_node, "%s@%s", tname, hostname);

	struct in_addr addr;
	addr.s_addr = htonl(INADDR_ANY);

	// fd to erlang node
	state_t* state = calloc (1, sizeof(state_t));
	state->looping = true;
	ei_x_new (&state->buffer);

	// initialize this node
	printf ("initialising %s\n", full_name);
	if (ei_connect_xinit (&state->ec, hostname, sname, full_name, &addr, cookie, 0) == -1)
	panic ("ei_connect_xinit failed.");

	// connect to target node
	printf ("connecting to %s\n", target_node);
	if ((state->fd = ei_connect (&state->ec, target_node)) < 0)
	panic ("ei_connect failed.");

	state->myhtml = myhtml_create ();
	myhtml_init (state->myhtml, MyHTML_OPTIONS_DEFAULT, 1, 0);

	// signal to stdout that we are ready
	printf ("%s ready\n", full_name);
	fflush (stdout);

	while (state->looping)
	{
	erlang_msg emsg;

	switch (ei_xreceive_msg (state->fd, &emsg, &state->buffer))
	{
	case ERL_TICK:
	break;
	case ERL_ERROR:
	panic ("ei_xreceive_msg: %s\n", strerror (erl_errno));
	break;
	default:
	handle_emsg (state, &emsg);
	break;
	}
	}

	// shutdown: free all state
	ei_x_free (&state->buffer);
	myhtml_destroy (state->myhtml);
	free (state);

	return EXIT_SUCCESS;
	}

	// handle an erlang_msg structure and call handle_send() if relevant
	static void handle_emsg (state_t * state, erlang_msg * emsg)
	{
	state->buffer.index = 0;

	switch (emsg->msgtype)
	{
	case ERL_REG_SEND:
	case ERL_SEND:
	handle_send (state, emsg);
	break;
	case ERL_LINK:
	case ERL_UNLINK:
	break;
	case ERL_EXIT:
	break;
	}
	}

	// handle ERL_SEND message type.
	-// we expect a tuple with arity of 3 in state->buffer.
	-// we expect the first argument to be an atom (`decode`),
	+// we expect a tuple with arity of 3 or 4 in state->buffer.
	+// we expect the first argument to be an atom (`decode` or `decode_fragment`),
	// the second argument to be the HTML payload, and the
	// third argument to be the argument list.
	+// In case of `decode_fragment`, the fourth argument should be
	+// the context tag name.
	// any other message: respond with an {error, unknown_call} tuple.
	static void handle_send (state_t * state, erlang_msg * emsg)
	{
	// response holds our response, prepare it
	ei_x_buff response;

	ei_x_new (&response);

	// check the protocol version, if it's unsupported, panic
	int version;
	if (ei_decode_version (state->buffer.buff, &state->buffer.index, &version) < 0)
	panic ("malformed message - bad version (%d).", version);

	- // decode the tuple header, make sure we have an arity of 3.
	+ // decode the tuple header
	int arity;
	- if (ei_decode_tuple_header (state->buffer.buff, &state->buffer.index, &arity) < 0 \|\| arity != 3)
	+ if (ei_decode_tuple_header (state->buffer.buff, &state->buffer.index, &arity) < 0)
	{
	err_term (&response, "badmatch");
	goto out;
	}

	// the tuple should begin with a `decode` atom.
	char atom[MAXATOMLEN];
	if (ei_decode_atom (state->buffer.buff, &state->buffer.index, atom) < 0)
	{
	err_term (&response, "badmatch");
	goto out;
	}

	+ bool fragment = false;
	if (strcmp (atom, "decode"))
	{
	- err_term (&response, "unknown_call");
	+ if (strcmp (atom, "decode_fragment")) {
	+ err_term (&response, "unknown_call");
	+ goto out;
	+ } else if (arity != 4) {
	+ err_term (&response, "badmatch");
	+ goto out;
	+ } else {
	+ fragment = true;
	+ }
	+ } else if (arity != 3) {
	+ err_term (&response, "badmatch");
	goto out;
	}

	// the next argument should be a binary, allocate it dynamically.
	int bin_type, bin_size;
	if (ei_get_type (state->buffer.buff, &state->buffer.index, &bin_type, &bin_size) < 0)
	panic ("failed to decode binary size in message");

	// verify the type
	if (bin_type != ERL_BINARY_EXT)
	{
	err_term (&response, "badmatch");
	goto out;
	}

	// decode the binary
	char * bin_data = calloc (1, bin_size + 1);
	if (ei_decode_binary (state->buffer.buff, &state->buffer.index, bin_data, NULL) < 0)
	panic ("failed to decode binary in message");

	// next should be the options list
	if (ei_decode_list_header (state->buffer.buff, &state->buffer.index, &arity) < 0)
	panic ("failed to decode options list header in message");

	parse_flags_t parse_flags = decode_parse_flags (state, arity);
	- decode (state, &response, bin_data, bin_size, parse_flags);
	+
	+ // if we are parsing a fragment, context tag name should come next
	+ myhtml_tag_id_t context_tag_id = 0;
	+ if (fragment) {
	+ int context_bin_type, context_bin_size;
	+ if (ei_get_type (state->buffer.buff, &state->buffer.index, &context_bin_type, &context_bin_size) < 0)
	+ panic ("failed to decode binary size in message");
	+
	+ // verify the type
	+ if (context_bin_type != ERL_BINARY_EXT)
	+ {
	+ err_term (&response, "badmatch");
	+ goto out;
	+ }
	+
	+ // decode the binary
	+ char * context_bin_data = calloc (1, context_bin_size + 1);
	+ if (ei_decode_binary (state->buffer.buff, &state->buffer.index, context_bin_data, NULL) < 0)
	+ panic ("failed to decode binary in message");
	+
	+ const myhtml_tag_context_t * context_tag_context = myhtml_tag_static_search(context_bin_data, context_bin_size);
	+
	+ free (context_bin_data);
	+
	+ if (context_tag_context == NULL) {
	+ err_term (&response, "unknown_context_tag");
	+ goto out;
	+ } else {
	+ context_tag_id = context_tag_context->id;
	+ }
	+ }
	+ decode (state, &response, bin_data, bin_size, parse_flags, fragment, context_tag_id);

	free (bin_data);

	out:
	// send response
	ei_send (state->fd, &emsg->from, response.buff, response.buffsz);

	// free response
	ei_x_free (&response);

	return;
	}

	static void err_term (ei_x_buff * response, const char * error_atom)
	{
	response->index = 0;
	ei_x_encode_version (response);
	+ ei_x_encode_tuple_header(response, 2);
	+ ei_x_encode_atom(response, "myhtml_worker");
	ei_x_encode_tuple_header (response, 2);
	ei_x_encode_atom (response, "error");
	ei_x_encode_atom (response, error_atom);
	}

	static parse_flags_t decode_parse_flags (state_t * state, int arity)
	{
	parse_flags_t parse_flags = 0;

	for (int i = 0; i < arity; i++)
	{
	char atom[MAXATOMLEN];

	if (ei_decode_atom (state->buffer.buff, &state->buffer.index, atom) < 0)
	continue;

	if (! strcmp ("html_atoms", atom))
	parse_flags \|= FLAG_HTML_ATOMS;
	else if (! strcmp ("nil_self_closing", atom))
	parse_flags \|= FLAG_NIL_SELF_CLOSING;
	else if (! strcmp ("comment_tuple3", atom))
	parse_flags \|= FLAG_COMMENT_TUPLE3;
	}

	return parse_flags;
	}

	-static void decode (state_t * state, ei_x_buff * response, const char * bin_data, size_t bin_size, parse_flags_t parse_flags)
	+static void decode (state_t * state, ei_x_buff * response, const char * bin_data, size_t bin_size, parse_flags_t parse_flags, bool fragment, myhtml_tag_id_t context_tag_id)
	{
	myhtml_tree_t * tree = myhtml_tree_create ();
	myhtml_tree_init (tree, state->myhtml);
	myhtml_tree_parse_flags_set (tree, MyHTML_TREE_PARSE_FLAGS_WITHOUT_DOCTYPE_IN_TREE);

	// parse tree
	- mystatus_t status = myhtml_parse (tree, MyENCODING_UTF_8, bin_data, bin_size);
	+ mystatus_t status;
	+ if (fragment) {
	+ status = myhtml_parse_fragment(tree, MyENCODING_UTF_8, bin_data, bin_size, context_tag_id, 0);
	+ } else {
	+ status = myhtml_parse (tree, MyENCODING_UTF_8, bin_data, bin_size);
	+ }
	+
	if (status != MyHTML_STATUS_OK)
	{
	err_term (response, "myhtml_parse_failed");
	return;
	}

	// build tree
	build_tree (response, tree, parse_flags);
	myhtml_tree_destroy (tree);
	}

	// a tag is sent as a tuple:
	// - a string or atom for the tag name
	// - an attribute list
	// - a children list
	// in this function, we prepare the atom and complete attribute list
	static void prepare_tag_header (ei_x_buff * response, const char * tag_string, myhtml_tree_node_t * node, parse_flags_t parse_flags)
	{
	myhtml_tag_id_t tag_id = myhtml_node_tag_id (node);
	myhtml_namespace_t tag_ns = myhtml_node_namespace (node);

	ei_x_encode_tuple_header (response, 3);

	if (! (parse_flags & FLAG_HTML_ATOMS) \|\| (tag_id == MyHTML_TAG__UNDEF \|\| tag_id == MyHTML_TAG_LAST_ENTRY \|\| tag_ns != MyHTML_NAMESPACE_HTML))
	ei_x_encode_binary (response, tag_string, strlen (tag_string));
	else
	ei_x_encode_atom (response, tag_string);

	prepare_node_attrs (response, node);
	}

	// prepare an attribute node
	static void prepare_node_attrs(ei_x_buff * response, myhtml_tree_node_t * node)
	{
	myhtml_tree_attr_t * attr;

	for (attr = myhtml_node_attribute_first (node); attr != NULL; attr = myhtml_attribute_next (attr))
	{
	size_t attr_name_len;
	const char *attr_name = myhtml_attribute_key (attr, &attr_name_len);
	size_t attr_value_len;
	const char *attr_value = myhtml_attribute_value (attr, &attr_value_len);

	/* guard against poisoned attribute nodes */
	if (! attr_name_len)
	continue;

	ei_x_encode_list_header (response, 1);
	ei_x_encode_tuple_header (response, 2);
	ei_x_encode_binary (response, attr_name, attr_name_len);

	if (attr_value_len)
	ei_x_encode_binary (response, attr_value, attr_value_len);
	else
	ei_x_encode_binary (response, attr_name, attr_name_len);
	}

	ei_x_encode_empty_list (response);
	}

	// dump a comment node
	static void prepare_comment (ei_x_buff * response, const char * node_comment, size_t comment_len, parse_flags_t parse_flags)
	{
	ei_x_encode_tuple_header (response, parse_flags & FLAG_COMMENT_TUPLE3 ? 3 : 2);
	ei_x_encode_atom (response, "comment");

	if (parse_flags & FLAG_COMMENT_TUPLE3)
	ei_x_encode_list_header (response, 0);

	ei_x_encode_binary (response, node_comment, comment_len);
	}

	#ifdef DEBUG_LIST_MANIP

	#define EMIT_LIST_HDR \
	printf ("list hdr for node %p\n", current_node); \
	fflush (stdout); \
	ei_x_encode_list_header (response, 1)

	#define EMIT_EMPTY_LIST_HDR \
	printf ("list empty for node %p\n", current_node); \
	fflush (stdout); \
	ei_x_encode_list_header (response, 0)

	#define EMIT_LIST_TAIL \
	printf ("list tail for node %p\n", current_node); \
	fflush (stdout); \
	ei_x_encode_empty_list (response)

	#else

	#define EMIT_LIST_HDR ei_x_encode_list_header (response, 1)
	#define EMIT_EMPTY_LIST_HDR ei_x_encode_list_header (response, 0)
	#define EMIT_LIST_TAIL ei_x_encode_empty_list (response)

	#endif

	static void build_tree (ei_x_buff * response, myhtml_tree_t * tree, parse_flags_t parse_flags)
	{
	myhtml_tree_node_t * node = myhtml_tree_get_document (tree);

	tstack stack;
	tstack_init (&stack, 30);
	tstack_push (&stack, node);

	myhtml_tree_node_t * current_node = node->child;

	// ok we're going to send an actual response so start encoding it
	response->index = 0;
	ei_x_encode_version (response);
	ei_x_encode_tuple_header(response, 2);
	ei_x_encode_atom(response, "myhtml_worker");
	+ ei_x_encode_tuple_header(response, 2);
	+ ei_x_encode_atom(response, "ok");

	while (current_node != NULL)
	{
	myhtml_tag_id_t tag_id = myhtml_node_tag_id (current_node);
	myhtml_namespace_t tag_ns = myhtml_node_namespace (current_node);

	if (tag_id == MyHTML_TAG__TEXT)
	{
	size_t text_len;
	const char * node_text = myhtml_node_text (current_node, &text_len);
	EMIT_LIST_HDR;
	ei_x_encode_binary (response, node_text, text_len);
	}
	else if (tag_id == MyHTML_TAG__COMMENT)
	{
	size_t comment_len;
	const char* node_comment = myhtml_node_text (current_node, &comment_len);

	EMIT_LIST_HDR;
	prepare_comment (response, node_comment, comment_len, parse_flags);
	}
	else
	{
	// get name of tag
	size_t tag_name_len;
	const char *tag_name = myhtml_tag_name_by_id (tree, tag_id, &tag_name_len);
	// get namespace of tag
	size_t tag_ns_len;
	const char *tag_ns_name_ptr = myhtml_namespace_name_by_id (tag_ns, &tag_ns_len);
	char buffer [tag_ns_len + tag_name_len + 2];
	char *tag_string = buffer;

	if (tag_ns != MyHTML_NAMESPACE_HTML)
	{
	// tag_ns_name_ptr is unmodifyable, copy it in our tag_ns_buffer to make it modifyable.
	// +1 because myhtml uses strlen for length returned, which doesn't include the null-byte
	// https://github.com/lexborisov/myhtml/blob/0ade0e564a87f46fd21693a7d8c8d1fa09ffb6b6/source/myhtml/mynamespace.c#L80
	char tag_ns_buffer[tag_ns_len + 1];
	strncpy (tag_ns_buffer, tag_ns_name_ptr, sizeof tag_ns_buffer);
	lowercase (tag_ns_buffer);

	snprintf (tag_string, sizeof buffer, "%s:%s", tag_ns_buffer, tag_name);
	}
	else
	{
	// strncpy length does not contain null, so blank the buffer before copying
	// and limit the copy length to buffer size minus one for safety.
	memset (tag_string, '\0', sizeof buffer);
	strncpy (tag_string, tag_name, sizeof buffer - 1);
	}

	EMIT_LIST_HDR;
	prepare_tag_header (response, tag_string, current_node, parse_flags);

	if (current_node->child)
	{
	tstack_push (&stack, current_node);
	current_node = current_node->child;

	continue;
	}
	else
	{
	if (parse_flags & FLAG_NIL_SELF_CLOSING && (myhtml_node_is_close_self(current_node) \|\| myhtml_node_is_void_element(current_node)))
	{
	#ifdef DEBUG_LIST_MANIP
	printf ("self-closing tag %s emit nil?\n", tag_string); fflush (stdout);
	#endif
	ei_x_encode_atom (response, "nil");
	}
	else
	{
	EMIT_EMPTY_LIST_HDR;
	}
	}
	}

	if (current_node->next)
	current_node = current_node->next;
	else
	{
	while (! current_node->next && stack.used != 0)
	{
	EMIT_LIST_TAIL;
	current_node = tstack_pop (&stack);
	}

	if (current_node->next)
	current_node = current_node->next;
	}

	// are we at root?
	if (current_node == node)
	break;
	}

	tstack_free (&stack);
	}

	static inline char * lowercase(char* c)
	{
	char * p = c;

	while (*p)
	{
	p = tolower ((unsigned char) p);
	p++;
	}

	return c;
	}
	diff --git a/lib/fast_html.ex b/lib/fast_html.ex
	index 05e6624..a5ffbad 100644
	--- a/lib/fast_html.ex
	+++ b/lib/fast_html.ex
	@@ -1,74 +1,96 @@
	defmodule :fast_html do
	@moduledoc """
	A module to decode html into a tree structure.
	"""

	@type tag() :: String.t() \| atom()
	@type attr() :: {String.t(), String.t()}
	@type attr_list() :: [] \| [attr()]
	@type comment_node() :: {:comment, String.t()}
	@type comment_node3() :: {:comment, [], String.t()}
	@type tree() ::
	{tag(), attr_list(), tree()}
	\| {tag(), attr_list(), nil}
	\| comment_node()
	\| comment_node3()
	@type format_flag() :: :html_atoms \| :nil_self_closing \| :comment_tuple3

	@doc """
	Returns a tree representation from the given html string.

	`opts` is a keyword list of options, the options available:
	* `timeout` - Call timeout
	* `format` - Format flags for the tree

	The following format flags are available:

	* `:html_atoms` uses atoms for known html tags (faster), binaries for everything else.
	* `:nil_self_closing` uses `nil` to designate self-closing tags and void elements.
	For example `<br>` is then being represented like `{"br", [], nil}`.
	See http://w3c.github.io/html-reference/syntax.html#void-elements for a full list of void elements.
	* `:comment_tuple3` uses 3-tuple elements for comments, instead of the default 2-tuple element.

	## Examples
	iex> :fast_html.decode("<h1>Hello world</h1>")
	{:ok, [{"html", [], [{"head", [], []}, {"body", [], [{"h1", [], ["Hello world"]}]}]}]}

	iex> :fast_html.decode("Hello world", timeout: 0)
	{:error, :timeout}

	iex> :fast_html.decode("<span class='hello'>Hi there</span>")
	{:ok, [{"html", [],
	[{"head", [], []},
	{"body", [], [{"span", [{"class", "hello"}], ["Hi there"]}]}]}]}

	iex> :fast_html.decode("<body><!-- a comment --!></body>")
	{:ok, [{"html", [], [{"head", [], []}, {"body", [], [comment: " a comment "]}]}]}

	iex> :fast_html.decode("<br>")
	{:ok, [{"html", [], [{"head", [], []}, {"body", [], [{"br", [], []}]}]}]}

	iex> :fast_html.decode("<h1>Hello world</h1>", format: [:html_atoms])
	{:ok, [{:html, [], [{:head, [], []}, {:body, [], [{:h1, [], ["Hello world"]}]}]}]}

	iex> :fast_html.decode("<br>", format: [:nil_self_closing])
	{:ok, [{"html", [], [{"head", [], []}, {"body", [], [{"br", [], nil}]}]}]}

	iex> :fast_html.decode("<body><!-- a comment --!></body>", format: [:comment_tuple3])
	{:ok, [{"html", [], [{"head", [], []}, {"body", [], [{:comment, [], " a comment "}]}]}]}

	iex> html = "<body><!-- a comment --!><unknown /></body>"
	iex> :fast_html.decode(html, format: [:html_atoms, :nil_self_closing, :comment_tuple3])
	{:ok, [{:html, [],
	[{:head, [], []},
	{:body, [], [{:comment, [], " a comment "}, {"unknown", [], nil}]}]}]}

	"""
	@spec decode(String.t(), format: [format_flag()]) ::
	{:ok, tree()} \| {:error, String.t() \| atom()}
	def decode(bin, opts \\ []) do
	flags = Keyword.get(opts, :format, [])
	timeout = Keyword.get(opts, :timeout, 10000)
	FastHtml.Cnode.call({:decode, bin, flags}, timeout)
	end
	+
	+ @doc """
	+ Like `decode/2`, but for parsing [HTML fragments](https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments).
	+
	+ `opts` is a keyword list of options, the options available are the same as in `decode/2` with addition of:
	+ * `context` - Name of the context element, defaults to `div`
	+ * `format` - Format flags for the tree
	+
	+ Example:
	+ iex> :fast_html.decode_fragment("rin is the <i>best</i> girl")
	+ {:ok, [{"html", [], ["rin is the ", {"i", [], ["best"]}, " girl"]}]}
	+ iex> :fast_html.decode_fragment("rin is the <i>best</i> girl", context: "title")
	+ {:ok, [{"html", [], ["rin is the <i>best</i> girl"]}]}
	+ iex> :fast_html.decode_fragment("rin is the <i>best</i> girl", context: "objective_truth")
	+ {:error, :unknown_context_tag}
	+ """
	+ def decode_fragment(bin, opts \\ []) do
	+ flags = Keyword.get(opts, :format, [])
	+ timeout = Keyword.get(opts, :timeout, 10000)
	+ context = Keyword.get(opts, :context, "div")
	+ FastHtml.Cnode.call({:decode_fragment, bin, flags, context}, timeout)
	+ end
	end
	diff --git a/lib/fast_html/cnode.ex b/lib/fast_html/cnode.ex
	index 209d125..c71858e 100644
	--- a/lib/fast_html/cnode.ex
	+++ b/lib/fast_html/cnode.ex
	@@ -1,146 +1,146 @@
	defmodule FastHtml.Cnode do
	@moduledoc """
	Manages myhtml c-node.

	## Configuration
	```elixir
	config :fast_html, :cnode,
	sname: "myhtml_worker", # Defaults to myhtml_<random bytes>
	spawn_inactive_timeout: 5000 # Defaults to 10000
	```
	"""

	@spawn_inactive_timeout 10000

	application = Mix.Project.config()[:app]

	use GenServer
	require Logger

	@doc false
	def start_link(args) do
	GenServer.start_link(__MODULE__, args, name: __MODULE__)
	end

	@doc false
	def init(args) do
	exec_path = Path.join(:code.priv_dir(unquote(application)), "myhtml_worker")

	sname = Keyword.get_lazy(args, :sname, &default_sname/0)
	hostname = master_hostname()
	addr = :"#{sname}@#{hostname}"

	spawn_inactive_timeout = Keyword.get(args, :spawn_inactive_timeout, @spawn_inactive_timeout)

	state = %{
	exec_path: exec_path,
	sname: sname,
	addr: addr,
	hostname: hostname,
	spawn_inactive_timeout: spawn_inactive_timeout
	}

	connect_or_spawn_cnode(state)
	end

	defp default_sname, do: "myhtml_#{FastHtml.Application.random_sname()}"
	defp master_sname, do: Node.self() \|> to_string \|> String.split("@") \|> List.first()
	defp master_hostname, do: Node.self() \|> to_string \|> String.split("@") \|> List.last()

	defp connect_or_spawn_cnode(state) do
	case connect_cnode(state) do
	{:stop, _} -> spawn_cnode(state)
	{:ok, state} -> state
	end
	end

	defp connect_cnode(%{addr: addr} = state) do
	if Node.connect(addr) do
	Logger.debug("connected to #{addr}")
	{:ok, state}
	else
	Logger.debug("connecting to #{addr} failed")
	{:stop, :cnode_connection_fail}
	end
	end

	defp spawn_cnode(%{exec_path: exec_path, sname: sname, hostname: hostname} = state) do
	Logger.debug("Spawning #{sname}@#{hostname}")

	cookie = :erlang.get_cookie()

	port =
	Port.open({:spawn_executable, exec_path}, [
	:binary,
	:exit_status,
	:stderr_to_stdout,
	line: 4096,
	args: [sname, hostname, cookie, master_sname()]
	])

	pid = Keyword.get(Port.info(port), :os_pid)
	state = Map.put(state, :pid, pid)
	await_cnode_ready(port, state)
	end

	defp await_cnode_ready(
	port,
	%{spawn_inactive_timeout: timeout, addr: addr} = state
	) do
	ready_line = to_string(addr) <> " ready"

	receive do
	{^port, {:data, {:eol, ^ready_line}}} ->
	connect_cnode(state)

	{^port, {:data, {:eol, line}}} ->
	Logger.debug("c-node is saying: #{line}")
	await_cnode_ready(port, state)

	{^port, {:exit_status, exit_status}} ->
	Logger.debug("unexpected c-node exit: #{exit_status}")
	{:stop, :cnode_unexpected_exit}

	message ->
	Logger.warn("unhandled message while waiting for cnode to be ready:\n#{inspect(message)}")
	await_cnode_ready(port, state)
	after
	timeout ->
	{:stop, :spawn_inactive_timeout}
	end
	end

	@doc false
	def handle_info({:nodedown, _cnode}, state) do
	{:stop, :nodedown, state}
	end

	@doc false
	def handle_info(msg, state) do
	Logger.warn("unhandled handle_info: #{inspect(msg)}")
	{:noreply, state}
	end

	@doc false
	def handle_call(:addr, _from, %{addr: addr} = state) do
	{:reply, addr, state}
	end

	@doc false
	def terminate(_reason, %{pid: pid}) when pid != nil do
	System.cmd("kill", ["-9", to_string(pid)])
	:normal
	end

	@doc "Call into myhtml cnode"
	def call(msg, timeout \\ 10000) do
	node = GenServer.call(__MODULE__, :addr)
	send({nil, node}, msg)

	receive do
	- {:myhtml_worker, res} -> {:ok, res}
	+ {:myhtml_worker, res} -> res
	after
	timeout -> {:error, :timeout}
	end
	end
	end
	diff --git a/mix.exs b/mix.exs
	index 50c97ad..2ce9a13 100644
	--- a/mix.exs
	+++ b/mix.exs
	@@ -1,157 +1,157 @@
	defmodule FastHtml.Mixfile do
	use Mix.Project

	def project do
	[
	app: :fast_html,
	- version: "1.0.0",
	+ version: "1.0.1",
	elixir: "~> 1.5",
	deps: deps(),
	package: package(),
	compilers: [:fast_html_cnode_make] ++ Mix.compilers(),
	build_embedded: Mix.env() == :prod,
	start_permanent: Mix.env() == :prod,
	name: "FastHtml",
	description: """
	A module to decode HTML into a tree,
	porting all properties of the underlying
	library myhtml, being fast and correct
	in regards to the html spec.

	Originally based on Myhtmlex.
	""",
	docs: docs()
	]
	end

	def package do
	[
	maintainers: ["Ariadne Conill", "rinpatch"],
	licenses: ["GNU LGPL"],
	links: %{
	"GitLab" => "https://git.pleroma.social/pleroma/fast_html",
	"Issues" => "https://git.pleroma.social/pleroma/fast_html/issues",
	"MyHTML" => "https://github.com/lexborisov/myhtml"
	},
	files:
	[
	"lib",
	"priv/.gitignore",
	"test",
	"Makefile",
	"mix.exs",
	"README.md",
	"LICENSE"
	] ++
	Enum.reject(
	Path.wildcard("c_src/*/"),
	fn path ->
	Path.extname(path) in [".so", ".a", ".o"] or
	String.starts_with?(Path.basename(path), ".") or File.dir?(path)
	end
	)
	]
	end

	def application do
	[
	extra_applications: [:logger],
	mod: {FastHtml.Application, []},
	# used to detect conflicts with other applications named processes
	registered: [FastHtml.Cnode, FastHtml.Supervisor]
	]
	end

	defp deps do
	[
	# documentation helpers
	{:ex_doc, "~> 0.19", only: :dev},
	# benchmarking helpers
	{:benchee, "~> 1.0", only: :bench, optional: true},
	{:myhtmlex, "~> 0.2.0", only: :bench, runtime: false, optional: true},
	{:mochiweb, "~> 2.18", only: :bench, optional: true},
	{:html5ever, "~> 0.7.0", only: :bench, optional: true}
	]
	end

	defp docs do
	[
	main: "readme",
	extras: ["README.md"]
	]
	end
	end

	defmodule Mix.Tasks.Compile.FastHtmlCnodeMake do
	@artifacts [
	"priv/myhtml_worker"
	]

	def find_make do
	_make_cmd =
	System.get_env("MAKE") \|\|
	case :os.type() do
	{:unix, :freebsd} -> "gmake"
	{:unix, :openbsd} -> "gmake"
	{:unix, :netbsd} -> "gmake"
	{:unix, :dragonfly} -> "gmake"
	_ -> "make"
	end
	end

	defp otp_version do
	:erlang.system_info(:otp_release)
	\|> to_string()
	\|> String.to_integer()
	end

	defp otp_22_or_newer? do
	otp_version() >= 22
	end

	def run(_) do
	make_cmd = find_make()

	exit_code =
	if match?({:win32, _}, :os.type()) do
	IO.warn("Windows is not yet a target.")
	1
	else
	{result, exit_code} =
	System.cmd(
	make_cmd,
	@artifacts,
	stderr_to_stdout: true,
	env: [
	{"MIX_ENV", to_string(Mix.env())},
	{"OTP22_DEF", (otp_22_or_newer?() && "YES") \|\| "NO"}
	]
	)

	IO.binwrite(result)
	exit_code
	end

	if exit_code == 0 do
	:ok
	else
	{:error,
	[
	%Mix.Task.Compiler.Diagnostic{
	compiler_name: "FastHtml Cnode",
	message: "Make exited with #{exit_code}",
	severity: :error,
	file: nil,
	position: nil
	}
	]}
	end
	end

	def clean() do
	make_cmd = find_make()
	{result, _error_code} = System.cmd(make_cmd, ["clean"], stderr_to_stdout: true)
	Mix.shell().info(result)
	:ok
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Wed, Oct 14, 8:54 AM (1 d, 21 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1787065
Default Alt Text: (32 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions