Page MenuHomePhorge

No OneTemporary

Size
45 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/gen_magic/config.ex b/lib/gen_magic/config.ex
index 0ee2ace..70ae355 100644
--- a/lib/gen_magic/config.ex
+++ b/lib/gen_magic/config.ex
@@ -1,57 +1,44 @@
defmodule GenMagic.Config do
@moduledoc false
@otp_app Mix.Project.config()[:app]
@executable_name "apprentice"
@startup_timeout 1_000
@process_timeout 30_000
@recycle_threshold :infinity
- @database_patterns [:default]
def get_port_name do
{:spawn_executable, to_charlist(get_executable_name())}
end
- def get_port_options(options) do
- arguments = [:use_stdio, :binary, :exit_status, {:packet, 2}]
-
- case get_executable_arguments(options) do
- [] -> arguments
- list -> [{:args, list} | arguments]
- end
+ def get_port_options(_options) do
+ [:use_stdio, :binary, :exit_status, {:packet, 2}]
end
def get_startup_timeout(options) do
get_value(options, :startup_timeout, @startup_timeout)
end
def get_process_timeout(options) do
get_value(options, :process_timeout, @process_timeout)
end
def get_recycle_threshold(options) do
get_value(options, :recycle_threshold, @recycle_threshold)
end
defp get_executable_name do
Path.join(:code.priv_dir(@otp_app), @executable_name)
end
- defp get_executable_arguments(options) do
- Enum.flat_map(List.wrap(get(options, :database_patterns, @database_patterns)), fn
- :default -> ["--database-default"]
- pattern -> pattern |> Path.wildcard() |> Enum.flat_map(&["--database-file", &1])
- end)
- end
-
defp get(options, key, default) do
Keyword.get(options, key, default)
end
defp get_value(options, key, default) do
case get(options, key, default) do
value when is_integer(value) and value > 0 -> value
:infinity -> :infinity
_ -> raise ArgumentError, message: "Invalid #{key}"
end
end
end
diff --git a/lib/gen_magic/server.ex b/lib/gen_magic/server.ex
index fc686b2..545b024 100644
--- a/lib/gen_magic/server.ex
+++ b/lib/gen_magic/server.ex
@@ -1,313 +1,420 @@
defmodule GenMagic.Server do
@moduledoc """
Provides access to the underlying libmagic client, which performs file introspection.
The Server needs to be supervised, since it will terminate if it receives any unexpected error.
"""
@behaviour :gen_statem
alias GenMagic.Result
alias GenMagic.Server.Data
alias GenMagic.Server.Status
import Kernel, except: [send: 2]
+ require Logger
@typedoc """
Represents the reference to the underlying server, as returned by `:gen_statem`.
"""
@type t :: :gen_statem.server_ref()
@typedoc """
Represents values accepted as startup options, which can be passed to `start_link/1`.
- `:name`: If present, this will be the registered name for the underlying process.
Note that `:gen_statem` requires `{:local, name}`, but given widespread GenServer convention,
atoms are accepted and will be converted to `{:local, name}`.
- `:startup_timeout`: Specifies how long the Server waits for the C program to initialise.
However, if the underlying C program exits, then the process exits immediately.
Can be set to `:infinity`.
- `:process_timeout`: Specifies how long the Server waits for each request to complete.
Can be set to `:infinity`.
Please note that, if you have chosen a custom timeout value, you should also pass it when
using `GenMagic.Server.perform/3`.
- `:recycle_threshold`: Specifies the number of requests processed before the underlying C
program is recycled.
Can be set to `:infinity` if you do not wish for the program to be recycled.
- `:database_patterns`: Specifies what magic databases to load; you can specify a list of either
Path Patterns (see `Path.wildcard/2`) or `:default` to instruct the C program to load the
appropriate databases.
For example, if you have had to add custom magics, then you can set this value to:
[:default, "path/to/my/magic"]
"""
+ @database_patterns [:default]
@type option ::
{:name, atom() | :gen_statem.server_name()}
| {:startup_timeout, timeout()}
| {:process_timeout, timeout()}
| {:recycle_threshold, non_neg_integer() | :infinity}
| {:database_patterns, nonempty_list(:default | Path.t())}
@typedoc """
Current state of the Server:
- `:pending`: This is the initial state; the Server will attempt to start the underlying Port
and the libmagic client, then automatically transition to either Available or Crashed.
- `:available`: This is the default state. In this state the Server is able to accept requests
and they will be replied in the same order.
- `:processing`: This is the state the Server will be in if it is processing requests. In this
state, further requests can still be lodged and they will be processed when the Server is
available again.
For proper concurrency, use a process pool like Poolboy, Sbroker, etc.
- `:recycling`: This is the state the Server will be in, if its underlying C program needs to be
recycled. This state is triggered whenever the cycle count reaches the defined value as per
`:recycle_threshold`.
In this state, the Server is able to accept requests, but they will not be processed until the
underlying C server program has been started again.
"""
@type state :: :starting | :processing | :available | :recycling
@spec child_spec([option()]) :: Supervisor.child_spec()
@spec start_link([option()]) :: :gen_statem.start_ret()
@spec perform(t(), Path.t() | {:bytes, binary()}, timeout()) ::
- {:ok, Result.t()} | {:error, term() | String.t()}
+ {:ok, Result.t()} | {:error, term() | String.t()}
@spec status(t(), timeout()) :: {:ok, Status.t()} | {:error, term()}
@spec stop(t(), term(), timeout()) :: :ok
@doc """
Returns the default Child Specification for this Server for use in Supervisors.
You can override this with `Supervisor.child_spec/2` as required.
"""
def child_spec(options) do
%{
id: __MODULE__,
start: {__MODULE__, :start_link, [options]},
type: :worker,
restart: :permanent,
shutdown: 500
}
end
@doc """
Starts a new Server.
See `t:option/0` for further details.
"""
def start_link(options) do
{name, options} = Keyword.pop(options, :name)
case name do
nil -> :gen_statem.start_link(__MODULE__, options, [])
name when is_atom(name) -> :gen_statem.start_link({:local, name}, __MODULE__, options, [])
{:global, _} -> :gen_statem.start_link(name, __MODULE__, options, [])
{:via, _, _} -> :gen_statem.start_link(name, __MODULE__, options, [])
{:local, _} -> :gen_statem.start_link(name, __MODULE__, options, [])
end
end
@doc """
Determines the type of the file provided.
"""
def perform(server_ref, path, timeout \\ 5000) do
case :gen_statem.call(server_ref, {:perform, path}, timeout) do
{:ok, %Result{} = result} -> {:ok, result}
{:error, reason} -> {:error, reason}
end
end
+ @doc """
+ Reloads a Server with a new set of databases.
+ """
+ def reload(server_ref, database_patterns \\ nil, timeout \\ 5000) do
+ :gen_statem.call(server_ref, {:reload, database_patterns}, timeout)
+ end
+
+ @doc """
+ Same as `reload/2,3` but with a full restart of the underlying C port.
+ """
+ def recycle(server_ref, database_patterns \\ nil, timeout \\ 5000) do
+ :gen_statem.call(server_ref, {:recycle, database_patterns}, timeout)
+ end
+
@doc """
Returns status of the Server.
"""
def status(server_ref, timeout \\ 5000) do
:gen_statem.call(server_ref, :status, timeout)
end
@doc """
Stops the Server with reason `:normal` and timeout `:infinity`.
"""
def stop(server_ref) do
:gen_statem.stop(server_ref)
end
@doc """
Stops the Server with the specified reason and timeout.
"""
def stop(server_ref, reason, timeout) do
:gen_statem.stop(server_ref, reason, timeout)
end
@impl :gen_statem
def init(options) do
import GenMagic.Config
data = %Data{
port_name: get_port_name(),
+ database_patterns: Keyword.get(options, :database_patterns, []),
port_options: get_port_options(options),
startup_timeout: get_startup_timeout(options),
process_timeout: get_process_timeout(options),
recycle_threshold: get_recycle_threshold(options)
}
{:ok, :starting, data}
end
@impl :gen_statem
def callback_mode do
[:state_functions, :state_enter]
end
@doc false
- def starting(:enter, _, %{request: nil, port: nil} = data) do
+ def starting(:enter, _, %{port: nil} = data) do
port = Port.open(data.port_name, data.port_options)
{:keep_state, %{data | port: port}, data.startup_timeout}
end
+ @doc false
+ def starting(:enter, _, data) do
+ {:keep_state_and_data, data.startup_timeout}
+ end
+
@doc false
def starting({:call, from}, :status, data) do
handle_status_call(from, :starting, data)
end
@doc false
def starting({:call, _from}, {:perform, _path}, _data) do
{:keep_state_and_data, :postpone}
end
@doc false
def starting(:info, {port, {:data, ready}}, %{port: port} = data) do
case :erlang.binary_to_term(ready) do
- :ready -> {:next_state, :available, data}
+ :ready -> {:next_state, :loading, data}
end
end
def starting(:info, {port, {:exit_status, code}}, %{port: port} = data) do
error =
case code do
- 1 -> :no_database
- 2 -> :no_argument
- 3 -> :missing_database
- 4 -> :ei_alloc_failed
- 5 -> :ei_bad_term
+ 1 -> :bad_db
+ 2 -> :ei_error
+ 3 -> :ei_bad_term
code -> {:unexpected_error, code}
end
{:stop, {:error, error}, data}
end
+ def loading(:enter, _old_state, data) do
+ databases = Enum.flat_map(List.wrap(data.database_patterns || @database_patterns), fn
+ :default -> [:default]
+ pattern -> Path.wildcard(pattern)
+ end)
+ databases = if databases == [] do
+ [:default]
+ else
+ databases
+ end
+ {:keep_state, {databases, data}, {:state_timeout, 0, :load}}
+ end
+
+ def loading(:state_timeout, :load_timeout, {[database | _], data}) do
+ {:stop, {:error, {:database_loading_timeout, database}}, data}
+ end
+
+ def loading(:state_timeout, :load, {[], data}) do
+ {:next_state, :available, data}
+ end
+
+ def loading(:state_timeout, :load, {[database | databases], data} = state) do
+ command = case database do
+ :default -> {:add_default_database, nil}
+ path -> {:add_database, database}
+ end
+ send(data.port, command)
+ {:keep_state, state, {:state_timeout, data.startup_timeout, :load_timeout}}
+ end
+
+ def loading(:info, {port, {:data, response}}, {[database | databases], %{port: port} = data}) do
+ case :erlang.binary_to_term(response) do
+ {:ok, :loaded} ->
+ {:keep_state, {databases, data}, {:state_timeout, 0, :load}}
+ end
+ end
+
+ def loading(:info, {port, {:exit_status, 1}}, {[database | _], %{port: port} = data}) do
+ {:stop, {:error, {:database_not_found, database}}, data}
+ end
+
+ @doc false
+ def loading({:call, from}, :status, {[database | _], data}) do
+ handle_status_call(from, :loading, data)
+ end
+
@doc false
+ def loading({:call, _from}, {:perform, _path}, _data) do
+ {:keep_state_and_data, :postpone}
+ end
+
+ @doc false
+ def available(:enter, _old_state, %{request: {:reload, from, _}}) do
+ response = {:reply, from, :ok}
+ {:keep_state_and_data, response}
+ end
+
def available(:enter, _old_state, %{request: nil}) do
:keep_state_and_data
end
@doc false
def available({:call, from}, {:perform, path}, data) do
data = %{data | cycles: data.cycles + 1, request: {path, from, :erlang.now()}}
arg =
case path do
path when is_binary(path) -> {:file, path}
{:bytes, bytes} -> {:bytes, bytes}
end
send(data.port, arg)
{:next_state, :processing, data}
end
+ def available({:call, from}, {:reload, databases}, data) do
+ send(data.port, {:reload, :reload})
+ {:next_state, :starting,
+ %{data | database_patterns: databases || data.database_patterns, request: {:reload, from, :reload}}}
+ end
+
+ def available({:call, from}, {:recycle, databases}, data) do
+ {:next_state, :recycling,
+ %{data | database_patterns: databases || data.database_patterns, request: {:reload, from, :recycle}}}
+ end
+
@doc false
def available({:call, from}, :status, data) do
handle_status_call(from, :available, data)
end
@doc false
def processing(:enter, _old_state, %{request: {_path, _from, _time}} = data) do
{:keep_state_and_data, data.process_timeout}
end
@doc false
def processing({:call, _from}, {:perform, _path}, _data) do
{:keep_state_and_data, :postpone}
end
@doc false
def processing({:call, from}, :status, data) do
handle_status_call(from, :processing, data)
end
+ @doc false
+ def processing(:state_timeout, _, %{port: port, request: {_, from, _}} = data) do
+ response = {:reply, from, {:error, :timeout}}
+ {:next_state, :recycling, %{data | request: nil}, [response, :hibernate]}
+ end
+
@doc false
def processing(:info, {port, {:data, response}}, %{port: port, request: {_, from, _}} = data) do
response = {:reply, from, handle_response(response)}
next_state = (data.cycles >= data.recycle_threshold && :recycling) || :available
{:next_state, next_state, %{data | request: nil}, [response, :hibernate]}
end
@doc false
- def recycling(:enter, _, %{request: nil, port: port} = data) when is_port(port) do
+ def recycling(:enter, _, %{port: port} = data) when is_port(port) do
send(data.port, {:stop, :recycle})
- {:keep_state_and_data, data.startup_timeout}
+ {:keep_state_and_data, {:state_timeout, data.startup_timeout, :stop}}
end
@doc false
def recycling({:call, _from}, {:perform, _path}, _data) do
{:keep_state_and_data, :postpone}
end
@doc false
def recycling({:call, from}, :status, data) do
handle_status_call(from, :recycling, data)
end
+ # In case of timeout, force close.
+ def recycling(:state_timeout, :stop, data) do
+ Kernel.send(data.port, {self(), :close})
+ {:keep_state_and_data, {:state_timeout, data.startup_timeout, :close}}
+ end
+
+ def recycling(:state_timeout, :close, data) do
+ {:stop, {:error, :port_close_failed}}
+ end
+
+ def recycling(:info, {port, :closed}, %{port: port} = data) do
+ {:next_state, :starting, %{data | port: nil, cycles: 0}}
+ end
+
@doc false
- def recycling(:info, {port, {:exit_status, 0}}, %{port: port} = data) do
+ def recycling(:info, {port, {:exit_status, _}}, %{port: port} = data) do
{:next_state, :starting, %{data | port: nil, cycles: 0}}
end
@doc false
@impl :gen_statem
def terminate(_, _, %{port: port}) do
Kernel.send(port, {self(), :close})
end
def terminate(_, _, _) do
:ok
end
defp send(port, command) do
Kernel.send(port, {self(), {:command, :erlang.term_to_binary(command)}})
end
@errnos %{
2 => :enoent,
13 => :eaccess,
20 => :enotdir,
12 => :enomem,
24 => :emfile,
36 => :enametoolong
}
@errno Map.keys(@errnos)
defp handle_response(data) do
case :erlang.binary_to_term(data) do
{:ok, {mime_type, encoding, content}} -> {:ok, Result.build(mime_type, encoding, content)}
{:error, {errno, _}} when errno in @errno -> {:error, @errnos[errno]}
{:error, {errno, string}} -> {:error, "#{errno}: #{string}"}
{:error, _} = error -> error
end
end
defp handle_status_call(from, state, data) do
response = {:ok, %__MODULE__.Status{state: state, cycles: data.cycles}}
{:keep_state_and_data, {:reply, from, response}}
end
end
diff --git a/lib/gen_magic/server/data.ex b/lib/gen_magic/server/data.ex
index 6836327..25d23e9 100644
--- a/lib/gen_magic/server/data.ex
+++ b/lib/gen_magic/server/data.ex
@@ -1,25 +1,26 @@
defmodule GenMagic.Server.Data do
@moduledoc false
@type request :: {Path.t(), {pid(), term()}, requested_at :: integer()}
@type t :: %__MODULE__{
port_name: Port.name(),
port_options: list(),
port: port(),
startup_timeout: timeout(),
process_timeout: timeout(),
recycle_threshold: non_neg_integer() | :infinity,
cycles: non_neg_integer(),
request: request | nil
}
defstruct port_name: nil,
port_options: nil,
port: nil,
startup_timeout: :infinity,
process_timeout: :infinity,
recycle_threshold: :infinity,
cycles: 0,
+ database_patterns: nil,
request: nil
end
diff --git a/src/apprentice.c b/src/apprentice.c
index 806a288..da8afaf 100644
--- a/src/apprentice.c
+++ b/src/apprentice.c
@@ -1,449 +1,425 @@
//
// The Sorcerer’s Apprentice
//
// To use this program, compile it with dynamically linked libmagic, as mirrored
// at https://github.com/file/file. You may install it with apt-get,
// yum or brew. Refer to the Makefile for further reference.
//
// This program is designed to run interactively as a backend daemon to the
-// GenMagic library, and follows the command line pattern:
-//
-// $ apprentice --database-file <file> --database-default
-//
-// Where each argument either refers to a compiled or uncompiled magic database,
-// or the default database. They will be loaded in the sequence that they were
-// specified. Note that you must specify at least one database.
+// GenMagic library.
//
// Communication is done over STDIN/STDOUT as binary packets of 2 bytes length
// plus X bytes payload, where the payload is an erlang term encoded with
// :erlang.term_to_binary/1 and decoded with :erlang.binary_to_term/1.
//
-// Once the program is ready, it sends the `:ready` atom. The startup can fail
-// for multiples reasons, and the program will exit accordingly:
-// - 1: No database
-// - 2: Missing/Bad argument
-// - 3: Missing database
+// Once the program is ready, it sends the `:ready` atom.
+//
+// It is then up to the Erlang side to load databases, by sending messages:
+// - `{:add_database, path}`
+// - `{:add_default_database, _}`
+//
+// If the requested database have been loaded, an `{:ok, :loaded}` message will
+// follow. Otherwise, the process will exit (exit code 1).
//
// Commands are sent to the program STDIN as an erlang term of `{Operation,
// Argument}`, and response of `{:ok | :error, Response}`.
//
-// Invalid packets will cause the program to exit (exit code 4). This will
+// Invalid packets will cause the program to exit (exit code 3). This will
// happen if your Erlang Term format doesn't match the version the program has
// been compiled with, or if you send a command too huge.
//
-// The program may exit with error codes 5 or 255 if something went wrong (such
-// as error allocating terms, or if stdin is lost).
+// The program may exit with exit code 3 if something went wrong with ei_*
+// functions.
//
// Commands:
+// {:reload, _} :: :ready
+// {:add_database, String.t()} :: {:ok, _} | {:error, _}
+// {:add_default_database, _} :: {:ok, _} | {:error, _}
// {:file, path :: String.t()} :: {:ok, {type, encoding, name}} | {:error,
// :badarg} | {:error, {errno :: integer(), String.t()}}
// {:bytes, binary()} :: same as :file
// {:stop, reason :: atom()} :: exit 0
#include <arpa/inet.h>
#include <ei.h>
#include <errno.h>
#include <getopt.h>
#include <libgen.h>
#include <magic.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#define ERROR_OK 0
-#define ERROR_NO_DATABASE 1
-#define ERROR_NO_ARGUMENT 2
-#define ERROR_MISSING_DATABASE 3
-#define ERROR_EI 4
-#define ERROR_BAD_TERM 5
+#define ERROR_DB 1
+#define ERROR_EI 2
+#define ERROR_BAD_TERM 3
// We use a bigger than possible valid command length (around 4111 bytes) to
// allow more precise errors when using too long paths.
#define COMMAND_LEN 8000
#define COMMAND_BUFFER_SIZE COMMAND_LEN + 1
#define MAGIC_FLAGS_COMMON (MAGIC_CHECK | MAGIC_ERROR)
magic_t magic_setup(int flags);
#define EI_ENSURE(result) \
do { \
if (result != 0) { \
+ fprintf(stderr, "EI ERROR, line: %d", __LINE__); \
exit(ERROR_EI); \
} \
} while (0);
typedef char byte;
void setup_environment();
-void setup_options(int argc, char **argv);
-void setup_options_file(char *optarg);
-void setup_options_default();
-void setup_system();
+void magic_open_all();
+int magic_load_all(char *path);
int process_command(uint16_t len, byte *buf);
void process_file(char *path, ei_x_buff *result);
void process_bytes(char *bytes, int size, ei_x_buff *result);
size_t read_cmd(byte *buf);
size_t write_cmd(byte *buf, size_t len);
void error(ei_x_buff *result, const char *error);
void handle_magic_error(magic_t handle, int errn, ei_x_buff *result);
void fdseek(uint16_t count);
-struct magic_file {
- struct magic_file *prev;
- struct magic_file *next;
- char *path;
-};
-
-static struct magic_file *magic_database;
static magic_t magic_mime_type; // MAGIC_MIME_TYPE
static magic_t magic_mime_encoding; // MAGIC_MIME_ENCODING
static magic_t magic_type_name; // MAGIC_NONE
int main(int argc, char **argv) {
- ei_init();
+ EI_ENSURE(ei_init());
setup_environment();
- setup_options(argc, argv);
- setup_system();
-
- ei_x_buff ok_buf;
- EI_ENSURE(ei_x_new_with_version(&ok_buf));
- EI_ENSURE(ei_x_encode_atom(&ok_buf, "ready"));
- write_cmd(ok_buf.buff, ok_buf.index);
- EI_ENSURE(ei_x_free(&ok_buf));
+ magic_open_all();
byte buf[COMMAND_BUFFER_SIZE];
uint16_t len;
while ((len = read_cmd(buf)) > 0) {
process_command(len, buf);
}
return 255;
}
int process_command(uint16_t len, byte *buf) {
ei_x_buff result;
char atom[128];
int index, version, arity, termtype, termsize;
index = 0;
// Initialize result
EI_ENSURE(ei_x_new_with_version(&result));
EI_ENSURE(ei_x_encode_tuple_header(&result, 2));
if (len >= COMMAND_LEN) {
error(&result, "badarg");
return 1;
}
if (ei_decode_version(buf, &index, &version) != 0) {
exit(ERROR_BAD_TERM);
}
if (ei_decode_tuple_header(buf, &index, &arity) != 0) {
error(&result, "badarg");
return 1;
}
if (arity != 2) {
error(&result, "badarg");
return 1;
}
if (ei_decode_atom(buf, &index, atom) != 0) {
error(&result, "badarg");
return 1;
}
+ // {:file, path}
if (strlen(atom) == 4 && strncmp(atom, "file", 4) == 0) {
char path[4097];
ei_get_type(buf, &index, &termtype, &termsize);
if (termtype == ERL_BINARY_EXT) {
if (termsize < 4096) {
long bin_length;
EI_ENSURE(ei_decode_binary(buf, &index, path, &bin_length));
path[termsize] = '\0';
process_file(path, &result);
} else {
error(&result, "enametoolong");
return 1;
}
} else {
error(&result, "badarg");
return 1;
}
+ // {:bytes, bytes}
} else if (strlen(atom) == 5 && strncmp(atom, "bytes", 5) == 0) {
int termtype;
int termsize;
char bytes[51];
EI_ENSURE(ei_get_type(buf, &index, &termtype, &termsize));
if (termtype == ERL_BINARY_EXT && termsize < 50) {
long bin_length;
EI_ENSURE(ei_decode_binary(buf, &index, bytes, &bin_length));
bytes[termsize] = '\0';
process_bytes(bytes, termsize, &result);
} else {
error(&result, "badarg");
return 1;
}
+ // {:add_database, path}
+ } else if (strlen(atom) == 12 && strncmp(atom, "add_database", 12) == 0) {
+ char path[4097];
+ ei_get_type(buf, &index, &termtype, &termsize);
+
+ if (termtype == ERL_BINARY_EXT) {
+ if (termsize < 4096) {
+ long bin_length;
+ EI_ENSURE(ei_decode_binary(buf, &index, path, &bin_length));
+ path[termsize] = '\0';
+ if (magic_load_all(path) == 0) {
+ EI_ENSURE(ei_x_encode_atom(&result, "ok"));
+ EI_ENSURE(ei_x_encode_atom(&result, "loaded"));
+ } else {
+ exit(ERROR_DB);
+ }
+ } else {
+ error(&result, "enametoolong");
+ return 1;
+ }
+ } else {
+ error(&result, "badarg");
+ return 1;
+ }
+ // {:add_default_database, _}
+ } else if (strlen(atom) == 20 &&
+ strncmp(atom, "add_default_database", 20) == 0) {
+ if (magic_load_all(NULL) == 0) {
+ EI_ENSURE(ei_x_encode_atom(&result, "ok"));
+ EI_ENSURE(ei_x_encode_atom(&result, "loaded"));
+ } else {
+ exit(ERROR_DB);
+ }
+ // {:reload, _}
+ } else if (strlen(atom) == 6 && strncmp(atom, "reload", 6) == 0) {
+ magic_open_all();
+ return 0;
+ // {:stop, _}
} else if (strlen(atom) == 4 && strncmp(atom, "stop", 4) == 0) {
exit(ERROR_OK);
+ // badarg
} else {
error(&result, "badarg");
return 1;
}
write_cmd(result.buff, result.index);
EI_ENSURE(ei_x_free(&result));
return 0;
}
void setup_environment() { opterr = 0; }
-void setup_options(int argc, char **argv) {
- const char *option_string = "f:";
- static struct option long_options[] = {
- {"database-file", required_argument, 0, 'f'},
- {"database-default", no_argument, 0, 'd'},
- {0, 0, 0, 0}};
-
- int option_character;
- while (1) {
- int option_index = 0;
- option_character =
- getopt_long(argc, argv, option_string, long_options, &option_index);
- if (-1 == option_character) {
- break;
- }
- switch (option_character) {
- case 'f': {
- setup_options_file(optarg);
- break;
- }
- case 'd': {
- setup_options_default();
- break;
- }
- case '?':
- default: {
- exit(ERROR_NO_ARGUMENT);
- break;
- }
- }
- }
-}
-
-void setup_options_file(char *optarg) {
- if (0 != access(optarg, R_OK)) {
- exit(ERROR_MISSING_DATABASE);
+void magic_open_all() {
+ if (magic_mime_encoding) {
+ magic_close(magic_mime_encoding);
}
-
- struct magic_file *next = malloc(sizeof(struct magic_file));
- size_t path_length = strlen(optarg) + 1;
- char *path = malloc(path_length);
- memcpy(path, optarg, path_length);
- next->path = path;
- next->prev = magic_database;
- if (magic_database) {
- magic_database->next = next;
+ if (magic_mime_type) {
+ magic_close(magic_mime_type);
}
- magic_database = next;
-}
-
-void setup_options_default() {
- struct magic_file *next = malloc(sizeof(struct magic_file));
- next->path = NULL;
- next->prev = magic_database;
- if (magic_database) {
- magic_database->next = next;
+ if (magic_type_name) {
+ magic_close(magic_type_name);
}
- magic_database = next;
-}
+ magic_mime_encoding = magic_open(MAGIC_FLAGS_COMMON | MAGIC_MIME_ENCODING);
+ magic_mime_type = magic_open(MAGIC_FLAGS_COMMON | MAGIC_MIME_TYPE);
+ magic_type_name = magic_open(MAGIC_FLAGS_COMMON | MAGIC_NONE);
-void setup_system() {
- magic_mime_encoding = magic_setup(MAGIC_FLAGS_COMMON | MAGIC_MIME_ENCODING);
- magic_mime_type = magic_setup(MAGIC_FLAGS_COMMON | MAGIC_MIME_TYPE);
- magic_type_name = magic_setup(MAGIC_FLAGS_COMMON | MAGIC_NONE);
+ ei_x_buff ok_buf;
+ EI_ENSURE(ei_x_new_with_version(&ok_buf));
+ EI_ENSURE(ei_x_encode_atom(&ok_buf, "ready"));
+ write_cmd(ok_buf.buff, ok_buf.index);
+ EI_ENSURE(ei_x_free(&ok_buf));
}
-magic_t magic_setup(int flags) {
+int magic_load_all(char *path) {
+ int res;
- magic_t magic = magic_open(flags);
- struct magic_file *current_database = magic_database;
- if (!current_database) {
- exit(ERROR_NO_DATABASE);
+ if ((res = magic_load(magic_mime_encoding, path)) != 0) {
+ return res;
}
-
- while (current_database->prev) {
- current_database = current_database->prev;
+ if ((res = magic_load(magic_mime_type, path)) != 0) {
+ return res;
}
- while (current_database) {
- magic_load(magic, current_database->path);
- current_database = current_database->next;
+ if ((res = magic_load(magic_type_name, path)) != 0) {
+ return res;
}
- return magic;
+ return 0;
}
void process_bytes(char *path, int size, ei_x_buff *result) {
const char *mime_type_result = magic_buffer(magic_mime_type, path, size);
const int mime_type_errno = magic_errno(magic_mime_type);
if (mime_type_errno > 0) {
handle_magic_error(magic_mime_type, mime_type_errno, result);
return;
}
const char *mime_encoding_result =
magic_buffer(magic_mime_encoding, path, size);
int mime_encoding_errno = magic_errno(magic_mime_encoding);
if (mime_encoding_errno > 0) {
handle_magic_error(magic_mime_encoding, mime_encoding_errno, result);
return;
}
const char *type_name_result = magic_buffer(magic_type_name, path, size);
int type_name_errno = magic_errno(magic_type_name);
if (type_name_errno > 0) {
handle_magic_error(magic_type_name, type_name_errno, result);
return;
}
EI_ENSURE(ei_x_encode_atom(result, "ok"));
EI_ENSURE(ei_x_encode_tuple_header(result, 3));
EI_ENSURE(
ei_x_encode_binary(result, mime_type_result, strlen(mime_type_result)));
EI_ENSURE(ei_x_encode_binary(result, mime_encoding_result,
strlen(mime_encoding_result)));
EI_ENSURE(
ei_x_encode_binary(result, type_name_result, strlen(type_name_result)));
return;
}
void handle_magic_error(magic_t handle, int errn, ei_x_buff *result) {
const char *error = magic_error(handle);
EI_ENSURE(ei_x_encode_atom(result, "error"));
EI_ENSURE(ei_x_encode_tuple_header(result, 2));
long errlon = (long)errn;
EI_ENSURE(ei_x_encode_long(result, errlon));
EI_ENSURE(ei_x_encode_binary(result, error, strlen(error)));
return;
}
void process_file(char *path, ei_x_buff *result) {
const char *mime_type_result = magic_file(magic_mime_type, path);
const int mime_type_errno = magic_errno(magic_mime_type);
if (mime_type_errno > 0) {
handle_magic_error(magic_mime_type, mime_type_errno, result);
return;
}
const char *mime_encoding_result = magic_file(magic_mime_encoding, path);
int mime_encoding_errno = magic_errno(magic_mime_encoding);
if (mime_encoding_errno > 0) {
handle_magic_error(magic_mime_encoding, mime_encoding_errno, result);
return;
}
const char *type_name_result = magic_file(magic_type_name, path);
int type_name_errno = magic_errno(magic_type_name);
if (type_name_errno > 0) {
handle_magic_error(magic_type_name, type_name_errno, result);
return;
}
EI_ENSURE(ei_x_encode_atom(result, "ok"));
EI_ENSURE(ei_x_encode_tuple_header(result, 3));
EI_ENSURE(
ei_x_encode_binary(result, mime_type_result, strlen(mime_type_result)));
EI_ENSURE(ei_x_encode_binary(result, mime_encoding_result,
strlen(mime_encoding_result)));
EI_ENSURE(
ei_x_encode_binary(result, type_name_result, strlen(type_name_result)));
return;
}
// Adapted from https://erlang.org/doc/tutorial/erl_interface.html
// Changed `read_cmd`, the original one was buggy given some length (due to
// endinaness).
// TODO: Check if `write_cmd` exhibits the same issue.
size_t read_exact(byte *buf, size_t len) {
int i, got = 0;
do {
if ((i = read(0, buf + got, len - got)) <= 0) {
return (i);
}
got += i;
} while (got < len);
return (len);
}
size_t write_exact(byte *buf, size_t len) {
int i, wrote = 0;
do {
if ((i = write(1, buf + wrote, len - wrote)) <= 0)
return (i);
wrote += i;
} while (wrote < len);
return (len);
}
size_t read_cmd(byte *buf) {
int i;
if ((i = read(0, buf, sizeof(uint16_t))) <= 0) {
return (i);
}
uint16_t len16 = *(uint16_t *)buf;
len16 = ntohs(len16);
// Buffer isn't large enough: just return possible len, without reading.
// Up to the caller of verifying the size again and return an error.
// buf left unchanged, stdin emptied of X bytes.
if (len16 > COMMAND_LEN) {
fdseek(len16);
return len16;
}
return read_exact(buf, len16);
}
size_t write_cmd(byte *buf, size_t len) {
byte li;
li = (len >> 8) & 0xff;
write_exact(&li, 1);
li = len & 0xff;
write_exact(&li, 1);
return write_exact(buf, len);
}
void error(ei_x_buff *result, const char *error) {
EI_ENSURE(ei_x_encode_atom(result, "error"));
EI_ENSURE(ei_x_encode_atom(result, error));
write_cmd(result->buff, result->index);
EI_ENSURE(ei_x_free(result));
}
void fdseek(uint16_t count) {
int i = 0;
while (i < count) {
getchar();
i += 1;
}
}
diff --git a/test/gen_magic/apprentice_test.exs b/test/gen_magic/apprentice_test.exs
index f7f0bb1..74c2081 100644
--- a/test/gen_magic/apprentice_test.exs
+++ b/test/gen_magic/apprentice_test.exs
@@ -1,159 +1,169 @@
defmodule GenMagic.ApprenticeTest do
use GenMagic.MagicCase
@tmp_path "/tmp/testgenmagicx"
require Logger
test "sends ready" do
port = Port.open(GenMagic.Config.get_port_name(), GenMagic.Config.get_port_options([]))
on_exit(fn() -> send(port, {self(), :close}) end)
- assert_ready(port)
+ assert_ready_and_init_default(port)
end
test "stops" do
port = Port.open(GenMagic.Config.get_port_name(), GenMagic.Config.get_port_options([]))
on_exit(fn() -> send(port, {self(), :close}) end)
- assert_ready(port)
+ assert_ready_and_init_default(port)
send(port, {self(), {:command, :erlang.term_to_binary({:stop, :stop})}})
assert_receive {^port, {:exit_status, 0}}
end
- test "exits with no database" do
+ test "exits with non existent database with an error" do
opts = [:use_stdio, :binary, :exit_status, {:packet, 2}, {:args, []}]
port = Port.open(GenMagic.Config.get_port_name(), opts)
on_exit(fn() -> send(port, {self(), :close}) end)
+ assert_ready(port)
+ send(port, {self(), {:command, :erlang.term_to_binary({:add_database, "/somewhere/nowhere"})}})
assert_receive {^port, {:exit_status, 1}}
end
- test "exits with a non existent database" do
- opts = [
- {:args, ["--database-file", "/no/such/database"]},
- :use_stdio,
- :binary,
- :exit_status,
- {:packet, 2}
- ]
-
- port = Port.open(GenMagic.Config.get_port_name(), opts)
- on_exit(fn() -> send(port, {self(), :close}) end)
- assert_receive {^port, {:exit_status, 3}}
- end
+ #test "exits with a non existent database" do
+ # opts = [
+ # {:args, ["--database-file", "/no/such/database"]},
+ # :use_stdio,
+ # :binary,
+ # :exit_status,
+ # {:packet, 2}
+ # ]
+ #
+ # port = Port.open(GenMagic.Config.get_port_name(), opts)
+ # on_exit(fn() -> send(port, {self(), :close}) end)
+ # assert_receive {^port, {:exit_status, 3}}
+ #end
describe "port" do
setup do
port = Port.open(GenMagic.Config.get_port_name(), GenMagic.Config.get_port_options([]))
on_exit(fn() -> send(port, {self(), :close}) end)
- assert_ready(port)
+ assert_ready_and_init_default(port)
%{port: port}
end
test "exits with badly formatted erlang terms", %{port: port} do
send(port, {self(), {:command, "i forgot to term_to_binary!!"}})
- assert_receive {^port, {:exit_status, 5}}
+ assert_receive {^port, {:exit_status, 3}}
end
test "errors with wrong command", %{port: port} do
send(port, {self(), {:command, :erlang.term_to_binary(:wrong)}})
assert_receive {^port, {:data, data}}
assert {:error, :badarg} = :erlang.binary_to_term(data)
refute_receive _
send(port, {self(), {:command, :erlang.term_to_binary({:file, 42})}})
assert_receive {^port, {:data, data}}
assert {:error, :badarg} = :erlang.binary_to_term(data)
refute_receive _
send(port, {self(), {:command, :erlang.term_to_binary("more wrong")}})
assert_receive {^port, {:data, data}}
assert {:error, :badarg} = :erlang.binary_to_term(data)
refute_receive _
send(port, {self(), {:command, :erlang.term_to_binary({"no", "no"})}})
assert_receive {^port, {:data, data}}
assert {:error, :badarg} = :erlang.binary_to_term(data)
refute_receive _
end
test "file works", %{port: port} do
send(port, {self(), {:command, :erlang.term_to_binary({:file, Path.expand("Makefile")})}})
assert_receive {^port, {:data, data}}
assert {:ok, _} = :erlang.binary_to_term(data)
end
test "bytes works", %{port: port} do
send(port, {self(), {:command, :erlang.term_to_binary({:bytes, "some bytes!"})}})
assert_receive {^port, {:data, data}}
assert {:ok, _} = :erlang.binary_to_term(data)
end
test "fails with non existent file", %{port: port} do
send(port, {self(), {:command, :erlang.term_to_binary({:file, "/path/to/nowhere"})}})
assert_receive {^port, {:data, data}}
assert {:error, _} = :erlang.binary_to_term(data)
end
test "works with big file path", %{port: port} do
# Test with longest valid path.
{dir, bigfile} = too_big(@tmp_path, "/a")
case File.mkdir_p(dir) do
:ok ->
File.touch!(bigfile)
on_exit(fn -> File.rm_rf!(@tmp_path) end)
send(port, {self(), {:command, :erlang.term_to_binary({:file, bigfile})}})
assert_receive {^port, {:data, data}}
assert {:ok, _} = :erlang.binary_to_term(data)
refute_receive _
# This path should be long enough for buffers, but larger than a valid path name.
# Magic will return an errno 36.
file = @tmp_path <> String.duplicate("a", 256)
send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
assert_receive {^port, {:data, data}}
assert {:error, {36, _}} = :erlang.binary_to_term(data)
refute_receive _
# Theses filename should be too big for the path buffer.
file = bigfile <> "aaaaaaaaaa"
send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
assert_receive {^port, {:data, data}}
assert {:error, :enametoolong} = :erlang.binary_to_term(data)
refute_receive _
# This call should be larger than the COMMAND_BUFFER_SIZE. Ensure nothing bad happens!
file = String.duplicate(bigfile, 4)
send(port, {self(), {:command, :erlang.term_to_binary({:file, file})}})
assert_receive {^port, {:data, data}}
assert {:error, :badarg} = :erlang.binary_to_term(data)
refute_receive _
# We re-run a valid call to ensure the buffer/... haven't been corrupted in port land.
send(port, {self(), {:command, :erlang.term_to_binary({:file, bigfile})}})
assert_receive {^port, {:data, data}}
assert {:ok, _} = :erlang.binary_to_term(data)
refute_receive _
{:error, :enametoolong} ->
Logger.info("Skipping test, operating system does not support max POSIX length for directories")
:ignore
end
end
end
def assert_ready(port) do
assert_receive {^port, {:data, data}}
assert :ready == :erlang.binary_to_term(data)
end
+ def assert_ready_and_init_default(port) do
+ assert_receive {^port, {:data, data}}
+ assert :ready == :erlang.binary_to_term(data)
+ send(port, {self(), {:command, :erlang.term_to_binary({:add_default_database, nil})}})
+ assert_receive {^port, {:data, data}}
+ assert {:ok, _} = :erlang.binary_to_term(data)
+ end
+
def too_big(path, filename, limit \\ 4095) do
last_len = byte_size(filename)
path_len = byte_size(path)
needed = limit - (last_len + path_len)
extra = make_too_big(needed, "")
{path <> extra, path <> extra <> filename}
end
def make_too_big(needed, acc) when needed <= 255 do
acc <> "/" <> String.duplicate("a", needed - 1)
end
def make_too_big(needed, acc) do
acc = acc <> "/" <> String.duplicate("a", 254)
make_too_big(needed - 255, acc)
end
end
diff --git a/test/gen_magic/gen_magic_test.exs b/test/gen_magic/gen_magic_test.exs
index 9699177..209b832 100644
--- a/test/gen_magic/gen_magic_test.exs
+++ b/test/gen_magic/gen_magic_test.exs
@@ -1,55 +1,72 @@
defmodule GenMagicTest do
use GenMagic.MagicCase
alias GenMagic.Result
doctest GenMagic
@iterations 100
test "Makefile is text file" do
{:ok, pid} = GenMagic.Server.start_link([])
path = absolute_path("Makefile")
assert {:ok, %{mime_type: "text/x-makefile"}} = GenMagic.Server.perform(pid, path)
end
@tag external: true
test "Load test local files" do
{:ok, pid} = GenMagic.Server.start_link([])
files_stream()
|> Stream.cycle()
|> Stream.take(@iterations)
|> Stream.map(&assert {:ok, %Result{}} = GenMagic.Server.perform(pid, &1))
|> Enum.all?()
|> assert
end
test "Non-existent file" do
- Process.flag(:trap_exit, true)
{:ok, pid} = GenMagic.Server.start_link([])
path = missing_filename()
assert_no_file(GenMagic.Server.perform(pid, path))
end
test "Named process" do
{:ok, pid} = GenMagic.Server.start_link(name: :gen_magic)
path = absolute_path("Makefile")
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(:gen_magic)
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
assert {:ok, %Result{} = result} = GenMagic.Server.perform(:gen_magic, path)
assert {:ok, %{cycles: 1}} = GenMagic.Server.status(:gen_magic)
assert {:ok, %{cycles: 1}} = GenMagic.Server.status(pid)
assert "text/x-makefile" = result.mime_type
end
- test "Custom database file recognises Elixir files" do
- database = absolute_path("elixir.mgc")
- on_exit(fn() -> File.rm(database) end)
- {_, 0} = System.cmd("file", ["-C", "-m", absolute_path("test/elixir")])
- {:ok, pid} = GenMagic.Server.start_link(database_patterns: [database])
- path = absolute_path("mix.exs")
- assert {:ok, %Result{} = result} = GenMagic.Server.perform(pid, path)
- assert "text/x-elixir" = result.mime_type
- assert "us-ascii" = result.encoding
- assert "Elixir module source text" = result.content
+ describe "custom database" do
+
+ setup do
+ database = absolute_path("elixir.mgc")
+ on_exit(fn() -> File.rm(database) end)
+ {_, 0} = System.cmd("file", ["-C", "-m", absolute_path("test/elixir")])
+ [database: database]
+ end
+
+ test "recognises Elixir files", %{database: database} do
+ {:ok, pid} = GenMagic.Server.start_link(database_patterns: [database])
+ path = absolute_path("mix.exs")
+ assert {:ok, %Result{} = result} = GenMagic.Server.perform(pid, path)
+ assert "text/x-elixir" = result.mime_type
+ assert "us-ascii" = result.encoding
+ assert "Elixir module source text" = result.content
+ end
+
+ test "recognises Elixir files after a reload", %{database: database} do
+ {:ok, pid} = GenMagic.Server.start_link([])
+ path = absolute_path("mix.exs")
+ {:ok, %Result{mime_type: mime}} = GenMagic.Server.perform(pid, path)
+ refute mime == "text/x-elixir"
+ :ok = GenMagic.Server.reload(pid, [database])
+ assert {:ok, %Result{mime_type: "text/x-elixir"}} = GenMagic.Server.perform(pid, path)
+ end
+
end
+
end
diff --git a/test/gen_magic/server_test.exs b/test/gen_magic/server_test.exs
index 53e5830..d89ea5f 100644
--- a/test/gen_magic/server_test.exs
+++ b/test/gen_magic/server_test.exs
@@ -1,34 +1,45 @@
defmodule GenMagic.ServerTest do
use GenMagic.MagicCase
doctest GenMagic.Server
describe "recycle_threshold" do
test "resets" do
{:ok, pid} = GenMagic.Server.start_link(recycle_threshold: 3)
path = absolute_path("Makefile")
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
assert {:ok, %{cycles: 1}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
assert {:ok, %{cycles: 2}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
Process.sleep(100)
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
end
test "resets before reply" do
{:ok, pid} = GenMagic.Server.start_link(recycle_threshold: 1)
path = absolute_path("Makefile")
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
Process.sleep(100)
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
Process.sleep(100)
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
assert {:ok, _} = GenMagic.Server.perform(pid, path)
Process.sleep(100)
assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
end
end
+
+ test "recycle" do
+ {:ok, pid} = GenMagic.Server.start_link([])
+ path = absolute_path("Makefile")
+ assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
+ assert {:ok, _} = GenMagic.Server.perform(pid, path)
+ assert {:ok, %{cycles: 1}} = GenMagic.Server.status(pid)
+ assert :ok = GenMagic.Server.recycle(pid)
+ assert {:ok, %{cycles: 0}} = GenMagic.Server.status(pid)
+ end
+
end

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 24, 9:13 PM (1 d, 5 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39488
Default Alt Text
(45 KB)

Event Timeline