No OneTemporary
Actions

Size

18 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/flake_id.ex b/lib/flake_id.ex
	index 1548aa3..ac1daed 100644
	--- a/lib/flake_id.ex
	+++ b/lib/flake_id.ex
	@@ -1,157 +1,157 @@
	# FlakeId: Decentralized, k-ordered ID generation service
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: LGPL-3.0-only

	defmodule FlakeId do
	@moduledoc """
	Decentralized, k-ordered ID generation service.
	"""

	import Kernel, except: [to_string: 1]
	alias FlakeId.Builder
	- alias FlakeId.Worker
	alias FlakeId.Local
	+ alias FlakeId.Worker

	@type error ::
	{:error, :clock_running_backwards} \| {:error, :clock_stuck} \| {:error, :invalid_state}

	@doc """
	Converts a binary Flake to a String

	## Examples

	iex> FlakeId.to_string(<<0, 0, 1, 109, 67, 124, 251, 125, 95, 28, 30, 59, 36, 42, 0, 0>>)
	"9n2ciuz1wdesFnrGJU"

	"""

	def to_string(<<_::integer-size(64), _::integer-size(48), _::integer-size(16)>> = binary_flake) do
	<<integer::integer-size(128)>> = binary_flake
	Base62.encode(integer)
	end

	def to_string(string), do: string

	@doc """
	Converts a String to a binary Flake

	## Examples

	iex> FlakeId.from_string("9n2ciuz1wdesFnrGJU")
	<<0, 0, 1, 109, 67, 124, 251, 125, 95, 28, 30, 59, 36, 42, 0, 0>>

	"""
	@spec from_string(String.t()) :: nil \| <<_::128>>
	def from_string(string)
	def from_string(<<_::integer-size(128)>> = flake), do: flake
	def from_string(string) when is_binary(string) and byte_size(string) < 18, do: nil
	def from_string(string), do: string \|> Base62.decode!() \|> from_integer

	@doc """
	Converts a binary Flake to an integer

	## Examples

	iex> FlakeId.to_integer(<<0, 0, 1, 109, 67, 124, 251, 125, 95, 28, 30, 59, 36, 42, 0, 0>>)
	28939165907792829150732718047232

	"""
	@spec to_integer(<<_::128>>) :: non_neg_integer
	def to_integer(binary_flake)
	def to_integer(<<integer::integer-size(128)>>), do: integer

	@doc """
	Converts an integer to a binary Flake

	## Examples

	iex> FlakeId.from_integer(28939165907792829150732718047232)
	<<0, 0, 1, 109, 67, 124, 251, 125, 95, 28, 30, 59, 36, 42, 0, 0>>

	"""
	@spec from_integer(integer) :: <<_::128>>
	def from_integer(integer) do
	<<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
	<<integer::integer-size(128)>>
	end

	@doc """
	Generates a string with Flake
	"""
	@spec get(atom()) :: {:ok, String.t()} \| error()
	def get(name \\ Worker) do
	case Worker.get(name) do
	{:ok, flake} -> {:ok, to_string(flake)}
	error -> error
	end
	end

	@doc """
	Generate a in-process flake, without hitting a node-wide worker. This is especially useful when you have
	long-running processes that do a lot of insertions/ID generation. Note that the first call will be slowest as it
	has to setup the Flake state.

	Each process can have its Flake state stored in the process dictionary and the Worker ID is derived from the global
	Worker ID and the current PID.
	"""
	@spec get_local :: {:ok, String.t()} \| error()
	def get_local do
	case Local.get() do
	{:ok, flake} -> {:ok, to_string(flake)}
	error -> error
	end
	end

	@doc """
	If you wish to insert historical data while preserving sorting, you can achieve this using a backdated flake. In that
	case you have to provide yourself a UNIQUE Worker ID (for the given time). Be very careful to ensure the uniqueness
	of the Worker ID otherwise you may generate the same flake twice.
	"""
	@spec backdate(any(), DateTime.t()) :: {:ok, String.t()} \| error()
	def backdate(datetime, unique) do
	hash = :erlang.phash2(unique)
	time = DateTime.to_unix(datetime, :millisecond)

	case Builder.get(time, %Builder{node: hash}) do
	{:ok, flake, _} -> {:ok, to_string(flake)}
	error -> error
	end
	end

	@doc """
	Generates a notional FlakeId (with an empty worker id) given a timestamp.

	This is useful for generating a lexical range of values that could have been generated in a span of time.
	"""
	def notional(datetime) do
	time = DateTime.to_unix(datetime, :millisecond)

	case Builder.get(time, %Builder{node: 0}) do
	{:ok, flake, _} -> {:ok, to_string(flake)}
	error -> error
	end
	end

	@doc """
	Checks that ID is a valid FlakeId

	## Examples

	iex> FlakeId.flake_id?("9n2ciuz1wdesFnrGJU")
	true

	iex> FlakeId.flake_id?("#cofe")
	false

	iex> FlakeId.flake_id?("pleroma.social")
	false
	"""
	@spec flake_id?(String.t()) :: boolean
	def flake_id?(id), do: flake_id?(String.to_charlist(id), true)

	defp flake_id?([c \| cs], true) when c >= ?0 and c <= ?9, do: flake_id?(cs, true)
	defp flake_id?([c \| cs], true) when c >= ?A and c <= ?Z, do: flake_id?(cs, true)
	defp flake_id?([c \| cs], true) when c >= ?a and c <= ?z, do: flake_id?(cs, true)
	defp flake_id?([], true), do: true
	defp flake_id?(_, _), do: false
	end
	diff --git a/lib/flake_id/application.ex b/lib/flake_id/application.ex
	index 72e42b4..383ebac 100644
	--- a/lib/flake_id/application.ex
	+++ b/lib/flake_id/application.ex
	@@ -1,36 +1,39 @@
	# FlakeId: Decentralized, k-ordered ID generation service
	# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: LGPL-3.0-only

	defmodule FlakeId.Application do
	@moduledoc false

	use Application
	require Logger

	def start(_type, _args) do
	- if elem(default_clock(), 1) == :os do
	+ unless multi_time_warp?() do
	Logger.error("""
	FlakeId: Time consistency is not optimal, using system time.
	More information at: https://hexdocs.pm/flake_id/readme.html#time-warp
	""")
	end

	children = [
	FlakeId.TimeHint,
	FlakeId.Worker
	]

	opts = [strategy: :one_for_one, name: FlakeId.Supervisor]
	Supervisor.start_link(children, opts)
	end

	- def default_clock() do
	- if :erlang.system_info(:time_warp_mode) == :multi_time_warp do
	+ def default_clock do
	+ if multi_time_warp?() do
	{:erlang, :system_time}
	else
	{:os, :system_time}
	end
	end

	+ def multi_time_warp? do
	+ :erlang.system_info(:time_warp_mode) == :multi_time_warp
	+ end
	end
	diff --git a/lib/flake_id/builder.ex b/lib/flake_id/builder.ex
	index 9164dcc..b446afe 100644
	--- a/lib/flake_id/builder.ex
	+++ b/lib/flake_id/builder.ex
	@@ -1,122 +1,124 @@
	# FlakeId: Decentralized, k-ordered ID generation service
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: LGPL-3.0-only

	defmodule FlakeId.Builder do
	@moduledoc false

	defstruct node: nil, time: 0, sq: 0, clock: nil

	@type t :: %__MODULE__{
	node: non_neg_integer,
	time: non_neg_integer,
	sq: non_neg_integer
	}

	- @spec new(options) :: t() when options: [worker_id \| time_hint \| clock],
	- worker_id: {:worker_id, :random \| :node \| {:iface, String.t()} \| Integer.t \| String.t()},
	- time_hint: {:time_hint, false \| true \| atom()},
	- clock: {module(), atom()}
	+ @spec new(options) :: t()
	+ when options: [worker_id \| time_hint \| clock],
	+ worker_id:
	+ {:worker_id, :random \| :node \| {:iface, String.t()} \| Integer.t() \| String.t()},
	+ time_hint: {:time_hint, false \| true \| atom()},
	+ clock: {module(), atom()}
	def new(options \\ []) do
	worker_id = worker_id(Keyword.get(options, :worker_id))

	time =
	case Keyword.get(options, :time_hint) do
	false -> 0
	name when is_atom(name) -> FlakeId.TimeHint.get(name)
	_ -> FlakeId.TimeHint.get()
	end

	clock = Keyword.get(options, :clock, FlakeId.Application.default_clock())
	%__MODULE__{node: worker_id, time: time, clock: clock}
	end

	@spec backwards?(t()) :: boolean()
	def backwards?(%__MODULE__{time: time, clock: clock}) do
	time > time(clock)
	end

	- def time() do
	+ def time do
	time(FlakeId.Application.default_clock())
	end

	@spec time(t() \| {module(), atom()}) :: non_neg_integer()
	def time(%__MODULE__{clock: clock}) do
	time(clock)
	end

	def time({mod, fun}) do
	apply(mod, fun, [:millisecond])
	end

	@spec get(t) :: {:ok, <<_::128>>, t} \| FlakeId.error()
	- def get(state = %__MODULE__{clock: clock}) do
	+ def get(%__MODULE__{clock: clock} = state) do
	get(time(clock), state)
	end

	@spec get(non_neg_integer(), t()) :: {:ok, <<_::128>>, t()} \| FlakeId.error()

	# Error when time didn't change and sequence is too big for 16bit int.
	def get(time, %__MODULE__{time: time, sq: sq}) when sq >= 65_535 do
	{:error, :clock_stuck}
	end

	# Matches when the calling time is the same as the state time. Incr. sq
	def get(time, %__MODULE__{node: node, time: time, sq: seq} = state)
	when is_integer(time) and is_integer(node) and is_integer(seq) do
	new_state = %__MODULE__{state \| sq: seq + 1}
	{:ok, gen_flake(new_state), new_state}
	end

	# Error when clock is running backwards
	def get(newtime, %__MODULE__{time: time}) when newtime < time do
	{:error, :clock_running_backwards}
	end

	# Matches when the times are different, reset sq
	def get(newtime, %__MODULE__{node: node, time: time, sq: seq} = state)
	when is_integer(time) and is_integer(node) and is_integer(seq) and newtime > time do
	new_state = %__MODULE__{state \| time: newtime, sq: 0}
	{:ok, gen_flake(new_state), new_state}
	end

	def get(_, _) do
	{:error, :invalid_state}
	end

	@spec gen_flake(t) :: <<_::128>>
	defp gen_flake(%__MODULE__{time: time, node: node, sq: seq}) do
	<<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
	end

	def worker_id(setting \\ nil)

	def worker_id(nil) do
	worker_id(Application.get_env(:flake_id, :worker_id, :random))
	end

	def worker_id(:node) do
	:erlang.phash2(node())
	end

	def worker_id({:mac, iface}) do
	{:ok, addresses} = :inet.getifaddrs()
	proplist = :proplists.get_value(iface, addresses)
	hwaddr = Enum.take(:proplists.get_value(:hwaddr, proplist), 6)
	<<worker::integer-size(48)>> = :binary.list_to_bin(hwaddr)
	worker
	end

	def worker_id(:random) do
	<<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
	worker
	end

	def worker_id(integer) when is_integer(integer) do
	integer
	end

	def worker_id(binary) when is_binary(binary) do
	:erlang.phash2(binary)
	end
	end
	diff --git a/lib/flake_id/local.ex b/lib/flake_id/local.ex
	index e624a96..ddd3cb8 100644
	--- a/lib/flake_id/local.ex
	+++ b/lib/flake_id/local.ex
	@@ -1,29 +1,29 @@
	# FlakeId: Decentralized, k-ordered ID generation service
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: LGPL-3.0-only

	defmodule FlakeId.Local do
	@moduledoc false

	alias FlakeId.Builder

	- @spec get() :: {:ok, String.t()} \| FlakeId.error()
	- def get() do
	+ @spec get :: {:ok, String.t()} \| FlakeId.error()
	+ def get do
	case Builder.get(get_state()) do
	{:ok, flake, state} ->
	Process.put(__MODULE__, state)
	{:ok, flake}

	error = {:error, _} ->
	error
	end
	end

	- defp get_state() do
	+ defp get_state do
	Process.get(__MODULE__, Builder.new(worker_id: worker_id()))
	end

	- defp worker_id() do
	+ defp worker_id do
	:erlang.phash2({Builder.worker_id(), self()})
	end
	end
	diff --git a/lib/flake_id/time_hint.ex b/lib/flake_id/time_hint.ex
	index 633e7e3..02498ef 100644
	--- a/lib/flake_id/time_hint.ex
	+++ b/lib/flake_id/time_hint.ex
	@@ -1,125 +1,129 @@
	# FlakeId: Decentralized, k-ordered ID generation service
	# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: LGPL-3.0-only

	defmodule FlakeId.TimeHint do
	@moduledoc false
	use GenServer
	require Logger

	@tick :timer.minutes(1)
	@default_env false
	@default_file "flake_id_time"

	@spec get() :: 0 \| non_neg_integer
	@doc "Returns the latest time hint, or zero if disabled"
	def get(name \\ nil) do
	if name \|\| Application.get_env(:flake_id, :time_hint, @default_env) do
	GenServer.call(name \|\| __MODULE__, :get)
	else
	0
	end
	end

	- @spec start_link([] \| options) :: GenServer.on_start when options: [name \| file \| tick \| clock],
	- name: {:name, atom()},
	- file: {:file, Path.t()},
	- tick: {:tick, non_neg_integer()},
	- clock: {module(), atom()}
	+ @spec start_link([] \| options) :: GenServer.on_start()
	+ when options: [name \| file \| tick \| clock],
	+ name: {:name, atom()},
	+ file: {:file, Path.t()},
	+ tick: {:tick, non_neg_integer()},
	+ clock: {module(), atom()}
	def start_link(options)

	def start_link([]) do
	if Application.get_env(:flake_id, :time_hint, false) do
	file =
	Application.get_env(
	:flake_id,
	:time_hint_file,
	Application.app_dir(:flake_id, ["priv", @default_file])
	)

	tick = Application.get_env(:flake_id, :time_hint_tick, @tick)
	- clock = Application.get_env(:flake_id, :time_hint_clock, FlakeId.Application.default_clock())
	+
	+ clock =
	+ Application.get_env(:flake_id, :time_hint_clock, FlakeId.Application.default_clock())
	+
	GenServer.start_link(__MODULE__, [file, tick, clock], name: __MODULE__)
	else
	:ignore
	end
	end

	def start_link(options) do
	name = Keyword.fetch!(options, :name)
	file = Keyword.fetch!(options, :file)
	tick = Keyword.get(options, :tick, @tick)
	clock = Keyword.get(options, :clock, FlakeId.Application.default_clock())
	GenServer.start_link(__MODULE__, [file, tick, clock], name: name)
	end

	@impl true
	def init([file, tick, clock]) do
	Process.send_after(self(), :tick, tick)
	{:ok, {file, tick, clock, read_and_update(file, clock)}}
	end

	@impl true
	- def handle_call(:get, _, state = {_, _, _, time}) do
	+ def handle_call(:get, _, {_, _, _, time} = state) do
	{:reply, time, state}
	end

	@impl true
	def handle_info(:tick, {file, tick, clock, prev_time}) do
	Process.send_after(self(), :tick, tick)
	{:noreply, {file, tick, clock, update(file, clock, prev_time)}}
	end

	if Mix.env() == :test do
	@impl true
	def handle_info(:stop, state) do
	{:stop, :normal, state}
	end
	end

	@impl true
	def terminate(_, {file, _, clock, prev_time}) do
	update(file, clock, prev_time)
	end

	defp read_and_update(file, clock) do
	update(file, clock, read(file))
	end

	- defp update(file, {mod,fun}, prev_time) do
	+ defp update(file, {mod, fun}, prev_time) do
	time = apply(mod, fun, [:millisecond])

	if time > prev_time do
	:ok = write(file, time)
	time
	else
	prev_time
	end
	end

	defp read(file) do
	case File.read(file) do
	{:ok, data} ->
	{__MODULE__, time} = :erlang.binary_to_term(data)
	time

	{:error, :enoent} ->
	0

	error ->
	Logger.error("#{__MODULE__}: could not read file #{file}: #{inspect(error)}")
	throw(error)
	end
	end

	defp write(file, time) do
	case File.write(file, :erlang.term_to_binary({__MODULE__, time})) do
	:ok ->
	:ok

	error ->
	Logger.error("#{__MODULE__}: could not write file #{file}: #{inspect(error)}")
	error
	end
	end
	end
	diff --git a/test/flake_id/builder_test.exs b/test/flake_id/builder_test.exs
	index c6801bb..e6b5072 100644
	--- a/test/flake_id/builder_test.exs
	+++ b/test/flake_id/builder_test.exs
	@@ -1,70 +1,70 @@
	defmodule FlakeId.BuilderTest do
	use ExUnit.Case, async: true

	alias FlakeId.Builder

	describe "get/2" do
	test "increment `:sq` when the calling time is the same as the state time" do
	builder = Builder.new()
	time = builder.time

	assert {:ok, <<_::integer-size(128)>>, %Builder{time: ^time, sq: 1}} =
	Builder.get(time, builder)
	end

	test "reset `:sq` when the times are different" do
	time = Builder.time()
	node = Builder.worker_id()
	state = %Builder{time: time - 1, node: node, sq: 42}

	assert {:ok, _, %Builder{time: ^time, sq: 0}} = Builder.get(time, state)
	end

	test "error when clock is running backwards" do
	time = Builder.time()
	state = %Builder{time: time + 1}

	assert Builder.get(time, state) == {:error, :clock_running_backwards}
	end

	test "error when clock is stuck and `sq` overflows" do
	time = Builder.time()
	- state = %Builder{node: 0, time: time, sq: 65534}
	+ state = %Builder{node: 0, time: time, sq: 65_534}
	assert {:ok, _, %Builder{sq: sq}} = Builder.get(time, state)
	state = %Builder{state \| node: 0, sq: sq}
	assert Builder.get(time, state) == {:error, :clock_stuck}
	end
	end

	test "uses initial time from TimeHint" do
	file = "TEST-flake-id-builder-initial-time-hint"
	on_exit(fn -> File.rm(file) end)
	FakeClock.set(42)

	{:ok, _} =
	FlakeId.TimeHint.start_link(name: TestTimeHint, file: file, clock: {FakeClock, :time})

	builder = Builder.new(time_hint: TestTimeHint)
	assert builder.time == FlakeId.TimeHint.get(TestTimeHint)
	end

	describe "worker_id" do
	test "random" do
	builder1 = Builder.new(worker_id: :random)
	builder2 = Builder.new(worker_id: :random)
	refute builder1.node == builder2.node
	end

	test "integer" do
	- integer = :rand.uniform(10000)
	+ integer = :rand.uniform(10_000)
	builder = Builder.new(worker_id: integer)
	assert builder.node == integer
	end

	test "binary" do
	hash = :erlang.phash2("hello")
	builder = Builder.new(worker_id: "hello")
	assert builder.node == hash
	end
	end
	end
	diff --git a/test/support/fake_clock.ex b/test/support/fake_clock.ex
	index 3a85455..01f4d6a 100644
	--- a/test/support/fake_clock.ex
	+++ b/test/support/fake_clock.ex
	@@ -1,4 +1,5 @@
	defmodule FakeClock do
	+ @moduledoc false
	def set(time), do: :persistent_term.put(__MODULE__, time)
	def time(_), do: :persistent_term.get(__MODULE__)
	end

File Metadata

Mime Type: text/x-diff
Expires: Mon, Nov 25, 4:52 AM (1 d, 13 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 39589
Default Alt Text: (18 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions