Page MenuHomePhorge

No OneTemporary

Size
52 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
index 3d37dfa24..3a124a221 100644
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@@ -1,117 +1,111 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Benchmark do
import Mix.Pleroma
use Mix.Task
def run(["search"]) do
start_pleroma()
Benchee.run(%{
"search" => fn ->
Pleroma.Activity.search(nil, "cofe")
end
})
end
def run(["tag"]) do
start_pleroma()
Benchee.run(%{
"tag" => fn ->
%{"type" => "Create", "tag" => "cofe"}
|> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
end
})
end
def run(["render_timeline", nickname | _] = args) do
start_pleroma()
user = Pleroma.User.get_by_nickname(nickname)
activities =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("limit", 4096)
|> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
|> Enum.reverse()
inputs = %{
"1 activity" => Enum.take_random(activities, 1),
"10 activities" => Enum.take_random(activities, 10),
"20 activities" => Enum.take_random(activities, 20),
"40 activities" => Enum.take_random(activities, 40),
"80 activities" => Enum.take_random(activities, 80)
}
inputs =
if Enum.at(args, 2) == "extended" do
Map.merge(inputs, %{
"200 activities" => Enum.take_random(activities, 200),
"500 activities" => Enum.take_random(activities, 500),
"2000 activities" => Enum.take_random(activities, 2000),
"4096 activities" => Enum.take_random(activities, 4096)
})
else
inputs
end
Benchee.run(
%{
"Standart rendering" => fn activities ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
activities: activities,
for: user,
as: :activity,
skip_relationships: true
})
end
},
inputs: inputs
)
end
def run(["adapters"]) do
start_pleroma()
- :ok =
- Pleroma.Gun.Conn.open(
- "https://httpbin.org/stream-bytes/1500",
- :gun_connections
- )
-
Process.sleep(1_500)
Benchee.run(
%{
"Without conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool, reuse_conn: false]
)
end,
"Without conn and with pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [reuse_conn: false]
)
end,
"With reused conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool]
)
end,
"With reused conn and with pool" => fn ->
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
end
},
parallel: 10
)
end
end
diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index de3173351..fb7153a16 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -1,199 +1,196 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Conn do
@moduledoc """
Struct for gun connection data
"""
alias Pleroma.Gun
alias Pleroma.Pool.Connections
require Logger
@type gun_state :: :init | :up | :down
@type conn_state :: :init | :active | :idle
@type t :: %__MODULE__{
conn: pid(),
gun_state: gun_state(),
conn_state: conn_state(),
used_by: [GenServer.from()],
awaited_by: [GenServer.from()],
last_reference: pos_integer(),
crf: float(),
retries: pos_integer()
}
defstruct conn: nil,
gun_state: :init,
conn_state: :init,
used_by: [],
awaited_by: [],
last_reference: 0,
crf: 1,
retries: 0
- @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
- def open(url, name, opts \\ [])
- def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
-
- def open(%URI{} = uri, name, opts) do
+ @spec open(URI.t(), atom(), keyword()) :: :ok
+ def open(%URI{} = uri, name, opts \\ []) do
pool_opts = Pleroma.Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|> maybe_add_tls_opts(uri)
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
max_connections = pool_opts[:max_connections] || 125
with {:ok, conn_pid} <- try_open(name, uri, opts, max_connections) do
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
Connections.update_conn(name, key, conn_pid)
else
_error -> Connections.remove_conn(name, key)
end
end
defp try_open(name, uri, opts, max_connections) do
- if Connections.count(name) < max_connections do
+ if Connections.count(name) <= max_connections do
do_open(uri, opts)
else
close_least_used_and_do_open(name, uri, opts)
end
end
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
charlist_host =
host
|> to_charlist()
|> :idna.encode()
tls_opts = [
verify: :verify_peer,
cacertfile: CAStore.file_path(),
depth: 20,
reuse_sessions: false,
verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: charlist_host]}
]
tls_opts =
if Keyword.keyword?(opts[:tls_opts]) do
Keyword.merge(tls_opts, opts[:tls_opts])
else
tls_opts
end
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.Connection.format_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp destination_opts(%URI{host: host, port: port}) do
host = Pleroma.HTTP.Connection.format_host(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp close_least_used_and_do_open(name, uri, opts) do
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
:ok <- Gun.close(conn.conn) do
Connections.remove_conn(name, key)
do_open(uri, opts)
else
[] -> {:error, :pool_overflowed}
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end
diff --git a/lib/pleroma/http/middleware/follow_redirects.ex b/lib/pleroma/http/middleware/follow_redirects.ex
index bed3ec9a0..c81755a48 100644
--- a/lib/pleroma/http/middleware/follow_redirects.ex
+++ b/lib/pleroma/http/middleware/follow_redirects.ex
@@ -1,106 +1,107 @@
# Pleroma: A lightweight social networking server
+# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex>
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Middleware.FollowRedirects do
@moduledoc """
Follow 3xx redirects
## Options
- `:max_redirects` - limit number of redirects (default: `5`)
"""
@behaviour Tesla.Middleware
@max_redirects 5
@redirect_statuses [301, 302, 303, 307, 308]
@impl Tesla.Middleware
def call(env, next, opts \\ []) do
max = Keyword.get(opts, :max_redirects, @max_redirects)
redirect(env, next, max)
end
defp redirect(env, next, left) do
opts = env.opts[:adapter]
adapter_opts =
if opts[:reuse_conn] do
checkin_conn(env.url, opts)
else
opts
end
env = %{env | opts: Keyword.put(env.opts, :adapter, adapter_opts)}
case Tesla.run(env, next) do
{:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 ->
checkout_conn(adapter_opts)
case Tesla.get_header(res, "location") do
nil ->
{:ok, res}
location ->
location = parse_location(location, res)
env
|> new_request(res.status, location)
|> redirect(next, left - 1)
end
{:ok, %{status: status}} when status in @redirect_statuses ->
checkout_conn(adapter_opts)
{:error, {__MODULE__, :too_many_redirects}}
other ->
unless adapter_opts[:body_as] == :chunks do
checkout_conn(adapter_opts)
end
other
end
end
defp checkin_conn(url, opts) do
uri = URI.parse(url)
case Pleroma.Pool.Connections.checkin(uri, :gun_connections, opts) do
nil ->
opts
conn when is_pid(conn) ->
Keyword.merge(opts, conn: conn, close_conn: false)
end
end
defp checkout_conn(opts) do
if is_pid(opts[:conn]) do
Pleroma.Pool.Connections.checkout(opts[:conn], self(), :gun_connections)
end
end
# The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally
# requested resource is not available, however a related resource (or another redirect)
# available via GET is available at the specified location.
# https://tools.ietf.org/html/rfc7231#section-6.4.4
defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []}
# The 307 (Temporary Redirect) status code indicates that the target
# resource resides temporarily under a different URI and the user agent
# MUST NOT change the request method (...)
# https://tools.ietf.org/html/rfc7231#section-6.4.7
defp new_request(env, 307, location), do: %{env | url: location}
defp new_request(env, _, location), do: %{env | url: location, query: []}
defp parse_location("https://" <> _rest = location, _env), do: location
defp parse_location("http://" <> _rest = location, _env), do: location
defp parse_location(location, env) do
env.url
|> URI.parse()
|> URI.merge(location)
|> URI.to_string()
end
end
diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex
index 86bb00bc5..87fa50a22 100644
--- a/lib/pleroma/pool/connections.ex
+++ b/lib/pleroma/pool/connections.ex
@@ -1,378 +1,374 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Connections do
use GenServer
alias Pleroma.Config
alias Pleroma.Gun
alias Pleroma.Gun.Conn
require Logger
@type domain :: String.t()
@type conn :: Conn.t()
@type seconds :: pos_integer()
@type t :: %__MODULE__{
- conns: %{domain() => conn()},
- opts: keyword()
+ conns: %{domain() => conn()}
}
- defstruct conns: %{}, opts: []
+ defstruct conns: %{}
- @spec start_link({atom(), keyword()}) :: {:ok, pid()}
- def start_link({name, opts}) do
- GenServer.start_link(__MODULE__, opts, name: name)
+ @spec start_link(atom()) :: {:ok, pid()}
+ def start_link(name) do
+ GenServer.start_link(__MODULE__, [], name: name)
end
@impl true
- def init(opts) do
+ def init(_) do
schedule_close_idle_conns()
- {:ok, %__MODULE__{conns: %{}, opts: opts}}
+ {:ok, %__MODULE__{conns: %{}}}
end
@spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
def checkin(url, name, opts \\ [])
def checkin(url, name, opts) when is_binary(url), do: checkin(URI.parse(url), name, opts)
def checkin(%URI{} = uri, name, opts) do
GenServer.call(name, {:checkin, uri, opts, name})
end
@spec alive?(atom()) :: boolean()
def alive?(name) do
- if pid = Process.whereis(name) do
- Process.alive?(pid)
- else
- false
- end
+ pid = Process.whereis(name)
+ is_pid(pid) and Process.alive?(pid)
end
@spec get_state(atom()) :: t()
def get_state(name) do
GenServer.call(name, :state)
end
@spec count(atom()) :: pos_integer()
def count(name) do
GenServer.call(name, :count)
end
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
def get_unused_conns(name) do
GenServer.call(name, :unused_conns)
end
@spec checkout(pid(), pid(), atom()) :: :ok
def checkout(conn, pid, name) do
GenServer.cast(name, {:checkout, conn, pid})
end
@spec add_conn(atom(), String.t(), Conn.t()) :: :ok
def add_conn(name, key, conn) do
GenServer.cast(name, {:add_conn, key, conn})
end
@spec update_conn(atom(), String.t(), pid()) :: :ok
def update_conn(name, key, conn_pid) do
GenServer.cast(name, {:update_conn, key, conn_pid})
end
@spec remove_conn(atom(), String.t()) :: :ok
def remove_conn(name, key) do
GenServer.cast(name, {:remove_conn, key})
end
@spec refresh(atom()) :: :ok
def refresh(name) do
GenServer.call(name, :refresh)
end
@impl true
def handle_cast({:add_conn, key, conn}, state) do
{:noreply, put_in(state.conns[key], conn)}
end
@impl true
def handle_cast({:update_conn, key, conn_pid}, state) do
conn = state.conns[key]
Process.monitor(conn_pid)
conn =
Enum.reduce(conn.awaited_by, conn, fn waiting, conn ->
GenServer.reply(waiting, conn_pid)
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
%{
conn
| last_reference: time,
crf: crf,
used_by: [waiting | conn.used_by]
}
end)
state =
put_in(state.conns[key], %{
conn
| conn: conn_pid,
conn_state: :active,
gun_state: :up,
awaited_by: []
})
{:noreply, state}
end
@impl true
def handle_cast({:checkout, conn_pid, pid}, state) do
state =
with true <- Process.alive?(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid),
used_by <- List.keydelete(conn.used_by, pid, 0) do
conn_state = if used_by == [], do: :idle, else: :active
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
else
false ->
Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
state
nil ->
Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
state
end
{:noreply, state}
end
@impl true
def handle_cast({:remove_conn, key}, state) do
conn = state.conns[key]
Enum.each(conn.awaited_by, fn waiting -> GenServer.reply(waiting, nil) end)
state = put_in(state.conns, Map.delete(state.conns, key))
{:noreply, state}
end
@impl true
def handle_call({:checkin, uri, opts, name}, from, state) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
case state.conns[key] do
%{conn: pid, gun_state: :up} = conn ->
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| last_reference: time,
crf: crf,
conn_state: :active,
used_by: [from | conn.used_by]
})
{:reply, pid, state}
%{gun_state: :down} ->
{:reply, nil, state}
%{gun_state: :init} = conn ->
state = put_in(state.conns[key], %{conn | awaited_by: [from | conn.awaited_by]})
{:noreply, state}
nil ->
state =
put_in(state.conns[key], %Conn{
awaited_by: [from]
})
Task.start(Conn, :open, [uri, name, opts])
{:noreply, state}
end
end
@impl true
def handle_call(:state, _from, state), do: {:reply, state, state}
@impl true
def handle_call(:refresh, _from, state) do
{:reply, :ok, put_in(state.conns, %{})}
end
@impl true
def handle_call(:count, _from, state) do
{:reply, Enum.count(state.conns), state}
end
@impl true
def handle_call(:unused_conns, _from, state) do
unused_conns =
state.conns
|> Enum.filter(&idle_conn?/1)
|> Enum.sort(&least_used/2)
{:reply, unused_conns, state}
end
defp idle_conn?({_, %{conn_state: :idle}}), do: true
defp idle_conn?(_), do: false
defp least_used({_, c1}, {_, c2}) do
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do
%{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
host =
case :inet.ntoa(host) do
{:error, :einval} -> host
ip -> ip
end
key = "#{scheme}:#{host}:#{port}"
state =
with {key, conn} <- find_conn(state.conns, conn_pid, key),
{true, key} <- {Process.alive?(conn_pid), key} do
conn_state = if conn.used_by == [], do: :idle, else: :active
put_in(state.conns[key], %{
conn
| gun_state: :up,
conn_state: conn_state,
retries: 0
})
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
:ok = Gun.close(conn_pid)
state
end
{:noreply, state}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
retries = Config.get([:connections_pool, :retry], 1)
# we can't get info on this pid, because pid is dead
state =
with {key, conn} <- find_conn(state.conns, conn_pid),
{true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do
:ok = Gun.close(conn.conn)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
put_in(state.conns[key], %{
conn
| gun_state: :down,
conn_state: :idle,
retries: conn.retries + 1
})
end
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
Logger.debug(":gun_down for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
state =
with {key, conn} <- find_conn(state.conns, conn_pid) do
Enum.each(conn.used_by, fn {pid, _ref} ->
Process.exit(pid, reason)
end)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
nil ->
Logger.debug(":DOWN for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:close_idle_conns, max_idle_time}, state) do
closing_time = :os.system_time(:second) - max_idle_time
idle_conns_keys =
state.conns
|> Enum.filter(&idle_more_than?(&1, closing_time))
|> Enum.map(fn {key, %{conn: conn}} ->
Gun.close(conn)
key
end)
schedule_close_idle_conns()
{:noreply, put_in(state.conns, Map.drop(state.conns, idle_conns_keys))}
end
defp schedule_close_idle_conns do
max_idle_time = Config.get([:connections_pool, :max_idle_time], 1) * 60
interval = Config.get([:connections_pool, :closing_idle_conns_interval], 1) * 60 * 1000
Process.send_after(self(), {:close_idle_conns, max_idle_time}, interval)
end
defp idle_more_than?(
{_, %{conn_state: :idle, last_reference: idle_since}},
closing_time
)
when closing_time >= idle_since do
true
end
defp idle_more_than?(_, _), do: false
defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid
end)
end
defp find_conn(conns, conn_pid, conn_key) do
Enum.find(conns, fn {key, conn} ->
key == conn_key and conn.conn == conn_pid
end)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end
diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex
index faf646cb2..5e200259d 100644
--- a/lib/pleroma/pool/supervisor.ex
+++ b/lib/pleroma/pool/supervisor.ex
@@ -1,42 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Supervisor do
use Supervisor
alias Pleroma.Config
alias Pleroma.Pool
def start_link(args) do
Supervisor.start_link(__MODULE__, args, name: __MODULE__)
end
def init(_) do
conns_child = %{
id: Pool.Connections,
- start:
- {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
+ start: {Pool.Connections, :start_link, [:gun_connections]}
}
Supervisor.init([conns_child | pools()], strategy: :one_for_one)
end
defp pools do
pools = Config.get(:pools)
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) == false do
Keyword.delete(pools, :upload)
else
pools
end
for {pool_name, pool_opts} <- pools do
pool_opts
|> Keyword.put(:id, {Pool, pool_name})
|> Keyword.put(:name, pool_name)
|> Pool.child_spec()
end
end
end
diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex
index 999b1a4cb..50736dabb 100644
--- a/lib/pleroma/reverse_proxy/client/tesla.ex
+++ b/lib/pleroma/reverse_proxy/client/tesla.ex
@@ -1,84 +1,82 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
{:ok, status(), headers}
| {:ok, status(), headers, map()}
| {:error, atom() | String.t()}
| no_return()
@impl true
def request(method, url, headers, body, opts \\ []) do
check_adapter()
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
Pleroma.HTTP.request(
method,
url,
body,
headers,
adapter: opts
) do
if is_map(response.body) and method != :head do
{:ok, response.status, response.headers, response.body}
else
{:ok, response.status, response.headers}
end
- else
- {:error, error} -> {:error, error}
end
end
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
:ok = Pleroma.Pool.Connections.checkout(pid, self(), :gun_connections)
:done
end
def stream_body(client) do
case read_chunk!(client) do
{:fin, body} ->
{:ok, body, Map.put(client, :fin, true)}
{:nofin, part} ->
{:ok, part, client}
{:error, error} ->
{:error, error}
end
end
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
adapter = check_adapter()
adapter.read_chunk(pid, stream, opts)
end
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid}) do
adapter = check_adapter()
adapter.close(pid)
end
defp check_adapter do
adapter = Application.get_env(:tesla, :adapter)
unless adapter == Tesla.Adapter.Gun do
raise "#{adapter} doesn't support reading body in chunks"
end
adapter
end
end
diff --git a/test/http/gun_test.exs b/test/http/gun_test.exs
index c64cf7125..1a5a5b60c 100644
--- a/test/http/gun_test.exs
+++ b/test/http/gun_test.exs
@@ -1,84 +1,92 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.GunTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.HTTP.Gun
describe "options/1" do
+ test "http" do
+ uri = URI.parse("http://example.com")
+
+ opts = Gun.options([reuse_conn: false], uri)
+
+ assert opts[:reuse_conn] == false
+ end
+
test "https url with default port" do
uri = URI.parse("https://example.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
assert opts[:tls_opts][:log_level] == :warning
assert opts[:reuse_conn] == false
end
test "https ipv4 with default port" do
uri = URI.parse("https://127.0.0.1")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "https ipv6 with default port" do
uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "https url with non standart port" do
uri = URI.parse("https://example.com:115")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "merges with defaul http adapter config" do
clear_config([:http, :adapter], a: 1, b: 2)
defaults = Gun.options([reuse_conn: false], URI.parse("https://example.com"))
assert Keyword.has_key?(defaults, :a)
assert Keyword.has_key?(defaults, :b)
end
test "default ssl adapter opts with connection" do
uri = URI.parse("https://some-domain.com")
opts = Gun.options(uri)
assert opts[:certificates_verification]
assert opts[:reuse_conn]
refute opts[:tls_opts] == []
end
test "parses string proxy host & port" do
clear_config([:http, :proxy_url], "localhost:8123")
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:proxy] == {'localhost', 8123}
end
test "parses tuple proxy scheme host and port" do
clear_config([:http, :proxy_url], {:socks, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:proxy] == {:socks, 'localhost', 1234}
end
test "passed opts have more weight than defaults" do
clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false, proxy: {'example.com', 4321}], uri)
assert opts[:proxy] == {'example.com', 4321}
end
end
end
diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs
index d8436997f..8490f6bf1 100644
--- a/test/pool/connections_test.exs
+++ b/test/pool/connections_test.exs
@@ -1,831 +1,894 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.ConnectionsTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
import ExUnit.CaptureLog
import Mox
alias Pleroma.Gun.Conn
alias Pleroma.GunMock
alias Pleroma.Pool.Connections
setup :verify_on_exit!
setup :set_mox_global
setup_all do
{:ok, pid} = Agent.start_link(fn -> %{} end, name: :gun_state)
on_exit(fn ->
if Process.alive?(pid), do: Agent.stop(pid)
end)
end
setup do
name = :test_connections
- {:ok, pid} = Connections.start_link({name, []})
+ {:ok, pid} = Connections.start_link(name)
on_exit(fn ->
if Process.alive?(pid), do: GenServer.stop(name)
end)
{:ok, name: name}
end
defp open_mock(num \\ 1) do
GunMock
|> expect(:open, num, &start_and_register(&1, &2, &3))
|> expect(:await_up, num, fn _, _ -> {:ok, :http} end)
|> expect(:set_owner, num, fn _, _ -> :ok end)
end
defp connect_mock(mock) do
mock
|> expect(:connect, &connect(&1, &2))
|> expect(:await, &await(&1, &2))
end
defp info_mock(mock), do: expect(mock, :info, &info(&1))
defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout}
defp start_and_register(host, port, _) do
{:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end)
scheme =
case port do
443 -> "https"
_ -> "http"
end
info = %{
origin_scheme: scheme,
origin_host: host,
origin_port: port
}
Agent.update(:gun_state, &Map.put(&1, pid, %{info: info, ref: nil}))
{:ok, pid}
end
defp info(pid), do: Agent.get(:gun_state, & &1[pid][:info])
defp connect(pid, _) do
ref = make_ref()
Agent.update(:gun_state, &put_in(&1[pid][:ref], ref))
ref
end
defp await(pid, _ref) do
Agent.get(:gun_state, & &1[pid][:ref])
{:response, :fin, 200, []}
end
defp now, do: :os.system_time(:second)
describe "alive?/2" do
test "is alive", %{name: name} do
assert Connections.alive?(name)
end
test "returns false if not started" do
refute Connections.alive?(:some_random_name)
end
end
+ describe "pool overflow" do
+ setup do: clear_config([:connections_pool, :max_connections], 2)
+
+ test "when all conns are active return nil", %{name: name} do
+ open_mock(2)
+ conn1 = Connections.checkin("https://example1.com", name)
+ conn2 = Connections.checkin("https://example2.com", name)
+ refute Connections.checkin("https://example3.com", name)
+
+ self = self()
+
+ assert match?(
+ %Connections{
+ conns: %{
+ "https:example1.com:443" => %Conn{
+ conn: ^conn1,
+ used_by: [{^self, _}]
+ },
+ "https:example2.com:443" => %Conn{
+ conn: ^conn2,
+ used_by: [{^self, _}]
+ }
+ }
+ },
+ Connections.get_state(name)
+ )
+
+ assert Connections.count(name) == 2
+ end
+
+ test "close idle conn", %{name: name} do
+ open_mock(3)
+ |> expect(:close, fn _ -> :ok end)
+
+ self = self()
+ conn1 = Connections.checkin("https://example1.com", name)
+ Connections.checkout(conn1, self, name)
+ conn2 = Connections.checkin("https://example2.com", name)
+ conn3 = Connections.checkin("https://example3.com", name)
+
+ assert match?(
+ %Connections{
+ conns: %{
+ "https:example2.com:443" => %Conn{
+ conn: ^conn2,
+ used_by: [{^self, _}]
+ },
+ "https:example3.com:443" => %Conn{
+ conn: ^conn3,
+ used_by: [{^self, _}]
+ }
+ }
+ },
+ Connections.get_state(name)
+ )
+
+ assert Connections.count(name) == 2
+ end
+ end
+
test "opens connection and reuse it on next request", %{name: name} do
open_mock()
url = "http://some-domain.com"
key = "http:some-domain.com:80"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}, {^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn, self, name)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn, self, name)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
},
Connections.get_state(name)
)
end
test "reuse connection for idna domains", %{name: name} do
open_mock()
url = "http://ですsome-domain.com"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:ですsome-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "reuse for ipv4", %{name: name} do
open_mock()
url = "http://127.0.0.1"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
:ok = Connections.checkout(conn, self, name)
:ok = Connections.checkout(reused_conn, self, name)
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
},
Connections.get_state(name)
)
end
test "reuse for ipv6", %{name: name} do
open_mock()
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "up and down ipv4", %{name: name} do
open_mock()
|> info_mock()
|> allow(self(), name)
self = self()
url = "http://127.0.0.1"
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
end
test "up and down ipv6", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
assert match?(
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
end
test "reuses connection based on protocol", %{name: name} do
open_mock(2)
http_url = "http://some-domain.com"
http_key = "http:some-domain.com:80"
https_url = "https://some-domain.com"
https_key = "https:some-domain.com:443"
conn = Connections.checkin(http_url, name)
assert is_pid(conn)
assert Process.alive?(conn)
https_conn = Connections.checkin(https_url, name)
refute conn == https_conn
reused_https = Connections.checkin(https_url, name)
refute conn == reused_https
assert reused_https == https_conn
assert match?(
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn,
gun_state: :up
},
^https_key => %Conn{
conn: ^https_conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
end
test "connection can't get up", %{name: name} do
expect(GunMock, :open, &start_and_register(&1, &2, &3))
url = "http://gun-not-up.com"
assert capture_log(fn ->
refute Connections.checkin(url, name)
end) =~
"Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
+
+ state = Connections.get_state(name)
+ assert state.conns == %{}
end
test "process gun_down message and then gun_up", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://gun-down-and-up.com"
key = "http:gun-down-and-up.com:80"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}]
}
}
},
Connections.get_state(name)
)
send(name, {:gun_down, conn, :http, nil, nil})
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :down,
used_by: [{^self, _}]
}
}
},
Connections.get_state(name)
)
send(name, {:gun_up, conn, :http})
conn2 = Connections.checkin(url, name)
assert conn == conn2
assert is_pid(conn2)
assert Process.alive?(conn2)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: _,
gun_state: :up,
used_by: [{^self, _}, {^self, _}]
}
}
},
Connections.get_state(name)
)
end
test "async processes get same conn for same domain", %{name: name} do
open_mock()
url = "http://some-domain.com"
tasks =
for _ <- 1..5 do
Task.async(fn ->
Connections.checkin(url, name)
end)
end
tasks_with_results = Task.yield_many(tasks)
results =
Enum.map(tasks_with_results, fn {task, res} ->
res || Task.shutdown(task, :brutal_kill)
end)
conns = for {:ok, value} <- results, do: value
state = Connections.get_state(name)
%{conn: conn} = Map.get(state.conns, "http:some-domain.com:80")
assert Enum.all?(conns, fn res -> res == conn end)
end
test "remove frequently used and idle", %{name: name} do
open_mock(3)
self = self()
http_url = "http://some-domain.com"
https_url = "https://some-domain.com"
conn1 = Connections.checkin(https_url, name)
[conn2 | _conns] =
for _ <- 1..4 do
Connections.checkin(http_url, name)
end
http_key = "http:some-domain.com:80"
assert match?(
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn2,
gun_state: :up,
conn_state: :active
},
"https:some-domain.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn1, self, name)
another_url = "http://another-domain.com"
conn = Connections.checkin(another_url, name)
assert match?(
%Connections{
conns: %{
"http:another-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up
},
^http_key => %Conn{
conn: _,
gun_state: :up
}
}
},
Connections.get_state(name)
)
end
describe "with proxy" do
test "as ip", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-string.com"
key = "http:proxy-string.com:80"
conn = Connections.checkin(url, name, proxy: {{127, 0, 0, 1}, 8123})
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-tuple-atom.com"
conn = Connections.checkin(url, name, proxy: {'localhost', 9050})
assert match?(
%Connections{
conns: %{
"http:proxy-tuple-atom.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as ip and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-string.com"
conn = Connections.checkin(url, name, proxy: {{127, 0, 0, 1}, 8123})
assert match?(
%Connections{
conns: %{
"https:proxy-string.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-tuple-atom.com"
conn = Connections.checkin(url, name, proxy: {'localhost', 9050})
assert match?(
%Connections{
conns: %{
"https:proxy-tuple-atom.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks type", %{name: name} do
open_mock()
url = "http://proxy-socks.com"
conn = Connections.checkin(url, name, proxy: {:socks5, 'localhost', 1234})
assert match?(
%Connections{
conns: %{
"http:proxy-socks.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks4 type and ssl", %{name: name} do
open_mock()
url = "https://proxy-socks.com"
conn = Connections.checkin(url, name, proxy: {:socks4, 'localhost', 1234})
assert match?(
%Connections{
conns: %{
"https:proxy-socks.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
end
describe "crf/3" do
setup do
crf = Connections.crf(1, 10, 1)
{:ok, crf: crf}
end
test "more used will have crf higher", %{crf: crf} do
# used 3 times
crf1 = Connections.crf(1, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 > crf2
end
test "recently used will have crf higher on equal references", %{crf: crf} do
# used 3 sec ago
crf1 = Connections.crf(3, 10, crf)
# used 4 sec ago
crf2 = Connections.crf(4, 10, crf)
assert crf1 > crf2
end
test "equal crf on equal reference and time", %{crf: crf} do
# used 2 times
crf1 = Connections.crf(1, 10, crf)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 == crf2
end
test "recently used will have higher crf", %{crf: crf} do
crf1 = Connections.crf(2, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
crf2 = Connections.crf(3, 10, crf)
crf2 = Connections.crf(4, 10, crf2)
assert crf1 > crf2
end
end
describe "get_unused_conns/1" do
setup %{name: name} do
Connections.refresh(name)
end
test "crf is equalent, sorting by reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "reference is equalent, sorting by crf", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.999
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "higher crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 3,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "lower crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.99,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
end
test "count/1" do
name = :test_count
- {:ok, _} = Connections.start_link({name, []})
+ {:ok, _} = Connections.start_link(name)
assert Connections.count(name) == 0
Connections.add_conn(name, "1", %Conn{conn: self()})
assert Connections.count(name) == 1
Connections.remove_conn(name, "1")
assert Connections.count(name) == 0
end
test "close_idle_conns/2", %{name: name} do
GunMock
|> expect(:close, fn _ -> :ok end)
|> allow(self(), name)
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
last_reference: now() - 30,
conn: self()
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now() - 10,
conn: self()
})
Connections.add_conn(name, "3", %Conn{
conn_state: :active,
conn: self()
})
name
|> Process.whereis()
|> send({:close_idle_conns, 15})
assert match?(
%Connections{
conns: %{
"3" => %Conn{},
"2" => %Conn{}
}
},
Connections.get_state(name)
)
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sat, Aug 29, 8:13 AM (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1736894
Default Alt Text
(52 KB)

Event Timeline