Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85649192
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Award Token
Flag For Later
Size
52 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
index 3d37dfa24..3a124a221 100644
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@@ -1,117 +1,111 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Benchmark do
import Mix.Pleroma
use Mix.Task
def run(["search"]) do
start_pleroma()
Benchee.run(%{
"search" => fn ->
Pleroma.Activity.search(nil, "cofe")
end
})
end
def run(["tag"]) do
start_pleroma()
Benchee.run(%{
"tag" => fn ->
%{"type" => "Create", "tag" => "cofe"}
|> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
end
})
end
def run(["render_timeline", nickname | _] = args) do
start_pleroma()
user = Pleroma.User.get_by_nickname(nickname)
activities =
%{}
|> Map.put("type", ["Create", "Announce"])
|> Map.put("blocking_user", user)
|> Map.put("muting_user", user)
|> Map.put("user", user)
|> Map.put("limit", 4096)
|> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
|> Enum.reverse()
inputs = %{
"1 activity" => Enum.take_random(activities, 1),
"10 activities" => Enum.take_random(activities, 10),
"20 activities" => Enum.take_random(activities, 20),
"40 activities" => Enum.take_random(activities, 40),
"80 activities" => Enum.take_random(activities, 80)
}
inputs =
if Enum.at(args, 2) == "extended" do
Map.merge(inputs, %{
"200 activities" => Enum.take_random(activities, 200),
"500 activities" => Enum.take_random(activities, 500),
"2000 activities" => Enum.take_random(activities, 2000),
"4096 activities" => Enum.take_random(activities, 4096)
})
else
inputs
end
Benchee.run(
%{
"Standart rendering" => fn activities ->
Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
activities: activities,
for: user,
as: :activity,
skip_relationships: true
})
end
},
inputs: inputs
)
end
def run(["adapters"]) do
start_pleroma()
- :ok =
- Pleroma.Gun.Conn.open(
- "https://httpbin.org/stream-bytes/1500",
- :gun_connections
- )
-
Process.sleep(1_500)
Benchee.run(
%{
"Without conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool, reuse_conn: false]
)
end,
"Without conn and with pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [reuse_conn: false]
)
end,
"With reused conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool]
)
end,
"With reused conn and with pool" => fn ->
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
end
},
parallel: 10
)
end
end
diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index de3173351..fb7153a16 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -1,199 +1,196 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Conn do
@moduledoc """
Struct for gun connection data
"""
alias Pleroma.Gun
alias Pleroma.Pool.Connections
require Logger
@type gun_state :: :init | :up | :down
@type conn_state :: :init | :active | :idle
@type t :: %__MODULE__{
conn: pid(),
gun_state: gun_state(),
conn_state: conn_state(),
used_by: [GenServer.from()],
awaited_by: [GenServer.from()],
last_reference: pos_integer(),
crf: float(),
retries: pos_integer()
}
defstruct conn: nil,
gun_state: :init,
conn_state: :init,
used_by: [],
awaited_by: [],
last_reference: 0,
crf: 1,
retries: 0
- @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
- def open(url, name, opts \\ [])
- def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
-
- def open(%URI{} = uri, name, opts) do
+ @spec open(URI.t(), atom(), keyword()) :: :ok
+ def open(%URI{} = uri, name, opts \\ []) do
pool_opts = Pleroma.Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|> maybe_add_tls_opts(uri)
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
max_connections = pool_opts[:max_connections] || 125
with {:ok, conn_pid} <- try_open(name, uri, opts, max_connections) do
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
Connections.update_conn(name, key, conn_pid)
else
_error -> Connections.remove_conn(name, key)
end
end
defp try_open(name, uri, opts, max_connections) do
- if Connections.count(name) < max_connections do
+ if Connections.count(name) <= max_connections do
do_open(uri, opts)
else
close_least_used_and_do_open(name, uri, opts)
end
end
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
charlist_host =
host
|> to_charlist()
|> :idna.encode()
tls_opts = [
verify: :verify_peer,
cacertfile: CAStore.file_path(),
depth: 20,
reuse_sessions: false,
verify_fun: {&:ssl_verify_hostname.verify_fun/3, [check_hostname: charlist_host]}
]
tls_opts =
if Keyword.keyword?(opts[:tls_opts]) do
Keyword.merge(tls_opts, opts[:tls_opts])
else
tls_opts
end
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.Connection.format_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
{:ok, conn}
else
error ->
Logger.warn(
"Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp destination_opts(%URI{host: host, port: port}) do
host = Pleroma.HTTP.Connection.format_host(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp close_least_used_and_do_open(name, uri, opts) do
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
:ok <- Gun.close(conn.conn) do
Connections.remove_conn(name, key)
do_open(uri, opts)
else
[] -> {:error, :pool_overflowed}
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end
diff --git a/lib/pleroma/http/middleware/follow_redirects.ex b/lib/pleroma/http/middleware/follow_redirects.ex
index bed3ec9a0..c81755a48 100644
--- a/lib/pleroma/http/middleware/follow_redirects.ex
+++ b/lib/pleroma/http/middleware/follow_redirects.ex
@@ -1,106 +1,107 @@
# Pleroma: A lightweight social networking server
+# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex>
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Middleware.FollowRedirects do
@moduledoc """
Follow 3xx redirects
## Options
- `:max_redirects` - limit number of redirects (default: `5`)
"""
@behaviour Tesla.Middleware
@max_redirects 5
@redirect_statuses [301, 302, 303, 307, 308]
@impl Tesla.Middleware
def call(env, next, opts \\ []) do
max = Keyword.get(opts, :max_redirects, @max_redirects)
redirect(env, next, max)
end
defp redirect(env, next, left) do
opts = env.opts[:adapter]
adapter_opts =
if opts[:reuse_conn] do
checkin_conn(env.url, opts)
else
opts
end
env = %{env | opts: Keyword.put(env.opts, :adapter, adapter_opts)}
case Tesla.run(env, next) do
{:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 ->
checkout_conn(adapter_opts)
case Tesla.get_header(res, "location") do
nil ->
{:ok, res}
location ->
location = parse_location(location, res)
env
|> new_request(res.status, location)
|> redirect(next, left - 1)
end
{:ok, %{status: status}} when status in @redirect_statuses ->
checkout_conn(adapter_opts)
{:error, {__MODULE__, :too_many_redirects}}
other ->
unless adapter_opts[:body_as] == :chunks do
checkout_conn(adapter_opts)
end
other
end
end
defp checkin_conn(url, opts) do
uri = URI.parse(url)
case Pleroma.Pool.Connections.checkin(uri, :gun_connections, opts) do
nil ->
opts
conn when is_pid(conn) ->
Keyword.merge(opts, conn: conn, close_conn: false)
end
end
defp checkout_conn(opts) do
if is_pid(opts[:conn]) do
Pleroma.Pool.Connections.checkout(opts[:conn], self(), :gun_connections)
end
end
# The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally
# requested resource is not available, however a related resource (or another redirect)
# available via GET is available at the specified location.
# https://tools.ietf.org/html/rfc7231#section-6.4.4
defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []}
# The 307 (Temporary Redirect) status code indicates that the target
# resource resides temporarily under a different URI and the user agent
# MUST NOT change the request method (...)
# https://tools.ietf.org/html/rfc7231#section-6.4.7
defp new_request(env, 307, location), do: %{env | url: location}
defp new_request(env, _, location), do: %{env | url: location, query: []}
defp parse_location("https://" <> _rest = location, _env), do: location
defp parse_location("http://" <> _rest = location, _env), do: location
defp parse_location(location, env) do
env.url
|> URI.parse()
|> URI.merge(location)
|> URI.to_string()
end
end
diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex
index 86bb00bc5..87fa50a22 100644
--- a/lib/pleroma/pool/connections.ex
+++ b/lib/pleroma/pool/connections.ex
@@ -1,378 +1,374 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Connections do
use GenServer
alias Pleroma.Config
alias Pleroma.Gun
alias Pleroma.Gun.Conn
require Logger
@type domain :: String.t()
@type conn :: Conn.t()
@type seconds :: pos_integer()
@type t :: %__MODULE__{
- conns: %{domain() => conn()},
- opts: keyword()
+ conns: %{domain() => conn()}
}
- defstruct conns: %{}, opts: []
+ defstruct conns: %{}
- @spec start_link({atom(), keyword()}) :: {:ok, pid()}
- def start_link({name, opts}) do
- GenServer.start_link(__MODULE__, opts, name: name)
+ @spec start_link(atom()) :: {:ok, pid()}
+ def start_link(name) do
+ GenServer.start_link(__MODULE__, [], name: name)
end
@impl true
- def init(opts) do
+ def init(_) do
schedule_close_idle_conns()
- {:ok, %__MODULE__{conns: %{}, opts: opts}}
+ {:ok, %__MODULE__{conns: %{}}}
end
@spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
def checkin(url, name, opts \\ [])
def checkin(url, name, opts) when is_binary(url), do: checkin(URI.parse(url), name, opts)
def checkin(%URI{} = uri, name, opts) do
GenServer.call(name, {:checkin, uri, opts, name})
end
@spec alive?(atom()) :: boolean()
def alive?(name) do
- if pid = Process.whereis(name) do
- Process.alive?(pid)
- else
- false
- end
+ pid = Process.whereis(name)
+ is_pid(pid) and Process.alive?(pid)
end
@spec get_state(atom()) :: t()
def get_state(name) do
GenServer.call(name, :state)
end
@spec count(atom()) :: pos_integer()
def count(name) do
GenServer.call(name, :count)
end
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
def get_unused_conns(name) do
GenServer.call(name, :unused_conns)
end
@spec checkout(pid(), pid(), atom()) :: :ok
def checkout(conn, pid, name) do
GenServer.cast(name, {:checkout, conn, pid})
end
@spec add_conn(atom(), String.t(), Conn.t()) :: :ok
def add_conn(name, key, conn) do
GenServer.cast(name, {:add_conn, key, conn})
end
@spec update_conn(atom(), String.t(), pid()) :: :ok
def update_conn(name, key, conn_pid) do
GenServer.cast(name, {:update_conn, key, conn_pid})
end
@spec remove_conn(atom(), String.t()) :: :ok
def remove_conn(name, key) do
GenServer.cast(name, {:remove_conn, key})
end
@spec refresh(atom()) :: :ok
def refresh(name) do
GenServer.call(name, :refresh)
end
@impl true
def handle_cast({:add_conn, key, conn}, state) do
{:noreply, put_in(state.conns[key], conn)}
end
@impl true
def handle_cast({:update_conn, key, conn_pid}, state) do
conn = state.conns[key]
Process.monitor(conn_pid)
conn =
Enum.reduce(conn.awaited_by, conn, fn waiting, conn ->
GenServer.reply(waiting, conn_pid)
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
%{
conn
| last_reference: time,
crf: crf,
used_by: [waiting | conn.used_by]
}
end)
state =
put_in(state.conns[key], %{
conn
| conn: conn_pid,
conn_state: :active,
gun_state: :up,
awaited_by: []
})
{:noreply, state}
end
@impl true
def handle_cast({:checkout, conn_pid, pid}, state) do
state =
with true <- Process.alive?(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid),
used_by <- List.keydelete(conn.used_by, pid, 0) do
conn_state = if used_by == [], do: :idle, else: :active
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
else
false ->
Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
state
nil ->
Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
state
end
{:noreply, state}
end
@impl true
def handle_cast({:remove_conn, key}, state) do
conn = state.conns[key]
Enum.each(conn.awaited_by, fn waiting -> GenServer.reply(waiting, nil) end)
state = put_in(state.conns, Map.delete(state.conns, key))
{:noreply, state}
end
@impl true
def handle_call({:checkin, uri, opts, name}, from, state) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
case state.conns[key] do
%{conn: pid, gun_state: :up} = conn ->
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| last_reference: time,
crf: crf,
conn_state: :active,
used_by: [from | conn.used_by]
})
{:reply, pid, state}
%{gun_state: :down} ->
{:reply, nil, state}
%{gun_state: :init} = conn ->
state = put_in(state.conns[key], %{conn | awaited_by: [from | conn.awaited_by]})
{:noreply, state}
nil ->
state =
put_in(state.conns[key], %Conn{
awaited_by: [from]
})
Task.start(Conn, :open, [uri, name, opts])
{:noreply, state}
end
end
@impl true
def handle_call(:state, _from, state), do: {:reply, state, state}
@impl true
def handle_call(:refresh, _from, state) do
{:reply, :ok, put_in(state.conns, %{})}
end
@impl true
def handle_call(:count, _from, state) do
{:reply, Enum.count(state.conns), state}
end
@impl true
def handle_call(:unused_conns, _from, state) do
unused_conns =
state.conns
|> Enum.filter(&idle_conn?/1)
|> Enum.sort(&least_used/2)
{:reply, unused_conns, state}
end
defp idle_conn?({_, %{conn_state: :idle}}), do: true
defp idle_conn?(_), do: false
defp least_used({_, c1}, {_, c2}) do
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do
%{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
host =
case :inet.ntoa(host) do
{:error, :einval} -> host
ip -> ip
end
key = "#{scheme}:#{host}:#{port}"
state =
with {key, conn} <- find_conn(state.conns, conn_pid, key),
{true, key} <- {Process.alive?(conn_pid), key} do
conn_state = if conn.used_by == [], do: :idle, else: :active
put_in(state.conns[key], %{
conn
| gun_state: :up,
conn_state: conn_state,
retries: 0
})
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
:ok = Gun.close(conn_pid)
state
end
{:noreply, state}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
retries = Config.get([:connections_pool, :retry], 1)
# we can't get info on this pid, because pid is dead
state =
with {key, conn} <- find_conn(state.conns, conn_pid),
{true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do
:ok = Gun.close(conn.conn)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
put_in(state.conns[key], %{
conn
| gun_state: :down,
conn_state: :idle,
retries: conn.retries + 1
})
end
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
Logger.debug(":gun_down for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
state =
with {key, conn} <- find_conn(state.conns, conn_pid) do
Enum.each(conn.used_by, fn {pid, _ref} ->
Process.exit(pid, reason)
end)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
nil ->
Logger.debug(":DOWN for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:close_idle_conns, max_idle_time}, state) do
closing_time = :os.system_time(:second) - max_idle_time
idle_conns_keys =
state.conns
|> Enum.filter(&idle_more_than?(&1, closing_time))
|> Enum.map(fn {key, %{conn: conn}} ->
Gun.close(conn)
key
end)
schedule_close_idle_conns()
{:noreply, put_in(state.conns, Map.drop(state.conns, idle_conns_keys))}
end
defp schedule_close_idle_conns do
max_idle_time = Config.get([:connections_pool, :max_idle_time], 1) * 60
interval = Config.get([:connections_pool, :closing_idle_conns_interval], 1) * 60 * 1000
Process.send_after(self(), {:close_idle_conns, max_idle_time}, interval)
end
defp idle_more_than?(
{_, %{conn_state: :idle, last_reference: idle_since}},
closing_time
)
when closing_time >= idle_since do
true
end
defp idle_more_than?(_, _), do: false
defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid
end)
end
defp find_conn(conns, conn_pid, conn_key) do
Enum.find(conns, fn {key, conn} ->
key == conn_key and conn.conn == conn_pid
end)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end
diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex
index faf646cb2..5e200259d 100644
--- a/lib/pleroma/pool/supervisor.ex
+++ b/lib/pleroma/pool/supervisor.ex
@@ -1,42 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Supervisor do
use Supervisor
alias Pleroma.Config
alias Pleroma.Pool
def start_link(args) do
Supervisor.start_link(__MODULE__, args, name: __MODULE__)
end
def init(_) do
conns_child = %{
id: Pool.Connections,
- start:
- {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
+ start: {Pool.Connections, :start_link, [:gun_connections]}
}
Supervisor.init([conns_child | pools()], strategy: :one_for_one)
end
defp pools do
pools = Config.get(:pools)
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) == false do
Keyword.delete(pools, :upload)
else
pools
end
for {pool_name, pool_opts} <- pools do
pool_opts
|> Keyword.put(:id, {Pool, pool_name})
|> Keyword.put(:name, pool_name)
|> Pool.child_spec()
end
end
end
diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex
index 999b1a4cb..50736dabb 100644
--- a/lib/pleroma/reverse_proxy/client/tesla.ex
+++ b/lib/pleroma/reverse_proxy/client/tesla.ex
@@ -1,84 +1,82 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
{:ok, status(), headers}
| {:ok, status(), headers, map()}
| {:error, atom() | String.t()}
| no_return()
@impl true
def request(method, url, headers, body, opts \\ []) do
check_adapter()
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
Pleroma.HTTP.request(
method,
url,
body,
headers,
adapter: opts
) do
if is_map(response.body) and method != :head do
{:ok, response.status, response.headers, response.body}
else
{:ok, response.status, response.headers}
end
- else
- {:error, error} -> {:error, error}
end
end
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
:ok = Pleroma.Pool.Connections.checkout(pid, self(), :gun_connections)
:done
end
def stream_body(client) do
case read_chunk!(client) do
{:fin, body} ->
{:ok, body, Map.put(client, :fin, true)}
{:nofin, part} ->
{:ok, part, client}
{:error, error} ->
{:error, error}
end
end
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
adapter = check_adapter()
adapter.read_chunk(pid, stream, opts)
end
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid}) do
adapter = check_adapter()
adapter.close(pid)
end
defp check_adapter do
adapter = Application.get_env(:tesla, :adapter)
unless adapter == Tesla.Adapter.Gun do
raise "#{adapter} doesn't support reading body in chunks"
end
adapter
end
end
diff --git a/test/http/gun_test.exs b/test/http/gun_test.exs
index c64cf7125..1a5a5b60c 100644
--- a/test/http/gun_test.exs
+++ b/test/http/gun_test.exs
@@ -1,84 +1,92 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.GunTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.HTTP.Gun
describe "options/1" do
+ test "http" do
+ uri = URI.parse("http://example.com")
+
+ opts = Gun.options([reuse_conn: false], uri)
+
+ assert opts[:reuse_conn] == false
+ end
+
test "https url with default port" do
uri = URI.parse("https://example.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
assert opts[:tls_opts][:log_level] == :warning
assert opts[:reuse_conn] == false
end
test "https ipv4 with default port" do
uri = URI.parse("https://127.0.0.1")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "https ipv6 with default port" do
uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "https url with non standart port" do
uri = URI.parse("https://example.com:115")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:certificates_verification]
end
test "merges with defaul http adapter config" do
clear_config([:http, :adapter], a: 1, b: 2)
defaults = Gun.options([reuse_conn: false], URI.parse("https://example.com"))
assert Keyword.has_key?(defaults, :a)
assert Keyword.has_key?(defaults, :b)
end
test "default ssl adapter opts with connection" do
uri = URI.parse("https://some-domain.com")
opts = Gun.options(uri)
assert opts[:certificates_verification]
assert opts[:reuse_conn]
refute opts[:tls_opts] == []
end
test "parses string proxy host & port" do
clear_config([:http, :proxy_url], "localhost:8123")
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:proxy] == {'localhost', 8123}
end
test "parses tuple proxy scheme host and port" do
clear_config([:http, :proxy_url], {:socks, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false], uri)
assert opts[:proxy] == {:socks, 'localhost', 1234}
end
test "passed opts have more weight than defaults" do
clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234})
uri = URI.parse("https://some-domain.com")
opts = Gun.options([reuse_conn: false, proxy: {'example.com', 4321}], uri)
assert opts[:proxy] == {'example.com', 4321}
end
end
end
diff --git a/test/pool/connections_test.exs b/test/pool/connections_test.exs
index d8436997f..8490f6bf1 100644
--- a/test/pool/connections_test.exs
+++ b/test/pool/connections_test.exs
@@ -1,831 +1,894 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.ConnectionsTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
import ExUnit.CaptureLog
import Mox
alias Pleroma.Gun.Conn
alias Pleroma.GunMock
alias Pleroma.Pool.Connections
setup :verify_on_exit!
setup :set_mox_global
setup_all do
{:ok, pid} = Agent.start_link(fn -> %{} end, name: :gun_state)
on_exit(fn ->
if Process.alive?(pid), do: Agent.stop(pid)
end)
end
setup do
name = :test_connections
- {:ok, pid} = Connections.start_link({name, []})
+ {:ok, pid} = Connections.start_link(name)
on_exit(fn ->
if Process.alive?(pid), do: GenServer.stop(name)
end)
{:ok, name: name}
end
defp open_mock(num \\ 1) do
GunMock
|> expect(:open, num, &start_and_register(&1, &2, &3))
|> expect(:await_up, num, fn _, _ -> {:ok, :http} end)
|> expect(:set_owner, num, fn _, _ -> :ok end)
end
defp connect_mock(mock) do
mock
|> expect(:connect, &connect(&1, &2))
|> expect(:await, &await(&1, &2))
end
defp info_mock(mock), do: expect(mock, :info, &info(&1))
defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout}
defp start_and_register(host, port, _) do
{:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end)
scheme =
case port do
443 -> "https"
_ -> "http"
end
info = %{
origin_scheme: scheme,
origin_host: host,
origin_port: port
}
Agent.update(:gun_state, &Map.put(&1, pid, %{info: info, ref: nil}))
{:ok, pid}
end
defp info(pid), do: Agent.get(:gun_state, & &1[pid][:info])
defp connect(pid, _) do
ref = make_ref()
Agent.update(:gun_state, &put_in(&1[pid][:ref], ref))
ref
end
defp await(pid, _ref) do
Agent.get(:gun_state, & &1[pid][:ref])
{:response, :fin, 200, []}
end
defp now, do: :os.system_time(:second)
describe "alive?/2" do
test "is alive", %{name: name} do
assert Connections.alive?(name)
end
test "returns false if not started" do
refute Connections.alive?(:some_random_name)
end
end
+ describe "pool overflow" do
+ setup do: clear_config([:connections_pool, :max_connections], 2)
+
+ test "when all conns are active return nil", %{name: name} do
+ open_mock(2)
+ conn1 = Connections.checkin("https://example1.com", name)
+ conn2 = Connections.checkin("https://example2.com", name)
+ refute Connections.checkin("https://example3.com", name)
+
+ self = self()
+
+ assert match?(
+ %Connections{
+ conns: %{
+ "https:example1.com:443" => %Conn{
+ conn: ^conn1,
+ used_by: [{^self, _}]
+ },
+ "https:example2.com:443" => %Conn{
+ conn: ^conn2,
+ used_by: [{^self, _}]
+ }
+ }
+ },
+ Connections.get_state(name)
+ )
+
+ assert Connections.count(name) == 2
+ end
+
+ test "close idle conn", %{name: name} do
+ open_mock(3)
+ |> expect(:close, fn _ -> :ok end)
+
+ self = self()
+ conn1 = Connections.checkin("https://example1.com", name)
+ Connections.checkout(conn1, self, name)
+ conn2 = Connections.checkin("https://example2.com", name)
+ conn3 = Connections.checkin("https://example3.com", name)
+
+ assert match?(
+ %Connections{
+ conns: %{
+ "https:example2.com:443" => %Conn{
+ conn: ^conn2,
+ used_by: [{^self, _}]
+ },
+ "https:example3.com:443" => %Conn{
+ conn: ^conn3,
+ used_by: [{^self, _}]
+ }
+ }
+ },
+ Connections.get_state(name)
+ )
+
+ assert Connections.count(name) == 2
+ end
+ end
+
test "opens connection and reuse it on next request", %{name: name} do
open_mock()
url = "http://some-domain.com"
key = "http:some-domain.com:80"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}, {^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn, self, name)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn, self, name)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
},
Connections.get_state(name)
)
end
test "reuse connection for idna domains", %{name: name} do
open_mock()
url = "http://ですsome-domain.com"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:ですsome-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "reuse for ipv4", %{name: name} do
open_mock()
url = "http://127.0.0.1"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
:ok = Connections.checkout(conn, self, name)
:ok = Connections.checkout(reused_conn, self, name)
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
},
Connections.get_state(name)
)
end
test "reuse for ipv6", %{name: name} do
open_mock()
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
assert match?(
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "up and down ipv4", %{name: name} do
open_mock()
|> info_mock()
|> allow(self(), name)
self = self()
url = "http://127.0.0.1"
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
assert match?(
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
end
test "up and down ipv6", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
assert match?(
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
},
Connections.get_state(name)
)
end
test "reuses connection based on protocol", %{name: name} do
open_mock(2)
http_url = "http://some-domain.com"
http_key = "http:some-domain.com:80"
https_url = "https://some-domain.com"
https_key = "https:some-domain.com:443"
conn = Connections.checkin(http_url, name)
assert is_pid(conn)
assert Process.alive?(conn)
https_conn = Connections.checkin(https_url, name)
refute conn == https_conn
reused_https = Connections.checkin(https_url, name)
refute conn == reused_https
assert reused_https == https_conn
assert match?(
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn,
gun_state: :up
},
^https_key => %Conn{
conn: ^https_conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
end
test "connection can't get up", %{name: name} do
expect(GunMock, :open, &start_and_register(&1, &2, &3))
url = "http://gun-not-up.com"
assert capture_log(fn ->
refute Connections.checkin(url, name)
end) =~
"Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
+
+ state = Connections.get_state(name)
+ assert state.conns == %{}
end
test "process gun_down message and then gun_up", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://gun-down-and-up.com"
key = "http:gun-down-and-up.com:80"
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}]
}
}
},
Connections.get_state(name)
)
send(name, {:gun_down, conn, :http, nil, nil})
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :down,
used_by: [{^self, _}]
}
}
},
Connections.get_state(name)
)
send(name, {:gun_up, conn, :http})
conn2 = Connections.checkin(url, name)
assert conn == conn2
assert is_pid(conn2)
assert Process.alive?(conn2)
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: _,
gun_state: :up,
used_by: [{^self, _}, {^self, _}]
}
}
},
Connections.get_state(name)
)
end
test "async processes get same conn for same domain", %{name: name} do
open_mock()
url = "http://some-domain.com"
tasks =
for _ <- 1..5 do
Task.async(fn ->
Connections.checkin(url, name)
end)
end
tasks_with_results = Task.yield_many(tasks)
results =
Enum.map(tasks_with_results, fn {task, res} ->
res || Task.shutdown(task, :brutal_kill)
end)
conns = for {:ok, value} <- results, do: value
state = Connections.get_state(name)
%{conn: conn} = Map.get(state.conns, "http:some-domain.com:80")
assert Enum.all?(conns, fn res -> res == conn end)
end
test "remove frequently used and idle", %{name: name} do
open_mock(3)
self = self()
http_url = "http://some-domain.com"
https_url = "https://some-domain.com"
conn1 = Connections.checkin(https_url, name)
[conn2 | _conns] =
for _ <- 1..4 do
Connections.checkin(http_url, name)
end
http_key = "http:some-domain.com:80"
assert match?(
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn2,
gun_state: :up,
conn_state: :active
},
"https:some-domain.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
conn_state: :active
}
}
},
Connections.get_state(name)
)
:ok = Connections.checkout(conn1, self, name)
another_url = "http://another-domain.com"
conn = Connections.checkin(another_url, name)
assert match?(
%Connections{
conns: %{
"http:another-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up
},
^http_key => %Conn{
conn: _,
gun_state: :up
}
}
},
Connections.get_state(name)
)
end
describe "with proxy" do
test "as ip", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-string.com"
key = "http:proxy-string.com:80"
conn = Connections.checkin(url, name, proxy: {{127, 0, 0, 1}, 8123})
assert match?(
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-tuple-atom.com"
conn = Connections.checkin(url, name, proxy: {'localhost', 9050})
assert match?(
%Connections{
conns: %{
"http:proxy-tuple-atom.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as ip and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-string.com"
conn = Connections.checkin(url, name, proxy: {{127, 0, 0, 1}, 8123})
assert match?(
%Connections{
conns: %{
"https:proxy-string.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-tuple-atom.com"
conn = Connections.checkin(url, name, proxy: {'localhost', 9050})
assert match?(
%Connections{
conns: %{
"https:proxy-tuple-atom.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks type", %{name: name} do
open_mock()
url = "http://proxy-socks.com"
conn = Connections.checkin(url, name, proxy: {:socks5, 'localhost', 1234})
assert match?(
%Connections{
conns: %{
"http:proxy-socks.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks4 type and ssl", %{name: name} do
open_mock()
url = "https://proxy-socks.com"
conn = Connections.checkin(url, name, proxy: {:socks4, 'localhost', 1234})
assert match?(
%Connections{
conns: %{
"https:proxy-socks.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
},
Connections.get_state(name)
)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
end
describe "crf/3" do
setup do
crf = Connections.crf(1, 10, 1)
{:ok, crf: crf}
end
test "more used will have crf higher", %{crf: crf} do
# used 3 times
crf1 = Connections.crf(1, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 > crf2
end
test "recently used will have crf higher on equal references", %{crf: crf} do
# used 3 sec ago
crf1 = Connections.crf(3, 10, crf)
# used 4 sec ago
crf2 = Connections.crf(4, 10, crf)
assert crf1 > crf2
end
test "equal crf on equal reference and time", %{crf: crf} do
# used 2 times
crf1 = Connections.crf(1, 10, crf)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 == crf2
end
test "recently used will have higher crf", %{crf: crf} do
crf1 = Connections.crf(2, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
crf2 = Connections.crf(3, 10, crf)
crf2 = Connections.crf(4, 10, crf2)
assert crf1 > crf2
end
end
describe "get_unused_conns/1" do
setup %{name: name} do
Connections.refresh(name)
end
test "crf is equalent, sorting by reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "reference is equalent, sorting by crf", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.999
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "higher crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 3,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "lower crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.99,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
end
test "count/1" do
name = :test_count
- {:ok, _} = Connections.start_link({name, []})
+ {:ok, _} = Connections.start_link(name)
assert Connections.count(name) == 0
Connections.add_conn(name, "1", %Conn{conn: self()})
assert Connections.count(name) == 1
Connections.remove_conn(name, "1")
assert Connections.count(name) == 0
end
test "close_idle_conns/2", %{name: name} do
GunMock
|> expect(:close, fn _ -> :ok end)
|> allow(self(), name)
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
last_reference: now() - 30,
conn: self()
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now() - 10,
conn: self()
})
Connections.add_conn(name, "3", %Conn{
conn_state: :active,
conn: self()
})
name
|> Process.whereis()
|> send({:close_idle_conns, 15})
assert match?(
%Connections{
conns: %{
"3" => %Conn{},
"2" => %Conn{}
}
},
Connections.get_state(name)
)
end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sat, Aug 29, 8:13 AM (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1736894
Default Alt Text
(52 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline
Log In to Comment