Page MenuHomePhorge
Files F85827268

No OneTemporary
Actions

Authored By
Unknown
Size
10 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 740e6e9ff..0728cbaa2 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -1,144 +1,125 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper do
@moduledoc """
Configure Tesla.Client with default and customized adapter options.
"""
@defaults [pool: :federation]
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | :inet.ip_address()
- @type pool() :: :federation | :upload | :media | :default
- alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
require Logger
@type proxy ::
{Connection.host(), pos_integer()}
| {Connection.proxy_type(), Connection.host(), pos_integer()}
@callback options(keyword(), URI.t()) :: keyword()
@callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()}
@spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
def format_proxy(nil), do: nil
def format_proxy(proxy_url) do
case parse_proxy(proxy_url) do
{:ok, host, port} -> {host, port}
{:ok, type, host, port} -> {type, host, port}
_ -> nil
end
end
@spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
def maybe_add_proxy(opts, nil), do: opts
def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
@doc """
Merge default connection & adapter options with received ones.
"""
@spec options(URI.t(), keyword()) :: keyword()
def options(%URI{} = uri, opts \\ []) do
@defaults
|> Keyword.merge(opts)
- |> put_timeout()
|> adapter_helper().options(uri)
end
- @spec pool_timeout(pool()) :: non_neg_integer()
- def pool_timeout(pool) do
- {config_key, default} =
- if adapter() == Tesla.Adapter.Gun do
- {:pools, Config.get([:pools, :default, :timeout], 5_000)}
- else
- {:hackney_pools, 10_000}
- end
-
- Config.get([config_key, pool, :timeout], default)
- end
-
- # For Hackney, this is the time a connection can stay idle in the pool.
- # For Gun, this is the timeout to receive a message from Gun.
- defp put_timeout(opts) do
- Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
- end
-
+ @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
+
defp adapter, do: Application.get_env(:tesla, :adapter)
defp adapter_helper do
case adapter() do
Tesla.Adapter.Gun -> AdapterHelper.Gun
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
_ -> AdapterHelper.Default
end
end
@spec parse_proxy(String.t() | tuple() | nil) ::
{:ok, host(), pos_integer()}
| {:ok, proxy_type(), host(), pos_integer()}
| {:error, atom()}
| nil
def parse_proxy(nil), do: nil
def parse_proxy(proxy) when is_binary(proxy) do
with [host, port] <- String.split(proxy, ":"),
{port, ""} <- Integer.parse(port) do
{:ok, parse_host(host), port}
else
{_, _} ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
:error ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
def parse_proxy(proxy) when is_tuple(proxy) do
with {type, host, port} <- proxy do
{:ok, type, parse_host(host), port}
else
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
@spec parse_host(String.t() | atom() | charlist()) :: charlist() | :inet.ip_address()
def parse_host(host) when is_list(host), do: host
def parse_host(host) when is_atom(host), do: to_charlist(host)
def parse_host(host) when is_binary(host) do
host = to_charlist(host)
case :inet.parse_address(host) do
{:error, :einval} -> host
{:ok, ip} -> ip
end
end
@spec format_host(String.t()) :: charlist()
def format_host(host) do
host_charlist = to_charlist(host)
case :inet.parse_address(host_charlist) do
{:error, :einval} ->
:idna.encode(host_charlist)
{:ok, _ip} ->
host_charlist
end
end
end
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index db0a298b3..02e20f2d1 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -1,78 +1,93 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Gun do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.Config
alias Pleroma.Gun.ConnectionPool
alias Pleroma.HTTP.AdapterHelper
require Logger
@defaults [
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
]
+ @type pool() :: :federation | :upload | :media | :default
+
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
proxy =
[:http, :proxy_url]
|> Config.get()
|> AdapterHelper.format_proxy()
config_opts = Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
+ |> put_timeout()
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
Keyword.put(opts, :certificates_verification, true)
end
+ defp put_timeout(opts) do
+ # this is the timeout to receive a message from Gun
+ Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+ end
+
+ @spec pool_timeout(pool()) :: non_neg_integer()
+ def pool_timeout(pool) do
+ default = Config.get([:pools, :default, :timeout], 5_000)
+
+ Config.get([:pools, pool, :timeout], default)
+ end
+
@spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
def get_conn(uri, opts) do
case ConnectionPool.get_conn(uri, opts) do
{:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)}
err -> err
end
end
@prefix Pleroma.Gun.ConnectionPool
def limiter_setup do
wait = Config.get([:connections_pool, :connection_acquisition_wait])
retries = Config.get([:connections_pool, :connection_acquisition_retries])
:pools
|> Config.get([])
|> Enum.each(fn {name, opts} ->
max_running = Keyword.get(opts, :size, 50)
max_waiting = Keyword.get(opts, :max_waiting, 10)
result =
ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting,
wait: wait,
max_retries: retries
)
case result do
:ok -> :ok
{:error, :existing} -> :ok
end
end)
:ok
end
end
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
index ed9794ca2..6dbef9085 100644
--- a/lib/pleroma/uploaders/s3.ex
+++ b/lib/pleroma/uploaders/s3.ex
@@ -1,97 +1,101 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.S3 do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Pleroma.Config
# The file name is re-encoded with S3's constraints here to comply with previous
# links with less strict filenames
@impl true
def get_file(file) do
config = Config.get([__MODULE__])
bucket = Keyword.fetch!(config, :bucket)
bucket_with_namespace =
cond do
truncated_namespace = Keyword.get(config, :truncated_namespace) ->
truncated_namespace
namespace = Keyword.get(config, :bucket_namespace) ->
namespace <> ":" <> bucket
true ->
bucket
end
{:ok,
{:url,
Path.join([
Keyword.fetch!(config, :public_endpoint),
bucket_with_namespace,
strict_encode(URI.decode(file))
])}}
end
@impl true
def put_file(%Pleroma.Upload{} = upload) do
config = Config.get([__MODULE__])
bucket = Keyword.get(config, :bucket)
streaming = Keyword.get(config, :streaming_enabled)
s3_name = strict_encode(upload.path)
op =
if streaming do
op =
upload.tempfile
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload(bucket, s3_name, [
{:acl, :public_read},
{:content_type, upload.content_type}
])
- # set s3 upload timeout to respect :upload pool timeout
- # timeout should be slightly larger, so s3 can retry upload on fail
- timeout = Pleroma.HTTP.AdapterHelper.pool_timeout(:upload) + 1_000
- opts = Keyword.put(op.opts, :timeout, timeout)
- Map.put(op, :opts, opts)
+ if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+ # set s3 upload timeout to respect :upload pool timeout
+ # timeout should be slightly larger, so s3 can retry upload on fail
+ timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000
+ opts = Keyword.put(op.opts, :timeout, timeout)
+ Map.put(op, :opts, opts)
+ else
+ op
+ end
else
{:ok, file_data} = File.read(upload.tempfile)
ExAws.S3.put_object(bucket, s3_name, file_data, [
{:acl, :public_read},
{:content_type, upload.content_type}
])
end
case ExAws.request(op) do
{:ok, _} ->
{:ok, {:file, s3_name}}
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "S3 Upload failed"}
end
end
@impl true
def delete_file(file) do
[__MODULE__, :bucket]
|> Config.get()
|> ExAws.S3.delete_object(file)
|> ExAws.request()
|> case do
{:ok, %{status_code: 204}} -> :ok
error -> {:error, inspect(error)}
end
end
@regex Regex.compile!("[^0-9a-zA-Z!.*/'()_-]")
def strict_encode(name) do
String.replace(name, @regex, "-")
end
end

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 30, 5:11 PM (14 h, 20 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1800343
Default Alt Text
(10 KB)

Event Timeline