Page MenuHomePhorge
Files F85824852

No OneTemporary
Actions

Authored By
Unknown
Size
21 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex
index 561b015be..5ac7f96f9 100644
--- a/lib/pleroma/http.ex
+++ b/lib/pleroma/http.ex
@@ -1,239 +1,244 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP do
@moduledoc """
Wrapper for `Tesla.request/2`.
"""
alias Pleroma.HTTP.AdapterHelper
alias Pleroma.HTTP.Request
alias Pleroma.HTTP.RequestBuilder, as: Builder
alias Tesla.Client
alias Tesla.Env
require Logger
require Pleroma.Constants
@type t :: __MODULE__
@type method() :: :get | :post | :put | :delete | :head
@doc """
Performs GET request.
See `Pleroma.HTTP.request/5`
"""
@spec get(Request.url() | nil, Request.headers(), keyword()) ::
nil | {:ok, Env.t()} | {:error, any()}
def get(url, headers \\ [], options \\ [])
def get(nil, _, _), do: nil
def get(url, headers, options), do: request(:get, url, "", headers, options)
@spec head(Request.url(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()}
def head(url, headers \\ [], options \\ []), do: request(:head, url, "", headers, options)
@doc """
Performs POST request.
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@doc """
Builds and performs http request.
# Arguments:
`method` - :get, :post, :put, :delete, :head
`url` - full url
`body` - request body
`headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
`options` - custom, per-request middleware or adapter options
# Returns:
`{:ok, %Tesla.Env{}}` or `{:error, error}`
"""
@spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def request(method, url, body, headers, options) when is_binary(url) do
uri = URI.parse(url)
adapter_opts = AdapterHelper.options(uri, options || [])
options = put_in(options[:adapter], adapter_opts)
params = options[:params] || []
request = build_request(method, headers, options, url, body, params)
adapter = Application.get_env(:tesla, :adapter)
extra_middleware = options[:tesla_middleware] || []
client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter)
maybe_limit(
fn ->
request(client, request)
end,
adapter,
adapter_opts
)
end
@spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
def request(client, request), do: Tesla.request(client, request)
defp build_request(method, headers, options, url, body, params) do
Builder.new()
|> Builder.method(method)
|> Builder.headers(headers)
|> Builder.opts(options)
|> Builder.url(url)
|> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
|> Builder.convert_to_keyword()
end
@prefix Pleroma.Gun.ConnectionPool
defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do
ConcurrentLimiter.limit(:"#{@prefix}.#{opts[:pool] || :default}", fun)
end
defp maybe_limit(fun, _, _) do
fun.()
end
defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do
default_middleware() ++
[Pleroma.Tesla.Middleware.ConnectionPool] ++
extra_middleware
end
defp adapter_middlewares(_, extra_middleware) do
# A lot of tests are written expecting unencoded URLs
# and the burden of fixing that is high. Also it makes
# them hard to read. Tests will opt-in when we want to validate
# the encoding is being done correctly.
cond do
Pleroma.Config.get(:env) == :test and Pleroma.Config.get(:test_url_encoding) ->
default_middleware()
Pleroma.Config.get(:env) == :test ->
# Emulate redirects in test env, which are handled by adapters in other environments
[Tesla.Middleware.FollowRedirects]
# Hackney and Finch
true ->
default_middleware() ++ extra_middleware
end
end
defp default_middleware,
do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
# We don't always want to decode the path first, like is the case in
# Pleroma.Upload.url_from_spec/3.
def encode_url(url, opts \\ []) when is_binary(url) and is_list(opts) do
bypass_parse = Keyword.get(opts, :bypass_parse, false)
bypass_decode = Keyword.get(opts, :bypass_decode, false)
cond do
bypass_parse ->
encode_path(url, bypass_decode)
true ->
URI.parse(url)
|> then(fn parsed ->
- path = encode_path(parsed.path, bypass_decode)
- |> maybe_apply_path_encoding_quirks()
+ path =
+ encode_path(parsed.path, bypass_decode)
+ |> maybe_apply_path_encoding_quirks()
+
query = encode_query(parsed.query)
%{parsed | path: path, query: query}
end)
|> URI.to_string()
end
end
defp encode_path(nil, _bypass_decode), do: nil
# URI.encode/2 deliberately does not encode all chars that are forbidden
# in the path component of a URI. It only encodes chars that are forbidden
# in the whole URI. A predicate in the 2nd argument is used to fix that here.
# URI.encode/2 uses the predicate function to determine whether each byte
# (in an integer representation) should be encoded or not.
defp encode_path(path, bypass_decode) when is_binary(path) do
path =
cond do
bypass_decode ->
path
true ->
URI.decode(path)
end
path
|> URI.encode(fn byte ->
URI.char_unreserved?(byte) ||
Enum.any?(
Pleroma.Constants.uri_path_allowed_reserved_chars(),
fn char ->
char == byte
end
)
end)
end
defp encode_query(nil), do: nil
# Order of kv pairs in query is not preserved when using URI.decode_query.
# URI.query_decoder/2 returns a stream which so far appears to not change order.
# Immediately switch to a list to prevent breakage for sites that expect
# the order of query keys to be always the same.
defp encode_query(query) when is_binary(query) do
query
|> URI.query_decoder()
|> Enum.to_list()
|> do_encode_query()
end
-
+
defp maybe_apply_path_encoding_quirks(path), do: path
# Always uses www_form encoding
defp do_encode_query(enumerable) do
Enum.map_join(enumerable, "&", &maybe_apply_query_quirk(&1))
end
defp maybe_apply_query_quirk({key, value}) do
case key do
"precrop" ->
query_encode_kv_pair({key, value}, ~c":,")
key ->
query_encode_kv_pair({key, value})
end
end
defp query_encode_kv_pair({key, value}, rules \\ []) when is_list(rules) do
cond do
length(rules) > 0 ->
# URI.encode_query/2 does not appear to follow spec and encodes all parts of our URI path Constant.
# This appears to work outside of edge-cases like The Guardian Rich Media Cards,
# keeping behavior same as with URI.encode_query/2 unless otherwise specified via rules.
- URI.encode_www_form(Kernel.to_string(key)) <> "=" <>
- URI.encode(value, fn byte ->
- URI.char_unreserved?(byte) ||
- Enum.any?(
- rules,
- fn char ->
- char == byte
- end)
- end)
- |> String.replace("%20", "+")
+ (URI.encode_www_form(Kernel.to_string(key)) <>
+ "=" <>
+ URI.encode(value, fn byte ->
+ URI.char_unreserved?(byte) ||
+ Enum.any?(
+ rules,
+ fn char ->
+ char == byte
+ end
+ )
+ end))
+ |> String.replace("%20", "+")
true ->
- URI.encode_www_form(Kernel.to_string(key)) <> "=" <> URI.encode_www_form(Kernel.to_string(value))
+ URI.encode_www_form(Kernel.to_string(key)) <>
+ "=" <> URI.encode_www_form(Kernel.to_string(value))
end
end
end
diff --git a/test/pleroma/http_test.exs b/test/pleroma/http_test.exs
index fe6afea04..9bdb6ec35 100644
--- a/test/pleroma/http_test.exs
+++ b/test/pleroma/http_test.exs
@@ -1,197 +1,214 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTPTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
import Tesla.Mock
alias Pleroma.HTTP
setup do
mock(fn
%{
method: :get,
url: "http://example.com/hello",
headers: [{"content-type", "application/json"}]
} ->
json(%{"my" => "data"})
%{method: :head, url: "http://example.com/hello"} ->
%Tesla.Env{status: 200, body: ""}
%{method: :get, url: "http://example.com/hello"} ->
%Tesla.Env{status: 200, body: "hello"}
%{method: :post, url: "http://example.com/world"} ->
%Tesla.Env{status: 200, body: "world"}
%{method: :get, url: "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz"} ->
%Tesla.Env{status: 200, body: "emoji data"}
%{
method: :get,
url: "https://example.com/media/foo/bar%20!$&'()*+,;=/:%20@a%20%5Bbaz%5D.mp4"
} ->
%Tesla.Env{status: 200, body: "video data"}
%{method: :get, url: "https://example.com/media/unicode%20%F0%9F%99%82%20.gif"} ->
%Tesla.Env{status: 200, body: "unicode data"}
- %{method: :get, url: "https://i.guim.co.uk/img/media/1069ef13c447908272c4de94174cec2b6352cb2f/0_91_2000_1201/master/2000.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&precrop=40:21,offset-x50,offset-y0&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctb3BpbmlvbnMtYWdlLTIwMTkucG5n&enable=upscale&s=cba21427a73512fdc9863c486c03fdd8"} ->
+ %{
+ method: :get,
+ url:
+ "https://i.guim.co.uk/img/media/1069ef13c447908272c4de94174cec2b6352cb2f/0_91_2000_1201/master/2000.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&precrop=40:21,offset-x50,offset-y0&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctb3BpbmlvbnMtYWdlLTIwMTkucG5n&enable=upscale&s=cba21427a73512fdc9863c486c03fdd8"
+ } ->
%Tesla.Env{status: 200, body: "Guardian image quirk"}
- %{method: :get, url: "https://example.com/emoji/Pack%201/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar+baz"} ->
+ %{
+ method: :get,
+ url:
+ "https://example.com/emoji/Pack%201/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar+baz"
+ } ->
%Tesla.Env{status: 200, body: "Space in query with Guardian quirk"}
- %{method: :get, url: "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY%2F20130721%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"} ->
+ %{
+ method: :get,
+ url:
+ "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY%2F20130721%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"
+ } ->
%Tesla.Env{status: 200, body: "AWS S3 data"}
end)
:ok
end
describe "head/1" do
test "returns successfully result" do
assert HTTP.head("http://example.com/hello") == {:ok, %Tesla.Env{status: 200, body: ""}}
end
end
describe "get/1" do
test "returns successfully result" do
assert HTTP.get("http://example.com/hello") == {
:ok,
%Tesla.Env{status: 200, body: "hello"}
}
end
end
describe "get/2 (with headers)" do
test "returns successfully result for json content-type" do
assert HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) ==
{
:ok,
%Tesla.Env{
status: 200,
body: "{\"my\":\"data\"}",
headers: [{"content-type", "application/json"}]
}
}
end
end
describe "post/2" do
test "returns successfully result" do
assert HTTP.post("http://example.com/world", "") == {
:ok,
%Tesla.Env{status: 200, body: "world"}
}
end
end
test "URL encoding properly encodes URLs with spaces" do
clear_config(:test_url_encoding, true)
url_with_space = "https://example.com/emoji/Pack 1/koronebless.png?foo=bar baz"
{:ok, result} = HTTP.get(url_with_space)
assert result.status == 200
properly_encoded_url = "https://example.com/emoji/Pack%201/koronebless.png?foo=bar+baz"
{:ok, result} = HTTP.get(properly_encoded_url)
assert result.status == 200
url_with_reserved_chars = "https://example.com/media/foo/bar !$&'()*+,;=/: @a [baz].mp4"
{:ok, result} = HTTP.get(url_with_reserved_chars)
assert result.status == 200
url_with_unicode = "https://example.com/media/unicode 🙂 .gif"
{:ok, result} = HTTP.get(url_with_unicode)
assert result.status == 200
end
test "decodes URL first by default" do
clear_config(:test_url_encoding, true)
normal_url = "https://example.com/media/file%20with%20space.jpg?name=a+space.jpg"
result = HTTP.encode_url(normal_url)
assert result == "https://example.com/media/file%20with%20space.jpg?name=a+space.jpg"
end
test "doesn't decode URL first when specified" do
clear_config(:test_url_encoding, true)
normal_url = "https://example.com/media/file%20with%20space.jpg"
result = HTTP.encode_url(normal_url, bypass_decode: true)
assert result == "https://example.com/media/file%2520with%2520space.jpg"
end
test "properly applies Guardian image query quirk" do
clear_config(:test_url_encoding, true)
- url = "https://i.guim.co.uk/img/media/1069ef13c447908272c4de94174cec2b6352cb2f/0_91_2000_1201/master/2000.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&precrop=40:21,offset-x50,offset-y0&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctb3BpbmlvbnMtYWdlLTIwMTkucG5n&enable=upscale&s=cba21427a73512fdc9863c486c03fdd8"
+ url =
+ "https://i.guim.co.uk/img/media/1069ef13c447908272c4de94174cec2b6352cb2f/0_91_2000_1201/master/2000.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&precrop=40:21,offset-x50,offset-y0&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctb3BpbmlvbnMtYWdlLTIwMTkucG5n&enable=upscale&s=cba21427a73512fdc9863c486c03fdd8"
result = HTTP.encode_url(url)
assert result == url
{:ok, result_get} = HTTP.get(result)
assert result_get.status == 200
end
test "properly encodes spaces as \"pluses\" in query when using quirks" do
clear_config(:test_url_encoding, true)
- url = "https://example.com/emoji/Pack 1/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar baz"
+ url =
+ "https://example.com/emoji/Pack 1/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar baz"
- properly_encoded_url = "https://example.com/emoji/Pack%201/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar+baz"
+ properly_encoded_url =
+ "https://example.com/emoji/Pack%201/koronebless.png?precrop=40:21,overlay-x0,overlay-y0&foo=bar+baz"
result = HTTP.encode_url(url)
assert result == properly_encoded_url
{:ok, result_get} = HTTP.get(result)
assert result_get.status == 200
end
test "properly encode AWS S3 queries" do
clear_config(:test_url_encoding, true)
- url = "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY%2F20130721%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"
- unencoded_url = "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"
+ url =
+ "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY%2F20130721%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"
+
+ unencoded_url =
+ "https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=accessKEY/20130721/us-east-1/s3/aws4_request&X-Amz-Date=20130721T201207Z&X-Amz-Expires=86400&X-Amz-Signature=SIGNATURE&X-Amz-SignedHeaders=host"
result = HTTP.encode_url(url)
result_unencoded = HTTP.encode_url(unencoded_url)
assert result == url
assert result == result_unencoded
{:ok, result_get} = HTTP.get(result)
assert result_get.status == 200
-
end
test "preserves query key order" do
clear_config(:test_url_encoding, true)
url = "https://example.com/foo?hjkl=qwertz&xyz=abc&bar=baz"
result = HTTP.encode_url(url)
assert result == url
end
end
diff --git a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs
index 13b4d24b8..b589068c7 100644
--- a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs
+++ b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs
@@ -1,115 +1,115 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do
use ExUnit.Case
use Pleroma.Tests.Helpers
alias Pleroma.HTTP
alias Pleroma.UnstubbedConfigMock, as: ConfigMock
alias Pleroma.Web.ActivityPub.MRF
alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy
import Mock
import Mox
@message %{
"type" => "Create",
"object" => %{
"type" => "Note",
"content" => "content",
"attachment" => [
%{"url" => [%{"href" => "http://example.com/image.jpg"}]}
]
}
}
@message_with_history %{
"type" => "Create",
"object" => %{
"type" => "Note",
"content" => "content",
"formerRepresentations" => %{
"orderedItems" => [
%{
"type" => "Note",
"content" => "content",
"attachment" => [
%{"url" => [%{"href" => "http://example.com/image.jpg"}]}
]
}
]
}
}
}
setup do
ConfigMock
|> stub_with(Pleroma.Test.StaticConfig)
:ok
end
setup do: clear_config([:media_proxy, :enabled], true)
test "it prefetches media proxy URIs" do
Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} ->
{:ok, %Tesla.Env{status: 200, body: ""}}
end)
with_mock HTTP,
get: fn _, _, _ -> {:ok, []} end,
encode_url: fn url -> :meck.passthrough([url]) end,
encode_url: fn url, opts -> :meck.passthrough([url, opts]) end do
- MediaProxyWarmingPolicy.filter(@message)
+ MediaProxyWarmingPolicy.filter(@message)
- assert called(HTTP.get(:_, :_, :_))
+ assert called(HTTP.get(:_, :_, :_))
end
end
test "it does nothing when no attachments are present" do
object =
@message["object"]
|> Map.delete("attachment")
message =
@message
|> Map.put("object", object)
with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
MediaProxyWarmingPolicy.filter(message)
refute called(HTTP.get(:_, :_, :_))
end
end
test "history-aware" do
Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} ->
{:ok, %Tesla.Env{status: 200, body: ""}}
end)
with_mock HTTP,
get: fn _, _, _ -> {:ok, []} end,
encode_url: fn url -> :meck.passthrough([url]) end,
encode_url: fn url, opts -> :meck.passthrough([url, opts]) end do
- MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history)
+ MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history)
- assert called(HTTP.get(:_, :_, :_))
+ assert called(HTTP.get(:_, :_, :_))
end
end
test "works with Updates" do
Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} ->
{:ok, %Tesla.Env{status: 200, body: ""}}
end)
with_mock HTTP,
get: fn _, _, _ -> {:ok, []} end,
encode_url: fn url -> :meck.passthrough([url]) end,
encode_url: fn url, opts -> :meck.passthrough([url, opts]) end do
- MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history |> Map.put("type", "Update"))
+ MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history |> Map.put("type", "Update"))
assert called(HTTP.get(:_, :_, :_))
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Thu, Oct 29, 7:29 PM (1 d, 21 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1798569
Default Alt Text
(21 KB)

Event Timeline