Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
41 KB
Referenced Files
None
Subscribers
None
diff --git a/changelog.d/search-remote-fetch.security b/changelog.d/search-remote-fetch.security
new file mode 100644
index 000000000..4211615ae
--- /dev/null
+++ b/changelog.d/search-remote-fetch.security
@@ -0,0 +1 @@
+Status search only fetches remote URLs with resolve=true, on the first page, and for anonymous users only if limit_to_local_content allows it
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index c66723d1d..dca69260d 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -1,173 +1,193 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.DatabaseSearch do
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
require Pleroma.Constants
import Ecto.Query
@behaviour Pleroma.Search.SearchBackend
@impl true
def search(user, search_query, options \\ []) do
index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
activities =
try do
with_fuzzy_limit(fn ->
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public(user)
|> query_with(index_type, search_query)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> Pagination.fetch_paginated(
%{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
:offset
)
end)
rescue
_ -> []
end
# Outside the query transaction, as it may fetch over the network.
- maybe_fetch(activities, user, search_query)
+ maybe_fetch(activities, user, search_query, options)
end
@doc """
Runs `fun` with `gin_fuzzy_search_limit` set, if configured under
`Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their
results incomplete and random, so it is only set for search queries.
"""
def with_fuzzy_limit(fun) do
case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do
limit when is_integer(limit) and limit > 0 ->
{:ok, result} =
Repo.transaction(fn ->
# SET LOCAL takes no parameters; limit is an integer.
Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}")
fun.()
end)
result
_ ->
fun.()
end
end
@impl true
def add_to_index(_activity), do: :ok
@impl true
def remove_from_index(_object), do: :ok
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
@impl true
def healthcheck_endpoints, do: nil
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
def maybe_restrict_author(query, _), do: query
def maybe_restrict_blocked(query, %User{} = user) do
Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
end
def maybe_restrict_blocked(query, _), do: query
defp restrict_public(q, user) when not is_nil(user) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp restrict_public(q, _user) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp query_with(q, :gin, search_query) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
^tsc,
o.data,
^search_query
),
order_by: [desc: :inserted_at]
)
end
defp query_with(q, :rum, search_query) do
from([a, o] in q,
where:
fragment(
"? @@ websearch_to_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
def maybe_restrict_local(q, user) do
limit = Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp restrict_local(q), do: where(q, local: true)
- def maybe_fetch(activities, user, search_query) do
+ @doc """
+ Adds the status a URL query points to, fetching it if needed. As in Mastodon,
+ this only happens with `resolve: true` and on the first page, and for
+ anonymous users only if `[:instance, :limit_to_local_content]` is `false`.
+ """
+ def maybe_fetch(activities, user, search_query, options) do
+ if fetch_allowed?(user, options),
+ do: fetch(activities, user, search_query),
+ else: activities
+ end
+
+ defp fetch_allowed?(user, options) do
+ options[:resolve] == true and (options[:offset] || 0) == 0 and
+ case Config.get([:instance, :limit_to_local_content], :unauthenticated) do
+ :all -> false
+ :unauthenticated -> match?(%User{}, user)
+ false -> true
+ end
+ end
+
+ defp fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
true <- Visibility.visible_for_user?(activity, user) do
[activity | activities]
else
_ -> activities
end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index dc10076e1..802870b75 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -1,159 +1,159 @@
defmodule Pleroma.Search.Meilisearch do
require Logger
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
import Pleroma.Search.DatabaseSearch
import Ecto.Query
@behaviour Pleroma.Search.SearchBackend
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
[{"Content-Type", "application/json"}] ++
if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
end
def meili_get(path) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.get(
Path.join(endpoint, path),
meili_headers()
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_post(path, params) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.post(
Path.join(endpoint, path),
Jason.encode!(params),
meili_headers()
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_put(path, params) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.request(
:put,
Path.join(endpoint, path),
Jason.encode!(params),
meili_headers(),
[]
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_delete(path) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
with {:ok, _} <-
Pleroma.HTTP.request(
:delete,
Path.join(endpoint, path),
"",
meili_headers(),
[]
) do
:ok
else
_ -> {:error, "Could not remove from index"}
end
end
@impl true
def search(user, query, options \\ []) do
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
res =
meili_post(
"/indexes/objects/search",
%{q: query, offset: offset, limit: limit}
)
with {:ok, result} <- res do
hits = result["hits"] |> Enum.map(& &1["ap"])
try do
hits
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
- |> maybe_fetch(user, query)
+ |> maybe_fetch(user, query, options)
|> order_by([object: obj], desc: obj.data["published"])
|> Pleroma.Repo.all()
rescue
- _ -> maybe_fetch([], user, query)
+ _ -> maybe_fetch([], user, query, options)
end
end
end
@impl true
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
result =
meili_put(
"/indexes/objects/documents",
[search_data]
)
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
end
@impl true
def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}")
end
@impl true
def healthcheck_endpoints do
endpoint =
Config.get([Pleroma.Search.Meilisearch, :url])
|> URI.parse()
|> Map.put(:path, "/health")
|> URI.to_string()
[endpoint]
end
end
diff --git a/lib/pleroma/search/parade_db.ex b/lib/pleroma/search/parade_db.ex
index ee00e74af..78d561505 100644
--- a/lib/pleroma/search/parade_db.ex
+++ b/lib/pleroma/search/parade_db.ex
@@ -1,287 +1,287 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB do
@behaviour Pleroma.Search.SearchBackend
require Logger
require Pleroma.Constants
import Ecto.Query
import Pleroma.Search.DatabaseSearch,
only: [
- maybe_fetch: 3,
+ maybe_fetch: 4,
maybe_restrict_author: 2,
maybe_restrict_blocked: 2,
maybe_restrict_local: 2
]
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Search
@max_limit 40
@default_table "pleroma_search_documents"
defp client_impl do
Config.get([Pleroma.Search.ParadeDB, :client_impl]) ||
Pleroma.Search.ParadeDB.Client.Postgres
end
defp table do
Config.get([Pleroma.Search.ParadeDB, :table], @default_table)
end
defp fuzzy_distance do
case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do
value when is_integer(value) and value in 0..2 ->
value
value when is_binary(value) ->
case Integer.parse(value) do
{parsed, ""} when parsed in 0..2 -> parsed
_ -> 0
end
_ ->
0
end
end
@impl true
def create_index do
with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []),
{:ok, _} <- client_impl().query(create_table_sql(table()), []),
{:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []),
{:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do
:ok
else
{:error, error} ->
Logger.error("ParadeDB create_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def drop_index do
case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB drop_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def healthcheck_endpoints, do: nil
@impl true
def add_to_index(activity) do
with true <- Search.indexable?(activity),
maybe_search_data when not is_nil(maybe_search_data) <-
Search.object_to_search_data(activity.object) do
dumped_activity_id = dump_activity_id(activity.id)
actor_ap_id = activity.data["actor"]
published_at = DateTime.from_unix!(maybe_search_data.published)
if is_nil(dumped_activity_id) do
Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}")
:ok
else
sql =
"""
INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (id) DO UPDATE SET
object_id = EXCLUDED.object_id,
object_ap_id = EXCLUDED.object_ap_id,
actor_ap_id = EXCLUDED.actor_ap_id,
content = EXCLUDED.content,
published_at = EXCLUDED.published_at
"""
params = [
dumped_activity_id,
maybe_search_data.id,
maybe_search_data.ap,
actor_ap_id,
maybe_search_data.content,
published_at
]
case client_impl().query(sql, params) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB add_to_index failed: #{inspect(error)}")
{:error, error}
end
end
else
_ -> :ok
end
end
@impl true
def remove_from_index(object) do
sql = "DELETE FROM #{table()} WHERE object_id = $1"
case client_impl().query(sql, [object.id]) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def search(user, query, options \\ []) do
limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit)
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
case search_ids(query, limit, offset, author) do
{:ok, []} ->
- maybe_fetch([], user, query)
+ maybe_fetch([], user, query, options)
{:ok, ids} ->
ids
- |> fetch_activities(user, query, author)
+ |> fetch_activities(user, query, author, options)
{:error, error} ->
Logger.error("ParadeDB search failed: #{inspect(error)}")
- maybe_fetch([], user, query)
+ maybe_fetch([], user, query, options)
end
end
- defp fetch_activities(ids, user, query, author) do
+ defp fetch_activities(ids, user, query, author, options) do
dumped_ids =
ids
|> Enum.map(&dump_activity_id/1)
|> Enum.reject(&is_nil/1)
from(a in Activity, where: a.id in ^dumped_ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_public(user)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> order_by(
[a],
fragment(
"array_position(?, ?)",
^dumped_ids,
a.id
)
)
|> Pleroma.Repo.all()
- |> maybe_fetch(user, query)
+ |> maybe_fetch(user, query, options)
end
defp maybe_restrict_public(query, %Pleroma.User{}) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp maybe_restrict_public(query, _user) do
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp search_ids(query, limit, offset, author) do
base_sql =
"SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}"
params = [query]
{base_sql, params} =
case author do
%Pleroma.User{ap_id: ap_id} when is_binary(ap_id) ->
{base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]}
_ ->
{base_sql, params}
end
limit_arg = length(params) + 1
offset_arg = length(params) + 2
sql =
base_sql <>
" ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}"
params = params ++ [limit, offset]
with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do
{:ok, Enum.map(rows, fn [id] -> id end)}
end
end
defp search_query_param_sql(0), do: "$1"
defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})"
defp create_table_sql(table) do
"""
CREATE TABLE IF NOT EXISTS #{table} (
id uuid PRIMARY KEY,
object_id bigint NOT NULL,
object_ap_id text NOT NULL,
actor_ap_id text,
content text NOT NULL,
published_at timestamptz
)
"""
end
defp create_object_id_index_sql(table) do
"CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)"
end
defp create_bm25_index_sql(table) do
"""
CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table}
USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at)
WITH (key_field = 'id')
"""
end
defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid
defp dump_activity_id(id) when is_binary(id) do
case Ecto.UUID.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} ->
uuid
:error ->
dump_flake_activity_id(id)
end
end
defp dump_activity_id(_), do: nil
defp dump_flake_activity_id(id) do
case FlakeId.Ecto.CompatType.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} -> uuid
_ -> nil
end
rescue
_ -> nil
end
end
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index a8c5a3eb9..a453d222f 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -1,207 +1,207 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Object
alias Pleroma.Search
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
- import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
+ import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 4]
@impl true
def create_index do
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
OpenAIClient.post("/v1/embeddings", %{
input: text,
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp actor_from_activity(%{data: %{"actor" => actor}}) do
actor
end
defp actor_from_activity(_), do: nil
defp build_index_payload(activity, embedding) do
actor = actor_from_activity(activity)
published_at = activity.data["published"]
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding,
payload: %{actor: actor, published_at: published_at}
}
]
}
end
defp build_search_payload(embedding, options) do
base = %{
vector: embedding,
limit: options[:limit] || 20,
offset: options[:offset] || 0
}
if author = options[:author] do
Map.put(base, :filter, %{
must: [%{key: "actor", match: %{value: author.ap_id}}]
})
else
base
end
end
@impl true
def add_to_index(%Activity{object: %Object{} = object} = activity) do
search_data = Search.object_to_search_data(object)
with {:ok, embedding} <- get_embedding(search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
end
@impl true
def remove_from_index(object) do
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
with {:ok, %{status: 200}} <-
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
:ok
else
e -> {:error, e}
end
end
@impl true
def search(user, original_query, options) do
query = "Represent this sentence for searching relevant passages: #{original_query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post(
"/collections/posts/points/search",
build_search_payload(embedding, options)
) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
- |> maybe_fetch(user, original_query)
+ |> maybe_fetch(user, original_query, options)
else
_ ->
[]
end
end
@impl true
def healthcheck_endpoints do
qdrant_health =
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|> URI.parse()
|> Map.put(:path, "/healthz")
|> URI.to_string()
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
[qdrant_health, openai_health] |> Enum.filter(& &1)
end
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
alias Pleroma.Config.Getting, as: Config
def post(path, body) do
Tesla.post(client(), path, body)
end
defp client do
Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"Authorization", "Bearer #{Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
]}
]
end
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
alias Pleroma.Config.Getting, as: Config
def delete(path) do
Tesla.delete(client(), path)
end
def post(path, body) do
Tesla.post(client(), path, body)
end
def put(path, body) do
Tesla.put(client(), path, body)
end
defp client do
Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
end
defp middleware do
[
{Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])},
Tesla.Middleware.JSON,
{Tesla.Middleware.Headers,
[
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
]}
]
end
end
diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
index f0c9c1901..368c07f35 100644
--- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
@@ -1,461 +1,487 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
use Pleroma.Web.ConnCase
alias Pleroma.Object
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
import ExUnit.CaptureLog
import Tesla.Mock
import Mock
setup do
Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig)
:ok
end
setup_all do
mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
describe ".search2" do
test "it returns empty result if user or status search return undefined error", %{conn: conn} do
with_mocks [
{Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]},
{Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]}
] do
capture_log(fn ->
results =
conn
|> get("/api/v2/search?q=2hu")
|> json_response_and_validate_schema(200)
assert results["accounts"] == []
assert results["statuses"] == []
end) =~
"[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}"
end
end
@tag :skip_darwin
test "search", %{conn: conn} do
user = insert(:user)
user_two = insert(:user, %{nickname: "shp@shitposter.club"})
user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
{:ok, activity} = CommonAPI.post(user, %{status: "This is about 2hu private 天子"})
{:ok, _activity} =
CommonAPI.post(user, %{
status: "This is about 2hu, but private",
visibility: "private"
})
{:ok, _} = CommonAPI.post(user_two, %{status: "This isn't"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "2hu #private"})}")
|> json_response_and_validate_schema(200)
[account | _] = results["accounts"]
assert account["id"] == to_string(user_three.id)
assert results["hashtags"] == []
[status] = results["statuses"]
assert status["id"] == to_string(activity.id)
results =
get(conn, "/api/v2/search?q=天子")
|> json_response_and_validate_schema(200)
assert results["hashtags"] == []
[status] = results["statuses"]
assert status["id"] == to_string(activity.id)
end
test "search local-only status as an authenticated user" do
user = insert(:user)
%{conn: conn} = oauth_access(["read:search"])
{:ok, activity} =
CommonAPI.post(user, %{status: "This is about 2hu private 天子", visibility: "local"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "2hu"})}")
|> json_response_and_validate_schema(200)
[status] = results["statuses"]
assert status["id"] == to_string(activity.id)
end
test "search local-only status as an unauthenticated user" do
user = insert(:user)
%{conn: conn} = oauth_access([])
{:ok, _activity} =
CommonAPI.post(user, %{status: "This is about 2hu private 天子", visibility: "local"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "2hu"})}")
|> json_response_and_validate_schema(200)
assert [] = results["statuses"]
end
test "search local-only status as an anonymous user" do
user = insert(:user)
{:ok, _activity} =
CommonAPI.post(user, %{status: "This is about 2hu private 天子", visibility: "local"})
results =
build_conn()
|> get("/api/v2/search?#{URI.encode_query(%{q: "2hu"})}")
|> json_response_and_validate_schema(200)
assert [] = results["statuses"]
end
test "returns empty results when no hashtags match", %{conn: conn} do
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "nonexistent"})}")
|> json_response_and_validate_schema(200)
assert results["hashtags"] == []
end
test "searches hashtags by multiple words in query", %{conn: conn} do
user = insert(:user)
{:ok, _activity1} = CommonAPI.post(user, %{status: "This is my new #computer"})
{:ok, _activity2} = CommonAPI.post(user, %{status: "Check out this #laptop"})
{:ok, _activity3} = CommonAPI.post(user, %{status: "My #desktop setup"})
{:ok, _activity4} = CommonAPI.post(user, %{status: "New #phone arrived"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "new computer"})}")
|> json_response_and_validate_schema(200)
hashtag_names = Enum.map(results["hashtags"], & &1["name"])
assert "computer" in hashtag_names
refute "laptop" in hashtag_names
refute "desktop" in hashtag_names
refute "phone" in hashtag_names
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "computer laptop"})}")
|> json_response_and_validate_schema(200)
hashtag_names = Enum.map(results["hashtags"], & &1["name"])
assert "computer" in hashtag_names
assert "laptop" in hashtag_names
refute "desktop" in hashtag_names
refute "phone" in hashtag_names
end
test "supports pagination of hashtags search results", %{conn: conn} do
user = insert(:user)
{:ok, _activity1} = CommonAPI.post(user, %{status: "First #alpha hashtag"})
{:ok, _activity2} = CommonAPI.post(user, %{status: "Second #beta hashtag"})
{:ok, _activity3} = CommonAPI.post(user, %{status: "Third #gamma hashtag"})
{:ok, _activity4} = CommonAPI.post(user, %{status: "Fourth #delta hashtag"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "a", limit: 2, offset: 1})}")
|> json_response_and_validate_schema(200)
hashtag_names = Enum.map(results["hashtags"], & &1["name"])
# Should return 2 hashtags (alpha, beta, gamma, delta all contain 'a')
# With offset 1, we skip the first one, so we get 2 of the remaining 3
assert length(hashtag_names) == 2
assert Enum.all?(hashtag_names, &String.contains?(&1, "a"))
end
test "searches real hashtags from database", %{conn: conn} do
user = insert(:user)
{:ok, _activity1} = CommonAPI.post(user, %{status: "Check out this #car"})
{:ok, _activity2} = CommonAPI.post(user, %{status: "Fast #racecar on the track"})
{:ok, _activity3} = CommonAPI.post(user, %{status: "NASCAR #nascar racing"})
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "car"})}")
|> json_response_and_validate_schema(200)
hashtag_names = Enum.map(results["hashtags"], & &1["name"])
# Should return car, racecar, and nascar since they all contain "car"
assert "car" in hashtag_names
assert "racecar" in hashtag_names
assert "nascar" in hashtag_names
# Search for "race" - should return racecar
results =
conn
|> get("/api/v2/search?#{URI.encode_query(%{q: "race"})}")
|> json_response_and_validate_schema(200)
hashtag_names = Enum.map(results["hashtags"], & &1["name"])
assert "racecar" in hashtag_names
refute "car" in hashtag_names
refute "nascar" in hashtag_names
end
test "excludes a blocked users from search results", %{conn: conn} do
user = insert(:user)
user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"})
user_neo = insert(:user, %{nickname: "Agent Neo", name: "Agent"})
{:ok, act1} = CommonAPI.post(user, %{status: "This is about 2hu private 天子"})
{:ok, act2} = CommonAPI.post(user_smith, %{status: "Agent Smith"})
{:ok, act3} = CommonAPI.post(user_neo, %{status: "Agent Smith"})
Pleroma.User.block(user, user_smith)
results =
conn
|> assign(:user, user)
|> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
|> get("/api/v2/search?q=Agent")
|> json_response_and_validate_schema(200)
status_ids = Enum.map(results["statuses"], fn g -> g["id"] end)
assert act3.id in status_ids
refute act2.id in status_ids
refute act1.id in status_ids
end
end
describe ".account_search" do
test "account search", %{conn: conn} do
user_two = insert(:user, %{nickname: "shp@shitposter.club"})
user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
results =
conn
|> get("/api/v1/accounts/search?q=shp")
|> json_response_and_validate_schema(200)
result_ids = for result <- results, do: result["acct"]
assert user_two.nickname in result_ids
assert user_three.nickname in result_ids
results =
conn
|> get("/api/v1/accounts/search?q=2hu")
|> json_response_and_validate_schema(200)
result_ids = for result <- results, do: result["acct"]
assert user_three.nickname in result_ids
end
test "returns account if query contains a space", %{conn: conn} do
insert(:user, %{nickname: "shp@shitposter.club"})
results =
conn
|> get("/api/v1/accounts/search?q=shp@shitposter.club xxx")
|> json_response_and_validate_schema(200)
assert length(results) == 1
end
end
describe ".search" do
test "it returns empty result if user or status search return undefined error", %{conn: conn} do
with_mocks [
{Pleroma.User, [], [search: fn _q, _o -> raise "Oops" end]},
{Pleroma.Activity, [], [search: fn _u, _q, _o -> raise "Oops" end]}
] do
capture_log(fn ->
results =
conn
|> get("/api/v1/search?q=2hu")
|> json_response_and_validate_schema(200)
assert results["accounts"] == []
assert results["statuses"] == []
end) =~
"[error] Elixir.Pleroma.Web.MastodonAPI.SearchController search error: %RuntimeError{message: \"Oops\"}"
end
end
test "search", %{conn: conn} do
user = insert(:user)
user_two = insert(:user, %{nickname: "shp@shitposter.club"})
user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
{:ok, activity} = CommonAPI.post(user, %{status: "This is about 2hu"})
{:ok, _activity} =
CommonAPI.post(user, %{
status: "This is about 2hu, but private",
visibility: "private"
})
{:ok, _} = CommonAPI.post(user_two, %{status: "This isn't"})
results =
conn
|> get("/api/v1/search?q=2hu")
|> json_response_and_validate_schema(200)
[account | _] = results["accounts"]
assert account["id"] == to_string(user_three.id)
assert results["hashtags"] == []
[status] = results["statuses"]
assert status["id"] == to_string(activity.id)
end
- test "search fetches remote statuses and prefers them over other results", %{conn: conn} do
+ test "search fetches remote statuses and prefers them over other results" do
+ %{conn: conn} = oauth_access(["read:search"])
+
{:ok, %{id: activity_id}} =
CommonAPI.post(insert(:user), %{
status: "check out http://mastodon.example.org/@admin/99541947525187367"
})
%{"url" => result_url, "id" => result_id} =
conn
- |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367")
+ |> get(
+ "/api/v1/search?resolve=true&q=http://mastodon.example.org/@admin/99541947525187367"
+ )
|> json_response_and_validate_schema(200)
|> Map.get("statuses")
|> List.first()
refute match?(^result_id, activity_id)
assert match?(^result_url, "http://mastodon.example.org/@admin/99541947525187367")
end
+ test "search fetches remote statuses only with resolve, when logged in, on the first page",
+ %{conn: anonymous} do
+ %{conn: conn} = oauth_access(["read:search"])
+ url = "http://mastodon.example.org/@admin/99541947525187367"
+
+ for {conn, params} <- [
+ {conn, ""},
+ {conn, "&resolve=false"},
+ {conn, "&resolve=true&offset=5"},
+ {anonymous, "&resolve=true"}
+ ] do
+ assert %{"statuses" => []} =
+ conn
+ |> get("/api/v1/search?q=#{url}#{params}")
+ |> json_response_and_validate_schema(200)
+ end
+
+ refute Pleroma.Object.get_by_ap_id(
+ "http://mastodon.example.org/users/admin/statuses/99541947525187367"
+ )
+ end
+
test "search doesn't show statuses that it shouldn't", %{conn: conn} do
{:ok, activity} =
CommonAPI.post(insert(:user), %{
status: "This is about 2hu, but private",
visibility: "private"
})
capture_log(fn ->
q = Object.normalize(activity, fetch: false).data["id"]
results =
conn
|> get("/api/v1/search?q=#{q}")
|> json_response_and_validate_schema(200)
[] = results["statuses"]
end)
end
test "search fetches remote accounts", %{conn: conn} do
user = insert(:user)
query = URI.encode_query(%{q: " mike@osada.macgirvin.com ", resolve: true})
results =
conn
|> assign(:user, user)
|> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
|> get("/api/v1/search?#{query}")
|> json_response_and_validate_schema(200)
[account] = results["accounts"]
assert account["acct"] == "mike@osada.macgirvin.com"
end
test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do
results =
conn
|> get("/api/v1/search?q=mike@osada.macgirvin.com&resolve=false")
|> json_response_and_validate_schema(200)
assert [] == results["accounts"]
end
test "search with limit and offset", %{conn: conn} do
user = insert(:user)
_user_two = insert(:user, %{nickname: "shp@shitposter.club"})
_user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
{:ok, _activity1} = CommonAPI.post(user, %{status: "This is about 2hu"})
{:ok, _activity2} = CommonAPI.post(user, %{status: "This is also about 2hu"})
result =
conn
|> get("/api/v1/search?q=2hu&limit=1")
assert results = json_response_and_validate_schema(result, 200)
assert [%{"id" => activity_id1}] = results["statuses"]
assert [_] = results["accounts"]
results =
conn
|> get("/api/v1/search?q=2hu&limit=1&offset=1")
|> json_response_and_validate_schema(200)
assert [%{"id" => activity_id2}] = results["statuses"]
assert [] = results["accounts"]
assert activity_id1 != activity_id2
end
test "search returns results only for the given type", %{conn: conn} do
user = insert(:user)
_user_two = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
{:ok, _activity} = CommonAPI.post(user, %{status: "This is about 2hu"})
assert %{"statuses" => [_activity], "accounts" => [], "hashtags" => []} =
conn
|> get("/api/v1/search?q=2hu&type=statuses")
|> json_response_and_validate_schema(200)
assert %{"statuses" => [], "accounts" => [_user_two], "hashtags" => []} =
conn
|> get("/api/v1/search?q=2hu&type=accounts")
|> json_response_and_validate_schema(200)
end
test "search uses account_id to filter statuses by the author", %{conn: conn} do
user = insert(:user, %{nickname: "shp@shitposter.club"})
user_two = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
{:ok, activity1} = CommonAPI.post(user, %{status: "This is about 2hu"})
{:ok, activity2} = CommonAPI.post(user_two, %{status: "This is also about 2hu"})
results =
conn
|> get("/api/v1/search?q=2hu&account_id=#{user.id}")
|> json_response_and_validate_schema(200)
assert [%{"id" => activity_id1}] = results["statuses"]
assert activity_id1 == activity1.id
assert [_] = results["accounts"]
results =
conn
|> get("/api/v1/search?q=2hu&account_id=#{user_two.id}")
|> json_response_and_validate_schema(200)
assert [%{"id" => activity_id2}] = results["statuses"]
assert activity_id2 == activity2.id
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Wed, Nov 18, 10:56 PM (1 d, 22 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1839450
Default Alt Text
(41 KB)

Event Timeline