Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85831231
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
25 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/changelog.d/search-url-no-fts.fix b/changelog.d/search-url-no-fts.fix
new file mode 100644
index 000000000..fd51c5d0b
--- /dev/null
+++ b/changelog.d/search-url-no-fts.fix
@@ -0,0 +1 @@
+User search: a URL query only returns exact matches (resolved account, ap_id, uri) instead of also text-searching the URL's fragments against every account.
diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index abc45f62a..1455288e9 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -1,322 +1,351 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.Search do
alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType
alias Pleroma.Instances.Instance
alias Pleroma.Pagination
alias Pleroma.User
import Ecto.Query
@limit 20
def search(query_string, opts \\ []) do
resolve = Keyword.get(opts, :resolve, false)
following = Keyword.get(opts, :following, false)
result_limit = Keyword.get(opts, :limit, @limit)
offset = Keyword.get(opts, :offset, 0)
capabilities = Keyword.get(opts, :capabilities, [])
for_user = Keyword.get(opts, :for_user)
query_string = format_query(query_string)
# If this returns anything, it should bounce to the top
maybe_resolved = maybe_resolve(resolve, for_user, query_string)
top_user_ids =
[]
|> maybe_add_resolved(maybe_resolved)
|> maybe_add_ap_id_match(query_string)
|> maybe_add_uri_match(query_string)
+ |> maybe_add_profile_url_match(query_string)
results =
query_string
|> search_query(for_user, following, top_user_ids, capabilities)
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
results
end
defp maybe_add_resolved(list, {:ok, %User{} = user}) do
[user.id | list]
end
defp maybe_add_resolved(list, _), do: list
defp maybe_add_ap_id_match(list, query) do
if user = User.get_cached_by_ap_id(query) do
[user.id | list]
else
list
end
end
defp maybe_add_uri_match(list, query) do
with {:ok, query} <- UriType.cast(query),
q = from(u in User, where: u.uri == ^query, select: u.id),
users = Pleroma.Repo.all(q) do
users ++ list
else
_ -> list
end
end
+ # "https://host/users/name" or "https://host/@name" -> the user known as name@host
+ defp maybe_add_profile_url_match(list, "http" <> _ = query) do
+ with %URI{host: host, path: path}
+ when is_binary(host) and host != "" and is_binary(path) <- URI.parse(query),
+ [_ | _] = segments <- String.split(path, "/", trim: true),
+ segment when segment != "" <- segments |> List.last() |> String.trim_leading("@"),
+ %User{id: id} <- get_by_nickname(segment <> "@" <> host) do
+ [id | list]
+ else
+ _ -> list
+ end
+ end
+
+ defp maybe_add_profile_url_match(list, _), do: list
+
+ # :idna.encode/1 exits rather than raises on hosts it cannot encode
+ defp get_by_nickname(nickname) do
+ User.get_by_nickname(format_query(nickname))
+ catch
+ _, _ -> nil
+ end
+
defp format_query(query_string) do
# Strip the beginning @ off if there is a query
query_string = String.trim_leading(query_string, "@")
with [name, domain] <- String.split(query_string, "@") do
encoded_domain =
domain
|> String.replace(~r/[!-\-|@|[-`|{-~|\/|:|\s]+/, "")
|> String.to_charlist()
|> :idna.encode()
|> to_string()
name <> "@" <> encoded_domain
else
_ -> query_string
end
end
defp search_query(query_string, for_user, following, top_user_ids, capabilities) do
for_user
|> base_query(following)
|> filter_blocked_user(for_user)
|> filter_invisible_users()
|> filter_internal_users()
|> filter_blocked_domains(for_user)
|> filter_unreachable_users()
|> fts_search(query_string)
|> select_top_users(top_user_ids)
|> trigram_rank(query_string)
|> boost_search_rank(for_user, top_user_ids)
|> subquery()
|> order_by_search_rank(for_user)
|> maybe_restrict_local(for_user)
|> maybe_restrict_accepting_chat_messages(capabilities)
|> filter_deactivated_users()
end
defp select_top_users(query, top_user_ids) do
from(u in query,
or_where: u.id in ^top_user_ids
)
end
+ # Clients search for URLs to resolve them. Those are matched exactly
+ # (ap_id, uri, resolve), and running the URL's fragments ("https", "de",
+ # "notice") through the text index would rank every account matching one
+ # of them, which on a large instance is tens of thousands of rows.
+ defp fts_search(query, "http" <> _), do: where(query, false)
+
defp fts_search(query, query_string) do
query_string = to_tsquery(query_string)
from(
u in query,
where:
fragment(
# The fragment must _exactly_ match `users_fts_index`, otherwise the index won't work
"""
(
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')
) @@ to_tsquery('simple', ?)
""",
u.nickname,
u.name,
^query_string
)
)
end
defp to_tsquery(query_string) do
String.trim_trailing(query_string, "@" <> local_domain())
|> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
|> String.trim()
|> String.split()
|> Enum.map(&(&1 <> ":*"))
|> Enum.join(" | ")
end
# Considers nickname match, localized nickname match, name match; preferences nickname match
defp trigram_rank(query, query_string) do
from(
u in query,
select_merge: %{
search_rank:
fragment(
"""
similarity(?, ?) +
similarity(?, regexp_replace(?, '@.+', '')) +
similarity(?, trim(coalesce(?, '')))
""",
^query_string,
u.nickname,
^query_string,
u.nickname,
^query_string,
u.name
)
}
)
end
defp base_query(%User{} = user, true), do: User.get_friends_query(user)
defp base_query(_user, _following), do: User
defp filter_invisible_users(query) do
from(q in query, where: q.invisible == false)
end
defp filter_internal_users(query) do
from(q in query, where: q.actor_type != "Application")
end
defp filter_deactivated_users(query) do
from(q in query, where: q.is_active == true)
end
defp filter_blocked_user(query, %User{} = blocker) do
query
|> join(:left, [u], b in Pleroma.UserRelationship,
as: :blocks,
on: b.relationship_type == ^:block and b.source_id == ^blocker.id and u.id == b.target_id
)
|> where([blocks: b], is_nil(b.target_id))
end
defp filter_blocked_user(query, _), do: query
defp filter_blocked_domains(query, %User{domain_blocks: domain_blocks})
when length(domain_blocks) > 0 do
domains = Enum.join(domain_blocks, ",")
from(
q in query,
where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains)
)
end
defp filter_blocked_domains(query, _), do: query
defp filter_unreachable_users(query) do
from(u in query,
left_join: i in Instance,
on: i.host == fragment("substring(? from '.*://([^/]*)')", u.ap_id),
where: is_nil(i.unreachable_since)
)
end
defp maybe_resolve(true, user, query) do
case {limit(), user} do
{:all, _} -> :noop
{:unauthenticated, %User{}} -> User.get_or_fetch(query)
{:unauthenticated, _} -> :noop
{false, _} -> User.get_or_fetch(query)
end
end
defp maybe_resolve(_, _, _), do: :noop
defp maybe_restrict_local(q, user) do
case {limit(), user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp maybe_restrict_accepting_chat_messages(query, capabilities) do
if "accepts_chat_messages" in capabilities do
from(q in query, where: q.accepts_chat_messages == true)
else
query
end
end
defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
defp restrict_local(q), do: where(q, [u], u.local == true)
defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
defp boost_search_rank(query, %User{} = for_user, top_user_ids) do
friends_ids = User.get_friends_ids(for_user)
followers_ids = User.get_followers_ids(for_user)
from(u in subquery(query),
select_merge: %{
search_type:
fragment(
"""
CASE WHEN (?) THEN 2
WHEN (?) THEN 1
ELSE 0 END
""",
u.id in ^top_user_ids,
u.id in ^friends_ids or u.id in ^followers_ids
),
search_rank:
fragment(
"""
CASE WHEN (?) THEN (?) * 1.5
WHEN (?) THEN (?) * 1.3
WHEN (?) THEN (?) * 1.1
WHEN (?) THEN 9001
ELSE (?) END
""",
u.id in ^friends_ids and u.id in ^followers_ids,
u.search_rank,
u.id in ^friends_ids,
u.search_rank,
u.id in ^followers_ids,
u.search_rank,
u.id in ^top_user_ids,
u.search_rank
)
}
)
end
defp boost_search_rank(query, _for_user, top_user_ids) do
from(u in subquery(query),
select_merge: %{
search_type:
fragment(
"""
CASE WHEN (?) THEN 2
ELSE 0 END
""",
u.id in ^top_user_ids
),
search_rank:
fragment(
"""
CASE WHEN (?) THEN 9001
ELSE (?) END
""",
u.id in ^top_user_ids,
u.search_rank
)
}
)
end
defp order_by_search_rank(query, %User{}) do
order_by(
query,
[u],
desc: u.search_type,
desc_nulls_last:
fragment(
"CASE WHEN ? = 1 THEN COALESCE(?, ?) ELSE NULL END",
u.search_type,
u.last_status_at,
u.last_active_at
),
desc: u.search_rank
)
end
defp order_by_search_rank(query, _), do: order_by(query, desc: :search_rank)
end
diff --git a/test/pleroma/user_search_test.exs b/test/pleroma/user_search_test.exs
index 75cfbd2db..6b83c7ca6 100644
--- a/test/pleroma/user_search_test.exs
+++ b/test/pleroma/user_search_test.exs
@@ -1,378 +1,407 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.UserSearchTest do
alias Pleroma.User
use Pleroma.DataCase
import Pleroma.Factory
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
describe "User.search" do
setup do: clear_config([:instance, :limit_to_local_content])
test "returns a resolved user as the first result" do
clear_config([:instance, :limit_to_local_content], false)
user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/users/lain", resolve: true)
+ [first_user] = User.search("https://lain.com/users/lain", resolve: true)
assert first_user.id == user.id
end
test "returns a user with matching ap_id as the first result" do
user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/users/lain")
+ [first_user] = User.search("https://lain.com/users/lain")
assert first_user.id == user.id
end
+ test "finds a user by profile URL" do
+ user = insert(:user, %{nickname: "lain@lain.com"})
+
+ assert [%{id: id}] = User.search("https://lain.com/users/lain")
+ assert id == user.id
+ assert [%{id: ^id}] = User.search("https://lain.com/@lain")
+ end
+
+ test "finds a local user by profile URL" do
+ user = insert(:user, %{nickname: "alice"})
+ url = "#{Pleroma.Web.Endpoint.url()}/users/alice/"
+
+ assert [%{id: id}] = User.search(url)
+ assert id == user.id
+ end
+
+ test "tolerates unusable URLs" do
+ assert [] = User.search("http:///users/x")
+ assert [] = User.search("https://" <> String.duplicate("a", 64) <> ".com/@x")
+ assert [] = User.search("https://example.com")
+ end
+
+ test "does not text-search the fragments of a URL" do
+ insert(:user, %{nickname: "com_user"})
+ insert(:user, %{name: "lain"})
+
+ assert [] = User.search("https://lain.com/users/lain")
+ end
+
test "doesn't die if two users have the same uri" do
insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"})
insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"})
assert [_first_user, _second_user] = User.search("https://gensokyo.2hu/@raymoo")
end
test "returns a user with matching uri as the first result" do
user =
insert(:user, %{
nickname: "no_relation",
ap_id: "https://lain.com/users/lain",
uri: "https://lain.com/@lain"
})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/@lain")
+ [first_user] = User.search("https://lain.com/@lain")
assert first_user.id == user.id
end
test "excludes invisible users from results" do
user = insert(:user, %{nickname: "john t1000"})
insert(:user, %{invisible: true, nickname: "john t800"})
[found_user] = User.search("john")
assert found_user.id == user.id
end
test "excludes deactivated users from results" do
user = insert(:user, %{nickname: "john t1000"})
insert(:user, %{is_active: false, nickname: "john t800"})
[found_user] = User.search("john")
assert found_user.id == user.id
end
# Note: as in Mastodon, `is_discoverable` doesn't anyhow relate to user searchability
test "includes non-discoverable users in results" do
insert(:user, %{nickname: "john 3000", is_discoverable: false})
insert(:user, %{nickname: "john 3001"})
users = User.search("john")
assert Enum.count(users) == 2
end
test "excludes service actors from results" do
insert(:user, actor_type: "Application", nickname: "user1")
service = insert(:user, actor_type: "Service", nickname: "user2")
person = insert(:user, actor_type: "Person", nickname: "user3")
assert [found_user1, found_user2] = User.search("user")
assert [found_user1.id, found_user2.id] -- [service.id, person.id] == []
end
test "accepts limit parameter" do
Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
assert length(User.search("john", limit: 3)) == 3
assert length(User.search("john")) == 5
end
test "accepts offset parameter" do
Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
assert length(User.search("john", limit: 3)) == 3
assert length(User.search("john", limit: 3, offset: 3)) == 2
end
defp clear_virtual_fields(user) do
Map.merge(user, %{search_rank: nil, search_type: nil})
end
test "finds a user by full nickname or its leading fragment" do
user = insert(:user, %{nickname: "john"})
Enum.each(["john", "jo", "j"], fn query ->
assert user ==
User.search(query)
|> List.first()
|> clear_virtual_fields()
end)
end
test "finds a user by full name or leading fragment(s) of its words" do
user = insert(:user, %{name: "John Doe"})
Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query ->
assert user ==
User.search(query)
|> List.first()
|> clear_virtual_fields()
end)
end
test "matches by leading fragment of user domain" do
user = insert(:user, %{nickname: "arandom@dude.com"})
insert(:user, %{nickname: "iamthedude"})
assert [user.id] == User.search("dud") |> Enum.map(& &1.id)
end
test "ranks full nickname match higher than full name match" do
nicknamed_user = insert(:user, %{nickname: "hj@shigusegubu.club"})
named_user = insert(:user, %{nickname: "xyz@sample.com", name: "HJ"})
results = User.search("hj")
assert [nicknamed_user.id, named_user.id] == Enum.map(results, & &1.id)
assert Enum.at(results, 0).search_rank > Enum.at(results, 1).search_rank
end
test "finds users, considering density of matched tokens" do
u1 = insert(:user, %{name: "Bar Bar plus Word Word"})
u2 = insert(:user, %{name: "Word Word Bar Bar Bar"})
assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id)
end
test "finds users, boosting ranks of friends and followers" do
u1 = insert(:user)
u2 = insert(:user, %{name: "Doe"})
follower = insert(:user, %{name: "Doe"})
friend = insert(:user, %{name: "Doe"})
{:ok, follower, u1} = User.follow(follower, u1)
{:ok, u1, friend} = User.follow(u1, friend)
assert [friend.id, follower.id, u2.id] --
Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == []
end
test "finds followings of user by partial name" do
lizz = insert(:user, %{name: "Lizz"})
jimi = insert(:user, %{name: "Jimi"})
following_lizz = insert(:user, %{name: "Jimi Hendrix"})
following_jimi = insert(:user, %{name: "Lizz Wright"})
follower_lizz = insert(:user, %{name: "Jimi"})
{:ok, lizz, following_lizz} = User.follow(lizz, following_lizz)
{:ok, _jimi, _following_jimi} = User.follow(jimi, following_jimi)
{:ok, _follower_lizz, _lizz} = User.follow(follower_lizz, lizz)
assert Enum.map(User.search("jimi", following: true, for_user: lizz), & &1.id) == [
following_lizz.id
]
assert User.search("lizz", following: true, for_user: lizz) == []
end
test "find local and remote users for authenticated users" do
u1 = insert(:user, %{name: "lain"})
u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
results =
"lain"
|> User.search(for_user: u1)
|> Enum.map(& &1.id)
|> Enum.sort()
assert [u1.id, u2.id, u3.id] == results
end
test "find only local users for unauthenticated users" do
%{id: id} = insert(:user, %{name: "lain"})
insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
assert [%{id: ^id}] = User.search("lain")
end
test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do
clear_config([:instance, :limit_to_local_content], :all)
%{id: id} = insert(:user, %{name: "lain"})
insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
assert [%{id: ^id}] = User.search("lain")
end
test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do
clear_config([:instance, :limit_to_local_content], false)
u1 = insert(:user, %{name: "lain"})
u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
results =
"lain"
|> User.search()
|> Enum.map(& &1.id)
|> Enum.sort()
assert [u1.id, u2.id, u3.id] == results
end
test "does not yield false-positive matches" do
insert(:user, %{name: "John Doe"})
Enum.each(["mary", "a", ""], fn query ->
assert [] == User.search(query)
end)
end
test "works with URIs" do
user = insert(:user)
results =
User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user)
result = results |> List.first()
user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin")
assert length(results) == 1
expected =
result
|> Map.put(:search_rank, nil)
|> Map.put(:search_type, nil)
|> Map.put(:last_digest_emailed_at, nil)
|> Map.put(:multi_factor_authentication_settings, nil)
|> Map.put(:notification_settings, nil)
assert user == expected
end
test "excludes a blocked users from search result" do
user = insert(:user, %{nickname: "Bill"})
[blocked_user | users] = Enum.map(0..3, &insert(:user, %{nickname: "john#{&1}"}))
blocked_user2 =
insert(
:user,
%{nickname: "john awful", ap_id: "https://awful-and-rude-instance.com/user/bully"}
)
User.block_domain(user, "awful-and-rude-instance.com")
User.block(user, blocked_user)
account_ids = User.search("john", for_user: refresh_record(user)) |> collect_ids
assert account_ids == collect_ids(users)
refute Enum.member?(account_ids, blocked_user.id)
refute Enum.member?(account_ids, blocked_user2.id)
assert length(account_ids) == 3
end
test "local user has the same search_rank as for users with the same nickname, but another domain" do
user = insert(:user)
insert(:user, nickname: "lain@mastodon.social")
insert(:user, nickname: "lain")
insert(:user, nickname: "lain@pleroma.social")
assert User.search("lain@localhost", resolve: true, for_user: user)
|> Enum.each(fn u -> u.search_rank == 0.5 end)
end
test "localhost is the part of the domain" do
user = insert(:user)
insert(:user, nickname: "another@somedomain")
insert(:user, nickname: "lain")
insert(:user, nickname: "lain@examplelocalhost")
result = User.search("lain@examplelocalhost", resolve: true, for_user: user)
assert Enum.each(result, fn u -> u.search_rank == 0.5 end)
assert length(result) == 2
end
test "local user search with users" do
user = insert(:user)
local_user = insert(:user, nickname: "lain")
insert(:user, nickname: "another@localhost.com")
insert(:user, nickname: "localhost@localhost.com")
[result] = User.search("lain@localhost", resolve: true, for_user: user)
assert Map.put(result, :search_rank, nil) |> Map.put(:search_type, nil) == local_user
end
test "works with idna domains" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results = User.search("lain@zetsubou.みんな", resolve: false, for_user: user)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains converted input" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("lain@zetsubou." <> to_string(:idna.encode("zetsubou.みんな")),
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains and bad chars in domain" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("lain@zetsubou!@#$%^&*()+,-/:;<=>?[]'_{}|~`.みんな",
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains and query as link" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("https://zetsubou.みんな/users/lain",
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "find users accepting chat messages only" do
user1 = insert(:user, nickname: "user1", accepts_chat_messages: true)
insert(:user, nickname: "user2", accepts_chat_messages: false)
[found_user1] = User.search("user", capabilities: ["accepts_chat_messages"])
assert found_user1.id == user1.id
end
end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sun, Nov 1, 12:06 PM (9 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1803008
Default Alt Text
(25 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline