Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
25 KB
Referenced Files
None
Subscribers
None
diff --git a/changelog.d/search-url-no-fts.fix b/changelog.d/search-url-no-fts.fix
new file mode 100644
index 000000000..fd51c5d0b
--- /dev/null
+++ b/changelog.d/search-url-no-fts.fix
@@ -0,0 +1 @@
+User search: a URL query only returns exact matches (resolved account, ap_id, uri) instead of also text-searching the URL's fragments against every account.
diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index abc45f62a..1455288e9 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -1,322 +1,351 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.Search do
alias Pleroma.EctoType.ActivityPub.ObjectValidators.Uri, as: UriType
alias Pleroma.Instances.Instance
alias Pleroma.Pagination
alias Pleroma.User
import Ecto.Query
@limit 20
def search(query_string, opts \\ []) do
resolve = Keyword.get(opts, :resolve, false)
following = Keyword.get(opts, :following, false)
result_limit = Keyword.get(opts, :limit, @limit)
offset = Keyword.get(opts, :offset, 0)
capabilities = Keyword.get(opts, :capabilities, [])
for_user = Keyword.get(opts, :for_user)
query_string = format_query(query_string)
# If this returns anything, it should bounce to the top
maybe_resolved = maybe_resolve(resolve, for_user, query_string)
top_user_ids =
[]
|> maybe_add_resolved(maybe_resolved)
|> maybe_add_ap_id_match(query_string)
|> maybe_add_uri_match(query_string)
+ |> maybe_add_profile_url_match(query_string)
results =
query_string
|> search_query(for_user, following, top_user_ids, capabilities)
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
results
end
defp maybe_add_resolved(list, {:ok, %User{} = user}) do
[user.id | list]
end
defp maybe_add_resolved(list, _), do: list
defp maybe_add_ap_id_match(list, query) do
if user = User.get_cached_by_ap_id(query) do
[user.id | list]
else
list
end
end
defp maybe_add_uri_match(list, query) do
with {:ok, query} <- UriType.cast(query),
q = from(u in User, where: u.uri == ^query, select: u.id),
users = Pleroma.Repo.all(q) do
users ++ list
else
_ -> list
end
end
+ # "https://host/users/name" or "https://host/@name" -> the user known as name@host
+ defp maybe_add_profile_url_match(list, "http" <> _ = query) do
+ with %URI{host: host, path: path}
+ when is_binary(host) and host != "" and is_binary(path) <- URI.parse(query),
+ [_ | _] = segments <- String.split(path, "/", trim: true),
+ segment when segment != "" <- segments |> List.last() |> String.trim_leading("@"),
+ %User{id: id} <- get_by_nickname(segment <> "@" <> host) do
+ [id | list]
+ else
+ _ -> list
+ end
+ end
+
+ defp maybe_add_profile_url_match(list, _), do: list
+
+ # :idna.encode/1 exits rather than raises on hosts it cannot encode
+ defp get_by_nickname(nickname) do
+ User.get_by_nickname(format_query(nickname))
+ catch
+ _, _ -> nil
+ end
+
defp format_query(query_string) do
# Strip the beginning @ off if there is a query
query_string = String.trim_leading(query_string, "@")
with [name, domain] <- String.split(query_string, "@") do
encoded_domain =
domain
|> String.replace(~r/[!-\-|@|[-`|{-~|\/|:|\s]+/, "")
|> String.to_charlist()
|> :idna.encode()
|> to_string()
name <> "@" <> encoded_domain
else
_ -> query_string
end
end
defp search_query(query_string, for_user, following, top_user_ids, capabilities) do
for_user
|> base_query(following)
|> filter_blocked_user(for_user)
|> filter_invisible_users()
|> filter_internal_users()
|> filter_blocked_domains(for_user)
|> filter_unreachable_users()
|> fts_search(query_string)
|> select_top_users(top_user_ids)
|> trigram_rank(query_string)
|> boost_search_rank(for_user, top_user_ids)
|> subquery()
|> order_by_search_rank(for_user)
|> maybe_restrict_local(for_user)
|> maybe_restrict_accepting_chat_messages(capabilities)
|> filter_deactivated_users()
end
defp select_top_users(query, top_user_ids) do
from(u in query,
or_where: u.id in ^top_user_ids
)
end
+ # Clients search for URLs to resolve them. Those are matched exactly
+ # (ap_id, uri, resolve), and running the URL's fragments ("https", "de",
+ # "notice") through the text index would rank every account matching one
+ # of them, which on a large instance is tens of thousands of rows.
+ defp fts_search(query, "http" <> _), do: where(query, false)
+
defp fts_search(query, query_string) do
query_string = to_tsquery(query_string)
from(
u in query,
where:
fragment(
# The fragment must _exactly_ match `users_fts_index`, otherwise the index won't work
"""
(
setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')
) @@ to_tsquery('simple', ?)
""",
u.nickname,
u.name,
^query_string
)
)
end
defp to_tsquery(query_string) do
String.trim_trailing(query_string, "@" <> local_domain())
|> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
|> String.trim()
|> String.split()
|> Enum.map(&(&1 <> ":*"))
|> Enum.join(" | ")
end
# Considers nickname match, localized nickname match, name match; preferences nickname match
defp trigram_rank(query, query_string) do
from(
u in query,
select_merge: %{
search_rank:
fragment(
"""
similarity(?, ?) +
similarity(?, regexp_replace(?, '@.+', '')) +
similarity(?, trim(coalesce(?, '')))
""",
^query_string,
u.nickname,
^query_string,
u.nickname,
^query_string,
u.name
)
}
)
end
defp base_query(%User{} = user, true), do: User.get_friends_query(user)
defp base_query(_user, _following), do: User
defp filter_invisible_users(query) do
from(q in query, where: q.invisible == false)
end
defp filter_internal_users(query) do
from(q in query, where: q.actor_type != "Application")
end
defp filter_deactivated_users(query) do
from(q in query, where: q.is_active == true)
end
defp filter_blocked_user(query, %User{} = blocker) do
query
|> join(:left, [u], b in Pleroma.UserRelationship,
as: :blocks,
on: b.relationship_type == ^:block and b.source_id == ^blocker.id and u.id == b.target_id
)
|> where([blocks: b], is_nil(b.target_id))
end
defp filter_blocked_user(query, _), do: query
defp filter_blocked_domains(query, %User{domain_blocks: domain_blocks})
when length(domain_blocks) > 0 do
domains = Enum.join(domain_blocks, ",")
from(
q in query,
where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains)
)
end
defp filter_blocked_domains(query, _), do: query
defp filter_unreachable_users(query) do
from(u in query,
left_join: i in Instance,
on: i.host == fragment("substring(? from '.*://([^/]*)')", u.ap_id),
where: is_nil(i.unreachable_since)
)
end
defp maybe_resolve(true, user, query) do
case {limit(), user} do
{:all, _} -> :noop
{:unauthenticated, %User{}} -> User.get_or_fetch(query)
{:unauthenticated, _} -> :noop
{false, _} -> User.get_or_fetch(query)
end
end
defp maybe_resolve(_, _, _), do: :noop
defp maybe_restrict_local(q, user) do
case {limit(), user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp maybe_restrict_accepting_chat_messages(query, capabilities) do
if "accepts_chat_messages" in capabilities do
from(q in query, where: q.accepts_chat_messages == true)
else
query
end
end
defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
defp restrict_local(q), do: where(q, [u], u.local == true)
defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
defp boost_search_rank(query, %User{} = for_user, top_user_ids) do
friends_ids = User.get_friends_ids(for_user)
followers_ids = User.get_followers_ids(for_user)
from(u in subquery(query),
select_merge: %{
search_type:
fragment(
"""
CASE WHEN (?) THEN 2
WHEN (?) THEN 1
ELSE 0 END
""",
u.id in ^top_user_ids,
u.id in ^friends_ids or u.id in ^followers_ids
),
search_rank:
fragment(
"""
CASE WHEN (?) THEN (?) * 1.5
WHEN (?) THEN (?) * 1.3
WHEN (?) THEN (?) * 1.1
WHEN (?) THEN 9001
ELSE (?) END
""",
u.id in ^friends_ids and u.id in ^followers_ids,
u.search_rank,
u.id in ^friends_ids,
u.search_rank,
u.id in ^followers_ids,
u.search_rank,
u.id in ^top_user_ids,
u.search_rank
)
}
)
end
defp boost_search_rank(query, _for_user, top_user_ids) do
from(u in subquery(query),
select_merge: %{
search_type:
fragment(
"""
CASE WHEN (?) THEN 2
ELSE 0 END
""",
u.id in ^top_user_ids
),
search_rank:
fragment(
"""
CASE WHEN (?) THEN 9001
ELSE (?) END
""",
u.id in ^top_user_ids,
u.search_rank
)
}
)
end
defp order_by_search_rank(query, %User{}) do
order_by(
query,
[u],
desc: u.search_type,
desc_nulls_last:
fragment(
"CASE WHEN ? = 1 THEN COALESCE(?, ?) ELSE NULL END",
u.search_type,
u.last_status_at,
u.last_active_at
),
desc: u.search_rank
)
end
defp order_by_search_rank(query, _), do: order_by(query, desc: :search_rank)
end
diff --git a/test/pleroma/user_search_test.exs b/test/pleroma/user_search_test.exs
index 75cfbd2db..6b83c7ca6 100644
--- a/test/pleroma/user_search_test.exs
+++ b/test/pleroma/user_search_test.exs
@@ -1,378 +1,407 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.UserSearchTest do
alias Pleroma.User
use Pleroma.DataCase
import Pleroma.Factory
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
:ok
end
describe "User.search" do
setup do: clear_config([:instance, :limit_to_local_content])
test "returns a resolved user as the first result" do
clear_config([:instance, :limit_to_local_content], false)
user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/users/lain", resolve: true)
+ [first_user] = User.search("https://lain.com/users/lain", resolve: true)
assert first_user.id == user.id
end
test "returns a user with matching ap_id as the first result" do
user = insert(:user, %{nickname: "no_relation", ap_id: "https://lain.com/users/lain"})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/users/lain")
+ [first_user] = User.search("https://lain.com/users/lain")
assert first_user.id == user.id
end
+ test "finds a user by profile URL" do
+ user = insert(:user, %{nickname: "lain@lain.com"})
+
+ assert [%{id: id}] = User.search("https://lain.com/users/lain")
+ assert id == user.id
+ assert [%{id: ^id}] = User.search("https://lain.com/@lain")
+ end
+
+ test "finds a local user by profile URL" do
+ user = insert(:user, %{nickname: "alice"})
+ url = "#{Pleroma.Web.Endpoint.url()}/users/alice/"
+
+ assert [%{id: id}] = User.search(url)
+ assert id == user.id
+ end
+
+ test "tolerates unusable URLs" do
+ assert [] = User.search("http:///users/x")
+ assert [] = User.search("https://" <> String.duplicate("a", 64) <> ".com/@x")
+ assert [] = User.search("https://example.com")
+ end
+
+ test "does not text-search the fragments of a URL" do
+ insert(:user, %{nickname: "com_user"})
+ insert(:user, %{name: "lain"})
+
+ assert [] = User.search("https://lain.com/users/lain")
+ end
+
test "doesn't die if two users have the same uri" do
insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"})
insert(:user, %{uri: "https://gensokyo.2hu/@raymoo"})
assert [_first_user, _second_user] = User.search("https://gensokyo.2hu/@raymoo")
end
test "returns a user with matching uri as the first result" do
user =
insert(:user, %{
nickname: "no_relation",
ap_id: "https://lain.com/users/lain",
uri: "https://lain.com/@lain"
})
_user = insert(:user, %{nickname: "com_user"})
- [first_user, _second_user] = User.search("https://lain.com/@lain")
+ [first_user] = User.search("https://lain.com/@lain")
assert first_user.id == user.id
end
test "excludes invisible users from results" do
user = insert(:user, %{nickname: "john t1000"})
insert(:user, %{invisible: true, nickname: "john t800"})
[found_user] = User.search("john")
assert found_user.id == user.id
end
test "excludes deactivated users from results" do
user = insert(:user, %{nickname: "john t1000"})
insert(:user, %{is_active: false, nickname: "john t800"})
[found_user] = User.search("john")
assert found_user.id == user.id
end
# Note: as in Mastodon, `is_discoverable` doesn't anyhow relate to user searchability
test "includes non-discoverable users in results" do
insert(:user, %{nickname: "john 3000", is_discoverable: false})
insert(:user, %{nickname: "john 3001"})
users = User.search("john")
assert Enum.count(users) == 2
end
test "excludes service actors from results" do
insert(:user, actor_type: "Application", nickname: "user1")
service = insert(:user, actor_type: "Service", nickname: "user2")
person = insert(:user, actor_type: "Person", nickname: "user3")
assert [found_user1, found_user2] = User.search("user")
assert [found_user1.id, found_user2.id] -- [service.id, person.id] == []
end
test "accepts limit parameter" do
Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
assert length(User.search("john", limit: 3)) == 3
assert length(User.search("john")) == 5
end
test "accepts offset parameter" do
Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
assert length(User.search("john", limit: 3)) == 3
assert length(User.search("john", limit: 3, offset: 3)) == 2
end
defp clear_virtual_fields(user) do
Map.merge(user, %{search_rank: nil, search_type: nil})
end
test "finds a user by full nickname or its leading fragment" do
user = insert(:user, %{nickname: "john"})
Enum.each(["john", "jo", "j"], fn query ->
assert user ==
User.search(query)
|> List.first()
|> clear_virtual_fields()
end)
end
test "finds a user by full name or leading fragment(s) of its words" do
user = insert(:user, %{name: "John Doe"})
Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query ->
assert user ==
User.search(query)
|> List.first()
|> clear_virtual_fields()
end)
end
test "matches by leading fragment of user domain" do
user = insert(:user, %{nickname: "arandom@dude.com"})
insert(:user, %{nickname: "iamthedude"})
assert [user.id] == User.search("dud") |> Enum.map(& &1.id)
end
test "ranks full nickname match higher than full name match" do
nicknamed_user = insert(:user, %{nickname: "hj@shigusegubu.club"})
named_user = insert(:user, %{nickname: "xyz@sample.com", name: "HJ"})
results = User.search("hj")
assert [nicknamed_user.id, named_user.id] == Enum.map(results, & &1.id)
assert Enum.at(results, 0).search_rank > Enum.at(results, 1).search_rank
end
test "finds users, considering density of matched tokens" do
u1 = insert(:user, %{name: "Bar Bar plus Word Word"})
u2 = insert(:user, %{name: "Word Word Bar Bar Bar"})
assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id)
end
test "finds users, boosting ranks of friends and followers" do
u1 = insert(:user)
u2 = insert(:user, %{name: "Doe"})
follower = insert(:user, %{name: "Doe"})
friend = insert(:user, %{name: "Doe"})
{:ok, follower, u1} = User.follow(follower, u1)
{:ok, u1, friend} = User.follow(u1, friend)
assert [friend.id, follower.id, u2.id] --
Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == []
end
test "finds followings of user by partial name" do
lizz = insert(:user, %{name: "Lizz"})
jimi = insert(:user, %{name: "Jimi"})
following_lizz = insert(:user, %{name: "Jimi Hendrix"})
following_jimi = insert(:user, %{name: "Lizz Wright"})
follower_lizz = insert(:user, %{name: "Jimi"})
{:ok, lizz, following_lizz} = User.follow(lizz, following_lizz)
{:ok, _jimi, _following_jimi} = User.follow(jimi, following_jimi)
{:ok, _follower_lizz, _lizz} = User.follow(follower_lizz, lizz)
assert Enum.map(User.search("jimi", following: true, for_user: lizz), & &1.id) == [
following_lizz.id
]
assert User.search("lizz", following: true, for_user: lizz) == []
end
test "find local and remote users for authenticated users" do
u1 = insert(:user, %{name: "lain"})
u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
results =
"lain"
|> User.search(for_user: u1)
|> Enum.map(& &1.id)
|> Enum.sort()
assert [u1.id, u2.id, u3.id] == results
end
test "find only local users for unauthenticated users" do
%{id: id} = insert(:user, %{name: "lain"})
insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
assert [%{id: ^id}] = User.search("lain")
end
test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do
clear_config([:instance, :limit_to_local_content], :all)
%{id: id} = insert(:user, %{name: "lain"})
insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
assert [%{id: ^id}] = User.search("lain")
end
test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do
clear_config([:instance, :limit_to_local_content], false)
u1 = insert(:user, %{name: "lain"})
u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
results =
"lain"
|> User.search()
|> Enum.map(& &1.id)
|> Enum.sort()
assert [u1.id, u2.id, u3.id] == results
end
test "does not yield false-positive matches" do
insert(:user, %{name: "John Doe"})
Enum.each(["mary", "a", ""], fn query ->
assert [] == User.search(query)
end)
end
test "works with URIs" do
user = insert(:user)
results =
User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user)
result = results |> List.first()
user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin")
assert length(results) == 1
expected =
result
|> Map.put(:search_rank, nil)
|> Map.put(:search_type, nil)
|> Map.put(:last_digest_emailed_at, nil)
|> Map.put(:multi_factor_authentication_settings, nil)
|> Map.put(:notification_settings, nil)
assert user == expected
end
test "excludes a blocked users from search result" do
user = insert(:user, %{nickname: "Bill"})
[blocked_user | users] = Enum.map(0..3, &insert(:user, %{nickname: "john#{&1}"}))
blocked_user2 =
insert(
:user,
%{nickname: "john awful", ap_id: "https://awful-and-rude-instance.com/user/bully"}
)
User.block_domain(user, "awful-and-rude-instance.com")
User.block(user, blocked_user)
account_ids = User.search("john", for_user: refresh_record(user)) |> collect_ids
assert account_ids == collect_ids(users)
refute Enum.member?(account_ids, blocked_user.id)
refute Enum.member?(account_ids, blocked_user2.id)
assert length(account_ids) == 3
end
test "local user has the same search_rank as for users with the same nickname, but another domain" do
user = insert(:user)
insert(:user, nickname: "lain@mastodon.social")
insert(:user, nickname: "lain")
insert(:user, nickname: "lain@pleroma.social")
assert User.search("lain@localhost", resolve: true, for_user: user)
|> Enum.each(fn u -> u.search_rank == 0.5 end)
end
test "localhost is the part of the domain" do
user = insert(:user)
insert(:user, nickname: "another@somedomain")
insert(:user, nickname: "lain")
insert(:user, nickname: "lain@examplelocalhost")
result = User.search("lain@examplelocalhost", resolve: true, for_user: user)
assert Enum.each(result, fn u -> u.search_rank == 0.5 end)
assert length(result) == 2
end
test "local user search with users" do
user = insert(:user)
local_user = insert(:user, nickname: "lain")
insert(:user, nickname: "another@localhost.com")
insert(:user, nickname: "localhost@localhost.com")
[result] = User.search("lain@localhost", resolve: true, for_user: user)
assert Map.put(result, :search_rank, nil) |> Map.put(:search_type, nil) == local_user
end
test "works with idna domains" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results = User.search("lain@zetsubou.みんな", resolve: false, for_user: user)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains converted input" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("lain@zetsubou." <> to_string(:idna.encode("zetsubou.みんな")),
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains and bad chars in domain" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("lain@zetsubou!@#$%^&*()+,-/:;<=>?[]'_{}|~`.みんな",
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "works with idna domains and query as link" do
user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
results =
User.search("https://zetsubou.みんな/users/lain",
resolve: false,
for_user: user
)
result = List.first(results)
assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
end
test "find users accepting chat messages only" do
user1 = insert(:user, nickname: "user1", accepts_chat_messages: true)
insert(:user, nickname: "user2", accepts_chat_messages: false)
[found_user1] = User.search("user", capabilities: ["accepts_chat_messages"])
assert found_user1.id == user1.id
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 1, 12:06 PM (9 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1803008
Default Alt Text
(25 KB)

Event Timeline