Page MenuHomePhorge
Files F85828045

No OneTemporary
Actions

Authored By
Unknown
Size
7 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/changelog.d/gin-fuzzy-search-limit.fix b/changelog.d/gin-fuzzy-search-limit.fix
new file mode 100644
index 000000000..276c3daa9
--- /dev/null
+++ b/changelog.d/gin-fuzzy-search-limit.fix
@@ -0,0 +1 @@
+gin_fuzzy_search_limit is now only set for search queries (configurable under Pleroma.Search.DatabaseSearch), so other GIN index scans such as recipient lookups and Oban job queries are exact again
diff --git a/config/config.exs b/config/config.exs
index 1b8baed1f..32f5b6c20 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -843,7 +843,7 @@ config :pleroma, :modules, runtime_dir: "instance/modules"
config :pleroma, configurable_from_database: false
config :pleroma, Pleroma.Repo,
- parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
+ parameters: [jit: "off"],
prepare: :unnamed
config :pleroma, :connections_pool,
@@ -948,6 +948,8 @@ config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fet
config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
+config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 500
+
config :pleroma, Pleroma.Search.Meilisearch,
url: "http://127.0.0.1:7700/",
private_key: nil,
diff --git a/config/description.exs b/config/description.exs
index f435628aa..67ae4a585 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3565,6 +3565,21 @@ config :pleroma, :config_description, [
}
]
},
+ %{
+ group: :pleroma,
+ key: Pleroma.Search.DatabaseSearch,
+ type: :group,
+ description: "Built-in database search settings.",
+ children: [
+ %{
+ key: :gin_fuzzy_search_limit,
+ type: :integer,
+ description:
+ "Limits how many matches PostgreSQL considers in GIN full-text search queries (see `gin_fuzzy_search_limit` in the PostgreSQL docs). Only set for search queries. Keeps search fast on large databases, but results become incomplete and change between requests. Set to `nil` for exact results, which is fine on small instances. Default: 500.",
+ suggestions: [500, nil]
+ }
+ ]
+ },
%{
group: :pleroma,
key: Pleroma.Search.Meilisearch,
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 5820cdbd9..c37629184 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -10,6 +10,14 @@ To use built-in search that has no external dependencies, set the search module
While it has no external dependencies, it has problems with performance and relevancy.
+With a GIN index (the default), search queries limit how many matches PostgreSQL considers (`gin_fuzzy_search_limit`, see [PostgreSQL's docs](https://www.postgresql.org/docs/current/gin.html#GIN-TIPS)). This keeps searches for common words fast on large databases, but the results are incomplete and can change between requests. On smaller instances you can turn it off for exact results:
+
+> config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil
+
+Without a limit, searches for common words can take longer than the database timeout on large databases, and then return nothing.
+
+Earlier versions of Pleroma set this limit for every database connection, which also made other GIN index scans (for example recipient lookups) incomplete. It is now only set for search queries.
+
## QdrantSearch
This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index e88d632cb..c66723d1d 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do
alias Pleroma.Config
alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
+ alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
@@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
- try do
- Activity
- |> Activity.with_preloaded_object()
- |> Activity.restrict_deactivated_users()
- |> restrict_public(user)
- |> query_with(index_type, search_query)
- |> maybe_restrict_local(user)
- |> maybe_restrict_author(author)
- |> maybe_restrict_blocked(user)
- |> Pagination.fetch_paginated(
- %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
- :offset
- )
- |> maybe_fetch(user, search_query)
- rescue
- _ -> maybe_fetch([], user, search_query)
+ activities =
+ try do
+ with_fuzzy_limit(fn ->
+ Activity
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> restrict_public(user)
+ |> query_with(index_type, search_query)
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> Pagination.fetch_paginated(
+ %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+ :offset
+ )
+ end)
+ rescue
+ _ -> []
+ end
+
+ # Outside the query transaction, as it may fetch over the network.
+ maybe_fetch(activities, user, search_query)
+ end
+
+ @doc """
+ Runs `fun` with `gin_fuzzy_search_limit` set, if configured under
+ `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their
+ results incomplete and random, so it is only set for search queries.
+ """
+ def with_fuzzy_limit(fun) do
+ case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do
+ limit when is_integer(limit) and limit > 0 ->
+ {:ok, result} =
+ Repo.transaction(fn ->
+ # SET LOCAL takes no parameters; limit is an integer.
+ Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}")
+ fun.()
+ end)
+
+ result
+
+ _ ->
+ fun.()
end
end
diff --git a/test/pleroma/search/database_search_test.exs b/test/pleroma/search/database_search_test.exs
index d8dd09063..cce810e99 100644
--- a/test/pleroma/search/database_search_test.exs
+++ b/test/pleroma/search/database_search_test.exs
@@ -44,4 +44,31 @@ defmodule Pleroma.Search.DatabaseSearchTest do
assert result.id == other_post.id
end
+
+ describe "gin_fuzzy_search_limit" do
+ defp fuzzy_limit do
+ %{rows: [[value]]} = Pleroma.Repo.query!("SHOW gin_fuzzy_search_limit")
+ value
+ end
+
+ test "is not set on normal database connections" do
+ assert fuzzy_limit() == "0"
+ end
+
+ test "is only set for the search query when configured" do
+ clear_config([Search, :gin_fuzzy_search_limit], nil)
+ assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "0"
+
+ clear_config([Search, :gin_fuzzy_search_limit], 10_000)
+ assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "10000"
+ end
+
+ test "search still works with a limit" do
+ clear_config([Search, :gin_fuzzy_search_limit], 10_000)
+ {:ok, post} = CommonAPI.post(insert(:user), %{status: "it's wednesday my dudes"})
+
+ assert [%{id: id}] = Search.search(nil, "wednesday")
+ assert id == post.id
+ end
+ end
end

File Metadata

Mime Type
text/x-diff
Expires
Sat, Oct 31, 2:00 AM (19 h, 10 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1800883
Default Alt Text
(7 KB)

Event Timeline