Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
6 KB
Referenced Files
None
Subscribers
None
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
index 4bd35c17c..2d9315a2f 100644
--- a/lib/pleroma/search/qdrant_search.ex
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -1,128 +1,131 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias __MODULE__.QdrantClient
alias __MODULE__.OllamaClient
+ alias Pleroma.Config.Getting, as: Config
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
@impl true
def create_index() do
- payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
+ payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index() do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"embedding" => embedding}}} <-
OllamaClient.post("/api/embeddings", %{
prompt: text,
- model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
+ model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp build_index_payload(activity, embedding) do
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding
}
]
}
end
defp build_search_payload(embedding) do
%{
vector: embedding,
limit: 20
}
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
:ok
end
end
@impl true
def search(_user, query, _options) do
query = "Represent this sentence for searching relevant passages: #{query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
else
_ ->
[]
end
end
@impl true
def remove_from_index(_object) do
:ok
end
end
defmodule Pleroma.Search.QdrantSearch.OllamaClient do
use Tesla
+ alias Pleroma.Config.Getting, as: Config
- plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
+ plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url]))
plug(Tesla.Middleware.JSON)
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
+ alias Pleroma.Config.Getting, as: Config
- plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
+ plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
end
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
new file mode 100644
index 000000000..9be246a9a
--- /dev/null
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.QdrantSearchTest do
+ use Pleroma.DataCase, async: true
+ use Oban.Testing, repo: Pleroma.Repo
+
+ import Pleroma.Factory
+ import Mox
+
+ alias Pleroma.Web.CommonAPI
+ alias Pleroma.UnstubbedConfigMock, as: Config
+ alias Pleroma.Search.QdrantSearch
+ alias Pleroma.Workers.SearchIndexingWorker
+
+ describe "Qdrant search" do
+ test "indexes a public post on creation" do
+ user = insert(:user)
+
+ Tesla.Mock.mock(fn
+ %{method: :post, url: "https://ollama.url/api/embeddings"} ->
+ send(self(), "posted_to_ollama")
+ Tesla.Mock.json(%{embedding: [1, 2, 3]})
+
+ %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
+ send(self(), "posted_to_qdrant")
+
+ assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body))
+
+ Tesla.Mock.json("ok")
+ end)
+
+ Config
+ |> expect(:get, 4, fn
+ [Pleroma.Search, :module], nil ->
+ QdrantSearch
+
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{
+ ollama_model: "a_model",
+ ollama_url: "https://ollama.url",
+ qdrant_url: "https://qdrant.url"
+ }[key]
+ end)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ args = %{"op" => "add_to_index", "activity" => activity.id}
+
+ assert_enqueued(
+ worker: SearchIndexingWorker,
+ args: args
+ )
+
+ assert :ok = perform_job(SearchIndexingWorker, args)
+ assert_received("posted_to_ollama")
+ assert_received("posted_to_qdrant")
+ end
+ end
+end

File Metadata

Mime Type
text/x-diff
Expires
Wed, Nov 18, 4:17 PM (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1840584
Default Alt Text
(6 KB)

Event Timeline