Page MenuHomePhorge
Files F85807231

No OneTemporary
Actions

Size
62 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add
new file mode 100644
index 000000000..9801131d1
--- /dev/null
+++ b/changelog.d/qdrant_search.add
@@ -0,0 +1 @@
+Add Qdrant/OpenAI embedding search
diff --git a/config/config.exs b/config/config.exs
index 9ba494b26..cd86816fe 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -1,938 +1,951 @@
# .i;;;;i.
# iYcviii;vXY:
# .YXi .i1c.
# .YC. . in7.
# .vc. ...... ;1c.
# i7, .. .;1;
# i7, .. ... .Y1i
# ,7v .6MMM@; .YX,
# .7;. ..IMMMMMM1 :t7.
# .;Y. ;$MMMMMM9. :tc.
# vY. .. .nMMM@MMU. ;1v.
# i7i ... .#MM@M@C. .....:71i
# it: .... $MMM@9;.,i;;;i,;tti
# :t7. ..... 0MMMWv.,iii:::,,;St.
# .nC. ..... IMMMQ..,::::::,.,czX.
# .ct: ....... .ZMMMI..,:::::::,,:76Y.
# c2: ......,i..Y$M@t..:::::::,,..inZY
# vov ......:ii..c$MBc..,,,,,,,,,,..iI9i
# i9Y ......iii:..7@MA,..,,,,,,,,,....;AA:
# iIS. ......:ii::..;@MI....,............;Ez.
# .I9. ......:i::::...8M1..................C0z.
# .z9; ......:i::::,.. .i:...................zWX.
# vbv ......,i::::,,. ................. :AQY
# c6Y. .,...,::::,,..:t0@@QY. ................ :8bi
# :6S. ..,,...,:::,,,..EMMMMMMI. ............... .;bZ,
# :6o, .,,,,..:::,,,..i#MMMMMM#v................. YW2.
# .n8i ..,,,,,,,::,,,,.. tMMMMM@C:.................. .1Wn
# 7Uc. .:::,,,,,::,,,,.. i1t;,..................... .UEi
# 7C...::::::::::::,,,,.. .................... vSi.
# ;1;...,,::::::,......... .................. Yz:
# v97,......... .voC.
# izAotX7777777777777777777777777777777777777777Y7n92:
# .;CoIIIIIUAA666666699999ZZZZZZZZZZZZZZZZZZZZ6ov.
#
# !!! ATTENTION !!!
# DO NOT EDIT THIS FILE! THIS FILE CONTAINS THE DEFAULT VALUES FOR THE CON-
# FIGURATION! EDIT YOUR SECRET FILE (either prod.secret.exs, dev.secret.exs).
#
# This file is responsible for configuring your application
# and its dependencies with the aid of the Config module.
#
# This configuration file is loaded before any dependency and
# is restricted to this project.
import Config
# General application configuration
config :pleroma, ecto_repos: [Pleroma.Repo]
config :pleroma, Pleroma.Repo,
telemetry_event: [Pleroma.Repo.Instrumenter],
migration_lock: nil
config :pleroma, Pleroma.Captcha,
enabled: true,
seconds_valid: 300,
method: Pleroma.Captcha.Native
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
# Upload configuration
config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local,
filters: [Pleroma.Upload.Filter.Dedupe],
link_name: false,
proxy_remote: false,
filename_display_max_length: 30,
default_description: nil,
base_url: nil
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
bucket_namespace: nil,
truncated_namespace: nil,
streaming_enabled: true
config :ex_aws, :s3,
# host: "s3.wasabisys.com", # required if not Amazon AWS
access_key_id: nil,
secret_access_key: nil,
# region: "us-east-1", # may be required for Amazon AWS
scheme: "https://"
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: nil,
get_gateway_url: nil
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
pack_extensions: [".png", ".gif"],
groups: [
Custom: ["/emoji/*.png", "/emoji/**/*.png"]
],
default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json",
shared_pack_cache_seconds_per_file: 60
config :pleroma, :uri_schemes,
valid_schemes: [
"https",
"http",
"dat",
"dweb",
"gopher",
"hyper",
"ipfs",
"ipns",
"irc",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
"xmpp"
]
# Configures the endpoint
config :pleroma, Pleroma.Web.Endpoint,
url: [host: "localhost"],
http: [
ip: {127, 0, 0, 1}
],
protocol: "https",
secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
live_view: [signing_salt: "U5ELgdEwTD3n1+D5s0rY0AMg8/y1STxZ3Zvsl3bWh+oBcGrYdil0rXqPMRd3Glcq"],
signing_salt: "CqaoopA2",
render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
pubsub_server: Pleroma.PubSub,
secure_cookie_flag: true,
extra_cookie_attrs: [
"SameSite=Lax"
]
# Configures Elixir's Logger
config :logger, :console,
level: :debug,
format: "\n$time $metadata[$level] $message\n",
metadata: [:actor, :path, :type, :user]
config :logger, :ex_syslogger,
level: :debug,
ident: "pleroma",
format: "$metadata[$level] $message",
metadata: [:actor, :path, :type, :user]
config :mime, :types, %{
"application/xml" => ["xml"],
"application/xrd+xml" => ["xrd+xml"],
"application/jrd+json" => ["jrd+json"],
"application/activity+json" => ["activity+json"],
"application/ld+json" => ["activity+json"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
proxy_url: nil,
send_user_agent: true,
user_agent: :default,
adapter: []
config :pleroma, :instance,
name: "Pleroma",
email: "example@example.com",
notify_email: "noreply@example.com",
description: "Pleroma: An efficient and flexible fediverse server",
short_description: "",
background_image: "/images/city.jpg",
instance_thumbnail: "/instance/thumbnail.jpeg",
favicon: "/favicon.png",
limit: 5_000,
description_limit: 5_000,
remote_limit: 100_000,
upload_limit: 16_000_000,
avatar_upload_limit: 2_000_000,
background_upload_limit: 4_000_000,
banner_upload_limit: 4_000_000,
poll_limits: %{
max_options: 20,
max_option_chars: 200,
min_expiration: 0,
max_expiration: 365 * 24 * 60 * 60
},
registrations_open: true,
invites_enabled: false,
account_activation_required: false,
account_approval_required: false,
federating: true,
federation_incoming_replies_max_depth: 100,
federation_reachability_timeout_days: 7,
allow_relay: true,
public: true,
quarantined_instances: [],
static_dir: "instance/static/",
allowed_post_formats: [
"text/plain",
"text/html",
"text/markdown",
"text/bbcode"
],
autofollowed_nicknames: [],
autofollowing_nicknames: [],
max_pinned_statuses: 1,
attachment_links: false,
max_report_comment_size: 1000,
report_strip_status: true,
safe_dm_mentions: false,
healthcheck: false,
remote_post_retention_days: 90,
skip_thread_containment: true,
limit_to_local_content: :unauthenticated,
user_bio_length: 5000,
user_name_length: 100,
max_account_fields: 10,
max_remote_account_fields: 20,
account_field_name_length: 512,
account_field_value_length: 2048,
registration_reason_length: 500,
external_user_synchronization: true,
extended_nickname_format: true,
cleanup_attachments: false,
multi_factor_authentication: [
totp: [
# digits 6 or 8
digits: 6,
period: 30
],
backup_codes: [
number: 5,
length: 16
]
],
show_reactions: true,
password_reset_token_validity: 60 * 60 * 24,
profile_directory: true,
admin_privileges: [
:users_read,
:users_manage_invites,
:users_manage_activation_state,
:users_manage_tags,
:users_manage_credentials,
:users_delete,
:messages_read,
:messages_delete,
:instances_delete,
:reports_manage_reports,
:moderation_log_read,
:announcements_manage_announcements,
:emoji_manage_emoji,
:statistics_read
],
moderator_privileges: [:messages_delete, :reports_manage_reports],
max_endorsed_users: 20,
birthday_required: false,
birthday_min_age: 0,
max_media_attachments: 1_000
config :pleroma, :welcome,
direct_message: [
enabled: false,
sender_nickname: nil,
message: nil
],
chat_message: [
enabled: false,
sender_nickname: nil,
message: nil
],
email: [
enabled: false,
sender: nil,
subject: "Welcome to <%= instance_name %>",
html: "Welcome to <%= instance_name %>",
text: "Welcome to <%= instance_name %>"
]
config :pleroma, :feed,
post_title: %{
max_length: 100,
omission: "..."
}
config :pleroma, :markup,
# XXX - unfortunately, inline images must be enabled by default right now, because
# of custom emoji. Issue #275 discusses defanging that somehow.
allow_inline_images: true,
allow_headings: false,
allow_tables: false,
allow_fonts: false,
scrub_policy: [
Pleroma.HTML.Scrubber.Default,
Pleroma.HTML.Transform.MediaProxy
]
config :pleroma, :frontend_configurations,
pleroma_fe: %{
alwaysShowSubjectInput: true,
background: "/images/city.jpg",
collapseMessageWithSubject: false,
disableChat: false,
greentext: false,
hideFilteredStatuses: false,
hideMutedPosts: false,
hidePostStats: false,
hideSitename: false,
hideUserStats: false,
loginMethod: "password",
logo: "/static/logo.svg",
logoMargin: ".1em",
logoMask: true,
minimalScopesMode: false,
noAttachmentLinks: false,
nsfwCensorImage: "",
postContentType: "text/plain",
redirectRootLogin: "/main/friends",
redirectRootNoLogin: "/main/all",
scopeCopy: true,
sidebarRight: false,
showFeaturesPanel: true,
showInstanceSpecificPanel: false,
subjectLineBehavior: "email",
theme: "pleroma-dark",
webPushNotifications: false
}
config :pleroma, :assets,
mascots: [
pleroma_fox_tan: %{
url: "/images/pleroma-fox-tan-smol.png",
mime_type: "image/png"
},
pleroma_fox_tan_shy: %{
url: "/images/pleroma-fox-tan-shy.png",
mime_type: "image/png"
}
],
default_mascot: :pleroma_fox_tan
config :pleroma, :manifest,
icons: [
%{
src: "/static/logo.svg",
sizes: "144x144",
purpose: "any",
type: "image/svg+xml"
}
],
theme_color: "#282c37",
background_color: "#191b22"
config :pleroma, :activitypub,
unfollow_blocked: true,
outgoing_blocks: true,
blockers_visible: true,
follow_handshake_timeout: 500,
note_replies_output_limit: 5,
sign_object_fetches: true,
authorized_fetch_mode: false
config :pleroma, :streamer,
workers: 3,
overflow_workers: 2
config :pleroma, :user, deny_follow_blocked: true
config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default
config :pleroma, :mrf_rejectnonpublic,
allow_followersonly: false,
allow_direct: false
config :pleroma, :mrf_hellthread,
delist_threshold: 10,
reject_threshold: 20
config :pleroma, :mrf_simple,
media_removal: [],
media_nsfw: [],
federated_timeline_removal: [],
report_removal: [],
reject: [],
followers_only: [],
accept: [],
avatar_removal: [],
banner_removal: [],
reject_deletes: []
config :pleroma, :mrf_keyword,
reject: [],
federated_timeline_removal: [],
replace: []
config :pleroma, :mrf_emoji,
remove_url: [],
remove_shortcode: [],
federated_timeline_removal_url: [],
federated_timeline_removal_shortcode: []
config :pleroma, :mrf_hashtag,
sensitive: ["nsfw"],
reject: [],
federated_timeline_removal: []
config :pleroma, :mrf_subchain, match_actor: %{}
config :pleroma, :mrf_activity_expiration, days: 365
config :pleroma, :mrf_vocabulary,
accept: [],
reject: []
config :pleroma, :mrf_dnsrbl,
nameserver: "127.0.0.1",
port: 53,
zone: "bl.pleroma.com"
# threshold of 7 days
config :pleroma, :mrf_object_age,
threshold: 604_800,
actions: [:delist, :strip_followers]
config :pleroma, :mrf_nsfw_api,
url: "http://127.0.0.1:5000/",
threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
config :pleroma, :mrf_follow_bot, follower_nickname: nil
config :pleroma, :mrf_inline_quote, template: "<bdi>RT:</bdi> {url}"
config :pleroma, :mrf_force_mention,
mention_parent: true,
mention_quoted: true
config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000
config :pleroma, :rich_media,
enabled: true,
ignore_hosts: [],
ignore_tld: ["local", "localdomain", "lan"],
parsers: [
Pleroma.Web.RichMedia.Parsers.TwitterCard,
Pleroma.Web.RichMedia.Parsers.OEmbed
],
failure_backoff: 60_000,
ttl_setters: [
Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl,
Pleroma.Web.RichMedia.Parser.TTL.Opengraph
],
max_body: 5_000_000
config :pleroma, :media_proxy,
enabled: false,
invalidation: [
enabled: false,
provider: Pleroma.Web.MediaProxy.Invalidation.Script
],
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,
# Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1
max_read_duration: 30_000,
http: [
follow_redirect: true,
pool: :media
]
],
whitelist: []
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
method: :purge,
headers: [],
options: []
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
script_path: nil,
url_format: nil
# Note: media preview proxy depends on media proxy to be enabled
config :pleroma, :media_preview_proxy,
enabled: false,
thumbnail_max_width: 600,
thumbnail_max_height: 600,
image_quality: 85,
min_content_length: 100 * 1024
config :pleroma, :shout,
enabled: true,
limit: 5_000
config :phoenix, :format_encoders, json: Jason, "activity+json": Jason
config :phoenix, :json_library, Jason
config :phoenix, :filter_parameters, ["password", "confirm"]
config :pleroma, :gopher,
enabled: false,
ip: {0, 0, 0, 0},
port: 9999
config :pleroma, Pleroma.Web.Metadata,
providers: [
Pleroma.Web.Metadata.Providers.OpenGraph,
Pleroma.Web.Metadata.Providers.TwitterCard
],
unfurl_nsfw: false
config :pleroma, Pleroma.Web.Preload,
providers: [
Pleroma.Web.Preload.Providers.Instance
]
config :pleroma, :http_security,
enabled: true,
sts: false,
sts_max_age: 31_536_000,
ct_max_age: 2_592_000,
referrer_policy: "same-origin"
config :cors_plug,
max_age: 86_400,
methods: ["POST", "PUT", "DELETE", "GET", "PATCH", "OPTIONS"],
expose: [
"Link",
"X-RateLimit-Reset",
"X-RateLimit-Limit",
"X-RateLimit-Remaining",
"X-Request-Id",
"Idempotency-Key"
],
credentials: true,
headers: ["Authorization", "Content-Type", "Idempotency-Key"]
config :pleroma, Pleroma.User,
restricted_nicknames: [
".well-known",
"~",
"about",
"activities",
"api",
"auth",
"check_password",
"dev",
"friend-requests",
"inbox",
"internal",
"main",
"media",
"nodeinfo",
"notice",
"oauth",
"objects",
"ostatus_subscribe",
"pleroma",
"proxy",
"push",
"registration",
"relay",
"settings",
"status",
"tag",
"user-search",
"user_exists",
"users",
"web",
"verify_credentials",
"update_credentials",
"relationships",
"search",
"confirmation_resend",
"mfa"
],
email_blacklist: []
config :pleroma, Oban,
repo: Pleroma.Repo,
log: false,
queues: [
activity_expiration: 10,
token_expiration: 5,
filter_expiration: 1,
backup: 1,
federator_incoming: 5,
federator_outgoing: 5,
ingestion_queue: 50,
web_push: 50,
mailer: 10,
transmogrifier: 20,
scheduled_activities: 10,
poll_notifications: 10,
background: 5,
remote_fetcher: 2,
attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
search_indexing: [limit: 10, paused: true],
rich_media_expiration: 2
],
plugins: [Oban.Plugins.Pruner],
crontab: [
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
]
config :pleroma, :workers,
retries: [
federator_incoming: 5,
federator_outgoing: 5,
search_indexing: 2
]
config :pleroma, Pleroma.Formatter,
class: false,
rel: "ugc",
new_window: false,
truncate: false,
strip_prefix: false,
extra: true,
validate_tld: :no_scheme
config :pleroma, :ldap,
enabled: System.get_env("LDAP_ENABLED") == "true",
host: System.get_env("LDAP_HOST") || "localhost",
port: String.to_integer(System.get_env("LDAP_PORT") || "389"),
ssl: System.get_env("LDAP_SSL") == "true",
sslopts: [],
tls: System.get_env("LDAP_TLS") == "true",
tlsopts: [],
base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
uid: System.get_env("LDAP_UID") || "cn"
oauth_consumer_strategies =
System.get_env("OAUTH_CONSUMER_STRATEGIES")
|> to_string()
|> String.split()
|> Enum.map(&hd(String.split(&1, ":")))
ueberauth_providers =
for strategy <- oauth_consumer_strategies do
strategy_module_name = "Elixir.Ueberauth.Strategy.#{String.capitalize(strategy)}"
strategy_module = String.to_atom(strategy_module_name)
{String.to_atom(strategy), {strategy_module, [callback_params: ["state"]]}}
end
config :ueberauth,
Ueberauth,
base_path: "/oauth",
providers: ueberauth_providers
config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies
config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false
config :pleroma, Pleroma.Emails.UserEmail,
logo: nil,
styling: %{
link_color: "#d8a070",
background_color: "#2C3645",
content_background_color: "#1B2635",
header_color: "#d8a070",
text_color: "#b9b9ba",
text_muted_color: "#b9b9ba"
}
config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
config :pleroma, Pleroma.PromEx,
disabled: false,
manual_metrics_start_delay: :no_delay,
drop_metrics_groups: [],
grafana: [
host: System.get_env("GRAFANA_HOST", "http://localhost:3000"),
auth_token: System.get_env("GRAFANA_TOKEN"),
upload_dashboards_on_start: false,
folder_name: "BEAM",
annotate_app_lifecycle: true
],
metrics_server: [
port: 4021,
path: "/metrics",
protocol: :http,
pool_size: 5,
cowboy_opts: [],
auth_strategy: :none
],
datasource: "Prometheus"
config :pleroma, Pleroma.ScheduledActivity,
daily_user_limit: 25,
total_user_limit: 300,
enabled: true
config :pleroma, :email_notifications,
digest: %{
active: false,
interval: 7,
inactivity_threshold: 7
}
config :pleroma, :oauth2,
token_expires_in: 3600 * 24 * 365 * 100,
issue_new_refresh_token: true,
clean_expired_tokens: false
config :pleroma, :database, rum_enabled: false
config :pleroma, :features, improved_hashtag_timeline: :auto
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env()
config :http_signatures,
adapter: Pleroma.Signature
config :pleroma, :rate_limit,
authentication: {60_000, 15},
timeline: {500, 3},
search: [{1000, 10}, {1000, 30}],
app_account_creation: {1_800_000, 25},
relations_actions: {10_000, 10},
relation_id_action: {60_000, 2},
statuses_actions: {10_000, 15},
status_id_action: {60_000, 3},
password_reset: {1_800_000, 5},
account_confirmation_resend: {8_640_000, 5},
ap_routes: {60_000, 15}
config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
config :pleroma, Pleroma.Web.Plugs.RemoteIp,
enabled: true,
headers: ["x-forwarded-for"],
proxies: [],
reserved: [
"127.0.0.0/8",
"::1/128",
"fc00::/7",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
config :pleroma, :static_fe, enabled: false
# Example of frontend configuration
# This example will make us serve the primary frontend from the
# frontends directory within your `:pleroma, :instance, static_dir`.
# e.g., instance/static/frontends/pleroma/develop/
#
# With no frontend configuration, the bundled files from the `static` directory will
# be used.
#
# config :pleroma, :frontends,
# primary: %{"name" => "pleroma-fe", "ref" => "develop"},
# admin: %{"name" => "admin-fe", "ref" => "stable"},
# available: %{...}
config :pleroma, :frontends,
available: %{
"kenoma" => %{
"name" => "kenoma",
"git" => "https://git.pleroma.social/lambadalambda/kenoma",
"build_url" =>
"https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "master"
},
"pleroma-fe" => %{
"name" => "pleroma-fe",
"git" => "https://git.pleroma.social/pleroma/pleroma-fe",
"build_url" =>
"https://git.pleroma.social/pleroma/pleroma-fe/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "develop"
},
"fedi-fe" => %{
"name" => "fedi-fe",
"git" => "https://git.pleroma.social/pleroma/fedi-fe",
"build_url" =>
"https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build_release",
"ref" => "master",
"custom-http-headers" => [
{"service-worker-allowed", "/"}
]
},
"admin-fe" => %{
"name" => "admin-fe",
"git" => "https://git.pleroma.social/pleroma/admin-fe",
"build_url" =>
"https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "develop"
},
"soapbox" => %{
"name" => "soapbox",
"git" => "https://gitlab.com/soapbox-pub/soapbox",
"build_url" =>
"https://gitlab.com/soapbox-pub/soapbox/-/jobs/artifacts/${ref}/download?job=build-production",
"ref" => "v3.0.0-beta.1",
"build_dir" => "static"
},
"glitch-lily" => %{
"name" => "glitch-lily",
"git" => "https://lily-is.land/infra/glitch-lily",
"build_url" =>
"https://lily-is.land/infra/glitch-lily/-/jobs/artifacts/${ref}/download?job=build",
"ref" => "servant",
"build_dir" => "public"
}
}
config :pleroma, :web_cache_ttl,
activity_pub: nil,
activity_pub_question: 30_000
config :pleroma, :modules, runtime_dir: "instance/modules"
config :pleroma, configurable_from_database: false
config :pleroma, Pleroma.Repo,
parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
prepare: :unnamed
config :pleroma, :connections_pool,
reclaim_multiplier: 0.1,
connection_acquisition_wait: 250,
connection_acquisition_retries: 5,
max_connections: 250,
max_idle_time: 30_000,
retry: 0,
connect_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_waiting: 10,
recv_timeout: 10_000
],
media: [
size: 50,
max_waiting: 20,
recv_timeout: 15_000
],
upload: [
size: 25,
max_waiting: 5,
recv_timeout: 15_000
],
default: [
size: 10,
max_waiting: 2,
recv_timeout: 5_000
]
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
config :pleroma, :majic_pool, size: 2
private_instance? = :if_instance_is_private
config :pleroma, :restrict_unauthenticated,
timelines: %{local: private_instance?, federated: private_instance?},
profiles: %{local: private_instance?, remote: private_instance?},
activities: %{local: private_instance?, remote: private_instance?}
config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
config :pleroma, :mrf,
policies: [
Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
Pleroma.Web.ActivityPub.MRF.TagPolicy,
Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy
],
transparency: true,
transparency_exclusions: []
config :tzdata, :http_client, Pleroma.HTTP.Tzdata
config :ex_aws, http_client: Pleroma.HTTP.ExAws
config :web_push_encryption, http_client: Pleroma.HTTP.WebPush
config :pleroma, :instances_favicons, enabled: false
config :floki, :html_parser, Floki.HTMLParser.FastHtml
config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
config :pleroma, Pleroma.User.Backup,
purge_after_days: 30,
limit_days: 7,
dir: nil,
process_wait_time: 30_000,
process_chunk_size: 100
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
{Pleroma.Search, [max_running: 30, max_waiting: 50]}
]
config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true
config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch,
url: "http://127.0.0.1:7700/",
private_key: nil,
initial_indexing_chunk_size: 100_000
config :pleroma, Pleroma.Application,
background_migrators: true,
internal_fetch: true,
load_custom_modules: true,
max_restarts: 3,
streamer_registry: true
config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
+config :pleroma, Pleroma.Search.QdrantSearch,
+ qdrant_url: "http://127.0.0.1:6333/",
+ qdrant_api_key: "",
+ openai_url: "http://127.0.0.1:11345",
+ # The healthcheck url has to be set to nil when used with the real openai
+ # API, as it doesn't have a healthcheck endpoint.
+ openai_healthcheck_url: "http://127.0.0.1:11345/health",
+ openai_model: "snowflake/snowflake-arctic-embed-xs",
+ openai_api_key: "",
+ qdrant_index_configuration: %{
+ vectors: %{size: 384, distance: "Cosine"}
+ }
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
index 0316c9bf4..d34f84d4f 100644
--- a/docs/configuration/search.md
+++ b/docs/configuration/search.md
@@ -1,123 +1,147 @@
# Configuring search
{! backend/administration/CLI_tasks/general_cli_task_info.include !}
## Built-in search
To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
While it has no external dependencies, it has problems with performance and relevancy.
+## QdrantSearch
+
+This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
+
+The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
+
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
+
+Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions.
+
+You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
+
+### Indexing and model options
+
+To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
+
+The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail.
+
+Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`.
+
+E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024.
+
+When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other.
+
## Meilisearch
Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million
posts while idle and up to 7G while indexing initially). The disk usage for this additional index is also
around 4 gigabytes. Like [RUM](./cheatsheet.md#rum-indexing-for-full-text-search) indexes, it offers considerably
higher performance and ordering by timestamp in a reasonable amount of time.
Additionally, the search results seem to be more accurate.
Due to high memory usage, it may be best to set it up on a different machine, if running pleroma on a low-resource
computer, and use private key authentication to secure the remote search instance.
To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`:
> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
You then need to set the address of the meilisearch instance, and optionally the private key for authentication. You might
also want to change the `initial_indexing_chunk_size` to be smaller if you're server is not very powerful, but not higher than `100_000`,
because meilisearch will refuse to process it if it's too big. However, in general you want this to be as big as possible, because meilisearch
indexes faster when it can process many posts in a single batch.
> config :pleroma, Pleroma.Search.Meilisearch,
> url: "http://127.0.0.1:7700/",
> private_key: "private key",
> initial_indexing_chunk_size: 100_000
Information about setting up meilisearch can be found in the
[official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
You probably want to start it with `MEILI_NO_ANALYTICS=true` environment variable to disable analytics.
At least version 0.25.0 is required, but you are strongly advised to use at least 0.26.0, as it introduces
the `--enable-auto-batching` option which drastically improves performance. Without this option, the search
is hardly usable on a somewhat big instance.
### Private key authentication (optional)
To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
you have to get the _private key_, which is actually used for authentication.
=== "OTP"
```sh
./bin/pleroma_ctl search.meilisearch show-keys <your master key here>
```
=== "From Source"
```sh
mix pleroma.search.meilisearch show-keys <your master key here>
```
You will see a "Default Admin API Key", this is the key you actually put into your configuration file.
### Initial indexing
After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only
have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
million posts while idle and up to 7G while indexing initially, but your experience may be different).
The sequence of actions is as follows:
1. First, change the configuration to use `Pleroma.Search.Meilisearch` as the search backend
2. Restart your instance, at this point it can be used while the search indexing is running, though search won't return anything
3. Start the initial indexing process (as described below with `index`),
and wait until the task says it sent everything from the database to index
4. Wait until everything is actually indexed (by checking with `stats` as described below),
at this point you don't have to do anything, just wait a while.
To start the initial indexing, run the `index` command:
=== "OTP"
```sh
./bin/pleroma_ctl search.meilisearch index
```
=== "From Source"
```sh
mix pleroma.search.meilisearch index
```
This will show you the total amount of posts to index, and then show you the amount of posts indexed currently, until the numbers eventually
become the same. The posts are indexed in big batches and meilisearch will take some time to actually index them, even after you have
inserted all the posts into it. Depending on the amount of posts, this may be as long as several hours. To get information about the status
of indexing and how many posts have actually been indexed, use the `stats` command:
=== "OTP"
```sh
./bin/pleroma_ctl search.meilisearch stats
```
=== "From Source"
```sh
mix pleroma.search.meilisearch stats
```
### Clearing the index
In case you need to clear the index (for example, to re-index from scratch, if that needs to happen for some reason), you can
use the `clear` command:
=== "OTP"
```sh
./bin/pleroma_ctl search.meilisearch clear
```
=== "From Source"
```sh
mix pleroma.search.meilisearch clear
```
This will clear **all** the posts from the search index. Note, that deleted posts are also removed from index by the instance itself, so
there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information
that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
depends on the amount of text in posts.
diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
new file mode 100644
index 000000000..81a9fced6
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Indexer do
+ import Mix.Pleroma
+ import Ecto.Query
+
+ alias Pleroma.Workers.SearchIndexingWorker
+
+ def run(["create_index"]) do
+ start_pleroma()
+
+ with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
+ IO.puts("Index created")
+ else
+ e -> IO.puts("Could not create index: #{inspect(e)}")
+ end
+ end
+
+ def run(["drop_index"]) do
+ start_pleroma()
+
+ with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
+ IO.puts("Index dropped")
+ else
+ e -> IO.puts("Could not drop index: #{inspect(e)}")
+ end
+ end
+
+ def run(["index" | options]) do
+ {options, [], []} =
+ OptionParser.parse(
+ options,
+ strict: [
+ limit: :integer
+ ]
+ )
+
+ start_pleroma()
+
+ limit = Keyword.get(options, :limit, 100_000)
+
+ per_step = 1000
+ chunks = max(div(limit, per_step), 1)
+
+ 1..chunks
+ |> Enum.each(fn step ->
+ q =
+ from(a in Pleroma.Activity,
+ limit: ^per_step,
+ offset: ^per_step * (^step - 1),
+ select: [:id],
+ order_by: [desc: :id]
+ )
+
+ {:ok, ids} =
+ Pleroma.Repo.transaction(fn ->
+ Pleroma.Repo.stream(q, timeout: :infinity)
+ |> Enum.map(fn a ->
+ a.id
+ end)
+ end)
+
+ IO.puts("Got #{length(ids)} activities, adding to indexer")
+
+ ids
+ |> Enum.chunk_every(100)
+ |> Enum.each(fn chunk ->
+ IO.puts("Adding #{length(chunk)} activities to indexing queue")
+
+ chunk
+ |> Enum.map(fn id ->
+ SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
+ end)
+ |> Oban.insert_all()
+ end)
+ end)
+ end
+end
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
index 11e99e7f1..c6fe8a9bd 100644
--- a/lib/pleroma/search/database_search.ex
+++ b/lib/pleroma/search/database_search.ex
@@ -1,168 +1,174 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.DatabaseSearch do
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Visibility
require Pleroma.Constants
import Ecto.Query
@behaviour Pleroma.Search.SearchBackend
@impl true
def search(user, search_query, options \\ []) do
index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
try do
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public(user)
|> query_with(index_type, search_query, :websearch)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> Pagination.fetch_paginated(
%{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
:offset
)
|> maybe_fetch(user, search_query)
rescue
_ -> maybe_fetch([], user, search_query)
end
end
@impl true
def add_to_index(_activity), do: :ok
@impl true
def remove_from_index(_object), do: :ok
+ @impl true
+ def create_index, do: :ok
+
+ @impl true
+ def drop_index, do: :ok
+
@impl true
def healthcheck_endpoints, do: nil
def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author)
end
def maybe_restrict_author(query, _), do: query
def maybe_restrict_blocked(query, %User{} = user) do
Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
end
def maybe_restrict_blocked(query, _), do: query
defp restrict_public(q, user) when not is_nil(user) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp restrict_public(q, _user) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
from([a, o] in q,
where:
fragment(
"? @@ websearch_to_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
def maybe_restrict_local(q, user) do
limit = Config.get([:instance, :limit_to_local_content], :unauthenticated)
case {limit, user} do
{:all, _} -> restrict_local(q)
{:unauthenticated, %User{}} -> q
{:unauthenticated, _} -> restrict_local(q)
{false, _} -> q
end
end
defp restrict_local(q), do: where(q, local: true)
def maybe_fetch(activities, user, search_query) do
with true <- Regex.match?(~r/https?:/, search_query),
{:ok, object} <- Fetcher.fetch_object_from_id(search_query),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
true <- Visibility.visible_for_user?(activity, user) do
[activity | activities]
else
_ -> activities
end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
index 08c2f3d86..9bba5b30f 100644
--- a/lib/pleroma/search/meilisearch.ex
+++ b/lib/pleroma/search/meilisearch.ex
@@ -1,192 +1,198 @@
defmodule Pleroma.Search.Meilisearch do
require Logger
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
import Pleroma.Search.DatabaseSearch
import Ecto.Query
@behaviour Pleroma.Search.SearchBackend
+ @impl true
+ def create_index, do: :ok
+
+ @impl true
+ def drop_index, do: :ok
+
defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
[{"Content-Type", "application/json"}] ++
if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}]
end
def meili_get(path) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.get(
Path.join(endpoint, path),
meili_headers()
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_post(path, params) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.post(
Path.join(endpoint, path),
Jason.encode!(params),
meili_headers()
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_put(path, params) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
result =
Pleroma.HTTP.request(
:put,
Path.join(endpoint, path),
Jason.encode!(params),
meili_headers(),
[]
)
with {:ok, res} <- result do
{:ok, Jason.decode!(res.body)}
end
end
def meili_delete(path) do
endpoint = Config.get([Pleroma.Search.Meilisearch, :url])
with {:ok, _} <-
Pleroma.HTTP.request(
:delete,
Path.join(endpoint, path),
"",
meili_headers(),
[]
) do
:ok
else
_ -> {:error, "Could not remove from index"}
end
end
@impl true
def search(user, query, options \\ []) do
limit = Enum.min([Keyword.get(options, :limit), 40])
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
res =
meili_post(
"/indexes/objects/search",
%{q: query, offset: offset, limit: limit}
)
with {:ok, result} <- res do
hits = result["hits"] |> Enum.map(& &1["ap"])
try do
hits
|> Activity.create_by_object_ap_id()
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> maybe_fetch(user, query)
|> order_by([object: obj], desc: obj.data["published"])
|> Pleroma.Repo.all()
rescue
_ -> maybe_fetch([], user, query)
end
end
end
def object_to_search_data(object) do
# Only index public or unlisted Notes
if not is_nil(object) and object.data["type"] == "Note" and
not is_nil(object.data["content"]) and
(Pleroma.Constants.as_public() in object.data["to"] or
Pleroma.Constants.as_public() in object.data["cc"]) and
object.data["content"] not in ["", "."] do
data = object.data
content_str =
case data["content"] do
[nil | rest] -> to_string(rest)
str -> str
end
content =
with {:ok, scrubbed} <-
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
# Make sure we have a non-empty string
if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
end
end
end
@impl true
def add_to_index(activity) do
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
result =
meili_put(
"/indexes/objects/documents",
[maybe_search_data]
)
with {:ok, %{"status" => "enqueued"}} <- result do
# Added successfully
:ok
else
_ ->
# There was an error, report it
Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}")
{:error, result}
end
else
# The post isn't something we can search, that's ok
:ok
end
end
@impl true
def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}")
end
@impl true
def healthcheck_endpoints do
endpoint =
Config.get([Pleroma.Search.Meilisearch, :url])
|> URI.parse()
|> Map.put(:path, "/health")
|> URI.to_string()
[endpoint]
end
end
diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex
new file mode 100644
index 000000000..b659bb682
--- /dev/null
+++ b/lib/pleroma/search/qdrant_search.ex
@@ -0,0 +1,182 @@
+defmodule Pleroma.Search.QdrantSearch do
+ @behaviour Pleroma.Search.SearchBackend
+ import Ecto.Query
+
+ alias Pleroma.Activity
+ alias Pleroma.Config.Getting, as: Config
+
+ alias __MODULE__.OpenAIClient
+ alias __MODULE__.QdrantClient
+
+ import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
+ import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
+
+ @impl true
+ def create_index do
+ payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
+
+ with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
+ :ok
+ else
+ e -> {:error, e}
+ end
+ end
+
+ @impl true
+ def drop_index do
+ with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
+ :ok
+ else
+ e -> {:error, e}
+ end
+ end
+
+ def get_embedding(text) do
+ with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
+ OpenAIClient.post("/v1/embeddings", %{
+ input: text,
+ model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
+ }) do
+ {:ok, embedding}
+ else
+ _ ->
+ {:error, "Failed to get embedding"}
+ end
+ end
+
+ defp actor_from_activity(%{data: %{"actor" => actor}}) do
+ actor
+ end
+
+ defp actor_from_activity(_), do: nil
+
+ defp build_index_payload(activity, embedding) do
+ actor = actor_from_activity(activity)
+ published_at = activity.data["published"]
+
+ %{
+ points: [
+ %{
+ id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
+ vector: embedding,
+ payload: %{actor: actor, published_at: published_at}
+ }
+ ]
+ }
+ end
+
+ defp build_search_payload(embedding, options) do
+ base = %{
+ vector: embedding,
+ limit: options[:limit] || 20,
+ offset: options[:offset] || 0
+ }
+
+ if author = options[:author] do
+ Map.put(base, :filter, %{
+ must: [%{key: "actor", match: %{value: author.ap_id}}]
+ })
+ else
+ base
+ end
+ end
+
+ @impl true
+ def add_to_index(activity) do
+ # This will only index public or unlisted notes
+ maybe_search_data = object_to_search_data(activity.object)
+
+ if activity.data["type"] == "Create" and maybe_search_data do
+ with {:ok, embedding} <- get_embedding(maybe_search_data.content),
+ {:ok, %{status: 200}} <-
+ QdrantClient.put(
+ "/collections/posts/points",
+ build_index_payload(activity, embedding)
+ ) do
+ :ok
+ else
+ e -> {:error, e}
+ end
+ else
+ :ok
+ end
+ end
+
+ @impl true
+ def remove_from_index(object) do
+ activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
+ id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
+
+ with {:ok, %{status: 200}} <-
+ QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
+ :ok
+ else
+ e -> {:error, e}
+ end
+ end
+
+ @impl true
+ def search(user, original_query, options) do
+ query = "Represent this sentence for searching relevant passages: #{original_query}"
+
+ with {:ok, embedding} <- get_embedding(query),
+ {:ok, %{body: %{"result" => result}}} <-
+ QdrantClient.post(
+ "/collections/posts/points/search",
+ build_search_payload(embedding, options)
+ ) do
+ ids =
+ Enum.map(result, fn %{"id" => id} ->
+ Ecto.UUID.dump!(id)
+ end)
+
+ from(a in Activity, where: a.id in ^ids)
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
+ |> Pleroma.Repo.all()
+ |> maybe_fetch(user, original_query)
+ else
+ _ ->
+ []
+ end
+ end
+
+ @impl true
+ def healthcheck_endpoints do
+ qdrant_health =
+ Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
+ |> URI.parse()
+ |> Map.put(:path, "/healthz")
+ |> URI.to_string()
+
+ openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
+
+ [qdrant_health, openai_health] |> Enum.filter(& &1)
+ end
+end
+
+defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
+ use Tesla
+ alias Pleroma.Config.Getting, as: Config
+
+ plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
+ plug(Tesla.Middleware.JSON)
+
+ plug(Tesla.Middleware.Headers, [
+ {"Authorization",
+ "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
+ ])
+end
+
+defmodule Pleroma.Search.QdrantSearch.QdrantClient do
+ use Tesla
+ alias Pleroma.Config.Getting, as: Config
+
+ plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
+ plug(Tesla.Middleware.JSON)
+
+ plug(Tesla.Middleware.Headers, [
+ {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
+ ])
+end
diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex
index 13c887bc2..f4ed13c36 100644
--- a/lib/pleroma/search/search_backend.ex
+++ b/lib/pleroma/search/search_backend.ex
@@ -1,32 +1,42 @@
defmodule Pleroma.Search.SearchBackend do
@doc """
Search statuses with a query, restricting to only those the user should have access to.
"""
@callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [
Pleroma.Activity.t()
]
@doc """
Add the object associated with the activity to the search index.
The whole activity is passed, to allow filtering on things such as scope.
"""
@callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()}
@doc """
Remove the object from the index.
Just the object, as opposed to the whole activity, is passed, since the object
is what contains the actual content and there is no need for filtering when removing
from index.
"""
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
+ @doc """
+ Create the index
+ """
+ @callback create_index() :: :ok | {:error, any()}
+
+ @doc """
+ Drop the index
+ """
+ @callback drop_index() :: :ok | {:error, any()}
+
@doc """
Healthcheck endpoints of search backend infrastructure to monitor for controlling
processing of jobs in the Oban queue.
It is expected a 200 response is healthy and other responses are unhealthy.
"""
@callback healthcheck_endpoints :: list() | nil
end
diff --git a/supplemental/search/fastembed-api/Dockerfile b/supplemental/search/fastembed-api/Dockerfile
new file mode 100644
index 000000000..c1e0ef51f
--- /dev/null
+++ b/supplemental/search/fastembed-api/Dockerfile
@@ -0,0 +1,9 @@
+FROM python:3.9
+
+WORKDIR /code
+COPY fastembed-server.py /workdir/fastembed-server.py
+COPY requirements.txt /workdir/requirements.txt
+
+RUN pip install -r /workdir/requirements.txt
+
+CMD ["python", "/workdir/fastembed-server.py"]
diff --git a/supplemental/search/fastembed-api/README.md b/supplemental/search/fastembed-api/README.md
new file mode 100644
index 000000000..63a037207
--- /dev/null
+++ b/supplemental/search/fastembed-api/README.md
@@ -0,0 +1,6 @@
+# About
+This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend.
+
+# Usage
+
+The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds.
diff --git a/supplemental/search/fastembed-api/compose.yml b/supplemental/search/fastembed-api/compose.yml
new file mode 100644
index 000000000..d4cb31722
--- /dev/null
+++ b/supplemental/search/fastembed-api/compose.yml
@@ -0,0 +1,5 @@
+services:
+ web:
+ build: .
+ ports:
+ - "11345:11345"
diff --git a/supplemental/search/fastembed-api/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py
new file mode 100644
index 000000000..02da69db2
--- /dev/null
+++ b/supplemental/search/fastembed-api/fastembed-server.py
@@ -0,0 +1,27 @@
+from fastembed import TextEmbedding
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+models = {}
+
+app = FastAPI()
+
+class EmbeddingRequest(BaseModel):
+ model: str
+ input: str
+
+@app.post("/v1/embeddings")
+def embeddings(request: EmbeddingRequest):
+ model = models.get(request.model) or TextEmbedding(request.model)
+ models[request.model] = model
+ embeddings = next(model.embed(request.input)).tolist()
+ return {"data": [{"embedding": embeddings}]}
+
+@app.get("/health")
+def health():
+ return {"status": "ok"}
+
+if __name__ == "__main__":
+ import uvicorn
+
+ uvicorn.run(app, host="0.0.0.0", port=11345)
diff --git a/supplemental/search/fastembed-api/requirements.txt b/supplemental/search/fastembed-api/requirements.txt
new file mode 100644
index 000000000..db67a8402
--- /dev/null
+++ b/supplemental/search/fastembed-api/requirements.txt
@@ -0,0 +1,4 @@
+fastapi==0.111.0
+fastembed==0.2.7
+pydantic==1.10.15
+uvicorn==0.29.0
diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs
new file mode 100644
index 000000000..47a77a391
--- /dev/null
+++ b/test/pleroma/search/qdrant_search_test.exs
@@ -0,0 +1,199 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.QdrantSearchTest do
+ use Pleroma.DataCase, async: true
+ use Oban.Testing, repo: Pleroma.Repo
+
+ import Pleroma.Factory
+ import Mox
+
+ alias Pleroma.Search.QdrantSearch
+ alias Pleroma.UnstubbedConfigMock, as: Config
+ alias Pleroma.Web.CommonAPI
+ alias Pleroma.Workers.SearchIndexingWorker
+
+ describe "Qdrant search" do
+ test "returns the correct healthcheck endpoints" do
+ # No openai healthcheck URL
+ Config
+ |> expect(:get, 2, fn
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{qdrant_url: "https://qdrant.url"}[key]
+ end)
+
+ [health_endpoint] = QdrantSearch.healthcheck_endpoints()
+
+ assert "https://qdrant.url/healthz" == health_endpoint
+
+ # Set openai healthcheck URL
+ Config
+ |> expect(:get, 2, fn
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[
+ key
+ ]
+ end)
+
+ [_, health_endpoint] = QdrantSearch.healthcheck_endpoints()
+
+ assert "https://openai.url/health" == health_endpoint
+ end
+
+ test "searches for a term by encoding it and sending it to qdrant" do
+ user = insert(:user)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ Config
+ |> expect(:get, 3, fn
+ [Pleroma.Search, :module], nil ->
+ QdrantSearch
+
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{
+ openai_model: "a_model",
+ openai_url: "https://openai.url",
+ qdrant_url: "https://qdrant.url"
+ }[key]
+ end)
+
+ Tesla.Mock.mock(fn
+ %{url: "https://openai.url/v1/embeddings", method: :post} ->
+ Tesla.Mock.json(%{
+ data: [%{embedding: [1, 2, 3]}]
+ })
+
+ %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
+ data = Jason.decode!(body)
+ refute data["filter"]
+
+ Tesla.Mock.json(%{
+ result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
+ })
+ end)
+
+ results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{})
+
+ assert results == [activity]
+ end
+
+ test "for a given actor, ask for only relevant matches" do
+ user = insert(:user)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ Config
+ |> expect(:get, 3, fn
+ [Pleroma.Search, :module], nil ->
+ QdrantSearch
+
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{
+ openai_model: "a_model",
+ openai_url: "https://openai.url",
+ qdrant_url: "https://qdrant.url"
+ }[key]
+ end)
+
+ Tesla.Mock.mock(fn
+ %{url: "https://openai.url/v1/embeddings", method: :post} ->
+ Tesla.Mock.json(%{
+ data: [%{embedding: [1, 2, 3]}]
+ })
+
+ %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} ->
+ data = Jason.decode!(body)
+
+ assert data["filter"] == %{
+ "must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}]
+ }
+
+ Tesla.Mock.json(%{
+ result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}]
+ })
+ end)
+
+ results =
+ QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user})
+
+ assert results == [activity]
+ end
+
+ test "indexes a public post on creation, deletes from the index on deletion" do
+ user = insert(:user)
+
+ Tesla.Mock.mock(fn
+ %{method: :post, url: "https://openai.url/v1/embeddings"} ->
+ send(self(), "posted_to_openai")
+
+ Tesla.Mock.json(%{
+ data: [%{embedding: [1, 2, 3]}]
+ })
+
+ %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} ->
+ send(self(), "posted_to_qdrant")
+
+ data = Jason.decode!(body)
+ %{"points" => [%{"vector" => vector, "payload" => payload}]} = data
+
+ assert vector == [1, 2, 3]
+ assert payload["actor"]
+ assert payload["published_at"]
+
+ Tesla.Mock.json("ok")
+
+ %{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} ->
+ send(self(), "deleted_from_qdrant")
+ Tesla.Mock.json("ok")
+ end)
+
+ Config
+ |> expect(:get, 6, fn
+ [Pleroma.Search, :module], nil ->
+ QdrantSearch
+
+ [Pleroma.Search.QdrantSearch, key], nil ->
+ %{
+ openai_model: "a_model",
+ openai_url: "https://openai.url",
+ qdrant_url: "https://qdrant.url"
+ }[key]
+ end)
+
+ {:ok, activity} =
+ CommonAPI.post(user, %{
+ status: "guys i just don't wanna leave the swamp",
+ visibility: "public"
+ })
+
+ args = %{"op" => "add_to_index", "activity" => activity.id}
+
+ assert_enqueued(
+ worker: SearchIndexingWorker,
+ args: args
+ )
+
+ assert :ok = perform_job(SearchIndexingWorker, args)
+ assert_received("posted_to_openai")
+ assert_received("posted_to_qdrant")
+
+ {:ok, _} = CommonAPI.delete(activity.id, user)
+
+ delete_args = %{"op" => "remove_from_index", "object" => activity.object.id}
+ assert_enqueued(worker: SearchIndexingWorker, args: delete_args)
+ assert :ok = perform_job(SearchIndexingWorker, delete_args)
+
+ assert_received("deleted_from_qdrant")
+ end
+ end
+end

File Metadata

Mime Type
text/x-diff
Expires
Wed, Oct 14, 4:27 AM (1 d, 19 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1787003
Default Alt Text
(62 KB)

Event Timeline