Page MenuHomePhorge
Files F85828771

No OneTemporary
Actions

Authored By
Unknown
Size
8 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/changelog.d/stats-peer-domain-splitting.change b/changelog.d/stats-peer-domain-splitting.change
new file mode 100644
index 000000000..b4dd59b0f
--- /dev/null
+++ b/changelog.d/stats-peer-domain-splitting.change
@@ -0,0 +1 @@
+Reduce the cost of collecting peer domains during statistics refreshes by avoiding citext regular expression splitting.
diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
index 47b30b951..0d527fb50 100644
--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@@ -1,137 +1,137 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Stats do
use GenServer
import Ecto.Query
alias Pleroma.CounterCache
alias Pleroma.Repo
alias Pleroma.User
@interval :timer.seconds(60)
def start_link(_) do
GenServer.start_link(
__MODULE__,
nil,
name: __MODULE__
)
end
@impl true
def init(_args) do
if Pleroma.Config.get(:env) != :test do
{:ok, nil, {:continue, :calculate_stats}}
else
{:ok, calculate_stat_data()}
end
end
@doc "Performs update stats"
def force_update do
GenServer.call(__MODULE__, :force_update)
end
@doc "Returns stats data"
@spec get_stats() :: %{
domain_count: non_neg_integer(),
status_count: non_neg_integer(),
user_count: non_neg_integer()
}
def get_stats do
%{stats: stats} = GenServer.call(__MODULE__, :get_state)
stats
end
@doc "Returns list peers"
@spec get_peers() :: list(String.t())
def get_peers do
%{peers: peers} = GenServer.call(__MODULE__, :get_state)
peers
end
@spec calculate_stat_data() :: %{
peers: list(),
stats: %{
domain_count: non_neg_integer(),
status_count: non_neg_integer(),
user_count: non_neg_integer()
}
}
def calculate_stat_data do
peers =
from(
u in User,
- select: fragment("distinct split_part(?, '@', 2)", u.nickname),
+ select: fragment("distinct (pg_catalog.string_to_array(?::text, '@'))[2]", u.nickname),
where: u.local != ^true
)
|> Repo.all()
|> Enum.filter(& &1)
domain_count = Enum.count(peers)
status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count)
users_query =
from(u in User,
where: u.is_active == true,
where: u.local == true,
where: not is_nil(u.nickname),
where: not u.invisible
)
user_count = Repo.aggregate(users_query, :count, :id)
%{
peers: peers,
stats: %{
domain_count: domain_count,
status_count: status_count || 0,
user_count: user_count
}
}
end
@spec get_status_visibility_count(String.t() | nil) :: map()
def get_status_visibility_count(instance \\ nil) do
if is_nil(instance) do
CounterCache.get_sum()
else
CounterCache.get_by_instance(instance)
end
end
@impl true
def handle_continue(:calculate_stats, _) do
stats = calculate_stat_data()
unless Pleroma.Config.get(:env) == :test do
Process.send_after(self(), :run_update, @interval)
end
{:noreply, stats}
end
@impl true
def handle_call(:force_update, _from, _state) do
new_stats = calculate_stat_data()
{:reply, new_stats, new_stats}
end
@impl true
def handle_call(:get_state, _from, state) do
{:reply, state, state}
end
@impl true
def handle_info(:run_update, _) do
new_stats = calculate_stat_data()
Process.send_after(self(), :run_update, @interval)
{:noreply, new_stats}
end
end
diff --git a/test/pleroma/stats_test.exs b/test/pleroma/stats_test.exs
index c70603ab9..29c4e3b8e 100644
--- a/test/pleroma/stats_test.exs
+++ b/test/pleroma/stats_test.exs
@@ -1,122 +1,148 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.StatsTest do
use Pleroma.DataCase, async: true
import Pleroma.Factory
alias Pleroma.Stats
alias Pleroma.Web.CommonAPI
+ describe "peers" do
+ test "it counts distinct remote domains and excludes local users" do
+ insert(:user, local: true, nickname: "local")
+ insert(:user, local: false, nickname: "alice@remote.example")
+ insert(:user, local: false, nickname: "bob@remote.example")
+ insert(:user, local: false, nickname: "carol@other.example")
+
+ assert %{peers: peers, stats: %{domain_count: 2}} = Stats.calculate_stat_data()
+ assert Enum.sort(peers) == ["other.example", "remote.example"]
+ end
+
+ test "it ignores nil nicknames" do
+ insert(:user, local: false, nickname: nil)
+
+ assert %{peers: [], stats: %{domain_count: 0}} = Stats.calculate_stat_data()
+ end
+
+ test "it preserves domain case" do
+ insert(:user, local: false, nickname: "alice@Remote.example")
+ insert(:user, local: false, nickname: "bob@remote.example")
+
+ assert %{peers: peers, stats: %{domain_count: 2}} = Stats.calculate_stat_data()
+ assert Enum.sort(peers) == ["Remote.example", "remote.example"]
+ end
+ end
+
describe "user count" do
test "it ignores internal users" do
_user = insert(:user, local: true)
_internal = insert(:user, local: true, nickname: nil)
_internal = Pleroma.Web.ActivityPub.Relay.get_actor()
assert match?(%{stats: %{user_count: 1}}, Stats.calculate_stat_data())
end
end
describe "status visibility sum count" do
test "on new status" do
instance2 = "instance2.tld"
user = insert(:user)
other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user, %{visibility: "public", status: "hey"})
Enum.each(0..1, fn _ ->
CommonAPI.post(user, %{
visibility: "unlisted",
status: "hey"
})
end)
Enum.each(0..2, fn _ ->
CommonAPI.post(user, %{
visibility: "direct",
status: "hey @#{other_user.nickname}"
})
end)
Enum.each(0..3, fn _ ->
CommonAPI.post(user, %{
visibility: "private",
status: "hey"
})
end)
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Stats.get_status_visibility_count()
end
test "on status delete" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{"public" => 1} = Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user)
assert %{"public" => 0} = Stats.get_status_visibility_count()
end
test "on status visibility update" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{"public" => 1, "private" => 0} = Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
assert %{"public" => 0, "private" => 1} = Stats.get_status_visibility_count()
end
test "doesn't count unrelated activities" do
user = insert(:user)
other_user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
_ = CommonAPI.follow(other_user, user)
CommonAPI.favorite(activity.id, other_user)
CommonAPI.repeat(activity.id, other_user)
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
Stats.get_status_visibility_count()
end
end
describe "status visibility by instance count" do
test "single instance" do
local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1)
instance2 = "instance2.tld"
user1 = insert(:user)
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
Enum.each(1..5, fn _ ->
CommonAPI.post(user1, %{
visibility: "unlisted",
status: "hey"
})
end)
Enum.each(1..10, fn _ ->
CommonAPI.post(user1, %{
visibility: "direct",
status: "hey @#{user2.nickname}"
})
end)
Enum.each(1..20, fn _ ->
CommonAPI.post(user2, %{
visibility: "private",
status: "hey"
})
end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
Stats.get_status_visibility_count(instance2)
end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Sat, Oct 31, 11:12 AM (21 h, 14 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1801332
Default Alt Text
(8 KB)

Event Timeline