Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
24 KB
Referenced Files
None
Subscribers
None
diff --git a/changelog.d/instance-nodeinfo.add b/changelog.d/instance-nodeinfo.add
index 2fa222ea3..b98460f7a 100644
--- a/changelog.d/instance-nodeinfo.add
+++ b/changelog.d/instance-nodeinfo.add
@@ -1 +1 @@
-Fetch remote servers' nodeinfo (software and features) in the background and show it on accounts as `pleroma.instance`
+Fetch remote servers' nodeinfo (software and features) in the background and show it on accounts as `pleroma.instance`; `pleroma_ctl instance refresh_nodeinfo` fetches it for all known servers once
diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md
index 88509cf5b..0ab0c5bf4 100644
--- a/docs/administration/CLI_tasks/instance.md
+++ b/docs/administration/CLI_tasks/instance.md
@@ -1,45 +1,65 @@
# Managing instance configuration
{! backend/administration/CLI_tasks/general_cli_task_info.include !}
## Generate a new configuration file
=== "OTP"
```sh
./bin/pleroma_ctl instance gen [option ...]
```
=== "From Source"
```sh
mix pleroma.instance gen [option ...]
```
If any of the options are left unspecified, you will be prompted interactively.
### Options
- `-f`, `--force` - overwrite any output files
- `-o <path>`, `--output <path>` - the output file for the generated configuration
- `--output-psql <path>` - the output file for the generated PostgreSQL setup
- `--domain <domain>` - the domain of your instance
- `--instance-name <instance_name>` - the name of your instance
- `--admin-email <email>` - the email address of the instance admin
- `--notify-email <email>` - email address for notifications
- `--dbhost <hostname>` - the hostname of the PostgreSQL database to use
- `--dbname <database_name>` - the name of the database to use
- `--dbuser <username>` - the user (aka role) to use for the database connection
- `--dbpass <password>` - the password to use for the database connection
- `--rum <Y|N>` - Whether to enable RUM indexes
- `--indexable <Y|N>` - Allow/disallow indexing site by search engines
- `--db-configurable <Y|N>` - Allow/disallow configuring instance from admin part
- `--uploads-dir <path>` - the directory uploads go in when using a local uploader
- `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
- `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
- `--listen-port <port>` - the port the app should listen to, defaults to 4000
- `--strip-uploads-location <Y|N>` - use ExifTool to strip uploads of sensitive location data
- `--read-uploads-description <Y|N>` - use ExifTool to read image descriptions from uploads
- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
- `--skip-release-env` - skip generation the release environment file
- `--release-env-file` - release environment file path
+
+## Fetch the nodeinfo of known servers
+
+Remote servers' nodeinfo (software and features, shown on accounts as
+`pleroma.instance`) is fetched when a post from the server arrives. After
+an upgrade, this fetches it once for every server with remote accounts, in
+the background. Servers fetched in the last day are skipped; nothing
+happens when `:instances_nodeinfo` is disabled.
+
+=== "OTP"
+
+ ```sh
+ ./bin/pleroma_ctl instance refresh_nodeinfo
+ ```
+
+=== "From Source"
+
+ ```sh
+ mix pleroma.instance refresh_nodeinfo
+ ```
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index aebad6984..218ad1005 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -1,155 +1,155 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Pleroma do
@apps [
:restarter,
:ecto,
:ecto_sql,
:postgrex,
:db_connection,
:cachex,
:flake_id,
:swoosh,
:timex,
:fast_html,
:oban,
:logger_backends
]
- @cachex_children ["object", "user", "scrubber", "web_resp"]
+ @cachex_children ["object", "user", "scrubber", "web_resp", "instances"]
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Pleroma.Config.Holder.save_default()
Pleroma.Config.Oban.warn()
Pleroma.Application.limiters_setup()
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
unless System.get_env("DEBUG") do
try do
Logger.remove_backend(:console)
catch
:exit, _ -> :ok
end
end
adapter = Application.get_env(:tesla, :adapter)
apps =
if adapter == Tesla.Adapter.Gun do
[:gun | @apps]
else
[:hackney | @apps]
end
Enum.each(apps, &Application.ensure_all_started/1)
oban_config = [
crontab: [],
repo: Pleroma.Repo,
log: false,
queues: [],
plugins: []
]
children =
[
Pleroma.Repo,
Pleroma.Emoji,
{Pleroma.Config.TransferTask, false},
Pleroma.Web.Endpoint,
{Oban, oban_config},
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
http_children(adapter) ++
search_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
Supervisor.start_link(children ++ cachex_children,
strategy: :one_for_one,
name: Pleroma.Supervisor
)
if Pleroma.Config.get(:env) not in [:test, :benchmark] do
pleroma_rebooted?()
end
end
defp pleroma_rebooted? do
if Restarter.Pleroma.rebooted?() do
:ok
else
Process.sleep(10)
pleroma_rebooted?()
end
end
def load_pleroma do
Application.load(:pleroma)
end
def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do
Keyword.get(options, opt) || shell_prompt(prompt, defval, defname)
end
def shell_prompt(prompt, defval \\ nil, defname \\ nil) do
prompt_message = "#{prompt} [#{defname || defval}] "
input =
if mix_shell?(),
do: Mix.shell().prompt(prompt_message),
else: :io.get_line(prompt_message)
case input do
"\n" ->
case defval do
nil ->
shell_prompt(prompt, defval, defname)
defval ->
defval
end
input ->
String.trim(input)
end
end
def shell_info(message) do
if mix_shell?(),
do: Mix.shell().info(message),
else: IO.puts(message)
end
def shell_error(message) do
if mix_shell?(),
do: Mix.shell().error(message),
else: IO.puts(:stderr, message)
end
@doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)"
def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0)
def escape_sh_path(path) do
~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
end
defp http_children(Tesla.Adapter.Gun) do
Pleroma.Gun.ConnectionPool.children() ++
[{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
end
defp http_children(_), do: []
defp search_children do
case Pleroma.Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
end
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 143af5cdd..0a2d53ce4 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -1,355 +1,378 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Instance do
use Mix.Task
import Mix.Pleroma
alias Pleroma.Config
@shortdoc "Manages Pleroma instance"
@moduledoc File.read!("docs/administration/CLI_tasks/instance.md")
+ # once after an upgrade: the nodeinfo of every server with remote accounts,
+ # fetched in the background (fresh ones and a disabled feature are skipped)
+ def run(["refresh_nodeinfo"]) do
+ import Ecto.Query
+
+ start_pleroma()
+
+ hosts =
+ from(u in Pleroma.User,
+ where: u.local == false,
+ distinct: true,
+ select: fragment("substring(? from '^https\\?://([^/:]+)')", u.ap_id)
+ )
+ |> Pleroma.Repo.all(timeout: :infinity)
+ |> Enum.reject(&is_nil/1)
+
+ Enum.each(hosts, &Pleroma.Instances.Instance.schedule_metadata/1)
+
+ shell_info(
+ "Checked #{length(hosts)} servers; the ones without fresh nodeinfo are fetched in the background."
+ )
+ end
+
def run(["gen" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [
force: :boolean,
output: :string,
output_psql: :string,
domain: :string,
instance_name: :string,
admin_email: :string,
notify_email: :string,
dbhost: :string,
dbname: :string,
dbuser: :string,
dbpass: :string,
rum: :string,
indexable: :string,
db_configurable: :string,
uploads_dir: :string,
static_dir: :string,
listen_ip: :string,
listen_port: :string,
strip_uploads_location: :string,
read_uploads_description: :string,
anonymize_uploads: :string,
dedupe_uploads: :string
],
aliases: [
o: :output,
f: :force
]
)
paths =
[config_path, psql_path] = [
Keyword.get(options, :output, "config/generated_config.exs"),
Keyword.get(options, :output_psql, "config/setup_db.psql")
]
will_overwrite = Enum.filter(paths, &File.exists?/1)
proceed? = Enum.empty?(will_overwrite) or Keyword.get(options, :force, false)
if proceed? do
[domain, port | _] =
String.split(
get_option(
options,
:domain,
"What domain will your instance use? (e.g pleroma.soykaf.com)"
),
":"
) ++ [443]
name =
get_option(
options,
:instance_name,
"What is the name of your instance? (e.g. The Corndog Emporium)",
domain
)
email = get_option(options, :admin_email, "What is your admin email address?")
notify_email =
get_option(
options,
:notify_email,
"What email address do you want to use for sending email notifications?",
email
)
indexable =
get_option(
options,
:indexable,
"Do you want search engines to index your site? (y/n)",
"y"
) === "y"
db_configurable? =
get_option(
options,
:db_configurable,
"Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)",
"n"
) === "y"
dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma")
dbuser =
get_option(
options,
:dbuser,
"What is the user used to connect to your database?",
"pleroma"
)
dbpass =
get_option(
options,
:dbpass,
"What is the password used to connect to your database?",
:crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64),
"autogenerated"
)
rum_enabled =
get_option(
options,
:rum,
"Would you like to use RUM indices?",
"n"
) === "y"
listen_port =
get_option(
options,
:listen_port,
"What port will the app listen to (leave it if you are using the default setup with nginx)?",
4000
)
listen_ip =
get_option(
options,
:listen_ip,
"What ip will the app listen to (leave it if you are using the default setup with nginx)?",
"127.0.0.1"
)
uploads_dir =
get_option(
options,
:uploads_dir,
"What directory should media uploads go in (when using the local uploader)?",
Config.get([Pleroma.Uploaders.Local, :uploads])
)
|> Path.expand()
static_dir =
get_option(
options,
:static_dir,
"What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
Config.get([:instance, :static_dir])
)
|> Path.expand()
{strip_uploads_location_message, strip_uploads_location_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
"y"}
else
{"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
"n"}
end
strip_uploads_location =
get_option(
options,
:strip_uploads_location,
strip_uploads_location_message,
strip_uploads_location_default
) === "y"
{read_uploads_description_message, read_uploads_description_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
"y"}
else
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
"n"}
end
read_uploads_description =
get_option(
options,
:read_uploads_description,
read_uploads_description_message,
read_uploads_description_default
) === "y"
anonymize_uploads =
get_option(
options,
:anonymize_uploads,
"Do you want to anonymize the filenames of uploads? (y/n)",
"n"
) === "y"
dedupe_uploads =
get_option(
options,
:dedupe_uploads,
"Do you want to deduplicate uploaded files? (y/n)",
"n"
) === "y"
Config.put([:instance, :static_dir], static_dir)
secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
jwt_secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
lv_signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
{web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
template_dir = Application.app_dir(:pleroma, "priv") <> "/templates"
result_config =
EEx.eval_file(
template_dir <> "/sample_config.eex",
domain: domain,
port: port,
email: email,
notify_email: notify_email,
name: name,
dbhost: dbhost,
dbname: dbname,
dbuser: dbuser,
dbpass: dbpass,
secret: secret,
jwt_secret: jwt_secret,
signing_salt: signing_salt,
lv_signing_salt: lv_signing_salt,
web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
db_configurable?: db_configurable?,
static_dir: static_dir,
uploads_dir: uploads_dir,
rum_enabled: rum_enabled,
listen_ip: listen_ip,
listen_port: listen_port,
upload_filters:
upload_filters(%{
strip_location: strip_uploads_location,
read_description: read_uploads_description,
anonymize: anonymize_uploads,
dedupe: dedupe_uploads
})
)
result_psql =
EEx.eval_file(
template_dir <> "/sample_psql.eex",
dbname: dbname,
dbuser: dbuser,
dbpass: dbpass,
rum_enabled: rum_enabled
)
config_dir = Path.dirname(config_path)
psql_dir = Path.dirname(psql_path)
# Note: Distros requiring group read (0o750) on those directories should
# pre-create the directories.
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
|> Enum.each(fn dir ->
Pleroma.Backports.mkdir_p!(dir)
File.chmod!(dir, 0o700)
end)
shell_info("Writing config to #{config_path}.")
# Sadly no fchmod(2) equivalent in Elixir…
File.touch!(config_path)
File.chmod!(config_path, 0o640)
File.write(config_path, result_config)
shell_info("Writing the postgres script to #{psql_path}.")
File.write(psql_path, result_psql)
write_robots_txt(static_dir, indexable, template_dir)
shell_info(
"\n All files successfully written! Refer to the installation instructions for your platform for next steps."
)
if db_configurable? do
shell_info(
" Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information."
)
end
else
shell_error(
"The task would have overwritten the following files:\n" <>
Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them."
)
end
end
defp write_robots_txt(static_dir, indexable, template_dir) do
robots_txt =
EEx.eval_file(
template_dir <> "/robots_txt.eex",
indexable: indexable
)
robots_txt_path = Path.join(static_dir, "robots.txt")
if File.exists?(robots_txt_path) do
File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak")
end
File.write(robots_txt_path, robots_txt)
shell_info("Writing #{robots_txt_path}.")
end
defp upload_filters(filters) when is_map(filters) do
enabled_filters =
if filters.strip_location do
[Pleroma.Upload.Filter.Exiftool.StripLocation]
else
[]
end
enabled_filters =
if filters.read_description do
enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
else
enabled_filters
end
enabled_filters =
if filters.anonymize do
enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]
else
enabled_filters
end
enabled_filters =
if filters.dedupe do
enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
else
enabled_filters
end
enabled_filters
end
end
diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs
index 5ecb6e445..c11a90a25 100644
--- a/test/mix/tasks/pleroma/instance_test.exs
+++ b/test/mix/tasks/pleroma/instance_test.exs
@@ -1,107 +1,139 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.InstanceTest do
# Modifies the Application Environment, has to stay synchronous.
use Pleroma.DataCase
+ use Oban.Testing, repo: Pleroma.Repo
+
+ import Pleroma.Factory
setup do
Pleroma.Backports.mkdir_p!(tmp_path())
on_exit(fn ->
File.rm_rf(tmp_path())
static_dir = Pleroma.Config.get([:instance, :static_dir], "test/instance_static/")
if File.exists?(static_dir) do
File.rm_rf(Path.join(static_dir, "robots.txt"))
end
end)
# Is being modified by the mix task.
clear_config([:instance, :static_dir])
:ok
end
@uuid Ecto.UUID.generate()
defp tmp_path do
"/tmp/generated_files/#{@uuid}/"
end
test "running gen" do
mix_task = fn ->
Mix.Tasks.Pleroma.Instance.run([
"gen",
"--output",
tmp_path() <> "generated_config.exs",
"--output-psql",
tmp_path() <> "setup.psql",
"--domain",
"test.pleroma.social",
"--instance-name",
"Pleroma",
"--admin-email",
"admin@example.com",
"--notify-email",
"notify@example.com",
"--dbhost",
"dbhost",
"--dbname",
"dbname",
"--dbuser",
"dbuser",
"--dbpass",
"dbpass",
"--indexable",
"y",
"--db-configurable",
"y",
"--rum",
"y",
"--listen-port",
"4000",
"--listen-ip",
"127.0.0.1",
"--uploads-dir",
"test/uploads",
"--static-dir",
"./test/../test/instance/static/",
"--strip-uploads-location",
"y",
"--read-uploads-description",
"y",
"--dedupe-uploads",
"n",
"--anonymize-uploads",
"n"
])
end
ExUnit.CaptureIO.capture_io(fn ->
mix_task.()
end)
generated_config = File.read!(tmp_path() <> "generated_config.exs")
assert generated_config =~ "host: \"test.pleroma.social\""
assert generated_config =~ "name: \"Pleroma\""
assert generated_config =~ "email: \"admin@example.com\""
assert generated_config =~ "notify_email: \"notify@example.com\""
assert generated_config =~ "hostname: \"dbhost\""
assert generated_config =~ "database: \"dbname\""
assert generated_config =~ "username: \"dbuser\""
assert generated_config =~ "password: \"dbpass\""
assert generated_config =~ "configurable_from_database: true"
assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
assert generated_config =~
"filters: [Pleroma.Upload.Filter.Exiftool.StripLocation, Pleroma.Upload.Filter.Exiftool.ReadDescription]"
assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
end
defp generated_setup_psql do
~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\nCREATE EXTENSION IF NOT EXISTS rum;\n)
end
+
+ describe "refresh_nodeinfo" do
+ setup do
+ Mix.shell(Mix.Shell.Process)
+ on_exit(fn -> Mix.shell(Mix.Shell.IO) end)
+ end
+
+ test "asks for the nodeinfo of every server with remote accounts, unless fresh" do
+ insert(:user, local: false, ap_id: "https://one.example/users/a")
+ insert(:user, local: false, ap_id: "https://one.example/users/b")
+ insert(:user, local: false, ap_id: "https://two.example/users/c")
+ insert(:user, local: false, ap_id: "https://fresh.example/users/d")
+ insert(:user)
+
+ insert(:instance, host: "fresh.example", metadata_updated_at: DateTime.utc_now())
+
+ Mix.Tasks.Pleroma.Instance.run(["refresh_nodeinfo"])
+
+ hosts =
+ [worker: Pleroma.Workers.NodeInfoFetcherWorker]
+ |> all_enqueued()
+ |> Enum.map(& &1.args["host"])
+ |> Enum.sort()
+
+ assert hosts == ["one.example", "two.example"]
+ assert_received {:mix_shell, :info, [message]}
+ assert message =~ "3 servers"
+ end
+ end
end

File Metadata

Mime Type
text/x-diff
Expires
Thu, Oct 29, 11:31 PM (1 d, 16 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1798863
Default Alt Text
(24 KB)

Event Timeline