Page MenuHomePhorge

No OneTemporary

Size
21 KB
Referenced Files
None
Subscribers
None
diff --git a/docs/administration/CLI_tasks/instance.md b/docs/administration/CLI_tasks/instance.md
index 0ab0c5bf4..0832e26a8 100644
--- a/docs/administration/CLI_tasks/instance.md
+++ b/docs/administration/CLI_tasks/instance.md
@@ -1,65 +1,66 @@
# Managing instance configuration
{! backend/administration/CLI_tasks/general_cli_task_info.include !}
## Generate a new configuration file
=== "OTP"
```sh
./bin/pleroma_ctl instance gen [option ...]
```
=== "From Source"
```sh
mix pleroma.instance gen [option ...]
```
If any of the options are left unspecified, you will be prompted interactively.
### Options
- `-f`, `--force` - overwrite any output files
- `-o <path>`, `--output <path>` - the output file for the generated configuration
- `--output-psql <path>` - the output file for the generated PostgreSQL setup
- `--domain <domain>` - the domain of your instance
- `--instance-name <instance_name>` - the name of your instance
- `--admin-email <email>` - the email address of the instance admin
- `--notify-email <email>` - email address for notifications
- `--dbhost <hostname>` - the hostname of the PostgreSQL database to use
- `--dbname <database_name>` - the name of the database to use
- `--dbuser <username>` - the user (aka role) to use for the database connection
- `--dbpass <password>` - the password to use for the database connection
- `--rum <Y|N>` - Whether to enable RUM indexes
- `--indexable <Y|N>` - Allow/disallow indexing site by search engines
- `--db-configurable <Y|N>` - Allow/disallow configuring instance from admin part
- `--uploads-dir <path>` - the directory uploads go in when using a local uploader
- `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
- `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
- `--listen-port <port>` - the port the app should listen to, defaults to 4000
- `--strip-uploads-location <Y|N>` - use ExifTool to strip uploads of sensitive location data
- `--read-uploads-description <Y|N>` - use ExifTool to read image descriptions from uploads
- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
- `--skip-release-env` - skip generation the release environment file
- `--release-env-file` - release environment file path
## Fetch the nodeinfo of known servers
Remote servers' nodeinfo (software and features, shown on accounts as
`pleroma.instance`) is fetched when a post from the server arrives. After
-an upgrade, this fetches it once for every server with remote accounts, in
-the background. Servers fetched in the last day are skipped; nothing
-happens when `:instances_nodeinfo` is disabled.
+an upgrade, this fetches it once, in the background, for every server with
+an account that posted in the last 30 days (`--days <n>` to change that;
+long-dead servers would only time out). Servers fetched in the last day
+are skipped; nothing happens when `:instances_nodeinfo` is disabled.
=== "OTP"
```sh
./bin/pleroma_ctl instance refresh_nodeinfo
```
=== "From Source"
```sh
mix pleroma.instance refresh_nodeinfo
```
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 0a2d53ce4..aa0dd8783 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -1,378 +1,382 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Instance do
use Mix.Task
import Mix.Pleroma
alias Pleroma.Config
@shortdoc "Manages Pleroma instance"
@moduledoc File.read!("docs/administration/CLI_tasks/instance.md")
- # once after an upgrade: the nodeinfo of every server with remote accounts,
- # fetched in the background (fresh ones and a disabled feature are skipped)
- def run(["refresh_nodeinfo"]) do
+ # once after an upgrade: the nodeinfo of the servers whose accounts posted
+ # in the last --days (default 30), fetched in the background; fresh ones
+ # and a disabled feature are skipped. Long-dead servers would only time out.
+ def run(["refresh_nodeinfo" | rest]) do
import Ecto.Query
+ {options, [], []} = OptionParser.parse(rest, strict: [days: :integer])
+ since = NaiveDateTime.add(NaiveDateTime.utc_now(), -Keyword.get(options, :days, 30) * 86_400)
+
start_pleroma()
hosts =
from(u in Pleroma.User,
- where: u.local == false,
+ where: u.local == false and u.last_status_at > ^since,
distinct: true,
select: fragment("substring(? from '^https\\?://([^/:]+)')", u.ap_id)
)
|> Pleroma.Repo.all(timeout: :infinity)
|> Enum.reject(&is_nil/1)
Enum.each(hosts, &Pleroma.Instances.Instance.schedule_metadata/1)
shell_info(
"Checked #{length(hosts)} servers; the ones without fresh nodeinfo are fetched in the background."
)
end
def run(["gen" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [
force: :boolean,
output: :string,
output_psql: :string,
domain: :string,
instance_name: :string,
admin_email: :string,
notify_email: :string,
dbhost: :string,
dbname: :string,
dbuser: :string,
dbpass: :string,
rum: :string,
indexable: :string,
db_configurable: :string,
uploads_dir: :string,
static_dir: :string,
listen_ip: :string,
listen_port: :string,
strip_uploads_location: :string,
read_uploads_description: :string,
anonymize_uploads: :string,
dedupe_uploads: :string
],
aliases: [
o: :output,
f: :force
]
)
paths =
[config_path, psql_path] = [
Keyword.get(options, :output, "config/generated_config.exs"),
Keyword.get(options, :output_psql, "config/setup_db.psql")
]
will_overwrite = Enum.filter(paths, &File.exists?/1)
proceed? = Enum.empty?(will_overwrite) or Keyword.get(options, :force, false)
if proceed? do
[domain, port | _] =
String.split(
get_option(
options,
:domain,
"What domain will your instance use? (e.g pleroma.soykaf.com)"
),
":"
) ++ [443]
name =
get_option(
options,
:instance_name,
"What is the name of your instance? (e.g. The Corndog Emporium)",
domain
)
email = get_option(options, :admin_email, "What is your admin email address?")
notify_email =
get_option(
options,
:notify_email,
"What email address do you want to use for sending email notifications?",
email
)
indexable =
get_option(
options,
:indexable,
"Do you want search engines to index your site? (y/n)",
"y"
) === "y"
db_configurable? =
get_option(
options,
:db_configurable,
"Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)",
"n"
) === "y"
dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma")
dbuser =
get_option(
options,
:dbuser,
"What is the user used to connect to your database?",
"pleroma"
)
dbpass =
get_option(
options,
:dbpass,
"What is the password used to connect to your database?",
:crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64),
"autogenerated"
)
rum_enabled =
get_option(
options,
:rum,
"Would you like to use RUM indices?",
"n"
) === "y"
listen_port =
get_option(
options,
:listen_port,
"What port will the app listen to (leave it if you are using the default setup with nginx)?",
4000
)
listen_ip =
get_option(
options,
:listen_ip,
"What ip will the app listen to (leave it if you are using the default setup with nginx)?",
"127.0.0.1"
)
uploads_dir =
get_option(
options,
:uploads_dir,
"What directory should media uploads go in (when using the local uploader)?",
Config.get([Pleroma.Uploaders.Local, :uploads])
)
|> Path.expand()
static_dir =
get_option(
options,
:static_dir,
"What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
Config.get([:instance, :static_dir])
)
|> Path.expand()
{strip_uploads_location_message, strip_uploads_location_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
"y"}
else
{"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
"n"}
end
strip_uploads_location =
get_option(
options,
:strip_uploads_location,
strip_uploads_location_message,
strip_uploads_location_default
) === "y"
{read_uploads_description_message, read_uploads_description_default} =
if Pleroma.Utils.command_available?("exiftool") do
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
"y"}
else
{"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
"n"}
end
read_uploads_description =
get_option(
options,
:read_uploads_description,
read_uploads_description_message,
read_uploads_description_default
) === "y"
anonymize_uploads =
get_option(
options,
:anonymize_uploads,
"Do you want to anonymize the filenames of uploads? (y/n)",
"n"
) === "y"
dedupe_uploads =
get_option(
options,
:dedupe_uploads,
"Do you want to deduplicate uploaded files? (y/n)",
"n"
) === "y"
Config.put([:instance, :static_dir], static_dir)
secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
jwt_secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
lv_signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
{web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
template_dir = Application.app_dir(:pleroma, "priv") <> "/templates"
result_config =
EEx.eval_file(
template_dir <> "/sample_config.eex",
domain: domain,
port: port,
email: email,
notify_email: notify_email,
name: name,
dbhost: dbhost,
dbname: dbname,
dbuser: dbuser,
dbpass: dbpass,
secret: secret,
jwt_secret: jwt_secret,
signing_salt: signing_salt,
lv_signing_salt: lv_signing_salt,
web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
db_configurable?: db_configurable?,
static_dir: static_dir,
uploads_dir: uploads_dir,
rum_enabled: rum_enabled,
listen_ip: listen_ip,
listen_port: listen_port,
upload_filters:
upload_filters(%{
strip_location: strip_uploads_location,
read_description: read_uploads_description,
anonymize: anonymize_uploads,
dedupe: dedupe_uploads
})
)
result_psql =
EEx.eval_file(
template_dir <> "/sample_psql.eex",
dbname: dbname,
dbuser: dbuser,
dbpass: dbpass,
rum_enabled: rum_enabled
)
config_dir = Path.dirname(config_path)
psql_dir = Path.dirname(psql_path)
# Note: Distros requiring group read (0o750) on those directories should
# pre-create the directories.
[config_dir, psql_dir, static_dir, uploads_dir]
|> Enum.reject(&File.exists?/1)
|> Enum.each(fn dir ->
Pleroma.Backports.mkdir_p!(dir)
File.chmod!(dir, 0o700)
end)
shell_info("Writing config to #{config_path}.")
# Sadly no fchmod(2) equivalent in Elixir…
File.touch!(config_path)
File.chmod!(config_path, 0o640)
File.write(config_path, result_config)
shell_info("Writing the postgres script to #{psql_path}.")
File.write(psql_path, result_psql)
write_robots_txt(static_dir, indexable, template_dir)
shell_info(
"\n All files successfully written! Refer to the installation instructions for your platform for next steps."
)
if db_configurable? do
shell_info(
" Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information."
)
end
else
shell_error(
"The task would have overwritten the following files:\n" <>
Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them."
)
end
end
defp write_robots_txt(static_dir, indexable, template_dir) do
robots_txt =
EEx.eval_file(
template_dir <> "/robots_txt.eex",
indexable: indexable
)
robots_txt_path = Path.join(static_dir, "robots.txt")
if File.exists?(robots_txt_path) do
File.cp!(robots_txt_path, "#{robots_txt_path}.bak")
shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak")
end
File.write(robots_txt_path, robots_txt)
shell_info("Writing #{robots_txt_path}.")
end
defp upload_filters(filters) when is_map(filters) do
enabled_filters =
if filters.strip_location do
[Pleroma.Upload.Filter.Exiftool.StripLocation]
else
[]
end
enabled_filters =
if filters.read_description do
enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
else
enabled_filters
end
enabled_filters =
if filters.anonymize do
enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]
else
enabled_filters
end
enabled_filters =
if filters.dedupe do
enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
else
enabled_filters
end
enabled_filters
end
end
diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs
index c11a90a25..fa3e54991 100644
--- a/test/mix/tasks/pleroma/instance_test.exs
+++ b/test/mix/tasks/pleroma/instance_test.exs
@@ -1,139 +1,162 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.InstanceTest do
# Modifies the Application Environment, has to stay synchronous.
use Pleroma.DataCase
use Oban.Testing, repo: Pleroma.Repo
import Pleroma.Factory
setup do
Pleroma.Backports.mkdir_p!(tmp_path())
on_exit(fn ->
File.rm_rf(tmp_path())
static_dir = Pleroma.Config.get([:instance, :static_dir], "test/instance_static/")
if File.exists?(static_dir) do
File.rm_rf(Path.join(static_dir, "robots.txt"))
end
end)
# Is being modified by the mix task.
clear_config([:instance, :static_dir])
:ok
end
@uuid Ecto.UUID.generate()
defp tmp_path do
"/tmp/generated_files/#{@uuid}/"
end
test "running gen" do
mix_task = fn ->
Mix.Tasks.Pleroma.Instance.run([
"gen",
"--output",
tmp_path() <> "generated_config.exs",
"--output-psql",
tmp_path() <> "setup.psql",
"--domain",
"test.pleroma.social",
"--instance-name",
"Pleroma",
"--admin-email",
"admin@example.com",
"--notify-email",
"notify@example.com",
"--dbhost",
"dbhost",
"--dbname",
"dbname",
"--dbuser",
"dbuser",
"--dbpass",
"dbpass",
"--indexable",
"y",
"--db-configurable",
"y",
"--rum",
"y",
"--listen-port",
"4000",
"--listen-ip",
"127.0.0.1",
"--uploads-dir",
"test/uploads",
"--static-dir",
"./test/../test/instance/static/",
"--strip-uploads-location",
"y",
"--read-uploads-description",
"y",
"--dedupe-uploads",
"n",
"--anonymize-uploads",
"n"
])
end
ExUnit.CaptureIO.capture_io(fn ->
mix_task.()
end)
generated_config = File.read!(tmp_path() <> "generated_config.exs")
assert generated_config =~ "host: \"test.pleroma.social\""
assert generated_config =~ "name: \"Pleroma\""
assert generated_config =~ "email: \"admin@example.com\""
assert generated_config =~ "notify_email: \"notify@example.com\""
assert generated_config =~ "hostname: \"dbhost\""
assert generated_config =~ "database: \"dbname\""
assert generated_config =~ "username: \"dbuser\""
assert generated_config =~ "password: \"dbpass\""
assert generated_config =~ "configurable_from_database: true"
assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
assert generated_config =~
"filters: [Pleroma.Upload.Filter.Exiftool.StripLocation, Pleroma.Upload.Filter.Exiftool.ReadDescription]"
assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
end
defp generated_setup_psql do
~s(CREATE USER dbuser WITH ENCRYPTED PASSWORD 'dbpass';\nCREATE DATABASE dbname OWNER dbuser;\n\\c dbname;\n--Extensions made by ecto.migrate that need superuser access\nCREATE EXTENSION IF NOT EXISTS citext;\nCREATE EXTENSION IF NOT EXISTS pg_trgm;\nCREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";\nCREATE EXTENSION IF NOT EXISTS rum;\n)
end
describe "refresh_nodeinfo" do
setup do
Mix.shell(Mix.Shell.Process)
on_exit(fn -> Mix.shell(Mix.Shell.IO) end)
end
- test "asks for the nodeinfo of every server with remote accounts, unless fresh" do
- insert(:user, local: false, ap_id: "https://one.example/users/a")
- insert(:user, local: false, ap_id: "https://one.example/users/b")
- insert(:user, local: false, ap_id: "https://two.example/users/c")
- insert(:user, local: false, ap_id: "https://fresh.example/users/d")
+ defp remote(ap_id, days_ago) do
+ insert(:user,
+ local: false,
+ ap_id: ap_id,
+ last_status_at: NaiveDateTime.add(NaiveDateTime.utc_now(), -days_ago * 86_400)
+ )
+ end
+
+ defp enqueued_hosts do
+ [worker: Pleroma.Workers.NodeInfoFetcherWorker]
+ |> all_enqueued()
+ |> Enum.map(& &1.args["host"])
+ |> Enum.sort()
+ end
+
+ test "asks for the nodeinfo of servers with recent posts, unless fresh" do
+ remote("https://one.example/users/a", 1)
+ remote("https://one.example/users/b", 2)
+ remote("https://two.example/users/c", 20)
+ remote("https://fresh.example/users/d", 1)
+ # quiet for longer than 30 days
+ remote("https://quiet.example/users/e", 40)
+ insert(:user, local: false, ap_id: "https://never.example/users/f")
insert(:user)
insert(:instance, host: "fresh.example", metadata_updated_at: DateTime.utc_now())
Mix.Tasks.Pleroma.Instance.run(["refresh_nodeinfo"])
- hosts =
- [worker: Pleroma.Workers.NodeInfoFetcherWorker]
- |> all_enqueued()
- |> Enum.map(& &1.args["host"])
- |> Enum.sort()
-
- assert hosts == ["one.example", "two.example"]
+ assert enqueued_hosts() == ["one.example", "two.example"]
assert_received {:mix_shell, :info, [message]}
assert message =~ "3 servers"
end
+
+ test "takes the period from --days" do
+ remote("https://recent.example/users/a", 5)
+ remote("https://quiet.example/users/b", 40)
+
+ Mix.Tasks.Pleroma.Instance.run(["refresh_nodeinfo", "--days", "3"])
+ assert enqueued_hosts() == []
+
+ Mix.Tasks.Pleroma.Instance.run(["refresh_nodeinfo", "--days", "60"])
+ assert enqueued_hosts() == ["quiet.example", "recent.example"]
+ end
end
end

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 9, 1:21 AM (1 d, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1783488
Default Alt Text
(21 KB)

Event Timeline