Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85949041
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
23 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex
index cba94248f..2c6378265 100644
--- a/lib/pleroma/user/backup.ex
+++ b/lib/pleroma/user/backup.ex
@@ -1,258 +1,242 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.Backup do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
import Pleroma.Web.Gettext
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
alias Pleroma.Workers.BackupWorker
schema "backups" do
field(:content_type, :string)
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
def create(user, admin_id \\ nil) do
- with :ok <- validate_email_enabled(),
- :ok <- validate_user_email(user),
- :ok <- validate_limit(user, admin_id),
+ with :ok <- validate_limit(user, admin_id),
{:ok, backup} <- user |> new() |> Repo.insert() do
BackupWorker.process(backup, admin_id)
end
end
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name
}
end
def delete(backup) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
Repo.delete(backup)
end
end
defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
defp validate_limit(user, nil) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = Pleroma.Config.get([__MODULE__, :limit_days])
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
if diff > days do
:ok
else
{:error,
dngettext(
"errors",
"Last export was less than a day ago",
"Last export was less than %{days} days ago",
days,
days: days
)}
end
nil ->
:ok
end
end
- defp validate_email_enabled do
- if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
- :ok
- else
- {:error, dgettext("errors", "Backups require enabled email")}
- end
- end
-
- defp validate_user_email(%User{email: nil}) do
- {:error, dgettext("errors", "Email is required")}
- end
-
- defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
-
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> limit(1)
|> Repo.one()
end
def list(%User{id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> Repo.all()
end
def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
__MODULE__
|> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id)
|> Repo.all()
|> Enum.each(&BackupWorker.delete/1)
end
def get(id), do: Repo.get(__MODULE__, id)
def process(%__MODULE__{} = backup) do
with {:ok, zip_file} <- export(backup),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
|> cast(%{file_size: size, processed: true}, [:file_size, :processed])
|> Repo.update()
end
end
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
def export(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
name = String.trim_trailing(backup.file_name, ".zip")
dir = dir(name)
with :ok <- File.mkdir(dir),
:ok <- actor(dir, backup.user),
:ok <- statuses(dir, backup.user),
:ok <- likes(dir, backup.user),
:ok <- bookmarks(dir, backup.user),
{:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, to_string(zip_path)}
end
end
def dir(name) do
dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
Path.join(dir, name)
end
def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
upload = %Pleroma.Upload{
name: backup.file_name,
tempfile: zip_path,
content_type: backup.content_type,
path: Path.join("backups", backup.file_name)
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
:ok <- File.rm(zip_path) do
{:ok, upload}
end
end
defp actor(dir, user) do
with {:ok, json} <-
UserView.render("user.json", %{user: user})
|> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
|> Jason.encode() do
File.write(Path.join(dir, "actor.json"), json)
end
end
defp write_header(file, name) do
IO.write(
file,
"""
{
"@context": "https://www.w3.org/ns/activitystreams",
"id": "#{name}.json",
"type": "OrderedCollection",
"orderedItems": [
"""
)
end
defp write(query, dir, name, fun) do
path = Path.join(dir, "#{name}.json")
with {:ok, file} <- File.open(path, [:write, :utf8]),
:ok <- write_header(file, name) do
total =
query
|> Pleroma.Repo.chunk_stream(100)
|> Enum.reduce(0, fn i, acc ->
with {:ok, data} <- fun.(i),
{:ok, str} <- Jason.encode(data),
:ok <- IO.write(file, str <> ",\n") do
acc + 1
else
_ -> acc
end
end)
with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do
File.close(file)
end
end
end
defp bookmarks(dir, %{id: user_id} = _user) do
Bookmark
|> where(user_id: ^user_id)
|> join(:inner, [b], activity in assoc(b, :activity))
|> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
|> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
end
defp likes(dir, user) do
user.ap_id
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Like")
|> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
|> write(dir, "likes", fn a -> {:ok, a.object} end)
end
defp statuses(dir, user) do
opts =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:actor_id, user.ap_id)
[
[Pleroma.Constants.as_public(), user.ap_id],
User.following(user),
Pleroma.List.memberships(user)
]
|> Enum.concat()
|> ActivityPub.fetch_activities_query(opts)
|> write(dir, "outbox", fn a ->
with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
{:ok, Map.delete(activity, "@context")}
end
end)
end
end
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 9b763b04b..66c5c3591 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -1,54 +1,70 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1
alias Oban.Job
alias Pleroma.User.Backup
def process(backup, admin_user_id \\ nil) do
%{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
|> new()
|> Oban.insert()
end
def schedule_deletion(backup) do
days = Pleroma.Config.get([Backup, :purge_after_days])
time = 60 * 60 * 24 * days
scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
%{"op" => "delete", "backup_id" => backup.id}
|> new(scheduled_at: scheduled_at)
|> Oban.insert()
end
def delete(backup) do
%{"op" => "delete", "backup_id" => backup.id}
|> new()
|> Oban.insert()
end
def perform(%Job{
args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id}
}) do
with {:ok, %Backup{} = backup} <-
backup_id |> Backup.get() |> Backup.process(),
{:ok, _job} <- schedule_deletion(backup),
:ok <- Backup.remove_outdated(backup),
- {:ok, _} <-
- backup
- |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
- |> Pleroma.Emails.Mailer.deliver() do
+ :ok <- maybe_deliver_email(backup, admin_user_id) do
{:ok, backup}
end
end
def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
case Backup.get(backup_id) do
%Backup{} = backup -> Backup.delete(backup)
nil -> :ok
end
end
+
+ defp has_email?(user) do
+ not is_nil(user.email) and user.email != ""
+ end
+
+ defp maybe_deliver_email(backup, admin_user_id) do
+ has_mailer = Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled])
+ backup = backup |> Pleroma.Repo.preload(:user)
+
+ if has_email?(backup.user) and has_mailer do
+ backup
+ |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
+ |> Pleroma.Emails.Mailer.deliver()
+
+ :ok
+ else
+ :ok
+ end
+ end
end
diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
index b16152876..029bbbf56 100644
--- a/test/pleroma/user/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -1,238 +1,275 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.User.BackupTest do
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.DataCase
import Mock
import Pleroma.Factory
import Swoosh.TestAssertions
alias Pleroma.Bookmark
alias Pleroma.Tests.ObanHelpers
alias Pleroma.User.Backup
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackupWorker
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
clear_config([Pleroma.Emails.Mailer, :enabled], true)
end
- test "it requries enabled email" do
+ test "it does not requrie enabled email" do
clear_config([Pleroma.Emails.Mailer, :enabled], false)
user = insert(:user)
- assert {:error, "Backups require enabled email"} == Backup.create(user)
+ assert {:ok, _} = Backup.create(user)
end
- test "it requries user's email" do
+ test "it does not require user's email" do
user = insert(:user, %{email: nil})
- assert {:error, "Email is required"} == Backup.create(user)
+ assert {:ok, _} = Backup.create(user)
end
test "it creates a backup record and an Oban job" do
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
assert_enqueued(worker: BackupWorker, args: args)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
end
test "it return an error if the export limit is over" do
%{id: user_id} = user = insert(:user)
limit_days = Pleroma.Config.get([Backup, :limit_days])
assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
backup = Backup.get(args["backup_id"])
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
end
test "it process a backup record" do
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
%{id: user_id} = user = insert(:user)
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
assert {:ok, backup} = perform_job(BackupWorker, args)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
assert_enqueued(worker: BackupWorker, args: delete_job_args)
assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
refute Backup.get(backup_id)
email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
assert_email_sent(
to: {user.name, user.email},
html_body: email.html_body
)
end
+ test "it does not send an email if the user does not have an email" do
+ clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ %{id: user_id} = user = insert(:user, %{email: nil})
+
+ assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+ assert {:ok, backup} = perform_job(BackupWorker, args)
+ assert backup.file_size > 0
+ assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+ assert_no_email_sent()
+ end
+
+ test "it does not send an email if mailer is not on" do
+ clear_config([Pleroma.Emails.Mailer, :enabled], false)
+ clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ %{id: user_id} = user = insert(:user)
+
+ assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+ assert {:ok, backup} = perform_job(BackupWorker, args)
+ assert backup.file_size > 0
+ assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+ assert_no_email_sent()
+ end
+
+ test "it does not send an email if the user has an empty email" do
+ clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ %{id: user_id} = user = insert(:user, %{email: ""})
+
+ assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+ assert {:ok, backup} = perform_job(BackupWorker, args)
+ assert backup.file_size > 0
+ assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+ assert_no_email_sent()
+ end
+
test "it removes outdated backups after creating a fresh one" do
clear_config([Backup, :limit_days], -1)
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
user = insert(:user)
assert {:ok, job1} = Backup.create(user)
assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
assert {:ok, job2} = Backup.create(user)
assert Pleroma.Repo.aggregate(Backup, :count) == 2
assert {:ok, backup2} = ObanHelpers.perform(job2)
ObanHelpers.perform_all()
assert [^backup2] = Pleroma.Repo.all(Backup)
end
test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} =
CommonAPI.post(user, %{status: "status1"})
{:ok, %{object: %{data: %{"id" => id2}}} = status2} =
CommonAPI.post(user, %{status: "status2"})
{:ok, %{object: %{data: %{"id" => id3}}} = status3} =
CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
assert %{
"@context" => [
"https://www.w3.org/ns/activitystreams",
"http://localhost:4001/schemas/litepub-0.1.jsonld",
%{"@language" => "und"}
],
"bookmarks" => "bookmarks.json",
"followers" => "http://cofe.io/users/cofe/followers",
"following" => "http://cofe.io/users/cofe/following",
"id" => "http://cofe.io/users/cofe",
"inbox" => "http://cofe.io/users/cofe/inbox",
"likes" => "likes.json",
"name" => "Cofe",
"outbox" => "http://cofe.io/users/cofe/outbox",
"preferredUsername" => "cofe",
"publicKey" => %{
"id" => "http://cofe.io/users/cofe#main-key",
"owner" => "http://cofe.io/users/cofe"
},
"type" => "Person",
"url" => "http://cofe.io/users/cofe"
} = Jason.decode!(json)
assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "outbox.json",
"orderedItems" => [
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status1",
"type" => "Note"
},
"type" => "Create"
},
%{
"object" => %{
"actor" => "http://cofe.io/users/cofe",
"content" => "status2"
}
},
%{
"actor" => "http://cofe.io/users/cofe",
"object" => %{
"content" => "status3"
}
}
],
"totalItems" => 3,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "likes.json",
"orderedItems" => [^id1, ^id2],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
assert %{
"@context" => "https://www.w3.org/ns/activitystreams",
"id" => "bookmarks.json",
"orderedItems" => [^id2, ^id3],
"totalItems" => 2,
"type" => "OrderedCollection"
} = Jason.decode!(json)
:zip.zip_close(zipfile)
File.rm!(path)
end
describe "it uploads and deletes a backup archive" do
setup do
clear_config([Pleroma.Upload, :base_url], "https://s3.amazonaws.com")
clear_config([Pleroma.Uploaders.S3, :bucket], "test_bucket")
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, status1} = CommonAPI.post(user, %{status: "status1"})
{:ok, status2} = CommonAPI.post(user, %{status: "status2"})
{:ok, status3} = CommonAPI.post(user, %{status: "status3"})
CommonAPI.favorite(user, status1.id)
CommonAPI.favorite(user, status2.id)
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.export(backup)
[path: path, backup: backup]
end
test "S3", %{path: path, backup: backup} do
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
clear_config([Pleroma.Uploaders.S3, :streaming_enabled], false)
with_mock ExAws,
request: fn
%{http_method: :put} -> {:ok, :ok}
%{http_method: :delete} -> {:ok, %{status_code: 204}}
end do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
end
test "Local", %{path: path, backup: backup} do
clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
assert {:ok, _backup} = Backup.delete(backup)
end
end
end
diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
index 3ee660a05..ba17636da 100644
--- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
@@ -1,85 +1,105 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
use Pleroma.Web.ConnCase
alias Pleroma.User.Backup
alias Pleroma.Web.PleromaAPI.BackupView
setup do
clear_config([Pleroma.Upload, :uploader])
clear_config([Backup, :limit_days])
oauth_access(["read:accounts"])
end
test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
backup = Backup.get(backup_id)
response =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] = response
assert url == BackupView.download_url(backup)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
end
test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
assert [
%{
"content_type" => "application/zip",
"url" => url,
"file_size" => 0,
"processed" => false,
"inserted_at" => _
}
] =
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
Pleroma.Tests.ObanHelpers.perform_all()
assert [
%{
"url" => ^url,
"processed" => true
}
] =
conn
|> get("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(:ok)
days = Pleroma.Config.get([Backup, :limit_days])
assert %{"error" => "Last export was less than #{days} days ago"} ==
conn
|> post("/api/v1/pleroma/backups")
|> json_response_and_validate_schema(400)
end
+
+ test "Backup without email address" do
+ user = Pleroma.Factory.insert(:user, email: nil)
+ %{conn: conn} = oauth_access(["read:accounts"], user: user)
+
+ assert is_nil(user.email)
+
+ assert [
+ %{
+ "content_type" => "application/zip",
+ "url" => _url,
+ "file_size" => 0,
+ "processed" => false,
+ "inserted_at" => _
+ }
+ ] =
+ conn
+ |> post("/api/v1/pleroma/backups")
+ |> json_response_and_validate_schema(:ok)
+ end
end
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Tue, Nov 17, 5:31 PM (12 h, 5 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1839691
Default Alt Text
(23 KB)
Attached To
Mode
rPUBE pleroma-upstream
Attached
Detach File
Event Timeline