No OneTemporary
Actions

Authored By

Unknown

Size

20 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/coveralls.json b/coveralls.json
	new file mode 100644
	index 000000000..75e845ade
	--- /dev/null
	+++ b/coveralls.json
	@@ -0,0 +1,6 @@
	+{
	+ "skip_files": [
	+ "test/support",
	+ "lib/mix/tasks/pleroma/benchmark.ex"
	+ ]
	+}
	\ No newline at end of file
	diff --git a/docs/administration/CLI_tasks/emoji.md b/docs/administration/CLI_tasks/emoji.md
	index efec8222c..3d524a52b 100644
	--- a/docs/administration/CLI_tasks/emoji.md
	+++ b/docs/administration/CLI_tasks/emoji.md
	@@ -1,46 +1,46 @@
	# Managing emoji packs

	{! backend/administration/CLI_tasks/general_cli_task_info.include !}

	## Lists emoji packs and metadata specified in the manifest

	```sh tab="OTP"
	./bin/pleroma_ctl emoji ls-packs [option ...]
	```

	```sh tab="From Source"
	mix pleroma.emoji ls-packs [option ...]
	```


	### Options
	- `-m, --manifest PATH/URL` - path to a custom manifest, it can either be an URL starting with `http`, in that case the manifest will be fetched from that address, or a local path

	## Fetch, verify and install the specified packs from the manifest into `STATIC-DIR/emoji/PACK-NAME`

	```sh tab="OTP"
	./bin/pleroma_ctl emoji get-packs [option ...] <pack ...>
	```

	```sh tab="From Source"
	mix pleroma.emoji get-packs [option ...] <pack ...>
	```

	### Options
	- `-m, --manifest PATH/URL` - same as [`ls-packs`](#ls-packs)

	## Create a new manifest entry and a file list from the specified remote pack file

	```sh tab="OTP"
	./bin/pleroma_ctl emoji gen-pack PACK-URL
	```

	```sh tab="From Source"
	mix pleroma.emoji gen-pack PACK-URL
	```

	-Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you.
	+Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you.

	- The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, replacing the old pack with the same name if it was in the file previously.
	+ The manifest entry will either be written to a newly created `pack_name.json` file (pack name is asked in questions) or appended to the existing one, replacing the old pack with the same name if it was in the file previously.

	The file list will be written to the file specified previously, replacing that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).
	diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
	index 429d763c7..cdffa88b2 100644
	--- a/lib/mix/tasks/pleroma/emoji.ex
	+++ b/lib/mix/tasks/pleroma/emoji.ex
	@@ -1,253 +1,271 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Mix.Tasks.Pleroma.Emoji do
	use Mix.Task
	import Mix.Pleroma

	@shortdoc "Manages emoji packs"
	@moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")

	def run(["ls-packs" \| args]) do
	start_pleroma()

	{options, [], []} = parse_global_opts(args)

	- manifest =
	- fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
	+ url_or_path = options[:manifest] \|\| default_manifest()
	+ manifest = fetch_manifest(url_or_path)

	Enum.each(manifest, fn {name, info} ->
	to_print = [
	{"Name", name},
	{"Homepage", info["homepage"]},
	{"Description", info["description"]},
	{"License", info["license"]},
	{"Source", info["src"]}
	]

	for {param, value} <- to_print do
	IO.puts(IO.ANSI.format([:bright, param, :normal, ": ", value]))
	end

	# A newline
	IO.puts("")
	end)
	end

	def run(["get-packs" \| args]) do
	start_pleroma()

	{options, pack_names, []} = parse_global_opts(args)

	- manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
	+ url_or_path = options[:manifest] \|\| default_manifest()

	- manifest = fetch_manifest(manifest_url)
	+ manifest = fetch_manifest(url_or_path)

	for pack_name <- pack_names do
	if Map.has_key?(manifest, pack_name) do
	pack = manifest[pack_name]
	src_url = pack["src"]

	IO.puts(
	IO.ANSI.format([
	"Downloading ",
	:bright,
	pack_name,
	:normal,
	" from ",
	:underline,
	src_url
	])
	)

	binary_archive = Tesla.get!(client(), src_url).body
	archive_sha = :crypto.hash(:sha256, binary_archive) \|> Base.encode16()

	sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]

	if archive_sha == String.upcase(pack["src_sha256"]) do
	IO.puts(IO.ANSI.format(sha_status_text ++ [:green, "OK"]))
	else
	IO.puts(IO.ANSI.format(sha_status_text ++ [:red, "BAD"]))

	raise "Bad SHA256 for #{pack_name}"
	end

	# The url specified in files should be in the same directory
	- files_url = Path.join(Path.dirname(manifest_url), pack["files"])
	+ files_url =
	+ url_or_path
	+ \|> Path.dirname()
	+ \|> Path.join(pack["files"])

	IO.puts(
	IO.ANSI.format([
	"Fetching the file list for ",
	:bright,
	pack_name,
	:normal,
	" from ",
	:underline,
	files_url
	])
	)

	files = Tesla.get!(client(), files_url).body \|> Jason.decode!()

	IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))

	pack_path =
	Path.join([
	Pleroma.Config.get!([:instance, :static_dir]),
	"emoji",
	pack_name
	])

	files_to_unzip =
	Enum.map(
	files,
	fn {_, f} -> to_charlist(f) end
	)

	{:ok, _} =
	:zip.unzip(binary_archive,
	cwd: pack_path,
	file_list: files_to_unzip
	)

	IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))

	pack_json = %{
	pack: %{
	"license" => pack["license"],
	"homepage" => pack["homepage"],
	"description" => pack["description"],
	"fallback-src" => pack["src"],
	"fallback-src-sha256" => pack["src_sha256"],
	"share-files" => true
	},
	files: files
	}

	File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true))
	else
	IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
	end
	end
	end

	- def run(["gen-pack", src]) do
	+ def run(["gen-pack" \| args]) do
	start_pleroma()

	- proposed_name = Path.basename(src) \|> Path.rootname()
	- name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
	- # If there's no name, use the default one
	- name = if String.length(name) > 0, do: name, else: proposed_name
	+ {opts, [src], []} =
	+ OptionParser.parse(
	+ args,
	+ strict: [
	+ name: :string,
	+ license: :string,
	+ homepage: :string,
	+ description: :string,
	+ files: :string,
	+ extensions: :string
	+ ]
	+ )

	- license = String.trim(IO.gets("License: "))
	- homepage = String.trim(IO.gets("Homepage: "))
	- description = String.trim(IO.gets("Description: "))
	+ proposed_name = Path.basename(src) \|> Path.rootname()
	+ name = get_option(opts, :name, "Pack name:", proposed_name)
	+ license = get_option(opts, :license, "License:")
	+ homepage = get_option(opts, :homepage, "Homepage:")
	+ description = get_option(opts, :description, "Description:")

	- proposed_files_name = "#{name}.json"
	- files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
	- files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
	+ proposed_files_name = "#{name}_files.json"
	+ files_name = get_option(opts, :files, "Save file list to:", proposed_files_name)

	default_exts = [".png", ".gif"]
	- default_exts_str = Enum.join(default_exts, " ")

	- exts =
	- String.trim(
	- IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
	+ custom_exts =
	+ get_option(
	+ opts,
	+ :extensions,
	+ "Emoji file extensions (separated with spaces):",
	+ Enum.join(default_exts, " ")
	)
	+ \|> String.split(" ", trim: true)

	exts =
	- if String.length(exts) > 0 do
	- String.split(exts, " ")
	- \|> Enum.filter(fn e -> e \|> String.trim() \|> String.length() > 0 end)
	- else
	+ if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do
	default_exts
	+ else
	+ custom_exts
	end

	+ IO.puts("Using #{Enum.join(exts, " ")} extensions")
	+
	IO.puts("Downloading the pack and generating SHA256")

	binary_archive = Tesla.get!(client(), src).body
	archive_sha = :crypto.hash(:sha256, binary_archive) \|> Base.encode16()

	IO.puts("SHA256 is #{archive_sha}")

	pack_json = %{
	name => %{
	license: license,
	homepage: homepage,
	description: description,
	src: src,
	src_sha256: archive_sha,
	files: files_name
	}
	}

	tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")

	{:ok, _} = :zip.unzip(binary_archive, cwd: String.to_charlist(tmp_pack_dir))

	emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)

	File.write!(files_name, Jason.encode!(emoji_map, pretty: true))

	IO.puts("""

	#{files_name} has been created and contains the list of all found emojis in the pack.
	- Please review the files in the remove those not needed.
	+ Please review the files in the pack and remove those not needed.
	""")

	- if File.exists?("index.json") do
	- existing_data = File.read!("index.json") \|> Jason.decode!()
	+ pack_file = "#{name}.json"
	+
	+ if File.exists?(pack_file) do
	+ existing_data = File.read!(pack_file) \|> Jason.decode!()

	File.write!(
	- "index.json",
	+ pack_file,
	Jason.encode!(
	Map.merge(
	existing_data,
	pack_json
	),
	pretty: true
	)
	)

	- IO.puts("index.json file has been update with the #{name} pack")
	+ IO.puts("#{pack_file} has been updated with the #{name} pack")
	else
	- File.write!("index.json", Jason.encode!(pack_json, pretty: true))
	+ File.write!(pack_file, Jason.encode!(pack_json, pretty: true))

	- IO.puts("index.json has been created with the #{name} pack")
	+ IO.puts("#{pack_file} has been created with the #{name} pack")
	end
	end

	defp fetch_manifest(from) do
	Jason.decode!(
	if String.starts_with?(from, "http") do
	Tesla.get!(client(), from).body
	else
	File.read!(from)
	end
	)
	end

	defp parse_global_opts(args) do
	OptionParser.parse(
	args,
	strict: [
	manifest: :string
	],
	aliases: [
	m: :manifest
	]
	)
	end

	defp client do
	middleware = [
	{Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
	]

	Tesla.client(middleware)
	end

	defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
	end
	diff --git a/test/fixtures/emoji/packs/blank.png.zip b/test/fixtures/emoji/packs/blank.png.zip
	new file mode 100644
	index 000000000..651daf127
	Binary files /dev/null and b/test/fixtures/emoji/packs/blank.png.zip differ
	diff --git a/test/fixtures/emoji/packs/default-manifest.json b/test/fixtures/emoji/packs/default-manifest.json
	new file mode 100644
	index 000000000..c8433808d
	--- /dev/null
	+++ b/test/fixtures/emoji/packs/default-manifest.json
	@@ -0,0 +1,10 @@
	+{
	+ "finmoji": {
	+ "license": "CC BY-NC-ND 4.0",
	+ "homepage": "https://finland.fi/emoji/",
	+ "description": "Finland is the first country in the world to publish its own set of country themed emojis. The Finland emoji collection contains 56 tongue-in-cheek emotions, which were created to explain some hard-to-describe Finnish emotions, Finnish words and customs.",
	+ "src": "https://finland.fi/wp-content/uploads/2017/06/finland-emojis.zip",
	+ "src_sha256": "384025A1AC6314473863A11AC7AB38A12C01B851A3F82359B89B4D4211D3291D",
	+ "files": "finmoji.json"
	+ }
	+}
	\ No newline at end of file
	diff --git a/test/fixtures/emoji/packs/finmoji.json b/test/fixtures/emoji/packs/finmoji.json
	new file mode 100644
	index 000000000..279770998
	--- /dev/null
	+++ b/test/fixtures/emoji/packs/finmoji.json
	@@ -0,0 +1,3 @@
	+{
	+ "blank": "blank.png"
	+}
	\ No newline at end of file
	diff --git a/test/fixtures/emoji/packs/manifest.json b/test/fixtures/emoji/packs/manifest.json
	new file mode 100644
	index 000000000..2d51a459b
	--- /dev/null
	+++ b/test/fixtures/emoji/packs/manifest.json
	@@ -0,0 +1,10 @@
	+{
	+ "blobs.gg": {
	+ "src_sha256": "3a12f3a181678d5b3584a62095411b0d60a335118135910d879920f8ade5a57f",
	+ "src": "https://git.pleroma.social/pleroma/emoji-index/raw/master/packs/blobs_gg.zip",
	+ "license": "Apache 2.0",
	+ "homepage": "https://blobs.gg",
	+ "files": "blobs_gg.json",
	+ "description": "Blob Emoji from blobs.gg repacked as apng"
	+ }
	+}
	\ No newline at end of file
	diff --git a/test/tasks/emoji_test.exs b/test/tasks/emoji_test.exs
	new file mode 100644
	index 000000000..f2930652a
	--- /dev/null
	+++ b/test/tasks/emoji_test.exs
	@@ -0,0 +1,226 @@
	+defmodule Mix.Tasks.Pleroma.EmojiTest do
	+ use ExUnit.Case, async: true
	+
	+ import ExUnit.CaptureIO
	+ import Tesla.Mock
	+
	+ alias Mix.Tasks.Pleroma.Emoji
	+
	+ describe "ls-packs" do
	+ test "with default manifest as url" do
	+ mock(fn
	+ %{
	+ method: :get,
	+ url: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/default-manifest.json")
	+ }
	+ end)
	+
	+ capture_io(fn -> Emoji.run(["ls-packs"]) end) =~
	+ "https://finland.fi/wp-content/uploads/2017/06/finland-emojis.zip"
	+ end
	+
	+ test "with passed manifest as file" do
	+ capture_io(fn ->
	+ Emoji.run(["ls-packs", "-m", "test/fixtures/emoji/packs/manifest.json"])
	+ end) =~ "https://git.pleroma.social/pleroma/emoji-index/raw/master/packs/blobs_gg.zip"
	+ end
	+ end
	+
	+ describe "get-packs" do
	+ test "download pack from default manifest" do
	+ mock(fn
	+ %{
	+ method: :get,
	+ url: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/default-manifest.json")
	+ }
	+
	+ %{
	+ method: :get,
	+ url: "https://finland.fi/wp-content/uploads/2017/06/finland-emojis.zip"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/blank.png.zip")
	+ }
	+
	+ %{
	+ method: :get,
	+ url: "https://git.pleroma.social/pleroma/emoji-index/raw/master/finmoji.json"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/finmoji.json")
	+ }
	+ end)
	+
	+ assert capture_io(fn -> Emoji.run(["get-packs", "finmoji"]) end) =~ "Writing pack.json for"
	+
	+ emoji_path =
	+ Path.join(
	+ Pleroma.Config.get!([:instance, :static_dir]),
	+ "emoji"
	+ )
	+
	+ assert File.exists?(Path.join([emoji_path, "finmoji", "pack.json"]))
	+ on_exit(fn -> File.rm_rf!("test/instance_static/emoji/finmoji") end)
	+ end
	+
	+ test "pack not found" do
	+ mock(fn
	+ %{
	+ method: :get,
	+ url: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/default-manifest.json")
	+ }
	+ end)
	+
	+ assert capture_io(fn -> Emoji.run(["get-packs", "not_found"]) end) =~
	+ "No pack named \"not_found\" found"
	+ end
	+
	+ test "raise on bad sha256" do
	+ mock(fn
	+ %{
	+ method: :get,
	+ url: "https://git.pleroma.social/pleroma/emoji-index/raw/master/packs/blobs_gg.zip"
	+ } ->
	+ %Tesla.Env{
	+ status: 200,
	+ body: File.read!("test/fixtures/emoji/packs/blank.png.zip")
	+ }
	+ end)
	+
	+ assert_raise RuntimeError, ~r/^Bad SHA256 for blobs.gg/, fn ->
	+ capture_io(fn ->
	+ Emoji.run(["get-packs", "blobs.gg", "-m", "test/fixtures/emoji/packs/manifest.json"])
	+ end)
	+ end
	+ end
	+ end
	+
	+ describe "gen-pack" do
	+ setup do
	+ url = "https://finland.fi/wp-content/uploads/2017/06/finland-emojis.zip"
	+
	+ mock(fn %{
	+ method: :get,
	+ url: ^url
	+ } ->
	+ %Tesla.Env{status: 200, body: File.read!("test/fixtures/emoji/packs/blank.png.zip")}
	+ end)
	+
	+ {:ok, url: url}
	+ end
	+
	+ test "with default extensions", %{url: url} do
	+ name = "pack1"
	+ pack_json = "#{name}.json"
	+ files_json = "#{name}_file.json"
	+ refute File.exists?(pack_json)
	+ refute File.exists?(files_json)
	+
	+ captured =
	+ capture_io(fn ->
	+ Emoji.run([
	+ "gen-pack",
	+ url,
	+ "--name",
	+ name,
	+ "--license",
	+ "license",
	+ "--homepage",
	+ "homepage",
	+ "--description",
	+ "description",
	+ "--files",
	+ files_json,
	+ "--extensions",
	+ ".png .gif"
	+ ])
	+ end)
	+
	+ assert captured =~ "#{pack_json} has been created with the pack1 pack"
	+ assert captured =~ "Using .png .gif extensions"
	+
	+ assert File.exists?(pack_json)
	+ assert File.exists?(files_json)
	+
	+ on_exit(fn ->
	+ File.rm_rf!(pack_json)
	+ File.rm_rf!(files_json)
	+ end)
	+ end
	+
	+ test "with custom extensions and update existing files", %{url: url} do
	+ name = "pack2"
	+ pack_json = "#{name}.json"
	+ files_json = "#{name}_file.json"
	+ refute File.exists?(pack_json)
	+ refute File.exists?(files_json)
	+
	+ captured =
	+ capture_io(fn ->
	+ Emoji.run([
	+ "gen-pack",
	+ url,
	+ "--name",
	+ name,
	+ "--license",
	+ "license",
	+ "--homepage",
	+ "homepage",
	+ "--description",
	+ "description",
	+ "--files",
	+ files_json,
	+ "--extensions",
	+ " .png .gif .jpeg "
	+ ])
	+ end)
	+
	+ assert captured =~ "#{pack_json} has been created with the pack2 pack"
	+ assert captured =~ "Using .png .gif .jpeg extensions"
	+
	+ assert File.exists?(pack_json)
	+ assert File.exists?(files_json)
	+
	+ captured =
	+ capture_io(fn ->
	+ Emoji.run([
	+ "gen-pack",
	+ url,
	+ "--name",
	+ name,
	+ "--license",
	+ "license",
	+ "--homepage",
	+ "homepage",
	+ "--description",
	+ "description",
	+ "--files",
	+ files_json,
	+ "--extensions",
	+ " .png .gif .jpeg "
	+ ])
	+ end)
	+
	+ assert captured =~ "#{pack_json} has been updated with the pack2 pack"
	+
	+ on_exit(fn ->
	+ File.rm_rf!(pack_json)
	+ File.rm_rf!(files_json)
	+ end)
	+ end
	+ end
	+end

File Metadata

Mime Type: text/x-diff
Expires: Thu, Oct 29, 10:36 PM (1 d, 16 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1798784
Default Alt Text: (20 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions