No OneTemporary
Actions

Size

20 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
	index c2225af7d..4f4099456 100644
	--- a/lib/mix/tasks/pleroma/emoji.ex
	+++ b/lib/mix/tasks/pleroma/emoji.ex
	@@ -1,301 +1,302 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Mix.Tasks.Pleroma.Emoji do
	use Mix.Task

	@shortdoc "Manages emoji packs"
	@moduledoc """
	Manages emoji packs

	## ls-packs

	mix pleroma.emoji ls-packs [OPTION...]

	Lists the emoji packs and metadata specified in the manifest.

	### Options

	- `-m, --manifest PATH/URL` - path to a custom manifest, it can
	either be an URL starting with `http`, in that case the
	manifest will be fetched from that address, or a local path

	## get-packs

	mix pleroma.emoji get-packs [OPTION...] PACKS

	Fetches, verifies and installs the specified PACKS from the
	- manifest into the `STATIC-DIR/emoji/PACK-NAME`
	+ manifest into the `/priv/static/emoji-packs/PACK-NAME`

	### Options

	- `-m, --manifest PATH/URL` - same as ls-packs

	## gen-pack

	mix pleroma.emoji gen-pack PACK-URL

	Creates a new manifest entry and a file list from the specified
	remote pack file. Currently, only .zip archives are recognized
	as remote pack files and packs are therefore assumed to be zip
	archives. This command is intended to run interactively and will
	first ask you some basic questions about the pack, then download
	the remote file and generate an SHA256 checksum for it, then
	generate an emoji file list for you.

	The manifest entry will either be written to a newly created
	`index.json` file or appended to the existing one, replacing
	the old pack with the same name if it was in the file previously.

	The file list will be written to the file specified previously,
	replacing that file. You _should_ check that the file list doesn't
	contain anything you don't need in the pack, that is, anything that is
	not an emoji (the whole pack is downloaded, but only emoji files
	are extracted).
	"""

	def run(["ls-packs" \| args]) do
	Application.ensure_all_started(:hackney)

	{options, [], []} = parse_global_opts(args)

	manifest =
	fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())

	Enum.each(manifest, fn {name, info} ->
	to_print = [
	{"Name", name},
	{"Homepage", info["homepage"]},
	{"Description", info["description"]},
	{"License", info["license"]},
	{"Source", info["src"]}
	]

	for {param, value} <- to_print do
	IO.puts(IO.ANSI.format([:bright, param, :normal, ": ", value]))
	end

	# A newline
	IO.puts("")
	end)
	end

	def run(["get-packs" \| args]) do
	Application.ensure_all_started(:hackney)

	{options, pack_names, []} = parse_global_opts(args)

	manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()

	manifest = fetch_manifest(manifest_url)

	for pack_name <- pack_names do
	if Map.has_key?(manifest, pack_name) do
	pack = manifest[pack_name]
	src_url = pack["src"]

	IO.puts(
	IO.ANSI.format([
	"Downloading ",
	:bright,
	pack_name,
	:normal,
	" from ",
	:underline,
	src_url
	])
	)

	binary_archive = Tesla.get!(client(), src_url).body
	archive_sha = :crypto.hash(:sha256, binary_archive) \|> Base.encode16()

	sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]

	if archive_sha == String.upcase(pack["src_sha256"]) do
	IO.puts(IO.ANSI.format(sha_status_text ++ [:green, "OK"]))
	else
	IO.puts(IO.ANSI.format(sha_status_text ++ [:red, "BAD"]))

	raise "Bad SHA256 for #{pack_name}"
	end

	# The url specified in files should be in the same directory
	files_url = Path.join(Path.dirname(manifest_url), pack["files"])

	IO.puts(
	IO.ANSI.format([
	"Fetching the file list for ",
	:bright,
	pack_name,
	:normal,
	" from ",
	:underline,
	files_url
	])
	)

	files = Tesla.get!(client(), files_url).body \|> Jason.decode!()

	IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))

	pack_path =
	Path.join([
	- Pleroma.Config.get!([:instance, :static_dir]),
	- "emoji",
	+ :code.priv_dir(:pleroma),
	+ "static",
	+ "emoji-packs"
	pack_name
	])

	files_to_unzip =
	Enum.map(
	files,
	fn {_, f} -> to_charlist(f) end
	)

	{:ok, _} =
	:zip.unzip(binary_archive,
	cwd: pack_path,
	file_list: files_to_unzip
	)

	IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))

	emoji_txt_str =
	Enum.map(
	files,
	fn {shortcode, path} ->
	emojo_path = Path.join("/emoji/#{pack_name}", path)
	"#{shortcode}, #{emojo_path}"
	end
	)
	\|> Enum.join("\n")

	File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
	else
	IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
	end
	end
	end

	def run(["gen-pack", src]) do
	Application.ensure_all_started(:hackney)

	proposed_name = Path.basename(src) \|> Path.rootname()
	name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
	# If there's no name, use the default one
	name = if String.length(name) > 0, do: name, else: proposed_name

	license = String.trim(IO.gets("License: "))
	homepage = String.trim(IO.gets("Homepage: "))
	description = String.trim(IO.gets("Description: "))

	proposed_files_name = "#{name}.json"
	files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
	files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name

	default_exts = [".png", ".gif"]
	default_exts_str = Enum.join(default_exts, " ")

	exts =
	String.trim(
	IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
	)

	exts =
	if String.length(exts) > 0 do
	String.split(exts, " ")
	\|> Enum.filter(fn e -> e \|> String.trim() \|> String.length() > 0 end)
	else
	default_exts
	end

	IO.puts("Downloading the pack and generating SHA256")

	binary_archive = Tesla.get!(client(), src).body
	archive_sha = :crypto.hash(:sha256, binary_archive) \|> Base.encode16()

	IO.puts("SHA256 is #{archive_sha}")

	pack_json = %{
	name => %{
	license: license,
	homepage: homepage,
	description: description,
	src: src,
	src_sha256: archive_sha,
	files: files_name
	}
	}

	tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")

	{:ok, _} =
	:zip.unzip(
	binary_archive,
	cwd: tmp_pack_dir
	)

	emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)

	File.write!(files_name, Jason.encode!(emoji_map, pretty: true))

	IO.puts("""

	#{files_name} has been created and contains the list of all found emojis in the pack.
	Please review the files in the remove those not needed.
	""")

	if File.exists?("index.json") do
	existing_data = File.read!("index.json") \|> Jason.decode!()

	File.write!(
	"index.json",
	Jason.encode!(
	Map.merge(
	existing_data,
	pack_json
	),
	pretty: true
	)
	)

	IO.puts("index.json file has been update with the #{name} pack")
	else
	File.write!("index.json", Jason.encode!(pack_json, pretty: true))

	IO.puts("index.json has been created with the #{name} pack")
	end
	end

	defp fetch_manifest(from) do
	Jason.decode!(
	if String.starts_with?(from, "http") do
	Tesla.get!(client(), from).body
	else
	File.read!(from)
	end
	)
	end

	defp parse_global_opts(args) do
	OptionParser.parse(
	args,
	strict: [
	manifest: :string
	],
	aliases: [
	m: :manifest
	]
	)
	end

	defp client do
	middleware = [
	{Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
	]

	Tesla.client(middleware)
	end

	defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
	end
	diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex
	index 66e20f0e4..f6ebf4897 100644
	--- a/lib/pleroma/emoji.ex
	+++ b/lib/pleroma/emoji.ex
	@@ -1,263 +1,264 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Emoji do
	@moduledoc """
	The emojis are loaded from:

	* emoji packs in INSTANCE-DIR/emoji
	* the files: `config/emoji.txt` and `config/custom_emoji.txt`
	* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder

	This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
	"""
	use GenServer

	require Logger

	@type pattern :: Regex.t() \| module() \| String.t()
	@type patterns :: pattern() \| [pattern()]
	@type group_patterns :: keyword(patterns())

	@ets __MODULE__.Ets
	@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]

	@doc false
	def start_link(_) do
	GenServer.start_link(__MODULE__, [], name: __MODULE__)
	end

	@doc "Reloads the emojis from disk."
	@spec reload() :: :ok
	def reload do
	GenServer.call(__MODULE__, :reload)
	end

	@doc "Returns the path of the emoji `name`."
	@spec get(String.t()) :: String.t() \| nil
	def get(name) do
	case :ets.lookup(@ets, name) do
	[{_, path}] -> path
	_ -> nil
	end
	end

	@doc "Returns all the emojos!!"
	@spec get_all() :: [{String.t(), String.t()}, ...]
	def get_all do
	:ets.tab2list(@ets)
	end

	@doc false
	def init(_) do
	@ets = :ets.new(@ets, @ets_options)
	GenServer.cast(self(), :reload)
	{:ok, nil}
	end

	@doc false
	def handle_cast(:reload, state) do
	load()
	{:noreply, state}
	end

	@doc false
	def handle_call(:reload, _from, state) do
	load()
	{:reply, :ok, state}
	end

	@doc false
	def terminate(_, _) do
	:ok
	end

	@doc false
	def code_change(_old_vsn, state, _extra) do
	load()
	{:ok, state}
	end

	defp load do
	emoji_dir_path =
	- Path.join(
	- Pleroma.Config.get!([:instance, :static_dir]),
	- "emoji"
	- )
	+ Path.join([
	+ :code.priv_dir(:pleroma),
	+ "static",
	+ "emoji-packs"
	+ ])

	emoji_groups = Pleroma.Config.get([:emoji, :groups])

	case File.ls(emoji_dir_path) do
	{:error, :enoent} ->
	# The custom emoji directory doesn't exist,
	# don't do anything
	nil

	{:error, e} ->
	# There was some other error
	Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")

	{:ok, results} ->
	grouped =
	Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end)

	packs = grouped[true] \|\| []
	files = grouped[false] \|\| []

	# Print the packs we've found
	Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")

	if not Enum.empty?(files) do
	Logger.warn(
	"Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
	Enum.join(files, ", ")
	}"
	)
	end

	emojis =
	Enum.flat_map(
	packs,
	fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
	)

	true = :ets.insert(@ets, emojis)
	end

	# Compat thing for old custom emoji handling & default emoji,
	# it should run even if there are no emoji packs
	shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])

	emojis =
	(load_from_file("config/emoji.txt", emoji_groups) ++
	load_from_file("config/custom_emoji.txt", emoji_groups) ++
	load_from_globs(shortcode_globs, emoji_groups))
	\|> Enum.reject(fn value -> value == nil end)

	true = :ets.insert(@ets, emojis)

	:ok
	end

	defp load_pack(pack_dir, emoji_groups) do
	pack_name = Path.basename(pack_dir)

	emoji_txt = Path.join(pack_dir, "emoji.txt")

	if File.exists?(emoji_txt) do
	load_from_file(emoji_txt, emoji_groups)
	else
	extensions = Pleroma.Config.get([:emoji, :pack_extensions])

	Logger.info(
	"No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
	)

	make_shortcode_to_file_map(pack_dir, extensions)
	\|> Enum.map(fn {shortcode, rel_file} ->
	filename = Path.join("/emoji/#{pack_name}", rel_file)

	{shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
	end)
	end
	end

	def make_shortcode_to_file_map(pack_dir, exts) do
	find_all_emoji(pack_dir, exts)
	\|> Enum.map(&Path.relative_to(&1, pack_dir))
	\|> Enum.map(fn f -> {f \|> Path.basename() \|> Path.rootname(), f} end)
	\|> Enum.into(%{})
	end

	def find_all_emoji(dir, exts) do
	Enum.reduce(
	File.ls!(dir),
	[],
	fn f, acc ->
	filepath = Path.join(dir, f)

	if File.dir?(filepath) do
	acc ++ find_all_emoji(filepath, exts)
	else
	acc ++ [filepath]
	end
	end
	)
	\|> Enum.filter(fn f -> Path.extname(f) in exts end)
	end

	defp load_from_file(file, emoji_groups) do
	if File.exists?(file) do
	load_from_file_stream(File.stream!(file), emoji_groups)
	else
	[]
	end
	end

	defp load_from_file_stream(stream, emoji_groups) do
	stream
	\|> Stream.map(&String.trim/1)
	\|> Stream.map(fn line ->
	case String.split(line, ~r/,\s*/) do
	[name, file] ->
	{name, file, [to_string(match_extra(emoji_groups, file))]}

	[name, file \| tags] ->
	{name, file, tags}

	_ ->
	nil
	end
	end)
	\|> Enum.to_list()
	end

	defp load_from_globs(globs, emoji_groups) do
	static_path = Path.join(:code.priv_dir(:pleroma), "static")

	paths =
	Enum.map(globs, fn glob ->
	Path.join(static_path, glob)
	\|> Path.wildcard()
	end)
	\|> Enum.concat()

	Enum.map(paths, fn path ->
	tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
	shortcode = Path.basename(path, Path.extname(path))
	external_path = Path.join("/", Path.relative_to(path, static_path))
	{shortcode, external_path, [to_string(tag)]}
	end)
	end

	@doc """
	Finds a matching group for the given emoji filename
	"""
	@spec match_extra(group_patterns(), String.t()) :: atom() \| nil
	def match_extra(group_patterns, filename) do
	match_group_patterns(group_patterns, fn pattern ->
	case pattern do
	%Regex{} = regex -> Regex.match?(regex, filename)
	string when is_binary(string) -> filename == string
	end
	end)
	end

	defp match_group_patterns(group_patterns, matcher) do
	Enum.find_value(group_patterns, fn {group, patterns} ->
	patterns =
	patterns
	\|> List.wrap()
	\|> Enum.map(fn pattern ->
	if String.contains?(pattern, "*") do
	~r(#{String.replace(pattern, "", ".")})
	else
	pattern
	end
	end)

	Enum.any?(patterns, matcher) && group
	end)
	end
	end
	diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
	index c123530dc..27f735dd4 100644
	--- a/lib/pleroma/web/endpoint.ex
	+++ b/lib/pleroma/web/endpoint.ex
	@@ -1,137 +1,137 @@
	# Pleroma: A lightweight social networking server
	# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
	# SPDX-License-Identifier: AGPL-3.0-only

	defmodule Pleroma.Web.Endpoint do
	use Phoenix.Endpoint, otp_app: :pleroma

	socket("/socket", Pleroma.Web.UserSocket)

	plug(Pleroma.Plugs.SetLocalePlug)
	plug(CORSPlug)
	plug(Pleroma.Plugs.HTTPSecurityPlug)
	plug(Pleroma.Plugs.UploadedMedia)

	@static_cache_control "public, no-cache"

	# InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
	# If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
	# Cache-control headers are duplicated in case we turn off etags in the future
	plug(Pleroma.Plugs.InstanceStatic,
	at: "/",
	gzip: true,
	cache_control_for_etags: @static_cache_control,
	headers: %{
	"cache-control" => @static_cache_control
	}
	)

	# Serve at "/" the static files from "priv/static" directory.
	#
	# You should set gzip to true if you are running phoenix.digest
	# when deploying your static files in production.
	plug(
	Plug.Static,
	at: "/",
	from: :pleroma,
	only:
	- ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc),
	+ ~w(index.html robots.txt static emoji emoji-packs packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc),
	# credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
	gzip: true,
	cache_control_for_etags: @static_cache_control,
	headers: %{
	"cache-control" => @static_cache_control
	}
	)

	plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")

	plug(Plug.Static,
	at: "/pleroma/admin/",
	from: {:pleroma, "priv/static/adminfe/"}
	)

	# Code reloading can be explicitly enabled under the
	# :code_reloader configuration of your endpoint.
	if code_reloading? do
	plug(Phoenix.CodeReloader)
	end

	plug(TrailingFormatPlug)
	plug(Plug.RequestId)
	plug(Plug.Logger)

	plug(
	Plug.Parsers,
	parsers: [:urlencoded, :multipart, :json],
	pass: ["/"],
	json_decoder: Jason,
	length: Pleroma.Config.get([:instance, :upload_limit]),
	body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []}
	)

	plug(Plug.MethodOverride)
	plug(Plug.Head)

	secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag])

	cookie_name =
	if secure_cookies,
	do: "__Host-pleroma_key",
	else: "pleroma_key"

	extra =
	Pleroma.Config.get([__MODULE__, :extra_cookie_attrs])
	\|> Enum.join(";")

	# The session will be stored in the cookie and signed,
	# this means its contents can be read but not tampered with.
	# Set :encryption_salt if you would also like to encrypt it.
	plug(
	Plug.Session,
	store: :cookie,
	key: cookie_name,
	signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
	http_only: true,
	secure: secure_cookies,
	extra: extra
	)

	# Note: the plug and its configuration is compile-time this can't be upstreamed yet
	if proxies = Pleroma.Config.get([__MODULE__, :reverse_proxies]) do
	plug(RemoteIp, proxies: proxies)
	end

	defmodule Instrumenter do
	use Prometheus.PhoenixInstrumenter
	end

	defmodule PipelineInstrumenter do
	use Prometheus.PlugPipelineInstrumenter
	end

	defmodule MetricsExporter do
	use Prometheus.PlugExporter
	end

	plug(PipelineInstrumenter)
	plug(MetricsExporter)

	plug(Pleroma.Web.Router)

	@doc """
	Dynamically loads configuration from the system environment
	on startup.

	It receives the endpoint configuration from the config files
	and must return the updated configuration.
	"""
	def load_from_system_env(config) do
	port = System.get_env("PORT") \|\| raise "expected the PORT environment variable to be set"
	{:ok, Keyword.put(config, :http, [:inet6, port: port])}
	end

	def websocket_url do
	String.replace_leading(url(), "http", "ws")
	end
	end

File Metadata

Mime Type: text/x-diff
Expires: Sat, Oct 10, 3:41 PM (1 d, 8 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1785310
Default Alt Text: (20 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions