No OneTemporary
Actions

Size

25 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/components/rich_content/rich_content.jsx b/src/components/rich_content/rich_content.jsx
	index e188763fb9..328e9201d4 100644
	--- a/src/components/rich_content/rich_content.jsx
	+++ b/src/components/rich_content/rich_content.jsx
	@@ -1,341 +1,349 @@
	import Vue from 'vue'
	import { unescape, flattenDeep } from 'lodash'
	import { getTagName, processTextForEmoji, getAttrs } from 'src/services/html_converter/utility.service.js'
	import { convertHtmlToTree } from 'src/services/html_converter/html_tree_converter.service.js'
	import { convertHtmlToLines } from 'src/services/html_converter/html_line_converter.service.js'
	import StillImage from 'src/components/still-image/still-image.vue'
	import MentionLink from 'src/components/mention_link/mention_link.vue'

	import './rich_content.scss'

	/**
	* RichContent, The Über-powered component for rendering Post HTML.
	*
	* This takes post HTML and does multiple things to it:
	* - Converts mention links to <MentionLink>-s
	* - Removes mentions from beginning and end (hellthread style only)
	* - Replaces emoji shortcodes with <StillImage>'d images.
	*
	* Stuff like removing mentions from beginning and end is done so that they could
	* be either replaced by collapsible <MentionsLine> or moved to separate place.
	* There are two problems with this component's architecture:
	* 1. Parsing HTML and rendering are inseparable. Attempts to separate the two
	* proven to be a massive overcomplication due to amount of things done here.
	* 2. We need to output both render and some extra data, which seems to be imp-
	* possible in vue. Current solution is to emit 'parseReady' event when parsing
	* is done within render() function.
	*
	* Apart from that one small hiccup with emit in render this _should_ be vue3-ready
	*/
	export default Vue.component('RichContent', {
	name: 'RichContent',
	props: {
	// Original html content
	html: {
	required: true,
	type: String
	},
	// Emoji object, as in status.emojis, note the "s" at the end...
	emoji: {
	required: true,
	type: Array
	},
	// Whether to handle links or not (posts: yes, everything else: no)
	handleLinks: {
	required: false,
	type: Boolean,
	default: false
	},
	// Meme arrows
	greentext: {
	required: false,
	type: Boolean,
	default: false
	}
	},
	// NEVER EVER TOUCH DATA INSIDE RENDER
	render (h) {
	// Pre-process HTML
	const { newHtml: html, lastMentions } = preProcessPerLine(this.html, this.greentext, this.handleLinks)
	const firstMentions = [] // Mentions that appear in the beginning of post body
	const lastTags = [] // Tags that appear at the end of post body
	const writtenMentions = [] // All mentions that appear in post body
	const writtenTags = [] // All tags that appear in post body
	// unique index for vue "tag" property
	let mentionIndex = 0
	let tagsIndex = 0

	const renderImage = (tag) => {
	return <StillImage
	{...{ attrs: getAttrs(tag) }}
	class="img"
	/>
	}

	const renderHashtag = (attrs, children, encounteredTextReverse) => {
	const linkData = getLinkData(attrs, children, tagsIndex++)
	writtenTags.push(linkData)
	attrs.target = '_blank'
	if (!encounteredTextReverse) {
	lastTags.push(linkData)
	attrs['data-parser-last'] = true
	}
	return <a {...{ attrs }}>
	{ children.map(processItem) }
	</a>
	}

	const renderMention = (attrs, children, encounteredText) => {
	const linkData = getLinkData(attrs, children, mentionIndex++)
	writtenMentions.push(linkData)
	if (!encounteredText) {
	firstMentions.push(linkData)
	return ''
	} else {
	return <MentionLink
	url={attrs.href}
	content={flattenDeep(children).join('')}
	/>
	}
	}

	// We stop treating mentions as "first" ones when we encounter
	// non-whitespace text
	let encounteredText = false
	// Processor to use with html_tree_converter
	const processItem = (item, index, array, what) => {
	// Handle text nodes - just add emoji
	if (typeof item === 'string') {
	const emptyText = item.trim() === ''
	if (emptyText) {
	return encounteredText ? item : item.trim()
	}
	let unescapedItem = unescape(item)
	if (!encounteredText) {
	unescapedItem = unescapedItem.trimStart()
	encounteredText = true
	}
	if (item.includes(':')) {
	unescapedItem = processTextForEmoji(
	unescapedItem,
	this.emoji,
	({ shortcode, url }) => {
	return <StillImage
	class="emoji img"
	src={url}
	title={`:${shortcode}:`}
	alt={`:${shortcode}:`}
	/>
	}
	)
	}
	return unescapedItem
	}

	// Handle tag nodes
	if (Array.isArray(item)) {
	const [opener, children] = item
	const Tag = getTagName(opener)
	const attrs = getAttrs(opener)
	switch (Tag) {
	case 'span': // replace images with StillImage
	if (attrs['class'] && attrs['class'].includes('lastMentions')) {
	if (firstMentions.length > 1) {
	break
	} else {
	return ''
	}
	} else {
	break
	}
	case 'img': // replace images with StillImage
	return renderImage(opener)
	case 'a': // replace mentions with MentionLink
	if (!this.handleLinks) break
	if (attrs['class'] && attrs['class'].includes('mention')) {
	return renderMention(attrs, children, encounteredText)
	} else if (attrs['class'] && attrs['class'].includes('hashtag')) {
	encounteredText = true
	return item // We'll handle it later
	} else {
	attrs.target = '_blank'
	return <a {...{ attrs }}>
	{ children.map(processItem) }
	</a>
	}
	}

	// Render tag as is
	if (children !== undefined) {
	return <Tag {...{ attrs: getAttrs(opener) }}>
	{ children.map(processItem) }
	</Tag>
	} else {
	return <Tag/>
	}
	}
	}

	// Processor for back direction (for finding "last" stuff, just easier this way)
	let encounteredTextReverse = false
	const processItemReverse = (item, index, array, what) => {
	// Handle text nodes - just add emoji
	if (typeof item === 'string') {
	const emptyText = item.trim() === ''
	if (emptyText) return item
	if (!encounteredTextReverse) encounteredTextReverse = true
	return item
	} else if (Array.isArray(item)) {
	// Handle tag nodes
	const [opener, children] = item
	const Tag = getTagName(opener)
	switch (Tag) {
	case 'a': // replace mentions with MentionLink
	if (!this.handleLinks) break
	const attrs = getAttrs(opener)
	// should only be this
	if (attrs['class'] && attrs['class'].includes('hashtag')) {
	return renderHashtag(attrs, children, encounteredTextReverse)
	}
	}
	}
	return item
	}

	// DO NOT USE SLOTS they cause a re-render feedback loop here.
	// slots updated -> rerender -> emit -> update up the tree -> rerender -> ...
	// at least until vue3?
	const result = <span class="RichContent">
	{ convertHtmlToTree(html).map(processItem).reverse().map(processItemReverse).reverse() }
	</span>

	const event = {
	firstMentions,
	lastMentions,
	lastTags,
	writtenMentions,
	writtenTags
	}

	// DO NOT MOVE TO UPDATE. BAD IDEA.
	this.$emit('parseReady', event)

	return result
	}
	})

	const getLinkData = (attrs, children, index) => {
	return {
	index,
	url: attrs.href,
	hashtag: attrs['data-tag'],
	content: flattenDeep(children).join('')
	}
	}

	/** Pre-processing HTML
	*
	* Currently this does two things:
	* - add green/cyantexting
	* - wrap and mark last line containing only mentions as ".lastMentionsLine" for
	* more compact hellthreads.
	*
	* @param {String} html - raw HTML to process
	* @param {Boolean} greentext - whether to enable greentexting or not
	* @param {Boolean} handleLinks - whether to handle links or not
	*/
	export const preProcessPerLine = (html, greentext, handleLinks) => {
	const lastMentions = []
	+ const greentextHandle = new Set(['p', 'div'])

	let nonEmptyIndex = -1
	const newHtml = convertHtmlToLines(html).reverse().map((item, index, array) => {
	// Going over each line in reverse to detect last mentions,
	// keeping non-text stuff as-is
	if (!item.text) return item
	const string = item.text
	nonEmptyIndex += 1

	// Greentext stuff
	- if (greentext && (string.includes('>') \|\| string.includes('<'))) {
	+ if (
	+ // Only if greentext is engaged
	+ greentext &&
	+ // Only handle p's and divs. Don't want to affect blocquotes, code etc
	+ item.level.every(l => greentextHandle.has(l)) &&
	+ // Only if line begins with '>' or '<'
	+ (string.includes('>') \|\| string.includes('<'))
	+ ) {
	const cleanedString = string.replace(/<[^>]+?>/gi, '') // remove all tags
	.replace(/@\w+/gi, '') // remove mentions (even failed ones)
	.trim()
	if (cleanedString.startsWith('>')) {
	return `<span class='greentext'>${string}</span>`
	} else if (cleanedString.startsWith('<')) {
	return `<span class='cyantext'>${string}</span>`
	}
	}

	// Converting that line part into tree
	const tree = convertHtmlToTree(string)

	// If line has loose text, i.e. text outside a mention or a tag
	// we won't touch mentions.
	let hasLooseText = false
	let hasMentions = false
	const process = (item) => {
	if (Array.isArray(item)) {
	const [opener, children, closer] = item
	const tag = getTagName(opener)
	// If we have a link we probably have mentions
	if (tag === 'a') {
	if (!handleLinks) return [opener, children, closer]
	const attrs = getAttrs(opener)
	if (attrs['class'] && attrs['class'].includes('mention')) {
	// Got mentions
	hasMentions = true
	return [opener, children, closer]
	} else {
	// Not a mention? Means we have loose text or whatever
	hasLooseText = true
	return [opener, children, closer]
	}
	} else if (tag === 'span' \|\| tag === 'p') {
	// For span and p we need to go deeper
	return [opener, [...children].map(process), closer]
	} else {
	// Everything else equals to a loose text
	hasLooseText = true
	return [opener, children, closer]
	}
	}

	if (typeof item === 'string') {
	if (item.trim() !== '') {
	// only meaningful strings are loose text
	hasLooseText = true
	}
	return item
	}
	}

	// We now processed our tree, now we need to mark line as lastMentions
	const result = [...tree].map(process)

	// Only check last (first since list is reversed) line
	if (handleLinks && hasMentions && !hasLooseText && nonEmptyIndex++ === 0) {
	let mentionIndex = 0
	const process = (item) => {
	if (Array.isArray(item)) {
	const [opener, children] = item
	const tag = getTagName(opener)
	if (tag === 'a') {
	const attrs = getAttrs(opener)
	lastMentions.push(getLinkData(attrs, children, mentionIndex++))
	} else if (children) {
	children.forEach(process)
	}
	}
	}
	result.forEach(process)
	// we DO need mentions here so that we conditionally remove them if don't
	// have first mentions
	return ['<span class="lastMentions">', flattenDeep(result).join(''), '</span>'].join('')
	} else {
	return flattenDeep(result).join('')
	}
	}).reverse().join('')

	return { newHtml, lastMentions }
	}
	diff --git a/src/services/html_converter/html_line_converter.service.js b/src/services/html_converter/html_line_converter.service.js
	index e448d5cd91..f43d162a5a 100644
	--- a/src/services/html_converter/html_line_converter.service.js
	+++ b/src/services/html_converter/html_line_converter.service.js
	@@ -1,99 +1,134 @@
	import { getTagName } from './utility.service.js'

	/**
	* This is a tiny purpose-built HTML parser/processor. This basically detects
	* any type of visual newline and converts entire HTML into a array structure.
	*
	* Text nodes are represented as object with single property - text - containing
	* the visual line. Intended usage is to process the array with .map() in which
	* map function returns a string and resulting array can be converted back to html
	* with a .join('').
	*
	* Generally this isn't very useful except for when you really need to either
	* modify visual lines (greentext i.e. simple quoting) or do something with
	* first/last line.
	*
	* known issue: doesn't handle CDATA so nested CDATA might not work well
	*
	* @param {Object} input - input data
	* @return {(string\|{ text: string })[]} processed html in form of a list.
	*/
	export const convertHtmlToLines = (html) => {
	- const ignoredTags = new Set(['code', 'blockquote'])
	- const handledTags = new Set(['p', 'br', 'div', 'pre', 'code', 'blockquote'])
	- const openCloseTags = new Set(['p', 'div', 'pre', 'code', 'blockquote'])
	+ // Elements that are implicitly self-closing
	+ // https://developer.mozilla.org/en-US/docs/Glossary/empty_element
	+ const emptyElements = new Set([
	+ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
	+ 'keygen', 'link', 'meta', 'param', 'source', 'track', 'wbr'
	+ ])
	+ // Block-level element (they make a visual line)
	+ // https://developer.mozilla.org/en-US/docs/Web/HTML/Block-level_elements
	+ const blockElements = new Set([
	+ 'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd',
	+ 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form',
	+ 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main',
	+ 'nav', 'ol', 'p', 'pre', 'section', 'table', 'ul'
	+ ])
	+ // br is very weird in a way that it's technically not block-level, it's
	+ // essentially converted to a \n (or \r\n). There's also wbr but it doesn't
	+ // guarantee linebreak, only suggest it.
	+ const linebreakElements = new Set(['br'])
	+
	+ const visualLineElements = new Set([
	+ ...blockElements.values(),
	+ ...linebreakElements.values()
	+ ])
	+
	+ // All block-level elements that aren't empty elements, i.e. not <hr>
	+ const nonEmptyElements = new Set(visualLineElements)
	+ // Difference
	+ for (let elem of emptyElements) {
	+ nonEmptyElements.delete(elem)
	+ }
	+
	+ // All elements that we are recognizing
	+ const allElements = new Set([
	+ ...nonEmptyElements.values(),
	+ ...emptyElements.values()
	+ ])

	let buffer = [] // Current output buffer
	const level = [] // How deep we are in tags and which tags were there
	let textBuffer = '' // Current line content
	let tagBuffer = null // Current tag buffer, if null = we are not currently reading a tag

	const flush = () => { // Processes current line buffer, adds it to output buffer and clears line buffer
	- if (textBuffer.trim().length > 0 && !level.some(l => ignoredTags.has(l))) {
	- buffer.push({ text: textBuffer })
	+ if (textBuffer.trim().length > 0) {
	+ buffer.push({ level: [...level], text: textBuffer })
	} else {
	buffer.push(textBuffer)
	}
	textBuffer = ''
	}

	const handleBr = (tag) => { // handles single newlines/linebreaks/selfclosing
	flush()
	buffer.push(tag)
	}

	const handleOpen = (tag) => { // handles opening tags
	flush()
	buffer.push(tag)
	level.unshift(getTagName(tag))
	}

	const handleClose = (tag) => { // handles closing tags
	- flush()
	- buffer.push(tag)
	if (level[0] === getTagName(tag)) {
	+ flush()
	+ buffer.push(tag)
	level.shift()
	+ } else { // Broken case
	+ textBuffer += tag
	}
	}

	for (let i = 0; i < html.length; i++) {
	const char = html[i]
	if (char === '<' && tagBuffer === null) {
	tagBuffer = char
	} else if (char !== '>' && tagBuffer !== null) {
	tagBuffer += char
	} else if (char === '>' && tagBuffer !== null) {
	tagBuffer += char
	const tagFull = tagBuffer
	tagBuffer = null
	const tagName = getTagName(tagFull)
	- if (handledTags.has(tagName)) {
	- if (tagName === 'br') {
	+ if (allElements.has(tagName)) {
	+ if (linebreakElements.has(tagName)) {
	handleBr(tagFull)
	- } else if (openCloseTags.has(tagName)) {
	+ } else if (nonEmptyElements.has(tagName)) {
	if (tagFull[1] === '/') {
	handleClose(tagFull)
	} else if (tagFull[tagFull.length - 2] === '/') {
	// self-closing
	handleBr(tagFull)
	} else {
	handleOpen(tagFull)
	}
	}
	} else {
	textBuffer += tagFull
	}
	} else if (char === '\n') {
	handleBr(char)
	} else {
	textBuffer += char
	}
	}
	if (tagBuffer) {
	textBuffer += tagBuffer
	}

	flush()

	return buffer
	}
	diff --git a/test/unit/specs/services/html_converter/html_line_converter.spec.js b/test/unit/specs/services/html_converter/html_line_converter.spec.js
	index 9485233fcc..c8c89700b3 100644
	--- a/test/unit/specs/services/html_converter/html_line_converter.spec.js
	+++ b/test/unit/specs/services/html_converter/html_line_converter.spec.js
	@@ -1,155 +1,164 @@
	import { convertHtmlToLines } from 'src/services/html_converter/html_line_converter.service.js'

	-const mapOnlyText = (processor) => (input) => input.text ? processor(input.text) : input
	-
	-describe('html_line_converter', () => {
	+const greentextHandle = new Set(['p', 'div'])
	+const mapOnlyText = (processor) => (input) => {
	+ if (input.text && input.level.every(l => greentextHandle.has(l))) {
	+ return processor(input.text)
	+ } else if (input.text) {
	+ return input.text
	+ } else {
	+ return input
	+ }
	+}
	+
	+describe.only('html_line_converter', () => {
	describe('with processor that keeps original line should not make any changes to HTML when', () => {
	const processorKeep = (line) => line
	it('fed with regular HTML with newlines', () => {
	const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with possibly broken HTML with invalid tags/composition', () => {
	const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with very broken HTML with broken composition', () => {
	const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with sorta valid HTML but tags aren\'t closed', () => {
	const inputOutput = 'just leaving a <div> hanging'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with not really HTML at this point... tags that aren\'t finished', () => {
	const inputOutput = 'do you expect me to finish this <div class='
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
	const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with maybe valid HTML? self-closing divs and ps', () => {
	const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})

	it('fed with valid XHTML containing a CDATA', () => {
	const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
	const result = convertHtmlToLines(inputOutput)
	const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
	expect(comparableResult).to.eql(inputOutput)
	})
	})
	describe('with processor that replaces lines with word "_" should match expected line when', () => {
	const processorReplace = (line) => '_'
	it('fed with regular HTML with newlines', () => {
	const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
	const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with possibly broken HTML with invalid tags/composition', () => {
	const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
	const output = '_'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with very broken HTML with broken composition', () => {
	const input = '</p> lmao what </div> whats going on <div> wha <p>'
	- const output = '</p>_</div>_<div>_<p>'
	+ const output = '_<div>_<p>'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with sorta valid HTML but tags aren\'t closed', () => {
	const input = 'just leaving a <div> hanging'
	const output = '_<div>_'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with not really HTML at this point... tags that aren\'t finished', () => {
	const input = 'do you expect me to finish this <div class='
	const output = '_'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
	const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
	const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	- it('fed with maybe valid HTML? self-closing divs and ps', () => {
	+ it('fed with maybe valid HTML? (XHTML) self-closing divs and ps', () => {
	const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
	const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('fed with valid XHTML containing a CDATA', () => {
	const input = 'Yes, it is me, <![CDATA[DIO]]>'
	const output = '_'
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})

	it('Testing handling ignored blocks', () => {
	const input = `
	<pre><code>> rei = "0"
	'0'
	> rei == 0
	true
	> rei == null
	false</code></pre><blockquote>That, christian-like JS diagram but it’s evangelion instead.</blockquote>
	`
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(input)
	})
	it('Testing handling ignored blocks 2', () => {
	const input = `
	<blockquote>An SSL error has happened.</blockquote><p>Shakespeare</p>
	`
	const output = `
	<blockquote>An SSL error has happened.</blockquote><p>_</p>
	`
	const result = convertHtmlToLines(input)
	const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
	expect(comparableResult).to.eql(output)
	})
	})
	})

File Metadata

Mime Type: text/x-diff
Expires: Sun, Aug 9, 1:14 PM (1 d, 10 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1724568
Default Alt Text: (25 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions