Page MenuHomePhorge
Files F85712676

No OneTemporary
Actions

Size
11 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/src/services/html_converter/html_line_converter.service.js b/src/services/html_converter/html_line_converter.service.js
index 80482c9aed..d8f5ecb867 100644
--- a/src/services/html_converter/html_line_converter.service.js
+++ b/src/services/html_converter/html_line_converter.service.js
@@ -1,102 +1,103 @@
/**
* This is a tiny purpose-built HTML parser/processor. This basically detects
* any type of visual newline and converts entire HTML into a array structure.
*
* Text nodes are represented as object with single property - text - containing
* the visual line. Intended usage is to process the array with .map() in which
* map function returns a string and resulting array can be converted back to html
* with a .join('').
*
* Generally this isn't very useful except for when you really need to either
* modify visual lines (greentext i.e. simple quoting) or do something with
* first/last line.
*
* known issue: doesn't handle CDATA so nested CDATA might not work well
*
* @param {Object} input - input data
* @return {(string|{ text: string })[]} processed html in form of a list.
*/
export const convertHtmlToLines = (html) => {
- const handledTags = new Set(['p', 'br', 'div'])
- const openCloseTags = new Set(['p', 'div'])
+ const ignoredTags = new Set(['code', 'blockquote'])
+ const handledTags = new Set(['p', 'br', 'div', 'pre', 'code', 'blockquote'])
+ const openCloseTags = new Set(['p', 'div', 'pre', 'code', 'blockquote'])
let buffer = [] // Current output buffer
const level = [] // How deep we are in tags and which tags were there
let textBuffer = '' // Current line content
let tagBuffer = null // Current tag buffer, if null = we are not currently reading a tag
// Extracts tag name from tag, i.e. <span a="b"> => span
const getTagName = (tag) => {
const result = /(?:<\/(\w+)>|<(\w+)\s?[^/]*?\/?>)/gi.exec(tag)
return result && (result[1] || result[2])
}
const flush = () => { // Processes current line buffer, adds it to output buffer and clears line buffer
- if (textBuffer.trim().length > 0) {
+ if (textBuffer.trim().length > 0 && !level.some(l => ignoredTags.has(l))) {
buffer.push({ text: textBuffer })
} else {
buffer.push(textBuffer)
}
textBuffer = ''
}
const handleBr = (tag) => { // handles single newlines/linebreaks/selfclosing
flush()
buffer.push(tag)
}
const handleOpen = (tag) => { // handles opening tags
flush()
buffer.push(tag)
- level.push(tag)
+ level.unshift(getTagName(tag))
}
const handleClose = (tag) => { // handles closing tags
flush()
buffer.push(tag)
- if (level[level.length - 1] === tag) {
- level.pop()
+ if (level[0] === getTagName(tag)) {
+ level.shift()
}
}
for (let i = 0; i < html.length; i++) {
const char = html[i]
if (char === '<' && tagBuffer === null) {
tagBuffer = char
} else if (char !== '>' && tagBuffer !== null) {
tagBuffer += char
} else if (char === '>' && tagBuffer !== null) {
tagBuffer += char
const tagFull = tagBuffer
tagBuffer = null
const tagName = getTagName(tagFull)
if (handledTags.has(tagName)) {
if (tagName === 'br') {
handleBr(tagFull)
} else if (openCloseTags.has(tagName)) {
if (tagFull[1] === '/') {
handleClose(tagFull)
} else if (tagFull[tagFull.length - 2] === '/') {
// self-closing
handleBr(tagFull)
} else {
handleOpen(tagFull)
}
}
} else {
textBuffer += tagFull
}
} else if (char === '\n') {
handleBr(char)
} else {
textBuffer += char
}
}
if (tagBuffer) {
textBuffer += tagBuffer
}
flush()
return buffer
}
diff --git a/test/unit/specs/services/html_converter/html_line_converter.spec.js b/test/unit/specs/services/html_converter/html_line_converter.spec.js
index 82cb4170ee..532ea187b0 100644
--- a/test/unit/specs/services/html_converter/html_line_converter.spec.js
+++ b/test/unit/specs/services/html_converter/html_line_converter.spec.js
@@ -1,130 +1,155 @@
import { convertHtmlToLines } from 'src/services/html_converter/html_line_converter.service.js'
const mapOnlyText = (processor) => (input) => input.text ? processor(input.text) : input
describe('TinyPostHTMLProcessor', () => {
describe('with processor that keeps original line should not make any changes to HTML when', () => {
const processorKeep = (line) => line
it('fed with regular HTML with newlines', () => {
const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with possibly broken HTML with invalid tags/composition', () => {
const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with very broken HTML with broken composition', () => {
const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with sorta valid HTML but tags aren\'t closed', () => {
const inputOutput = 'just leaving a <div> hanging'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with not really HTML at this point... tags that aren\'t finished', () => {
const inputOutput = 'do you expect me to finish this <div class='
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with maybe valid HTML? self-closing divs and ps', () => {
const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
it('fed with valid XHTML containing a CDATA', () => {
const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
const result = convertHtmlToLines(inputOutput)
const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
expect(comparableResult).to.eql(inputOutput)
})
})
describe('with processor that replaces lines with word "_" should match expected line when', () => {
const processorReplace = (line) => '_'
it('fed with regular HTML with newlines', () => {
const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with possibly broken HTML with invalid tags/composition', () => {
const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
const output = '_'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with very broken HTML with broken composition', () => {
const input = '</p> lmao what </div> whats going on <div> wha <p>'
const output = '</p>_</div>_<div>_<p>'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with sorta valid HTML but tags aren\'t closed', () => {
const input = 'just leaving a <div> hanging'
const output = '_<div>_'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with not really HTML at this point... tags that aren\'t finished', () => {
const input = 'do you expect me to finish this <div class='
const output = '_'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with maybe valid HTML? self-closing divs and ps', () => {
const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
it('fed with valid XHTML containing a CDATA', () => {
const input = 'Yes, it is me, <![CDATA[DIO]]>'
const output = '_'
const result = convertHtmlToLines(input)
const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
expect(comparableResult).to.eql(output)
})
+
+ it('Testing handling ignored blocks', () => {
+ const input = `
+ <pre><code>&gt; rei = &quot;0&quot;
+ &#39;0&#39;
+ &gt; rei == 0
+ true
+ &gt; rei == null
+ false</code></pre><blockquote>That, christian-like JS diagram but it’s evangelion instead.</blockquote>
+ `
+ const result = convertHtmlToLines(input)
+ const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+ expect(comparableResult).to.eql(input)
+ })
+ it('Testing handling ignored blocks 2', () => {
+ const input = `
+ <blockquote>An SSL error has happened.</blockquote><p>Shakespeare</p>
+ `
+ const output = `
+ <blockquote>An SSL error has happened.</blockquote><p>_</p>
+ `
+ const result = convertHtmlToLines(input)
+ const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+ expect(comparableResult).to.eql(output)
+ })
})
})

File Metadata

Mime Type
text/x-diff
Expires
Sat, Sep 19, 12:09 PM (1 d, 23 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1769453
Default Alt Text
(11 KB)

Event Timeline