Merge branch 'develop' into 'themeApply'

# Conflicts:
#   CHANGELOG.md

6 files changed, 821 insertions, 178 deletions

diff --git a/test/unit/specs/components/rich_content.spec.js b/test/unit/specs/components/rich_content.spec.js
new file mode 100644
index 00000000..f6c478a9
--- /dev/null
+++ b/test/unit/specs/components/rich_content.spec.js
@@ -0,0 +1,480 @@
+import { mount, shallowMount, createLocalVue } from '@vue/test-utils'
+import RichContent from 'src/components/rich_content/rich_content.jsx'
+
+const localVue = createLocalVue()
+const attentions = []
+
+const makeMention = (who) => {
+  attentions.push({ statusnet_profile_url: `https://fake.tld/@${who}` })
+  return `<span class="h-card"><a class="u-url mention" href="https://fake.tld/@${who}">@<span>${who}</span></a></span>`
+}
+const p = (...data) => `<p>${data.join('')}</p>`
+const compwrap = (...data) => `<span class="RichContent">${data.join('')}</span>`
+const mentionsLine = (times) => [
+  '<mentionsline-stub mentions="',
+  new Array(times).fill('[object Object]').join(','),
+  '"></mentionsline-stub>'
+].join('')
+
+describe('RichContent', () => {
+  it('renders simple post without exploding', () => {
+    const html = p('Hello world!')
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('unescapes everything as needed', () => {
+    const html = [
+      p('Testing &#39;em all'),
+      'Testing &#39;em all'
+    ].join('')
+    const expected = [
+      p('Testing \'em all'),
+      'Testing \'em all'
+    ].join('')
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('replaces mention with mentionsline', () => {
+    const html = p(
+      makeMention('John'),
+      ' how are you doing today?'
+    )
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(p(
+      mentionsLine(1),
+      ' how are you doing today?'
+    )))
+  })
+
+  it('replaces mentions at the end of the hellpost', () => {
+    const html = [
+      p('How are you doing today, fine gentlemen?'),
+      p(
+        makeMention('John'),
+        makeMention('Josh'),
+        makeMention('Jeremy')
+      )
+    ].join('')
+    const expected = [
+      p(
+        'How are you doing today, fine gentlemen?'
+      ),
+      // TODO fix this extra line somehow?
+      p(
+        '<mentionsline-stub mentions="',
+        '[object Object],',
+        '[object Object],',
+        '[object Object]',
+        '"></mentionsline-stub>'
+      )
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Does not touch links if link handling is disabled', () => {
+    const html = [
+      [
+        makeMention('Jack'),
+        'let\'s meet up with ',
+        makeMention('Janet')
+      ].join(''),
+      [
+        makeMention('John'),
+        makeMention('Josh'),
+        makeMention('Jeremy')
+      ].join('')
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Adds greentext and cyantext to the post', () => {
+    const html = [
+      '&gt;preordering videogames',
+      '&gt;any year'
+    ].join('\n')
+    const expected = [
+      '<span class="greentext">&gt;preordering videogames</span>',
+      '<span class="greentext">&gt;any year</span>'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Does not add greentext and cyantext if setting is set to false', () => {
+    const html = [
+      '&gt;preordering videogames',
+      '&gt;any year'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Adds emoji to post', () => {
+    const html = p('Ebin :DDDD :spurdo:')
+    const expected = p(
+      'Ebin :DDDD ',
+      '<anonymous-stub alt=":spurdo:" src="about:blank" title=":spurdo:" class="emoji img"></anonymous-stub>'
+    )
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [{ url: 'about:blank', shortcode: 'spurdo' }],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Doesn\'t add nonexistent emoji to post', () => {
+    const html = p('Lol :lol:')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Greentext + last mentions', () => {
+    const html = [
+      '&gt;quote',
+      makeMention('lol'),
+      '&gt;quote',
+      '&gt;quote'
+    ].join('\n')
+    const expected = [
+      '<span class="greentext">&gt;quote</span>',
+      mentionsLine(1),
+      '<span class="greentext">&gt;quote</span>',
+      '<span class="greentext">&gt;quote</span>'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('One buggy example', () => {
+    const html = [
+      'Bruh',
+      'Bruh',
+      [
+        makeMention('foo'),
+        makeMention('bar'),
+        makeMention('baz')
+      ].join(''),
+      'Bruh'
+    ].join('<br>')
+    const expected = [
+      'Bruh',
+      'Bruh',
+      mentionsLine(3),
+      'Bruh'
+    ].join('<br>')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('buggy example/hashtags', () => {
+    const html = [
+      '<p>',
+      '<a href="http://macrochan.org/images/N/H/NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg">',
+      'NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg</a>',
+      ' <a class="hashtag" data-tag="nou" href="https://shitposter.club/tag/nou">',
+      '#nou</a>',
+      ' <a class="hashtag" data-tag="screencap" href="https://shitposter.club/tag/screencap">',
+      '#screencap</a>',
+      ' </p>'
+    ].join('')
+    const expected = [
+      '<p>',
+      '<a href="http://macrochan.org/images/N/H/NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg" target="_blank">',
+      'NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg</a>',
+      ' <hashtaglink-stub url="https://shitposter.club/tag/nou" content="#nou" tag="nou">',
+      '</hashtaglink-stub>',
+      ' <hashtaglink-stub url="https://shitposter.club/tag/screencap" content="#screencap" tag="screencap">',
+      '</hashtaglink-stub>',
+      ' </p>'
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('rich contents of a mention are handled properly', () => {
+    attentions.push({ statusnet_profile_url: 'lol' })
+    const html = [
+      p(
+        '<a href="lol" class="mention">',
+        '<span>',
+        'https://</span>',
+        '<span>',
+        'lol.tld/</span>',
+        '<span>',
+        '</span>',
+        '</a>'
+      ),
+      p(
+        'Testing'
+      )
+    ].join('')
+    const expected = [
+      p(
+        '<span class="MentionsLine">',
+        '<span class="MentionLink mention-link">',
+        '<a href="lol" target="_blank" class="original">',
+        '<span>',
+        'https://</span>',
+        '<span>',
+        'lol.tld/</span>',
+        '<span>',
+        '</span>',
+        '</a>',
+        ' ',
+        '<!---->', // v-if placeholder, mentionlink's "new" (i.e. rich) display
+        '</span>',
+        '<!---->', // v-if placeholder, mentionsline's extra mentions and stuff
+        '</span>'
+      ),
+      p(
+        'Testing'
+      )
+    ].join('')
+
+    const wrapper = mount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('rich contents of a link are handled properly', () => {
+    const html = [
+      '<p>',
+      'Freenode is dead.</p>',
+      '<p>',
+      '<a href="https://isfreenodedeadyet.com/">',
+      '<span>',
+      'https://</span>',
+      '<span>',
+      'isfreenodedeadyet.com/</span>',
+      '<span>',
+      '</span>',
+      '</a>',
+      '</p>'
+    ].join('')
+    const expected = [
+      '<p>',
+      'Freenode is dead.</p>',
+      '<p>',
+      '<a href="https://isfreenodedeadyet.com/" target="_blank">',
+      '<span>',
+      'https://</span>',
+      '<span>',
+      'isfreenodedeadyet.com/</span>',
+      '<span>',
+      '</span>',
+      '</a>',
+      '</p>'
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it.skip('[INFORMATIVE] Performance testing, 10 000 simple posts', () => {
+    const amount = 20
+
+    const onePost = p(
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      ' i just landed in l a where are you'
+    )
+
+    const TestComponent = {
+      template: `
+      <div v-if="!vhtml">
+        ${new Array(amount).fill(`<RichContent html="${onePost}" :greentext="true" :handleLinks="handeLinks" :emoji="[]" :attentions="attentions"/>`)}
+      </div>
+      <div v-else="vhtml">
+        ${new Array(amount).fill(`<div v-html="${onePost}"/>`)}
+      </div>
+      `,
+      props: ['handleLinks', 'attentions', 'vhtml']
+    }
+    console.log(1)
+
+    const ptest = (handleLinks, vhtml) => {
+      const t0 = performance.now()
+
+      const wrapper = mount(TestComponent, {
+        localVue,
+        propsData: {
+          attentions,
+          handleLinks,
+          vhtml
+        }
+      })
+
+      const t1 = performance.now()
+
+      wrapper.destroy()
+
+      const t2 = performance.now()
+
+      return `Mount: ${t1 - t0}ms, destroy: ${t2 - t1}ms, avg ${(t1 - t0) / amount}ms - ${(t2 - t1) / amount}ms per item`
+    }
+
+    console.log(`${amount} items with links handling:`)
+    console.log(ptest(true))
+    console.log(`${amount} items without links handling:`)
+    console.log(ptest(false))
+    console.log(`${amount} items plain v-html:`)
+    console.log(ptest(false, true))
+  })
+})
diff --git a/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js b/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
index 759539e0..03fb32c9 100644
--- a/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
+++ b/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
@@ -1,4 +1,4 @@
-import { parseStatus, parseUser, parseNotification, addEmojis, parseLinkHeaderPagination } from '../../../../../src/services/entity_normalizer/entity_normalizer.service.js'
+import { parseStatus, parseUser, parseNotification, parseLinkHeaderPagination } from '../../../../../src/services/entity_normalizer/entity_normalizer.service.js'
 import mastoapidata from '../../../../fixtures/mastoapi.json'
 import qvitterapidata from '../../../../fixtures/statuses.json'
 
@@ -23,7 +23,6 @@ const makeMockStatusQvitter = (overrides = {}) => {
     repeat_num: 0,
     repeated: false,
     statusnet_conversation_id: '16300488',
-    statusnet_html: '<p>haha benis</p>',
     summary: null,
     tags: [],
     text: 'haha benis',
@@ -232,22 +231,6 @@ describe('API Entities normalizer', () => {
         expect(parsedRepeat).to.have.property('retweeted_status')
         expect(parsedRepeat).to.have.deep.property('retweeted_status.id', 'deadbeef')
       })
-
-      it('adds emojis to post content', () => {
-        const post = makeMockStatusMasto({ emojis: makeMockEmojiMasto(), content: 'Makes you think :thinking:' })
-
-        const parsedPost = parseStatus(post)
-
-        expect(parsedPost).to.have.property('statusnet_html').that.contains('<img')
-      })
-
-      it('adds emojis to subject line', () => {
-        const post = makeMockStatusMasto({ emojis: makeMockEmojiMasto(), spoiler_text: 'CW: 300 IQ :thinking:' })
-
-        const parsedPost = parseStatus(post)
-
-        expect(parsedPost).to.have.property('summary_html').that.contains('<img')
-      })
     })
   })
 
@@ -261,35 +244,6 @@ describe('API Entities normalizer', () => {
       expect(parseUser(remote)).to.have.property('is_local', false)
     })
 
-    it('adds emojis to user name', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), display_name: 'The :thinking: thinker' })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('name_html').that.contains('<img')
-    })
-
-    it('adds emojis to user bio', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), note: 'Hello i like to :thinking: a lot' })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('description_html').that.contains('<img')
-    })
-
-    it('adds emojis to user profile fields', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), fields: [{ name: ':thinking:', value: ':image:' }] })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('fields_html').to.be.an('array')
-
-      const field = parsedUser.fields_html[0]
-
-      expect(field).to.have.property('name').that.contains('<img')
-      expect(field).to.have.property('value').that.contains('<img')
-    })
-
     it('removes html tags from user profile fields', () => {
       const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), fields: [{ name: 'user', value: '<a rel="me" href="https://example.com/@user">@user</a>' }] })
 
@@ -355,41 +309,6 @@ describe('API Entities normalizer', () => {
     })
   })
 
-  describe('MastoAPI emoji adder', () => {
-    const emojis = makeMockEmojiMasto()
-    const imageHtml = '<img src="https://example.com/image.png" alt=":image:" title=":image:" class="emoji" />'
-      .replace(/"/g, '\'')
-    const thinkHtml = '<img src="https://example.com/think.png" alt=":thinking:" title=":thinking:" class="emoji" />'
-      .replace(/"/g, '\'')
-
-    it('correctly replaces shortcodes in supplied string', () => {
-      const result = addEmojis('This post has :image: emoji and :thinking: emoji', emojis)
-      expect(result).to.include(thinkHtml)
-      expect(result).to.include(imageHtml)
-    })
-
-    it('handles consecutive emojis correctly', () => {
-      const result = addEmojis('Lelel emoji spam :thinking::thinking::thinking::thinking:', emojis)
-      expect(result).to.include(thinkHtml + thinkHtml + thinkHtml + thinkHtml)
-    })
-
-    it('Doesn\'t replace nonexistent emojis', () => {
-      const result = addEmojis('Admin add the :tenshi: emoji', emojis)
-      expect(result).to.equal('Admin add the :tenshi: emoji')
-    })
-
-    it('Doesn\'t blow up on regex special characters', () => {
-      const emojis = makeMockEmojiMasto([{
-        shortcode: 'c++'
-      }, {
-        shortcode: '[a-z] {|}*'
-      }])
-      const result = addEmojis('This post has :c++: emoji and :[a-z] {|}*: emoji', emojis)
-      expect(result).to.include('title=\':c++:\'')
-      expect(result).to.include('title=\':[a-z] {|}*:\'')
-    })
-  })
-
   describe('Link header pagination', () => {
     it('Parses min and max ids as integers', () => {
       const linkHeader = '<https://example.com/api/v1/notifications?max_id=861676>; rel="next", <https://example.com/api/v1/notifications?min_id=861741>; rel="prev"'
diff --git a/test/unit/specs/services/html_converter/html_line_converter.spec.js b/test/unit/specs/services/html_converter/html_line_converter.spec.js
new file mode 100644
index 00000000..86bd7e8b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/html_line_converter.spec.js
@@ -0,0 +1,171 @@
+import { convertHtmlToLines } from 'src/services/html_converter/html_line_converter.service.js'
+
+const greentextHandle = new Set(['p', 'div'])
+const mapOnlyText = (processor) => (input) => {
+  if (input.text && input.level.every(l => greentextHandle.has(l))) {
+    return processor(input.text)
+  } else if (input.text) {
+    return input.text
+  } else {
+    return input
+  }
+}
+
+describe('html_line_converter', () => {
+  describe('with processor that keeps original line should not make any changes to HTML when', () => {
+    const processorKeep = (line) => line
+    it('fed with regular HTML with newlines', () => {
+      const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with possibly broken HTML with invalid tags/composition', () => {
+      const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with very broken HTML with broken composition', () => {
+      const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with sorta valid HTML but tags aren\'t closed', () => {
+      const inputOutput = 'just leaving a <div> hanging'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
+      const inputOutput = 'do you expect me to finish this <div class='
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
+      const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with maybe valid HTML? self-closing divs and ps', () => {
+      const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with valid XHTML containing a CDATA', () => {
+      const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with some recognized but not handled elements', () => {
+      const inputOutput = 'testing images\n\n<img src="benis.png">'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+  })
+  describe('with processor that replaces lines with word "_" should match expected line when', () => {
+    const processorReplace = (line) => '_'
+    it('fed with regular HTML with newlines', () => {
+      const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
+      const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with possibly broken HTML with invalid tags/composition', () => {
+      const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with very broken HTML with broken composition', () => {
+      const input = '</p> lmao what </div> whats going on <div> wha <p>'
+      const output = '_<div>_<p>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with sorta valid HTML but tags aren\'t closed', () => {
+      const input = 'just leaving a <div> hanging'
+      const output = '_<div>_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
+      const input = 'do you expect me to finish this <div class='
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
+      const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
+      const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with maybe valid HTML? (XHTML) self-closing divs and ps', () => {
+      const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
+      const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with valid XHTML containing a CDATA', () => {
+      const input = 'Yes, it is me, <![CDATA[DIO]]>'
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('Testing handling ignored blocks', () => {
+      const input = `
+      <pre><code>&gt; rei = &quot;0&quot;
+      &#39;0&#39;
+      &gt; rei == 0
+      true
+      &gt; rei == null
+      false</code></pre><blockquote>That, christian-like JS diagram but it’s evangelion instead.</blockquote>
+      `
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(input)
+    })
+    it('Testing handling ignored blocks 2', () => {
+      const input = `
+      <blockquote>An SSL error has happened.</blockquote><p>Shakespeare</p>
+      `
+      const output = `
+      <blockquote>An SSL error has happened.</blockquote><p>_</p>
+      `
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+  })
+})
diff --git a/test/unit/specs/services/html_converter/html_tree_converter.spec.js b/test/unit/specs/services/html_converter/html_tree_converter.spec.js
new file mode 100644
index 00000000..7283021b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/html_tree_converter.spec.js
@@ -0,0 +1,132 @@
+import { convertHtmlToTree } from 'src/services/html_converter/html_tree_converter.service.js'
+
+describe('html_tree_converter', () => {
+  describe('convertHtmlToTree', () => {
+    it('converts html into a tree structure', () => {
+      const input = '1 <p>2</p> <b>3<img src="a">4</b>5'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        [
+          '<p>',
+          ['2'],
+          '</p>'
+        ],
+        ' ',
+        [
+          '<b>',
+          [
+            '3',
+            ['<img src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts html to tree while preserving tag formatting', () => {
+      const input = '1 <p >2</p><b >3<img   src="a">4</b>5'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        [
+          '<p >',
+          ['2'],
+          '</p>'
+        ],
+        [
+          '<b >',
+          [
+            '3',
+            ['<img   src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts semi-broken html', () => {
+      const input = '1 <br> 2 <p> 42'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        ['<br>'],
+        ' 2 ',
+        [
+          '<p>',
+          [' 42']
+        ]
+      ])
+    })
+    it('realistic case 1', () => {
+      const input = '<p><span class="h-card"><a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">@<span>benis</span></a></span> <span class="h-card"><a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">@<span>hj</span></a></span> nice</p>'
+      expect(convertHtmlToTree(input)).to.eql([
+        [
+          '<p>',
+          [
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'benis'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' ',
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'hj'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' nice'
+          ],
+          '</p>'
+        ]
+      ])
+    })
+    it('realistic case 2', () => {
+      const inputOutput = 'Country improv: give me a city<br/>Audience: Memphis<br/>Improv troupe: come on, a better one<br/>Audience: el paso'
+      expect(convertHtmlToTree(inputOutput)).to.eql([
+        'Country improv: give me a city',
+        [
+          '<br/>'
+        ],
+        'Audience: Memphis',
+        [
+          '<br/>'
+        ],
+        'Improv troupe: come on, a better one',
+        [
+          '<br/>'
+        ],
+        'Audience: el paso'
+      ])
+    })
+  })
+})
diff --git a/test/unit/specs/services/html_converter/utility.spec.js b/test/unit/specs/services/html_converter/utility.spec.js
new file mode 100644
index 00000000..cf6fd99b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/utility.spec.js
@@ -0,0 +1,37 @@
+import { processTextForEmoji, getAttrs } from 'src/services/html_converter/utility.service.js'
+
+describe('html_converter utility', () => {
+  describe('processTextForEmoji', () => {
+    it('processes all emoji in text', () => {
+      const input = 'Hello from finland! :lol: We have best water! :lmao:'
+      const emojis = [
+        { shortcode: 'lol', src: 'LOL' },
+        { shortcode: 'lmao', src: 'LMAO' }
+      ]
+      const processor = ({ shortcode, src }) => ({ shortcode, src })
+      expect(processTextForEmoji(input, emojis, processor)).to.eql([
+        'Hello from finland! ',
+        { shortcode: 'lol', src: 'LOL' },
+        ' We have best water! ',
+        { shortcode: 'lmao', src: 'LMAO' }
+      ])
+    })
+    it('leaves text as is', () => {
+      const input = 'Number one: that\'s terror'
+      const emojis = []
+      const processor = ({ shortcode, src }) => ({ shortcode, src })
+      expect(processTextForEmoji(input, emojis, processor)).to.eql([
+        'Number one: that\'s terror'
+      ])
+    })
+  })
+
+  describe('getAttrs', () => {
+    it('extracts arguments from tag', () => {
+      const input = '<img src="boop" cool ebin=\'true\'>'
+      const output = { src: 'boop', cool: true, ebin: 'true' }
+
+      expect(getAttrs(input)).to.eql(output)
+    })
+  })
+})
diff --git a/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js b/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js
deleted file mode 100644
index f301429d..00000000
--- a/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js
+++ /dev/null
@@ -1,96 +0,0 @@
-import { processHtml } from 'src/services/tiny_post_html_processor/tiny_post_html_processor.service.js'
-
-describe('TinyPostHTMLProcessor', () => {
-  describe('with processor that keeps original line should not make any changes to HTML when', () => {
-    const processorKeep = (line) => line
-    it('fed with regular HTML with newlines', () => {
-      const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with possibly broken HTML with invalid tags/composition', () => {
-      const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with very broken HTML with broken composition', () => {
-      const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with sorta valid HTML but tags aren\'t closed', () => {
-      const inputOutput = 'just leaving a <div> hanging'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
-      const inputOutput = 'do you expect me to finish this <div class='
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
-      const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with maybe valid HTML? self-closing divs and ps', () => {
-      const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with valid XHTML containing a CDATA', () => {
-      const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-  })
-  describe('with processor that replaces lines with word "_" should match expected line when', () => {
-    const processorReplace = (line) => '_'
-    it('fed with regular HTML with newlines', () => {
-      const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
-      const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with possibly broken HTML with invalid tags/composition', () => {
-      const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with very broken HTML with broken composition', () => {
-      const input = '</p> lmao what </div> whats going on <div> wha <p>'
-      const output = '</p>_</div>_<div>_<p>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with sorta valid HTML but tags aren\'t closed', () => {
-      const input = 'just leaving a <div> hanging'
-      const output = '_<div>_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
-      const input = 'do you expect me to finish this <div class='
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
-      const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
-      const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with maybe valid HTML? self-closing divs and ps', () => {
-      const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
-      const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with valid XHTML containing a CDATA', () => {
-      const input = 'Yes, it is me, <![CDATA[DIO]]>'
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-  })
-})