Merge remote-tracking branch 'origin/develop' into settings-and-filtering

* origin/develop: (169 commits)
  Improve the user card for deactivated users
  Update CHANGELOG.md
  Update CHANGELOG.md
  Allow canceling a follow request
  Simple policy reasons for instance specific policies
  entity_normalizer: Escape name when parsing user
  Translated using Weblate (Spanish)
  Translated using Weblate (Catalan)
  Translated using Weblate (Korean)
  Translated using Weblate (Japanese (ja_PEDANTIC))
  Translated using Weblate (Indonesian)
  Translated using Weblate (Esperanto)
  Translated using Weblate (Vietnamese)
  Translated using Weblate (Italian)
  Translated using Weblate (Vietnamese)
  Translated using Weblate (Indonesian)
  Translated using Weblate (Italian)
  Translated using Weblate (Vietnamese)
  Translated using Weblate (Indonesian)
  Translated using Weblate (Chinese (Simplified))
  ...

6 files changed, 821 insertions, 178 deletions

diff --git a/test/unit/specs/components/rich_content.spec.js b/test/unit/specs/components/rich_content.spec.js
new file mode 100644
index 00000000..f6c478a9
--- /dev/null
+++ b/test/unit/specs/components/rich_content.spec.js
@@ -0,0 +1,480 @@
+import { mount, shallowMount, createLocalVue } from '@vue/test-utils'
+import RichContent from 'src/components/rich_content/rich_content.jsx'
+
+const localVue = createLocalVue()
+const attentions = []
+
+const makeMention = (who) => {
+  attentions.push({ statusnet_profile_url: `https://fake.tld/@${who}` })
+  return `<span class="h-card"><a class="u-url mention" href="https://fake.tld/@${who}">@<span>${who}</span></a></span>`
+}
+const p = (...data) => `<p>${data.join('')}</p>`
+const compwrap = (...data) => `<span class="RichContent">${data.join('')}</span>`
+const mentionsLine = (times) => [
+  '<mentionsline-stub mentions="',
+  new Array(times).fill('[object Object]').join(','),
+  '"></mentionsline-stub>'
+].join('')
+
+describe('RichContent', () => {
+  it('renders simple post without exploding', () => {
+    const html = p('Hello world!')
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('unescapes everything as needed', () => {
+    const html = [
+      p('Testing &#39;em all'),
+      'Testing &#39;em all'
+    ].join('')
+    const expected = [
+      p('Testing \'em all'),
+      'Testing \'em all'
+    ].join('')
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('replaces mention with mentionsline', () => {
+    const html = p(
+      makeMention('John'),
+      ' how are you doing today?'
+    )
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(p(
+      mentionsLine(1),
+      ' how are you doing today?'
+    )))
+  })
+
+  it('replaces mentions at the end of the hellpost', () => {
+    const html = [
+      p('How are you doing today, fine gentlemen?'),
+      p(
+        makeMention('John'),
+        makeMention('Josh'),
+        makeMention('Jeremy')
+      )
+    ].join('')
+    const expected = [
+      p(
+        'How are you doing today, fine gentlemen?'
+      ),
+      // TODO fix this extra line somehow?
+      p(
+        '<mentionsline-stub mentions="',
+        '[object Object],',
+        '[object Object],',
+        '[object Object]',
+        '"></mentionsline-stub>'
+      )
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Does not touch links if link handling is disabled', () => {
+    const html = [
+      [
+        makeMention('Jack'),
+        'let\'s meet up with ',
+        makeMention('Janet')
+      ].join(''),
+      [
+        makeMention('John'),
+        makeMention('Josh'),
+        makeMention('Jeremy')
+      ].join('')
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Adds greentext and cyantext to the post', () => {
+    const html = [
+      '&gt;preordering videogames',
+      '&gt;any year'
+    ].join('\n')
+    const expected = [
+      '<span class="greentext">&gt;preordering videogames</span>',
+      '<span class="greentext">&gt;any year</span>'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Does not add greentext and cyantext if setting is set to false', () => {
+    const html = [
+      '&gt;preordering videogames',
+      '&gt;any year'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Adds emoji to post', () => {
+    const html = p('Ebin :DDDD :spurdo:')
+    const expected = p(
+      'Ebin :DDDD ',
+      '<anonymous-stub alt=":spurdo:" src="about:blank" title=":spurdo:" class="emoji img"></anonymous-stub>'
+    )
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [{ url: 'about:blank', shortcode: 'spurdo' }],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('Doesn\'t add nonexistent emoji to post', () => {
+    const html = p('Lol :lol:')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: false,
+        greentext: false,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(html))
+  })
+
+  it('Greentext + last mentions', () => {
+    const html = [
+      '&gt;quote',
+      makeMention('lol'),
+      '&gt;quote',
+      '&gt;quote'
+    ].join('\n')
+    const expected = [
+      '<span class="greentext">&gt;quote</span>',
+      mentionsLine(1),
+      '<span class="greentext">&gt;quote</span>',
+      '<span class="greentext">&gt;quote</span>'
+    ].join('\n')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('One buggy example', () => {
+    const html = [
+      'Bruh',
+      'Bruh',
+      [
+        makeMention('foo'),
+        makeMention('bar'),
+        makeMention('baz')
+      ].join(''),
+      'Bruh'
+    ].join('<br>')
+    const expected = [
+      'Bruh',
+      'Bruh',
+      mentionsLine(3),
+      'Bruh'
+    ].join('<br>')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('buggy example/hashtags', () => {
+    const html = [
+      '<p>',
+      '<a href="http://macrochan.org/images/N/H/NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg">',
+      'NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg</a>',
+      ' <a class="hashtag" data-tag="nou" href="https://shitposter.club/tag/nou">',
+      '#nou</a>',
+      ' <a class="hashtag" data-tag="screencap" href="https://shitposter.club/tag/screencap">',
+      '#screencap</a>',
+      ' </p>'
+    ].join('')
+    const expected = [
+      '<p>',
+      '<a href="http://macrochan.org/images/N/H/NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg" target="_blank">',
+      'NHCMDUXJPPZ6M3Z2CQ6D2EBRSWGE7MZY.jpg</a>',
+      ' <hashtaglink-stub url="https://shitposter.club/tag/nou" content="#nou" tag="nou">',
+      '</hashtaglink-stub>',
+      ' <hashtaglink-stub url="https://shitposter.club/tag/screencap" content="#screencap" tag="screencap">',
+      '</hashtaglink-stub>',
+      ' </p>'
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('rich contents of a mention are handled properly', () => {
+    attentions.push({ statusnet_profile_url: 'lol' })
+    const html = [
+      p(
+        '<a href="lol" class="mention">',
+        '<span>',
+        'https://</span>',
+        '<span>',
+        'lol.tld/</span>',
+        '<span>',
+        '</span>',
+        '</a>'
+      ),
+      p(
+        'Testing'
+      )
+    ].join('')
+    const expected = [
+      p(
+        '<span class="MentionsLine">',
+        '<span class="MentionLink mention-link">',
+        '<a href="lol" target="_blank" class="original">',
+        '<span>',
+        'https://</span>',
+        '<span>',
+        'lol.tld/</span>',
+        '<span>',
+        '</span>',
+        '</a>',
+        ' ',
+        '<!---->', // v-if placeholder, mentionlink's "new" (i.e. rich) display
+        '</span>',
+        '<!---->', // v-if placeholder, mentionsline's extra mentions and stuff
+        '</span>'
+      ),
+      p(
+        'Testing'
+      )
+    ].join('')
+
+    const wrapper = mount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it('rich contents of a link are handled properly', () => {
+    const html = [
+      '<p>',
+      'Freenode is dead.</p>',
+      '<p>',
+      '<a href="https://isfreenodedeadyet.com/">',
+      '<span>',
+      'https://</span>',
+      '<span>',
+      'isfreenodedeadyet.com/</span>',
+      '<span>',
+      '</span>',
+      '</a>',
+      '</p>'
+    ].join('')
+    const expected = [
+      '<p>',
+      'Freenode is dead.</p>',
+      '<p>',
+      '<a href="https://isfreenodedeadyet.com/" target="_blank">',
+      '<span>',
+      'https://</span>',
+      '<span>',
+      'isfreenodedeadyet.com/</span>',
+      '<span>',
+      '</span>',
+      '</a>',
+      '</p>'
+    ].join('')
+
+    const wrapper = shallowMount(RichContent, {
+      localVue,
+      propsData: {
+        attentions,
+        handleLinks: true,
+        greentext: true,
+        emoji: [],
+        html
+      }
+    })
+
+    expect(wrapper.html()).to.eql(compwrap(expected))
+  })
+
+  it.skip('[INFORMATIVE] Performance testing, 10 000 simple posts', () => {
+    const amount = 20
+
+    const onePost = p(
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      makeMention('Lain'),
+      ' i just landed in l a where are you'
+    )
+
+    const TestComponent = {
+      template: `
+      <div v-if="!vhtml">
+        ${new Array(amount).fill(`<RichContent html="${onePost}" :greentext="true" :handleLinks="handeLinks" :emoji="[]" :attentions="attentions"/>`)}
+      </div>
+      <div v-else="vhtml">
+        ${new Array(amount).fill(`<div v-html="${onePost}"/>`)}
+      </div>
+      `,
+      props: ['handleLinks', 'attentions', 'vhtml']
+    }
+    console.log(1)
+
+    const ptest = (handleLinks, vhtml) => {
+      const t0 = performance.now()
+
+      const wrapper = mount(TestComponent, {
+        localVue,
+        propsData: {
+          attentions,
+          handleLinks,
+          vhtml
+        }
+      })
+
+      const t1 = performance.now()
+
+      wrapper.destroy()
+
+      const t2 = performance.now()
+
+      return `Mount: ${t1 - t0}ms, destroy: ${t2 - t1}ms, avg ${(t1 - t0) / amount}ms - ${(t2 - t1) / amount}ms per item`
+    }
+
+    console.log(`${amount} items with links handling:`)
+    console.log(ptest(true))
+    console.log(`${amount} items without links handling:`)
+    console.log(ptest(false))
+    console.log(`${amount} items plain v-html:`)
+    console.log(ptest(false, true))
+  })
+})
diff --git a/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js b/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
index 759539e0..03fb32c9 100644
--- a/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
+++ b/test/unit/specs/services/entity_normalizer/entity_normalizer.spec.js
@@ -1,4 +1,4 @@
-import { parseStatus, parseUser, parseNotification, addEmojis, parseLinkHeaderPagination } from '../../../../../src/services/entity_normalizer/entity_normalizer.service.js'
+import { parseStatus, parseUser, parseNotification, parseLinkHeaderPagination } from '../../../../../src/services/entity_normalizer/entity_normalizer.service.js'
 import mastoapidata from '../../../../fixtures/mastoapi.json'
 import qvitterapidata from '../../../../fixtures/statuses.json'
 
@@ -23,7 +23,6 @@ const makeMockStatusQvitter = (overrides = {}) => {
     repeat_num: 0,
     repeated: false,
     statusnet_conversation_id: '16300488',
-    statusnet_html: '<p>haha benis</p>',
     summary: null,
     tags: [],
     text: 'haha benis',
@@ -232,22 +231,6 @@ describe('API Entities normalizer', () => {
         expect(parsedRepeat).to.have.property('retweeted_status')
         expect(parsedRepeat).to.have.deep.property('retweeted_status.id', 'deadbeef')
       })
-
-      it('adds emojis to post content', () => {
-        const post = makeMockStatusMasto({ emojis: makeMockEmojiMasto(), content: 'Makes you think :thinking:' })
-
-        const parsedPost = parseStatus(post)
-
-        expect(parsedPost).to.have.property('statusnet_html').that.contains('<img')
-      })
-
-      it('adds emojis to subject line', () => {
-        const post = makeMockStatusMasto({ emojis: makeMockEmojiMasto(), spoiler_text: 'CW: 300 IQ :thinking:' })
-
-        const parsedPost = parseStatus(post)
-
-        expect(parsedPost).to.have.property('summary_html').that.contains('<img')
-      })
     })
   })
 
@@ -261,35 +244,6 @@ describe('API Entities normalizer', () => {
       expect(parseUser(remote)).to.have.property('is_local', false)
     })
 
-    it('adds emojis to user name', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), display_name: 'The :thinking: thinker' })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('name_html').that.contains('<img')
-    })
-
-    it('adds emojis to user bio', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), note: 'Hello i like to :thinking: a lot' })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('description_html').that.contains('<img')
-    })
-
-    it('adds emojis to user profile fields', () => {
-      const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), fields: [{ name: ':thinking:', value: ':image:' }] })
-
-      const parsedUser = parseUser(user)
-
-      expect(parsedUser).to.have.property('fields_html').to.be.an('array')
-
-      const field = parsedUser.fields_html[0]
-
-      expect(field).to.have.property('name').that.contains('<img')
-      expect(field).to.have.property('value').that.contains('<img')
-    })
-
     it('removes html tags from user profile fields', () => {
       const user = makeMockUserMasto({ emojis: makeMockEmojiMasto(), fields: [{ name: 'user', value: '<a rel="me" href="https://example.com/@user">@user</a>' }] })
 
@@ -355,41 +309,6 @@ describe('API Entities normalizer', () => {
     })
   })
 
-  describe('MastoAPI emoji adder', () => {
-    const emojis = makeMockEmojiMasto()
-    const imageHtml = '<img src="https://example.com/image.png" alt=":image:" title=":image:" class="emoji" />'
-      .replace(/"/g, '\'')
-    const thinkHtml = '<img src="https://example.com/think.png" alt=":thinking:" title=":thinking:" class="emoji" />'
-      .replace(/"/g, '\'')
-
-    it('correctly replaces shortcodes in supplied string', () => {
-      const result = addEmojis('This post has :image: emoji and :thinking: emoji', emojis)
-      expect(result).to.include(thinkHtml)
-      expect(result).to.include(imageHtml)
-    })
-
-    it('handles consecutive emojis correctly', () => {
-      const result = addEmojis('Lelel emoji spam :thinking::thinking::thinking::thinking:', emojis)
-      expect(result).to.include(thinkHtml + thinkHtml + thinkHtml + thinkHtml)
-    })
-
-    it('Doesn\'t replace nonexistent emojis', () => {
-      const result = addEmojis('Admin add the :tenshi: emoji', emojis)
-      expect(result).to.equal('Admin add the :tenshi: emoji')
-    })
-
-    it('Doesn\'t blow up on regex special characters', () => {
-      const emojis = makeMockEmojiMasto([{
-        shortcode: 'c++'
-      }, {
-        shortcode: '[a-z] {|}*'
-      }])
-      const result = addEmojis('This post has :c++: emoji and :[a-z] {|}*: emoji', emojis)
-      expect(result).to.include('title=\':c++:\'')
-      expect(result).to.include('title=\':[a-z] {|}*:\'')
-    })
-  })
-
   describe('Link header pagination', () => {
     it('Parses min and max ids as integers', () => {
       const linkHeader = '<https://example.com/api/v1/notifications?max_id=861676>; rel="next", <https://example.com/api/v1/notifications?min_id=861741>; rel="prev"'
diff --git a/test/unit/specs/services/html_converter/html_line_converter.spec.js b/test/unit/specs/services/html_converter/html_line_converter.spec.js
new file mode 100644
index 00000000..86bd7e8b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/html_line_converter.spec.js
@@ -0,0 +1,171 @@
+import { convertHtmlToLines } from 'src/services/html_converter/html_line_converter.service.js'
+
+const greentextHandle = new Set(['p', 'div'])
+const mapOnlyText = (processor) => (input) => {
+  if (input.text && input.level.every(l => greentextHandle.has(l))) {
+    return processor(input.text)
+  } else if (input.text) {
+    return input.text
+  } else {
+    return input
+  }
+}
+
+describe('html_line_converter', () => {
+  describe('with processor that keeps original line should not make any changes to HTML when', () => {
+    const processorKeep = (line) => line
+    it('fed with regular HTML with newlines', () => {
+      const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with possibly broken HTML with invalid tags/composition', () => {
+      const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with very broken HTML with broken composition', () => {
+      const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with sorta valid HTML but tags aren\'t closed', () => {
+      const inputOutput = 'just leaving a <div> hanging'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
+      const inputOutput = 'do you expect me to finish this <div class='
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
+      const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with maybe valid HTML? self-closing divs and ps', () => {
+      const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with valid XHTML containing a CDATA', () => {
+      const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+
+    it('fed with some recognized but not handled elements', () => {
+      const inputOutput = 'testing images\n\n<img src="benis.png">'
+      const result = convertHtmlToLines(inputOutput)
+      const comparableResult = result.map(mapOnlyText(processorKeep)).join('')
+      expect(comparableResult).to.eql(inputOutput)
+    })
+  })
+  describe('with processor that replaces lines with word "_" should match expected line when', () => {
+    const processorReplace = (line) => '_'
+    it('fed with regular HTML with newlines', () => {
+      const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
+      const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with possibly broken HTML with invalid tags/composition', () => {
+      const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with very broken HTML with broken composition', () => {
+      const input = '</p> lmao what </div> whats going on <div> wha <p>'
+      const output = '_<div>_<p>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with sorta valid HTML but tags aren\'t closed', () => {
+      const input = 'just leaving a <div> hanging'
+      const output = '_<div>_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
+      const input = 'do you expect me to finish this <div class='
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
+      const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
+      const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with maybe valid HTML? (XHTML) self-closing divs and ps', () => {
+      const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
+      const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('fed with valid XHTML containing a CDATA', () => {
+      const input = 'Yes, it is me, <![CDATA[DIO]]>'
+      const output = '_'
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+
+    it('Testing handling ignored blocks', () => {
+      const input = `
+      <pre><code>&gt; rei = &quot;0&quot;
+      &#39;0&#39;
+      &gt; rei == 0
+      true
+      &gt; rei == null
+      false</code></pre><blockquote>That, christian-like JS diagram but it’s evangelion instead.</blockquote>
+      `
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(input)
+    })
+    it('Testing handling ignored blocks 2', () => {
+      const input = `
+      <blockquote>An SSL error has happened.</blockquote><p>Shakespeare</p>
+      `
+      const output = `
+      <blockquote>An SSL error has happened.</blockquote><p>_</p>
+      `
+      const result = convertHtmlToLines(input)
+      const comparableResult = result.map(mapOnlyText(processorReplace)).join('')
+      expect(comparableResult).to.eql(output)
+    })
+  })
+})
diff --git a/test/unit/specs/services/html_converter/html_tree_converter.spec.js b/test/unit/specs/services/html_converter/html_tree_converter.spec.js
new file mode 100644
index 00000000..7283021b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/html_tree_converter.spec.js
@@ -0,0 +1,132 @@
+import { convertHtmlToTree } from 'src/services/html_converter/html_tree_converter.service.js'
+
+describe('html_tree_converter', () => {
+  describe('convertHtmlToTree', () => {
+    it('converts html into a tree structure', () => {
+      const input = '1 <p>2</p> <b>3<img src="a">4</b>5'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        [
+          '<p>',
+          ['2'],
+          '</p>'
+        ],
+        ' ',
+        [
+          '<b>',
+          [
+            '3',
+            ['<img src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts html to tree while preserving tag formatting', () => {
+      const input = '1 <p >2</p><b >3<img   src="a">4</b>5'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        [
+          '<p >',
+          ['2'],
+          '</p>'
+        ],
+        [
+          '<b >',
+          [
+            '3',
+            ['<img   src="a">'],
+            '4'
+          ],
+          '</b>'
+        ],
+        '5'
+      ])
+    })
+    it('converts semi-broken html', () => {
+      const input = '1 <br> 2 <p> 42'
+      expect(convertHtmlToTree(input)).to.eql([
+        '1 ',
+        ['<br>'],
+        ' 2 ',
+        [
+          '<p>',
+          [' 42']
+        ]
+      ])
+    })
+    it('realistic case 1', () => {
+      const input = '<p><span class="h-card"><a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">@<span>benis</span></a></span> <span class="h-card"><a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">@<span>hj</span></a></span> nice</p>'
+      expect(convertHtmlToTree(input)).to.eql([
+        [
+          '<p>',
+          [
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="9wRC6T2ZZiKWJ0vUi8" href="https://cawfee.club/users/benis" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'benis'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' ',
+            [
+              '<span class="h-card">',
+              [
+                [
+                  '<a class="u-url mention" data-user="194" href="https://shigusegubu.club/users/hj" rel="ugc">',
+                  [
+                    '@',
+                    [
+                      '<span>',
+                      [
+                        'hj'
+                      ],
+                      '</span>'
+                    ]
+                  ],
+                  '</a>'
+                ]
+              ],
+              '</span>'
+            ],
+            ' nice'
+          ],
+          '</p>'
+        ]
+      ])
+    })
+    it('realistic case 2', () => {
+      const inputOutput = 'Country improv: give me a city<br/>Audience: Memphis<br/>Improv troupe: come on, a better one<br/>Audience: el paso'
+      expect(convertHtmlToTree(inputOutput)).to.eql([
+        'Country improv: give me a city',
+        [
+          '<br/>'
+        ],
+        'Audience: Memphis',
+        [
+          '<br/>'
+        ],
+        'Improv troupe: come on, a better one',
+        [
+          '<br/>'
+        ],
+        'Audience: el paso'
+      ])
+    })
+  })
+})
diff --git a/test/unit/specs/services/html_converter/utility.spec.js b/test/unit/specs/services/html_converter/utility.spec.js
new file mode 100644
index 00000000..cf6fd99b
--- /dev/null
+++ b/test/unit/specs/services/html_converter/utility.spec.js
@@ -0,0 +1,37 @@
+import { processTextForEmoji, getAttrs } from 'src/services/html_converter/utility.service.js'
+
+describe('html_converter utility', () => {
+  describe('processTextForEmoji', () => {
+    it('processes all emoji in text', () => {
+      const input = 'Hello from finland! :lol: We have best water! :lmao:'
+      const emojis = [
+        { shortcode: 'lol', src: 'LOL' },
+        { shortcode: 'lmao', src: 'LMAO' }
+      ]
+      const processor = ({ shortcode, src }) => ({ shortcode, src })
+      expect(processTextForEmoji(input, emojis, processor)).to.eql([
+        'Hello from finland! ',
+        { shortcode: 'lol', src: 'LOL' },
+        ' We have best water! ',
+        { shortcode: 'lmao', src: 'LMAO' }
+      ])
+    })
+    it('leaves text as is', () => {
+      const input = 'Number one: that\'s terror'
+      const emojis = []
+      const processor = ({ shortcode, src }) => ({ shortcode, src })
+      expect(processTextForEmoji(input, emojis, processor)).to.eql([
+        'Number one: that\'s terror'
+      ])
+    })
+  })
+
+  describe('getAttrs', () => {
+    it('extracts arguments from tag', () => {
+      const input = '<img src="boop" cool ebin=\'true\'>'
+      const output = { src: 'boop', cool: true, ebin: 'true' }
+
+      expect(getAttrs(input)).to.eql(output)
+    })
+  })
+})
diff --git a/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js b/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js
deleted file mode 100644
index f301429d..00000000
--- a/test/unit/specs/services/tiny_post_html_processor/tiny_post_html_processor.spec.js
+++ /dev/null
@@ -1,96 +0,0 @@
-import { processHtml } from 'src/services/tiny_post_html_processor/tiny_post_html_processor.service.js'
-
-describe('TinyPostHTMLProcessor', () => {
-  describe('with processor that keeps original line should not make any changes to HTML when', () => {
-    const processorKeep = (line) => line
-    it('fed with regular HTML with newlines', () => {
-      const inputOutput = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with possibly broken HTML with invalid tags/composition', () => {
-      const inputOutput = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with very broken HTML with broken composition', () => {
-      const inputOutput = '</p> lmao what </div> whats going on <div> wha <p>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with sorta valid HTML but tags aren\'t closed', () => {
-      const inputOutput = 'just leaving a <div> hanging'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
-      const inputOutput = 'do you expect me to finish this <div class='
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
-      const inputOutput = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with maybe valid HTML? self-closing divs and ps', () => {
-      const inputOutput = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-
-    it('fed with valid XHTML containing a CDATA', () => {
-      const inputOutput = 'Yes, it is me, <![CDATA[DIO]]>'
-      expect(processHtml(inputOutput, processorKeep)).to.eql(inputOutput)
-    })
-  })
-  describe('with processor that replaces lines with word "_" should match expected line when', () => {
-    const processorReplace = (line) => '_'
-    it('fed with regular HTML with newlines', () => {
-      const input = '1<br/>2<p class="lol">3 4</p> 5 \n 6 <p > 7 <br> 8 </p> <br>\n<br/>'
-      const output = '_<br/>_<p class="lol">_</p>_\n_<p >_<br>_</p> <br>\n<br/>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with possibly broken HTML with invalid tags/composition', () => {
-      const input = '<feeee dwdwddddddw> <i>ayy<b>lm</i>ao</b> </section>'
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with very broken HTML with broken composition', () => {
-      const input = '</p> lmao what </div> whats going on <div> wha <p>'
-      const output = '</p>_</div>_<div>_<p>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with sorta valid HTML but tags aren\'t closed', () => {
-      const input = 'just leaving a <div> hanging'
-      const output = '_<div>_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with not really HTML at this point... tags that aren\'t finished', () => {
-      const input = 'do you expect me to finish this <div class='
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with dubiously valid HTML (p within p and also div inside p)', () => {
-      const input = 'look ma <p> p \nwithin <p> p! </p> and a <br/><div>div!</div></p>'
-      const output = '_<p>_\n_<p>_</p>_<br/><div>_</div></p>'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with maybe valid HTML? self-closing divs and ps', () => {
-      const input = 'a <div class="what"/> what now <p aria-label="wtf"/> ?'
-      const output = '_<div class="what"/>_<p aria-label="wtf"/>_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-
-    it('fed with valid XHTML containing a CDATA', () => {
-      const input = 'Yes, it is me, <![CDATA[DIO]]>'
-      const output = '_'
-      expect(processHtml(input, processorReplace)).to.eql(output)
-    })
-  })
-})