Strip Unicode format characters prior post processing

lib/diaspora/message_renderer.rb

@@ -14,6 +14,10 @@ module Diaspora
           processor.instance_exec(&block)
           processor.message
         end
+
+        def normalize message
+          message.delete("\u202a-\u202e\u200b-\u200f")
+        end
       end
 
       attr_reader :message, :options
@@ -91,6 +95,10 @@ module Diaspora
       def camo_urls
         @message = Diaspora::Camo.from_markdown(@message)
       end
+
+      def normalize
+        @message = self.class.normalize(@message)
+      end
     end
 
     DEFAULTS = {mentioned_people: [],
@@ -172,6 +180,7 @@ module Diaspora
     # @param [Hash] opts Override global output options, see {#initialize}
     def plain_text_for_json opts={}
       process(opts) {
+        normalize
         camo_urls if AppConfig.privacy.camo.proxy_markdown_images?
       }
     end
@@ -180,6 +189,7 @@ module Diaspora
     def html opts={}
       process(opts) {
         escape
+        normalize
         render_mentions
         render_tags
         squish
@@ -191,6 +201,7 @@ module Diaspora
     def markdownified opts={}
       process(opts) {
         process_newlines
+        normalize
         camo_urls if AppConfig.privacy.camo.proxy_markdown_images?
         markdownify
         render_mentions

lib/diaspora/taggable.rb

@@ -25,7 +25,7 @@ module Diaspora
     end
 
     def tag_strings
-      (send(self.class.field_with_tags) || "")
+      MessageRenderer::Processor.normalize(send(self.class.field_with_tags) || "")
         .scan(/(?:^|\s)#([#{ActsAsTaggableOn::Tag.tag_text_regexp}]+|<3)/u)
         .map(&:first)
         .uniq(&:downcase)

spec/lib/diaspora/message_renderer_spec.rb

@@ -50,6 +50,12 @@ describe Diaspora::MessageRenderer do
       expect(message(entities).html).to eq entities
     end
 
+    it 'normalizes' do
+      expect(
+        message("\u202a#\u200eUSA\u202c").markdownified
+      ).to eq %(<p><a class="tag" href="/tags/USA">#USA</a></p>\n)
+    end
+
     context 'with mentions' do
       it 'makes hovercard links for mentioned people' do
         expect(
@@ -107,8 +113,14 @@ describe Diaspora::MessageRenderer do
 
     it 'autolinks standard url links' do
       expect(
-        message("http://joindiaspora.com/"
-      ).markdownified).to include 'href="http://joindiaspora.com/"'
+        message("http://joindiaspora.com/").markdownified
+      ).to include 'href="http://joindiaspora.com/"'
+    end
+
+    it 'normalizes' do
+      expect(
+        message("\u202a#\u200eUSA\u202c").markdownified
+      ).to eq %(<p><a class="tag" href="/tags/USA">#USA</a></p>\n)
     end
 
     context 'when formatting status messages' do
@@ -184,4 +196,12 @@ describe Diaspora::MessageRenderer do
       expect(message(text).urls).to eq ["https://www.antifainfoblatt.de/sites/default/files/public/styles/front_full/public/jockpalfreeman.png?itok=OPjHKpmt", "https://www.antifainfoblatt.de/artikel/%E2%80%9Eschlie%C3%9Flich-waren-es-zu-viele%E2%80%9C"]
     end
   end
+
+  describe "#plain_text_for_json" do
+    it 'normalizes' do
+      expect(
+        message("\u202a#\u200eUSA\u202c").plain_text_for_json
+      ).to eq '#USA'
+    end
+  end
 end

spec/shared_behaviors/taggable.rb

@@ -139,6 +139,7 @@ shared_examples_for "it is taggable" do
         '#-initialhyphen'        => '-initialhyphen',
         '#-initialhyphen tag'    => '-initialhyphen',
         '#-initial-hyphen'       => '-initial-hyphen',
+        "\u202a#\u200eUSA\u202c" => 'USA'
       }
 
       expected.each do |text,hashtag|