SM IZ pared down markdownify to basic markdown, bumped version of RedCarpet

Gemfile

@@ -62,7 +62,7 @@ gem 'rails-i18n'
 # parsing
 
 gem 'nokogiri'
-gem 'redcarpet', :git => 'git://github.com/tanoku/redcarpet'
+gem 'redcarpet', "2.0.0b5"
 gem 'roxml', :git => 'git://github.com/Empact/roxml.git', :ref => '7ea9a9ffd2338aaef5b0'
 
 # queue

Gemfile.lock

@@ -56,12 +56,6 @@ GIT
       addressable (>= 2.1.1)
       eventmachine (>= 0.12.9)
 
-GIT
-  remote: git://github.com/tanoku/redcarpet
-  revision: 3c6b5807e8a5be51d769149e555b09bae8ae3228
-  specs:
-    redcarpet (2.0.0b3)
-
 GEM
   remote: http://rubygems.org/
   specs:
@@ -363,6 +357,7 @@ GEM
     rash (0.3.0)
       hashie (~> 1.0.0)
     rdoc (3.9.4)
+    redcarpet (2.0.0b5)
     redis (2.2.2)
     redis-namespace (0.8.0)
       redis (< 3.0.0)
@@ -518,7 +513,7 @@ DEPENDENCIES
   omniauth (= 0.2.6)
   rails (= 3.0.10)
   rails-i18n
-  redcarpet!
+  redcarpet (= 2.0.0b5)
   resque (= 1.10.0)
   resque-ensure-connected
   resque-timeout (= 1.0.0)

app/helpers/markdownify_helper.rb

@@ -31,6 +31,7 @@ module MarkdownifyHelper
 
     return '' if message.blank?
 
+    #renderer = Redcarpet::Render::HTML.new(render_options)
     renderer = Diaspora::Markdownify::HTML.new(render_options)
     markdown = Redcarpet::Markdown.new(renderer, markdown_options)
 

lib/diaspora/markdownify.rb

@@ -3,186 +3,13 @@ require 'erb'
 module Diaspora
   module Markdownify
     class HTML < Redcarpet::Render::HTML
-      attr_accessor :newlines, :specialchars, :youtube_maps, :vimeo_maps
 
-      def initialize(options={})
+      include ActionView::Helpers::TextHelper
-        super
+      include ActionView::Helpers::TagHelper
-
-        @newlines     = options.fetch(:newlines, true)
-        @specialchars = options.fetch(:specialchars, true)
-        @youtube_maps = options[:youtube_maps]||{}
-        @vimeo_maps   = options[:vimeo_maps] || {}
-      end
 
       def autolink(link, type)
-        return link if type == :email
+        auto_link(link, :link => :urls )
-        autolink_youtube(link) || autolink_vimeo(link) || autolink_simple(link)
       end
-
-      def autolink_simple(link)
-
-        # If there isn't *some* protocol, assume http
-        if link !~ %r{^\w+://}
-          link = "http://#{link}"
-        end
-
-        content = link.gsub(%r{^\w+://}, '')
-
-        %Q{<a target="_blank" href="#{link}">#{content}</a>}
-      end
-
-      def autolink_vimeo(link)
-        regex = %r{https?://(?:w{3}\.)?vimeo.com/(\d{6,})/?}
-        if link =~ regex
-          video_id = $1
-          if @vimeo_maps[video_id]
-            title = ERB::Util.h(CGI::unescape(@vimeo_maps[video_id]))
-          else
-            title = I18n.t 'application.helper.video_title.unknown'
-          end
-          return ' <a class="video-link" data-host="vimeo.com" data-video-id="' + 
-            video_id + '" href="' + link + '" target="_blank">Vimeo: ' + title + '</a>'
-        end
-        return
-      end
-
-      def autolink_youtube(link)
-        if link =~ YoutubeTitles::YOUTUBE_ID_REGEX
-          video_id = $1
-          anchor = $2 || ''
-
-          if @youtube_maps[video_id]
-            title = ERB::Util.h(CGI::unescape(@youtube_maps[video_id]))
-          else
-            title = I18n.t 'application.helper.video_title.unknown'
-          end
-          return ' <a class="video-link" data-host="youtube.com" data-video-id="' + 
-            video_id + '" data-anchor="' + anchor + 
-            '" href="'+ link + '" target="_blank">Youtube: ' + title + '</a>'
-        end
-        return
-      end
-
-      def block_code(text, language)
-        "<pre><code>\n#{text}</code></pre>"
-      end
-
-      def double_emphasis(text)
-        "<strong>#{text}</strong>"
-      end
-
-      def linebreak()
-        "<br />"
-      end
-
-      def link(link, title, content)
-        #hax
-        content ||=''
-
-        return autolink(link, 'url') if link == content 
-        
-        if link =~ Regexp.new(Regexp.escape(content))
-          return autolink(link, 'url')
-        end
-
-        if link !~ %r{^\w+://}
-          link = "http://#{link}"
-        end
-
-        tag = if title and content
-                %Q{<a target="_blank" href="#{link}" title="#{title}">#{content}</a>}
-              elsif content
-                %Q{<a target="_blank" href="#{link}">#{content}</a>}
-              else
-                autolink(link, 'url')
-              end
-        return tag
-      end
-
-      def paragraph(text)
-        #hax again... why is markdownify passing us nil?
-        text ||=''
-
-        if @newlines
-          br = linebreak
-
-          # in very clear cases, let newlines become <br /> tags
-          # Grabbed from Github flavored Markdown
-          text = text.gsub(/^[\w\<][^\n]*\n+/) do |x|
-            x =~ /\n{2}/ ? x : (x = x.strip; x << br)
-          end
-        end
-        return "<p>#{text}</p>"
-      end
-
-      def preprocess(full_document)
-        entities = [
-          ['&', '&amp;'],
-          ['>', '&gt;'],
-          ['<', '&lt;']
-        ]
-        entities.each do |original, replacement|
-          full_document = full_document.gsub(original, replacement)
-        end
-
-        if @specialchars
-          full_document = specialchars(full_document)
-        end
-
-        our_unsafe_chars = '()'
-        full_document = full_document.gsub(%r{
-          \[ \s*? ( [^ \] ]+ ) \s*? \]
-          (?: 
-            \( \s*? (\S+) \s*? (?: "([^"]+)" )? \) \s*? 
-          )
-        }xm) do |m|
-          content = $1
-          link = URI.escape($2, our_unsafe_chars)
-          title = $3
-
-          title_chunk = if title 
-                          %W{" #{title}"} 
-                        else 
-                          '' 
-                        end
-          %Q{[#{content}](#{link}#{title_chunk})}
-        end
-
-        return full_document
-      end
-
-
-
-      def single_emphasis(text)
-        "<em>#{text}</em>"
-      end
-
-      def specialchars(text)
-        if @specialchars
-          map = [
-            ["&lt;3", "&hearts;"],
-            ["&lt;-&gt;", "&#8596;"],
-            ["-&gt;", "&rarr;"],
-            ["&lt;-", "&larr;"],
-            ["\.\.\.", "&hellip;"],
-            ["(tm)", "&trade;"],
-            ["(r)", "&reg;"],
-            ["(c)", "&copy;"]
-          ]
-        end
-
-        map.each do |mapping|
-          text = text.gsub(mapping[0], mapping[1])
-        end
-
-        return text
-      end
-
-
-      def triple_emphasis(text)
-        single_emphasis(double_emphasis(text))
-      end
-
     end
   end
 end

spec/helpers/markdownify_helper_spec.rb

@@ -7,302 +7,27 @@ require 'spec_helper'
 describe MarkdownifyHelper do
 
   describe "#markdownify" do
-    it 'does not error if youtube_maps in the hash is explicitly set to nil' do
+    describe "not doing something dumb" do
-      expect{
+      it "strips out script tags" do
-        markdownify("http://www.youtube.com/watch?v=pZROlhHOvuo", :youtube_maps => nil)
+        markdownify("<script>alert('XSS is evil')</script>").should == 
-      }.should_not raise_error
+          "<p>alert(&#39;XSS is evil&#39;)</p>\n"
-    end
-
-    it 'does not error if youtube_maps in the hash is explicitly set to nil' do
-      expect{
-        markdownify("http://vimeo.com/18589934", :vimeo_maps => nil)
-      }.should_not raise_error
-    end
-    
-    describe "autolinks" do
-      it "should not allow basic XSS/HTML" do
-        markdownify("<script>alert('XSS is evil')</script>").should == "<p>&lt;script&gt;alert('XSS is evil')&lt;/script&gt;</p>"
       end
 
-      it "should recognize basic http links (1/3)" do
+      it 'strips onClick handlers from links' do
-        proto="http"
+        omghax = '[XSS](http://joindiaspora.com/" onClick="$\(\'a\'\).remove\(\))'
-        url="bugs.joindiaspora.com/issues/332"
+        markdownify(omghax).should_not match(/ onClick/i)
-        full_url = "#{proto}://#{url}"
-        markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
-      end
-
-      it "should recognize basic http links (2/3)" do
-        proto="http"
-        url="webmail.example.com?~()!*/"
-        full_url = "#{proto}://#{url}"
-        markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
-      end
-
-      it "should recognize basic http links (3/3)" do
-        proto="http"
-        url="127.0.0.1:3000/users/sign_in"
-        full_url = "#{proto}://#{url}"
-        markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
-      end
-
-      it "should recognize secure https links" do
-        proto="https"
-        url="127.0.0.1:3000/users/sign_in"
-        full_url = "#{proto}://#{url}"
-        markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
-      end
-
-      it "doesn't muck up code text" do
-        message = %{`puts "Hello"`}
-        markdownify(message).should =~ %r{<code>puts &quot;Hello&quot;</code>}
-        message = %Q{~~~\nA\nB\n~~~\n}
-        markdownify(message).should =~ %r{<pre><code>\nA\nB\n</code></pre>}
-      end
-
-      it "doesn't double parse video links" do
-        message = "http://www.vimeo.com/17449557
-                   http://www.youtube.com/watch?v=0x__dDWdf23&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1
-                   http://youtu.be/x_CzD0GBD-4"
-        res = markdownify(message)
-        res.should =~ /href.+href.+href/m
-        res.should_not =~ /href.+href.+href.+href/m
-      end
-
-      describe "video links" do
-        it "recognizes vimeo links" do
-          video_id = "17449557"
-          url = "http://www.vimeo.com/#{video_id}"
-          res = markdownify(url)
-          res.should =~ /data-host="vimeo.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-        end
-
-        it "matches a trailing slash in a vimeo link" do
-          video_id = "17449557"
-          url = "http://www.vimeo.com/#{video_id}/"
-          res = markdownify(url)
-          res.should =~ /data-host="vimeo.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-          res.should_not =~ />\//
-        end
-
-        it "recognizes youtube links" do
-          video_id = "0x__dDWdf23"
-          url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-
-          url = "www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related"
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="BARFOO-----"/
-        end
-
-        it "recognizes youtu.be links" do
-          video_id = "x_CzD0GBD-4"
-          url =  "http://youtu.be/#{video_id}"
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-        end
-
-        it "recognizes youtube links with hyphens" do
-          video_id = "ABYnqp-bxvg"
-          url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-        end
-
-        it "keeps anchors" do
-          anchor = "#t=11m34"
-          video_id = "DHRoHuv3I8E"
-          url = "http://www.youtube.com/watch?v=" + video_id + anchor
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-          res.should =~ /data-anchor="#{anchor}"/
-        end
-
-        it "has an empty data-anchor attribute if there is no anchor" do
-          video_id = "DHRoHuv3I8E"
-          url = "http://www.youtube.com/watch?v=" + video_id
-          res = markdownify(url)
-          res.should =~ /Youtube:/
-          res.should =~ /data-host="youtube.com"/
-          res.should =~ /data-video-id="#{video_id}"/
-          res.should =~ /data-anchor=""/
-        end
-
-        it "leaves the links in the href of the #a tag" do
-          video_id = "ABYnqp-bxvg"
-          start_url ="http://www.youtube.com/watch?v=" + video_id
-          url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
-          res = markdownify(url)
-          res.should =~ /href=[\S]+v=#{video_id}/
-        end
-
-        it 'does not autolink inside the link' do
-          video_id = "ABYnqp-bxvg"
-          start_url ="http://www.youtube.com/watch?v=" + video_id
-          url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
-          res = markdownify(url)
-          res.match(/href="<a/).should be_nil
-        end
-      end
-
-      it "recognizes multiple links of different types" do
-        message = "http:// Hello World, this is for www.joindiaspora.com and not for http://www.google.com though their Youtube service is neat, take http://www.youtube.com/watch?v=foobar----- or www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related It is a good idea we finally have youtube, so enjoy this video http://www.youtube.com/watch?v=rickrolld--"
-        res = markdownify(message)
-        res.should =~ /a target=\"_blank\" href=\"http:\/\/www.joindiaspora.com\"/
-        res.should =~ /a target=\"_blank\" href=\"http:\/\/www.google.com\"/
-        res.should =~ /data-video-id="foobar-----"/
-        res.should =~ /data-video-id="BARFOO-----"/
-        res.should =~ /data-video-id="rickrolld--"/
-      end
-
-      it "should recognize basic ftp links" do
-        proto="ftp"
-        url="ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4"
-        # I did not watch that one, but the title sounds nice :P
-        markdownify(proto+"://"+url).should == "<p><a target=\"_blank\" href=\""+proto+"://"+url+"\">"+url+"</a></p>"
-      end
-
-      it "should recognize www links" do
-        url="www.joindiaspora.com"
-        markdownify(url).should == %Q{<p><a target="_blank" href="http://#{url}">#{url}</a></p>}
       end
     end
 
-    describe "specialchars" do
+    it 'does not barf if message is nil' do
-      it "replaces &lt;3 with &hearts;" do
-        message = "i <3 you"
-        markdownify(message).should == "<p>i &hearts; you</p>"
-      end
-
-      it "replaces various things with (their) HTML entities" do
-        message = "... <-> -> <- (tm) (r) (c)"
-        markdownify(message).should == "<p>&hellip; &#8596; &rarr; &larr; &trade; &reg; &copy;</p>"
-      end
-
-      it "skips doing it if you say so" do
-        message = "... -> <-"
-        markdownify(message, :specialchars => false).should == "<p>... -&gt; &lt;-</p>"
-      end
-    end
-
-    describe "weak emphasis" do
-      it "should be recognized (1/2)" do
-        message = "*some text* some text *some text* some text"
-        markdownify(message).should == "<p><em>some text</em> some text <em>some text</em> some text</p>"
-      end
-
-      it "should be recognized (2/2)" do
-        message = "_some text_ some text _some text_ some text"
-        markdownify(message).should == "<p><em>some text</em> some text <em>some text</em> some text</p>"
-      end
-    end
-
-    describe "strong emphasis" do
-      it "should be recognized (1/2)" do
-        message = "**some text** some text **some text** some text"
-        markdownify(message).should == "<p><strong>some text</strong> some text <strong>some text</strong> some text</p>"
-      end
-
-      it "should be recognized (2/2)" do
-        message = "__some text__ some text __some text__ some text"
-        markdownify(message).should == "<p><strong>some text</strong> some text <strong>some text</strong> some text</p>"
-      end
-    end
-
-    describe "nested weak and strong emphasis" do
-      it "should be rendered correctly" do
-        message = "__this is _some_ text__"
-        markdownify(message).should == "<p><strong>this is <em>some</em> text</strong></p>"
-        message = "*this is **some** text*"
-        markdownify(message).should == "<p><em>this is <strong>some</strong> text</em></p>"
-        message = "___some text___"
-        markdownify(message).should == "<p><em><strong>some text</strong></em></p>"
-      end
-    end
-
-    describe "links" do
-      it "should be recognized without title attribute" do
-        message = "[link text](http://someurl.com) [link text](http://someurl.com)"
-        markdownify(message).should == '<p><a target="_blank" href="http://someurl.com">link text</a> <a target="_blank" href="http://someurl.com">link text</a></p>'
-      end
-
-      it "should be recognized with title attribute" do
-        message = '[link text](http://someurl.com "some title") [link text](http://someurl.com "some title")'
-        markdownify(message).should == '<p><a target="_blank" href="http://someurl.com" title="some title">link text</a> <a target="_blank" href="http://someurl.com" title="some title">link text</a></p>'
-      end
-
-      it "should have a robust link parsing" do
-        message = "[wikipedia](http://en.wikipedia.org/wiki/Text_(literary_theory))"
-        link = markdownify(message)
-        link.should =~ %r{href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"}
-        
-        message = "[  links]( google.com)"
-        markdownify(message).should == %Q{<p><a target="_blank" href="http://google.com">links</a></p>}
-
-        message = "[_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks )"
-        markdownify(message).should == %Q{<p><a target="_blank" href="http://google.com/search?q=with_multiple__underscores*and**asterisks"><em>http</em></a></p>}
-        message = %{[___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol')}
-        markdownify(message).should == %{<p><a target="_blank" href="ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4" title="File Transfer Protocol"><em><strong>FTP</strong></em></a></p>}
-
-        message = %{[**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)}
-        markdownify(message).should == %{<p><a target="_blank" href="foo://bar.example.org/yes_it*makes*no_sense"><strong>any protocol</strong></a></p>}
-
-        message = "This [ *text* ]( http://en.wikipedia.org/wiki/Text_(literary_theory) ) with many [ links]( google.com) tests [_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks ), [___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol'), [**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)"
-        markdownify(message).should == '<p>This <a target="_blank" href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"><em>text</em></a> with many <a target="_blank" href="http://google.com">links</a> tests <a target="_blank" href="http://google.com/search?q=with_multiple__underscores*and**asterisks"><em>http</em></a>, <a target="_blank" href="ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4" title="File Transfer Protocol"><em><strong>FTP</strong></em></a>, <a target="_blank" href="foo://bar.example.org/yes_it*makes*no_sense"><strong>any protocol</strong></a></p>'
-      end
-
-    end
-
-    describe "nested emphasis and links tags" do
-      it "should be rendered correctly" do
-        message = '[**some *link* text**](someurl.com "some title")'
-        markdownify(message).should == '<p><a target="_blank" href="http://someurl.com" title="some title"><strong>some <em>link</em> text</strong></a></p>'
-      end
-    end
-
-    it "should allow escaping" do
-      message = '*some text* \*some text* \**some text* _some text_ \_some text_ \__some text_'
-      markdownify(message).should == "<p><em>some text</em> *some text* *<em>some text</em> <em>some text</em> _some text_ _<em>some text</em></p>"
-    end
-
-    describe "newlines" do
-      it 'skips inserting newlines if you pass the newlines option' do
-        message = "These\nare\n\some\nnew\lines"
-        res = markdownify(message, :newlines => false)
-        res.should == "<p>#{message}</p>"
-      end
-
-      it 'generates breaklines' do
-        message = "These\nare\nsome\nnew\nlines"
-        res = markdownify(message)
-        res.should == "<p>These<br /\>are<br /\>some<br /\>new<br /\>lines</p>"
-      end
-
-      it 'should render newlines and basic http links correctly' do
-        message = "Some text, then a line break and a link\nhttp://joindiaspora.com\nsome more text"
-        res = markdownify(message)
-        res.should == '<p>Some text, then a line break and a link<br /><a target="_blank" href="http://joindiaspora.com">joindiaspora.com</a><br />some more text</p>'
-      end
-    end
-
-    it 'does not barf is message is nil' do
       markdownify(nil).should == ''
     end
 
-    context 'when formatting status messages' do
+    it 'autolinks standard url links' do
+      puts markdownify("http://joindiaspora.com/")
+    end
 
+    context 'when formatting status messages' do
       it "should leave tags intact" do
         message = Factory.create(:status_message, 
                                  :author => alice.person,
@@ -328,17 +53,5 @@ describe MarkdownifyHelper do
         formatted.should =~ /hovercard/
       end
     end
-
-    context 'performance', :performance => true do
-      before do
-        @message = "</p>HHello,Hello_, I _am a strong robot.*Hello, I am *a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a **strong robot.Hello, I am _a _strong *robot**.Hello*, I am a strong</p>"
-      end
-
-      it 'is sub millisecond' do
-        Benchmark.realtime{
-          markdownify(@message)
-        }.should < 0.001
-      end
-    end
   end
 end