From 6b9ad7c4c6ba2d1ceda4981966523ea46d9df968 Mon Sep 17 00:00:00 2001 From: Ilya Zhitomirskiy Date: Tue, 13 Sep 2011 20:20:15 -0700 Subject: [PATCH] SM IZ pared down markdownify to basic markdown, bumped version of RedCarpet --- Gemfile | 2 +- Gemfile.lock | 9 +- app/helpers/markdownify_helper.rb | 1 + lib/diaspora/markdownify.rb | 179 +------------- spec/helpers/markdownify_helper_spec.rb | 311 +----------------------- 5 files changed, 19 insertions(+), 483 deletions(-) diff --git a/Gemfile b/Gemfile index a47ffec3b..9a77f1c58 100644 --- a/Gemfile +++ b/Gemfile @@ -62,7 +62,7 @@ gem 'rails-i18n' # parsing gem 'nokogiri' -gem 'redcarpet', :git => 'git://github.com/tanoku/redcarpet' +gem 'redcarpet', "2.0.0b5" gem 'roxml', :git => 'git://github.com/Empact/roxml.git', :ref => '7ea9a9ffd2338aaef5b0' # queue diff --git a/Gemfile.lock b/Gemfile.lock index 546eaf952..64d5104f5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,12 +56,6 @@ GIT addressable (>= 2.1.1) eventmachine (>= 0.12.9) -GIT - remote: git://github.com/tanoku/redcarpet - revision: 3c6b5807e8a5be51d769149e555b09bae8ae3228 - specs: - redcarpet (2.0.0b3) - GEM remote: http://rubygems.org/ specs: @@ -363,6 +357,7 @@ GEM rash (0.3.0) hashie (~> 1.0.0) rdoc (3.9.4) + redcarpet (2.0.0b5) redis (2.2.2) redis-namespace (0.8.0) redis (< 3.0.0) @@ -518,7 +513,7 @@ DEPENDENCIES omniauth (= 0.2.6) rails (= 3.0.10) rails-i18n - redcarpet! + redcarpet (= 2.0.0b5) resque (= 1.10.0) resque-ensure-connected resque-timeout (= 1.0.0) diff --git a/app/helpers/markdownify_helper.rb b/app/helpers/markdownify_helper.rb index 13cb7c5cb..e6c66dc98 100644 --- a/app/helpers/markdownify_helper.rb +++ b/app/helpers/markdownify_helper.rb @@ -31,6 +31,7 @@ module MarkdownifyHelper return '' if message.blank? + #renderer = Redcarpet::Render::HTML.new(render_options) renderer = Diaspora::Markdownify::HTML.new(render_options) markdown = Redcarpet::Markdown.new(renderer, markdown_options) diff --git a/lib/diaspora/markdownify.rb b/lib/diaspora/markdownify.rb index c21fdaa68..a937afbb2 100644 --- a/lib/diaspora/markdownify.rb +++ b/lib/diaspora/markdownify.rb @@ -3,186 +3,13 @@ require 'erb' module Diaspora module Markdownify class HTML < Redcarpet::Render::HTML - attr_accessor :newlines, :specialchars, :youtube_maps, :vimeo_maps - def initialize(options={}) - super - - @newlines = options.fetch(:newlines, true) - @specialchars = options.fetch(:specialchars, true) - @youtube_maps = options[:youtube_maps]||{} - @vimeo_maps = options[:vimeo_maps] || {} - end + include ActionView::Helpers::TextHelper + include ActionView::Helpers::TagHelper def autolink(link, type) - return link if type == :email - autolink_youtube(link) || autolink_vimeo(link) || autolink_simple(link) + auto_link(link, :link => :urls ) end - - def autolink_simple(link) - - # If there isn't *some* protocol, assume http - if link !~ %r{^\w+://} - link = "http://#{link}" - end - - content = link.gsub(%r{^\w+://}, '') - - %Q{#{content}} - end - - def autolink_vimeo(link) - regex = %r{https?://(?:w{3}\.)?vimeo.com/(\d{6,})/?} - if link =~ regex - video_id = $1 - if @vimeo_maps[video_id] - title = ERB::Util.h(CGI::unescape(@vimeo_maps[video_id])) - else - title = I18n.t 'application.helper.video_title.unknown' - end - return ' Vimeo: ' + title + '' - end - return - end - - def autolink_youtube(link) - if link =~ YoutubeTitles::YOUTUBE_ID_REGEX - video_id = $1 - anchor = $2 || '' - - if @youtube_maps[video_id] - title = ERB::Util.h(CGI::unescape(@youtube_maps[video_id])) - else - title = I18n.t 'application.helper.video_title.unknown' - end - return ' Youtube: ' + title + '' - end - return - end - - def block_code(text, language) - "
\n#{text}
" - end - - def double_emphasis(text) - "#{text}" - end - - def linebreak() - "
" - end - - def link(link, title, content) - #hax - content ||='' - - return autolink(link, 'url') if link == content - - if link =~ Regexp.new(Regexp.escape(content)) - return autolink(link, 'url') - end - - if link !~ %r{^\w+://} - link = "http://#{link}" - end - - tag = if title and content - %Q{#{content}} - elsif content - %Q{#{content}} - else - autolink(link, 'url') - end - return tag - end - - def paragraph(text) - #hax again... why is markdownify passing us nil? - text ||='' - - if @newlines - br = linebreak - - # in very clear cases, let newlines become
tags - # Grabbed from Github flavored Markdown - text = text.gsub(/^[\w\<][^\n]*\n+/) do |x| - x =~ /\n{2}/ ? x : (x = x.strip; x << br) - end - end - return "

#{text}

" - end - - def preprocess(full_document) - entities = [ - ['&', '&'], - ['>', '>'], - ['<', '<'] - ] - entities.each do |original, replacement| - full_document = full_document.gsub(original, replacement) - end - - if @specialchars - full_document = specialchars(full_document) - end - - our_unsafe_chars = '()' - full_document = full_document.gsub(%r{ - \[ \s*? ( [^ \] ]+ ) \s*? \] - (?: - \( \s*? (\S+) \s*? (?: "([^"]+)" )? \) \s*? - ) - }xm) do |m| - content = $1 - link = URI.escape($2, our_unsafe_chars) - title = $3 - - title_chunk = if title - %W{" #{title}"} - else - '' - end - %Q{[#{content}](#{link}#{title_chunk})} - end - - return full_document - end - - - - def single_emphasis(text) - "#{text}" - end - - def specialchars(text) - if @specialchars - map = [ - ["<3", "♥"], - ["<->", "↔"], - ["->", "→"], - ["<-", "←"], - ["\.\.\.", "…"], - ["(tm)", "™"], - ["(r)", "®"], - ["(c)", "©"] - ] - end - - map.each do |mapping| - text = text.gsub(mapping[0], mapping[1]) - end - - return text - end - - - def triple_emphasis(text) - single_emphasis(double_emphasis(text)) - end - end end end diff --git a/spec/helpers/markdownify_helper_spec.rb b/spec/helpers/markdownify_helper_spec.rb index 08ba42f90..95c960d64 100644 --- a/spec/helpers/markdownify_helper_spec.rb +++ b/spec/helpers/markdownify_helper_spec.rb @@ -7,302 +7,27 @@ require 'spec_helper' describe MarkdownifyHelper do describe "#markdownify" do - it 'does not error if youtube_maps in the hash is explicitly set to nil' do - expect{ - markdownify("http://www.youtube.com/watch?v=pZROlhHOvuo", :youtube_maps => nil) - }.should_not raise_error - end - - it 'does not error if youtube_maps in the hash is explicitly set to nil' do - expect{ - markdownify("http://vimeo.com/18589934", :vimeo_maps => nil) - }.should_not raise_error - end - - describe "autolinks" do - it "should not allow basic XSS/HTML" do - markdownify("").should == "

<script>alert('XSS is evil')</script>

" + describe "not doing something dumb" do + it "strips out script tags" do + markdownify("").should == + "

alert('XSS is evil')

\n" end - it "should recognize basic http links (1/3)" do - proto="http" - url="bugs.joindiaspora.com/issues/332" - full_url = "#{proto}://#{url}" - markdownify(full_url).should == %Q{

#{url}

} - end - - it "should recognize basic http links (2/3)" do - proto="http" - url="webmail.example.com?~()!*/" - full_url = "#{proto}://#{url}" - markdownify(full_url).should == %Q{

#{url}

} - end - - it "should recognize basic http links (3/3)" do - proto="http" - url="127.0.0.1:3000/users/sign_in" - full_url = "#{proto}://#{url}" - markdownify(full_url).should == %Q{

#{url}

} - end - - it "should recognize secure https links" do - proto="https" - url="127.0.0.1:3000/users/sign_in" - full_url = "#{proto}://#{url}" - markdownify(full_url).should == %Q{

#{url}

} - end - - it "doesn't muck up code text" do - message = %{`puts "Hello"`} - markdownify(message).should =~ %r{puts "Hello"} - message = %Q{~~~\nA\nB\n~~~\n} - markdownify(message).should =~ %r{
\nA\nB\n
} - end - - it "doesn't double parse video links" do - message = "http://www.vimeo.com/17449557 - http://www.youtube.com/watch?v=0x__dDWdf23&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1 - http://youtu.be/x_CzD0GBD-4" - res = markdownify(message) - res.should =~ /href.+href.+href/m - res.should_not =~ /href.+href.+href.+href/m - end - - describe "video links" do - it "recognizes vimeo links" do - video_id = "17449557" - url = "http://www.vimeo.com/#{video_id}" - res = markdownify(url) - res.should =~ /data-host="vimeo.com"/ - res.should =~ /data-video-id="#{video_id}"/ - end - - it "matches a trailing slash in a vimeo link" do - video_id = "17449557" - url = "http://www.vimeo.com/#{video_id}/" - res = markdownify(url) - res.should =~ /data-host="vimeo.com"/ - res.should =~ /data-video-id="#{video_id}"/ - res.should_not =~ />\// - end - - it "recognizes youtube links" do - video_id = "0x__dDWdf23" - url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1" - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="#{video_id}"/ - - url = "www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related" - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="BARFOO-----"/ - end - - it "recognizes youtu.be links" do - video_id = "x_CzD0GBD-4" - url = "http://youtu.be/#{video_id}" - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="#{video_id}"/ - end - - it "recognizes youtube links with hyphens" do - video_id = "ABYnqp-bxvg" - url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1" - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="#{video_id}"/ - end - - it "keeps anchors" do - anchor = "#t=11m34" - video_id = "DHRoHuv3I8E" - url = "http://www.youtube.com/watch?v=" + video_id + anchor - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="#{video_id}"/ - res.should =~ /data-anchor="#{anchor}"/ - end - - it "has an empty data-anchor attribute if there is no anchor" do - video_id = "DHRoHuv3I8E" - url = "http://www.youtube.com/watch?v=" + video_id - res = markdownify(url) - res.should =~ /Youtube:/ - res.should =~ /data-host="youtube.com"/ - res.should =~ /data-video-id="#{video_id}"/ - res.should =~ /data-anchor=""/ - end - - it "leaves the links in the href of the #a tag" do - video_id = "ABYnqp-bxvg" - start_url ="http://www.youtube.com/watch?v=" + video_id - url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1" - res = markdownify(url) - res.should =~ /href=[\S]+v=#{video_id}/ - end - - it 'does not autolink inside the link' do - video_id = "ABYnqp-bxvg" - start_url ="http://www.youtube.com/watch?v=" + video_id - url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1" - res = markdownify(url) - res.match(/href=""+url+"

" - end - - it "should recognize www links" do - url="www.joindiaspora.com" - markdownify(url).should == %Q{

#{url}

} + it 'strips onClick handlers from links' do + omghax = '[XSS](http://joindiaspora.com/" onClick="$\(\'a\'\).remove\(\))' + markdownify(omghax).should_not match(/ onClick/i) end end - describe "specialchars" do - it "replaces <3 with ♥" do - message = "i <3 you" - markdownify(message).should == "

i ♥ you

" - end - - it "replaces various things with (their) HTML entities" do - message = "... <-> -> <- (tm) (r) (c)" - markdownify(message).should == "

… ↔ → ← ™ ® ©

" - end - - it "skips doing it if you say so" do - message = "... -> <-" - markdownify(message, :specialchars => false).should == "

... -> <-

" - end - end - - describe "weak emphasis" do - it "should be recognized (1/2)" do - message = "*some text* some text *some text* some text" - markdownify(message).should == "

some text some text some text some text

" - end - - it "should be recognized (2/2)" do - message = "_some text_ some text _some text_ some text" - markdownify(message).should == "

some text some text some text some text

" - end - end - - describe "strong emphasis" do - it "should be recognized (1/2)" do - message = "**some text** some text **some text** some text" - markdownify(message).should == "

some text some text some text some text

" - end - - it "should be recognized (2/2)" do - message = "__some text__ some text __some text__ some text" - markdownify(message).should == "

some text some text some text some text

" - end - end - - describe "nested weak and strong emphasis" do - it "should be rendered correctly" do - message = "__this is _some_ text__" - markdownify(message).should == "

this is some text

" - message = "*this is **some** text*" - markdownify(message).should == "

this is some text

" - message = "___some text___" - markdownify(message).should == "

some text

" - end - end - - describe "links" do - it "should be recognized without title attribute" do - message = "[link text](http://someurl.com) [link text](http://someurl.com)" - markdownify(message).should == '

link text link text

' - end - - it "should be recognized with title attribute" do - message = '[link text](http://someurl.com "some title") [link text](http://someurl.com "some title")' - markdownify(message).should == '

link text link text

' - end - - it "should have a robust link parsing" do - message = "[wikipedia](http://en.wikipedia.org/wiki/Text_(literary_theory))" - link = markdownify(message) - link.should =~ %r{href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"} - - message = "[ links]( google.com)" - markdownify(message).should == %Q{

links

} - - message = "[_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks )" - markdownify(message).should == %Q{

http

} - message = %{[___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol')} - markdownify(message).should == %{

FTP

} - - message = %{[**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)} - markdownify(message).should == %{

any protocol

} - - message = "This [ *text* ]( http://en.wikipedia.org/wiki/Text_(literary_theory) ) with many [ links]( google.com) tests [_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks ), [___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol'), [**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)" - markdownify(message).should == '

This text with many links tests http, FTP, any protocol

' - end - - end - - describe "nested emphasis and links tags" do - it "should be rendered correctly" do - message = '[**some *link* text**](someurl.com "some title")' - markdownify(message).should == '

some link text

' - end - end - - it "should allow escaping" do - message = '*some text* \*some text* \**some text* _some text_ \_some text_ \__some text_' - markdownify(message).should == "

some text *some text* *some text some text _some text_ _some text

" - end - - describe "newlines" do - it 'skips inserting newlines if you pass the newlines option' do - message = "These\nare\n\some\nnew\lines" - res = markdownify(message, :newlines => false) - res.should == "

#{message}

" - end - - it 'generates breaklines' do - message = "These\nare\nsome\nnew\nlines" - res = markdownify(message) - res.should == "

These
are
some
new
lines

" - end - - it 'should render newlines and basic http links correctly' do - message = "Some text, then a line break and a link\nhttp://joindiaspora.com\nsome more text" - res = markdownify(message) - res.should == '

Some text, then a line break and a link
joindiaspora.com
some more text

' - end - end - - it 'does not barf is message is nil' do + it 'does not barf if message is nil' do markdownify(nil).should == '' end - context 'when formatting status messages' do + it 'autolinks standard url links' do + puts markdownify("http://joindiaspora.com/") + end + context 'when formatting status messages' do it "should leave tags intact" do message = Factory.create(:status_message, :author => alice.person, @@ -328,17 +53,5 @@ describe MarkdownifyHelper do formatted.should =~ /hovercard/ end end - - context 'performance', :performance => true do - before do - @message = "

HHello,Hello_, I _am a strong robot.*Hello, I am *a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a **strong robot.Hello, I am _a _strong *robot**.Hello*, I am a strong

" - end - - it 'is sub millisecond' do - Benchmark.realtime{ - markdownify(@message) - }.should < 0.001 - end - end end end