SM IZ pared down markdownify to basic markdown, bumped version of RedCarpet

This commit is contained in:
Ilya Zhitomirskiy 2011-09-13 20:20:15 -07:00
parent 810d8ab97a
commit 6b9ad7c4c6
5 changed files with 19 additions and 483 deletions

View file

@ -62,7 +62,7 @@ gem 'rails-i18n'
# parsing
gem 'nokogiri'
gem 'redcarpet', :git => 'git://github.com/tanoku/redcarpet'
gem 'redcarpet', "2.0.0b5"
gem 'roxml', :git => 'git://github.com/Empact/roxml.git', :ref => '7ea9a9ffd2338aaef5b0'
# queue

View file

@ -56,12 +56,6 @@ GIT
addressable (>= 2.1.1)
eventmachine (>= 0.12.9)
GIT
remote: git://github.com/tanoku/redcarpet
revision: 3c6b5807e8a5be51d769149e555b09bae8ae3228
specs:
redcarpet (2.0.0b3)
GEM
remote: http://rubygems.org/
specs:
@ -363,6 +357,7 @@ GEM
rash (0.3.0)
hashie (~> 1.0.0)
rdoc (3.9.4)
redcarpet (2.0.0b5)
redis (2.2.2)
redis-namespace (0.8.0)
redis (< 3.0.0)
@ -518,7 +513,7 @@ DEPENDENCIES
omniauth (= 0.2.6)
rails (= 3.0.10)
rails-i18n
redcarpet!
redcarpet (= 2.0.0b5)
resque (= 1.10.0)
resque-ensure-connected
resque-timeout (= 1.0.0)

View file

@ -31,6 +31,7 @@ module MarkdownifyHelper
return '' if message.blank?
#renderer = Redcarpet::Render::HTML.new(render_options)
renderer = Diaspora::Markdownify::HTML.new(render_options)
markdown = Redcarpet::Markdown.new(renderer, markdown_options)

View file

@ -3,186 +3,13 @@ require 'erb'
module Diaspora
module Markdownify
class HTML < Redcarpet::Render::HTML
attr_accessor :newlines, :specialchars, :youtube_maps, :vimeo_maps
def initialize(options={})
super
@newlines = options.fetch(:newlines, true)
@specialchars = options.fetch(:specialchars, true)
@youtube_maps = options[:youtube_maps]||{}
@vimeo_maps = options[:vimeo_maps] || {}
end
include ActionView::Helpers::TextHelper
include ActionView::Helpers::TagHelper
def autolink(link, type)
return link if type == :email
autolink_youtube(link) || autolink_vimeo(link) || autolink_simple(link)
auto_link(link, :link => :urls )
end
def autolink_simple(link)
# If there isn't *some* protocol, assume http
if link !~ %r{^\w+://}
link = "http://#{link}"
end
content = link.gsub(%r{^\w+://}, '')
%Q{<a target="_blank" href="#{link}">#{content}</a>}
end
def autolink_vimeo(link)
regex = %r{https?://(?:w{3}\.)?vimeo.com/(\d{6,})/?}
if link =~ regex
video_id = $1
if @vimeo_maps[video_id]
title = ERB::Util.h(CGI::unescape(@vimeo_maps[video_id]))
else
title = I18n.t 'application.helper.video_title.unknown'
end
return ' <a class="video-link" data-host="vimeo.com" data-video-id="' +
video_id + '" href="' + link + '" target="_blank">Vimeo: ' + title + '</a>'
end
return
end
def autolink_youtube(link)
if link =~ YoutubeTitles::YOUTUBE_ID_REGEX
video_id = $1
anchor = $2 || ''
if @youtube_maps[video_id]
title = ERB::Util.h(CGI::unescape(@youtube_maps[video_id]))
else
title = I18n.t 'application.helper.video_title.unknown'
end
return ' <a class="video-link" data-host="youtube.com" data-video-id="' +
video_id + '" data-anchor="' + anchor +
'" href="'+ link + '" target="_blank">Youtube: ' + title + '</a>'
end
return
end
def block_code(text, language)
"<pre><code>\n#{text}</code></pre>"
end
def double_emphasis(text)
"<strong>#{text}</strong>"
end
def linebreak()
"<br />"
end
def link(link, title, content)
#hax
content ||=''
return autolink(link, 'url') if link == content
if link =~ Regexp.new(Regexp.escape(content))
return autolink(link, 'url')
end
if link !~ %r{^\w+://}
link = "http://#{link}"
end
tag = if title and content
%Q{<a target="_blank" href="#{link}" title="#{title}">#{content}</a>}
elsif content
%Q{<a target="_blank" href="#{link}">#{content}</a>}
else
autolink(link, 'url')
end
return tag
end
def paragraph(text)
#hax again... why is markdownify passing us nil?
text ||=''
if @newlines
br = linebreak
# in very clear cases, let newlines become <br /> tags
# Grabbed from Github flavored Markdown
text = text.gsub(/^[\w\<][^\n]*\n+/) do |x|
x =~ /\n{2}/ ? x : (x = x.strip; x << br)
end
end
return "<p>#{text}</p>"
end
def preprocess(full_document)
entities = [
['&', '&amp;'],
['>', '&gt;'],
['<', '&lt;']
]
entities.each do |original, replacement|
full_document = full_document.gsub(original, replacement)
end
if @specialchars
full_document = specialchars(full_document)
end
our_unsafe_chars = '()'
full_document = full_document.gsub(%r{
\[ \s*? ( [^ \] ]+ ) \s*? \]
(?:
\( \s*? (\S+) \s*? (?: "([^"]+)" )? \) \s*?
)
}xm) do |m|
content = $1
link = URI.escape($2, our_unsafe_chars)
title = $3
title_chunk = if title
%W{" #{title}"}
else
''
end
%Q{[#{content}](#{link}#{title_chunk})}
end
return full_document
end
def single_emphasis(text)
"<em>#{text}</em>"
end
def specialchars(text)
if @specialchars
map = [
["&lt;3", "&hearts;"],
["&lt;-&gt;", "&#8596;"],
["-&gt;", "&rarr;"],
["&lt;-", "&larr;"],
["\.\.\.", "&hellip;"],
["(tm)", "&trade;"],
["(r)", "&reg;"],
["(c)", "&copy;"]
]
end
map.each do |mapping|
text = text.gsub(mapping[0], mapping[1])
end
return text
end
def triple_emphasis(text)
single_emphasis(double_emphasis(text))
end
end
end
end

View file

@ -7,302 +7,27 @@ require 'spec_helper'
describe MarkdownifyHelper do
describe "#markdownify" do
it 'does not error if youtube_maps in the hash is explicitly set to nil' do
expect{
markdownify("http://www.youtube.com/watch?v=pZROlhHOvuo", :youtube_maps => nil)
}.should_not raise_error
describe "not doing something dumb" do
it "strips out script tags" do
markdownify("<script>alert('XSS is evil')</script>").should ==
"<p>alert(&#39;XSS is evil&#39;)</p>\n"
end
it 'does not error if youtube_maps in the hash is explicitly set to nil' do
expect{
markdownify("http://vimeo.com/18589934", :vimeo_maps => nil)
}.should_not raise_error
end
describe "autolinks" do
it "should not allow basic XSS/HTML" do
markdownify("<script>alert('XSS is evil')</script>").should == "<p>&lt;script&gt;alert('XSS is evil')&lt;/script&gt;</p>"
end
it "should recognize basic http links (1/3)" do
proto="http"
url="bugs.joindiaspora.com/issues/332"
full_url = "#{proto}://#{url}"
markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
end
it "should recognize basic http links (2/3)" do
proto="http"
url="webmail.example.com?~()!*/"
full_url = "#{proto}://#{url}"
markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
end
it "should recognize basic http links (3/3)" do
proto="http"
url="127.0.0.1:3000/users/sign_in"
full_url = "#{proto}://#{url}"
markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
end
it "should recognize secure https links" do
proto="https"
url="127.0.0.1:3000/users/sign_in"
full_url = "#{proto}://#{url}"
markdownify(full_url).should == %Q{<p><a target="_blank" href="#{full_url}">#{url}</a></p>}
end
it "doesn't muck up code text" do
message = %{`puts "Hello"`}
markdownify(message).should =~ %r{<code>puts &quot;Hello&quot;</code>}
message = %Q{~~~\nA\nB\n~~~\n}
markdownify(message).should =~ %r{<pre><code>\nA\nB\n</code></pre>}
end
it "doesn't double parse video links" do
message = "http://www.vimeo.com/17449557
http://www.youtube.com/watch?v=0x__dDWdf23&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1
http://youtu.be/x_CzD0GBD-4"
res = markdownify(message)
res.should =~ /href.+href.+href/m
res.should_not =~ /href.+href.+href.+href/m
end
describe "video links" do
it "recognizes vimeo links" do
video_id = "17449557"
url = "http://www.vimeo.com/#{video_id}"
res = markdownify(url)
res.should =~ /data-host="vimeo.com"/
res.should =~ /data-video-id="#{video_id}"/
end
it "matches a trailing slash in a vimeo link" do
video_id = "17449557"
url = "http://www.vimeo.com/#{video_id}/"
res = markdownify(url)
res.should =~ /data-host="vimeo.com"/
res.should =~ /data-video-id="#{video_id}"/
res.should_not =~ />\//
end
it "recognizes youtube links" do
video_id = "0x__dDWdf23"
url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="#{video_id}"/
url = "www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related"
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="BARFOO-----"/
end
it "recognizes youtu.be links" do
video_id = "x_CzD0GBD-4"
url = "http://youtu.be/#{video_id}"
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="#{video_id}"/
end
it "recognizes youtube links with hyphens" do
video_id = "ABYnqp-bxvg"
url = "http://www.youtube.com/watch?v=" + video_id + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="#{video_id}"/
end
it "keeps anchors" do
anchor = "#t=11m34"
video_id = "DHRoHuv3I8E"
url = "http://www.youtube.com/watch?v=" + video_id + anchor
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="#{video_id}"/
res.should =~ /data-anchor="#{anchor}"/
end
it "has an empty data-anchor attribute if there is no anchor" do
video_id = "DHRoHuv3I8E"
url = "http://www.youtube.com/watch?v=" + video_id
res = markdownify(url)
res.should =~ /Youtube:/
res.should =~ /data-host="youtube.com"/
res.should =~ /data-video-id="#{video_id}"/
res.should =~ /data-anchor=""/
end
it "leaves the links in the href of the #a tag" do
video_id = "ABYnqp-bxvg"
start_url ="http://www.youtube.com/watch?v=" + video_id
url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
res = markdownify(url)
res.should =~ /href=[\S]+v=#{video_id}/
end
it 'does not autolink inside the link' do
video_id = "ABYnqp-bxvg"
start_url ="http://www.youtube.com/watch?v=" + video_id
url = start_url + "&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1"
res = markdownify(url)
res.match(/href="<a/).should be_nil
it 'strips onClick handlers from links' do
omghax = '[XSS](http://joindiaspora.com/" onClick="$\(\'a\'\).remove\(\))'
markdownify(omghax).should_not match(/ onClick/i)
end
end
it "recognizes multiple links of different types" do
message = "http:// Hello World, this is for www.joindiaspora.com and not for http://www.google.com though their Youtube service is neat, take http://www.youtube.com/watch?v=foobar----- or www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related It is a good idea we finally have youtube, so enjoy this video http://www.youtube.com/watch?v=rickrolld--"
res = markdownify(message)
res.should =~ /a target=\"_blank\" href=\"http:\/\/www.joindiaspora.com\"/
res.should =~ /a target=\"_blank\" href=\"http:\/\/www.google.com\"/
res.should =~ /data-video-id="foobar-----"/
res.should =~ /data-video-id="BARFOO-----"/
res.should =~ /data-video-id="rickrolld--"/
end
it "should recognize basic ftp links" do
proto="ftp"
url="ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4"
# I did not watch that one, but the title sounds nice :P
markdownify(proto+"://"+url).should == "<p><a target=\"_blank\" href=\""+proto+"://"+url+"\">"+url+"</a></p>"
end
it "should recognize www links" do
url="www.joindiaspora.com"
markdownify(url).should == %Q{<p><a target="_blank" href="http://#{url}">#{url}</a></p>}
end
end
describe "specialchars" do
it "replaces &lt;3 with &hearts;" do
message = "i <3 you"
markdownify(message).should == "<p>i &hearts; you</p>"
end
it "replaces various things with (their) HTML entities" do
message = "... <-> -> <- (tm) (r) (c)"
markdownify(message).should == "<p>&hellip; &#8596; &rarr; &larr; &trade; &reg; &copy;</p>"
end
it "skips doing it if you say so" do
message = "... -> <-"
markdownify(message, :specialchars => false).should == "<p>... -&gt; &lt;-</p>"
end
end
describe "weak emphasis" do
it "should be recognized (1/2)" do
message = "*some text* some text *some text* some text"
markdownify(message).should == "<p><em>some text</em> some text <em>some text</em> some text</p>"
end
it "should be recognized (2/2)" do
message = "_some text_ some text _some text_ some text"
markdownify(message).should == "<p><em>some text</em> some text <em>some text</em> some text</p>"
end
end
describe "strong emphasis" do
it "should be recognized (1/2)" do
message = "**some text** some text **some text** some text"
markdownify(message).should == "<p><strong>some text</strong> some text <strong>some text</strong> some text</p>"
end
it "should be recognized (2/2)" do
message = "__some text__ some text __some text__ some text"
markdownify(message).should == "<p><strong>some text</strong> some text <strong>some text</strong> some text</p>"
end
end
describe "nested weak and strong emphasis" do
it "should be rendered correctly" do
message = "__this is _some_ text__"
markdownify(message).should == "<p><strong>this is <em>some</em> text</strong></p>"
message = "*this is **some** text*"
markdownify(message).should == "<p><em>this is <strong>some</strong> text</em></p>"
message = "___some text___"
markdownify(message).should == "<p><em><strong>some text</strong></em></p>"
end
end
describe "links" do
it "should be recognized without title attribute" do
message = "[link text](http://someurl.com) [link text](http://someurl.com)"
markdownify(message).should == '<p><a target="_blank" href="http://someurl.com">link text</a> <a target="_blank" href="http://someurl.com">link text</a></p>'
end
it "should be recognized with title attribute" do
message = '[link text](http://someurl.com "some title") [link text](http://someurl.com "some title")'
markdownify(message).should == '<p><a target="_blank" href="http://someurl.com" title="some title">link text</a> <a target="_blank" href="http://someurl.com" title="some title">link text</a></p>'
end
it "should have a robust link parsing" do
message = "[wikipedia](http://en.wikipedia.org/wiki/Text_(literary_theory))"
link = markdownify(message)
link.should =~ %r{href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"}
message = "[ links]( google.com)"
markdownify(message).should == %Q{<p><a target="_blank" href="http://google.com">links</a></p>}
message = "[_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks )"
markdownify(message).should == %Q{<p><a target="_blank" href="http://google.com/search?q=with_multiple__underscores*and**asterisks"><em>http</em></a></p>}
message = %{[___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol')}
markdownify(message).should == %{<p><a target="_blank" href="ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4" title="File Transfer Protocol"><em><strong>FTP</strong></em></a></p>}
message = %{[**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)}
markdownify(message).should == %{<p><a target="_blank" href="foo://bar.example.org/yes_it*makes*no_sense"><strong>any protocol</strong></a></p>}
message = "This [ *text* ]( http://en.wikipedia.org/wiki/Text_(literary_theory) ) with many [ links]( google.com) tests [_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks ), [___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 'File Transfer Protocol'), [**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)"
markdownify(message).should == '<p>This <a target="_blank" href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"><em>text</em></a> with many <a target="_blank" href="http://google.com">links</a> tests <a target="_blank" href="http://google.com/search?q=with_multiple__underscores*and**asterisks"><em>http</em></a>, <a target="_blank" href="ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4" title="File Transfer Protocol"><em><strong>FTP</strong></em></a>, <a target="_blank" href="foo://bar.example.org/yes_it*makes*no_sense"><strong>any protocol</strong></a></p>'
end
end
describe "nested emphasis and links tags" do
it "should be rendered correctly" do
message = '[**some *link* text**](someurl.com "some title")'
markdownify(message).should == '<p><a target="_blank" href="http://someurl.com" title="some title"><strong>some <em>link</em> text</strong></a></p>'
end
end
it "should allow escaping" do
message = '*some text* \*some text* \**some text* _some text_ \_some text_ \__some text_'
markdownify(message).should == "<p><em>some text</em> *some text* *<em>some text</em> <em>some text</em> _some text_ _<em>some text</em></p>"
end
describe "newlines" do
it 'skips inserting newlines if you pass the newlines option' do
message = "These\nare\n\some\nnew\lines"
res = markdownify(message, :newlines => false)
res.should == "<p>#{message}</p>"
end
it 'generates breaklines' do
message = "These\nare\nsome\nnew\nlines"
res = markdownify(message)
res.should == "<p>These<br /\>are<br /\>some<br /\>new<br /\>lines</p>"
end
it 'should render newlines and basic http links correctly' do
message = "Some text, then a line break and a link\nhttp://joindiaspora.com\nsome more text"
res = markdownify(message)
res.should == '<p>Some text, then a line break and a link<br /><a target="_blank" href="http://joindiaspora.com">joindiaspora.com</a><br />some more text</p>'
end
end
it 'does not barf is message is nil' do
it 'does not barf if message is nil' do
markdownify(nil).should == ''
end
context 'when formatting status messages' do
it 'autolinks standard url links' do
puts markdownify("http://joindiaspora.com/")
end
context 'when formatting status messages' do
it "should leave tags intact" do
message = Factory.create(:status_message,
:author => alice.person,
@ -328,17 +53,5 @@ describe MarkdownifyHelper do
formatted.should =~ /hovercard/
end
end
context 'performance', :performance => true do
before do
@message = "</p>HHello,Hello_, I _am a strong robot.*Hello, I am *a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a strong robot.Hello, I am a **strong robot.Hello, I am _a _strong *robot**.Hello*, I am a strong</p>"
end
it 'is sub millisecond' do
Benchmark.realtime{
markdownify(@message)
}.should < 0.001
end
end
end
end