diff --git a/Gemfile b/Gemfile index e6cb7af26..d1b6564f5 100644 --- a/Gemfile +++ b/Gemfile @@ -50,6 +50,7 @@ gem 'json', '1.4.6' gem 'http_accept_language', :git => 'git://github.com/iain/http_accept_language.git', :ref => '0b78aa7849fc90cf9e12' gem 'thin', '1.2.11', :require => false +gem 'redcarpet', :git => 'git://github.com/tanoku/redcarpet' #Websocket gem 'em-websocket', :git => 'git://github.com/igrigorik/em-websocket', :ref => 'e278f5a1c4db60be7485' diff --git a/Gemfile.lock b/Gemfile.lock index 96285f1e3..b85ac8c09 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,6 +56,12 @@ GIT addressable (>= 2.1.1) eventmachine (>= 0.12.9) +GIT + remote: git://github.com/tanoku/redcarpet + revision: 3c6b5807e8a5be51d769149e555b09bae8ae3228 + specs: + redcarpet (2.0.0b3) + GEM remote: http://rubygems.org/ specs: @@ -87,8 +93,9 @@ GEM activesupport (= 3.0.9) arel (~> 2.0.10) tzinfo (~> 0.3.23) - activerecord-import (0.2.7) - activerecord (~> 3.0.0) + activerecord-import (0.2.8) + activerecord (~> 3.0pre) + activerecord (~> 3.0pre) activeresource (3.0.9) activemodel (= 3.0.9) activesupport (= 3.0.9) @@ -136,9 +143,9 @@ GEM ohai (>= 0.5.7) rest-client (< 1.7.0, >= 1.0.4) uuidtools - childprocess (0.2.1) + childprocess (0.2.2) ffi (~> 1.0.6) - closure-compiler (1.1.1) + closure-compiler (1.1.3) cloudfiles (1.4.10) mime-types (>= 1.16) columnize (0.3.4) @@ -167,7 +174,7 @@ GEM eventmachine (0.12.10) excon (0.2.4) extlib (0.9.15) - factory_girl (2.0.3) + factory_girl (2.0.5) factory_girl_rails (1.1.0) factory_girl (~> 2.0.0) railties (>= 3.0.0) @@ -194,12 +201,12 @@ GEM ruby-hmac foreigner (0.9.1) formatador (0.2.0) - fuubar (0.0.5) + fuubar (0.0.6) rspec (~> 2.0) - rspec-instafail (~> 0.1.4) + rspec-instafail (~> 0.1.8) ruby-progressbar (~> 0.0.10) gem_plugin (0.2.3) - gherkin (2.4.5) + gherkin (2.4.16) json (>= 1.4.6) haml (3.1.2) hashie (1.0.0) @@ -209,9 +216,9 @@ GEM builder http_connection (1.4.1) i18n (0.5.0) - i18n-inflector (2.6.2) + i18n-inflector (2.6.3) i18n (>= 0.4.1) - i18n-inflector-rails (1.0.5) + i18n-inflector-rails (1.0.6) actionpack (~> 3.0) i18n-inflector (~> 2.6) railties (~> 3.0) @@ -343,9 +350,9 @@ GEM rake (0.9.2) rash (0.3.0) hashie (~> 1.0.0) - rcov (0.9.9) - rdoc (3.9.1) - redis (2.2.1) + rcov (0.9.10) + rdoc (3.9.4) + redis (2.2.2) redis-namespace (0.8.0) redis (< 3.0.0) resque (1.10.0) @@ -407,14 +414,14 @@ GEM tilt (< 2.0, >= 1.2.2) sqlite3 (1.3.4) subexec (0.0.4) - systemu (2.2.0) + systemu (2.3.0) term-ansicolor (1.0.6) thin (1.2.11) daemons (>= 1.0.9) eventmachine (>= 0.12.6) rack (>= 1.0.0) thor (0.14.6) - tilt (1.3.2) + tilt (1.3.3) treetop (1.4.10) polyglot polyglot (>= 0.3.1) @@ -496,6 +503,7 @@ DEPENDENCIES rails (= 3.0.9) rails-i18n rcov + redcarpet! resque (= 1.10.0) resque-ensure-connected rest-client (= 1.6.1) diff --git a/app/helpers/markdownify_helper.rb b/app/helpers/markdownify_helper.rb index edf3c7f74..1f4b1b303 100644 --- a/app/helpers/markdownify_helper.rb +++ b/app/helpers/markdownify_helper.rb @@ -2,134 +2,32 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. +require 'lib/diaspora/markdownify' + module MarkdownifyHelper - def markdownify(message, options={}) - message = h(message).to_str + def markdownify(message, render_options={}) + markdown_options = { + :autolink => true, + :fenced_code_blocks => true, + :space_after_headers => true, + :strikethrough => true, + :superscript => true, + :tables => true + } - options[:newlines] = true if !options.has_key?(:newlines) - options[:specialchars] = true if !options.has_key?(:specialchars) + render_options[:filter_html] = true - message = process_links(message) - message = process_autolinks(message) - message = process_emphasis(message) - message = process_youtube(message, options[:youtube_maps]) - message = process_vimeo(message, options[:vimeo_maps]) - message = process_specialchars(message) if options[:specialchars] - message = process_newlines(message) if options[:newlines] - - message.html_safe + renderer = Diaspora::Markdownify::HTML.new(render_options) + markdown = Redcarpet::Markdown.new(renderer, markdown_options) + message = markdown.render(message) + return message.html_safe end def process_newlines(message) - message.gsub!(/\n+/, '
') - message - end - - def process_links(message) - message.gsub!(/\[\s*([^\[]+?)\s*\]\(\s*([^ ]+\s*) \"(([^&]|(&[^q])|(&q[^u])|(&qu[^o])|(&quo[^t])|("[^;]))+)\"\s*\)/) do |m| - escape = "\\" - link = $1 - url = $2 - title = $3 - url.gsub!("_", "\\_") - url.gsub!("*", "\\*") - protocol = (url =~ /^\w+:\/\//) ? '' :'http://' - res = "#{link}" - res + # in very clear cases, let newlines become
tags + # Grabbed from Github flavored Markdown + message.gsub(/^[\w\<][^\n]*\n+/) do |x| + x =~ /\n{2}/ ? x : (x.strip!; x << " \n") end - - message.gsub!(/\[\s*([^\[]+?)\s*\]\(\s*([^ ]+)\s*\)/) do |m| - escape = "\\" - link = $1 - url = $2 - url.gsub!("_", "\\_") - url.gsub!("*", "\\*") - protocol = (url =~ /^\w+:\/\//) ? '' :'http://' - res = "#{link}" - res - end - - message - end - - def process_youtube(message, youtube_maps) - processed_message = message.gsub(YoutubeTitles::YOUTUBE_ID_REGEX) do |matched_string| - match_data = matched_string.match(YoutubeTitles::YOUTUBE_ID_REGEX) - video_id = match_data[1] - anchor = match_data[2] - anchor ||= '' - if youtube_maps && youtube_maps[video_id] - title = h(CGI::unescape(youtube_maps[video_id])) - else - title = I18n.t 'application.helper.video_title.unknown' - end - ' Youtube: ' + title + '' - end - processed_message - end - - def process_autolinks(message) - message.gsub!(/( |^)(www\.[^\s]+\.[^\s])/, '\1http://\2') - message.gsub!(/(#{captures[2]}} - res.gsub!(/(\*|_)/) { |m| "\\#{$1}" } - res - end - end - message - end - - def process_emphasis(message) - message.gsub!("\\**", "-^doublestar^-") - message.gsub!("\\__", "-^doublescore^-") - message.gsub!("\\*", "-^star^-") - message.gsub!("\\_", "-^score^-") - message.gsub!(/(\*\*\*|___)(.+?)\1/m, '\2') - message.gsub!(/(\*\*|__)(.+?)\1/m, '\2') - message.gsub!(/(\*|_)(.+?)\1/m, '\2') - message.gsub!("-^doublestar^-", "**") - message.gsub!("-^doublescore^-", "__") - message.gsub!("-^star^-", "*") - message.gsub!("-^score^-", "_") - message - end - - def process_vimeo(message, vimeo_maps) - regex = /https?:\/\/(?:w{3}\.)?vimeo.com\/(\d{6,})\/?/ - processed_message = message.gsub(regex) do |matched_string| - match_data = message.match(regex) - video_id = match_data[1] - if vimeo_maps && vimeo_maps[video_id] - title = h(CGI::unescape(vimeo_maps[video_id])) - else - title = I18n.t 'application.helper.video_title.unknown' - end - ' Vimeo: ' + title + '' - end - processed_message - end - - def process_specialchars(message) - map = [ - ["<3", "♥"], - ["<->", "↔"], - ["->", "→"], - ["<-", "←"], - ["...", "…"], - ["(tm)", "™"], - ["(r)", "®"], - ["(c)", "©"] - ] - - map.each do |mapping| - message.gsub!(mapping[0], mapping[1]) - end - message end end diff --git a/app/views/status_messages/_status_message.haml b/app/views/status_messages/_status_message.haml index 133cde0b5..c3be5bad9 100644 --- a/app/views/status_messages/_status_message.haml +++ b/app/views/status_messages/_status_message.haml @@ -16,4 +16,4 @@ = link_to (image_tag photo.url(:thumb_small), :class => 'stream-photo thumb_small', 'data-small-photo' => photo.url(:thumb_medium), 'data-full-photo' => photo.url), photo_path(photo), :class => 'stream-photo-link' %p{:class => direction_for(post.text)} - = markdownify(post.text, :youtube_maps => post[:youtube_titles]) + != markdownify(post.text, :youtube_maps => post[:youtube_titles]) diff --git a/features/follows_tags.feature b/features/follows_tags.feature index ccd5be3e3..3b165deca 100644 --- a/features/follows_tags.feature +++ b/features/follows_tags.feature @@ -22,7 +22,7 @@ Feature: posting Scenario: see a tag that I am following When I go to the home page And I follow "#boss" - Then I should see "I am da #boss" + Then I should see "I am da #boss" within "body" Scenario: can stop following a tag from the tag page When I hover over the ".button.tag_following" diff --git a/lib/diaspora/markdownify.rb b/lib/diaspora/markdownify.rb new file mode 100644 index 000000000..6e38faadd --- /dev/null +++ b/lib/diaspora/markdownify.rb @@ -0,0 +1,177 @@ +require 'erb' + +module Diaspora + module Markdownify + class HTML < Redcarpet::Render::HTML + attr_accessor :newlines, :specialchars, :youtube_maps, :vimeo_maps + + def initialize(options={}) + super + + @expand_tags = options.fetch(:expand_tabs, true) + @newlines = options.fetch(:newlines, true) + @specialchars = options.fetch(:specialchars, true) + @youtube_maps = options.fetch(:youtube_maps, {}) + @vimeo_maps = options.fetch(:vimeo_maps, {}) + end + + def autolink(link, type) + autolink_youtube(link) || autolink_vimeo(link) || autolink_simple(link) + end + + def autolink_simple(link) + + # If there isn't *some* protocol, assume http + if link !~ %r{^\w+://} + link = "http://#{link}" + end + + content = link.gsub(%r{^\w+://}, '') + + %Q{#{content}} + end + + def autolink_vimeo(link) + regex = %r{https?://(?:w{3}\.)?vimeo.com/(\d{6,})/?} + if link =~ regex + video_id = $1 + if @vimeo_maps[video_id] + title = ERB::Util.h(CGI::unescape(@vimeo_maps[video_id])) + else + title = I18n.t 'application.helper.video_title.unknown' + end + return ' Vimeo: ' + title + '' + end + return + end + + def autolink_youtube(link) + if link =~ YoutubeTitles::YOUTUBE_ID_REGEX + video_id = $1 + anchor = $2 || '' + + if @youtube_maps[video_id] + title = ERB::Util.h(CGI::unescape(@youtube_maps[video_id])) + else + title = I18n.t 'application.helper.video_title.unknown' + end + return ' Youtube: ' + title + '' + end + return + end + + def block_code(text, language) + "
\n#{text}
" + end + + def double_emphasis(text) + "#{text}" + end + + def linebreak() + "
" + end + + def link(link, title, content) + return autolink(link, 'url') if link == content + + if link =~ Regexp.new(Regexp.escape(content)) + return autolink(link, 'url') + end + + if link !~ %r{^\w+://} + link = "http://#{link}" + end + + tag = if title and content + %Q{#{content}} + elsif content + %Q{#{content}} + else + autolink(link, 'url') + end + return tag + end + + def paragraph(text) + if @expand_tags + text = Diaspora::Taggable.format_tags(text, :no_escape => true) + end + + if @newlines + br = linebreak + + # in very clear cases, let newlines become
tags + # Grabbed from Github flavored Markdown + text = text.gsub(/^[\w\<][^\n]*\n+/) do |x| + x =~ /\n{2}/ ? x : (x = x.strip; x << br) + end + end + + return "

#{text}

" + end + + def preprocess(full_document) + if @specialchars + full_document = specialchars(full_document) + end + + our_unsafe_chars = '()' + full_document = full_document.gsub(%r{ + \[ \s*? ( [^ \] ]+ ) \s*? \] + (?: + \( \s*? (\S+) \s*? (?: "([^"]+)" )? \) \s*? + ) + }xm) do |m| + content = $1 + link = URI.escape($2, our_unsafe_chars) + title = $3 + + title_chunk = if title + %W{" #{title}"} + else + '' + end + %Q{[#{content}](#{link}#{title_chunk})} + end + + return full_document + end + + + def single_emphasis(text) + "#{text}" + end + + def specialchars(text) + if @specialchars + map = [ + ["<3", "♥"], + ["<->", "↔"], + ["->", "→"], + ["<-", "←"], + ["\.\.\.", "…"], + ["(tm)", "™"], + ["(r)", "®"], + ["(c)", "©"] + ] + end + + map.each do |mapping| + text = text.gsub(mapping[0], mapping[1]) + end + + return text + end + + + def triple_emphasis(text) + single_emphasis(double_emphasis(text)) + end + + end + end +end diff --git a/public/stylesheets/sass/application.sass b/public/stylesheets/sass/application.sass index ac4d6b2ce..34e633d2d 100644 --- a/public/stylesheets/sass/application.sass +++ b/public/stylesheets/sass/application.sass @@ -38,6 +38,9 @@ h1 ul > li :list-style none +.content ul li + :list-style disc + form :position relative input, diff --git a/spec/helpers/markdownify_helper_spec.rb b/spec/helpers/markdownify_helper_spec.rb index 1e036aa8d..1349b0033 100644 --- a/spec/helpers/markdownify_helper_spec.rb +++ b/spec/helpers/markdownify_helper_spec.rb @@ -8,31 +8,42 @@ describe MarkdownifyHelper do describe "#markdownify" do describe "autolinks" do it "should not allow basic XSS/HTML" do - markdownify("").should == "<script>alert('XSS is evil')</script>" + markdownify("").should == "

alert('XSS is evil')

" end it "should recognize basic http links (1/3)" do proto="http" url="bugs.joindiaspora.com/issues/332" - markdownify(proto+"://"+url).should == ""+url+"" + full_url = "#{proto}://#{url}" + markdownify(full_url).should == %Q{

#{url}

} end it "should recognize basic http links (2/3)" do proto="http" url="webmail.example.com?~()!*/" - markdownify(proto+"://"+url).should == ""+url+"" + full_url = "#{proto}://#{url}" + markdownify(full_url).should == %Q{

#{url}

} end it "should recognize basic http links (3/3)" do proto="http" url="127.0.0.1:3000/users/sign_in" - markdownify(proto+"://"+url).should == ""+url+"" + full_url = "#{proto}://#{url}" + markdownify(full_url).should == %Q{

#{url}

} end it "should recognize secure https links" do proto="https" url="127.0.0.1:3000/users/sign_in" - markdownify(proto+"://"+url).should == ""+url+"" + full_url = "#{proto}://#{url}" + markdownify(full_url).should == %Q{

#{url}

} + end + + it "doesn't muck up code text" do + message = %{`puts "Hello"`} + markdownify(message).should =~ %r{

puts "Hello"

} + message = %Q{~~~\nA\nB\n~~~\n} + markdownify(message).should =~ %r{
\nA\nB\n
} end it "doesn't double parse video links" do @@ -40,8 +51,8 @@ describe MarkdownifyHelper do http://www.youtube.com/watch?v=0x__dDWdf23&a=GxdCwVVULXdvEBKmx_f5ywvZ0zZHHHDU&list=ML&playnext=1 http://youtu.be/x_CzD0GBD-4" res = markdownify(message) - res.should =~ /href.+href.+href/ - res.should_not =~ /href.+href.+href.+href/ + res.should =~ /href.+href.+href/m + res.should_not =~ /href.+href.+href.+href/m end describe "video links" do @@ -69,6 +80,12 @@ describe MarkdownifyHelper do res.should =~ /Youtube:/ res.should =~ /data-host="youtube.com"/ res.should =~ /data-video-id="#{video_id}"/ + + url = "www.youtube.com/watch?foo=bar&v=BARFOO-----&whatever=related" + res = markdownify(url) + res.should =~ /Youtube:/ + res.should =~ /data-host="youtube.com"/ + res.should =~ /data-video-id="BARFOO-----"/ end it "recognizes youtu.be links" do @@ -141,113 +158,135 @@ describe MarkdownifyHelper do proto="ftp" url="ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4" # I did not watch that one, but the title sounds nice :P - markdownify(proto+"://"+url).should == ""+url+"" + markdownify(proto+"://"+url).should == "

"+url+"

" end it "should recognize www links" do url="www.joindiaspora.com" - markdownify(url).should == ""+url+"" + markdownify(url).should == %Q{

#{url}

} end end describe "specialchars" do it "replaces <3 with ♥" do message = "i <3 you" - markdownify(message).should == "i ♥ you" + markdownify(message).should == "

i ♥ you

" end it "replaces various things with (their) HTML entities" do message = "... <-> -> <- (tm) (r) (c)" - markdownify(message).should == "… ↔ → ← ™ ® ©" + markdownify(message).should == "

… ↔ → ← ™ ® ©

" end it "skips doing it if you say so" do message = "... -> <-" - markdownify(message, :specialchars => false).should == "... -> <-" + markdownify(message, :specialchars => false).should == "

... -> <-

" end end describe "weak emphasis" do it "should be recognized (1/2)" do message = "*some text* some text *some text* some text" - markdownify(message).should == "some text some text some text some text" + markdownify(message).should == "

some text some text some text some text

" end it "should be recognized (2/2)" do message = "_some text_ some text _some text_ some text" - markdownify(message).should == "some text some text some text some text" + markdownify(message).should == "

some text some text some text some text

" end end describe "strong emphasis" do it "should be recognized (1/2)" do message = "**some text** some text **some text** some text" - markdownify(message).should == "some text some text some text some text" + markdownify(message).should == "

some text some text some text some text

" end it "should be recognized (2/2)" do message = "__some text__ some text __some text__ some text" - markdownify(message).should == "some text some text some text some text" + markdownify(message).should == "

some text some text some text some text

" end end describe "nested weak and strong emphasis" do it "should be rendered correctly" do message = "__this is _some_ text__" - markdownify(message).should == "this is some text" + markdownify(message).should == "

this is some text

" message = "*this is **some** text*" - markdownify(message).should == "this is some text" + markdownify(message).should == "

this is some text

" message = "___some text___" - markdownify(message).should == "some text" + markdownify(message).should == "

some text

" end end describe "links" do it "should be recognized without title attribute" do message = "[link text](http://someurl.com) [link text](http://someurl.com)" - markdownify(message).should == 'link text link text' + markdownify(message).should == '

link text link text

' end it "should be recognized with title attribute" do message = '[link text](http://someurl.com "some title") [link text](http://someurl.com "some title")' - markdownify(message).should == 'link text link text' + markdownify(message).should == '

link text link text

' end it "should have a robust link parsing" do + message = "[wikipedia](http://en.wikipedia.org/wiki/Text_(literary_theory))" + link = markdownify(message) + link.should =~ %r{href="http://en.wikipedia.org/wiki/Text_%28literary_theory%29"} + + message = "[ links]( google.com)" + markdownify(message).should == %Q{

links

} + + message = "[_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks )" + markdownify(message).should == %Q{

http

} + message = %{[___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 \"File Transfer Protocol\")} + markdownify(message).should == %{

FTP

} + + message = %{[**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)} + markdownify(message).should == %{

any protocol

} + message = "This [ *text* ]( http://en.wikipedia.org/wiki/Text_(literary_theory) ) with many [ links]( google.com) tests [_http_](http://google.com/search?q=with_multiple__underscores*and**asterisks ), [___FTP___]( ftp://ftp.uni-kl.de/CCC/26C3/mp4/26c3-3540-en-a_hackers_utopia.mp4 \"File Transfer Protocol\"), [**any protocol**](foo://bar.example.org/yes_it*makes*no_sense)" - markdownify(message).should == 'This text with many links tests http, FTP, any protocol' + markdownify(message).should == '

This text with many links tests http, FTP, any protocol

' + end + + it "should leave #tag links intact" do + message = %{#tagged} + markdownify(message).should == "

#{message}

" + message = %{alice - 1 - #tagged} + markdownify(message).should == "

#{message}

" end end describe "nested emphasis and links tags" do it "should be rendered correctly" do message = '[**some *link* text**](someurl.com "some title")' - markdownify(message).should == 'some link text' + markdownify(message).should == '

some link text

' end end it "should allow escaping" do - message = '*some text* \\*some text* \\**some text* _some text_ \\_some text_ \\__some text_' - markdownify(message).should == "some text *some text **some text some text _some text __some text" + message = '*some text* \*some text* \**some text* _some text_ \_some text_ \__some text_' + markdownify(message).should == "

some text *some text* *some text some text _some text_ _some text

" end describe "newlines" do it 'skips inserting newlines if you pass the newlines option' do message = "These\nare\n\some\nnew\lines" res = markdownify(message, :newlines => false) - res.should == message + res.should == "

#{message}

" end it 'generates breaklines' do message = "These\nare\nsome\nnew\nlines" res = markdownify(message) - res.should == "These
are
some
new
lines" + res.should == "

These
are
some
new
lines

" end it 'should render newlines and basic http links correctly' do message = "Some text, then a line break and a link\nhttp://joindiaspora.com\nsome more text" res = markdownify(message) - res.should == 'Some text, then a line break and a link
joindiaspora.com
some more text' + res.should == '

Some text, then a line break and a link
joindiaspora.com
some more text

' end end @@ -263,4 +302,4 @@ describe MarkdownifyHelper do end end end -end \ No newline at end of file +end