From 9024d1a9b7c952d01f4e7a6ad9db082f6f57f522 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 19 Oct 2014 03:43:14 +0200 Subject: [PATCH 01/14] Add Camo configs --- config/defaults.yml | 4 ++ config/diaspora.yml.example | 133 ++++++++++++++++++++---------------- 2 files changed, 78 insertions(+), 59 deletions(-) diff --git a/config/defaults.yml b/config/defaults.yml index 841fc2643..ac1a7c55a 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -53,6 +53,10 @@ defaults: user_counts: false post_counts: false comment_counts: false + camo: + enable: false + root: + key: settings: pod_name: 'diaspora*' enable_registrations: true diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index 3a4153edf..6fb2b4a0e 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -24,11 +24,11 @@ ## heroku config:set SERVICES_FACEBOOK_APP_ID=yourappid SERVICES_FACEBOOK_SECRET=yourappsecret configuration: ## Section - + ## You need to change or at least review the settings in this section ## in order for your pod to work environment: ## Section - + ## Set the hostname of the machine you're running Diaspora on, as seen ## from the internet. This should be the URL you want to use to ## access the pod. So if you plan to use a reverse proxy, it should be @@ -37,7 +37,7 @@ configuration: ## Section ## If you do change the URL, you will have to start again as the URL ## will be hardcoded into the database. #url: "https://example.org/" - + ## Set the bundle of certificate authorities (CA) certificates. ## This is specific to your operating system. ## Examples (uncomment the relevant one or add your own): @@ -45,20 +45,20 @@ configuration: ## Section #certificate_authorities: '/etc/ssl/certs/ca-certificates.crt' ## For CentOS, Fedora: #certificate_authorities: '/etc/pki/tls/certs/ca-bundle.crt' - + ## URL for a remote Redis (default=localhost) ## Don't forget to restrict IP access if you uncomment these! #redis: 'redis://example_host' #redis: 'redis://username:password@host:6379/0' #redis: 'unix:///tmp/redis.sock' - + ## Require SSL (default=true) ## When set, your pod will force the use of HTTPS in production mode. Since ## OAuth2 requires SSL Diaspora's future API might not work if you're not using ## SSL. Also there is no guarantee that posting to services will be possible ## if SSL is disabled. Do not change this default unless you are sure! #require_ssl: true - + ## Single-process mode (default=false) ## If set to true Diaspora will work with just the appserver (Unicorn by default) ## running. However, this makes it quite slow as intensive jobs must be run @@ -71,7 +71,7 @@ configuration: ## Section ## Number of parallel threads Sidekiq uses (default=5) ## If you touch this please set the pool setting in your database.yml to - ## a value that's at minimum close to this! You can safely increase it + ## a value that's at minimum close to this! You can safely increase it ## to 25 and more on a medium-sized pod. This applies per started Sidekiq ## worker, so if you set it to 25 and start two workers you'll process ## up to 50 jobs in parallel. @@ -94,7 +94,7 @@ configuration: ## Section ## Log file for Sidekiq (default="log/sidekiq.log") #log: "log/sidekiq.log" - + ## Use Amazon S3 instead of your local filesystem ## to handle uploaded pictures (disabled by default) s3: ## Section @@ -110,23 +110,23 @@ configuration: ## Section ## 1 year. This can improve load speed and save requests to the image host. ## Set to false to revert to browser defaults (usually less than 1 year). #cache : true - + ## Set redirect URL for an external image host (Amazon S3 or other) ## If hosting images for your pod on an external server (even your own), ## add its URL here. All requests made to images under /uploads/images ## will be redirected to https://yourhost.tld/uploads/images/ #image_redirect_url: 'https://images.example.org' - + assets: ## Section ## Serve static assets via the appserver (default=false) ## This is highly discouraged for production use. Let your reverse ## proxy/webserver do it by serving the files under public/ directly. #serve: false - + ## Upload your assets to S3 (default=false) #upload: false - + ## Specify an asset host. Ensure it does not have a trailing slash (/). #host: http://cdn.example.org/diaspora @@ -134,7 +134,7 @@ configuration: ## Section ## Diaspora is only tested against the default pubsub server. ## You probably don't want to uncomment or change this. #pubsub_server: 'https://pubsubhubbub.appspot.com/' - + ## Settings affecting how ./script/server behaves. server: ## Section @@ -145,11 +145,11 @@ configuration: ## Section ## The environment in which the server should be started by default. ## Change this if you wish to run a production environment. #rails_environment: 'development' - + ## Write unicorn stderr and stdout log #stderr_log: '/usr/local/app/diaspora/log/unicorn-stderr.log' #stdout_log: '/usr/local/app/diaspora/log/unicorn-stdout.log' - + ## Number of Unicorn worker processes (default=2) ## Increase this if you have many users. #unicorn_worker: 2 @@ -159,7 +159,7 @@ configuration: ## Section ## Decrease if you're under heavy load and don't care if some ## requests fail. #unicorn_timeout: 90 - + ## Embed a Sidekiq worker inside the unicorn process (default=false) ## Useful for minimal Heroku setups. #embed_sidekiq_worker: false @@ -168,20 +168,20 @@ configuration: ## Section ## In most cases it is better to ## increase environment.sidekiq.concurrency instead! #sidekiq_workers: 1 - - ## Settings potentially affecting the privacy of your users + + ## Settings potentially affecting the privacy of your users privacy: ## Section - + ## Include jQuery from jquery.com's CDN (default=false) ## Enabling this can reduce traffic and speed up load time since most ## clients already have this one cached. When set to false (the default), ## the jQuery library will be loaded from your pod's own resources. #jquery_cdn: false - + ## Google Analytics (disabled by default) - ## Provide a key to enable tracking by Google Analytics + ## Provide a key to enable tracking by Google Analytics #google_analytics_key: - + ## Piwik Tracking (disabled by default) ## Provide a site ID and the host piwik is running on to enable ## tracking through Piwik. @@ -190,17 +190,17 @@ configuration: ## Section #enable: true #host: 'stats.example.org' #site_id: 1 - + ## Mixpanel event tracking (disabled by default) #mixpanel_uid: - + ## Chartbeat tracking (disabled by default) #chartbeat_uid: - + ## Statistics ## Your pod will report its name, software version and whether ## or not registrations are open via /statistics.json. - ## Uncomment the options below to enable more statistics. + ## Uncomment the options below to enable more statistics. statistics: ## Section ## Local user total and 6 month active counts @@ -209,10 +209,25 @@ configuration: ## Section ## Local post total count #post_counts: true #comment_counts: true - + + ## Use Camo to proxy embedded remote images + ## Do not enable this setting unless you have a working Camo setup. Using + ## camo to proxy embedded images will improve the privacy and security of + ## your pod's frontend, but it will increase the traffic on your server. + camo: ## Section + + ## Enable Camo (default=false) + #enable: true + + ## Root of your Camo installation + #root: "https://camo.example.com/" + + ## Shared key of your Camo installation + #key: "example123example456example!" + ## General settings settings: ## Section - + ## Pod name (default="diaspora*") ## The pod name displayed in various locations, including the header. #pod_name: "diaspora*" @@ -222,13 +237,13 @@ configuration: ## Section ## without an invitation. Note that this needs to be set to true ## (or commented out) to enable the first registration (you). #enable_registrations: true - + ## Auto-follow on sign-up (default=true) ## Users will automatically follow a specified account on creation. ## Set this to false if you don't want your users to automatically ## follow an account upon creation. #autofollow_on_join: true - + ## Auto-follow account (default='diasporahq@joindiaspora.com') ## The diaspora* HQ account keeps users up to date with news about Diaspora. ## If you set another auto-follow account (for example your podmin account), @@ -237,26 +252,26 @@ configuration: ## Section ## Invitation settings invitations: ## Section - + ## Enable invitations (default=true) ## Set this to false if you don't want users to be able to send invites. #open: true - + ## Number of invitations per invite link (default=25) ## Every user will see such a link if you have enabled invitations on your pod. #count: 25 - + ## Paypal donations ## You can provide the ID of a hosted Paypal button here to allow your users ## to send donations to help run their pod. If you leave this out your users ## will see a button to donate to the Diaspora Foundation instead :) #paypal_hosted_button_id: "change_me" - + ## Bitcoin donations ## You can provide a bitcoin address here to allow your users to provide ## donations towards the running of their pod. #bitcoin_address: "change_me" - + ## Community spotlight (disabled by default) ## The community spotlight shows new users public posts from people you ## think are interesting in Diaspora's community. To add an account @@ -268,12 +283,12 @@ configuration: ## Section ## E-mail address to which users can make suggestions about who ## should be in the community spotlight (optional). #suggest_email: 'admin@example.org' - + ## CURL debug (default=false) ## Turn on extra verbose output when sending stuff. Note: you ## don't need to touch this unless explicitly told to. #typhoeus_verbose: false - + ## Maximum number of parallel HTTP requests made to other pods (default=20) ## Be careful, raising this setting will heavily increase the memory usage ## of your Sidekiq workers. @@ -302,7 +317,7 @@ configuration: ## Section ## Sets the level of image distortion used in the captcha. ## Available options are: 'low', 'medium', 'high', 'random' #distortion: 'low' - + ## Terms of Service ## Show a default or customized terms of service for users. ## You can create a custom Terms of Service by placing a template @@ -330,7 +345,7 @@ configuration: ## Section ## Set a number to activate this setting. This age limit is shown ## in the default ToS document. #minimum_age: false - + ## Maintenance ## Various pod maintenance related settings are controlled from here. maintenance: ## Section @@ -355,42 +370,42 @@ configuration: ## Section #enable: true #app_id: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Twitter: twitter: ## Section #enable: true #key: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Tumblr tumblr: ## Section #enable: true #key: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Wordpress wordpress: ## Section #enable: true #client_id: 'abcdef' #secret: 'change_me' - + ## Enable pod users to send e-mails from Diaspora (disabled by default) mail: ## Section - + ## First you need to enable it. #enable: true - + ## Sender address used in mail sent by Diaspora #sender_address: 'no-reply@example.org' - + ## This selects which mailer should be used. Use 'smtp' for a smtp ## connection, 'sendmail' to use the sendmail binary or ## 'messagebus' to use the messagebus service. #method: 'smtp' - + ## Ignore if method isn't 'smtp' smtp: ## Section @@ -399,44 +414,44 @@ configuration: ## Section ## the SMTP server, if it sends one. (default port=587) #host: 'smtp.example.org' #port: 587 - + ## Authentication required to send mail (default='plain') ## Use one of 'plain', 'login' or 'cram_md5'. Use 'none' ## if server does not support authentication. #authentication: 'plain' - + ## Credentials to log in to the SMTP server ## May be necessary if authentication is not 'none'. #username: 'change_me' #password: 'change_me' - + ## Automatically enable TLS (default=true) ## Leave this commented out if authentication is set to 'none'. #starttls_auto: true - + ## The domain for the HELO command, if needed #domain: 'smtp.example.org' - + ## OpenSSL verify mode used when connecting to a SMTP server with TLS ## Set this to 'none' if you have a self-signed certificate. Possible ## values: 'none', 'peer', 'client_once', 'fail_if_no_peer_cert'. #openssl_verify_mode: 'none' - + ## Ignore if method isn't 'sendmail' sendmail: ## Section ## The path to the sendmail binary (default='/usr/sbin/sendmail') #location: '/usr/sbin/sendmail' - + ## Use exim and sendmail (default=false) #exim_fix: false - + ## Ignore if method isn't 'messagebus' #message_bus_api_key: 'abcdef' - + ## Administrator settings admins: ## Section - + ## Set the admin account ## This doesn't make the user an admin but is used when a generic ## admin contact is needed, much like the postmaster role in mail @@ -445,13 +460,13 @@ configuration: ## Section ## E-mail address to contact the administrator #podmin_email: 'podmin@example.org' - + ## Here you can override settings defined above if you need ## to have them different in different environments. production: ## Section environment: ## Section #redis_url: 'redis://production.example.org:6379' - + development: ## Section environment: ## Section #redis_url: 'redis://production.example.org:6379' From 92cd4e6b78284a2740964492382d4d10714056f2 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 19 Oct 2014 17:45:30 +0200 Subject: [PATCH 02/14] Use camo for Markdown images just a quick proof of concept --- app/presenters/post_presenter.rb | 2 +- config/defaults.yml | 4 +++- config/diaspora.yml.example | 18 ++++++++++++++++-- lib/diaspora/camo_url.rb | 16 ++++++++++++++++ lib/diaspora/message_renderer.rb | 13 +++++++++++++ 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 lib/diaspora/camo_url.rb diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index e6dd6fa31..a01c090b2 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -16,7 +16,7 @@ class PostPresenter { :id => @post.id, :guid => @post.guid, - :text => @post.raw_message, + :text => @post.message.plain_text_for_json, :public => @post.public, :created_at => @post.created_at, :interacted_at => @post.interacted_at, diff --git a/config/defaults.yml b/config/defaults.yml index ac1a7c55a..8cf35d41b 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -54,7 +54,9 @@ defaults: post_counts: false comment_counts: false camo: - enable: false + proxy_markdown_images: false + proxy_opengraph_thumbnails: false + proxy_remote_pod_images: false root: key: settings: diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index 6fb2b4a0e..9f3ee4335 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -216,8 +216,22 @@ configuration: ## Section ## your pod's frontend, but it will increase the traffic on your server. camo: ## Section - ## Enable Camo (default=false) - #enable: true + ## Proxy imaged embedded via markdown (default=false) + ## Embedded images are quite often from non-SSL sites and may cause a + ## partial content warning, so this is recommended. + #proxy_markdown_images: true + + ## Proxy Open Graph thumbnails (default=false) + ## Open Graph thumbnails may or may not be encrypted and loaded from + ## servers outside the network. Recommended. + #proxy_opengraph_thumbnails: true + + ## Proxy remote pod's images (default=false) + ## Profile pictures and photos from other pods usually are encrypted, + ## so enabling this is only useful if you want to avoid HTTP requests to + ## third-party servers. This will create a lot of traffic on your camo + ## instance. You have been warned. + #proxy_remote_pod_images: true ## Root of your Camo installation #root: "https://camo.example.com/" diff --git a/lib/diaspora/camo_url.rb b/lib/diaspora/camo_url.rb new file mode 100644 index 000000000..f0cbb0da2 --- /dev/null +++ b/lib/diaspora/camo_url.rb @@ -0,0 +1,16 @@ +# implicitly requires OpenSSL + +module Diaspora + module CamoUrl + def self.image_url(url) + digest = OpenSSL::HMAC.hexdigest( + OpenSSL::Digest.new("sha1"), + AppConfig.privacy.camo.key, + url + ) + encoded_url = url.to_enum(:each_byte).map {|byte| "%02x" % byte}.join + + "#{AppConfig.privacy.camo.root}#{digest}/#{encoded_url}" + end + end +end diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index 3714e4487..3549d3721 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -87,6 +87,12 @@ module Diaspora def render_tags @message = Diaspora::Taggable.format_tags message, no_escape: !options[:escape_tags] end + + def camo_urls + @message = @message.gsub(/!\[.*?\]\((.+?)\)/) do |link| + link.gsub($1, Diaspora::CamoUrl::image_url($1)) + end + end end DEFAULTS = {mentioned_people: [], @@ -165,6 +171,13 @@ module Diaspora } end + # @param [Hash] opts Override global output options, see {#initialize} + def plain_text_for_json opts={} + process(opts) { + camo_urls if AppConfig.privacy.camo.proxy_markdown_images? + } + end + # @param [Hash] opts Override global output options, see {#initialize} def html opts={} process(opts) { From cca0c9eec4d7e10242e21bc9fde9aba83122ae30 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sat, 8 Nov 2014 20:03:34 +0100 Subject: [PATCH 03/14] Some refactorings, safer regex --- lib/diaspora.rb | 11 +++---- lib/diaspora/camo.rb | 31 ++++++++++++++++++++ lib/diaspora/camo_url.rb | 16 ---------- lib/diaspora/message_renderer.rb | 4 +-- spec/lib/diaspora/camo_spec.rb | 50 ++++++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 24 deletions(-) create mode 100644 lib/diaspora/camo.rb delete mode 100644 lib/diaspora/camo_url.rb create mode 100644 spec/lib/diaspora/camo_spec.rb diff --git a/lib/diaspora.rb b/lib/diaspora.rb index 161dbae8f..6f6d2002c 100644 --- a/lib/diaspora.rb +++ b/lib/diaspora.rb @@ -3,12 +3,13 @@ # the COPYRIGHT file. module Diaspora + require 'diaspora/camo' require 'diaspora/exceptions' - require 'diaspora/parser' - require 'diaspora/fetcher' - require 'diaspora/markdownify' - require 'diaspora/message_renderer' - require 'diaspora/mentionable' require 'diaspora/exporter' require 'diaspora/federated' + require 'diaspora/fetcher' + require 'diaspora/markdownify' + require 'diaspora/mentionable' + require 'diaspora/message_renderer' + require 'diaspora/parser' end diff --git a/lib/diaspora/camo.rb b/lib/diaspora/camo.rb new file mode 100644 index 000000000..6338b9076 --- /dev/null +++ b/lib/diaspora/camo.rb @@ -0,0 +1,31 @@ +# implicitly requires OpenSSL +module Diaspora + module Camo + def self.from_markdown(markdown_text) + markdown_text.gsub(/(!\[(.*?)\]\s?\([ \t]*()?[ \t]*((['"])(.*?)\6[ \t]*)?\))/m) do |link| + link.gsub($4, self.image_url($4)) + end + end + + def self.image_url(url) + return unless url + return url unless self.url_eligible?(url) + + digest = OpenSSL::HMAC.hexdigest( + OpenSSL::Digest.new('sha1'), + AppConfig.privacy.camo.key, + url + ) + + encoded_url = url.to_enum(:each_byte).map {|byte| '%02x' % byte}.join + "#{AppConfig.privacy.camo.root}#{digest}/#{encoded_url}" + end + + def self.url_eligible?(url) + return false unless url.start_with?('http', '//') + return false if url.start_with?(AppConfig.environment.url.to_s, + AppConfig.privacy.camo.root.to_s) + true + end + end +end diff --git a/lib/diaspora/camo_url.rb b/lib/diaspora/camo_url.rb deleted file mode 100644 index f0cbb0da2..000000000 --- a/lib/diaspora/camo_url.rb +++ /dev/null @@ -1,16 +0,0 @@ -# implicitly requires OpenSSL - -module Diaspora - module CamoUrl - def self.image_url(url) - digest = OpenSSL::HMAC.hexdigest( - OpenSSL::Digest.new("sha1"), - AppConfig.privacy.camo.key, - url - ) - encoded_url = url.to_enum(:each_byte).map {|byte| "%02x" % byte}.join - - "#{AppConfig.privacy.camo.root}#{digest}/#{encoded_url}" - end - end -end diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index 3549d3721..cd007d925 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -89,9 +89,7 @@ module Diaspora end def camo_urls - @message = @message.gsub(/!\[.*?\]\((.+?)\)/) do |link| - link.gsub($1, Diaspora::CamoUrl::image_url($1)) - end + @message = Diaspora::Camo::from_markdown(@message) end end diff --git a/spec/lib/diaspora/camo_spec.rb b/spec/lib/diaspora/camo_spec.rb new file mode 100644 index 000000000..c2def5fad --- /dev/null +++ b/spec/lib/diaspora/camo_spec.rb @@ -0,0 +1,50 @@ +# Copyright (c) 2010, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +require 'spec_helper' + +describe Diaspora::Camo do + before do + AppConfig.privacy.camo.root = 'http://localhost:3000/camo/' + AppConfig.privacy.camo.key = 'kittenpower' + + @raw_image_url = 'http://example.com/kitten.jpg' + @camo_image_url = "#{AppConfig.privacy.camo.root}5bc5b9d7ebd202841ab0667c4fc8d4304278f902/687474703a2f2f6578616d706c652e636f6d2f6b697474656e2e6a7067" + end + + describe '#image_url' do + it 'should not rewrite local URLs' do + local_image = "#{AppConfig.environment.url}kitten.jpg" + expect(Diaspora::Camo::image_url(local_image)).to eq(local_image) + end + + it 'should not rewrite relative URLs' do + relative_image = "/kitten.jpg" + expect(Diaspora::Camo::image_url(relative_image)).to eq(relative_image) + end + + it 'should not rewrite already camo-fied URLs' do + camo_image = "#{AppConfig.privacy.camo.root}1234/56789abcd" + expect(Diaspora::Camo::image_url(camo_image)).to eq(camo_image) + end + + it 'should rewrite external URLs' do + expect(Diaspora::Camo::image_url(@raw_image_url)).to eq(@camo_image_url) + end + end + + describe '#from_markdown' do + it 'should rewrite plain markdown images' do + expect(Diaspora::Camo::from_markdown("![](#{@raw_image_url})")).to include(@camo_image_url) + end + + it 'should rewrite markdown images with alt texts' do + expect(Diaspora::Camo::from_markdown("![a kitten](#{@raw_image_url})")).to include(@camo_image_url) + end + + it 'should rewrite markdown images with title texts' do + expect(Diaspora::Camo::from_markdown("![](#{@raw_image_url}) \"title\"")).to include(@camo_image_url) + end + end +end From 613f2d114272a9f90dece1c9ce3243462be1bdb6 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sat, 8 Nov 2014 21:20:44 +0100 Subject: [PATCH 04/14] Add Camo for comments --- app/models/comment.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/comment.rb b/app/models/comment.rb index b71fcaa22..abc89f302 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -83,6 +83,14 @@ class Comment < ActiveRecord::Base @message ||= Diaspora::MessageRenderer.new text end + def text + if AppConfig.privacy.camo.proxy_markdown_images? + Diaspora::Camo::from_markdown(self[:text]) + else + self[:text] + end + end + def text= text self[:text] = text.to_s.strip #to_s if for nil, for whatever reason end From 0a619749f2df12dfae08b764bf4191f2df333749 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sat, 8 Nov 2014 21:29:18 +0100 Subject: [PATCH 05/14] Add Camo for profile images --- app/models/profile.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/profile.rb b/app/models/profile.rb index bb73d7d72..11d44cd4f 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -74,7 +74,16 @@ class Profile < ActiveRecord::Base else self[:image_url] end - result || ActionController::Base.helpers.image_path('user/default.png') + + unless result + ActionController::Base.helpers.image_path('user/default.png') + else + if AppConfig.privacy.camo.proxy_remote_pod_images? + Diaspora::Camo::image_url(result) + else + result + end + end end def from_omniauth_hash(omniauth_user_hash) From 89ad5b96ad10c50c89fb6774cf3d7d1d84d7e06f Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sat, 8 Nov 2014 22:54:38 +0100 Subject: [PATCH 06/14] Add Camo for a profiles bio and location yes, we do have Markdown there. But please don't tell anyone. --- app/models/profile.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/models/profile.rb b/app/models/profile.rb index 11d44cd4f..6d8b4722b 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -142,6 +142,22 @@ class Profile < ActiveRecord::Base birthday.to_s(:long).gsub(', 1000', '') if birthday.present? end + def bio + if AppConfig.privacy.camo.proxy_markdown_images? + Diaspora::Camo::from_markdown(self[:bio]) + else + self[:bio] + end + end + + def location + if AppConfig.privacy.camo.proxy_markdown_images? + Diaspora::Camo::from_markdown(self[:location]) + else + self[:location] + end + end + def bio_message @bio_message ||= Diaspora::MessageRenderer.new(bio) end From db2560d7fc2e309bcaa3f74f35d68ff940f4b93e Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 01:36:16 +0100 Subject: [PATCH 07/14] Add Camo for OpenGraph images --- app/models/open_graph_cache.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/models/open_graph_cache.rb b/app/models/open_graph_cache.rb index d0c94362e..fac1a6320 100644 --- a/app/models/open_graph_cache.rb +++ b/app/models/open_graph_cache.rb @@ -15,6 +15,14 @@ class OpenGraphCache < ActiveRecord::Base t.add :url end + def image + if AppConfig.privacy.camo.proxy_opengraph_thumbnails? + Diaspora::Camo::image_url(self[:image]) + else + self[:image] + end + end + def self.find_or_create_by(opts) cache = OpenGraphCache.find_or_initialize_by(opts) cache.fetch_and_save_opengraph_data! unless cache.persisted? From 5c9cc4936314e91cfdd504db9d1113651f236a86 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 02:16:28 +0100 Subject: [PATCH 08/14] Only RegEx if there is something to RegEx --- lib/diaspora/camo.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/diaspora/camo.rb b/lib/diaspora/camo.rb index 6338b9076..51ee17d33 100644 --- a/lib/diaspora/camo.rb +++ b/lib/diaspora/camo.rb @@ -2,6 +2,7 @@ module Diaspora module Camo def self.from_markdown(markdown_text) + return unless markdown_text markdown_text.gsub(/(!\[(.*?)\]\s?\([ \t]*()?[ \t]*((['"])(.*?)\6[ \t]*)?\))/m) do |link| link.gsub($4, self.image_url($4)) end From 5bcd5087bf9fe6d185e7b4673f55f6a8a82f058e Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 02:17:43 +0100 Subject: [PATCH 09/14] Add Camo for remote images --- app/models/photo.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/models/photo.rb b/app/models/photo.rb index 1c36cd641..b6053f657 100644 --- a/app/models/photo.rb +++ b/app/models/photo.rb @@ -114,7 +114,12 @@ class Photo < ActiveRecord::Base def url(name = nil) if remote_photo_path name = name.to_s + '_' if name - remote_photo_path + name.to_s + remote_photo_name + image_url = remote_photo_path + name.to_s + remote_photo_name + if AppConfig.privacy.camo.proxy_remote_pod_images? + Diaspora::Camo::image_url(image_url) + else + image_url + end elsif processed? processed_image.url(name) else From a4a63c4c854ab952b326b50a71c06e096d0044a0 Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 02:27:17 +0100 Subject: [PATCH 10/14] A little bit of cleanup --- spec/lib/diaspora/camo_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/lib/diaspora/camo_spec.rb b/spec/lib/diaspora/camo_spec.rb index c2def5fad..fb291f38a 100644 --- a/spec/lib/diaspora/camo_spec.rb +++ b/spec/lib/diaspora/camo_spec.rb @@ -10,22 +10,22 @@ describe Diaspora::Camo do AppConfig.privacy.camo.key = 'kittenpower' @raw_image_url = 'http://example.com/kitten.jpg' - @camo_image_url = "#{AppConfig.privacy.camo.root}5bc5b9d7ebd202841ab0667c4fc8d4304278f902/687474703a2f2f6578616d706c652e636f6d2f6b697474656e2e6a7067" + @camo_image_url = AppConfig.privacy.camo.root + '5bc5b9d7ebd202841ab0667c4fc8d4304278f902/687474703a2f2f6578616d706c652e636f6d2f6b697474656e2e6a7067' end describe '#image_url' do it 'should not rewrite local URLs' do - local_image = "#{AppConfig.environment.url}kitten.jpg" + local_image = AppConfig.environment.url + 'kitten.jpg' expect(Diaspora::Camo::image_url(local_image)).to eq(local_image) end it 'should not rewrite relative URLs' do - relative_image = "/kitten.jpg" + relative_image = '/kitten.jpg' expect(Diaspora::Camo::image_url(relative_image)).to eq(relative_image) end it 'should not rewrite already camo-fied URLs' do - camo_image = "#{AppConfig.privacy.camo.root}1234/56789abcd" + camo_image = AppConfig.privacy.camo.root + '1234/56789abcd' expect(Diaspora::Camo::image_url(camo_image)).to eq(camo_image) end From df4063fd45904c94285ef1e801214185029514eb Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 06:51:17 +0100 Subject: [PATCH 11/14] Documentation improvements and changelog entry I just assume camo is cool enough to get an own section. --- Changelog.md | 5 +++++ config/diaspora.yml.example | 2 ++ 2 files changed, 7 insertions(+) diff --git a/Changelog.md b/Changelog.md index 0d4199887..b7586dfdb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -42,6 +42,11 @@ Removing of old inactive users can now be done automatically by background proce This maintenance is not enabled by default. Podmins can enable it by for example copying over the new settings under `settings.maintenance` to their `diaspora.yml` file and setting it enabled. The default setting is to expire accounts that have been inactive for 2 years (no login). +## Camo integration to proxy external assets +It is now possible to enable an automatic proxying of external assets, for example images embedded via Markdown or OpenGraph thumbnails loaded from insecure third party servers through a [Camo proxy](https://github.com/atmos/camo). + +This is disabled by default since it requires the installation of additional packages and might cause some traffic. Check the [wiki page](https://wiki.diasporafoundation.org/Installation/Camo) for more information and detailed installation instructions. + ## Refactor * Redesign contacts page [#5153](https://github.com/diaspora/diaspora/pull/5153) * Improve profile page design on mobile [#5084](https://github.com/diaspora/diaspora/pull/5084) diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index 9f3ee4335..ea8682eba 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -214,6 +214,8 @@ configuration: ## Section ## Do not enable this setting unless you have a working Camo setup. Using ## camo to proxy embedded images will improve the privacy and security of ## your pod's frontend, but it will increase the traffic on your server. + ## Check out https://wiki.diasporafoundation.org/Installation/Camo for more + ## details and installation instructions. camo: ## Section ## Proxy imaged embedded via markdown (default=false) From fd63fca7d52a3b9439dec95cb20da19dfa5b7d4f Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 07:29:36 +0100 Subject: [PATCH 12/14] Add Camo to MessageRenderer#markdownified for mobile views --- lib/diaspora/message_renderer.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index cd007d925..3eb818814 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -191,6 +191,7 @@ module Diaspora def markdownified opts={} process(opts) { process_newlines + camo_urls if AppConfig.privacy.camo.proxy_markdown_images? markdownify render_mentions render_tags From 68081509203b9021930f2c5c4d829d888cd9fcbe Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 07:20:30 -0800 Subject: [PATCH 13/14] Method calling convention and doc improvements ... and a stealth-commit of an already introduced bug prevention system. ;) --- app/models/comment.rb | 2 +- app/models/open_graph_cache.rb | 2 +- app/models/photo.rb | 2 +- app/models/profile.rb | 6 +++--- app/presenters/post_presenter.rb | 7 ++++++- config/diaspora.yml.example | 2 +- lib/diaspora/camo.rb | 2 +- lib/diaspora/message_renderer.rb | 2 +- spec/lib/diaspora/camo_spec.rb | 14 +++++++------- 9 files changed, 22 insertions(+), 17 deletions(-) diff --git a/app/models/comment.rb b/app/models/comment.rb index abc89f302..918b3f440 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -85,7 +85,7 @@ class Comment < ActiveRecord::Base def text if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo::from_markdown(self[:text]) + Diaspora::Camo.from_markdown(self[:text]) else self[:text] end diff --git a/app/models/open_graph_cache.rb b/app/models/open_graph_cache.rb index fac1a6320..072d30a98 100644 --- a/app/models/open_graph_cache.rb +++ b/app/models/open_graph_cache.rb @@ -17,7 +17,7 @@ class OpenGraphCache < ActiveRecord::Base def image if AppConfig.privacy.camo.proxy_opengraph_thumbnails? - Diaspora::Camo::image_url(self[:image]) + Diaspora::Camo.image_url(self[:image]) else self[:image] end diff --git a/app/models/photo.rb b/app/models/photo.rb index b6053f657..7c57d1cb2 100644 --- a/app/models/photo.rb +++ b/app/models/photo.rb @@ -116,7 +116,7 @@ class Photo < ActiveRecord::Base name = name.to_s + '_' if name image_url = remote_photo_path + name.to_s + remote_photo_name if AppConfig.privacy.camo.proxy_remote_pod_images? - Diaspora::Camo::image_url(image_url) + Diaspora::Camo.image_url(image_url) else image_url end diff --git a/app/models/profile.rb b/app/models/profile.rb index 6d8b4722b..75cf450e8 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -79,7 +79,7 @@ class Profile < ActiveRecord::Base ActionController::Base.helpers.image_path('user/default.png') else if AppConfig.privacy.camo.proxy_remote_pod_images? - Diaspora::Camo::image_url(result) + Diaspora::Camo.image_url(result) else result end @@ -144,7 +144,7 @@ class Profile < ActiveRecord::Base def bio if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo::from_markdown(self[:bio]) + Diaspora::Camo.from_markdown(self[:bio]) else self[:bio] end @@ -152,7 +152,7 @@ class Profile < ActiveRecord::Base def location if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo::from_markdown(self[:location]) + Diaspora::Camo.from_markdown(self[:location]) else self[:location] end diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index a01c090b2..ff3877bf7 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -13,10 +13,15 @@ class PostPresenter end def as_json(options={}) + text = if @post.message + @post.message.plain_text_for_json + else + @post.raw_message + end { :id => @post.id, :guid => @post.guid, - :text => @post.message.plain_text_for_json, + :text => text, :public => @post.public, :created_at => @post.created_at, :interacted_at => @post.interacted_at, diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index ea8682eba..152f370bb 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -236,7 +236,7 @@ configuration: ## Section #proxy_remote_pod_images: true ## Root of your Camo installation - #root: "https://camo.example.com/" + #root: "https://example.com/camo/" ## Shared key of your Camo installation #key: "example123example456example!" diff --git a/lib/diaspora/camo.rb b/lib/diaspora/camo.rb index 51ee17d33..eac55e3a8 100644 --- a/lib/diaspora/camo.rb +++ b/lib/diaspora/camo.rb @@ -19,7 +19,7 @@ module Diaspora ) encoded_url = url.to_enum(:each_byte).map {|byte| '%02x' % byte}.join - "#{AppConfig.privacy.camo.root}#{digest}/#{encoded_url}" + File.join(AppConfig.privacy.camo.root, digest, encoded_url) end def self.url_eligible?(url) diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index 3eb818814..f0eb11bd8 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -89,7 +89,7 @@ module Diaspora end def camo_urls - @message = Diaspora::Camo::from_markdown(@message) + @message = Diaspora::Camo.from_markdown(@message) end end diff --git a/spec/lib/diaspora/camo_spec.rb b/spec/lib/diaspora/camo_spec.rb index fb291f38a..e5561d40a 100644 --- a/spec/lib/diaspora/camo_spec.rb +++ b/spec/lib/diaspora/camo_spec.rb @@ -16,35 +16,35 @@ describe Diaspora::Camo do describe '#image_url' do it 'should not rewrite local URLs' do local_image = AppConfig.environment.url + 'kitten.jpg' - expect(Diaspora::Camo::image_url(local_image)).to eq(local_image) + expect(Diaspora::Camo.image_url(local_image)).to eq(local_image) end it 'should not rewrite relative URLs' do relative_image = '/kitten.jpg' - expect(Diaspora::Camo::image_url(relative_image)).to eq(relative_image) + expect(Diaspora::Camo.image_url(relative_image)).to eq(relative_image) end it 'should not rewrite already camo-fied URLs' do camo_image = AppConfig.privacy.camo.root + '1234/56789abcd' - expect(Diaspora::Camo::image_url(camo_image)).to eq(camo_image) + expect(Diaspora::Camo.image_url(camo_image)).to eq(camo_image) end it 'should rewrite external URLs' do - expect(Diaspora::Camo::image_url(@raw_image_url)).to eq(@camo_image_url) + expect(Diaspora::Camo.image_url(@raw_image_url)).to eq(@camo_image_url) end end describe '#from_markdown' do it 'should rewrite plain markdown images' do - expect(Diaspora::Camo::from_markdown("![](#{@raw_image_url})")).to include(@camo_image_url) + expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url})")).to include(@camo_image_url) end it 'should rewrite markdown images with alt texts' do - expect(Diaspora::Camo::from_markdown("![a kitten](#{@raw_image_url})")).to include(@camo_image_url) + expect(Diaspora::Camo.from_markdown("![a kitten](#{@raw_image_url})")).to include(@camo_image_url) end it 'should rewrite markdown images with title texts' do - expect(Diaspora::Camo::from_markdown("![](#{@raw_image_url}) \"title\"")).to include(@camo_image_url) + expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url}) \"title\"")).to include(@camo_image_url) end end end From 4c98445f1c070c677d23a9c6c07a0cab148589aa Mon Sep 17 00:00:00 2001 From: Dennis Schubert Date: Sun, 9 Nov 2014 07:47:40 -0800 Subject: [PATCH 14/14] Fix presenters instead of modifying the getters --- app/models/comment.rb | 8 -------- app/models/profile.rb | 16 ---------------- app/presenters/comment_presenter.rb | 4 ++-- app/presenters/profile_presenter.rb | 4 ++-- 4 files changed, 4 insertions(+), 28 deletions(-) diff --git a/app/models/comment.rb b/app/models/comment.rb index 918b3f440..b71fcaa22 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -83,14 +83,6 @@ class Comment < ActiveRecord::Base @message ||= Diaspora::MessageRenderer.new text end - def text - if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo.from_markdown(self[:text]) - else - self[:text] - end - end - def text= text self[:text] = text.to_s.strip #to_s if for nil, for whatever reason end diff --git a/app/models/profile.rb b/app/models/profile.rb index 75cf450e8..bfbb535e5 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -142,22 +142,6 @@ class Profile < ActiveRecord::Base birthday.to_s(:long).gsub(', 1000', '') if birthday.present? end - def bio - if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo.from_markdown(self[:bio]) - else - self[:bio] - end - end - - def location - if AppConfig.privacy.camo.proxy_markdown_images? - Diaspora::Camo.from_markdown(self[:location]) - else - self[:location] - end - end - def bio_message @bio_message ||= Diaspora::MessageRenderer.new(bio) end diff --git a/app/presenters/comment_presenter.rb b/app/presenters/comment_presenter.rb index 725d206c3..95eef095c 100644 --- a/app/presenters/comment_presenter.rb +++ b/app/presenters/comment_presenter.rb @@ -7,9 +7,9 @@ class CommentPresenter < BasePresenter { :id => @comment.id, :guid => @comment.guid, - :text => @comment.text, + :text => @comment.message.plain_text_for_json, :author => @comment.author.as_api_response(:backbone), :created_at => @comment.created_at } end -end \ No newline at end of file +end diff --git a/app/presenters/profile_presenter.rb b/app/presenters/profile_presenter.rb index 047955e71..3581ef9f9 100644 --- a/app/presenters/profile_presenter.rb +++ b/app/presenters/profile_presenter.rb @@ -4,8 +4,8 @@ class ProfilePresenter < BasePresenter def base_hash { id: id, tags: tags.pluck(:name), - bio: bio, - location: location, + bio: bio_message.plain_text_for_json, + location: location_message.plain_text_for_json, gender: gender, birthday: formatted_birthday, searchable: searchable