diff --git a/Changelog.md b/Changelog.md index 0d4199887..b7586dfdb 100644 --- a/Changelog.md +++ b/Changelog.md @@ -42,6 +42,11 @@ Removing of old inactive users can now be done automatically by background proce This maintenance is not enabled by default. Podmins can enable it by for example copying over the new settings under `settings.maintenance` to their `diaspora.yml` file and setting it enabled. The default setting is to expire accounts that have been inactive for 2 years (no login). +## Camo integration to proxy external assets +It is now possible to enable an automatic proxying of external assets, for example images embedded via Markdown or OpenGraph thumbnails loaded from insecure third party servers through a [Camo proxy](https://github.com/atmos/camo). + +This is disabled by default since it requires the installation of additional packages and might cause some traffic. Check the [wiki page](https://wiki.diasporafoundation.org/Installation/Camo) for more information and detailed installation instructions. + ## Refactor * Redesign contacts page [#5153](https://github.com/diaspora/diaspora/pull/5153) * Improve profile page design on mobile [#5084](https://github.com/diaspora/diaspora/pull/5084) diff --git a/app/models/open_graph_cache.rb b/app/models/open_graph_cache.rb index d0c94362e..072d30a98 100644 --- a/app/models/open_graph_cache.rb +++ b/app/models/open_graph_cache.rb @@ -15,6 +15,14 @@ class OpenGraphCache < ActiveRecord::Base t.add :url end + def image + if AppConfig.privacy.camo.proxy_opengraph_thumbnails? + Diaspora::Camo.image_url(self[:image]) + else + self[:image] + end + end + def self.find_or_create_by(opts) cache = OpenGraphCache.find_or_initialize_by(opts) cache.fetch_and_save_opengraph_data! unless cache.persisted? diff --git a/app/models/photo.rb b/app/models/photo.rb index 1c36cd641..7c57d1cb2 100644 --- a/app/models/photo.rb +++ b/app/models/photo.rb @@ -114,7 +114,12 @@ class Photo < ActiveRecord::Base def url(name = nil) if remote_photo_path name = name.to_s + '_' if name - remote_photo_path + name.to_s + remote_photo_name + image_url = remote_photo_path + name.to_s + remote_photo_name + if AppConfig.privacy.camo.proxy_remote_pod_images? + Diaspora::Camo.image_url(image_url) + else + image_url + end elsif processed? processed_image.url(name) else diff --git a/app/models/profile.rb b/app/models/profile.rb index bb73d7d72..bfbb535e5 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -74,7 +74,16 @@ class Profile < ActiveRecord::Base else self[:image_url] end - result || ActionController::Base.helpers.image_path('user/default.png') + + unless result + ActionController::Base.helpers.image_path('user/default.png') + else + if AppConfig.privacy.camo.proxy_remote_pod_images? + Diaspora::Camo.image_url(result) + else + result + end + end end def from_omniauth_hash(omniauth_user_hash) diff --git a/app/presenters/comment_presenter.rb b/app/presenters/comment_presenter.rb index 725d206c3..95eef095c 100644 --- a/app/presenters/comment_presenter.rb +++ b/app/presenters/comment_presenter.rb @@ -7,9 +7,9 @@ class CommentPresenter < BasePresenter { :id => @comment.id, :guid => @comment.guid, - :text => @comment.text, + :text => @comment.message.plain_text_for_json, :author => @comment.author.as_api_response(:backbone), :created_at => @comment.created_at } end -end \ No newline at end of file +end diff --git a/app/presenters/post_presenter.rb b/app/presenters/post_presenter.rb index e6dd6fa31..ff3877bf7 100644 --- a/app/presenters/post_presenter.rb +++ b/app/presenters/post_presenter.rb @@ -13,10 +13,15 @@ class PostPresenter end def as_json(options={}) + text = if @post.message + @post.message.plain_text_for_json + else + @post.raw_message + end { :id => @post.id, :guid => @post.guid, - :text => @post.raw_message, + :text => text, :public => @post.public, :created_at => @post.created_at, :interacted_at => @post.interacted_at, diff --git a/app/presenters/profile_presenter.rb b/app/presenters/profile_presenter.rb index 047955e71..3581ef9f9 100644 --- a/app/presenters/profile_presenter.rb +++ b/app/presenters/profile_presenter.rb @@ -4,8 +4,8 @@ class ProfilePresenter < BasePresenter def base_hash { id: id, tags: tags.pluck(:name), - bio: bio, - location: location, + bio: bio_message.plain_text_for_json, + location: location_message.plain_text_for_json, gender: gender, birthday: formatted_birthday, searchable: searchable diff --git a/config/defaults.yml b/config/defaults.yml index 841fc2643..8cf35d41b 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -53,6 +53,12 @@ defaults: user_counts: false post_counts: false comment_counts: false + camo: + proxy_markdown_images: false + proxy_opengraph_thumbnails: false + proxy_remote_pod_images: false + root: + key: settings: pod_name: 'diaspora*' enable_registrations: true diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index 3a4153edf..152f370bb 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -24,11 +24,11 @@ ## heroku config:set SERVICES_FACEBOOK_APP_ID=yourappid SERVICES_FACEBOOK_SECRET=yourappsecret configuration: ## Section - + ## You need to change or at least review the settings in this section ## in order for your pod to work environment: ## Section - + ## Set the hostname of the machine you're running Diaspora on, as seen ## from the internet. This should be the URL you want to use to ## access the pod. So if you plan to use a reverse proxy, it should be @@ -37,7 +37,7 @@ configuration: ## Section ## If you do change the URL, you will have to start again as the URL ## will be hardcoded into the database. #url: "https://example.org/" - + ## Set the bundle of certificate authorities (CA) certificates. ## This is specific to your operating system. ## Examples (uncomment the relevant one or add your own): @@ -45,20 +45,20 @@ configuration: ## Section #certificate_authorities: '/etc/ssl/certs/ca-certificates.crt' ## For CentOS, Fedora: #certificate_authorities: '/etc/pki/tls/certs/ca-bundle.crt' - + ## URL for a remote Redis (default=localhost) ## Don't forget to restrict IP access if you uncomment these! #redis: 'redis://example_host' #redis: 'redis://username:password@host:6379/0' #redis: 'unix:///tmp/redis.sock' - + ## Require SSL (default=true) ## When set, your pod will force the use of HTTPS in production mode. Since ## OAuth2 requires SSL Diaspora's future API might not work if you're not using ## SSL. Also there is no guarantee that posting to services will be possible ## if SSL is disabled. Do not change this default unless you are sure! #require_ssl: true - + ## Single-process mode (default=false) ## If set to true Diaspora will work with just the appserver (Unicorn by default) ## running. However, this makes it quite slow as intensive jobs must be run @@ -71,7 +71,7 @@ configuration: ## Section ## Number of parallel threads Sidekiq uses (default=5) ## If you touch this please set the pool setting in your database.yml to - ## a value that's at minimum close to this! You can safely increase it + ## a value that's at minimum close to this! You can safely increase it ## to 25 and more on a medium-sized pod. This applies per started Sidekiq ## worker, so if you set it to 25 and start two workers you'll process ## up to 50 jobs in parallel. @@ -94,7 +94,7 @@ configuration: ## Section ## Log file for Sidekiq (default="log/sidekiq.log") #log: "log/sidekiq.log" - + ## Use Amazon S3 instead of your local filesystem ## to handle uploaded pictures (disabled by default) s3: ## Section @@ -110,23 +110,23 @@ configuration: ## Section ## 1 year. This can improve load speed and save requests to the image host. ## Set to false to revert to browser defaults (usually less than 1 year). #cache : true - + ## Set redirect URL for an external image host (Amazon S3 or other) ## If hosting images for your pod on an external server (even your own), ## add its URL here. All requests made to images under /uploads/images ## will be redirected to https://yourhost.tld/uploads/images/ #image_redirect_url: 'https://images.example.org' - + assets: ## Section ## Serve static assets via the appserver (default=false) ## This is highly discouraged for production use. Let your reverse ## proxy/webserver do it by serving the files under public/ directly. #serve: false - + ## Upload your assets to S3 (default=false) #upload: false - + ## Specify an asset host. Ensure it does not have a trailing slash (/). #host: http://cdn.example.org/diaspora @@ -134,7 +134,7 @@ configuration: ## Section ## Diaspora is only tested against the default pubsub server. ## You probably don't want to uncomment or change this. #pubsub_server: 'https://pubsubhubbub.appspot.com/' - + ## Settings affecting how ./script/server behaves. server: ## Section @@ -145,11 +145,11 @@ configuration: ## Section ## The environment in which the server should be started by default. ## Change this if you wish to run a production environment. #rails_environment: 'development' - + ## Write unicorn stderr and stdout log #stderr_log: '/usr/local/app/diaspora/log/unicorn-stderr.log' #stdout_log: '/usr/local/app/diaspora/log/unicorn-stdout.log' - + ## Number of Unicorn worker processes (default=2) ## Increase this if you have many users. #unicorn_worker: 2 @@ -159,7 +159,7 @@ configuration: ## Section ## Decrease if you're under heavy load and don't care if some ## requests fail. #unicorn_timeout: 90 - + ## Embed a Sidekiq worker inside the unicorn process (default=false) ## Useful for minimal Heroku setups. #embed_sidekiq_worker: false @@ -168,20 +168,20 @@ configuration: ## Section ## In most cases it is better to ## increase environment.sidekiq.concurrency instead! #sidekiq_workers: 1 - - ## Settings potentially affecting the privacy of your users + + ## Settings potentially affecting the privacy of your users privacy: ## Section - + ## Include jQuery from jquery.com's CDN (default=false) ## Enabling this can reduce traffic and speed up load time since most ## clients already have this one cached. When set to false (the default), ## the jQuery library will be loaded from your pod's own resources. #jquery_cdn: false - + ## Google Analytics (disabled by default) - ## Provide a key to enable tracking by Google Analytics + ## Provide a key to enable tracking by Google Analytics #google_analytics_key: - + ## Piwik Tracking (disabled by default) ## Provide a site ID and the host piwik is running on to enable ## tracking through Piwik. @@ -190,17 +190,17 @@ configuration: ## Section #enable: true #host: 'stats.example.org' #site_id: 1 - + ## Mixpanel event tracking (disabled by default) #mixpanel_uid: - + ## Chartbeat tracking (disabled by default) #chartbeat_uid: - + ## Statistics ## Your pod will report its name, software version and whether ## or not registrations are open via /statistics.json. - ## Uncomment the options below to enable more statistics. + ## Uncomment the options below to enable more statistics. statistics: ## Section ## Local user total and 6 month active counts @@ -209,10 +209,41 @@ configuration: ## Section ## Local post total count #post_counts: true #comment_counts: true - + + ## Use Camo to proxy embedded remote images + ## Do not enable this setting unless you have a working Camo setup. Using + ## camo to proxy embedded images will improve the privacy and security of + ## your pod's frontend, but it will increase the traffic on your server. + ## Check out https://wiki.diasporafoundation.org/Installation/Camo for more + ## details and installation instructions. + camo: ## Section + + ## Proxy imaged embedded via markdown (default=false) + ## Embedded images are quite often from non-SSL sites and may cause a + ## partial content warning, so this is recommended. + #proxy_markdown_images: true + + ## Proxy Open Graph thumbnails (default=false) + ## Open Graph thumbnails may or may not be encrypted and loaded from + ## servers outside the network. Recommended. + #proxy_opengraph_thumbnails: true + + ## Proxy remote pod's images (default=false) + ## Profile pictures and photos from other pods usually are encrypted, + ## so enabling this is only useful if you want to avoid HTTP requests to + ## third-party servers. This will create a lot of traffic on your camo + ## instance. You have been warned. + #proxy_remote_pod_images: true + + ## Root of your Camo installation + #root: "https://example.com/camo/" + + ## Shared key of your Camo installation + #key: "example123example456example!" + ## General settings settings: ## Section - + ## Pod name (default="diaspora*") ## The pod name displayed in various locations, including the header. #pod_name: "diaspora*" @@ -222,13 +253,13 @@ configuration: ## Section ## without an invitation. Note that this needs to be set to true ## (or commented out) to enable the first registration (you). #enable_registrations: true - + ## Auto-follow on sign-up (default=true) ## Users will automatically follow a specified account on creation. ## Set this to false if you don't want your users to automatically ## follow an account upon creation. #autofollow_on_join: true - + ## Auto-follow account (default='diasporahq@joindiaspora.com') ## The diaspora* HQ account keeps users up to date with news about Diaspora. ## If you set another auto-follow account (for example your podmin account), @@ -237,26 +268,26 @@ configuration: ## Section ## Invitation settings invitations: ## Section - + ## Enable invitations (default=true) ## Set this to false if you don't want users to be able to send invites. #open: true - + ## Number of invitations per invite link (default=25) ## Every user will see such a link if you have enabled invitations on your pod. #count: 25 - + ## Paypal donations ## You can provide the ID of a hosted Paypal button here to allow your users ## to send donations to help run their pod. If you leave this out your users ## will see a button to donate to the Diaspora Foundation instead :) #paypal_hosted_button_id: "change_me" - + ## Bitcoin donations ## You can provide a bitcoin address here to allow your users to provide ## donations towards the running of their pod. #bitcoin_address: "change_me" - + ## Community spotlight (disabled by default) ## The community spotlight shows new users public posts from people you ## think are interesting in Diaspora's community. To add an account @@ -268,12 +299,12 @@ configuration: ## Section ## E-mail address to which users can make suggestions about who ## should be in the community spotlight (optional). #suggest_email: 'admin@example.org' - + ## CURL debug (default=false) ## Turn on extra verbose output when sending stuff. Note: you ## don't need to touch this unless explicitly told to. #typhoeus_verbose: false - + ## Maximum number of parallel HTTP requests made to other pods (default=20) ## Be careful, raising this setting will heavily increase the memory usage ## of your Sidekiq workers. @@ -302,7 +333,7 @@ configuration: ## Section ## Sets the level of image distortion used in the captcha. ## Available options are: 'low', 'medium', 'high', 'random' #distortion: 'low' - + ## Terms of Service ## Show a default or customized terms of service for users. ## You can create a custom Terms of Service by placing a template @@ -330,7 +361,7 @@ configuration: ## Section ## Set a number to activate this setting. This age limit is shown ## in the default ToS document. #minimum_age: false - + ## Maintenance ## Various pod maintenance related settings are controlled from here. maintenance: ## Section @@ -355,42 +386,42 @@ configuration: ## Section #enable: true #app_id: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Twitter: twitter: ## Section #enable: true #key: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Tumblr tumblr: ## Section #enable: true #key: 'abcdef' #secret: 'change_me' - + ## OAuth credentials for Wordpress wordpress: ## Section #enable: true #client_id: 'abcdef' #secret: 'change_me' - + ## Enable pod users to send e-mails from Diaspora (disabled by default) mail: ## Section - + ## First you need to enable it. #enable: true - + ## Sender address used in mail sent by Diaspora #sender_address: 'no-reply@example.org' - + ## This selects which mailer should be used. Use 'smtp' for a smtp ## connection, 'sendmail' to use the sendmail binary or ## 'messagebus' to use the messagebus service. #method: 'smtp' - + ## Ignore if method isn't 'smtp' smtp: ## Section @@ -399,44 +430,44 @@ configuration: ## Section ## the SMTP server, if it sends one. (default port=587) #host: 'smtp.example.org' #port: 587 - + ## Authentication required to send mail (default='plain') ## Use one of 'plain', 'login' or 'cram_md5'. Use 'none' ## if server does not support authentication. #authentication: 'plain' - + ## Credentials to log in to the SMTP server ## May be necessary if authentication is not 'none'. #username: 'change_me' #password: 'change_me' - + ## Automatically enable TLS (default=true) ## Leave this commented out if authentication is set to 'none'. #starttls_auto: true - + ## The domain for the HELO command, if needed #domain: 'smtp.example.org' - + ## OpenSSL verify mode used when connecting to a SMTP server with TLS ## Set this to 'none' if you have a self-signed certificate. Possible ## values: 'none', 'peer', 'client_once', 'fail_if_no_peer_cert'. #openssl_verify_mode: 'none' - + ## Ignore if method isn't 'sendmail' sendmail: ## Section ## The path to the sendmail binary (default='/usr/sbin/sendmail') #location: '/usr/sbin/sendmail' - + ## Use exim and sendmail (default=false) #exim_fix: false - + ## Ignore if method isn't 'messagebus' #message_bus_api_key: 'abcdef' - + ## Administrator settings admins: ## Section - + ## Set the admin account ## This doesn't make the user an admin but is used when a generic ## admin contact is needed, much like the postmaster role in mail @@ -445,13 +476,13 @@ configuration: ## Section ## E-mail address to contact the administrator #podmin_email: 'podmin@example.org' - + ## Here you can override settings defined above if you need ## to have them different in different environments. production: ## Section environment: ## Section #redis_url: 'redis://production.example.org:6379' - + development: ## Section environment: ## Section #redis_url: 'redis://production.example.org:6379' diff --git a/lib/diaspora.rb b/lib/diaspora.rb index 161dbae8f..6f6d2002c 100644 --- a/lib/diaspora.rb +++ b/lib/diaspora.rb @@ -3,12 +3,13 @@ # the COPYRIGHT file. module Diaspora + require 'diaspora/camo' require 'diaspora/exceptions' - require 'diaspora/parser' - require 'diaspora/fetcher' - require 'diaspora/markdownify' - require 'diaspora/message_renderer' - require 'diaspora/mentionable' require 'diaspora/exporter' require 'diaspora/federated' + require 'diaspora/fetcher' + require 'diaspora/markdownify' + require 'diaspora/mentionable' + require 'diaspora/message_renderer' + require 'diaspora/parser' end diff --git a/lib/diaspora/camo.rb b/lib/diaspora/camo.rb new file mode 100644 index 000000000..eac55e3a8 --- /dev/null +++ b/lib/diaspora/camo.rb @@ -0,0 +1,32 @@ +# implicitly requires OpenSSL +module Diaspora + module Camo + def self.from_markdown(markdown_text) + return unless markdown_text + markdown_text.gsub(/(!\[(.*?)\]\s?\([ \t]*()?[ \t]*((['"])(.*?)\6[ \t]*)?\))/m) do |link| + link.gsub($4, self.image_url($4)) + end + end + + def self.image_url(url) + return unless url + return url unless self.url_eligible?(url) + + digest = OpenSSL::HMAC.hexdigest( + OpenSSL::Digest.new('sha1'), + AppConfig.privacy.camo.key, + url + ) + + encoded_url = url.to_enum(:each_byte).map {|byte| '%02x' % byte}.join + File.join(AppConfig.privacy.camo.root, digest, encoded_url) + end + + def self.url_eligible?(url) + return false unless url.start_with?('http', '//') + return false if url.start_with?(AppConfig.environment.url.to_s, + AppConfig.privacy.camo.root.to_s) + true + end + end +end diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index 3714e4487..f0eb11bd8 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -87,6 +87,10 @@ module Diaspora def render_tags @message = Diaspora::Taggable.format_tags message, no_escape: !options[:escape_tags] end + + def camo_urls + @message = Diaspora::Camo.from_markdown(@message) + end end DEFAULTS = {mentioned_people: [], @@ -165,6 +169,13 @@ module Diaspora } end + # @param [Hash] opts Override global output options, see {#initialize} + def plain_text_for_json opts={} + process(opts) { + camo_urls if AppConfig.privacy.camo.proxy_markdown_images? + } + end + # @param [Hash] opts Override global output options, see {#initialize} def html opts={} process(opts) { @@ -180,6 +191,7 @@ module Diaspora def markdownified opts={} process(opts) { process_newlines + camo_urls if AppConfig.privacy.camo.proxy_markdown_images? markdownify render_mentions render_tags diff --git a/spec/lib/diaspora/camo_spec.rb b/spec/lib/diaspora/camo_spec.rb new file mode 100644 index 000000000..e5561d40a --- /dev/null +++ b/spec/lib/diaspora/camo_spec.rb @@ -0,0 +1,50 @@ +# Copyright (c) 2010, Diaspora Inc. This file is +# licensed under the Affero General Public License version 3 or later. See +# the COPYRIGHT file. + +require 'spec_helper' + +describe Diaspora::Camo do + before do + AppConfig.privacy.camo.root = 'http://localhost:3000/camo/' + AppConfig.privacy.camo.key = 'kittenpower' + + @raw_image_url = 'http://example.com/kitten.jpg' + @camo_image_url = AppConfig.privacy.camo.root + '5bc5b9d7ebd202841ab0667c4fc8d4304278f902/687474703a2f2f6578616d706c652e636f6d2f6b697474656e2e6a7067' + end + + describe '#image_url' do + it 'should not rewrite local URLs' do + local_image = AppConfig.environment.url + 'kitten.jpg' + expect(Diaspora::Camo.image_url(local_image)).to eq(local_image) + end + + it 'should not rewrite relative URLs' do + relative_image = '/kitten.jpg' + expect(Diaspora::Camo.image_url(relative_image)).to eq(relative_image) + end + + it 'should not rewrite already camo-fied URLs' do + camo_image = AppConfig.privacy.camo.root + '1234/56789abcd' + expect(Diaspora::Camo.image_url(camo_image)).to eq(camo_image) + end + + it 'should rewrite external URLs' do + expect(Diaspora::Camo.image_url(@raw_image_url)).to eq(@camo_image_url) + end + end + + describe '#from_markdown' do + it 'should rewrite plain markdown images' do + expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url})")).to include(@camo_image_url) + end + + it 'should rewrite markdown images with alt texts' do + expect(Diaspora::Camo.from_markdown("![a kitten](#{@raw_image_url})")).to include(@camo_image_url) + end + + it 'should rewrite markdown images with title texts' do + expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url}) \"title\"")).to include(@camo_image_url) + end + end +end