Merge pull request #5386 from denschub/camo

Add Camo support

Changelog.md

@@ -42,6 +42,11 @@ Removing of old inactive users can now be done automatically by background proce
 
 This maintenance is not enabled by default. Podmins can enable it by for example copying over the new settings under `settings.maintenance` to their `diaspora.yml` file and setting it enabled. The default setting is to expire accounts that have been inactive for 2 years (no login).
 
+## Camo integration to proxy external assets
+It is now possible to enable an automatic proxying of external assets, for example images embedded via Markdown or OpenGraph thumbnails loaded from insecure third party servers through a [Camo proxy](https://github.com/atmos/camo).
+
+This is disabled by default since it requires the installation of additional packages and might cause some traffic. Check the [wiki page](https://wiki.diasporafoundation.org/Installation/Camo) for more information and detailed installation instructions.
+
 ## Refactor
 * Redesign contacts page [#5153](https://github.com/diaspora/diaspora/pull/5153)
 * Improve profile page design on mobile [#5084](https://github.com/diaspora/diaspora/pull/5084)

app/models/open_graph_cache.rb

@@ -15,6 +15,14 @@ class OpenGraphCache < ActiveRecord::Base
     t.add :url
   end
 
+  def image
+    if AppConfig.privacy.camo.proxy_opengraph_thumbnails?
+      Diaspora::Camo.image_url(self[:image])
+    else
+      self[:image]
+    end
+  end
+
   def self.find_or_create_by(opts)
     cache = OpenGraphCache.find_or_initialize_by(opts)
     cache.fetch_and_save_opengraph_data! unless cache.persisted?

app/models/photo.rb

@@ -114,7 +114,12 @@ class Photo < ActiveRecord::Base
   def url(name = nil)
     if remote_photo_path
       name = name.to_s + '_' if name
-      remote_photo_path + name.to_s + remote_photo_name
+      image_url = remote_photo_path + name.to_s + remote_photo_name
+      if AppConfig.privacy.camo.proxy_remote_pod_images?
+        Diaspora::Camo.image_url(image_url)
+      else
+        image_url
+      end
     elsif processed?
       processed_image.url(name)
     else

app/models/profile.rb

@@ -74,7 +74,16 @@ class Profile < ActiveRecord::Base
              else
                self[:image_url]
              end
-    result || ActionController::Base.helpers.image_path('user/default.png')
+
+    unless result
+      ActionController::Base.helpers.image_path('user/default.png')
+    else
+      if AppConfig.privacy.camo.proxy_remote_pod_images?
+        Diaspora::Camo.image_url(result)
+      else
+        result
+      end
+    end
   end
 
   def from_omniauth_hash(omniauth_user_hash)

app/presenters/comment_presenter.rb

@@ -7,7 +7,7 @@ class CommentPresenter < BasePresenter
     {
       :id => @comment.id,
       :guid => @comment.guid,
-      :text  => @comment.text,
+      :text  => @comment.message.plain_text_for_json,
       :author => @comment.author.as_api_response(:backbone),
       :created_at => @comment.created_at
     }

app/presenters/post_presenter.rb

@@ -13,10 +13,15 @@ class PostPresenter
   end
 
   def as_json(options={})
+    text = if @post.message
+      @post.message.plain_text_for_json
+    else
+      @post.raw_message
+    end
     {
         :id => @post.id,
         :guid => @post.guid,
-        :text => @post.raw_message,
+        :text => text,
         :public => @post.public,
         :created_at => @post.created_at,
         :interacted_at => @post.interacted_at,

app/presenters/profile_presenter.rb

@@ -4,8 +4,8 @@ class ProfilePresenter < BasePresenter
   def base_hash
     {  id: id,
        tags: tags.pluck(:name),
-       bio: bio,
-       location: location,
+       bio: bio_message.plain_text_for_json,
+       location: location_message.plain_text_for_json,
        gender: gender,
        birthday: formatted_birthday,
        searchable: searchable

config/defaults.yml

@@ -53,6 +53,12 @@ defaults:
       user_counts: false
       post_counts: false
       comment_counts: false
+    camo:
+      proxy_markdown_images: false
+      proxy_opengraph_thumbnails: false
+      proxy_remote_pod_images: false
+      root:
+      key:
   settings:
     pod_name: 'diaspora*'
     enable_registrations: true

config/diaspora.yml.example

@@ -210,6 +210,37 @@ configuration: ## Section
       #post_counts: true
       #comment_counts: true
 
+    ## Use Camo to proxy embedded remote images
+    ## Do not enable this setting unless you have a working Camo setup. Using
+    ## camo to proxy embedded images will improve the privacy and security of
+    ## your pod's frontend, but it will increase the traffic on your server.
+    ## Check out https://wiki.diasporafoundation.org/Installation/Camo for more
+    ## details and installation instructions.
+    camo: ## Section
+
+      ## Proxy imaged embedded via markdown (default=false)
+      ## Embedded images are quite often from non-SSL sites and may cause a
+      ## partial content warning, so this is recommended.
+      #proxy_markdown_images: true
+
+      ## Proxy Open Graph thumbnails (default=false)
+      ## Open Graph thumbnails may or may not be encrypted and loaded from
+      ## servers outside the network. Recommended.
+      #proxy_opengraph_thumbnails: true
+
+      ## Proxy remote pod's images (default=false)
+      ## Profile pictures and photos from other pods usually are encrypted,
+      ## so enabling this is only useful if you want to avoid HTTP requests to
+      ## third-party servers. This will create a lot of traffic on your camo
+      ## instance. You have been warned.
+      #proxy_remote_pod_images: true
+
+      ## Root of your Camo installation
+      #root: "https://example.com/camo/"
+
+      ## Shared key of your Camo installation
+      #key: "example123example456example!"
+
   ## General settings
   settings: ## Section
 

lib/diaspora.rb

@@ -3,12 +3,13 @@
 #   the COPYRIGHT file.
 
 module Diaspora
+  require 'diaspora/camo'
   require 'diaspora/exceptions'
-  require 'diaspora/parser'
-  require 'diaspora/fetcher'
-  require 'diaspora/markdownify'
-  require 'diaspora/message_renderer'
-  require 'diaspora/mentionable'
   require 'diaspora/exporter'
   require 'diaspora/federated'
+  require 'diaspora/fetcher'
+  require 'diaspora/markdownify'
+  require 'diaspora/mentionable'
+  require 'diaspora/message_renderer'
+  require 'diaspora/parser'
 end

lib/diaspora/camo.rb

@@ -0,0 +1,32 @@
+# implicitly requires OpenSSL
+module Diaspora
+  module Camo
+    def self.from_markdown(markdown_text)
+      return unless markdown_text
+      markdown_text.gsub(/(!\[(.*?)\]\s?\([ \t]*()<?(\S+?)>?[ \t]*((['"])(.*?)\6[ \t]*)?\))/m) do |link|
+        link.gsub($4, self.image_url($4))
+      end
+    end
+
+    def self.image_url(url)
+      return unless url
+      return url unless self.url_eligible?(url)
+
+      digest = OpenSSL::HMAC.hexdigest(
+        OpenSSL::Digest.new('sha1'),
+        AppConfig.privacy.camo.key,
+        url
+      )
+
+      encoded_url = url.to_enum(:each_byte).map {|byte| '%02x' % byte}.join
+      File.join(AppConfig.privacy.camo.root, digest, encoded_url)
+    end
+
+    def self.url_eligible?(url)
+      return false unless url.start_with?('http', '//')
+      return false if url.start_with?(AppConfig.environment.url.to_s,
+                                      AppConfig.privacy.camo.root.to_s)
+      true
+    end
+  end
+end

lib/diaspora/message_renderer.rb

@@ -87,6 +87,10 @@ module Diaspora
       def render_tags
         @message = Diaspora::Taggable.format_tags message, no_escape: !options[:escape_tags]
       end
+
+      def camo_urls
+        @message = Diaspora::Camo.from_markdown(@message)
+      end
     end
 
     DEFAULTS = {mentioned_people: [],
@@ -165,6 +169,13 @@ module Diaspora
       }
     end
 
+    # @param [Hash] opts Override global output options, see {#initialize}
+    def plain_text_for_json opts={}
+      process(opts) {
+        camo_urls if AppConfig.privacy.camo.proxy_markdown_images?
+      }
+    end
+
     # @param [Hash] opts Override global output options, see {#initialize}
     def html opts={}
       process(opts) {
@@ -180,6 +191,7 @@ module Diaspora
     def markdownified opts={}
       process(opts) {
         process_newlines
+        camo_urls if AppConfig.privacy.camo.proxy_markdown_images?
         markdownify
         render_mentions
         render_tags

spec/lib/diaspora/camo_spec.rb

@@ -0,0 +1,50 @@
+#   Copyright (c) 2010, Diaspora Inc.  This file is
+#   licensed under the Affero General Public License version 3 or later.  See
+#   the COPYRIGHT file.
+
+require 'spec_helper'
+
+describe Diaspora::Camo do
+  before do
+    AppConfig.privacy.camo.root = 'http://localhost:3000/camo/'
+    AppConfig.privacy.camo.key = 'kittenpower'
+
+    @raw_image_url = 'http://example.com/kitten.jpg'
+    @camo_image_url = AppConfig.privacy.camo.root + '5bc5b9d7ebd202841ab0667c4fc8d4304278f902/687474703a2f2f6578616d706c652e636f6d2f6b697474656e2e6a7067'
+  end
+
+  describe '#image_url' do
+    it 'should not rewrite local URLs' do
+      local_image = AppConfig.environment.url + 'kitten.jpg'
+      expect(Diaspora::Camo.image_url(local_image)).to eq(local_image)
+    end
+
+    it 'should not rewrite relative URLs' do
+      relative_image = '/kitten.jpg'
+      expect(Diaspora::Camo.image_url(relative_image)).to eq(relative_image)
+    end
+
+    it 'should not rewrite already camo-fied URLs' do
+      camo_image = AppConfig.privacy.camo.root + '1234/56789abcd'
+      expect(Diaspora::Camo.image_url(camo_image)).to eq(camo_image)
+    end
+
+    it 'should rewrite external URLs' do
+      expect(Diaspora::Camo.image_url(@raw_image_url)).to eq(@camo_image_url)
+    end
+  end
+
+  describe '#from_markdown' do
+    it 'should rewrite plain markdown images' do
+      expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url})")).to include(@camo_image_url)
+    end
+
+    it 'should rewrite markdown images with alt texts' do
+      expect(Diaspora::Camo.from_markdown("![a kitten](#{@raw_image_url})")).to include(@camo_image_url)
+    end
+
+    it 'should rewrite markdown images with title texts' do
+      expect(Diaspora::Camo.from_markdown("![](#{@raw_image_url}) \"title\"")).to include(@camo_image_url)
+    end
+  end
+end