Merge pull request #78 from SuperTux88/fetch-linked-posts

Fetch posts linked with diaspora:// URLs

docs/_includes/federation_tree.html

@@ -1,9 +1,10 @@
 <ul class="{{ include.ulclasses | default: "nav nav-pills nav-stacked" }}">
   <li class="{% include active_class.html url="/federation/types.html" %}"><a href="{{ site.baseurl }}/federation/types.html">Value Types</a></li>
   <li class="{% include active_class.html url="/federation/magicsig.html" %}"><a href="{{ site.baseurl }}/federation/magicsig.html">Magic Signatures</a></li>
+  <li class="{% include active_class.html url="/federation/xml_serialization.html" %}"><a href="{{ site.baseurl }}/federation/xml_serialization.html">XML Serialization</a></li>
   <li class="{% include active_class.html url="/federation/encryption.html" %}"><a href="{{ site.baseurl }}/federation/encryption.html">Encryption</a></li>
   <li class="{% include active_class.html url="/federation/routes.html" %}"><a href="{{ site.baseurl }}/federation/routes.html">Routes</a></li>
   <li class="{% include active_class.html url="/federation/fetching.html" %}"><a href="{{ site.baseurl }}/federation/fetching.html">Fetching</a></li>
-  <li class="{% include active_class.html url="/federation/xml_serialization.html" %}"><a href="{{ site.baseurl }}/federation/xml_serialization.html">XML Serialization</a></li>
+  <li class="{% include active_class.html url="/federation/diaspora_scheme.html" %}"><a href="{{ site.baseurl }}/federation/diaspora_scheme.html">diaspora:// URI scheme</a></li>
   <li class="{% include active_class.html url="/federation/relayable.html" %}"><a href="{{ site.baseurl }}/federation/relayable.html">Relayable</a></li>
 </ul>

docs/federation/diaspora_scheme.md

@@ -0,0 +1,32 @@
+---
+title: diaspora:// URI scheme
+---
+
+## Server and software independent links
+
+A `diaspora://` URL is used if a user wants to link to another post. It doesn't
+contain a server hostname so it is independent of the senders server. And it
+isn't software specific, it is thought to be compatible with every software
+that is compatible with the protocol, so the receiving software can display
+it as software specific URL. 
+
+The format is similar to the route used for [fetching][fetching], so if the
+receiving server doesn't know the linked entity yet, it can just be fetched.
+
+### Format
+
+`diaspora://:type/:guid`
+
+#### Parameters
+
+| Name   | Description                                    |
+| ------ | ---------------------------------------------- |
+| `type` | The type of the linked entity in `snake_case`. |
+| `guid` | The [GUID][guid] of the linked entity.         |
+
+#### Example
+
+`diaspora://post/17faf230675101350d995254001bd39e`
+
+[fetching]: {{ site.baseurl }}/federation/fetching.html
+[guid]: {{ site.baseurl }}/federation/types.html#guid

docs/federation/types.md

@@ -31,6 +31,8 @@ A network-wide, unique identifier. A random string of at least 16 and at most 25
 * Numbers: `0-9`
 * Special chars: `-`, `_`, `@`, `.` and `:`
 
+ Special chars aren't allowed at the end.
+
 Example: `298962a0b8dc0133e40d406c8f31e210`
 
 ## String
@@ -55,7 +57,9 @@ Example: `12.3456`
 
 Text formatted with markdown using the [CommonMark spec][commonmark].
 
-Example: `Some *Text* with **markdown**.`
+It can also contain [diaspora:// URLs][diaspora_scheme].
+
+Example: `Some *Text* with **markdown** and a [link](diaspora://post/298962a0b8dc0133e40d406c8f31e210).`
 
 ## URL
 

lib/diaspora_federation/entities/event.rb

@@ -24,7 +24,7 @@ module DiasporaFederation
       # @!attribute [r] description
       #   Description of the event
       #   @return [String] event description
-      property :description, :string, optional: true
+      property :description, :string, alias: :text, optional: true
 
       # @!attribute [r] start
       #   The start time of the event

lib/diaspora_federation/entities/profile.rb

@@ -43,9 +43,12 @@ module DiasporaFederation
       #   @return [String] url to the small avatar (50x50)
       property :image_url_small, :string, optional: true
 
+      # @!attribute [r] bio
+      #   @return [String] bio of the person
+      property :bio, :string, alias: :text, optional: true
+
       property :birthday, :string, optional: true
       property :gender, :string, optional: true
-      property :bio, :string, optional: true
       property :location, :string, optional: true
 
       # @!attribute [r] searchable

lib/diaspora_federation/entity.rb

@@ -40,6 +40,9 @@ module DiasporaFederation
     # @see https://www.w3.org/TR/REC-xml/#charsets "Extensible Markup Language (XML) 1.0"
     INVALID_XML_REGEX = /[^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}]/
 
+    # Regex to validate and find entity names
+    ENTITY_NAME_REGEX = "[a-z]*(?:_[a-z]*)*".freeze
+
     # Initializes the Entity with the given attribute hash and freezes the created
     # instance it returns.
     #
@@ -145,7 +148,7 @@ module DiasporaFederation
     # @param [String] entity_name "snake_case" class name
     # @return [Class] entity class
     def self.entity_class(entity_name)
-      raise InvalidEntityName, "'#{entity_name}' is invalid" unless entity_name =~ /\A[a-z]*(_[a-z]*)*\z/
+      raise InvalidEntityName, "'#{entity_name}' is invalid" unless entity_name =~ /\A#{ENTITY_NAME_REGEX}\z/
       class_name = entity_name.sub(/\A[a-z]/, &:upcase)
       class_name.gsub!(/_([a-z])/) { Regexp.last_match[1].upcase }
 

lib/diaspora_federation/federation.rb

@@ -4,6 +4,7 @@ module DiasporaFederation
   end
 end
 
+require "diaspora_federation/federation/diaspora_url_parser"
 require "diaspora_federation/federation/fetcher"
 require "diaspora_federation/federation/receiver"
 require "diaspora_federation/federation/sender"

lib/diaspora_federation/federation/diaspora_url_parser.rb

@@ -0,0 +1,29 @@
+module DiasporaFederation
+  module Federation
+    # This module is for parsing and fetching linked entities.
+    module DiasporaUrlParser
+      include Logging
+
+      # Regex to find diaspora:// URLs
+      DIASPORA_URL_REGEX = %r{diaspora://(#{Entity::ENTITY_NAME_REGEX})/(#{Validation::Rule::Guid::VALID_CHARS})}
+
+      # Parses all diaspora:// URLs from the text and fetches the entities from
+      # the remote server if needed.
+      # @param [String] sender the diaspora* ID of the sender of the entity
+      # @param [String] text text with diaspora:// URLs to fetch
+      def self.fetch_linked_entities(sender, text)
+        text.scan(DIASPORA_URL_REGEX).each do |type, guid|
+          fetch_entity(sender, type, guid)
+        end
+      end
+
+      private_class_method def self.fetch_entity(sender, type, guid)
+        class_name = Entity.entity_class(type).to_s.rpartition("::").last
+        return if DiasporaFederation.callbacks.trigger(:fetch_related_entity, class_name, guid)
+        Fetcher.fetch_public(sender, type, guid)
+      rescue => e
+        logger.error "Failed to fetch linked entity #{type}:#{guid}: #{e.class}: #{e.message}"
+      end
+    end
+  end
+end

lib/diaspora_federation/federation/fetcher.rb

@@ -21,7 +21,7 @@ module DiasporaFederation
       end
 
       private_class_method def self.entity_name(class_name)
-        return class_name if class_name =~ /\A[a-z]*(_[a-z]*)*\z/
+        return class_name if class_name =~ /\A#{Entity::ENTITY_NAME_REGEX}\z/
 
         raise DiasporaFederation::Entity::UnknownEntity, class_name unless Entities.const_defined?(class_name)
 

lib/diaspora_federation/federation/receiver/abstract_receiver.rb

@@ -30,6 +30,7 @@ module DiasporaFederation
           validate
           DiasporaFederation.callbacks.trigger(:receive_entity, entity, sender, recipient_id)
           logger.info "successfully received #{entity} from person #{sender}#{" for #{recipient_id}" if recipient_id}"
+          fetch_linked_entities_from_text
         end
 
         def validate
@@ -43,6 +44,10 @@ module DiasporaFederation
             sender == entity.author
           end
         end
+
+        def fetch_linked_entities_from_text
+          DiasporaUrlParser.fetch_linked_entities(sender, entity.text) if entity.respond_to?(:text) && entity.text
+        end
       end
     end
   end

lib/diaspora_federation/validators/rules/guid.rb

@@ -6,7 +6,11 @@ module Validation
     # * Letters: a-z
     # * Numbers: 0-9
     # * Special chars: '-', '_', '@', '.' and ':'
+    # Special chars aren't allowed at the end.
     class Guid
+      # Allowed chars to validate a GUID with a regex
+      VALID_CHARS = "[0-9A-Za-z\\-_@.:]{15,254}[0-9a-z]".freeze
+
       # The error key for this rule
       # @return [Symbol] error key
       def error_key
@@ -15,7 +19,7 @@ module Validation
 
       # Determines if value is a valid +GUID+
       def valid_value?(value)
-        value.is_a?(String) && value.downcase =~ /\A[0-9a-z\-_@.:]{16,255}\z/
+        value.is_a?(String) && value =~ /\A#{VALID_CHARS}\z/
       end
 
       # This rule has no params.

spec/lib/diaspora_federation/entities/account_migration_spec.rb

@@ -24,9 +24,9 @@ module DiasporaFederation
     <image_url>#{data[:profile].image_url}</image_url>
     <image_url_medium>#{data[:profile].image_url}</image_url_medium>
     <image_url_small>#{data[:profile].image_url}</image_url_small>
+    <bio>#{data[:profile].bio}</bio>
     <birthday>#{data[:profile].birthday}</birthday>
     <gender>#{data[:profile].gender}</gender>
-    <bio>#{data[:profile].bio}</bio>
     <location>#{data[:profile].location}</location>
     <searchable>#{data[:profile].searchable}</searchable>
     <public>#{data[:profile].public}</public>

spec/lib/diaspora_federation/entities/person_spec.rb

@@ -13,9 +13,9 @@ module DiasporaFederation
     <image_url>#{data[:profile].image_url}</image_url>
     <image_url_medium>#{data[:profile].image_url}</image_url_medium>
     <image_url_small>#{data[:profile].image_url}</image_url_small>
+    <bio>#{data[:profile].bio}</bio>
     <birthday>#{data[:profile].birthday}</birthday>
     <gender>#{data[:profile].gender}</gender>
-    <bio>#{data[:profile].bio}</bio>
     <location>#{data[:profile].location}</location>
     <searchable>#{data[:profile].searchable}</searchable>
     <public>#{data[:profile].public}</public>

spec/lib/diaspora_federation/entities/profile_spec.rb

@@ -9,9 +9,9 @@ module DiasporaFederation
   <image_url>#{data[:image_url]}</image_url>
   <image_url_medium>#{data[:image_url]}</image_url_medium>
   <image_url_small>#{data[:image_url]}</image_url_small>
+  <bio>#{data[:bio]}</bio>
   <birthday>#{data[:birthday]}</birthday>
   <gender>#{data[:gender]}</gender>
-  <bio>#{data[:bio]}</bio>
   <location>#{data[:location]}</location>
   <searchable>#{data[:searchable]}</searchable>
   <public>#{data[:public]}</public>
@@ -29,9 +29,9 @@ XML
     "image_url": "#{data[:image_url]}",
     "image_url_medium": "#{data[:image_url]}",
     "image_url_small": "#{data[:image_url]}",
+    "bio": "#{data[:bio]}",
     "birthday": "#{data[:birthday]}",
     "gender": "#{data[:gender]}",
-    "bio": "#{data[:bio]}",
     "location": "#{data[:location]}",
     "searchable": #{data[:searchable]},
     "public": #{data[:public]},

spec/lib/diaspora_federation/federation/diaspora_url_parser_spec.rb

@@ -0,0 +1,68 @@
+module DiasporaFederation
+  describe Federation::DiasporaUrlParser do
+    let(:sender) { Fabricate.sequence(:diaspora_id) }
+    let(:guid) { Fabricate.sequence(:guid) }
+
+    describe ".fetch_linked_entities" do
+      it "parses linked posts from the text" do
+        guid2 = Fabricate.sequence(:guid)
+        guid3 = Fabricate.sequence(:guid)
+        expect_callback(:fetch_related_entity, "Post", guid).and_return(double)
+        expect_callback(:fetch_related_entity, "Post", guid2).and_return(double)
+        expect_callback(:fetch_related_entity, "Post", guid3).and_return(double)
+
+        text = "This is a [link to a post with markdown](diaspora://post/#{guid}) and one without " \
+               "diaspora://post/#{guid2} and finally a last one diaspora://post/#{guid3}."
+
+        Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+      end
+
+      it "ignores invalid diaspora:// urls" do
+        expect(DiasporaFederation.callbacks).not_to receive(:trigger)
+
+        text = "This is an invalid link diaspora://Post/#{guid}) and another one: " \
+               "diaspora://post/abcd."
+
+        Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+      end
+
+      it "allows to link other entities" do
+        expect_callback(:fetch_related_entity, "Event", guid).and_return(double)
+
+        text = "This is a link to an event diaspora://event/#{guid}."
+
+        Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+      end
+
+      it "handles unknown entities gracefully" do
+        expect(DiasporaFederation.callbacks).not_to receive(:trigger)
+
+        text = "This is a link to an event diaspora://unknown/#{guid}."
+
+        Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+      end
+
+      it "fetches entities from sender when not found locally" do
+        expect_callback(:fetch_related_entity, "Post", guid).and_return(nil)
+        expect(Federation::Fetcher).to receive(:fetch_public).with(sender, "post", guid)
+
+        text = "This is a link to a post: diaspora://post/#{guid}."
+
+        Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+      end
+
+      it "handles fetch errors gracefully" do
+        expect_callback(:fetch_related_entity, "Post", guid).and_return(nil)
+        expect(Federation::Fetcher).to receive(:fetch_public).with(
+          sender, "post", guid
+        ).and_raise(Federation::Fetcher::NotFetchable, "Something went wrong!")
+
+        text = "This is a link to a post: diaspora://post/#{guid}."
+
+        expect {
+          Federation::DiasporaUrlParser.fetch_linked_entities(sender, text)
+        }.not_to raise_error
+      end
+    end
+  end
+end

spec/lib/diaspora_federation/federation/receiver/private_spec.rb

@@ -112,6 +112,36 @@ module DiasporaFederation
           end
         end
       end
+
+      context "with text" do
+        before do
+          expect(DiasporaFederation.callbacks).to receive(:trigger)
+        end
+
+        it "fetches linked entities when the received entity has a text property" do
+          expect(Federation::DiasporaUrlParser).to receive(:fetch_linked_entities).with(post.author, post.text)
+
+          described_class.new(magic_env, recipient).receive
+        end
+
+        it "fetches linked entities for the profile bio" do
+          profile = Fabricate(:profile_entity)
+          magic_env = Salmon::MagicEnvelope.new(profile, profile.author)
+
+          expect(Federation::DiasporaUrlParser).to receive(:fetch_linked_entities).with(profile.author, profile.bio)
+
+          described_class.new(magic_env, recipient).receive
+        end
+
+        it "doesn't try to fetch linked entities when the text is nil" do
+          photo = Fabricate(:photo_entity, public: false, text: nil)
+          magic_env = Salmon::MagicEnvelope.new(photo, photo.author)
+
+          expect(Federation::DiasporaUrlParser).not_to receive(:fetch_linked_entities)
+
+          described_class.new(magic_env, recipient).receive
+        end
+      end
     end
   end
 end

spec/lib/diaspora_federation/federation/receiver/public_spec.rb

@@ -134,6 +134,36 @@ module DiasporaFederation
           described_class.new(magic_env).receive
         end
       end
+
+      context "with text" do
+        before do
+          expect(DiasporaFederation.callbacks).to receive(:trigger)
+        end
+
+        it "fetches linked entities when the received entity has a text property" do
+          expect(Federation::DiasporaUrlParser).to receive(:fetch_linked_entities).with(post.author, post.text)
+
+          described_class.new(magic_env).receive
+        end
+
+        it "fetches linked entities for the profile bio" do
+          profile = Fabricate(:profile_entity, public: true)
+          magic_env = Salmon::MagicEnvelope.new(profile, profile.author)
+
+          expect(Federation::DiasporaUrlParser).to receive(:fetch_linked_entities).with(profile.author, profile.bio)
+
+          described_class.new(magic_env).receive
+        end
+
+        it "doesn't try to fetch linked entities when the text is nil" do
+          photo = Fabricate(:photo_entity, text: nil)
+          magic_env = Salmon::MagicEnvelope.new(photo, photo.author)
+
+          expect(Federation::DiasporaUrlParser).not_to receive(:fetch_linked_entities)
+
+          described_class.new(magic_env).receive
+        end
+      end
     end
   end
 end

spec/lib/diaspora_federation/validators/rules/guid_spec.rb

@@ -45,6 +45,14 @@ describe Validation::Rule::Guid do
       expect(validator.errors).to include(:guid)
     end
 
+    it "fails if the string contains special chars at the end" do
+      validator = Validation::Validator.new(OpenStruct.new(guid: "abcdef0123456789."))
+      validator.rule(:guid, :guid)
+
+      expect(validator).not_to be_valid
+      expect(validator.errors).to include(:guid)
+    end
+
     it "fails if the string contains invalid chars" do
       validator = Validation::Validator.new(OpenStruct.new(guid: "ghijklmnopqrstuvwxyz++"))
       validator.rule(:guid, :guid)