diff --git a/Changelog.md b/Changelog.md index 4c49c2585..cdf7f8b7c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -21,6 +21,7 @@ * Enable frozen string literals [#7595](https://github.com/diaspora/diaspora/pull/7595) * Remove `rails_admin_histories` table [#7597](https://github.com/diaspora/diaspora/pull/7597) * Optimize memory usage on profile export [#7627](https://github.com/diaspora/diaspora/pull/7627) +* Limit the number of parallel exports [#7629](https://github.com/diaspora/diaspora/pull/7629) ## Bug fixes * Fix displaying polls with long answers [#7579](https://github.com/diaspora/diaspora/pull/7579) @@ -32,12 +33,16 @@ * Fix invalid data in the database for user data export [#7614](https://github.com/diaspora/diaspora/pull/7614) * Fix local migration run without old private key [#7558](https://github.com/diaspora/diaspora/pull/7558) * Fix export not downloadable because the filename was resetted on access [#7622](https://github.com/diaspora/diaspora/pull/7622) +* Delete invalid oEmbed caches with binary titles [#7620](https://github.com/diaspora/diaspora/pull/7620) ## Features * Ask for confirmation when leaving a submittable comment field [#7530](https://github.com/diaspora/diaspora/pull/7530) * Show users vote in polls [#7550](https://github.com/diaspora/diaspora/pull/7550) * Add explanation of ignore function to in-app help section [#7585](https://github.com/diaspora/diaspora/pull/7585) * Add camo information to NodeInfo [#7617](https://github.com/diaspora/diaspora/pull/7617) +* Add support for `diaspora://` links [#7625](https://github.com/diaspora/diaspora/pull/7625) +* Add support to relay likes for comments [#7625](https://github.com/diaspora/diaspora/pull/7625) +* Implement RFC 7033 WebFinger [#7625](https://github.com/diaspora/diaspora/pull/7625) # 0.7.0.1 diff --git a/Gemfile b/Gemfile index 5d55323e2..8234f47be 100644 --- a/Gemfile +++ b/Gemfile @@ -15,8 +15,8 @@ gem "unicorn-worker-killer", "0.4.4" # Federation -gem "diaspora_federation-json_schema", "0.2.1" -gem "diaspora_federation-rails", "0.2.1" +gem "diaspora_federation-json_schema", "0.2.2" +gem "diaspora_federation-rails", "0.2.2" # API and JSON @@ -292,7 +292,7 @@ group :test do gem "timecop", "0.9.1" gem "webmock", "3.0.1", require: false - gem "diaspora_federation-test", "0.2.1" + gem "diaspora_federation-test", "0.2.2" # Coverage gem "coveralls", "0.8.21", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 7e55f4293..625eff57f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -166,20 +166,20 @@ GEM devise rails (>= 3.0.4) diaspora-prosody-config (0.0.7) - diaspora_federation (0.2.1) - faraday (>= 0.9.0, < 0.13.0) + diaspora_federation (0.2.2) + faraday (>= 0.9.0, < 0.14.0) faraday_middleware (>= 0.10.0, < 0.13.0) nokogiri (~> 1.6, >= 1.6.8) typhoeus (~> 1.0) valid (~> 1.0) - diaspora_federation-json_schema (0.2.1) - diaspora_federation-rails (0.2.1) + diaspora_federation-json_schema (0.2.2) + diaspora_federation-rails (0.2.2) actionpack (>= 4.2, < 6) - diaspora_federation (= 0.2.1) - diaspora_federation-test (0.2.1) - diaspora_federation (= 0.2.1) - fabrication (~> 2.16.0) - uuid (~> 2.3.8) + diaspora_federation (= 0.2.2) + diaspora_federation-test (0.2.2) + diaspora_federation (= 0.2.2) + fabrication (~> 2.16) + uuid (~> 2.3, >= 2.3.8) diff-lcs (1.3) docile (1.1.5) domain_name (0.5.20170404) @@ -783,9 +783,9 @@ DEPENDENCIES devise (= 4.3.0) devise_lastseenable (= 0.0.6) diaspora-prosody-config (= 0.0.7) - diaspora_federation-json_schema (= 0.2.1) - diaspora_federation-rails (= 0.2.1) - diaspora_federation-test (= 0.2.1) + diaspora_federation-json_schema (= 0.2.2) + diaspora_federation-rails (= 0.2.2) + diaspora_federation-test (= 0.2.2) entypo-rails (= 3.0.0) eye (= 0.9.2) factory_girl_rails (= 4.8.0) diff --git a/app/controllers/people_controller.rb b/app/controllers/people_controller.rb index e6123d378..c47ff2ae2 100644 --- a/app/controllers/people_controller.rb +++ b/app/controllers/people_controller.rb @@ -189,7 +189,7 @@ class PeopleController < ApplicationController end def diaspora_id?(query) - !(query.nil? || query.lstrip.empty?) && Validation::Rule::DiasporaId.new.valid_value?(query) + !(query.nil? || query.lstrip.empty?) && Validation::Rule::DiasporaId.new.valid_value?(query.downcase).present? end # view this profile on the home pod, if you don't want to sign in... diff --git a/app/models/comment.rb b/app/models/comment.rb index e165b09ff..a766566fc 100644 --- a/app/models/comment.rb +++ b/app/models/comment.rb @@ -14,6 +14,7 @@ class Comment < ApplicationRecord include Diaspora::Taggable include Diaspora::Likeable include Diaspora::MentionsContainer + include Reference::Source acts_as_taggable_on :tags extract_tags_from :text diff --git a/app/models/message.rb b/app/models/message.rb index 7e1857fe9..e0c738d00 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -5,6 +5,8 @@ class Message < ApplicationRecord include Diaspora::Fields::Guid include Diaspora::Fields::Author + include Reference::Source + belongs_to :conversation, touch: true delegate :name, to: :author, prefix: true diff --git a/app/models/reference.rb b/app/models/reference.rb new file mode 100644 index 000000000..8312ceb61 --- /dev/null +++ b/app/models/reference.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class Reference < ApplicationRecord + belongs_to :source, polymorphic: true + belongs_to :target, polymorphic: true + validates :target_id, uniqueness: {scope: %i[target_type source_id source_type]} + + module Source + extend ActiveSupport::Concern + + included do + after_create :create_references + has_many :references, as: :source, dependent: :destroy + end + + def create_references + text&.scan(DiasporaFederation::Federation::DiasporaUrlParser::DIASPORA_URL_REGEX)&.each do |author, type, guid| + class_name = DiasporaFederation::Entity.entity_class(type).to_s.rpartition("::").last + entity = Diaspora::Federation::Mappings.model_class_for(class_name).find_by(guid: guid) + references.find_or_create_by(target: entity) if entity.diaspora_handle == author + end + end + end + + module Target + extend ActiveSupport::Concern + + included do + has_many :referenced_by, as: :target, class_name: "Reference", dependent: :destroy + end + end +end diff --git a/app/models/status_message.rb b/app/models/status_message.rb index 43ae1697d..a44afca90 100644 --- a/app/models/status_message.rb +++ b/app/models/status_message.rb @@ -7,6 +7,9 @@ class StatusMessage < Post include Diaspora::Taggable + include Reference::Source + include Reference::Target + include PeopleHelper acts_as_taggable_on :tags diff --git a/app/workers/export_user.rb b/app/workers/export_user.rb index b73535d17..247b8a6e3 100644 --- a/app/workers/export_user.rb +++ b/app/workers/export_user.rb @@ -4,12 +4,25 @@ # licensed under the Affero General Public License version 3 or later. See # the COPYRIGHT file. - module Workers class ExportUser < Base sidekiq_options queue: :low + include Diaspora::Logging + def perform(user_id) + if currently_running_exports >= AppConfig.settings.export_concurrency.to_i + logger.info "Already the maximum number of parallel user exports running, " \ + "scheduling export for User:#{user_id} in 5 minutes." + self.class.perform_in(5.minutes + rand(30), user_id) + else + export_user(user_id) + end + end + + private + + def export_user(user_id) @user = User.find(user_id) @user.perform_export! @@ -19,5 +32,13 @@ module Workers ExportMailer.export_failure_for(@user).deliver_now end end + + def currently_running_exports + return 0 if AppConfig.environment.single_process_mode? + Sidekiq::Workers.new.count do |process_id, thread_id, work| + !(Process.pid.to_s == process_id.split(":")[1] && Thread.current.object_id.to_s(36) == thread_id) && + work["payload"]["class"] == self.class.to_s + end + end end end diff --git a/config/defaults.yml b/config/defaults.yml index 322a6f436..221448318 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -111,6 +111,7 @@ defaults: suggest_email: typhoeus_verbose: false typhoeus_concurrency: 20 + export_concurrency: 1 username_blacklist: - 'admin' - 'administrator' diff --git a/config/diaspora.yml.example b/config/diaspora.yml.example index b2573625d..d52378c29 100644 --- a/config/diaspora.yml.example +++ b/config/diaspora.yml.example @@ -455,6 +455,11 @@ configuration: ## Section ## of your Sidekiq workers. #typhoeus_concurrency: 20 + ## Maximum number of parallel user data export jobs (default=1) + ## Be careful, exports of big/old profiles can use a lot of memory, running + ## many of them in parallel can be a problem for small servers. + #export_concurrency: 1 + ## Captcha settings captcha: ## Section diff --git a/config/initializers/diaspora_federation.rb b/config/initializers/diaspora_federation.rb index cc0c9f215..050a70575 100644 --- a/config/initializers/diaspora_federation.rb +++ b/config/initializers/diaspora_federation.rb @@ -7,6 +7,8 @@ DiasporaFederation.configure do |config| config.certificate_authorities = AppConfig.environment.certificate_authorities.get + config.webfinger_http_fallback = Rails.env == "development" + config.http_concurrency = AppConfig.settings.typhoeus_concurrency.to_i config.http_verbose = AppConfig.settings.typhoeus_verbose? diff --git a/db/migrate/20170917163640_cleanup_invalid_o_embed_caches.rb b/db/migrate/20170917163640_cleanup_invalid_o_embed_caches.rb new file mode 100644 index 000000000..89272d2ff --- /dev/null +++ b/db/migrate/20170917163640_cleanup_invalid_o_embed_caches.rb @@ -0,0 +1,12 @@ +class CleanupInvalidOEmbedCaches < ActiveRecord::Migration[5.1] + class OEmbedCache < ApplicationRecord + end + class Post < ApplicationRecord + end + + def up + ids = OEmbedCache.where("data LIKE '%!binary%'").ids + Post.where(o_embed_cache_id: ids).update_all(o_embed_cache_id: nil) # rubocop:disable Rails/SkipsModelValidations + OEmbedCache.where(id: ids).delete_all + end +end diff --git a/db/migrate/20170920214158_create_references_table.rb b/db/migrate/20170920214158_create_references_table.rb new file mode 100644 index 000000000..75ce6c109 --- /dev/null +++ b/db/migrate/20170920214158_create_references_table.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class CreateReferencesTable < ActiveRecord::Migration[5.1] + def change + create_table :references do |t| + t.integer :source_id, null: false + t.string :source_type, limit: 60, null: false + t.integer :target_id, null: false + t.string :target_type, limit: 60, null: false + end + + add_index :references, %i[source_id source_type target_id target_type], + name: :index_references_on_source_and_target, unique: true + add_index :references, %i[source_id source_type], name: :index_references_on_source_id_and_source_type + end +end diff --git a/lib/diaspora/federation/entities.rb b/lib/diaspora/federation/entities.rb index b560263dc..e6dbff2a0 100644 --- a/lib/diaspora/federation/entities.rb +++ b/lib/diaspora/federation/entities.rb @@ -176,13 +176,11 @@ module Diaspora def self.reshare(reshare) DiasporaFederation::Entities::Reshare.new( - root_author: reshare.root_diaspora_id, - root_guid: reshare.root_guid, - author: reshare.diaspora_handle, - guid: reshare.guid, - public: reshare.public, - created_at: reshare.created_at, - provider_display_name: reshare.provider_display_name + root_author: reshare.root_diaspora_id, + root_guid: reshare.root_guid, + author: reshare.diaspora_handle, + guid: reshare.guid, + created_at: reshare.created_at ) end diff --git a/lib/diaspora/federation/receive.rb b/lib/diaspora/federation/receive.rb index 43fc399c2..adddf4000 100644 --- a/lib/diaspora/federation/receive.rb +++ b/lib/diaspora/federation/receive.rb @@ -142,12 +142,10 @@ module Diaspora author = author_of(entity) ignore_existing_guid(Reshare, entity.guid, author) do Reshare.create!( - author: author, - guid: entity.guid, - created_at: entity.created_at, - provider_display_name: entity.provider_display_name, - public: entity.public, - root_guid: entity.root_guid + author: author, + guid: entity.guid, + created_at: entity.created_at, + root_guid: entity.root_guid ) end end @@ -160,10 +158,10 @@ module Diaspora when Person User.find(recipient_id).disconnected_by(object) when Diaspora::Relayable - if object.parent.author.local? - parent_author = object.parent.author.owner + if object.root.author.local? + root_author = object.root.author.owner retraction = Retraction.for(object) - retraction.defer_dispatch(parent_author, false) + retraction.defer_dispatch(root_author, false) retraction.perform else object.destroy! @@ -259,7 +257,7 @@ module Diaspora yield.tap do |relayable| retract_if_author_ignored(relayable) - relayable.signature = build_signature(klass, entity) if relayable.parent.author.local? + relayable.signature = build_signature(klass, entity) if relayable.root.author.local? relayable.save! end end @@ -274,18 +272,18 @@ module Diaspora end private_class_method def self.retract_if_author_ignored(relayable) - parent_author = relayable.parent.author.owner - return unless parent_author && parent_author.ignored_people.include?(relayable.author) + root_author = relayable.root.author.owner + return unless root_author && root_author.ignored_people.include?(relayable.author) retraction = Retraction.for(relayable) - Diaspora::Federation::Dispatcher.build(parent_author, retraction, subscribers: [relayable.author]).dispatch + Diaspora::Federation::Dispatcher.build(root_author, retraction, subscribers: [relayable.author]).dispatch raise Diaspora::Federation::AuthorIgnored end private_class_method def self.relay_relayable(relayable) - parent_author = relayable.parent.author.owner - Diaspora::Federation::Dispatcher.defer_dispatch(parent_author, relayable) if parent_author + root_author = relayable.root.author.owner + Diaspora::Federation::Dispatcher.defer_dispatch(root_author, relayable) if root_author end # check if the object already exists, otherwise save it. diff --git a/lib/diaspora/message_renderer.rb b/lib/diaspora/message_renderer.rb index bf36efde2..38bdf5863 100644 --- a/lib/diaspora/message_renderer.rb +++ b/lib/diaspora/message_renderer.rb @@ -95,6 +95,12 @@ module Diaspora def normalize @message = self.class.normalize(@message) end + + def diaspora_links + @message = @message.gsub(DiasporaFederation::Federation::DiasporaUrlParser::DIASPORA_URL_REGEX) {|match_str| + Regexp.last_match(2) == "post" ? AppConfig.url_to("/posts/#{Regexp.last_match(3)}") : match_str + } + end end DEFAULTS = {mentioned_people: [], @@ -158,6 +164,7 @@ module Diaspora def plain_text opts={} process(opts) { make_mentions_plain_text + diaspora_links squish append_and_truncate } @@ -167,6 +174,7 @@ module Diaspora def plain_text_without_markdown opts={} process(opts) { make_mentions_plain_text + diaspora_links strip_markdown squish append_and_truncate @@ -177,6 +185,7 @@ module Diaspora def plain_text_for_json opts={} process(opts) { normalize + diaspora_links camo_urls if AppConfig.privacy.camo.proxy_markdown_images? } end @@ -186,6 +195,7 @@ module Diaspora process(opts) { escape normalize + diaspora_links render_mentions render_tags squish @@ -198,6 +208,7 @@ module Diaspora process(opts) { process_newlines normalize + diaspora_links camo_urls if AppConfig.privacy.camo.proxy_markdown_images? markdownify render_mentions diff --git a/lib/diaspora/relayable.rb b/lib/diaspora/relayable.rb index b8cb9da91..55e3d8ff4 100644 --- a/lib/diaspora/relayable.rb +++ b/lib/diaspora/relayable.rb @@ -17,9 +17,15 @@ module Diaspora end end + def root + @root ||= parent + @root = @root.parent while @root.is_a?(Relayable) + @root + end + def author_is_not_ignored - unless new_record? && parent.present? && parent.author.local? && - parent.author.owner.ignored_people.include?(author) + unless new_record? && root.present? && root.author.local? && + root.author.owner.ignored_people.include?(author) return end @@ -28,19 +34,19 @@ module Diaspora # @return [Array] def subscribers - if parent.author.local? + if root.author.local? if author.local? - parent.subscribers + root.subscribers else - parent.subscribers.select(&:remote?).reject {|person| person.pod_id == author.pod_id } + root.subscribers.select(&:remote?).reject {|person| person.pod_id == author.pod_id } end else - [parent.author, author] + [root.author, author] end end def sender_for_dispatch - parent.author.owner if parent.author.local? + root.author.owner if root.author.local? end # @abstract diff --git a/lib/schemas/archive-format.json b/lib/schemas/archive-format.json index 11ba60a72..314331506 100644 --- a/lib/schemas/archive-format.json +++ b/lib/schemas/archive-format.json @@ -81,7 +81,8 @@ { "oneOf": [ { "$ref": "#/definitions/remote_subscription/public" }, - { "$ref": "#/definitions/remote_subscription/private" } + { "$ref": "#/definitions/remote_subscription/private" }, + { "$ref": "#/definitions/remote_subscription/reshare" } ] } ] @@ -223,7 +224,7 @@ } }, "required": [ - "entity_data" + "entity_data", "subscribed_pods_uris" ] }, @@ -248,7 +249,29 @@ "public" ] } - } + }, + "required": [ + "entity_data", "subscribed_users_ids" + ] + }, + + "reshare": { + "type": "object", + "properties": { + "entity_type": { + "type": "string", + "pattern": "^reshare$" + }, + "subscribed_pods_uris": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "entity_type", "subscribed_pods_uris" + ] } } } diff --git a/spec/factories.rb b/spec/factories.rb index 0c246e7c1..0c08448d0 100644 --- a/spec/factories.rb +++ b/spec/factories.rb @@ -258,6 +258,11 @@ FactoryGirl.define do end end + factory :reference do + association :source, factory: :status_message + association :target, factory: :status_message + end + factory(:notification, class: Notifications::AlsoCommented) do association :recipient, :factory => :user association :target, :factory => :comment diff --git a/spec/federation_callbacks_spec.rb b/spec/federation_callbacks_spec.rb index ce5be23d5..f67259666 100644 --- a/spec/federation_callbacks_spec.rb +++ b/spec/federation_callbacks_spec.rb @@ -428,7 +428,6 @@ describe "diaspora federation callbacks" do expect(entity.guid).to eq(post.guid) expect(entity.author).to eq(alice.diaspora_handle) - expect(entity.public).to be_truthy end it "does not fetch a private post" do diff --git a/spec/integration/exporter_spec.rb b/spec/integration/exporter_spec.rb index 00110dafa..908a92532 100644 --- a/spec/integration/exporter_spec.rb +++ b/spec/integration/exporter_spec.rb @@ -210,7 +210,6 @@ describe Diaspora::Exporter do "author": user.diaspora_handle, "guid": reshare.guid, "created_at": reshare.created_at.iso8601, - "public": true, "root_author": reshare.root_author.diaspora_handle, "root_guid": reshare.root_guid } diff --git a/spec/lib/diaspora/federation/entities_spec.rb b/spec/lib/diaspora/federation/entities_spec.rb index 69e342539..0b80c1cc8 100644 --- a/spec/lib/diaspora/federation/entities_spec.rb +++ b/spec/lib/diaspora/federation/entities_spec.rb @@ -212,9 +212,7 @@ describe Diaspora::Federation::Entities do expect(federation_entity.guid).to eq(diaspora_entity.guid) expect(federation_entity.root_author).to eq(diaspora_entity.root.author.diaspora_handle) expect(federation_entity.root_guid).to eq(diaspora_entity.root.guid) - expect(federation_entity.public).to be_truthy expect(federation_entity.created_at).to eq(diaspora_entity.created_at) - expect(federation_entity.provider_display_name).to eq(diaspora_entity.provider_display_name) end context "Retraction" do diff --git a/spec/lib/diaspora/federation/receive_spec.rb b/spec/lib/diaspora/federation/receive_spec.rb index 9a7a8ca21..466dd4bfe 100644 --- a/spec/lib/diaspora/federation/receive_spec.rb +++ b/spec/lib/diaspora/federation/receive_spec.rb @@ -74,7 +74,7 @@ describe Diaspora::Federation::Receive do let(:entity) { comment_entity } it_behaves_like "it ignores existing object received twice", Comment - it_behaves_like "it rejects if the parent author ignores the author", Comment + it_behaves_like "it rejects if the root author ignores the author", Comment it_behaves_like "it relays relayables", Comment end @@ -241,8 +241,49 @@ describe Diaspora::Federation::Receive do let(:entity) { like_entity } it_behaves_like "it ignores existing object received twice", Like - it_behaves_like "it rejects if the parent author ignores the author", Like + it_behaves_like "it rejects if the root author ignores the author", Like it_behaves_like "it relays relayables", Like + + context "like for a comment" do + let(:comment) { FactoryGirl.create(:comment, post: post) } + let(:like_entity) { + build_relayable_federation_entity( + :like, + { + author: sender.diaspora_handle, + parent_guid: comment.guid, + parent_type: "Comment", + author_signature: "aa" + }, + "new_property" => "data" + ) + } + + it "attaches the like to the comment" do + Diaspora::Federation::Receive.perform(like_entity) + + like = Like.find_by!(guid: like_entity.guid) + + expect(comment.likes).to include(like) + expect(like.target).to eq(comment) + end + + it "saves the signature data" do + Diaspora::Federation::Receive.perform(like_entity) + + like = Like.find_by!(guid: like_entity.guid) + + expect(like.signature).not_to be_nil + expect(like.signature.author_signature).to eq("aa") + expect(like.signature.additional_data).to eq("new_property" => "data") + expect(like.signature.order).to eq(like_entity.signature_order.map(&:to_s)) + end + + let(:entity) { like_entity } + it_behaves_like "it ignores existing object received twice", Like + it_behaves_like "it rejects if the root author ignores the author", Like + it_behaves_like "it relays relayables", Like + end end describe ".message" do @@ -408,7 +449,7 @@ describe Diaspora::Federation::Receive do let(:entity) { poll_participation_entity } it_behaves_like "it ignores existing object received twice", PollParticipation - it_behaves_like "it rejects if the parent author ignores the author", PollParticipation + it_behaves_like "it rejects if the root author ignores the author", PollParticipation it_behaves_like "it relays relayables", PollParticipation end @@ -584,17 +625,6 @@ describe Diaspora::Federation::Receive do Diaspora::Federation::Receive.perform(status_message_entity) end - - it "finds the correct author if the author is not lowercase" do - status_message_entity = Fabricate(:status_message_entity, author: sender.diaspora_handle.upcase) - - received = Diaspora::Federation::Receive.perform(status_message_entity) - - status_message = StatusMessage.find_by!(guid: status_message_entity.guid) - - expect(received).to eq(status_message) - expect(status_message.author).to eq(sender) - end end context "with poll" do diff --git a/spec/lib/diaspora/message_renderer_spec.rb b/spec/lib/diaspora/message_renderer_spec.rb index 700e3c348..7789e243e 100644 --- a/spec/lib/diaspora/message_renderer_spec.rb +++ b/spec/lib/diaspora/message_renderer_spec.rb @@ -1,11 +1,6 @@ # frozen_string_literal: true describe Diaspora::MessageRenderer do - MESSAGE_NORMALIZTIONS = { - "\u202a#\u200eUSA\u202c" => "#USA", - "ള്‍" => "ള്‍" - } - def message(text, opts={}) Diaspora::MessageRenderer.new(text, opts) end @@ -100,6 +95,20 @@ describe Diaspora::MessageRenderer do end end end + + context "with diaspora:// links" do + it "replaces diaspora:// links with pod-local links" do + target = FactoryGirl.create(:status_message) + expect( + message("Have a look at diaspora://#{target.diaspora_handle}/post/#{target.guid}.").html + ).to match(/Have a look at #{AppConfig.url_to("/posts/#{target.guid}")}./) + end + + it "doesn't touch invalid diaspora:// links" do + text = "You can create diaspora://author/type/guid links!" + expect(message(text).html).to match(/#{text}/) + end + end end describe "#markdownified" do @@ -128,8 +137,11 @@ describe Diaspora::MessageRenderer do end it "normalizes" do - MESSAGE_NORMALIZTIONS.each do |input, output| - expect(message(input).plain_text_for_json).to eq output + { + "\u202a#\u200eUSA\u202c" => "

#USA

\n", + "ള്‍" => "

ള്‍

\n" + }.each do |input, output| + expect(message(input).markdownified).to eq output end end @@ -180,6 +192,25 @@ describe Diaspora::MessageRenderer do entities = '& ß ' ' "' expect(message(entities).markdownified).to eq "

#{entities}

\n" end + + context "with diaspora:// links" do + it "replaces diaspora:// links with pod-local links" do + target1 = FactoryGirl.create(:status_message) + target2 = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target1.diaspora_handle}/post/#{target1.guid}) and " \ + "this one too diaspora://#{target2.diaspora_handle}/post/#{target2.guid}." + + rendered = message(text).markdownified + + expect(rendered).to match(%r{at this post and}) + expect(rendered).to match(/this one too #{AppConfig.url_to("/posts/#{target2.guid}")}./) + end + + it "doesn't touch invalid diaspora:// links" do + text = "You can create diaspora://author/type/guid links!" + expect(message(text).markdownified).to match(/#{text}/) + end + end end end @@ -210,6 +241,25 @@ describe Diaspora::MessageRenderer do expect(msg.plain_text_without_markdown).to eq "@#{alice.diaspora_handle} is cool" end end + + context "with diaspora:// links" do + it "replaces diaspora:// links with pod-local links" do + target1 = FactoryGirl.create(:status_message) + target2 = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target1.diaspora_handle}/post/#{target1.guid}) and " \ + "this one too diaspora://#{target2.diaspora_handle}/post/#{target2.guid}." + + rendered = message(text).plain_text_without_markdown + + expect(rendered).to match(/look at this post \(#{AppConfig.url_to("/posts/#{target1.guid}")}\) and/) + expect(rendered).to match(/this one too #{AppConfig.url_to("/posts/#{target2.guid}")}./) + end + + it "doesn't touch invalid diaspora:// links" do + text = "You can create diaspora://author/type/guid links!" + expect(message(text).plain_text_without_markdown).to match(/#{text}/) + end + end end describe "#urls" do @@ -241,9 +291,31 @@ describe Diaspora::MessageRenderer do describe "#plain_text_for_json" do it "normalizes" do - MESSAGE_NORMALIZTIONS.each do |input, output| + { + "\u202a#\u200eUSA\u202c" => "#USA", + "ള്‍" => "ള്‍" + }.each do |input, output| expect(message(input).plain_text_for_json).to eq output end end + + context "with diaspora:// links" do + it "replaces diaspora:// links with pod-local links" do + target1 = FactoryGirl.create(:status_message) + target2 = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target1.diaspora_handle}/post/#{target1.guid}) and " \ + "this one too diaspora://#{target2.diaspora_handle}/post/#{target2.guid}." + + rendered = message(text).plain_text_for_json + + expect(rendered).to match(/look at \[this post\]\(#{AppConfig.url_to("/posts/#{target1.guid}")}\) and/) + expect(rendered).to match(/this one too #{AppConfig.url_to("/posts/#{target2.guid}")}./) + end + + it "doesn't touch invalid diaspora:// links" do + text = "You can create diaspora://author/type/guid links!" + expect(message(text).plain_text_for_json).to match(/#{text}/) + end + end end end diff --git a/spec/models/comment_spec.rb b/spec/models/comment_spec.rb index 8e84dca2f..9b426b224 100644 --- a/spec/models/comment_spec.rb +++ b/spec/models/comment_spec.rb @@ -10,6 +10,7 @@ describe Comment, type: :model do let(:comment_alice) { alice.comment!(status_bob, "why so formal?") } it_behaves_like "it is mentions container" + it_behaves_like "a reference source" describe "#destroy" do it "should delete a participation" do diff --git a/spec/models/like_spec.rb b/spec/models/like_spec.rb index 2701d88dc..d96121d78 100644 --- a/spec/models/like_spec.rb +++ b/spec/models/like_spec.rb @@ -62,4 +62,17 @@ describe Like, type: :model do let(:remote_object_on_local_parent) { FactoryGirl.create(:like, target: local_parent, author: remote_raphael) } let(:relayable) { Like::Generator.new(alice, status).build } end + + context "like for a comment" do + it_behaves_like "it is relayable" do + let(:local_parent) { local_luke.post(:status_message, text: "hi", to: local_luke.aspects.first) } + let(:remote_parent) { FactoryGirl.create(:status_message, author: remote_raphael) } + let(:comment_on_local_parent) { FactoryGirl.create(:comment, post: local_parent) } + let(:comment_on_remote_parent) { FactoryGirl.create(:comment, post: remote_parent) } + let(:object_on_local_parent) { local_luke.like!(comment_on_local_parent) } + let(:object_on_remote_parent) { local_luke.like!(comment_on_remote_parent) } + let(:remote_object_on_local_parent) { FactoryGirl.create(:like, target: local_parent, author: remote_raphael) } + let(:relayable) { Like::Generator.new(alice, status).build } + end + end end diff --git a/spec/models/message_spec.rb b/spec/models/message_spec.rb index 6127c008f..031df4a9a 100644 --- a/spec/models/message_spec.rb +++ b/spec/models/message_spec.rb @@ -52,4 +52,6 @@ describe Message, type: :model do expect(conf.reload.unread).to eq(1) end end + + it_behaves_like "a reference source" end diff --git a/spec/models/reference_spec.rb b/spec/models/reference_spec.rb new file mode 100644 index 000000000..6d5514e32 --- /dev/null +++ b/spec/models/reference_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +describe Reference, type: :model do + context "validation" do + it "validates a valid reference" do + expect(FactoryGirl.build(:reference)).to be_valid + end + + it "requires a source" do + expect(FactoryGirl.build(:reference, source: nil)).not_to be_valid + end + + it "requires a target" do + expect(FactoryGirl.build(:reference, target: nil)).not_to be_valid + end + + it "disallows to link the same target twice from one source" do + reference = FactoryGirl.create(:reference) + expect(FactoryGirl.build(:reference, source: reference.source, target: reference.target)).not_to be_valid + end + end +end diff --git a/spec/models/status_message_spec.rb b/spec/models/status_message_spec.rb index 8c1165a21..8dfca6f67 100644 --- a/spec/models/status_message_spec.rb +++ b/spec/models/status_message_spec.rb @@ -147,6 +147,9 @@ describe StatusMessage, type: :model do end end + it_behaves_like "a reference source" + it_behaves_like "a reference target" + describe "#nsfw" do it "returns MatchObject (true) if the post contains #nsfw (however capitalised)" do status = FactoryGirl.build(:status_message, text: "This message is #nSFw") diff --git a/spec/shared_behaviors/receiving.rb b/spec/shared_behaviors/receiving.rb index 930d71aea..6a563e237 100644 --- a/spec/shared_behaviors/receiving.rb +++ b/spec/shared_behaviors/receiving.rb @@ -15,7 +15,7 @@ shared_examples_for "it ignores existing object received twice" do |klass| end end -shared_examples_for "it rejects if the parent author ignores the author" do |klass| +shared_examples_for "it rejects if the root author ignores the author" do |klass| it "saves the relayable if the author is not ignored" do Diaspora::Federation::Receive.perform(entity) diff --git a/spec/shared_behaviors/references.rb b/spec/shared_behaviors/references.rb new file mode 100644 index 000000000..61567a652 --- /dev/null +++ b/spec/shared_behaviors/references.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +shared_examples_for "a reference source" do + let!(:source) { FactoryGirl.create(described_class.to_s.underscore.to_sym) } + let!(:reference) { FactoryGirl.create(:reference, source: source) } + + describe "references" do + it "returns the references" do + expect(source.references).to match_array([reference]) + end + + it "destroys the reference when the source is destroyed" do + source.destroy + expect(Reference.where(id: reference.id)).not_to exist + end + end + + describe "#create_references" do + it "creates a reference for every referenced post after create" do + target1 = FactoryGirl.create(:status_message) + target2 = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target1.diaspora_handle}/post/#{target1.guid}) and " \ + "this one too diaspora://#{target2.diaspora_handle}/post/#{target2.guid}." + + post = FactoryGirl.build(described_class.to_s.underscore.to_sym, text: text) + post.save + + expect(post.references.map(&:target).map(&:guid)).to match_array([target1, target2].map(&:guid)) + end + + it "only creates one reference, even when it is referenced twice" do + target = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target.diaspora_handle}/post/#{target.guid}) and " \ + "this one too diaspora://#{target.diaspora_handle}/post/#{target.guid}." + + post = FactoryGirl.build(described_class.to_s.underscore.to_sym, text: text) + post.save + + expect(post.references.map(&:target).map(&:guid)).to match_array([target.guid]) + end + + it "only creates references, when the author of the known entity matches" do + target1 = FactoryGirl.create(:status_message) + target2 = FactoryGirl.create(:status_message) + text = "Have a look at [this post](diaspora://#{target1.diaspora_handle}/post/#{target1.guid}) and " \ + "this one too diaspora://#{target1.diaspora_handle}/post/#{target2.guid}." + + post = FactoryGirl.build(described_class.to_s.underscore.to_sym, text: text) + post.save + + expect(post.references.map(&:target).map(&:guid)).to match_array([target1.guid]) + end + end +end + +shared_examples_for "a reference target" do + let!(:target) { FactoryGirl.create(described_class.to_s.underscore.to_sym) } + let!(:reference) { FactoryGirl.create(:reference, target: target) } + + describe "referenced_by" do + it "returns the references where the target is referenced" do + expect(target.referenced_by).to match_array([reference]) + end + + it "destroys the reference when the target is destroyed" do + target.destroy + expect(Reference.where(id: reference.id)).not_to exist + end + end +end diff --git a/spec/workers/export_user_spec.rb b/spec/workers/export_user_spec.rb index 72b0c341f..e72410149 100644 --- a/spec/workers/export_user_spec.rb +++ b/spec/workers/export_user_spec.rb @@ -22,4 +22,68 @@ describe Workers::ExportUser do expect(ExportMailer).to receive(:export_failure_for).with(alice).and_call_original Workers::ExportUser.new.perform(alice.id) end + + context "concurrency" do + before do + AppConfig.environment.single_process_mode = false + AppConfig.settings.export_concurrency = 1 + end + + after :all do + AppConfig.environment.single_process_mode = true + end + + let(:pid) { "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(6)}" } + + it "schedules a job for later when already another parallel export job is running" do + expect(Sidekiq::Workers).to receive(:new).and_return( + [[pid, SecureRandom.hex(4), {"payload" => {"class" => "Workers::ExportUser"}}]] + ) + + expect(Workers::ExportUser).to receive(:perform_in).with(kind_of(Integer), alice.id) + expect(alice).not_to receive(:perform_export!) + + Workers::ExportUser.new.perform(alice.id) + end + + it "runs the export when the own running job" do + expect(Sidekiq::Workers).to receive(:new).and_return( + [[pid, Thread.current.object_id.to_s(36), {"payload" => {"class" => "Workers::ExportUser"}}]] + ) + + expect(Workers::ExportUser).not_to receive(:perform_in).with(kind_of(Integer), alice.id) + expect(alice).to receive(:perform_export!) + + Workers::ExportUser.new.perform(alice.id) + end + + it "runs the export when no other job is running" do + expect(Sidekiq::Workers).to receive(:new).and_return([]) + + expect(Workers::ExportUser).not_to receive(:perform_in).with(kind_of(Integer), alice.id) + expect(alice).to receive(:perform_export!) + + Workers::ExportUser.new.perform(alice.id) + end + + it "runs the export when some other job is running" do + expect(Sidekiq::Workers).to receive(:new).and_return( + [[pid, SecureRandom.hex(4), {"payload" => {"class" => "Workers::OtherJob"}}]] + ) + + expect(Workers::ExportUser).not_to receive(:perform_in).with(kind_of(Integer), alice.id) + expect(alice).to receive(:perform_export!) + + Workers::ExportUser.new.perform(alice.id) + end + + it "runs the export when diaspora is in single process mode" do + AppConfig.environment.single_process_mode = true + expect(Sidekiq::Workers).not_to receive(:new) + expect(Workers::ExportUser).not_to receive(:perform_in).with(kind_of(Integer), alice.id) + expect(alice).to receive(:perform_export!) + + Workers::ExportUser.new.perform(alice.id) + end + end end