From dda5f71124ce17929291da92f7fe18d4b1545da9 Mon Sep 17 00:00:00 2001 From: Dumitru Ursu Date: Tue, 6 Jan 2015 14:45:46 +0200 Subject: [PATCH] Add warning about encoding change Add warning about MySQL collation Fix database index length This allows new databases to be created with utf8mb4, on MySQL. The maximum column size is 767 bytes. Each character is 4 bytes long -> 767 / 4 = 191 characters for the column. --- Changelog.md | 4 ++++ config/database.yml.example | 3 +-- db/migrate/0000_create_schema.rb | 12 +++++----- .../20130429073928_add_tweet_id_to_post.rb | 2 +- ...50_remove_limit_from_root_guid_in_posts.rb | 4 ++-- .../20140801101230_create_chat_contacts.rb | 13 +++++----- ...20150106050733_set_mysql_to_unicode_mb4.rb | 24 ++++++++++--------- db/schema.rb | 7 +++--- lib/configuration_methods.rb | 7 ++++-- 9 files changed, 42 insertions(+), 34 deletions(-) diff --git a/Changelog.md b/Changelog.md index 07f354cfc..6f8007db6 100644 --- a/Changelog.md +++ b/Changelog.md @@ -32,6 +32,10 @@ series and run our comprehensive test suite against it. ## Change in defaults.yml The default for including jQuery from a CDN has changed. If you want to continue to include it from a CDN, please explicitly set the `jquery_cdn` setting to `true` in diaspora.yml. +## Change in database.yml +For MySQL databases, replace `charset: utf8` with `encoding: utf8mb4`. This is enables full UTF8 support (4bytes characters), including standard emoji characters. See `database.yml.example` for reference. +Also, do not forget to remove `collation: utf8_bin`. It will choose a compatible one automatically. + ## Experimental chat feature This release adds experimental integration with XMPP for real-time chat. Please see [our wiki](https://wiki.diasporafoundation.org/Vines) for further informations. diff --git a/config/database.yml.example b/config/database.yml.example index 38f15f3c8..e62f7a9a6 100644 --- a/config/database.yml.example +++ b/config/database.yml.example @@ -5,8 +5,7 @@ mysql: &mysql username: "root" password: "" # socket: /tmp/mysql.sock - charset: utf8mb4 - collation: utf8_bin + encoding: utf8mb4 postgres: &postgres adapter: postgresql diff --git a/db/migrate/0000_create_schema.rb b/db/migrate/0000_create_schema.rb index c9fe48356..92cc18d27 100644 --- a/db/migrate/0000_create_schema.rb +++ b/db/migrate/0000_create_schema.rb @@ -199,7 +199,7 @@ class CreateSchema < ActiveRecord::Migration t.text "data", :null => false end - add_index "o_embed_caches", ["url"], :name => "index_o_embed_caches_on_url", :length => {"url"=>255} + add_index "o_embed_caches", ["url"], :name => "index_o_embed_caches_on_url", :length => {"url"=> 191}, using: :btree create_table "participations", :force => true do |t| t.string "guid" @@ -227,7 +227,7 @@ class CreateSchema < ActiveRecord::Migration t.integer "fetch_status", :default => 0 end - add_index "people", ["diaspora_handle"], :name => "index_people_on_diaspora_handle", :unique => true + add_index "people", ["diaspora_handle"], :name => "index_people_on_diaspora_handle", :unique => true, :length => {"diaspora_handle" => 191} add_index "people", ["guid"], :name => "index_people_on_guid", :unique => true add_index "people", ["owner_id"], :name => "index_people_on_owner_id", :unique => true @@ -265,7 +265,7 @@ class CreateSchema < ActiveRecord::Migration t.integer "author_id", :null => false t.boolean "public", :default => false, :null => false t.string "diaspora_handle" - t.string "guid", :null => false + t.string "guid" t.boolean "pending", :default => false, :null => false t.string "type", :limit => 40, :null => false t.text "text" @@ -337,7 +337,7 @@ class CreateSchema < ActiveRecord::Migration t.datetime "updated_at", :null => false end - add_index "rails_admin_histories", ["item", "table", "month", "year"], :name => "index_rails_admin_histories" + add_index "rails_admin_histories", ["item", "table", "month", "year"], :name => "index_rails_admin_histories", :length => {"table" => 180} create_table "roles", :force => true do |t| t.integer "person_id" @@ -404,7 +404,7 @@ class CreateSchema < ActiveRecord::Migration t.string "name" end - add_index "tags", ["name"], :name => "index_tags_on_name", :unique => true + add_index "tags", ["name"], :name => "index_tags_on_name", :unique => true, :length => {"name" => 191} create_table "user_preferences", :force => true do |t| t.string "email_type" @@ -449,7 +449,7 @@ class CreateSchema < ActiveRecord::Migration end add_index "users", ["authentication_token"], :name => "index_users_on_authentication_token", :unique => true - add_index "users", ["email"], :name => "index_users_on_email" + add_index "users", ["email"], :name => "index_users_on_email", length: {"email" => "191"} add_index "users", ["invitation_service", "invitation_identifier"], :name => "index_users_on_invitation_service_and_invitation_identifier", :unique => true add_index "users", ["invitation_token"], :name => "index_users_on_invitation_token" add_index "users", ["username"], :name => "index_users_on_username", :unique => true diff --git a/db/migrate/20130429073928_add_tweet_id_to_post.rb b/db/migrate/20130429073928_add_tweet_id_to_post.rb index 3d42da98c..a01869d42 100644 --- a/db/migrate/20130429073928_add_tweet_id_to_post.rb +++ b/db/migrate/20130429073928_add_tweet_id_to_post.rb @@ -1,6 +1,6 @@ class AddTweetIdToPost < ActiveRecord::Migration def change - add_column :posts, :tweet_id, :string + add_column :posts, :tweet_id, :string, limit: 64 add_index :posts, :tweet_id end end diff --git a/db/migrate/20130613203350_remove_limit_from_root_guid_in_posts.rb b/db/migrate/20130613203350_remove_limit_from_root_guid_in_posts.rb index 8b2fe2b37..612e927a6 100644 --- a/db/migrate/20130613203350_remove_limit_from_root_guid_in_posts.rb +++ b/db/migrate/20130613203350_remove_limit_from_root_guid_in_posts.rb @@ -1,9 +1,9 @@ class RemoveLimitFromRootGuidInPosts < ActiveRecord::Migration def up - change_column :posts, :root_guid, :string + change_column :posts, :root_guid, :string, limit: 64 end def down - change_column :posts, :root_guid, :string, limit: 30 + change_column :posts, :root_guid, :string, limit: 64 end end diff --git a/db/migrate/20140801101230_create_chat_contacts.rb b/db/migrate/20140801101230_create_chat_contacts.rb index 346ad9c20..42a01b68c 100644 --- a/db/migrate/20140801101230_create_chat_contacts.rb +++ b/db/migrate/20140801101230_create_chat_contacts.rb @@ -1,13 +1,14 @@ class CreateChatContacts < ActiveRecord::Migration def up create_table :chat_contacts do |t| - t.integer :user_id, null: false - t.string :jid, null: false - t.string :name, limit: 255, null: true - t.string :ask, limit: 128, null: true - t.string :subscription, limit: 128, null: false + t.integer :user_id, null: false + ## JID <= 3071 bytes http://tools.ietf.org/html/rfc6122 + t.string :jid, limit: 3071, null: false + t.string :name, limit: 255, null: true + t.string :ask, limit: 128, null: true + t.string :subscription, limit: 128, null: false end - add_index :chat_contacts, [:user_id, :jid], unique: true + add_index :chat_contacts, [:user_id, :jid], unique: true, length: {"jid": 100} end def down diff --git a/db/migrate/20150106050733_set_mysql_to_unicode_mb4.rb b/db/migrate/20150106050733_set_mysql_to_unicode_mb4.rb index a4abd5b36..4f732a86d 100644 --- a/db/migrate/20150106050733_set_mysql_to_unicode_mb4.rb +++ b/db/migrate/20150106050733_set_mysql_to_unicode_mb4.rb @@ -1,18 +1,20 @@ class SetMysqlToUnicodeMb4 < ActiveRecord::Migration - UTF8_PAIRS = { - 'comments': 'text', - 'messages': 'text', - 'poll_answers': 'answer', - 'polls': 'question', - 'posts': 'text', - } + UTF8_PAIRS = {} + tables = ActiveRecord::Base.connection.tables + + tables.each do |table| + ActiveRecord::Base.connection.columns(table).each do |column| + # build a hash with all the columns that contain text + UTF8_PAIRS[table] = column.name if (column.type == :string) || (column.type == :text) + end + end def self.up - if ENV['DB'] == 'mysql' + if AppConfig.mysql? execute "ALTER DATABASE `#{ActiveRecord::Base.connection.current_database}` CHARACTER SET utf8mb4;" - ActiveRecord::Base.connection.tables.each do |table| + tables.each do |table| execute "ALTER TABLE `#{table}` CHARACTER SET = utf8mb4;" end @@ -23,10 +25,10 @@ class SetMysqlToUnicodeMb4 < ActiveRecord::Migration end def self.down - if ENV['DB'] == 'mysql' + if AppConfig.mysql? execute "ALTER DATABASE `#{ActiveRecord::Base.connection.current_database}` CHARACTER SET utf8;" - ActiveRecord::Base.connection.tables.each do |table| + tables.each do |table| execute "ALTER TABLE `#{table}` CHARACTER SET = utf8;" end diff --git a/db/schema.rb b/db/schema.rb index 0521d0fa3..47e3c00cf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150106050733) do +ActiveRecord::Schema.define(version: 20150106050734) do create_table "account_deletions", force: true do |t| t.string "diaspora_handle" @@ -66,7 +66,6 @@ ActiveRecord::Schema.define(version: 20150106050733) do t.string "name" t.string "ask", limit: 128 t.string "subscription", limit: 128, null: false - t.text "groups" end add_index "chat_contacts", ["user_id", "jid"], name: "index_chat_contacts_on_user_id_and_jid", unique: true, using: :btree @@ -242,7 +241,7 @@ ActiveRecord::Schema.define(version: 20150106050733) do t.text "data", null: false end - add_index "o_embed_caches", ["url"], name: "index_o_embed_caches_on_url", using: :btree + add_index "o_embed_caches", ["url"], name: "index_o_embed_caches_on_url", length: {"url"=>255}, using: :btree create_table "open_graph_caches", force: true do |t| t.string "title" @@ -430,11 +429,11 @@ ActiveRecord::Schema.define(version: 20150106050733) do create_table "reports", force: true do |t| t.integer "item_id", null: false + t.string "item_type", null: false t.boolean "reviewed", default: false t.text "text" t.datetime "created_at" t.datetime "updated_at" - t.string "item_type", null: false t.integer "user_id", null: false end diff --git a/lib/configuration_methods.rb b/lib/configuration_methods.rb index 9c302b794..8f471319b 100644 --- a/lib/configuration_methods.rb +++ b/lib/configuration_methods.rb @@ -119,8 +119,11 @@ module Configuration end def postgres? - defined?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) && - ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) + ActiveRecord::Base.connection.adapter_name == "PostgreSQL" + end + + def mysql? + ActiveRecord::Base.connection.adapter_name == "Mysql2" end def bitcoin_donation_address