Change hashtags to preserve first-used casing (mastodon#11416)

Author: Gargron

app/lib/activitypub/activity/create.rb

@@ -148,12 +148,9 @@ def process_tags
   def process_hashtag(tag)
     return if tag['name'].blank?
 
-    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
-
-    return if @tags.include?(hashtag)
-
-    @tags << hashtag
+    Tag.find_or_create_by_names(tag['name']) do |hashtag|
+      @tags << hashtag unless @tags.include?(hashtag)
+    end
   rescue ActiveRecord::RecordInvalid
     nil
   end

app/models/tag.rb

@@ -20,7 +20,7 @@ class Tag < ApplicationRecord
   HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
@@ -64,22 +64,48 @@ def history
   end
 
   class << self
+    def find_or_create_by_names(name_or_names)
+      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+        tag = matching_name(normalized_name).first || create(name: normalized_name)
+
+        yield tag if block_given?
+
+        tag
+      end
+    end
+
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(term.strip) + '%'
+      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where('lower(name) like lower(?)', pattern)
+      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
          .order(:name)
          .limit(limit)
          .offset(offset)
     end
 
     def find_normalized(name)
-      find_by(name: name.mb_chars.downcase.to_s)
+      matching_name(name).first
     end
 
     def find_normalized!(name)
       find_normalized(name) || raise(ActiveRecord::RecordNotFound)
     end
+
+    def matching_name(name_or_names)
+      names = Array(name_or_names).map { |name| normalize(name).downcase }
+
+      if names.size == 1
+        where(arel_table[:name].lower.eq(names.first))
+      else
+        where(arel_table[:name].lower.in(names))
+      end
+    end
+
+    private
+
+    def normalize(str)
+      str.gsub(/\A#/, '').mb_chars.to_s
+    end
   end
 
   private

app/services/hashtag_query_service.rb

@@ -14,7 +14,7 @@ def call(tag, params, account = nil, local = false)
 
   private
 
-  def tags_for(tags)
-    Tag.where(name: tags.map(&:downcase)) if tags.presence
+  def tags_for(names)
+    Tag.matching_name(names) if names.presence
   end
 end

app/services/process_hashtags_service.rb

@@ -5,9 +5,7 @@ def call(status, tags = [])
     tags    = Extractor.extract_hashtags(status.text) if status.local?
     records = []
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      tag = Tag.where(name: name).first_or_create(name: name)
-
+    Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
 

db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

@@ -0,0 +1,15 @@
+class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name'
+    remove_index :tags, name: 'hashtag_search_index'
+  end
+
+  def down
+    add_index :tags, :name, unique: true, algorithm: :concurrently
+    safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_15_164535) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -652,8 +652,7 @@
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index"
-    t.index ["name"], name: "index_tags_on_name", unique: true
+    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|