Merge commit 'cb9f96036cf78835e22aafb30b80717b5a8b7394' into glitch-soc/merge-upstream

app/lib/importer/public_statuses_index_importer.rb

@@ -2,23 +2,14 @@
 
 class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
   def import!
-    indexable_statuses_scope.find_in_batches(batch_size: @batch_size) do |batch|
-      in_work_unit(batch.map(&:status_id)) do |status_ids|
+    scope.select(:id).find_in_batches(batch_size: @batch_size) do |batch|
+      in_work_unit(batch.pluck(:id)) do |status_ids|
         bulk = ActiveRecord::Base.connection_pool.with_connection do
-          Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
+          Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll, :preview_cards).where(id: status_ids)).bulk_body
         end
 
-        indexed = 0
-        deleted = 0
-
-        bulk.map! do |entry|
-          if entry[:index]
-            indexed += 1
-          else
-            deleted += 1
-          end
-          entry
-        end
+        indexed = bulk.count { |entry| entry[:index] }
+        deleted = bulk.count { |entry| entry[:delete] }
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 
@@ -35,7 +26,7 @@ class Importer::PublicStatusesIndexImporter < Importer::BaseImporter
     PublicStatusesIndex
   end
 
-  def indexable_statuses_scope
-    Status.indexable.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
+  def scope
+    Status.indexable
   end
 end

app/lib/importer/statuses_index_importer.rb

@@ -14,7 +14,7 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
       scope.find_in_batches(batch_size: @batch_size) do |tmp|
         in_work_unit(tmp.map(&:status_id)) do |status_ids|
           bulk = ActiveRecord::Base.connection_pool.with_connection do
-            Chewy::Index::Import::BulkBuilder.new(index, to_index: Status.includes(:media_attachments, :preloadable_poll).where(id: status_ids)).bulk_body
+            Chewy::Index::Import::BulkBuilder.new(index, to_index: index.adapter.default_scope.where(id: status_ids)).bulk_body
           end
 
           indexed = 0

app/lib/plain_text_formatter.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class PlainTextFormatter
-  include ActionView::Helpers::TextHelper
-
   NEWLINE_TAGS_RE = %r{(<br />|<br>|</p>)+}
 
   attr_reader :text, :local
@@ -18,7 +16,10 @@ class PlainTextFormatter
     if local?
       text
     else
-      html_entities.decode(strip_tags(insert_newlines)).chomp
+      node = Nokogiri::HTML.fragment(insert_newlines)
+      # Elements that are entirely removed with our Sanitize config
+      node.xpath('.//iframe|.//math|.//noembed|.//noframes|.//noscript|.//plaintext|.//script|.//style|.//svg|.//xmp').remove
+      node.text.chomp
     end
   end
 
@@ -27,8 +28,4 @@ class PlainTextFormatter
   def insert_newlines
     text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
   end
-
-  def html_entities
-    HTMLEntities.new
-  end
 end

app/lib/search_query_transformer.rb

@@ -98,7 +98,7 @@ class SearchQueryTransformer < Parslet::Transform
       when 'language'
         @filter = :language
         @type = :term
-        @term = term
+        @term = language_code_from_term(term)
       when 'from'
         @filter = :account_id
         @type = :term
@@ -137,6 +137,22 @@ class SearchQueryTransformer < Parslet::Transform
       # an ID that does not exist
       account&.id || -1
     end
+
+    def language_code_from_term(term)
+      language_code = term
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      language_code = term.split(/[_-]/).first.downcase
+
+      return language_code if LanguagesHelper::SUPPORTED_LOCALES.key?(language_code.to_sym)
+
+      term
+    end
   end
 
   rule(clause: subtree(:clause)) do