Merge commit 'f877aa9d70d0d600961989b8e97c0e0ce3ac1db6' into glitch-soc/merge-upstream

Conflicts:
- `.github/dependabot.yml`:
  Upstream made changes, but we had removed it.
  Discarded upstream changes.
- `.rubocop_todo.yml`:
  Upstream regenerated the file, we had some glitch-soc-specific ignores.
- `app/models/account_statuses_filter.rb`:
  Minor upstream code style change where glitch-soc had slightly different code
  due to handling of local-only posts.
  Updated to match upstream's code style.
- `app/models/status.rb`:
  Upstream moved ActiveRecord callback definitions, glitch-soc had an extra one.
  Moved the definitions as upstream did.
- `app/services/backup_service.rb`:
  Upstream rewrote a lot of the backup service, glitch-soc had changes because
  of exporting local-only posts.
  Took upstream changes and added back code to deal with local-only posts.
- `config/routes.rb`:
  Upstream split the file into different files, while glitch-soc had a few
  extra routes.
  Extra routes added to `config/routes/settings.rb`, `config/routes/api.rb`
  and `config/routes/admin.rb`
- `db/schema.rb`:
  Upstream has new migrations, while glitch-soc had an extra migration.
  Updated the expected serial number to match upstream's.
- `lib/mastodon/version.rb`:
  Upstream added support to set version tags from environment variables, while
  glitch-soc has an extra `+glitch` tag.
  Changed the code to support upstream's feature but prepending a `+glitch`.
- `spec/lib/activitypub/activity/create_spec.rb`:
  Minor code style change upstream, while glitch-soc has extra tests due to
  `directMessage` handling.
  Applied upstream's changes while keeping glitch-soc's extra tests.
- `spec/models/concerns/account_interactions_spec.rb`:
  Minor code style change upstream, while glitch-soc has extra tests.
  Applied upstream's changes while keeping glitch-soc's extra tests.

app/controllers/admin/domain_blocks_controller.rb

@@ -33,7 +33,7 @@ module Admin
 
       if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block)
         @domain_block.save
-        flash.now[:alert] = I18n.t('admin.domain_blocks.existing_domain_block_html', name: existing_domain_block.domain, unblock_url: admin_domain_block_path(existing_domain_block)).html_safe # rubocop:disable Rails/OutputSafety
+        flash.now[:alert] = I18n.t('admin.domain_blocks.existing_domain_block_html', name: existing_domain_block.domain, unblock_url: admin_domain_block_path(existing_domain_block)).html_safe
         @domain_block.errors.delete(:domain)
         render :new
       else

app/controllers/api/v1/media_controller.rb

@@ -15,7 +15,8 @@ class Api::V1::MediaController < Api::BaseController
     render json: @media_attachment, serializer: REST::MediaAttachmentSerializer
   rescue Paperclip::Errors::NotIdentifiedByImageMagickError
     render json: file_type_error, status: 422
-  rescue Paperclip::Error
+  rescue Paperclip::Error => e
+    Rails.logger.error "#{e.class}: #{e.message}"
     render json: processing_error, status: 500
   end
 

app/controllers/api/v2/media_controller.rb

@@ -6,7 +6,8 @@ class Api::V2::MediaController < Api::V1::MediaController
     render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: @media_attachment.not_processed? ? 202 : 200
   rescue Paperclip::Errors::NotIdentifiedByImageMagickError
     render json: file_type_error, status: 422
-  rescue Paperclip::Error
+  rescue Paperclip::Error => e
+    Rails.logger.error "#{e.class}: #{e.message}"
     render json: processing_error, status: 500
   end
 end

app/controllers/authorize_interactions_controller.rb

@@ -60,7 +60,7 @@ class AuthorizeInteractionsController < ApplicationController
   end
 
   def uri_param
-    params[:uri] || params.fetch(:acct, '').gsub(/\Aacct:/, '')
+    params[:uri] || params.fetch(:acct, '').delete_prefix('acct:')
   end
 
   def set_body_classes

app/controllers/concerns/signature_verification.rb

@@ -180,14 +180,15 @@ module SignatureVerification
 
   def build_signed_string
     signed_headers.map do |signed_header|
-      if signed_header == Request::REQUEST_TARGET
+      case signed_header
+      when Request::REQUEST_TARGET
         "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
-      elsif signed_header == '(created)'
+      when '(created)'
         raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank?
 
         "(created): #{signature_params['created']}"
-      elsif signed_header == '(expires)'
+      when '(expires)'
         raise SignatureVerificationError, 'Invalid pseudo-header (expires) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (expires) used but corresponding argument missing' if signature_params['expires'].blank?
 
@@ -244,7 +245,7 @@ module SignatureVerification
     end
 
     if key_id.start_with?('acct:')
-      stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, ''), suppress_errors: false) }
+      stoplight_wrap_request { ResolveAccountService.new.call(key_id.delete_prefix('acct:'), suppress_errors: false) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
       account   = ActivityPub::TagManager.instance.uri_to_actor(key_id)
       account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false, suppress_errors: false) }

app/controllers/intents_controller.rb

@@ -9,7 +9,7 @@ class IntentsController < ApplicationController
     if uri.scheme == 'web+mastodon'
       case uri.host
       when 'follow'
-        return redirect_to authorize_interaction_path(uri: uri.query_values['uri'].gsub(/\Aacct:/, ''))
+        return redirect_to authorize_interaction_path(uri: uri.query_values['uri'].delete_prefix('acct:'))
       when 'share'
         return redirect_to share_path(text: uri.query_values['text'])
       end

app/controllers/media_proxy_controller.rb

@@ -16,7 +16,7 @@ class MediaProxyController < ApplicationController
   rescue_from HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, with: :internal_server_error
 
   def show
-    with_lock("media_download:#{params[:id]}") do
+    with_redis_lock("media_download:#{params[:id]}") do
       @media_attachment = MediaAttachment.remote.attached.find(params[:id])
       authorize @media_attachment.status, :show?
       redownload! if @media_attachment.needs_redownload? && !reject_media?

app/controllers/settings/exports_controller.rb

@@ -15,7 +15,7 @@ class Settings::ExportsController < Settings::BaseController
   def create
     backup = nil
 
-    with_lock("backup:#{current_user.id}") do
+    with_redis_lock("backup:#{current_user.id}") do
       authorize :backup, :create?
       backup = current_user.backups.create!
     end

app/controllers/settings/imports_controller.rb

@@ -1,31 +1,97 @@
 # frozen_string_literal: true
 
-class Settings::ImportsController < Settings::BaseController
-  before_action :set_account
+require 'csv'
 
-  def show
-    @import = Import.new
+class Settings::ImportsController < Settings::BaseController
+  before_action :set_bulk_import, only: [:show, :confirm, :destroy]
+  before_action :set_recent_imports, only: [:index]
+
+  TYPE_TO_FILENAME_MAP = {
+    following: 'following_accounts_failures.csv',
+    blocking: 'blocked_accounts_failures.csv',
+    muting: 'muted_accounts_failures.csv',
+    domain_blocking: 'blocked_domains_failures.csv',
+    bookmarks: 'bookmarks_failures.csv',
+  }.freeze
+
+  TYPE_TO_HEADERS_MAP = {
+    following: ['Account address', 'Show boosts', 'Notify on new posts', 'Languages'],
+    blocking: false,
+    muting: ['Account address', 'Hide notifications'],
+    domain_blocking: false,
+    bookmarks: false,
+  }.freeze
+
+  def index
+    @import = Form::Import.new(current_account: current_account)
+  end
+
+  def show; end
+
+  def failures
+    @bulk_import = current_account.bulk_imports.where(state: :finished).find(params[:id])
+
+    respond_to do |format|
+      format.csv do
+        filename = TYPE_TO_FILENAME_MAP[@bulk_import.type.to_sym]
+        headers = TYPE_TO_HEADERS_MAP[@bulk_import.type.to_sym]
+
+        export_data = CSV.generate(headers: headers, write_headers: true) do |csv|
+          @bulk_import.rows.find_each do |row|
+            case @bulk_import.type.to_sym
+            when :following
+              csv << [row.data['acct'], row.data.fetch('show_reblogs', true), row.data.fetch('notify', false), row.data['languages']&.join(', ')]
+            when :blocking
+              csv << [row.data['acct']]
+            when :muting
+              csv << [row.data['acct'], row.data.fetch('hide_notifications', true)]
+            when :domain_blocking
+              csv << [row.data['domain']]
+            when :bookmarks
+              csv << [row.data['uri']]
+            end
+          end
+        end
+
+        send_data export_data, filename: filename
+      end
+    end
+  end
+
+  def confirm
+    @bulk_import.update!(state: :scheduled)
+    BulkImportWorker.perform_async(@bulk_import.id)
+    redirect_to settings_imports_path, notice: I18n.t('imports.success')
   end
 
   def create
-    @import = Import.new(import_params)
-    @import.account = @account
+    @import = Form::Import.new(import_params.merge(current_account: current_account))
 
     if @import.save
-      ImportWorker.perform_async(@import.id)
-      redirect_to settings_import_path, notice: I18n.t('imports.success')
+      redirect_to settings_import_path(@import.bulk_import.id)
     else
-      render :show
+      # We need to set recent imports as we are displaying the index again
+      set_recent_imports
+      render :index
     end
   end
 
+  def destroy
+    @bulk_import.destroy!
+    redirect_to settings_imports_path
+  end
+
   private
 
-  def set_account
-    @account = current_user.account
+  def import_params
+    params.require(:form_import).permit(:data, :type, :mode)
   end
 
-  def import_params
-    params.require(:import).permit(:data, :type, :mode)
+  def set_bulk_import
+    @bulk_import = current_account.bulk_imports.where(state: :unconfirmed).find(params[:id])
+  end
+
+  def set_recent_imports
+    @recent_imports = current_account.bulk_imports.reorder(id: :desc).limit(10)
   end
 end

app/controllers/settings/preferences/appearance_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Settings::Preferences::AppearanceController < Settings::PreferencesController
+class Settings::Preferences::AppearanceController < Settings::Preferences::BaseController
   private
 
   def after_update_redirect_path

app/controllers/settings/preferences/base_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Settings::PreferencesController < Settings::BaseController
+class Settings::Preferences::BaseController < Settings::BaseController
   def show; end
 
   def update
@@ -15,7 +15,7 @@ class Settings::PreferencesController < Settings::BaseController
   private
 
   def after_update_redirect_path
-    settings_preferences_path
+    raise 'Override in controller'
   end
 
   def user_params

app/controllers/settings/preferences/notifications_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Settings::Preferences::NotificationsController < Settings::PreferencesController
+class Settings::Preferences::NotificationsController < Settings::Preferences::BaseController
   private
 
   def after_update_redirect_path

app/controllers/settings/preferences/other_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Settings::Preferences::OtherController < Settings::PreferencesController
+class Settings::Preferences::OtherController < Settings::Preferences::BaseController
   private
 
   def after_update_redirect_path

app/controllers/well_known/webfinger_controller.rb

@@ -18,7 +18,14 @@ module WellKnown
     private
 
     def set_account
-      @account = Account.find_local!(username_from_resource)
+      username = username_from_resource
+      @account = begin
+        if username == Rails.configuration.x.local_domain
+          Account.representative
+        else
+          Account.find_local!(username)
+        end
+      end
     end
 
     def username_from_resource

app/helpers/application_helper.rb

@@ -32,10 +32,6 @@ module ApplicationHelper
     paths.any? { |path| current_page?(path) } ? 'active' : ''
   end
 
-  def active_link_to(label, path, **options)
-    link_to label, path, options.merge(class: active_nav_class(path))
-  end
-
   def show_landing_strip?
     !user_signed_in? && !single_user_mode?
   end
@@ -147,7 +143,7 @@ module ApplicationHelper
     if prefers_autoplay?
       image_tag(custom_emoji.image.url, class: 'emojione', alt: ":#{custom_emoji.shortcode}:")
     else
-      image_tag(custom_emoji.image.url(:static), class: 'emojione custom-emoji', alt: ":#{custom_emoji.shortcode}", 'data-original' => full_asset_url(custom_emoji.image.url), 'data-static' => full_asset_url(custom_emoji.image.url(:static)))
+      image_tag(custom_emoji.image.url(:static), :class => 'emojione custom-emoji', :alt => ":#{custom_emoji.shortcode}", 'data-original' => full_asset_url(custom_emoji.image.url), 'data-static' => full_asset_url(custom_emoji.image.url(:static)))
     end
   end
 
@@ -174,11 +170,11 @@ module ApplicationHelper
   end
 
   def storage_host
-    "https://#{ENV['S3_ALIAS_HOST'].presence || ENV['S3_CLOUDFRONT_HOST']}"
+    URI::HTTPS.build(host: storage_host_name).to_s
   end
 
   def storage_host?
-    ENV['S3_ALIAS_HOST'].present? || ENV['S3_CLOUDFRONT_HOST'].present?
+    storage_host_name.present?
   end
 
   def quote_wrap(text, line_width: 80, break_sequence: "\n")
@@ -236,4 +232,10 @@ module ApplicationHelper
   def prerender_custom_emojis(html, custom_emojis, other_options = {})
     EmojiFormatter.new(html, custom_emojis, other_options.merge(animate: prefers_autoplay?)).to_s
   end
+
+  private
+
+  def storage_host_name
+    ENV.fetch('S3_ALIAS_HOST', nil) || ENV.fetch('S3_CLOUDFRONT_HOST', nil)
+  end
 end

app/javascript/mastodon/actions/app.js

@@ -1,17 +0,0 @@
-export const APP_FOCUS   = 'APP_FOCUS';
-export const APP_UNFOCUS = 'APP_UNFOCUS';
-
-export const focusApp = () => ({
-  type: APP_FOCUS,
-});
-
-export const unfocusApp = () => ({
-  type: APP_UNFOCUS,
-});
-
-export const APP_LAYOUT_CHANGE = 'APP_LAYOUT_CHANGE';
-
-export const changeLayout = layout => ({
-  type: APP_LAYOUT_CHANGE,
-  layout,
-});

app/javascript/mastodon/actions/app.ts

@@ -0,0 +1,10 @@
+import { createAction } from '@reduxjs/toolkit';
+
+export const focusApp = createAction('APP_FOCUS');
+export const unfocusApp = createAction('APP_UNFOCUS');
+
+type ChangeLayoutPayload = {
+  layout: 'mobile' | 'single-column' | 'multi-column';
+};
+export const changeLayout =
+  createAction<ChangeLayoutPayload>('APP_LAYOUT_CHANGE');

app/javascript/mastodon/blurhash.ts

@@ -84,7 +84,7 @@ const DIGIT_CHARACTERS = [
   '~',
 ];
 
-export const decode83 = (str) => {
+export const decode83 = (str: string) => {
   let value = 0;
   let c, digit;
 
@@ -97,13 +97,13 @@ export const decode83 = (str) => {
   return value;
 };
 
-export const intToRGB = int => ({
+export const intToRGB = (int: number) => ({
   r: Math.max(0, (int >> 16)),
   g: Math.max(0, (int >> 8) & 255),
   b: Math.max(0, (int & 255)),
 });
 
-export const getAverageFromBlurhash = blurhash => {
+export const getAverageFromBlurhash = (blurhash: string) => {
   if (!blurhash) {
     return null;
   }

app/javascript/mastodon/compare_id.ts

@@ -1,4 +1,4 @@
-export default function compareId (id1, id2) {
+export default function compareId (id1: string, id2: string) {
   if (id1 === id2) {
     return 0;
   }

app/javascript/mastodon/components/blurhash.jsx

@@ -1,65 +0,0 @@
-// @ts-check
-
-import { decode } from 'blurhash';
-import React, { useRef, useEffect } from 'react';
-import PropTypes from 'prop-types';
-
-/**
- * @typedef BlurhashPropsBase
- * @property {string?} hash Hash to render
- * @property {number} width
- * Width of the blurred region in pixels. Defaults to 32
- * @property {number} [height]
- * Height of the blurred region in pixels. Defaults to width
- * @property {boolean} [dummy]
- * Whether dummy mode is enabled. If enabled, nothing is rendered
- * and canvas left untouched
- */
-
-/** @typedef {JSX.IntrinsicElements['canvas'] & BlurhashPropsBase} BlurhashProps */
-
-/**
- * Component that is used to render blurred of blurhash string
- * @param {BlurhashProps} param1 Props of the component
- * @returns {JSX.Element} Canvas which will render blurred region element to embed
- */
-function Blurhash({
-  hash,
-  width = 32,
-  height = width,
-  dummy = false,
-  ...canvasProps
-}) {
-  const canvasRef = /** @type {import('react').MutableRefObject<HTMLCanvasElement>} */ (useRef());
-
-  useEffect(() => {
-    const { current: canvas } = canvasRef;
-    canvas.width = canvas.width; // resets canvas
-
-    if (dummy || !hash) return;
-
-    try {
-      const pixels = decode(hash, width, height);
-      const ctx = canvas.getContext('2d');
-      const imageData = new ImageData(pixels, width, height);
-
-      // @ts-expect-error
-      ctx.putImageData(imageData, 0, 0);
-    } catch (err) {
-      console.error('Blurhash decoding failure', { err, hash });
-    }
-  }, [dummy, hash, width, height]);
-
-  return (
-    <canvas {...canvasProps} ref={canvasRef} width={width} height={height} />
-  );
-}
-
-Blurhash.propTypes = {
-  hash: PropTypes.string.isRequired,
-  width: PropTypes.number,
-  height: PropTypes.number,
-  dummy: PropTypes.bool,
-};
-
-export default React.memo(Blurhash);

app/javascript/mastodon/components/blurhash.tsx

@@ -0,0 +1,45 @@
+import { decode } from 'blurhash';
+import React, { useRef, useEffect } from 'react';
+
+type Props = {
+  hash: string;
+  width?: number;
+  height?: number;
+  dummy?: boolean; // Whether dummy mode is enabled. If enabled, nothing is rendered and canvas left untouched
+  children?: never;
+  [key: string]: any;
+}
+function Blurhash({
+  hash,
+  width = 32,
+  height = width,
+  dummy = false,
+  ...canvasProps
+}: Props) {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+
+  useEffect(() => {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const canvas = canvasRef.current!;
+    // eslint-disable-next-line no-self-assign
+    canvas.width = canvas.width; // resets canvas
+
+    if (dummy || !hash) return;
+
+    try {
+      const pixels = decode(hash, width, height);
+      const ctx = canvas.getContext('2d');
+      const imageData = new ImageData(pixels, width, height);
+
+      ctx?.putImageData(imageData, 0, 0);
+    } catch (err) {
+      console.error('Blurhash decoding failure', { err, hash });
+    }
+  }, [dummy, hash, width, height]);
+
+  return (
+    <canvas {...canvasProps} ref={canvasRef} width={width} height={height} />
+  );
+}
+
+export default React.memo(Blurhash);

app/javascript/mastodon/components/column_back_button.jsx

@@ -21,7 +21,9 @@ export default class ColumnBackButton extends React.PureComponent {
 
     if (onClick) {
       onClick();
-    } else if (window.history && window.history.state) {
+    // Check if there is a previous page in the app to go back to per https://stackoverflow.com/a/70532858/9703201
+    // When upgrading to V6, check `location.key !== 'default'` instead per https://github.com/remix-run/history/blob/main/docs/api-reference.md#location
+    } else if (router.route.location.key) {
       router.history.goBack();
     } else {
       router.history.push('/');

app/javascript/mastodon/components/icon_button.tsx

@@ -1,34 +1,36 @@
 import React from 'react';
-import PropTypes from 'prop-types';
 import classNames from 'classnames';
-import Icon from 'mastodon/components/icon';
-import AnimatedNumber from 'mastodon/components/animated_number';
+import { Icon } from './icon';
+import { AnimatedNumber } from './animated_number';
 
-export default class IconButton extends React.PureComponent {
-
-  static propTypes = {
-    className: PropTypes.string,
-    title: PropTypes.string.isRequired,
-    icon: PropTypes.string.isRequired,
-    onClick: PropTypes.func,
-    onMouseDown: PropTypes.func,
-    onKeyDown: PropTypes.func,
-    onKeyPress: PropTypes.func,
-    size: PropTypes.number,
-    active: PropTypes.bool,
-    expanded: PropTypes.bool,
-    style: PropTypes.object,
-    activeStyle: PropTypes.object,
-    disabled: PropTypes.bool,
-    inverted: PropTypes.bool,
-    animate: PropTypes.bool,
-    overlay: PropTypes.bool,
-    tabIndex: PropTypes.number,
-    counter: PropTypes.number,
-    obfuscateCount: PropTypes.bool,
-    href: PropTypes.string,
-    ariaHidden: PropTypes.bool,
-  };
+type Props = {
+  className?: string;
+  title: string;
+  icon: string;
+  onClick?: React.MouseEventHandler<HTMLButtonElement>;
+  onMouseDown?: React.MouseEventHandler<HTMLButtonElement>;
+  onKeyDown?: React.KeyboardEventHandler<HTMLButtonElement>;
+  onKeyPress?: React.KeyboardEventHandler<HTMLButtonElement>;
+  size: number;
+  active: boolean;
+  expanded?: boolean;
+  style?: React.CSSProperties;
+  activeStyle?: React.CSSProperties;
+  disabled: boolean;
+  inverted?: boolean;
+  animate: boolean;
+  overlay: boolean;
+  tabIndex: number;
+  counter?: number;
+  obfuscateCount?: boolean;
+  href?: string;
+  ariaHidden: boolean;
+}
+type States = {
+  activate: boolean,
+  deactivate: boolean,
+}
+export default class IconButton extends React.PureComponent<Props, States> {
 
   static defaultProps = {
     size: 18,
@@ -45,7 +47,7 @@ export default class IconButton extends React.PureComponent {
     deactivate: false,
   };
 
-  componentWillReceiveProps (nextProps) {
+  UNSAFE_componentWillReceiveProps (nextProps: Props) {
     if (!nextProps.animate) return;
 
     if (this.props.active && !nextProps.active) {
@@ -55,27 +57,27 @@ export default class IconButton extends React.PureComponent {
     }
   }
 
-  handleClick = (e) =>  {
+  handleClick: React.MouseEventHandler<HTMLButtonElement> = (e) =>  {
     e.preventDefault();
 
-    if (!this.props.disabled) {
+    if (!this.props.disabled && this.props.onClick != null) {
       this.props.onClick(e);
     }
   };
 
-  handleKeyPress = (e) => {
+  handleKeyPress: React.KeyboardEventHandler<HTMLButtonElement> = (e) => {
     if (this.props.onKeyPress && !this.props.disabled) {
       this.props.onKeyPress(e);
     }
   };
 
-  handleMouseDown = (e) => {
+  handleMouseDown: React.MouseEventHandler<HTMLButtonElement> = (e) => {
     if (!this.props.disabled && this.props.onMouseDown) {
       this.props.onMouseDown(e);
     }
   };
 
-  handleKeyDown = (e) => {
+  handleKeyDown: React.KeyboardEventHandler<HTMLButtonElement> = (e) => {
     if (!this.props.disabled && this.props.onKeyDown) {
       this.props.onKeyDown(e);
     }
@@ -132,7 +134,7 @@ export default class IconButton extends React.PureComponent {
       </React.Fragment>
     );
 
-    if (href && !this.prop) {
+    if (href != null) {
       contents = (
         <a href={href} target='_blank' rel='noopener noreferrer'>
           {contents}

app/javascript/mastodon/components/media_gallery.jsx

@@ -81,12 +81,10 @@ class Item extends React.PureComponent {
   render () {
     const { attachment, lang, index, size, standalone, displayWidth, visible } = this.props;
 
+    let badges = [], thumbnail;
+
     let width  = 50;
     let height = 100;
-    let top    = 'auto';
-    let left   = 'auto';
-    let bottom = 'auto';
-    let right  = 'auto';
 
     if (size === 1) {
       width = 100;
@@ -96,45 +94,13 @@ class Item extends React.PureComponent {
       height = 50;
     }
 
-    if (size === 2) {
-      if (index === 0) {
-        right = '2px';
-      } else {
-        left = '2px';
-      }
-    } else if (size === 3) {
-      if (index === 0) {
-        right = '2px';
-      } else if (index > 0) {
-        left = '2px';
-      }
-
-      if (index === 1) {
-        bottom = '2px';
-      } else if (index > 1) {
-        top = '2px';
-      }
-    } else if (size === 4) {
-      if (index === 0 || index === 2) {
-        right = '2px';
-      }
-
-      if (index === 1 || index === 3) {
-        left = '2px';
-      }
-
-      if (index < 2) {
-        bottom = '2px';
-      } else {
-        top = '2px';
-      }
+    if (attachment.get('description')?.length > 0) {
+      badges.push(<span key='alt' className='media-gallery__gifv__label'>ALT</span>);
     }
 
-    let thumbnail = '';
-
     if (attachment.get('type') === 'unknown') {
       return (
-        <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
+        <div className={classNames('media-gallery__item', { standalone, 'media-gallery__item--tall': height === 100, 'media-gallery__item--wide': width === 100 })} key={attachment.get('id')}>
           <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url') || attachment.get('url')} style={{ cursor: 'pointer' }} title={attachment.get('description')} lang={lang} target='_blank' rel='noopener noreferrer'>
             <Blurhash
               hash={attachment.get('blurhash')}
@@ -184,6 +150,8 @@ class Item extends React.PureComponent {
     } else if (attachment.get('type') === 'gifv') {
       const autoPlay = this.getAutoPlay();
 
+      badges.push(<span key='gif' className='media-gallery__gifv__label'>GIF</span>);
+
       thumbnail = (
         <div className={classNames('media-gallery__gifv', { autoplay: autoPlay })}>
           <video
@@ -201,14 +169,12 @@ class Item extends React.PureComponent {
             loop
             muted
           />
-
-          <span className='media-gallery__gifv__label'>GIF</span>
         </div>
       );
     }
 
     return (
-      <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
+      <div className={classNames('media-gallery__item', { standalone, 'media-gallery__item--tall': height === 100, 'media-gallery__item--wide': width === 100 })} key={attachment.get('id')}>
         <Blurhash
           hash={attachment.get('blurhash')}
           dummy={!useBlurhash}
@@ -216,7 +182,14 @@ class Item extends React.PureComponent {
             'media-gallery__preview--hidden': visible && this.state.loaded,
           })}
         />
+
         {visible && thumbnail}
+
+        {badges && (
+          <div className='media-gallery__item__badges'>
+            {badges}
+          </div>
+        )}
       </div>
     );
   }
@@ -313,7 +286,7 @@ class MediaGallery extends React.PureComponent {
   }
 
   render () {
-    const { media, lang, intl, sensitive, height, defaultWidth, standalone, autoplay } = this.props;
+    const { media, lang, intl, sensitive, defaultWidth, standalone, autoplay } = this.props;
     const { visible } = this.state;
     const width = this.state.width || defaultWidth;
 
@@ -322,13 +295,9 @@ class MediaGallery extends React.PureComponent {
     const style = {};
 
     if (this.isFullSizeEligible() && (standalone || !cropImages)) {
-      if (width) {
-        style.height = width / this.props.media.getIn([0, 'meta', 'small', 'aspect']);
-      }
-    } else if (width) {
-      style.height = width / (16/9);
+      style.aspectRatio = `${this.props.media.getIn([0, 'meta', 'small', 'aspect'])}`;
     } else {
-      style.height = height;
+      style.aspectRatio = '16 / 9';
     }
 
     const size     = media.take(4).size;

app/javascript/mastodon/components/picture_in_picture_placeholder.jsx

@@ -3,62 +3,22 @@ import PropTypes from 'prop-types';
 import Icon from 'mastodon/components/icon';
 import { removePictureInPicture } from 'mastodon/actions/picture_in_picture';
 import { connect } from 'react-redux';
-import { debounce } from 'lodash';
 import { FormattedMessage } from 'react-intl';
 
 class PictureInPicturePlaceholder extends React.PureComponent {
 
   static propTypes = {
-    width: PropTypes.number,
     dispatch: PropTypes.func.isRequired,
   };
 
-  state = {
-    width: this.props.width,
-    height: this.props.width && (this.props.width / (16/9)),
-  };
-
   handleClick = () => {
     const { dispatch } = this.props;
     dispatch(removePictureInPicture());
   };
 
-  setRef = c => {
-    this.node = c;
-
-    if (this.node) {
-      this._setDimensions();
-    }
-  };
-
-  _setDimensions () {
-    const width  = this.node.offsetWidth;
-    const height = width / (16/9);
-
-    this.setState({ width, height });
-  }
-
-  componentDidMount () {
-    window.addEventListener('resize', this.handleResize, { passive: true });
-  }
-
-  componentWillUnmount () {
-    window.removeEventListener('resize', this.handleResize);
-  }
-
-  handleResize = debounce(() => {
-    if (this.node) {
-      this._setDimensions();
-    }
-  }, 250, {
-    trailing: true,
-  });
-
   render () {
-    const { height } = this.state;
-
     return (
-      <div ref={this.setRef} className='picture-in-picture-placeholder' style={{ height }} role='button' tabIndex={0} onClick={this.handleClick}>
+      <div className='picture-in-picture-placeholder' role='button' tabIndex={0} onClick={this.handleClick}>
         <Icon id='window-restore' />
         <FormattedMessage id='picture_in_picture.restore' defaultMessage='Put it back' />
       </div>

app/javascript/mastodon/components/status.jsx

@@ -411,7 +411,7 @@ class Status extends ImmutablePureComponent {
     }
 
     if (pictureInPicture.get('inUse')) {
-      media = <PictureInPicturePlaceholder width={this.props.cachedMediaWidth} />;
+      media = <PictureInPicturePlaceholder />;
     } else if (status.get('media_attachments').size > 0) {
       if (this.props.muted) {
         media = (
@@ -460,12 +460,9 @@ class Status extends ImmutablePureComponent {
                 src={attachment.get('url')}
                 alt={attachment.get('description')}
                 lang={status.get('language')}
-                width={this.props.cachedMediaWidth}
-                height={110}
                 inline
                 sensitive={status.get('sensitive')}
                 onOpenVideo={this.handleOpenVideo}
-                cacheWidth={this.props.cacheMediaWidth}
                 deployPictureInPicture={pictureInPicture.get('available') ? this.handleDeployPictureInPicture : undefined}
                 visible={this.state.showMedia}
                 onToggleVisibility={this.handleToggleMediaVisibility}
@@ -498,8 +495,6 @@ class Status extends ImmutablePureComponent {
           onOpenMedia={this.handleOpenMedia}
           card={status.get('card')}
           compact
-          cacheWidth={this.props.cacheMediaWidth}
-          defaultWidth={this.props.cachedMediaWidth}
           sensitive={status.get('sensitive')}
         />
       );

app/javascript/mastodon/containers/compose_container.jsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { Provider } from 'react-redux';
 import PropTypes from 'prop-types';
-import configureStore from '../store/configureStore';
+import { store } from '../store/configureStore';
 import { hydrateStore } from '../actions/store';
 import { IntlProvider, addLocaleData } from 'react-intl';
 import { getLocale } from '../locales';
@@ -12,8 +12,6 @@ import { fetchCustomEmojis } from '../actions/custom_emojis';
 const { localeData, messages } = getLocale();
 addLocaleData(localeData);
 
-const store = configureStore();
-
 if (initialState) {
   store.dispatch(hydrateStore(initialState));
 }

app/javascript/mastodon/containers/mastodon.jsx

@@ -5,7 +5,7 @@ import { IntlProvider, addLocaleData } from 'react-intl';
 import { Provider as ReduxProvider } from 'react-redux';
 import { BrowserRouter, Route } from 'react-router-dom';
 import { ScrollContext } from 'react-router-scroll-4';
-import configureStore from 'mastodon/store/configureStore';
+import { store } from 'mastodon/store/configureStore';
 import UI from 'mastodon/features/ui';
 import { fetchCustomEmojis } from 'mastodon/actions/custom_emojis';
 import { hydrateStore } from 'mastodon/actions/store';
@@ -19,7 +19,6 @@ addLocaleData(localeData);
 
 const title = process.env.NODE_ENV === 'production' ? siteTitle : `${siteTitle} (Dev)`;
 
-export const store = configureStore();
 const hydrateAction = hydrateStore(initialState);
 
 store.dispatch(hydrateAction);

app/javascript/mastodon/features/audio/index.jsx

@@ -384,7 +384,7 @@ class Audio extends React.PureComponent {
   }
 
   _getRadius () {
-    return parseInt(((this.state.height || this.props.height) - (PADDING * this._getScaleCoefficient()) * 2) / 2);
+    return parseInt((this.state.height || this.props.height) / 2 - PADDING * this._getScaleCoefficient());
   }
 
   _getScaleCoefficient () {
@@ -396,7 +396,7 @@ class Audio extends React.PureComponent {
   }
 
   _getCY() {
-    return Math.floor(this._getRadius() + (PADDING * this._getScaleCoefficient()));
+    return Math.floor((this.state.height || this.props.height) / 2);
   }
 
   _getAccentColor () {
@@ -470,7 +470,7 @@ class Audio extends React.PureComponent {
     }
 
     return (
-      <div className={classNames('audio-player', { editable, inactive: !revealed })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), height: this.props.fullscreen ? '100%' : (this.state.height || this.props.height) }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave} tabIndex={0} onKeyDown={this.handleKeyDown}>
+      <div className={classNames('audio-player', { editable, inactive: !revealed })} ref={this.setPlayerRef} style={{ backgroundColor: this._getBackgroundColor(), color: this._getForegroundColor(), aspectRatio: '16 / 9' }} onMouseEnter={this.handleMouseEnter} onMouseLeave={this.handleMouseLeave} tabIndex={0} onKeyDown={this.handleKeyDown}>
 
         <Blurhash
           hash={blurhash}
@@ -515,9 +515,16 @@ class Audio extends React.PureComponent {
         {(revealed || editable) && <img
           src={this.props.poster}
           alt=''
-          width={(this._getRadius() - TICK_SIZE) * 2}
-          height={(this._getRadius() - TICK_SIZE) * 2}
-          style={{ position: 'absolute', left: this._getCX(), top: this._getCY(), transform: 'translate(-50%, -50%)', borderRadius: '50%', pointerEvents: 'none' }}
+          style={{
+            position: 'absolute',
+            left: '50%',
+            top: '50%',
+            height: `calc(${(100 - 2 * 100 * PADDING / 982)}% - ${TICK_SIZE * 2}px)`,
+            aspectRatio: '1',
+            transform: 'translate(-50%, -50%)',
+            borderRadius: '50%',
+            pointerEvents: 'none',
+          }}
         />}
 
         <div className='video-player__seek' onMouseDown={this.handleMouseDown} ref={this.setSeekRef}>

app/javascript/mastodon/features/status/components/card.jsx

@@ -8,7 +8,6 @@ import classnames from 'classnames';
 import Icon from 'mastodon/components/icon';
 import { useBlurhash } from 'mastodon/initial_state';
 import Blurhash from 'mastodon/components/blurhash';
-import { debounce } from 'lodash';
 
 const IDNA_PREFIX = 'xn--';
 
@@ -54,8 +53,6 @@ export default class Card extends React.PureComponent {
     card: ImmutablePropTypes.map,
     onOpenMedia: PropTypes.func.isRequired,
     compact: PropTypes.bool,
-    defaultWidth: PropTypes.number,
-    cacheWidth: PropTypes.func,
     sensitive: PropTypes.bool,
   };
 
@@ -64,7 +61,6 @@ export default class Card extends React.PureComponent {
   };
 
   state = {
-    width: this.props.defaultWidth || 280,
     previewLoaded: false,
     embedded: false,
     revealed: !this.props.sensitive,
@@ -87,24 +83,6 @@ export default class Card extends React.PureComponent {
     window.removeEventListener('resize', this.handleResize);
   }
 
-  _setDimensions () {
-    const width = this.node.offsetWidth;
-
-    if (this.props.cacheWidth) {
-      this.props.cacheWidth(width);
-    }
-
-    this.setState({ width });
-  }
-
-  handleResize = debounce(() => {
-    if (this.node) {
-      this._setDimensions();
-    }
-  }, 250, {
-    trailing: true,
-  });
-
   handlePhotoClick = () => {
     const { card, onOpenMedia } = this.props;
 
@@ -138,10 +116,6 @@ export default class Card extends React.PureComponent {
 
   setRef = c => {
     this.node = c;
-
-    if (this.node) {
-      this._setDimensions();
-    }
   };
 
   handleImageLoad = () => {
@@ -157,36 +131,31 @@ export default class Card extends React.PureComponent {
   renderVideo () {
     const { card }  = this.props;
     const content   = { __html: addAutoPlay(card.get('html')) };
-    const { width } = this.state;
-    const ratio     = card.get('width') / card.get('height');
-    const height    = width / ratio;
 
     return (
       <div
         ref={this.setRef}
         className='status-card__image status-card-video'
         dangerouslySetInnerHTML={content}
-        style={{ height }}
+        style={{ aspectRatio: `${card.get('width')} / ${card.get('height')}` }}
       />
     );
   }
 
   render () {
     const { card, compact } = this.props;
-    const { width, embedded, revealed } = this.state;
+    const { embedded, revealed } = this.state;
 
     if (card === null) {
       return null;
     }
 
     const provider    = card.get('provider_name').length === 0 ? decodeIDNA(getHostname(card.get('url'))) : card.get('provider_name');
-    const horizontal  = (!compact && card.get('width') > card.get('height') && (card.get('width') + 100 >= width)) || card.get('type') !== 'link' || embedded;
+    const horizontal  = (!compact && card.get('width') > card.get('height')) || card.get('type') !== 'link' || embedded;
     const interactive = card.get('type') !== 'link';
     const className   = classnames('status-card', { horizontal, compact, interactive });
     const title       = interactive ? <a className='status-card__title' href={card.get('url')} title={card.get('title')} rel='noopener noreferrer' target='_blank'><strong>{card.get('title')}</strong></a> : <strong className='status-card__title' title={card.get('title')}>{card.get('title')}</strong>;
     const language    = card.get('language') || '';
-    const ratio       = card.get('width') / card.get('height');
-    const height      = (compact && !embedded) ? (width / (16 / 9)) : (width / ratio);
 
     const description = (
       <div className='status-card__content' lang={language}>
@@ -196,6 +165,14 @@ export default class Card extends React.PureComponent {
       </div>
     );
 
+    const thumbnailStyle = {
+      visibility: revealed? null : 'hidden',
+    };
+
+    if (horizontal) {
+      thumbnailStyle.aspectRatio = (compact && !embedded) ? '16 / 9' : `${card.get('width')} / ${card.get('height')}`;
+    }
+
     let embed     = '';
     let canvas = (
       <Blurhash
@@ -206,7 +183,7 @@ export default class Card extends React.PureComponent {
         dummy={!useBlurhash}
       />
     );
-    let thumbnail = <img src={card.get('image')} alt='' style={{ width: horizontal ? width : null, height: horizontal ? height : null, visibility: revealed ? null : 'hidden' }} onLoad={this.handleImageLoad} className='status-card__image-image' />;
+    let thumbnail = <img src={card.get('image')} alt='' style={thumbnailStyle} onLoad={this.handleImageLoad} className='status-card__image-image' />;
     let spoilerButton = (
       <button type='button' onClick={this.handleReveal} className='spoiler-button__overlay'>
         <span className='spoiler-button__overlay__label'><FormattedMessage id='status.sensitive_warning' defaultMessage='Sensitive content' /></span>

app/javascript/mastodon/features/ui/index.jsx

@@ -362,7 +362,7 @@ class UI extends React.PureComponent {
 
     if (layout !== this.props.layout) {
       this.handleLayoutChange.cancel();
-      this.props.dispatch(changeLayout(layout));
+      this.props.dispatch(changeLayout({ layout }));
     } else {
       this.handleLayoutChange();
     }

app/javascript/mastodon/features/video/index.jsx

@@ -2,7 +2,7 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
 import { is } from 'immutable';
-import { throttle, debounce } from 'lodash';
+import { throttle } from 'lodash';
 import classNames from 'classnames';
 import { isFullscreen, requestFullscreen, exitFullscreen } from '../ui/util/fullscreen';
 import { displayMedia, useBlurhash } from '../../initial_state';
@@ -102,8 +102,6 @@ class Video extends React.PureComponent {
     src: PropTypes.string.isRequired,
     alt: PropTypes.string,
     lang: PropTypes.string,
-    width: PropTypes.number,
-    height: PropTypes.number,
     sensitive: PropTypes.bool,
     currentTime: PropTypes.number,
     onOpenVideo: PropTypes.func,
@@ -112,7 +110,6 @@ class Video extends React.PureComponent {
     inline: PropTypes.bool,
     editable: PropTypes.bool,
     alwaysVisible: PropTypes.bool,
-    cacheWidth: PropTypes.func,
     visible: PropTypes.bool,
     onToggleVisibility: PropTypes.func,
     deployPictureInPicture: PropTypes.func,
@@ -135,7 +132,6 @@ class Video extends React.PureComponent {
     volume: 0.5,
     paused: true,
     dragging: false,
-    containerWidth: this.props.width,
     fullscreen: false,
     hovered: false,
     muted: false,
@@ -144,24 +140,8 @@ class Video extends React.PureComponent {
 
   setPlayerRef = c => {
     this.player = c;
-
-    if (this.player) {
-      this._setDimensions();
-    }
   };
 
-  _setDimensions () {
-    const width = this.player.offsetWidth;
-
-    if (this.props.cacheWidth) {
-      this.props.cacheWidth(width);
-    }
-
-    this.setState({
-      containerWidth: width,
-    });
-  }
-
   setVideoRef = c => {
     this.video = c;
 
@@ -370,12 +350,10 @@ class Video extends React.PureComponent {
     document.addEventListener('MSFullscreenChange', this.handleFullscreenChange, true);
 
     window.addEventListener('scroll', this.handleScroll);
-    window.addEventListener('resize', this.handleResize, { passive: true });
   }
 
   componentWillUnmount () {
     window.removeEventListener('scroll', this.handleScroll);
-    window.removeEventListener('resize', this.handleResize);
 
     document.removeEventListener('fullscreenchange', this.handleFullscreenChange, true);
     document.removeEventListener('webkitfullscreenchange', this.handleFullscreenChange, true);
@@ -404,14 +382,6 @@ class Video extends React.PureComponent {
     }
   }
 
-  handleResize = debounce(() => {
-    if (this.player) {
-      this._setDimensions();
-    }
-  }, 250, {
-    trailing: true,
-  });
-
   handleScroll = throttle(() => {
     if (!this.video) {
       return;
@@ -525,17 +495,12 @@ class Video extends React.PureComponent {
 
   render () {
     const { preview, src, inline, onOpenVideo, onCloseVideo, intl, alt, lang, detailed, sensitive, editable, blurhash, autoFocus } = this.props;
-    const { containerWidth, currentTime, duration, volume, buffer, dragging, paused, fullscreen, hovered, muted, revealed } = this.state;
+    const { currentTime, duration, volume, buffer, dragging, paused, fullscreen, hovered, muted, revealed } = this.state;
     const progress = Math.min((currentTime / duration) * 100, 100);
     const playerStyle = {};
 
-    let { width, height } = this.props;
-
-    if (inline && containerWidth) {
-      width  = containerWidth;
-      height = containerWidth / (16/9);
-
-      playerStyle.height = height;
+    if (inline) {
+      playerStyle.aspectRatio = '16 / 9';
     }
 
     let preload;
@@ -586,8 +551,6 @@ class Video extends React.PureComponent {
           aria-label={alt}
           title={alt}
           lang={lang}
-          width={width}
-          height={height}
           volume={volume}
           onClick={this.togglePlay}
           onKeyDown={this.handleVideoKeyDown}
@@ -596,6 +559,7 @@ class Video extends React.PureComponent {
           onLoadedData={this.handleLoadedData}
           onProgress={this.handleProgress}
           onVolumeChange={this.handleVolumeChange}
+          style={{ ...playerStyle, width: '100%' }}
         />}
 
         <div className={classNames('spoiler-button', { 'spoiler-button--hidden': revealed || editable })}>

app/javascript/mastodon/is_mobile.ts

@@ -1,21 +1,12 @@
-// @ts-check
-
 import { supportsPassiveEvents } from 'detect-passive-events';
-// @ts-expect-error
-import { forceSingleColumn } from 'mastodon/initial_state';
+import { forceSingleColumn } from './initial_state';
 
 const LAYOUT_BREAKPOINT = 630;
 
-/**
- * @param {number} width
- * @returns {boolean}
- */
-export const isMobile = width => width <= LAYOUT_BREAKPOINT;
+export const isMobile = (width: number) => width <= LAYOUT_BREAKPOINT;
 
-/**
- * @returns {string}
- */
-export const layoutFromWindow = () => {
+export type LayoutType = 'mobile' | 'single-column' | 'multi-column';
+export const layoutFromWindow = (): LayoutType => {
   if (isMobile(window.innerWidth)) {
     return 'mobile';
   } else if (forceSingleColumn) {
@@ -25,8 +16,9 @@ export const layoutFromWindow = () => {
   }
 };
 
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-expect-error
-const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream;
+const iOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && window.MSStream != null;
 
 const listenerOptions = supportsPassiveEvents ? { passive: true } : false;
 

app/javascript/mastodon/main.jsx

@@ -1,7 +1,8 @@
 import React from 'react';
 import ReactDOM from 'react-dom';
 import { setupBrowserNotifications } from 'mastodon/actions/notifications';
-import Mastodon, { store } from 'mastodon/containers/mastodon';
+import Mastodon from 'mastodon/containers/mastodon';
+import { store } from 'mastodon/store/configureStore';
 import { me } from 'mastodon/initial_state';
 import ready from 'mastodon/ready';
 

app/javascript/mastodon/reducers/meta.js

@@ -1,5 +1,5 @@
 import { STORE_HYDRATE } from 'mastodon/actions/store';
-import { APP_LAYOUT_CHANGE } from 'mastodon/actions/app';
+import { changeLayout } from 'mastodon/actions/app';
 import { Map as ImmutableMap } from 'immutable';
 import { layoutFromWindow } from 'mastodon/is_mobile';
 
@@ -14,8 +14,8 @@ export default function meta(state = initialState, action) {
   switch(action.type) {
   case STORE_HYDRATE:
     return state.merge(action.state.get('meta')).set('permissions', action.state.getIn(['role', 'permissions']));
-  case APP_LAYOUT_CHANGE:
-    return state.set('layout', action.layout);
+  case changeLayout.type:
+    return state.set('layout', action.payload.layout);
   default:
     return state;
   }

app/javascript/mastodon/reducers/missed_updates.js

@@ -1,21 +0,0 @@
-import { Map as ImmutableMap } from 'immutable';
-import { NOTIFICATIONS_UPDATE } from 'mastodon/actions/notifications';
-import { APP_FOCUS, APP_UNFOCUS } from 'mastodon/actions/app';
-
-const initialState = ImmutableMap({
-  focused: true,
-  unread: 0,
-});
-
-export default function missed_updates(state = initialState, action) {
-  switch(action.type) {
-  case APP_FOCUS:
-    return state.set('focused', true).set('unread', 0);
-  case APP_UNFOCUS:
-    return state.set('focused', false);
-  case NOTIFICATIONS_UPDATE:
-    return state.get('focused') ? state : state.update('unread', x => x + 1);
-  default:
-    return state;
-  }
-}

app/javascript/mastodon/reducers/missed_updates.ts

@@ -0,0 +1,31 @@
+import { Record } from 'immutable';
+import type { Action } from 'redux';
+import { NOTIFICATIONS_UPDATE } from '../actions/notifications';
+import { focusApp, unfocusApp } from '../actions/app';
+
+type MissedUpdatesState = {
+  focused: boolean;
+  unread: number;
+};
+const initialState = Record<MissedUpdatesState>({
+  focused: true,
+  unread: 0,
+})();
+
+export default function missed_updates(
+  state = initialState,
+  action: Action<string>,
+) {
+  switch (action.type) {
+  case focusApp.type:
+    return state.set('focused', true).set('unread', 0);
+  case unfocusApp.type:
+    return state.set('focused', false);
+  case NOTIFICATIONS_UPDATE:
+    return state.get('focused')
+      ? state
+      : state.update('unread', (x) => x + 1);
+  default:
+    return state;
+  }
+}

app/javascript/mastodon/reducers/notifications.js

@@ -23,8 +23,8 @@ import {
   MARKERS_FETCH_SUCCESS,
 } from '../actions/markers';
 import {
-  APP_FOCUS,
-  APP_UNFOCUS,
+  focusApp,
+  unfocusApp,
 } from '../actions/app';
 import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from '../actions/timelines';
@@ -258,9 +258,9 @@ export default function notifications(state = initialState, action) {
     return updateMounted(state);
   case NOTIFICATIONS_UNMOUNT:
     return state.update('mounted', count => count - 1);
-  case APP_FOCUS:
+  case focusApp.type:
     return updateVisibility(state, true);
-  case APP_UNFOCUS:
+  case unfocusApp.type:
     return updateVisibility(state, false);
   case NOTIFICATIONS_LOAD_PENDING:
     return state.update('items', list => state.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0);

app/javascript/mastodon/scroll.ts

@@ -1,6 +1,5 @@
-const easingOutQuint = (x, t, b, c, d) => c * ((t = t / d - 1) * t * t * t * t + 1) + b;
-
-const scroll = (node, key, target) => {
+const easingOutQuint = (x: number, t: number, b: number, c: number, d: number) => c * ((t = t / d - 1) * t * t * t * t + 1) + b;
+const scroll = (node: Element, key: 'scrollTop' | 'scrollLeft', target: number) => {
   const startTime = Date.now();
   const offset    = node[key];
   const gap       = target - offset;
@@ -28,5 +27,5 @@ const scroll = (node, key, target) => {
 
 const isScrollBehaviorSupported = 'scrollBehavior' in document.documentElement.style;
 
-export const scrollRight = (node, position) => isScrollBehaviorSupported ? node.scrollTo({ left: position, behavior: 'smooth' }) : scroll(node, 'scrollLeft', position);
-export const scrollTop = (node) => isScrollBehaviorSupported ? node.scrollTo({ top: 0, behavior: 'smooth' }) : scroll(node, 'scrollTop', 0);
+export const scrollRight = (node: Element, position: number) => isScrollBehaviorSupported ? node.scrollTo({ left: position, behavior: 'smooth' }) : scroll(node, 'scrollLeft', position);
+export const scrollTop = (node: Element) => isScrollBehaviorSupported ? node.scrollTo({ top: 0, behavior: 'smooth' }) : scroll(node, 'scrollTop', 0);

app/javascript/mastodon/store/configureStore.js

@@ -1,15 +1,16 @@
-import { createStore, applyMiddleware, compose } from 'redux';
+import { configureStore } from '@reduxjs/toolkit';
 import thunk from 'redux-thunk';
 import appReducer from '../reducers';
 import loadingBarMiddleware from '../middleware/loading_bar';
 import errorsMiddleware from '../middleware/errors';
 import soundsMiddleware from '../middleware/sounds';
 
-export default function configureStore() {
-  return createStore(appReducer, compose(applyMiddleware(
+export const store = configureStore({
+  reducer: appReducer,
+  middleware: [
     thunk,
     loadingBarMiddleware({ promiseTypeSuffixes: ['REQUEST', 'SUCCESS', 'FAIL'] }),
     errorsMiddleware(),
     soundsMiddleware(),
-  ), window.__REDUX_DEVTOOLS_EXTENSION__ ? window.__REDUX_DEVTOOLS_EXTENSION__() : f => f));
-}
+  ],
+});

app/javascript/mastodon/utils/base64.ts

@@ -1,4 +1,4 @@
-export const decode = base64 => {
+export const decode = (base64: string): Uint8Array => {
   const rawData = window.atob(base64);
   const outputArray = new Uint8Array(rawData.length);
 

app/javascript/mastodon/utils/filters.ts

@@ -1,4 +1,4 @@
-export const toServerSideType = columnType => {
+export const toServerSideType = (columnType: string) => {
   switch (columnType) {
   case 'home':
   case 'notifications':

app/javascript/mastodon/utils/numbers.ts

@@ -1,23 +1,19 @@
-// @ts-check
+import type { ValueOf } from '../../types/util';
 
 export const DECIMAL_UNITS = Object.freeze({
   ONE: 1,
   TEN: 10,
-  HUNDRED: Math.pow(10, 2),
-  THOUSAND: Math.pow(10, 3),
-  MILLION: Math.pow(10, 6),
-  BILLION: Math.pow(10, 9),
-  TRILLION: Math.pow(10, 12),
+  HUNDRED: 100,
+  THOUSAND: 1_000,
+  MILLION: 1_000_000,
+  BILLION: 1_000_000_000,
+  TRILLION: 1_000_000_000_000,
 });
+export type DecimalUnits = ValueOf<typeof DECIMAL_UNITS>;
 
 const TEN_THOUSAND = DECIMAL_UNITS.THOUSAND * 10;
 const TEN_MILLIONS = DECIMAL_UNITS.MILLION * 10;
 
-/**
- * @typedef {[number, number, number]} ShortNumber
- * Array of: shorten number, unit of shorten number and maximum fraction digits
- */
-
 /**
  * @param {number} sourceNumber Number to convert to short number
  * @returns {ShortNumber} Calculated short number
@@ -25,7 +21,8 @@ const TEN_MILLIONS = DECIMAL_UNITS.MILLION * 10;
  * shortNumber(5936);
  * // => [5.936, 1000, 1]
  */
-export function toShortNumber(sourceNumber) {
+export type ShortNumber = [number, DecimalUnits, 0 | 1] // Array of: shorten number, unit of shorten number and maximum fraction digits
+export function toShortNumber(sourceNumber: number): ShortNumber {
   if (sourceNumber < DECIMAL_UNITS.THOUSAND) {
     return [sourceNumber, DECIMAL_UNITS.ONE, 0];
   } else if (sourceNumber < DECIMAL_UNITS.MILLION) {
@@ -59,20 +56,16 @@ export function toShortNumber(sourceNumber) {
  * pluralReady(1793, DECIMAL_UNITS.THOUSAND)
  * // => 1790
  */
-export function pluralReady(sourceNumber, division) {
+export function pluralReady(sourceNumber: number, division: DecimalUnits): number {
   if (division == null || division < DECIMAL_UNITS.HUNDRED) {
     return sourceNumber;
   }
 
-  let closestScale = division / DECIMAL_UNITS.TEN;
+  const closestScale = division / DECIMAL_UNITS.TEN;
 
   return Math.trunc(sourceNumber / closestScale) * closestScale;
 }
 
-/**
- * @param {number} num
- * @returns {number}
- */
-export function roundTo10(num) {
+export function roundTo10(num: number): number {
   return Math.round(num * 0.1) / 0.1;
 }

app/javascript/styles/mastodon/components.scss

@@ -1784,7 +1784,6 @@ a.account__display-name {
 .status__avatar {
   width: 46px;
   height: 46px;
-  box-shadow: 0 0 0 2px $ui-base-color;
 }
 
 .muted {
@@ -3110,6 +3109,10 @@ $ui-header-height: 55px;
 }
 
 .compose-form__highlightable {
+  display: flex;
+  flex-direction: column;
+  overflow: hidden;
+  flex: 0 1 auto;
   border-radius: 4px;
   transition: box-shadow 300ms linear;
 
@@ -3804,6 +3807,10 @@ a.status-card {
 }
 
 .status-card-video {
+  // Firefox has a bug where frameborder=0 iframes add some extra blank space
+  // see https://bugzilla.mozilla.org/show_bug.cgi?id=155174
+  overflow: hidden;
+
   iframe {
     width: 100%;
     height: 100%;
@@ -6326,30 +6333,25 @@ a.status-card.compact:hover {
   z-index: 9999;
 }
 
-.media-gallery__gifv__label {
-  display: block;
+.media-gallery__item__badges {
   position: absolute;
-  color: $primary-text-color;
-  background: rgba($base-overlay-background, 0.5);
   bottom: 6px;
   inset-inline-start: 6px;
-  padding: 2px 6px;
-  border-radius: 2px;
-  font-size: 11px;
-  font-weight: 600;
-  z-index: 1;
-  pointer-events: none;
-  opacity: 0.9;
-  transition: opacity 0.1s ease;
-  line-height: 18px;
+  display: flex;
+  gap: 2px;
 }
 
-.media-gallery__gifv {
-  &:hover {
-    .media-gallery__gifv__label {
-      opacity: 1;
-    }
-  }
+.media-gallery__gifv__label {
+  display: block;
+  color: $white;
+  background: rgba($black, 0.65);
+  padding: 2px 6px;
+  border-radius: 4px;
+  font-size: 11px;
+  font-weight: 700;
+  z-index: 1;
+  pointer-events: none;
+  line-height: 18px;
 }
 
 .attachment-list {
@@ -6424,17 +6426,28 @@ a.status-card.compact:hover {
   position: relative;
   width: 100%;
   min-height: 64px;
+  display: grid;
+  grid-template-columns: 50% 50%;
+  grid-template-rows: 50% 50%;
+  gap: 2px;
 }
 
 .media-gallery__item {
   border: 0;
   box-sizing: border-box;
   display: block;
-  float: left;
   position: relative;
   border-radius: 4px;
   overflow: hidden;
 
+  &--tall {
+    grid-row: span 2;
+  }
+
+  &--wide {
+    grid-column: span 2;
+  }
+
   &.standalone {
     .media-gallery__item-gifv-thumbnail {
       transform: none;
@@ -8332,6 +8345,7 @@ noscript {
   font-weight: 500;
   cursor: pointer;
   color: $darker-text-color;
+  aspect-ratio: 16 / 9;
 
   i {
     display: block;

app/javascript/types/resources.ts

@@ -1,8 +1,4 @@
-interface MastodonMap<T> {
-  get<K extends keyof T>(key: K): T[K];
-  has<K extends keyof T>(key: K): boolean;
-  set<K extends keyof T>(key: K, value: T[K]): this;
-}
+import type { Record } from 'immutable';
 
 type AccountValues = {
   id: number;
@@ -10,4 +6,5 @@ type AccountValues = {
   avatar_static: string;
   [key: string]: any;
 };
-export type Account = MastodonMap<AccountValues>;
+
+export type Account = Record<AccountValues>;

app/javascript/types/util.ts

@@ -0,0 +1 @@
+export type ValueOf<T> = T[keyof T];

app/lib/activity_tracker.rb

@@ -43,7 +43,7 @@ class ActivityTracker
 
     case @type
     when :basic
-      redis.mget(*keys).map(&:to_i).sum
+      redis.mget(*keys).sum(&:to_i)
     when :unique
       redis.pfcount(*keys)
     end

app/lib/activitypub/activity/announce.rb

@@ -4,7 +4,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   def perform
     return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity?
 
-    with_lock("announce:#{value_or_id(@object)}") do
+    with_redis_lock("announce:#{value_or_id(@object)}") do
       original_status = status_from_object
 
       return reject_payload! if original_status.nil? || !announceable?(original_status)

app/lib/activitypub/activity/create.rb

@@ -47,7 +47,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def create_status
     return reject_payload! if unsupported_object_type? || non_matching_uri_hosts?(@account.uri, object_uri) || tombstone_exists? || !related_to_local_activity?
 
-    with_lock("create:#{object_uri}") do
+    with_redis_lock("create:#{object_uri}") do
       return if delete_arrived_first?(object_uri) || poll_vote?
 
       @status = find_existing_status
@@ -313,7 +313,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     poll = replied_to_status.preloadable_poll
     already_voted = true
 
-    with_lock("vote:#{replied_to_status.poll_id}:#{@account.id}") do
+    with_redis_lock("vote:#{replied_to_status.poll_id}:#{@account.id}") do
       already_voted = poll.votes.where(account: @account).exists?
       poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
     end

app/lib/activitypub/activity/delete.rb

@@ -12,7 +12,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   private
 
   def delete_person
-    with_lock("delete_in_progress:#{@account.id}", autorelease: 2.hours, raise_on_failure: false) do
+    with_redis_lock("delete_in_progress:#{@account.id}", autorelease: 2.hours, raise_on_failure: false) do
       DeleteAccountService.new.call(@account, reserve_username: false, skip_activitypub: true)
     end
   end
@@ -20,14 +20,14 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   def delete_note
     return if object_uri.nil?
 
-    with_lock("delete_status_in_progress:#{object_uri}", raise_on_failure: false) do
+    with_redis_lock("delete_status_in_progress:#{object_uri}", raise_on_failure: false) do
       unless non_matching_uri_hosts?(@account.uri, object_uri)
         # This lock ensures a concurrent `ActivityPub::Activity::Create` either
         # does not create a status at all, or has finished saving it to the
         # database before we try to load it.
         # Without the lock, `delete_later!` could be called after `delete_arrived_first?`
         # and `Status.find` before `Status.create!`
-        with_lock("create:#{object_uri}") { delete_later!(object_uri) }
+        with_redis_lock("create:#{object_uri}") { delete_later!(object_uri) }
 
         Tombstone.find_or_create_by(uri: object_uri, account: @account)
       end

app/lib/activitypub/case_transform.rb

@@ -13,7 +13,7 @@ module ActivityPub::CaseTransform
       when Symbol then camel_lower(value.to_s).to_sym
       when String
         camel_lower_cache[value] ||= if value.start_with?('_:')
-                                       "_:#{value.gsub(/\A_:/, '').underscore.camelize(:lower)}"
+                                       "_:#{value.delete_prefix('_:').underscore.camelize(:lower)}"
                                      else
                                        value.underscore.camelize(:lower)
                                      end

app/lib/feed_manager.rb

@@ -407,10 +407,10 @@ class FeedManager
     return true  if crutches[:languages][status.account_id].present? && status.language.present? && !crutches[:languages][status.account_id].include?(status.language)
 
     check_for_blocks = crutches[:active_mentions][status.id] || []
-    check_for_blocks.concat([status.account_id])
+    check_for_blocks.push(status.account_id)
 
     if status.reblog?
-      check_for_blocks.concat([status.reblog.account_id])
+      check_for_blocks.push(status.reblog.account_id)
       check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || [])
     end
 
@@ -446,7 +446,7 @@ class FeedManager
     # the notification has been checked for mute/block. Therefore, it's not
     # necessary to check the author of the toot for mute/block again
     check_for_blocks = status.active_mentions.pluck(:account_id)
-    check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
+    check_for_blocks.push(status.in_reply_to_account) if status.reply? && !status.in_reply_to_account_id.nil?
 
     should_filter   = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions)                                                         # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted)
     should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them
@@ -593,10 +593,10 @@ class FeedManager
 
     check_for_blocks = statuses.flat_map do |s|
       arr = crutches[:active_mentions][s.id] || []
-      arr.concat([s.account_id])
+      arr.push(s.account_id)
 
       if s.reblog?
-        arr.concat([s.reblog.account_id])
+        arr.push(s.reblog.account_id)
         arr.concat(crutches[:active_mentions][s.reblog_of_id] || [])
       end
 

app/lib/importer/accounts_index_importer.rb

@@ -6,8 +6,8 @@ class Importer::AccountsIndexImporter < Importer::BaseImporter
       in_work_unit(tmp) do |accounts|
         bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: accounts).bulk_body
 
-        indexed = bulk.select { |entry| entry[:index] }.size
-        deleted = bulk.select { |entry| entry[:delete] }.size
+        indexed = bulk.count { |entry| entry[:index] }
+        deleted = bulk.count { |entry| entry[:delete] }
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/importer/tags_index_importer.rb

@@ -6,8 +6,8 @@ class Importer::TagsIndexImporter < Importer::BaseImporter
       in_work_unit(tmp) do |tags|
         bulk = Chewy::Index::Import::BulkBuilder.new(index, to_index: tags).bulk_body
 
-        indexed = bulk.select { |entry| entry[:index] }.size
-        deleted = bulk.select { |entry| entry[:delete] }.size
+        indexed = bulk.count { |entry| entry[:index] }
+        deleted = bulk.count { |entry| entry[:delete] }
 
         Chewy::Index::Import::BulkRequest.new(index).perform(bulk)
 

app/lib/permalink_redirector.rb

@@ -8,21 +8,51 @@ class PermalinkRedirector
   end
 
   def redirect_path
-    if path_segments[0].present? && path_segments[0].start_with?('@') && path_segments[1] =~ /\d/
-      find_status_url_by_id(path_segments[1])
-    elsif path_segments[0].present? && path_segments[0].start_with?('@')
-      find_account_url_by_name(path_segments[0])
-    elsif path_segments[0] == 'statuses' && path_segments[1] =~ /\d/
-      find_status_url_by_id(path_segments[1])
-    elsif path_segments[0] == 'accounts' && path_segments[1] =~ /\d/
-      find_account_url_by_id(path_segments[1])
+    if at_username_status_request? || statuses_status_request?
+      find_status_url_by_id(second_segment)
+    elsif at_username_request?
+      find_account_url_by_name(first_segment)
+    elsif accounts_request? && record_integer_id_request?
+      find_account_url_by_id(second_segment)
     end
   end
 
   private
 
+  def at_username_status_request?
+    at_username_request? && record_integer_id_request?
+  end
+
+  def statuses_status_request?
+    statuses_request? && record_integer_id_request?
+  end
+
+  def at_username_request?
+    first_segment.present? && first_segment.start_with?('@')
+  end
+
+  def statuses_request?
+    first_segment == 'statuses'
+  end
+
+  def accounts_request?
+    first_segment == 'accounts'
+  end
+
+  def record_integer_id_request?
+    second_segment =~ /\d/
+  end
+
+  def first_segment
+    path_segments.first
+  end
+
+  def second_segment
+    path_segments.second
+  end
+
   def path_segments
-    @path_segments ||= @path.gsub(/\A\//, '').split('/')
+    @path_segments ||= @path.delete_prefix('/').split('/')
   end
 
   def find_status_url_by_id(id)

app/lib/vacuum/imports_vacuum.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Vacuum::ImportsVacuum
+  def perform
+    clean_unconfirmed_imports!
+    clean_old_imports!
+  end
+
+  private
+
+  def clean_unconfirmed_imports!
+    BulkImport.where(state: :unconfirmed).where('created_at <= ?', 10.minutes.ago).reorder(nil).in_batches.delete_all
+  end
+
+  def clean_old_imports!
+    BulkImport.where('created_at <= ?', 1.week.ago).reorder(nil).in_batches.delete_all
+  end
+end

app/lib/webfinger_resource.rb

@@ -57,7 +57,7 @@ class WebfingerResource
   end
 
   def resource_without_acct_string
-    resource.gsub(/\Aacct:/, '')
+    resource.delete_prefix('acct:')
   end
 
   def local_username

app/models/account.rb

@@ -78,6 +78,7 @@ class Account < ApplicationRecord
   include DomainNormalizable
   include DomainMaterializable
   include AccountMerging
+  include AccountSearch
 
   MAX_DISPLAY_NAME_LENGTH = (ENV['MAX_DISPLAY_NAME_CHARS'] || 30).to_i
   MAX_NOTE_LENGTH = (ENV['MAX_BIO_CHARS'] || 500).to_i
@@ -408,14 +409,6 @@ class Account < ApplicationRecord
   end
 
   class << self
-    DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/
-    TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"
-
-    REPUTATION_SCORE_FUNCTION = '(greatest(0, coalesce(s.followers_count, 0)) / (greatest(0, coalesce(s.following_count, 0)) + 1.0))'
-    FOLLOWERS_SCORE_FUNCTION  = 'log(greatest(0, coalesce(s.followers_count, 0)) + 2)'
-    TIME_DISTANCE_FUNCTION    = '(case when s.last_status_at is null then 0 else exp(-1.0 * ((greatest(0, abs(extract(DAY FROM age(s.last_status_at))) - 30.0)^2) / (2.0 * ((-1.0 * 30^2) / (2.0 * ln(0.3)))))) end)'
-    BOOST                     = "((#{REPUTATION_SCORE_FUNCTION} + #{FOLLOWERS_SCORE_FUNCTION} + #{TIME_DISTANCE_FUNCTION}) / 3.0)"
-
     def readonly_attributes
       super - %w(statuses_count following_count followers_count)
     end
@@ -425,37 +418,6 @@ class Account < ApplicationRecord
       DeliveryFailureTracker.without_unavailable(urls)
     end
 
-    def search_for(terms, limit: 10, offset: 0)
-      tsquery = generate_query_for_search(terms)
-
-      sql = <<-SQL.squish
-        SELECT
-          accounts.*,
-          #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
-        FROM accounts
-        LEFT JOIN users ON accounts.id = users.account_id
-        LEFT JOIN account_stats AS s ON accounts.id = s.account_id
-        WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
-          AND accounts.suspended_at IS NULL
-          AND accounts.moved_to_account_id IS NULL
-          AND (accounts.domain IS NOT NULL OR (users.approved = TRUE AND users.confirmed_at IS NOT NULL))
-        ORDER BY rank DESC
-        LIMIT :limit OFFSET :offset
-      SQL
-
-      records = find_by_sql([sql, limit: limit, offset: offset, tsquery: tsquery])
-      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
-      records
-    end
-
-    def advanced_search_for(terms, account, limit: 10, following: false, offset: 0)
-      tsquery = generate_query_for_search(terms)
-      sql = advanced_search_for_sql_template(following)
-      records = find_by_sql([sql, id: account.id, limit: limit, offset: offset, tsquery: tsquery])
-      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
-      records
-    end
-
     def from_text(text)
       return [] if text.blank?
 
@@ -469,73 +431,15 @@ class Account < ApplicationRecord
         EntityCache.instance.mention(username, domain)
       end
     end
-
-    private
-
-    def generate_query_for_search(unsanitized_terms)
-      terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ')
-
-      # The final ":*" is for prefix search.
-      # The trailing space does not seem to fit any purpose, but `to_tsquery`
-      # behaves differently with and without a leading space if the terms start
-      # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep
-      # the same query.
-      "' #{terms} ':*"
-    end
-
-    def advanced_search_for_sql_template(following)
-      if following
-        <<-SQL.squish
-          WITH first_degree AS (
-            SELECT target_account_id
-            FROM follows
-            WHERE account_id = :id
-            UNION ALL
-            SELECT :id
-          )
-          SELECT
-            accounts.*,
-            (count(f.id) + 1) * #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id)
-          LEFT JOIN account_stats AS s ON accounts.id = s.account_id
-          WHERE accounts.id IN (SELECT * FROM first_degree)
-            AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-          GROUP BY accounts.id, s.id
-          ORDER BY rank DESC
-          LIMIT :limit OFFSET :offset
-        SQL
-      else
-        <<-SQL.squish
-          SELECT
-            accounts.*,
-            #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank,
-            count(f.id) AS followed
-          FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id)
-          LEFT JOIN users ON accounts.id = users.account_id
-          LEFT JOIN account_stats AS s ON accounts.id = s.account_id
-          WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH}
-            AND accounts.suspended_at IS NULL
-            AND accounts.moved_to_account_id IS NULL
-            AND (accounts.domain IS NOT NULL OR (users.approved = TRUE AND users.confirmed_at IS NOT NULL))
-          GROUP BY accounts.id, s.id
-          ORDER BY followed DESC, rank DESC
-          LIMIT :limit OFFSET :offset
-        SQL
-      end
-    end
   end
 
   def emojis
     @emojis ||= CustomEmoji.from_text(emojifiable_text, domain)
   end
 
-  before_create :generate_keys
   before_validation :prepare_contents, if: :local?
   before_validation :prepare_username, on: :create
+  before_create :generate_keys
   before_destroy :clean_feed_manager
 
   def ensure_keys!

app/models/account_conversation.rb

@@ -17,14 +17,13 @@
 class AccountConversation < ApplicationRecord
   include Redisable
 
+  before_validation :set_last_status
   after_commit :push_to_streaming_api
 
   belongs_to :account
   belongs_to :conversation
   belongs_to :last_status, class_name: 'Status'
 
-  before_validation :set_last_status
-
   def participant_account_ids=(arr)
     self[:participant_account_ids] = arr.sort
   end

app/models/account_migration.rb

@@ -42,7 +42,7 @@ class AccountMigration < ApplicationRecord
 
     return false unless errors.empty?
 
-    with_lock("account_migration:#{account.id}") do
+    with_redis_lock("account_migration:#{account.id}") do
       save
     end
   end

app/models/account_statuses_filter.rb

@@ -32,9 +32,9 @@ class AccountStatusesFilter
   private
 
   def initial_scope
-    if suspended?
-      Status.none
-    elsif anonymous?
+    return Status.none if suspended?
+
+    if anonymous?
       account.statuses.not_local_only.where(visibility: %i(public unlisted))
     elsif author?
       account.statuses.all # NOTE: #merge! does not work without the #all

app/models/account_suggestions/source.rb

@@ -18,7 +18,7 @@ class AccountSuggestions::Source
   def as_ordered_suggestions(scope, ordered_list)
     return [] if ordered_list.empty?
 
-    map = scope.index_by(&method(:to_ordered_list_key))
+    map = scope.index_by { |account| to_ordered_list_key(account) }
 
     ordered_list.map { |ordered_list_key| map[ordered_list_key] }.compact.map do |account|
       AccountSuggestions::Suggestion.new(

app/models/admin/appeal_filter.rb

@@ -5,6 +5,8 @@ class Admin::AppealFilter
     status
   ).freeze
 
+  IGNORED_PARAMS = %w(page).freeze
+
   attr_reader :params
 
   def initialize(params)
@@ -15,7 +17,7 @@ class Admin::AppealFilter
     scope = Appeal.order(id: :desc)
 
     params.each do |key, value|
-      next if %w(page).include?(key.to_s)
+      next if IGNORED_PARAMS.include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end

app/models/admin/status_filter.rb

@@ -6,6 +6,8 @@ class Admin::StatusFilter
     report_id
   ).freeze
 
+  IGNORED_PARAMS = %w(page report_id).freeze
+
   attr_reader :params
 
   def initialize(account, params)
@@ -17,7 +19,7 @@ class Admin::StatusFilter
     scope = @account.statuses.where(visibility: [:public, :unlisted])
 
     params.each do |key, value|
-      next if %w(page report_id).include?(key.to_s)
+      next if IGNORED_PARAMS.include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end

app/models/announcement_reaction.rb

@@ -14,6 +14,7 @@
 #
 
 class AnnouncementReaction < ApplicationRecord
+  before_validation :set_custom_emoji
   after_commit :queue_publish
 
   belongs_to :account
@@ -23,8 +24,6 @@ class AnnouncementReaction < ApplicationRecord
   validates :name, presence: true
   validates_with ReactionValidator
 
-  before_validation :set_custom_emoji
-
   private
 
   def set_custom_emoji

app/models/block.rb

@@ -25,8 +25,8 @@ class Block < ApplicationRecord
     false # Force uri_for to use uri attribute
   end
 
-  after_commit :remove_blocking_cache
   before_validation :set_uri, only: :create
+  after_commit :remove_blocking_cache
 
   private
 

app/models/bulk_import.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: bulk_imports
+#
+#  id                :bigint(8)        not null, primary key
+#  type              :integer          not null
+#  state             :integer          not null
+#  total_items       :integer          default(0), not null
+#  imported_items    :integer          default(0), not null
+#  processed_items   :integer          default(0), not null
+#  finished_at       :datetime
+#  overwrite         :boolean          default(FALSE), not null
+#  likely_mismatched :boolean          default(FALSE), not null
+#  original_filename :string           default(""), not null
+#  account_id        :bigint(8)        not null
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#
+class BulkImport < ApplicationRecord
+  self.inheritance_column = false
+
+  belongs_to :account
+  has_many :rows, class_name: 'BulkImportRow', inverse_of: :bulk_import, dependent: :delete_all
+
+  enum type: {
+    following: 0,
+    blocking: 1,
+    muting: 2,
+    domain_blocking: 3,
+    bookmarks: 4,
+  }
+
+  enum state: {
+    unconfirmed: 0,
+    scheduled: 1,
+    in_progress: 2,
+    finished: 3,
+  }
+
+  validates :type, presence: true
+
+  def self.progress!(bulk_import_id, imported: false)
+    # Use `increment_counter` so that the incrementation is done atomically in the database
+    BulkImport.increment_counter(:processed_items, bulk_import_id) # rubocop:disable Rails/SkipsModelValidations
+    BulkImport.increment_counter(:imported_items, bulk_import_id) if imported # rubocop:disable Rails/SkipsModelValidations
+
+    # Since the incrementation has been done atomically, concurrent access to `bulk_import` is now bening
+    bulk_import = BulkImport.find(bulk_import_id)
+    bulk_import.update!(state: :finished, finished_at: Time.now.utc) if bulk_import.processed_items == bulk_import.total_items
+  end
+end

app/models/bulk_import_row.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: bulk_import_rows
+#
+#  id             :bigint(8)        not null, primary key
+#  bulk_import_id :bigint(8)        not null
+#  data           :jsonb
+#  created_at     :datetime         not null
+#  updated_at     :datetime         not null
+#
+class BulkImportRow < ApplicationRecord
+  belongs_to :bulk_import
+end

app/models/concerns/account_associations.rb

@@ -68,5 +68,8 @@ module AccountAssociations
 
     # Account statuses cleanup policy
     has_one :statuses_cleanup_policy, class_name: 'AccountStatusesCleanupPolicy', inverse_of: :account, dependent: :destroy
+
+    # Imports
+    has_many :bulk_imports, inverse_of: :account, dependent: :delete_all
   end
 end

app/models/concerns/account_interactions.rb

@@ -271,7 +271,8 @@ module AccountInteractions
   end
 
   def lists_for_local_distribution
-    lists.joins(account: :user)
+    scope = lists.joins(account: :user)
+    scope.where.not(list_accounts: { follow_id: nil }).or(scope.where(account_id: id))
          .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago)
   end
 

app/models/concerns/account_search.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module AccountSearch
+  extend ActiveSupport::Concern
+
+  DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/
+
+  TEXT_SEARCH_RANKS = <<~SQL.squish
+    (
+        setweight(to_tsvector('simple', accounts.display_name), 'A') ||
+        setweight(to_tsvector('simple', accounts.username), 'B') ||
+        setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C')
+    )
+  SQL
+
+  REPUTATION_SCORE_FUNCTION = <<~SQL.squish
+    (
+        greatest(0, coalesce(s.followers_count, 0)) / (
+            greatest(0, coalesce(s.following_count, 0)) + 1.0
+        )
+    )
+  SQL
+
+  FOLLOWERS_SCORE_FUNCTION = <<~SQL.squish
+    log(
+        greatest(0, coalesce(s.followers_count, 0)) + 2
+    )
+  SQL
+
+  TIME_DISTANCE_FUNCTION = <<~SQL.squish
+    (
+        case
+            when s.last_status_at is null then 0
+            else exp(
+                -1.0 * (
+                    (
+                        greatest(0, abs(extract(DAY FROM age(s.last_status_at))) - 30.0)^2) /#{' '}
+                        (2.0 * ((-1.0 * 30^2) / (2.0 * ln(0.3)))
+                    )
+                )
+            )
+        end
+    )
+  SQL
+
+  BOOST = <<~SQL.squish
+    (
+        (#{REPUTATION_SCORE_FUNCTION} + #{FOLLOWERS_SCORE_FUNCTION} + #{TIME_DISTANCE_FUNCTION}) / 3.0
+    )
+  SQL
+
+  BASIC_SEARCH_SQL = <<~SQL.squish
+    SELECT
+      accounts.*,
+      #{BOOST} * ts_rank_cd(#{TEXT_SEARCH_RANKS}, to_tsquery('simple', :tsquery), 32) AS rank
+    FROM accounts
+    LEFT JOIN users ON accounts.id = users.account_id
+    LEFT JOIN account_stats AS s ON accounts.id = s.account_id
+    WHERE to_tsquery('simple', :tsquery) @@ #{TEXT_SEARCH_RANKS}
+      AND accounts.suspended_at IS NULL
+      AND accounts.moved_to_account_id IS NULL
+      AND (accounts.domain IS NOT NULL OR (users.approved = TRUE AND users.confirmed_at IS NOT NULL))
+    ORDER BY rank DESC
+    LIMIT :limit OFFSET :offset
+  SQL
+
+  ADVANCED_SEARCH_WITH_FOLLOWING = <<~SQL.squish
+    WITH first_degree AS (
+      SELECT target_account_id
+      FROM follows
+      WHERE account_id = :id
+      UNION ALL
+      SELECT :id
+    )
+    SELECT
+      accounts.*,
+      (count(f.id) + 1) * #{BOOST} * ts_rank_cd(#{TEXT_SEARCH_RANKS}, to_tsquery('simple', :tsquery), 32) AS rank
+    FROM accounts
+    LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id)
+    LEFT JOIN account_stats AS s ON accounts.id = s.account_id
+    WHERE accounts.id IN (SELECT * FROM first_degree)
+      AND to_tsquery('simple', :tsquery) @@ #{TEXT_SEARCH_RANKS}
+      AND accounts.suspended_at IS NULL
+      AND accounts.moved_to_account_id IS NULL
+    GROUP BY accounts.id, s.id
+    ORDER BY rank DESC
+    LIMIT :limit OFFSET :offset
+  SQL
+
+  ADVANCED_SEARCH_WITHOUT_FOLLOWING = <<~SQL.squish
+    SELECT
+      accounts.*,
+      #{BOOST} * ts_rank_cd(#{TEXT_SEARCH_RANKS}, to_tsquery('simple', :tsquery), 32) AS rank,
+      count(f.id) AS followed
+    FROM accounts
+    LEFT OUTER JOIN follows AS f ON
+      (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id)
+    LEFT JOIN users ON accounts.id = users.account_id
+    LEFT JOIN account_stats AS s ON accounts.id = s.account_id
+    WHERE to_tsquery('simple', :tsquery) @@ #{TEXT_SEARCH_RANKS}
+      AND accounts.suspended_at IS NULL
+      AND accounts.moved_to_account_id IS NULL
+      AND (accounts.domain IS NOT NULL OR (users.approved = TRUE AND users.confirmed_at IS NOT NULL))
+    GROUP BY accounts.id, s.id
+    ORDER BY followed DESC, rank DESC
+    LIMIT :limit OFFSET :offset
+  SQL
+
+  class_methods do
+    def search_for(terms, limit: 10, offset: 0)
+      tsquery = generate_query_for_search(terms)
+
+      find_by_sql([BASIC_SEARCH_SQL, { limit: limit, offset: offset, tsquery: tsquery }]).tap do |records|
+        ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      end
+    end
+
+    def advanced_search_for(terms, account, limit: 10, following: false, offset: 0)
+      tsquery = generate_query_for_search(terms)
+      sql_template = following ? ADVANCED_SEARCH_WITH_FOLLOWING : ADVANCED_SEARCH_WITHOUT_FOLLOWING
+
+      find_by_sql([sql_template, { id: account.id, limit: limit, offset: offset, tsquery: tsquery }]).tap do |records|
+        ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      end
+    end
+
+    private
+
+    def generate_query_for_search(unsanitized_terms)
+      terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ')
+
+      # The final ":*" is for prefix search.
+      # The trailing space does not seem to fit any purpose, but `to_tsquery`
+      # behaves differently with and without a leading space if the terms start
+      # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep
+      # the same query.
+      "' #{terms} ':*"
+    end
+  end
+end

app/models/concerns/lockable.rb

@@ -5,7 +5,7 @@ module Lockable
   # @param [ActiveSupport::Duration] autorelease Automatically release the lock after this time
   # @param [Boolean] raise_on_failure Raise an error if a lock cannot be acquired, or fail silently
   # @raise [Mastodon::RaceConditionError]
-  def with_lock(lock_name, autorelease: 15.minutes, raise_on_failure: true)
+  def with_redis_lock(lock_name, autorelease: 15.minutes, raise_on_failure: true)
     with_redis do |redis|
       RedisLock.acquire(redis: redis, key: "lock:#{lock_name}", autorelease: autorelease.seconds) do |lock|
         if lock.acquired?

app/models/concerns/status_safe_reblog_insert.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module StatusSafeReblogInsert
+  extend ActiveSupport::Concern
+
+  class_methods do
+    # This is a hack to ensure that no reblogs of discarded statuses are created,
+    # as this cannot be enforced through database constraints the same way we do
+    # for reblogs of deleted statuses.
+    #
+    # To achieve this, we redefine the internal method responsible for issuing
+    # the "INSERT" statement and replace the "INSERT INTO ... VALUES ..." query
+    # with an "INSERT INTO ... SELECT ..." query with a "WHERE deleted_at IS NULL"
+    # clause on the reblogged status to ensure consistency at the database level.
+    #
+    # Otherwise, the code is kept as close as possible to ActiveRecord::Persistence
+    # code, and actually calls it if we are not handling a reblog.
+    def _insert_record(values)
+      return super unless values.is_a?(Hash) && values['reblog_of_id'].present?
+
+      primary_key = self.primary_key
+      primary_key_value = nil
+
+      if primary_key
+        primary_key_value = values[primary_key]
+
+        if !primary_key_value && prefetch_primary_key?
+          primary_key_value = next_sequence_value
+          values[primary_key] = primary_key_value
+        end
+      end
+
+      # The following line is where we differ from stock ActiveRecord implementation
+      im = _compile_reblog_insert(values)
+
+      # Since we are using SELECT instead of VALUES, a non-error `nil` return is possible.
+      # For our purposes, it's equivalent to a foreign key constraint violation
+      result = connection.insert(im, "#{self} Create", primary_key || false, primary_key_value)
+      raise ActiveRecord::InvalidForeignKey, "(reblog_of_id)=(#{values['reblog_of_id']}) is not present in table \"statuses\"" if result.nil?
+
+      result
+    end
+
+    def _compile_reblog_insert(values)
+      # This is somewhat equivalent to the following code of ActiveRecord::Persistence:
+      # `arel_table.compile_insert(_substitute_values(values))`
+      # The main difference is that we use a `SELECT` instead of a `VALUES` clause,
+      # which means we have to build the `SELECT` clause ourselves and do a bit more
+      # manual work.
+
+      # Instead of using Arel::InsertManager#values, we are going to use Arel::InsertManager#select
+      im = Arel::InsertManager.new
+      im.into(arel_table)
+
+      binds = []
+      reblog_bind = nil
+      values.each do |name, value|
+        attr = arel_table[name]
+        bind = predicate_builder.build_bind_attribute(attr.name, value)
+
+        im.columns << attr
+        binds << bind
+
+        reblog_bind = bind if name == 'reblog_of_id'
+      end
+
+      im.select(arel_table.where(arel_table[:id].eq(reblog_bind)).where(arel_table[:deleted_at].eq(nil)).project(*binds))
+
+      im
+    end
+  end
+end

app/models/follow_request.rb

@@ -32,7 +32,8 @@ class FollowRequest < ApplicationRecord
   validates :languages, language: true
 
   def authorize!
-    account.follow!(target_account, reblogs: show_reblogs, notify: notify, languages: languages, uri: uri, bypass_limit: true)
+    follow = account.follow!(target_account, reblogs: show_reblogs, notify: notify, languages: languages, uri: uri, bypass_limit: true)
+    ListAccount.where(follow_request: self).update_all(follow_request_id: nil, follow_id: follow.id) # rubocop:disable Rails/SkipsModelValidations
     MergeWorker.perform_async(target_account.id, account.id) if account.local?
     destroy!
   end

app/models/form/import.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+# A non-ActiveRecord helper class for CSV uploads.
+# Handles saving contents to database.
+class Form::Import
+  include ActiveModel::Model
+
+  MODES = %i(merge overwrite).freeze
+
+  FILE_SIZE_LIMIT       = 20.megabytes
+  ROWS_PROCESSING_LIMIT = 20_000
+
+  EXPECTED_HEADERS_BY_TYPE = {
+    following: ['Account address', 'Show boosts', 'Notify on new posts', 'Languages'],
+    blocking: ['Account address'],
+    muting: ['Account address', 'Hide notifications'],
+    domain_blocking: ['#domain'],
+    bookmarks: ['#uri'],
+  }.freeze
+
+  KNOWN_FIRST_HEADERS = EXPECTED_HEADERS_BY_TYPE.values.map(&:first).uniq.freeze
+
+  ATTRIBUTE_BY_HEADER = {
+    'Account address' => 'acct',
+    'Show boosts' => 'show_reblogs',
+    'Notify on new posts' => 'notify',
+    'Languages' => 'languages',
+    'Hide notifications' => 'hide_notifications',
+    '#domain' => 'domain',
+    '#uri' => 'uri',
+  }.freeze
+
+  class EmptyFileError < StandardError; end
+
+  attr_accessor :current_account, :data, :type, :overwrite, :bulk_import
+
+  validates :type, presence: true
+  validates :data, presence: true
+  validate :validate_data
+
+  def guessed_type
+    return :muting if csv_data.headers.include?('Hide notifications')
+    return :following if csv_data.headers.include?('Show boosts') || csv_data.headers.include?('Notify on new posts') || csv_data.headers.include?('Languages')
+    return :following if data.original_filename&.start_with?('follows') || data.original_filename&.start_with?('following_accounts')
+    return :blocking if data.original_filename&.start_with?('blocks') || data.original_filename&.start_with?('blocked_accounts')
+    return :muting if data.original_filename&.start_with?('mutes') || data.original_filename&.start_with?('muted_accounts')
+    return :domain_blocking if data.original_filename&.start_with?('domain_blocks') || data.original_filename&.start_with?('blocked_domains')
+    return :bookmarks if data.original_filename&.start_with?('bookmarks')
+  end
+
+  # Whether the uploaded CSV file seems to correspond to a different import type than the one selected
+  def likely_mismatched?
+    guessed_type.present? && guessed_type != type.to_sym
+  end
+
+  def save
+    return false unless valid?
+
+    ApplicationRecord.transaction do
+      now = Time.now.utc
+      @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
+      nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length # rubocop:disable Rails/SkipsModelValidations
+      @bulk_import.update(total_items: nb_items)
+    end
+  end
+
+  def mode
+    overwrite ? :overwrite : :merge
+  end
+
+  def mode=(str)
+    self.overwrite = str.to_sym == :overwrite
+  end
+
+  private
+
+  def default_csv_header
+    case type.to_sym
+    when :following, :blocking, :muting
+      'Account address'
+    when :domain_blocking
+      '#domain'
+    when :bookmarks
+      '#uri'
+    end
+  end
+
+  def csv_data
+    return @csv_data if defined?(@csv_data)
+
+    csv_converter = lambda do |field, field_info|
+      case field_info.header
+      when 'Show boosts', 'Notify on new posts', 'Hide notifications'
+        ActiveModel::Type::Boolean.new.cast(field)
+      when 'Languages'
+        field&.split(',')&.map(&:strip)&.presence
+      when 'Account address'
+        field.strip.gsub(/\A@/, '')
+      when '#domain', '#uri'
+        field.strip
+      else
+        field
+      end
+    end
+
+    @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: true, converters: csv_converter)
+    @csv_data.take(1) # Ensure the headers are read
+    raise EmptyFileError if @csv_data.headers == true
+
+    @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: [default_csv_header], converters: csv_converter) unless KNOWN_FIRST_HEADERS.include?(@csv_data.headers&.first)
+    @csv_data
+  end
+
+  def csv_row_count
+    return @csv_row_count if defined?(@csv_row_count)
+
+    csv_data.rewind
+    @csv_row_count = csv_data.take(ROWS_PROCESSING_LIMIT + 2).count
+  end
+
+  def parsed_rows
+    csv_data.rewind
+
+    expected_headers = EXPECTED_HEADERS_BY_TYPE[type.to_sym]
+
+    csv_data.take(ROWS_PROCESSING_LIMIT + 1).map do |row|
+      row.to_h.slice(*expected_headers).transform_keys { |key| ATTRIBUTE_BY_HEADER[key] }
+    end
+  end
+
+  def validate_data
+    return if data.nil?
+    return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
+    return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless csv_data.headers.include?(default_csv_header)
+
+    errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT
+
+    if type.to_sym == :following
+      base_limit = FollowLimitValidator.limit_for_account(current_account)
+      limit = base_limit
+      limit -= current_account.following_count unless overwrite
+      errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
+    end
+  rescue CSV::MalformedCSVError => e
+    errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
+  rescue EmptyFileError
+    errors.add(:data, I18n.t('imports.errors.empty'))
+  end
+end

app/models/import.rb

@@ -17,6 +17,9 @@
 #  overwrite         :boolean          default(FALSE), not null
 #
 
+# NOTE: This is a deprecated model, only kept to not break ongoing imports
+# on upgrade. See `BulkImport` and `Form::Import` for its replacements.
+
 class Import < ApplicationRecord
   FILE_TYPES = %w(text/plain text/csv application/csv).freeze
   MODES = %i(merge overwrite).freeze
@@ -28,7 +31,6 @@ class Import < ApplicationRecord
   enum type: { following: 0, blocking: 1, muting: 2, domain_blocking: 3, bookmarks: 4 }
 
   validates :type, presence: true
-  validates_with ImportValidator, on: :create
 
   has_attached_file :data
   validates_attachment_content_type :data, content_type: FILE_TYPES

app/models/list_account.rb

@@ -4,24 +4,39 @@
 #
 # Table name: list_accounts
 #
-#  id         :bigint(8)        not null, primary key
-#  list_id    :bigint(8)        not null
-#  account_id :bigint(8)        not null
-#  follow_id  :bigint(8)
+#  id                :bigint(8)        not null, primary key
+#  list_id           :bigint(8)        not null
+#  account_id        :bigint(8)        not null
+#  follow_id         :bigint(8)
+#  follow_request_id :bigint(8)
 #
 
 class ListAccount < ApplicationRecord
   belongs_to :list
   belongs_to :account
   belongs_to :follow, optional: true
+  belongs_to :follow_request, optional: true
 
   validates :account_id, uniqueness: { scope: :list_id }
+  validate :validate_relationship
 
   before_validation :set_follow
 
   private
 
   def set_follow
-    self.follow = Follow.find_by!(account_id: list.account_id, target_account_id: account.id) unless list.account_id == account.id
+    return if list.account_id == account.id
+
+    self.follow = Follow.find_by!(account_id: list.account_id, target_account_id: account.id)
+  rescue ActiveRecord::RecordNotFound
+    self.follow_request = FollowRequest.find_by!(account_id: list.account_id, target_account_id: account.id)
+  end
+
+  def validate_relationship
+    return if list.account_id == account_id
+
+    errors.add(:account_id, 'follow relationship missing') if follow_id.nil? && follow_request_id.nil?
+    errors.add(:follow, 'mismatched accounts') if follow_id.present? && follow.target_account_id != account_id
+    errors.add(:follow_request, 'mismatched accounts') if follow_request_id.present? && follow_request.target_account_id != account_id
   end
 end

app/models/media_attachment.rb

@@ -34,8 +34,8 @@ class MediaAttachment < ApplicationRecord
 
   include Attachmentable
 
-  enum type: { :image => 0, :gifv => 1, :video => 2, :unknown => 3, :audio => 4 }
-  enum processing: { :queued => 0, :in_progress => 1, :complete => 2, :failed => 3 }, _prefix: true
+  enum type: { image: 0, gifv: 1, video: 2, unknown: 3, audio: 4 }
+  enum processing: { queued: 0, in_progress: 1, complete: 2, failed: 3 }, _prefix: true
 
   MAX_DESCRIPTION_LENGTH = 1_500
 
@@ -135,7 +135,7 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
-          vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
+          :vf => 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
         }.freeze,
       }.freeze,
       format: 'png',
@@ -169,6 +169,8 @@ class MediaAttachment < ApplicationRecord
     original: IMAGE_STYLES[:small].freeze,
   }.freeze
 
+  DEFAULT_STYLES = [:original].freeze
+
   GLOBAL_CONVERT_OPTIONS = {
     all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date',
   }.freeze
@@ -271,12 +273,12 @@ class MediaAttachment < ApplicationRecord
     delay_processing? && attachment_name == :file
   end
 
-  after_commit :enqueue_processing, on: :create
-  after_commit :reset_parent_cache, on: :update
-
   before_create :set_unknown_type
   before_create :set_processing
 
+  after_commit :enqueue_processing, on: :create
+  after_commit :reset_parent_cache, on: :update
+
   after_post_process :set_meta
 
   class << self

app/models/relationship_filter.rb

@@ -10,6 +10,8 @@ class RelationshipFilter
     location
   ).freeze
 
+  IGNORED_PARAMS = %w(relationship page).freeze
+
   attr_reader :params, :account
 
   def initialize(account, params)
@@ -23,7 +25,7 @@ class RelationshipFilter
     scope = scope_for('relationship', params['relationship'].to_s.strip)
 
     params.each do |key, value|
-      next if %w(relationship page).include?(key)
+      next if IGNORED_PARAMS.include?(key)
 
       scope.merge!(scope_for(key.to_s, value.to_s.strip)) if value.present?
     end

app/models/session_activation.rb

@@ -36,8 +36,8 @@ class SessionActivation < ApplicationRecord
     detection.platform.id
   end
 
-  before_create :assign_access_token
   before_save   :assign_user_agent
+  before_create :assign_access_token
 
   class << self
     def active?(id)

app/models/status.rb

@@ -32,14 +32,13 @@
 #
 
 class Status < ApplicationRecord
-  before_destroy :unlink_from_conversations!
-
   include Discard::Model
   include Paginable
   include Cacheable
   include StatusThreadingConcern
   include StatusSnapshotConcern
   include RateLimitable
+  include StatusSafeReblogInsert
 
   rate_limit by: :account, family: :statuses
 
@@ -119,6 +118,28 @@ class Status < ApplicationRecord
   after_create_commit :trigger_create_webhooks
   after_update_commit :trigger_update_webhooks
 
+  after_create_commit  :increment_counter_caches
+  after_destroy_commit :decrement_counter_caches
+
+  after_create_commit :store_uri, if: :local?
+  after_create_commit :update_statistics, if: :local?
+
+  before_validation :prepare_contents, if: :local?
+  before_validation :set_reblog
+  before_validation :set_visibility
+  before_validation :set_conversation
+  before_validation :set_local
+
+  before_create :set_local_only
+
+  around_create Mastodon::Snowflake::Callbacks
+
+  after_create :set_poll_id
+
+  # The `prepend: true` option below ensures this runs before
+  # the `dependent: destroy` callbacks remove relevant records
+  before_destroy :unlink_from_conversations!, prepend: true
+
   cache_associated :application,
                    :media_attachments,
                    :conversation,
@@ -316,23 +337,6 @@ class Status < ApplicationRecord
     attributes['trendable'].nil? && account.requires_review_notification?
   end
 
-  after_create_commit  :increment_counter_caches
-  after_destroy_commit :decrement_counter_caches
-
-  after_create_commit :store_uri, if: :local?
-  after_create_commit :update_statistics, if: :local?
-
-  before_validation :prepare_contents, if: :local?
-  before_validation :set_reblog
-  before_validation :set_visibility
-  before_validation :set_conversation
-  before_validation :set_local
-  before_create :set_locality
-
-  around_create Mastodon::Snowflake::Callbacks
-
-  after_create :set_poll_id
-
   class << self
     def selectable_visibilities
       visibilities.keys - %w(direct limited)
@@ -442,71 +446,6 @@ class Status < ApplicationRecord
     super || build_status_stat
   end
 
-  # This is a hack to ensure that no reblogs of discarded statuses are created,
-  # as this cannot be enforced through database constraints the same way we do
-  # for reblogs of deleted statuses.
-  #
-  # To achieve this, we redefine the internal method responsible for issuing
-  # the "INSERT" statement and replace the "INSERT INTO ... VALUES ..." query
-  # with an "INSERT INTO ... SELECT ..." query with a "WHERE deleted_at IS NULL"
-  # clause on the reblogged status to ensure consistency at the database level.
-  #
-  # Otherwise, the code is kept as close as possible to ActiveRecord::Persistence
-  # code, and actually calls it if we are not handling a reblog.
-  def self._insert_record(values)
-    return super unless values.is_a?(Hash) && values['reblog_of_id'].present?
-
-    primary_key = self.primary_key
-    primary_key_value = nil
-
-    if primary_key
-      primary_key_value = values[primary_key]
-
-      if !primary_key_value && prefetch_primary_key?
-        primary_key_value = next_sequence_value
-        values[primary_key] = primary_key_value
-      end
-    end
-
-    # The following line is where we differ from stock ActiveRecord implementation
-    im = _compile_reblog_insert(values)
-
-    # Since we are using SELECT instead of VALUES, a non-error `nil` return is possible.
-    # For our purposes, it's equivalent to a foreign key constraint violation
-    result = connection.insert(im, "#{self} Create", primary_key || false, primary_key_value)
-    raise ActiveRecord::InvalidForeignKey, "(reblog_of_id)=(#{values['reblog_of_id']}) is not present in table \"statuses\"" if result.nil?
-
-    result
-  end
-
-  def self._compile_reblog_insert(values)
-    # This is somewhat equivalent to the following code of ActiveRecord::Persistence:
-    # `arel_table.compile_insert(_substitute_values(values))`
-    # The main difference is that we use a `SELECT` instead of a `VALUES` clause,
-    # which means we have to build the `SELECT` clause ourselves and do a bit more
-    # manual work.
-
-    # Instead of using Arel::InsertManager#values, we are going to use Arel::InsertManager#select
-    im = Arel::InsertManager.new
-    im.into(arel_table)
-
-    binds = []
-    reblog_bind = nil
-    values.each do |name, value|
-      attr = arel_table[name]
-      bind = predicate_builder.build_bind_attribute(attr.name, value)
-
-      im.columns << attr
-      binds << bind
-
-      reblog_bind = bind if name == 'reblog_of_id'
-    end
-
-    im.select(arel_table.where(arel_table[:id].eq(reblog_bind)).where(arel_table[:deleted_at].eq(nil)).project(*binds))
-
-    im
-  end
-
   def discard_with_reblogs
     discard_time = Time.current
     Status.unscoped.where(reblog_of_id: id, deleted_at: [nil, deleted_at]).in_batches.update_all(deleted_at: discard_time) unless reblog?
@@ -555,7 +494,7 @@ class Status < ApplicationRecord
     self.sensitive  = false if sensitive.nil?
   end
 
-  def set_locality
+  def set_local_only
     return unless account.domain.nil? && !attribute_changed?(:local_only)
 
     self.local_only = marked_local_only?

app/models/trends/history.rb

@@ -11,7 +11,7 @@ class Trends::History
     end
 
     def uses
-      with_redis { |redis| redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum }
+      with_redis { |redis| redis.mget(*@days.map { |day| day.key_for(:uses) }).sum(&:to_i) }
     end
 
     def accounts

app/models/trends/preview_card_filter.rb

@@ -6,6 +6,8 @@ class Trends::PreviewCardFilter
     locale
   ).freeze
 
+  IGNORED_PARAMS = %w(page).freeze
+
   attr_reader :params
 
   def initialize(params)
@@ -16,7 +18,7 @@ class Trends::PreviewCardFilter
     scope = initial_scope
 
     params.each do |key, value|
-      next if %w(page).include?(key.to_s)
+      next if IGNORED_PARAMS.include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end

app/models/trends/status_filter.rb

@@ -6,6 +6,8 @@ class Trends::StatusFilter
     locale
   ).freeze
 
+  IGNORED_PARAMS = %w(page).freeze
+
   attr_reader :params
 
   def initialize(params)
@@ -16,7 +18,7 @@ class Trends::StatusFilter
     scope = initial_scope
 
     params.each do |key, value|
-      next if %w(page).include?(key.to_s)
+      next if IGNORED_PARAMS.include?(key.to_s)
 
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end

app/presenters/status_relationships_presenter.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class StatusRelationshipsPresenter
+  PINNABLE_VISIBILITIES = %w(public unlisted private).freeze
+
   attr_reader :reblogs_map, :favourites_map, :mutes_map, :pins_map,
               :bookmarks_map, :filters_map
 
@@ -16,7 +18,7 @@ class StatusRelationshipsPresenter
       statuses            = statuses.compact
       status_ids          = statuses.flat_map { |s| [s.id, s.reblog_of_id] }.uniq.compact
       conversation_ids    = statuses.filter_map(&:conversation_id).uniq
-      pinnable_status_ids = statuses.map(&:proper).filter_map { |s| s.id if s.account_id == current_account_id && %w(public unlisted private).include?(s.visibility) }
+      pinnable_status_ids = statuses.map(&:proper).filter_map { |s| s.id if s.account_id == current_account_id && PINNABLE_VISIBILITIES.include?(s.visibility) }
 
       @filters_map     = build_filters_map(statuses, current_account_id).merge(options[:filters_map] || {})
       @reblogs_map     = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {})

app/services/activitypub/fetch_remote_actor_service.rb

@@ -67,7 +67,7 @@ class ActivityPub::FetchRemoteActorService < BaseService
   end
 
   def split_acct(acct)
-    acct.gsub(/\Aacct:/, '').split('@')
+    acct.delete_prefix('acct:').split('@')
   end
 
   def supported_context?

app/services/activitypub/fetch_remote_status_service.rb

@@ -2,12 +2,15 @@
 
 class ActivityPub::FetchRemoteStatusService < BaseService
   include JsonLdHelper
+  include DomainControlHelper
   include Redisable
 
   DISCOVERIES_PER_REQUEST = 1000
 
   # Should be called when uri has already been checked for locality
   def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
+    return if domain_not_allowed?(uri)
+
     @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = if prefetched_body.nil?
               fetch_resource(uri, id, on_behalf_of)

app/services/activitypub/process_account_service.rb

@@ -24,7 +24,7 @@ class ActivityPub::ProcessAccountService < BaseService
     # The key does not need to be unguessable, it just needs to be somewhat unique
     @options[:request_id] ||= "#{Time.now.utc.to_i}-#{username}@#{domain}"
 
-    with_lock("process_account:#{@uri}") do
+    with_redis_lock("process_account:#{@uri}") do
       @account            = Account.remote.find_by(uri: @uri) if @options[:only_key]
       @account          ||= Account.find_remote(@username, @domain)
       @old_public_key     = @account&.public_key

app/services/activitypub/process_status_update_service.rb

@@ -35,7 +35,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
     last_edit_date = @status.edited_at.presence || @status.created_at
 
     # Only allow processing one create/update per status at a time
-    with_lock("create:#{@uri}") do
+    with_redis_lock("create:#{@uri}") do
       Status.transaction do
         record_previous_edit!
         update_media_attachments!
@@ -58,7 +58,7 @@ class ActivityPub::ProcessStatusUpdateService < BaseService
   end
 
   def handle_implicit_update!
-    with_lock("create:#{@uri}") do
+    with_redis_lock("create:#{@uri}") do
       update_poll!(allow_significant_changes: false)
       queue_poll_notifications!
     end

app/services/backup_service.rb

@@ -1,59 +1,67 @@
 # frozen_string_literal: true
 
-require 'rubygems/package'
+require 'zip'
 
 class BackupService < BaseService
   include Payloadable
+  include ContextHelper
 
-  attr_reader :account, :backup, :collection
+  attr_reader :account, :backup
 
   def call(backup)
     @backup  = backup
     @account = backup.user.account
 
-    build_json!
     build_archive!
   end
 
   private
 
-  def build_json!
-    @collection = serialize(collection_presenter, ActivityPub::CollectionSerializer)
+  def build_outbox_json!(file)
+    skeleton = serialize(collection_presenter, ActivityPub::CollectionSerializer)
+    skeleton[:@context] = full_context
+    skeleton[:orderedItems] = ['!PLACEHOLDER!']
+    skeleton = Oj.dump(skeleton)
+    prepend, append = skeleton.split('"!PLACEHOLDER!"')
+    add_comma = false
+
+    file.write(prepend)
 
     account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
-      statuses.each do |status|
-        item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, signer: @account, allow_local_only: true)
-        item.delete(:@context)
+      file.write(',') if add_comma
+      add_comma = true
+
+      file.write(statuses.map do |status|
+        item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, allow_local_only: true)
+        item.delete('@context')
 
         unless item[:type] == 'Announce' || item[:object][:attachment].blank?
           item[:object][:attachment].each do |attachment|
-            attachment[:url] = Addressable::URI.parse(attachment[:url]).path.gsub(/\A\/system\//, '')
+            attachment[:url] = Addressable::URI.parse(attachment[:url]).path.delete_prefix('/system/')
           end
         end
 
-        @collection[:orderedItems] << item
-      end
+        Oj.dump(item)
+      end.join(','))
 
       GC.start
     end
+
+    file.write(append)
   end
 
   def build_archive!
-    tmp_file = Tempfile.new(%w(archive .tar.gz))
+    tmp_file = Tempfile.new(%w(archive .zip))
 
-    File.open(tmp_file, 'wb') do |file|
-      Zlib::GzipWriter.wrap(file) do |gz|
-        Gem::Package::TarWriter.new(gz) do |tar|
-          dump_media_attachments!(tar)
-          dump_outbox!(tar)
-          dump_likes!(tar)
-          dump_bookmarks!(tar)
-          dump_actor!(tar)
-        end
-      end
+    Zip::File.open(tmp_file, create: true) do |zipfile|
+      dump_outbox!(zipfile)
+      dump_media_attachments!(zipfile)
+      dump_likes!(zipfile)
+      dump_bookmarks!(zipfile)
+      dump_actor!(zipfile)
     end
 
-    archive_filename = "#{['archive', Time.now.utc.strftime('%Y%m%d%H%M%S'), SecureRandom.hex(16)].join('-')}.tar.gz"
+    archive_filename = "#{['archive', Time.now.utc.strftime('%Y%m%d%H%M%S'), SecureRandom.hex(16)].join('-')}.zip"
 
     @backup.dump      = ActionDispatch::Http::UploadedFile.new(tempfile: tmp_file, filename: archive_filename)
     @backup.processed = true
@@ -63,27 +71,28 @@ class BackupService < BaseService
     tmp_file.unlink
   end
 
-  def dump_media_attachments!(tar)
+  def dump_media_attachments!(zipfile)
     MediaAttachment.attached.where(account: account).reorder(nil).find_in_batches do |media_attachments|
       media_attachments.each do |m|
-        next unless m.file&.path
+        path = m.file&.path
+        next unless path
 
-        download_to_tar(tar, m.file, m.file.path)
+        path = path.gsub(/\A.*\/system\//, '')
+        path = path.gsub(/\A\/+/, '')
+        download_to_zip(zipfile, m.file, path)
       end
 
       GC.start
     end
   end
 
-  def dump_outbox!(tar)
-    json = Oj.dump(collection)
-
-    tar.add_file_simple('outbox.json', 0o444, json.bytesize) do |io|
-      io.write(json)
+  def dump_outbox!(zipfile)
+    zipfile.get_output_stream('outbox.json') do |io|
+      build_outbox_json!(io)
     end
   end
 
-  def dump_actor!(tar)
+  def dump_actor!(zipfile)
     actor = serialize(account, ActivityPub::ActorSerializer)
 
     actor[:icon][:url]  = "avatar#{File.extname(actor[:icon][:url])}"  if actor[:icon]
@@ -92,51 +101,66 @@ class BackupService < BaseService
     actor[:likes]       = 'likes.json'
     actor[:bookmarks]   = 'bookmarks.json'
 
-    download_to_tar(tar, account.avatar, "avatar#{File.extname(account.avatar.path)}") if account.avatar.exists?
-    download_to_tar(tar, account.header, "header#{File.extname(account.header.path)}") if account.header.exists?
+    download_to_zip(tar, account.avatar, "avatar#{File.extname(account.avatar.path)}") if account.avatar.exists?
+    download_to_zip(tar, account.header, "header#{File.extname(account.header.path)}") if account.header.exists?
 
     json = Oj.dump(actor)
 
-    tar.add_file_simple('actor.json', 0o444, json.bytesize) do |io|
+    zipfile.get_output_stream('actor.json') do |io|
       io.write(json)
     end
   end
 
-  def dump_likes!(tar)
-    collection = serialize(ActivityPub::CollectionPresenter.new(id: 'likes.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+  def dump_likes!(zipfile)
+    skeleton = serialize(ActivityPub::CollectionPresenter.new(id: 'likes.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+    skeleton.delete(:totalItems)
+    skeleton[:orderedItems] = ['!PLACEHOLDER!']
+    skeleton = Oj.dump(skeleton)
+    prepend, append = skeleton.split('"!PLACEHOLDER!"')
 
-    Status.reorder(nil).joins(:favourites).includes(:account).merge(account.favourites).find_in_batches do |statuses|
-      statuses.each do |status|
-        collection[:totalItems] += 1
-        collection[:orderedItems] << ActivityPub::TagManager.instance.uri_for(status)
+    zipfile.get_output_stream('likes.json') do |io|
+      io.write(prepend)
+
+      add_comma = false
+
+      Status.reorder(nil).joins(:favourites).includes(:account).merge(account.favourites).find_in_batches do |statuses|
+        io.write(',') if add_comma
+        add_comma = true
+
+        io.write(statuses.map do |status|
+          Oj.dump(ActivityPub::TagManager.instance.uri_for(status))
+        end.join(','))
+
+        GC.start
       end
 
-      GC.start
-    end
-
-    json = Oj.dump(collection)
-
-    tar.add_file_simple('likes.json', 0o444, json.bytesize) do |io|
-      io.write(json)
+      io.write(append)
     end
   end
 
-  def dump_bookmarks!(tar)
-    collection = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+  def dump_bookmarks!(zipfile)
+    skeleton = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+    skeleton.delete(:totalItems)
+    skeleton[:orderedItems] = ['!PLACEHOLDER!']
+    skeleton = Oj.dump(skeleton)
+    prepend, append = skeleton.split('"!PLACEHOLDER!"')
 
-    Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
-      statuses.each do |status|
-        collection[:totalItems] += 1
-        collection[:orderedItems] << ActivityPub::TagManager.instance.uri_for(status)
+    zipfile.get_output_stream('bookmarks.json') do |io|
+      io.write(prepend)
+
+      add_comma = false
+      Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
+        io.write(',') if add_comma
+        add_comma = true
+
+        io.write(statuses.map do |status|
+          Oj.dump(ActivityPub::TagManager.instance.uri_for(status))
+        end.join(','))
+
+        GC.start
       end
 
-      GC.start
-    end
-
-    json = Oj.dump(collection)
-
-    tar.add_file_simple('bookmarks.json', 0o444, json.bytesize) do |io|
-      io.write(json)
+      io.write(append)
     end
   end
 
@@ -160,10 +184,10 @@ class BackupService < BaseService
 
   CHUNK_SIZE = 1.megabyte
 
-  def download_to_tar(tar, attachment, filename)
+  def download_to_zip(zipfile, attachment, filename)
     adapter = Paperclip.io_adapters.for(attachment)
 
-    tar.add_file_simple(filename, 0o444, adapter.size) do |io|
+    zipfile.get_output_stream(filename) do |io|
       while (buffer = adapter.read(CHUNK_SIZE))
         io.write(buffer)
       end

app/services/bulk_import_row_service.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class BulkImportRowService
+  def call(row)
+    @account = row.bulk_import.account
+    @data    = row.data
+    @type    = row.bulk_import.type.to_sym
+
+    case @type
+    when :following, :blocking, :muting
+      target_acct     = @data['acct']
+      target_domain   = domain(target_acct)
+      @target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_acct, { check_delivery_availability: true }) }
+      return false if @target_account.nil?
+    when :bookmarks
+      target_uri      = @data['uri']
+      target_domain   = Addressable::URI.parse(target_uri).normalized_host
+      @target_status = ActivityPub::TagManager.instance.uri_to_resource(target_uri, Status)
+      return false if @target_status.nil? && ActivityPub::TagManager.instance.local_uri?(target_uri)
+
+      @target_status ||= stoplight_wrap_request(target_domain) { ActivityPub::FetchRemoteStatusService.new.call(target_uri) }
+      return false if @target_status.nil?
+    end
+
+    case @type
+    when :following
+      FollowService.new.call(@account, @target_account, reblogs: @data['show_reblogs'], notify: @data['notify'], languages: @data['languages'])
+    when :blocking
+      BlockService.new.call(@account, @target_account)
+    when :muting
+      MuteService.new.call(@account, @target_account, notifications: @data['hide_notifications'])
+    when :bookmarks
+      return false unless StatusPolicy.new(@account, @target_status).show?
+
+      @account.bookmarks.find_or_create_by!(status: @target_status)
+    end
+
+    true
+  rescue ActiveRecord::RecordNotFound
+    false
+  end
+
+  def domain(uri)
+    domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1]
+    TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain)
+  end
+
+  def stoplight_wrap_request(domain, &block)
+    if domain.present?
+      Stoplight("source:#{domain}", &block)
+        .with_fallback { nil }
+        .with_threshold(1)
+        .with_cool_off_time(5.minutes.seconds)
+        .with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) }
+        .run
+    else
+      yield
+    end
+  end
+end

app/services/bulk_import_service.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+class BulkImportService < BaseService
+  def call(import)
+    @import  = import
+    @account = @import.account
+
+    case @import.type.to_sym
+    when :following
+      import_follows!
+    when :blocking
+      import_blocks!
+    when :muting
+      import_mutes!
+    when :domain_blocking
+      import_domain_blocks!
+    when :bookmarks
+      import_bookmarks!
+    end
+
+    @import.update!(state: :finished, finished_at: Time.now.utc) if @import.processed_items == @import.total_items
+  rescue
+    @import.update!(state: :finished, finished_at: Time.now.utc)
+
+    raise
+  end
+
+  private
+
+  def extract_rows_by_acct
+    local_domain_suffix = "@#{Rails.configuration.x.local_domain}"
+    @import.rows.to_a.index_by { |row| row.data['acct'].delete_suffix(local_domain_suffix) }
+  end
+
+  def import_follows!
+    rows_by_acct = extract_rows_by_acct
+
+    if @import.overwrite?
+      @account.following.find_each do |followee|
+        row = rows_by_acct.delete(followee.acct)
+
+        if row.nil?
+          UnfollowService.new.call(@account, followee)
+        else
+          row.destroy
+          @import.processed_items += 1
+          @import.imported_items += 1
+
+          # Since we're updating the settings of an existing relationship, we can safely call
+          # FollowService directly
+          FollowService.new.call(@account, followee, reblogs: row.data['show_reblogs'], notify: row.data['notify'], languages: row.data['languages'])
+        end
+      end
+
+      # Save pending infos due to `overwrite?` handling
+      @import.save!
+    end
+
+    Import::RowWorker.push_bulk(rows_by_acct.values) do |row|
+      [row.id]
+    end
+  end
+
+  def import_blocks!
+    rows_by_acct = extract_rows_by_acct
+
+    if @import.overwrite?
+      @account.blocking.find_each do |blocked_account|
+        row = rows_by_acct.delete(blocked_account.acct)
+
+        if row.nil?
+          UnblockService.new.call(@account, blocked_account)
+        else
+          row.destroy
+          @import.processed_items += 1
+          @import.imported_items += 1
+          BlockService.new.call(@account, blocked_account)
+        end
+      end
+
+      # Save pending infos due to `overwrite?` handling
+      @import.save!
+    end
+
+    Import::RowWorker.push_bulk(rows_by_acct.values) do |row|
+      [row.id]
+    end
+  end
+
+  def import_mutes!
+    rows_by_acct = extract_rows_by_acct
+
+    if @import.overwrite?
+      @account.muting.find_each do |muted_account|
+        row = rows_by_acct.delete(muted_account.acct)
+
+        if row.nil?
+          UnmuteService.new.call(@account, muted_account)
+        else
+          row.destroy
+          @import.processed_items += 1
+          @import.imported_items += 1
+          MuteService.new.call(@account, muted_account, notifications: row.data['hide_notifications'])
+        end
+      end
+
+      # Save pending infos due to `overwrite?` handling
+      @import.save!
+    end
+
+    Import::RowWorker.push_bulk(rows_by_acct.values) do |row|
+      [row.id]
+    end
+  end
+
+  def import_domain_blocks!
+    domains = @import.rows.map { |row| row.data['domain'] }
+
+    if @import.overwrite?
+      @account.domain_blocks.find_each do |domain_block|
+        domain = domains.delete(domain_block)
+
+        @account.unblock_domain!(domain_block.domain) if domain.nil?
+      end
+    end
+
+    @import.rows.delete_all
+    domains.each { |domain| @account.block_domain!(domain) }
+    @import.update!(processed_items: @import.total_items, imported_items: @import.total_items)
+
+    AfterAccountDomainBlockWorker.push_bulk(domains) do |domain|
+      [@account.id, domain]
+    end
+  end
+
+  def import_bookmarks!
+    rows_by_uri = @import.rows.index_by { |row| row.data['uri'] }
+
+    if @import.overwrite?
+      @account.bookmarks.includes(:status).find_each do |bookmark|
+        row = rows_by_uri.delete(ActivityPub::TagManager.instance.uri_for(bookmark.status))
+
+        if row.nil?
+          bookmark.destroy!
+        else
+          row.destroy
+          @import.processed_items += 1
+          @import.imported_items += 1
+        end
+      end
+
+      # Save pending infos due to `overwrite?` handling
+      @import.save!
+    end
+
+    Import::RowWorker.push_bulk(rows_by_uri.values) do |row|
+      [row.id]
+    end
+  end
+end

app/services/fetch_link_card_service.rb

@@ -23,7 +23,7 @@ class FetchLinkCardService < BaseService
 
     @url = @original_url.to_s
 
-    with_lock("fetch:#{@original_url}") do
+    with_redis_lock("fetch:#{@original_url}") do
       @card = PreviewCard.find_by(url: @url)
       process_url if @card.nil? || @card.updated_at <= 2.weeks.ago || @card.missing_image?
     end

app/services/fetch_resource_service.rb

@@ -4,6 +4,7 @@ class FetchResourceService < BaseService
   include JsonLdHelper
 
   ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html;q=0.1'
+  ACTIVITY_STREAM_LINK_TYPES = ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].freeze
 
   attr_reader :response_code
 
@@ -65,7 +66,7 @@ class FetchResourceService < BaseService
 
   def process_html(response)
     page      = Nokogiri::HTML(response.body_with_limit)
-    json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
+    json_link = page.xpath('//link[@rel="alternate"]').find { |link| ACTIVITY_STREAM_LINK_TYPES.include?(link['type']) }
 
     process(json_link['href'], terminal: true) unless json_link.nil?
   end

app/services/follow_migration_service.rb

@@ -9,10 +9,10 @@ class FollowMigrationService < FollowService
   def call(source_account, target_account, old_target_account, bypass_locked: false)
     @old_target_account = old_target_account
 
-    follow    = source_account.active_relationships.find_by(target_account: old_target_account)
-    reblogs   = follow&.show_reblogs?
-    notify    = follow&.notify?
-    languages = follow&.languages
+    @original_follow = source_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs          = @original_follow&.show_reblogs?
+    notify           = @original_follow&.notify?
+    languages        = @original_follow&.languages
 
     super(source_account, target_account, reblogs: reblogs, notify: notify, languages: languages, bypass_locked: bypass_locked, bypass_limit: true)
   end
@@ -21,6 +21,7 @@ class FollowMigrationService < FollowService
 
   def request_follow!
     follow_request = @source_account.request_follow!(@target_account, **follow_options.merge(rate_limit: @options[:with_rate_limit], bypass_limit: @options[:bypass_limit]))
+    migrate_list_accounts!
 
     if @target_account.local?
       LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
@@ -32,9 +33,30 @@ class FollowMigrationService < FollowService
     follow_request
   end
 
+  def change_follow_options!
+    migrate_list_accounts!
+    super
+  end
+
+  def change_follow_request_options!
+    migrate_list_accounts!
+    super
+  end
+
   def direct_follow!
     follow = super
+
+    migrate_list_accounts!
     UnfollowService.new.call(@source_account, @old_target_account, skip_unmerge: true)
+
     follow
   end
+
+  def migrate_list_accounts!
+    ListAccount.where(follow_id: @original_follow.id).includes(:list).find_each do |list_account|
+      list_account.list.accounts << @target_account
+    rescue ActiveRecord::RecordInvalid
+      nil
+    end
+  end
 end

app/services/import_service.rb

@@ -2,6 +2,9 @@
 
 require 'csv'
 
+# NOTE: This is a deprecated service, only kept to not break ongoing imports
+# on upgrade. See `BulkImportService` for its replacement.
+
 class ImportService < BaseService
   ROWS_PROCESSING_LIMIT = 20_000
 

app/services/remove_status_service.rb

@@ -18,7 +18,7 @@ class RemoveStatusService < BaseService
     @account  = status.account
     @options  = options
 
-    with_lock("distribute:#{@status.id}") do
+    with_redis_lock("distribute:#{@status.id}") do
       @status.discard_with_reblogs
 
       StatusPin.find_by(status: @status)&.destroy

app/services/resolve_account_service.rb

@@ -100,13 +100,13 @@ class ResolveAccountService < BaseService
   end
 
   def split_acct(acct)
-    acct.gsub(/\Aacct:/, '').split('@')
+    acct.delete_prefix('acct:').split('@')
   end
 
   def fetch_account!
     return unless activitypub_ready?
 
-    with_lock("resolve:#{@username}@#{@domain}") do
+    with_redis_lock("resolve:#{@username}@#{@domain}") do
       @account = ActivityPub::FetchRemoteAccountService.new.call(actor_url, suppress_errors: @options[:suppress_errors])
     end