weeeeeeeee
This commit is contained in:
tobi
@@ -1,44 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ActiveRecord
|
||||
module Batches
|
||||
def pluck_each(*column_names)
|
||||
relation = self
|
||||
|
||||
options = column_names.extract_options!
|
||||
|
||||
flatten = column_names.size == 1
|
||||
batch_limit = options[:batch_limit] || 1_000
|
||||
order = options[:order] || :asc
|
||||
|
||||
column_names.unshift(primary_key)
|
||||
|
||||
relation = relation.reorder(batch_order(order)).limit(batch_limit)
|
||||
relation.skip_query_cache!
|
||||
|
||||
batch_relation = relation
|
||||
|
||||
loop do
|
||||
batch = batch_relation.pluck(*column_names)
|
||||
|
||||
break if batch.empty?
|
||||
|
||||
primary_key_offset = batch.last[0]
|
||||
|
||||
batch.each do |record|
|
||||
if flatten
|
||||
yield record[1]
|
||||
else
|
||||
yield record[1..]
|
||||
end
|
||||
end
|
||||
|
||||
break if batch.size < batch_limit
|
||||
|
||||
batch_relation = relation.where(
|
||||
predicate_builder[primary_key, primary_key_offset, order == :desc ? :lt : :gt]
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,20 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../mastodon/snowflake'
|
||||
|
||||
module ActiveRecord
|
||||
module Tasks
|
||||
module DatabaseTasks
|
||||
original_load_schema = instance_method(:load_schema)
|
||||
|
||||
define_method(:load_schema) do |db_config, *args|
|
||||
ActiveRecord::Base.establish_connection(db_config)
|
||||
Mastodon::Snowflake.define_timestamp_id
|
||||
|
||||
original_load_schema.bind_call(self, db_config, *args)
|
||||
|
||||
Mastodon::Snowflake.ensure_id_sequences_exist
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,18 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Chewy
|
||||
module IndexExtensions
|
||||
def index_preset(base_options = {})
|
||||
case ENV['ES_PRESET'].presence
|
||||
when 'single_node_cluster', nil
|
||||
base_options.merge(number_of_replicas: 0)
|
||||
when 'small_cluster'
|
||||
base_options.merge(number_of_replicas: 1)
|
||||
when 'large_cluster'
|
||||
base_options.merge(number_of_replicas: 1, number_of_shards: (base_options[:number_of_shards] || 1) * 2)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Chewy::Index.extend(Chewy::IndexExtensions)
|
||||
@@ -1,11 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Chewy
|
||||
module SettingsExtensions
|
||||
def enabled?
|
||||
settings[:enabled]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Chewy.extend(Chewy::SettingsExtensions)
|
||||
@@ -1,12 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Chewy
|
||||
class Strategy
|
||||
class BypassWithWarning < Base
|
||||
def update(...)
|
||||
Rails.logger.warn 'Chewy update without a root strategy' unless @warning_issued
|
||||
@warning_issued = true
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,27 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Chewy
|
||||
class Strategy
|
||||
class Mastodon < Base
|
||||
def initialize
|
||||
super
|
||||
|
||||
@stash = Hash.new { |hash, key| hash[key] = [] }
|
||||
end
|
||||
|
||||
def update(type, objects, _options = {})
|
||||
@stash[type].concat(type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)) if Chewy.enabled?
|
||||
end
|
||||
|
||||
def leave
|
||||
RedisConfiguration.with do |redis|
|
||||
redis.pipelined do |pipeline|
|
||||
@stash.each do |type, ids|
|
||||
pipeline.sadd("chewy:queue:#{type.name}", ids)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,32 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'net/ldap'
|
||||
require 'devise/strategies/base'
|
||||
|
||||
module Devise
|
||||
module Strategies
|
||||
class TwoFactorLdapAuthenticatable < Base
|
||||
def valid?
|
||||
valid_params? && mapping.to.respond_to?(:authenticate_with_ldap)
|
||||
end
|
||||
|
||||
def authenticate!
|
||||
resource = mapping.to.authenticate_with_ldap(params[scope])
|
||||
|
||||
if resource && !resource.otp_required_for_login?
|
||||
success!(resource)
|
||||
else
|
||||
fail(:invalid)
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def valid_params?
|
||||
params[scope] && params[scope][:password].present?
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Warden::Strategies.add(:two_factor_ldap_authenticatable, Devise::Strategies::TwoFactorLdapAuthenticatable)
|
||||
@@ -1,31 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'devise/strategies/base'
|
||||
|
||||
module Devise
|
||||
module Strategies
|
||||
class TwoFactorPamAuthenticatable < Base
|
||||
def valid?
|
||||
valid_params? && mapping.to.respond_to?(:authenticate_with_pam)
|
||||
end
|
||||
|
||||
def authenticate!
|
||||
resource = mapping.to.authenticate_with_pam(params[scope])
|
||||
|
||||
if resource && !resource.otp_required_for_login?
|
||||
success!(resource)
|
||||
else
|
||||
fail(:invalid)
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def valid_params?
|
||||
params[scope] && params[scope][:password].present?
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Warden::Strategies.add(:two_factor_pam_authenticatable, Devise::Strategies::TwoFactorPamAuthenticatable)
|
||||
@@ -1,38 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Mastodon
|
||||
class Error < StandardError; end
|
||||
class NotPermittedError < Error; end
|
||||
class ValidationError < Error; end
|
||||
class HostValidationError < ValidationError; end
|
||||
class LengthValidationError < ValidationError; end
|
||||
class DimensionsValidationError < ValidationError; end
|
||||
class StreamValidationError < ValidationError; end
|
||||
class RaceConditionError < Error; end
|
||||
class RateLimitExceededError < Error; end
|
||||
class SyntaxError < Error; end
|
||||
class InvalidParameterError < Error; end
|
||||
|
||||
class UnexpectedResponseError < Error
|
||||
attr_reader :response
|
||||
|
||||
def initialize(response = nil)
|
||||
@response = response
|
||||
|
||||
if response.respond_to? :uri
|
||||
super("#{response.uri} returned code #{response.code}")
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class PrivateNetworkAddressError < HostValidationError
|
||||
attr_reader :host
|
||||
|
||||
def initialize(host)
|
||||
@host = host
|
||||
super()
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,10 +0,0 @@
|
||||
Description:
|
||||
Generate a Rails migration in the db/post_migrate/ dir.
|
||||
|
||||
Interacts with the post_deployment_migrations initializer.
|
||||
|
||||
Example:
|
||||
bin/rails generate post_deployment_migration IsolateChanges
|
||||
|
||||
Creates a migration in db/post_migrate/<timestamp>_isolate_changes.rb
|
||||
which will have `disable_ddl_transaction!` and a `change` method included.
|
||||
@@ -1,17 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rails/generators/active_record'
|
||||
|
||||
class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase
|
||||
source_root File.expand_path('templates', __dir__)
|
||||
|
||||
include Rails::Generators::Migration
|
||||
|
||||
def create_post_deployment_migration
|
||||
migration_template 'migration.erb', "db/post_migrate/#{file_name}.rb"
|
||||
end
|
||||
|
||||
def self.next_migration_number(path)
|
||||
ActiveRecord::Generators::Base.next_migration_number(path)
|
||||
end
|
||||
end
|
||||
@@ -1,8 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class <%= migration_class_name %> < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
||||
disable_ddl_transaction!
|
||||
|
||||
def change
|
||||
end
|
||||
end
|
||||
@@ -1,8 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Monkey patching until https://github.com/httprb/http/pull/757 is merged
|
||||
unless HTTP::Request::METHODS.include?(:purge)
|
||||
methods = HTTP::Request::METHODS.dup
|
||||
HTTP::Request.send(:remove_const, :METHODS)
|
||||
HTTP::Request.const_set(:METHODS, methods.push(:purge).freeze)
|
||||
end
|
||||
@@ -1,86 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
# This file generated automatically from http://w3id.org/identity/v1
|
||||
require 'json/ld'
|
||||
class JSON::LD::Context
|
||||
add_preloaded("http://w3id.org/identity/v1") do
|
||||
new(term_definitions: {
|
||||
"Credential" => TermDefinition.new("Credential", id: "https://w3id.org/credentials#Credential", simple: true),
|
||||
"CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true),
|
||||
"CryptographicKeyCredential" => TermDefinition.new("CryptographicKeyCredential", id: "https://w3id.org/credentials#CryptographicKeyCredential", simple: true),
|
||||
"EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true),
|
||||
"GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true),
|
||||
"Group" => TermDefinition.new("Group", id: "https://www.w3.org/ns/activitystreams#Group", simple: true),
|
||||
"Identity" => TermDefinition.new("Identity", id: "https://w3id.org/identity#Identity", simple: true),
|
||||
"LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true),
|
||||
"Organization" => TermDefinition.new("Organization", id: "http://schema.org/Organization", simple: true),
|
||||
"Person" => TermDefinition.new("Person", id: "http://schema.org/Person", simple: true),
|
||||
"PostalAddress" => TermDefinition.new("PostalAddress", id: "http://schema.org/PostalAddress", simple: true),
|
||||
"about" => TermDefinition.new("about", id: "http://schema.org/about", type_mapping: "@id"),
|
||||
"accessControl" => TermDefinition.new("accessControl", id: "https://w3id.org/permissions#accessControl", type_mapping: "@id"),
|
||||
"address" => TermDefinition.new("address", id: "http://schema.org/address", type_mapping: "@id"),
|
||||
"addressCountry" => TermDefinition.new("addressCountry", id: "http://schema.org/addressCountry", simple: true),
|
||||
"addressLocality" => TermDefinition.new("addressLocality", id: "http://schema.org/addressLocality", simple: true),
|
||||
"addressRegion" => TermDefinition.new("addressRegion", id: "http://schema.org/addressRegion", simple: true),
|
||||
"cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true),
|
||||
"cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true),
|
||||
"cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true),
|
||||
"claim" => TermDefinition.new("claim", id: "https://w3id.org/credentials#claim", type_mapping: "@id"),
|
||||
"comment" => TermDefinition.new("comment", id: "http://www.w3.org/2000/01/rdf-schema#comment", simple: true),
|
||||
"created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"),
|
||||
"cred" => TermDefinition.new("cred", id: "https://w3id.org/credentials#", simple: true, prefix: true),
|
||||
"credential" => TermDefinition.new("credential", id: "https://w3id.org/credentials#credential", type_mapping: "@id"),
|
||||
"dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true),
|
||||
"description" => TermDefinition.new("description", id: "http://schema.org/description", simple: true),
|
||||
"digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true),
|
||||
"digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true),
|
||||
"domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true),
|
||||
"email" => TermDefinition.new("email", id: "http://schema.org/email", simple: true),
|
||||
"expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"familyName" => TermDefinition.new("familyName", id: "http://schema.org/familyName", simple: true),
|
||||
"givenName" => TermDefinition.new("givenName", id: "http://schema.org/givenName", simple: true),
|
||||
"id" => TermDefinition.new("id", id: "@id", simple: true),
|
||||
"identity" => TermDefinition.new("identity", id: "https://w3id.org/identity#", simple: true, prefix: true),
|
||||
"identityService" => TermDefinition.new("identityService", id: "https://w3id.org/identity#identityService", type_mapping: "@id"),
|
||||
"idp" => TermDefinition.new("idp", id: "https://w3id.org/identity#idp", type_mapping: "@id"),
|
||||
"image" => TermDefinition.new("image", id: "http://schema.org/image", type_mapping: "@id"),
|
||||
"initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true),
|
||||
"issued" => TermDefinition.new("issued", id: "https://w3id.org/credentials#issued", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"issuer" => TermDefinition.new("issuer", id: "https://w3id.org/credentials#issuer", type_mapping: "@id"),
|
||||
"label" => TermDefinition.new("label", id: "http://www.w3.org/2000/01/rdf-schema#label", simple: true),
|
||||
"member" => TermDefinition.new("member", id: "http://schema.org/member", type_mapping: "@id"),
|
||||
"memberOf" => TermDefinition.new("memberOf", id: "http://schema.org/memberOf", type_mapping: "@id"),
|
||||
"name" => TermDefinition.new("name", id: "http://schema.org/name", simple: true),
|
||||
"nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true),
|
||||
"normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true),
|
||||
"owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"),
|
||||
"password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true),
|
||||
"paymentProcessor" => TermDefinition.new("paymentProcessor", id: "https://w3id.org/payswarm#processor", simple: true),
|
||||
"perm" => TermDefinition.new("perm", id: "https://w3id.org/permissions#", simple: true, prefix: true),
|
||||
"postalCode" => TermDefinition.new("postalCode", id: "http://schema.org/postalCode", simple: true),
|
||||
"preferences" => TermDefinition.new("preferences", id: "https://w3id.org/payswarm#preferences", type_mapping: "@vocab"),
|
||||
"privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"),
|
||||
"privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true),
|
||||
"ps" => TermDefinition.new("ps", id: "https://w3id.org/payswarm#", simple: true, prefix: true),
|
||||
"publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"),
|
||||
"publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true),
|
||||
"publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"),
|
||||
"rdf" => TermDefinition.new("rdf", id: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", simple: true, prefix: true),
|
||||
"rdfs" => TermDefinition.new("rdfs", id: "http://www.w3.org/2000/01/rdf-schema#", simple: true, prefix: true),
|
||||
"recipient" => TermDefinition.new("recipient", id: "https://w3id.org/credentials#recipient", type_mapping: "@id"),
|
||||
"revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"schema" => TermDefinition.new("schema", id: "http://schema.org/", simple: true, prefix: true),
|
||||
"sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true),
|
||||
"signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true),
|
||||
"signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signatureAlgorithm", simple: true),
|
||||
"signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true),
|
||||
"streetAddress" => TermDefinition.new("streetAddress", id: "http://schema.org/streetAddress", simple: true),
|
||||
"title" => TermDefinition.new("title", id: "http://purl.org/dc/terms/title", simple: true),
|
||||
"type" => TermDefinition.new("type", id: "@type", simple: true),
|
||||
"url" => TermDefinition.new("url", id: "http://schema.org/url", type_mapping: "@id"),
|
||||
"writePermission" => TermDefinition.new("writePermission", id: "https://w3id.org/permissions#writePermission", type_mapping: "@id"),
|
||||
"xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true)
|
||||
})
|
||||
end
|
||||
alias_preloaded("https://w3id.org/identity/v1", "http://w3id.org/identity/v1")
|
||||
end
|
||||
@@ -1,50 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
# This file generated automatically from http://w3id.org/security/v1
|
||||
require 'json/ld'
|
||||
class JSON::LD::Context
|
||||
add_preloaded("http://w3id.org/security/v1") do
|
||||
new(processingMode: "json-ld-1.0", term_definitions: {
|
||||
"CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true),
|
||||
"EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true),
|
||||
"EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true),
|
||||
"GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true),
|
||||
"LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true),
|
||||
"LinkedDataSignature2016" => TermDefinition.new("LinkedDataSignature2016", id: "https://w3id.org/security#LinkedDataSignature2016", simple: true),
|
||||
"authenticationTag" => TermDefinition.new("authenticationTag", id: "https://w3id.org/security#authenticationTag", simple: true),
|
||||
"canonicalizationAlgorithm" => TermDefinition.new("canonicalizationAlgorithm", id: "https://w3id.org/security#canonicalizationAlgorithm", simple: true),
|
||||
"cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true),
|
||||
"cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true),
|
||||
"cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true),
|
||||
"created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"),
|
||||
"dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true),
|
||||
"digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true),
|
||||
"digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true),
|
||||
"domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true),
|
||||
"encryptionKey" => TermDefinition.new("encryptionKey", id: "https://w3id.org/security#encryptionKey", simple: true),
|
||||
"expiration" => TermDefinition.new("expiration", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"id" => TermDefinition.new("id", id: "@id", simple: true),
|
||||
"initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true),
|
||||
"iterationCount" => TermDefinition.new("iterationCount", id: "https://w3id.org/security#iterationCount", simple: true),
|
||||
"nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true),
|
||||
"normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true),
|
||||
"owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"),
|
||||
"password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true),
|
||||
"privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"),
|
||||
"privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true),
|
||||
"publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"),
|
||||
"publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true),
|
||||
"publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"),
|
||||
"revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
|
||||
"salt" => TermDefinition.new("salt", id: "https://w3id.org/security#salt", simple: true),
|
||||
"sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true),
|
||||
"signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true),
|
||||
"signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signingAlgorithm", simple: true),
|
||||
"signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true),
|
||||
"type" => TermDefinition.new("type", id: "@type", simple: true),
|
||||
"xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true)
|
||||
})
|
||||
end
|
||||
alias_preloaded("https://w3id.org/security/v1", "http://w3id.org/security/v1")
|
||||
end
|
||||
@@ -1,26 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module HamlLint
|
||||
# Bans the usage of “•” (bullet) in HTML/HAML in favor of “·” (middle dot) in anything that will end up as a text node. (including string literals in Ruby code)
|
||||
class Linter::MiddleDot < Linter
|
||||
include LinterRegistry
|
||||
|
||||
# rubocop:disable Style/MiddleDot
|
||||
BULLET = '•'
|
||||
# rubocop:enable Style/MiddleDot
|
||||
MIDDLE_DOT = '·'
|
||||
MESSAGE = "Use '#{MIDDLE_DOT}' (middle dot) instead of '#{BULLET}' (bullet)".freeze
|
||||
|
||||
def visit_plain(node)
|
||||
return unless node.text.include?(BULLET)
|
||||
|
||||
record_lint(node, MESSAGE)
|
||||
end
|
||||
|
||||
def visit_script(node)
|
||||
return unless node.script.include?(BULLET)
|
||||
|
||||
record_lint(node, MESSAGE)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,31 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module RuboCop
|
||||
module Cop
|
||||
module Style
|
||||
# Bans the usage of “•” (bullet) in HTML/HAML in favor of “·” (middle dot) in string literals
|
||||
class MiddleDot < Base
|
||||
extend AutoCorrector
|
||||
extend Util
|
||||
|
||||
# rubocop:disable Style/MiddleDot
|
||||
BULLET = '•'
|
||||
# rubocop:enable Style/MiddleDot
|
||||
MIDDLE_DOT = '·'
|
||||
MESSAGE = "Use '#{MIDDLE_DOT}' (middle dot) instead of '#{BULLET}' (bullet)".freeze
|
||||
|
||||
def on_str(node)
|
||||
# Constants like __FILE__ are handled as strings,
|
||||
# but don't respond to begin.
|
||||
return unless node.loc.respond_to?(:begin) && node.loc.begin
|
||||
|
||||
return unless node.value.include?(BULLET)
|
||||
|
||||
add_offense(node, message: MESSAGE) do |corrector|
|
||||
corrector.replace(node, node.source.gsub(BULLET, MIDDLE_DOT))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,674 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'set'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Accounts < Base
|
||||
option :all, type: :boolean
|
||||
desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
|
||||
long_desc <<-LONG_DESC
|
||||
Generate and broadcast new RSA keys as part of security
|
||||
maintenance.
|
||||
|
||||
With the --all option, all local accounts will be subject
|
||||
to the rotation. Otherwise, and by default, only a single
|
||||
account specified by the USERNAME argument will be
|
||||
processed.
|
||||
LONG_DESC
|
||||
def rotate(username = nil)
|
||||
if options[:all]
|
||||
processed = 0
|
||||
delay = 0
|
||||
scope = Account.local.without_suspended
|
||||
progress = create_progress_bar(scope.count)
|
||||
|
||||
scope.find_in_batches do |accounts|
|
||||
accounts.each do |account|
|
||||
rotate_keys_for_account(account, delay)
|
||||
progress.increment
|
||||
processed += 1
|
||||
end
|
||||
|
||||
delay += 5.minutes
|
||||
end
|
||||
|
||||
progress.finish
|
||||
say("OK, rotated keys for #{processed} accounts", :green)
|
||||
elsif username.present?
|
||||
rotate_keys_for_account(Account.find_local(username))
|
||||
say('OK', :green)
|
||||
else
|
||||
say('No account(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
option :email, required: true
|
||||
option :confirmed, type: :boolean
|
||||
option :role
|
||||
option :reattach, type: :boolean
|
||||
option :force, type: :boolean
|
||||
option :approve, type: :boolean
|
||||
desc 'create USERNAME', 'Create a new user account'
|
||||
long_desc <<-LONG_DESC
|
||||
Create a new user account with a given USERNAME and an
|
||||
e-mail address provided with --email.
|
||||
|
||||
With the --confirmed option, the confirmation e-mail will
|
||||
be skipped and the account will be active straight away.
|
||||
|
||||
With the --role option, the role can be supplied.
|
||||
|
||||
With the --reattach option, the new user will be reattached
|
||||
to a given existing username of an old account. If the old
|
||||
account is still in use by someone else, you can supply
|
||||
the --force option to delete the old record and reattach the
|
||||
username to the new account anyway.
|
||||
|
||||
With the --approve option, the account will be approved.
|
||||
LONG_DESC
|
||||
def create(username)
|
||||
role_id = nil
|
||||
|
||||
if options[:role]
|
||||
role = UserRole.find_by(name: options[:role])
|
||||
|
||||
if role.nil?
|
||||
say('Cannot find user role with that name', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
role_id = role.id
|
||||
end
|
||||
|
||||
account = Account.new(username: username)
|
||||
password = SecureRandom.hex
|
||||
user = User.new(email: options[:email], password: password, agreement: true, role_id: role_id, confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
|
||||
|
||||
if options[:reattach]
|
||||
account = Account.find_local(username) || Account.new(username: username)
|
||||
|
||||
if account.user.present? && !options[:force]
|
||||
say('The chosen username is currently in use', :red)
|
||||
say('Use --force to reattach it anyway and delete the other user')
|
||||
return
|
||||
elsif account.user.present?
|
||||
DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false)
|
||||
account = Account.new(username: username)
|
||||
end
|
||||
end
|
||||
|
||||
account.suspended_at = nil
|
||||
user.account = account
|
||||
|
||||
if user.save
|
||||
if options[:confirmed]
|
||||
user.confirmed_at = nil
|
||||
user.confirm!
|
||||
end
|
||||
|
||||
user.approve! if options[:approve]
|
||||
|
||||
say('OK', :green)
|
||||
say("New password: #{password}")
|
||||
else
|
||||
report_errors(user.errors)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
option :role
|
||||
option :remove_role, type: :boolean
|
||||
option :email
|
||||
option :confirm, type: :boolean
|
||||
option :enable, type: :boolean
|
||||
option :disable, type: :boolean
|
||||
option :disable_2fa, type: :boolean
|
||||
option :approve, type: :boolean
|
||||
option :reset_password, type: :boolean
|
||||
desc 'modify USERNAME', 'Modify a user account'
|
||||
long_desc <<-LONG_DESC
|
||||
Modify a user account.
|
||||
|
||||
With the --role option, update the user's role. To remove the user's
|
||||
role, i.e. demote to normal user, use --remove-role.
|
||||
|
||||
With the --email option, update the user's e-mail address. With
|
||||
the --confirm option, mark the user's e-mail as confirmed.
|
||||
|
||||
With the --disable option, lock the user out of their account. The
|
||||
--enable option is the opposite.
|
||||
|
||||
With the --approve option, the account will be approved, if it was
|
||||
previously not due to not having open registrations.
|
||||
|
||||
With the --disable-2fa option, the two-factor authentication
|
||||
requirement for the user can be removed.
|
||||
|
||||
With the --reset-password option, the user's password is replaced by
|
||||
a randomly-generated one, printed in the output.
|
||||
LONG_DESC
|
||||
def modify(username)
|
||||
user = Account.find_local(username)&.user
|
||||
|
||||
if user.nil?
|
||||
say('No user with such username', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if options[:role]
|
||||
role = UserRole.find_by(name: options[:role])
|
||||
|
||||
if role.nil?
|
||||
say('Cannot find user role with that name', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
user.role_id = role.id
|
||||
elsif options[:remove_role]
|
||||
user.role_id = nil
|
||||
end
|
||||
|
||||
password = SecureRandom.hex if options[:reset_password]
|
||||
user.password = password if options[:reset_password]
|
||||
user.email = options[:email] if options[:email]
|
||||
user.disabled = false if options[:enable]
|
||||
user.disabled = true if options[:disable]
|
||||
user.approved = true if options[:approve]
|
||||
user.otp_required_for_login = false if options[:disable_2fa]
|
||||
|
||||
if user.save
|
||||
user.confirm if options[:confirm]
|
||||
|
||||
say('OK', :green)
|
||||
say("New password: #{password}") if options[:reset_password]
|
||||
else
|
||||
report_errors(user.errors)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
option :email
|
||||
option :dry_run, type: :boolean
|
||||
desc 'delete [USERNAME]', 'Delete a user'
|
||||
long_desc <<-LONG_DESC
|
||||
Remove a user account with a given USERNAME.
|
||||
|
||||
With the --email option, the user is selected based on email
|
||||
rather than username.
|
||||
LONG_DESC
|
||||
def delete(username = nil)
|
||||
if username.present? && options[:email].present?
|
||||
say('Use username or --email, not both', :red)
|
||||
exit(1)
|
||||
elsif username.blank? && options[:email].blank?
|
||||
say('No username provided', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
account = nil
|
||||
|
||||
if username.present?
|
||||
account = Account.find_local(username)
|
||||
if account.nil?
|
||||
say('No user with such username', :red)
|
||||
exit(1)
|
||||
end
|
||||
else
|
||||
account = Account.left_joins(:user).find_by(user: { email: options[:email] })
|
||||
if account.nil?
|
||||
say('No user with such email', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run_mode_suffix}")
|
||||
DeleteAccountService.new.call(account, reserve_email: false) unless dry_run?
|
||||
say("OK#{dry_run_mode_suffix}", :green)
|
||||
end
|
||||
|
||||
option :force, type: :boolean, aliases: [:f], description: 'Override public key check'
|
||||
desc 'merge FROM TO', 'Merge two remote accounts into one'
|
||||
long_desc <<-LONG_DESC
|
||||
Merge two remote accounts specified by their username@domain
|
||||
into one, whereby the TO account is the one being merged into
|
||||
and kept, while the FROM one is removed. It is primarily meant
|
||||
to fix duplicates caused by other servers changing their domain.
|
||||
|
||||
The command by default only works if both accounts have the same
|
||||
public key to prevent mistakes. To override this, use the --force.
|
||||
LONG_DESC
|
||||
def merge(from_acct, to_acct)
|
||||
username, domain = from_acct.split('@')
|
||||
from_account = Account.find_remote(username, domain)
|
||||
|
||||
if from_account.nil? || from_account.local?
|
||||
say("No such account (#{from_acct})", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
username, domain = to_acct.split('@')
|
||||
to_account = Account.find_remote(username, domain)
|
||||
|
||||
if to_account.nil? || to_account.local?
|
||||
say("No such account (#{to_acct})", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if from_account.public_key != to_account.public_key && !options[:force]
|
||||
say("Accounts don't have the same public key, might not be duplicates!", :red)
|
||||
say('Override with --force', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
to_account.merge_with!(from_account)
|
||||
from_account.destroy
|
||||
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
desc 'fix-duplicates', 'Find duplicate remote accounts and merge them'
|
||||
option :dry_run, type: :boolean
|
||||
long_desc <<-LONG_DESC
|
||||
Merge known remote accounts sharing an ActivityPub actor identifier.
|
||||
|
||||
Such duplicates can occur when a remote server admin misconfigures their
|
||||
domain configuration.
|
||||
LONG_DESC
|
||||
def fix_duplicates
|
||||
Account.remote.select(:uri, 'count(*)').group(:uri).having('count(*) > 1').pluck(:uri).each do |uri|
|
||||
say("Duplicates found for #{uri}")
|
||||
begin
|
||||
ActivityPub::FetchRemoteAccountService.new.call(uri) unless dry_run?
|
||||
rescue => e
|
||||
say("Error processing #{uri}: #{e}", :red)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
desc 'backup USERNAME', 'Request a backup for a user'
|
||||
long_desc <<-LONG_DESC
|
||||
Request a new backup for an account with a given USERNAME.
|
||||
|
||||
The backup will be created in Sidekiq asynchronously, and
|
||||
the user will receive an e-mail with a link to it once
|
||||
it's done.
|
||||
LONG_DESC
|
||||
def backup(username)
|
||||
account = Account.find_local(username)
|
||||
|
||||
if account.nil?
|
||||
say('No user with such username', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
backup = account.user.backups.create!
|
||||
BackupWorker.perform_async(backup.id)
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :dry_run, type: :boolean
|
||||
desc 'cull [DOMAIN...]', 'Remove remote accounts that no longer exist'
|
||||
long_desc <<-LONG_DESC
|
||||
Query every single remote account in the database to determine
|
||||
if it still exists on the origin server, and if it doesn't,
|
||||
remove it from the database.
|
||||
|
||||
Accounts that have had confirmed activity within the last week
|
||||
are excluded from the checks.
|
||||
LONG_DESC
|
||||
def cull(*domains)
|
||||
skip_threshold = 7.days.ago
|
||||
skip_domains = Concurrent::Set.new
|
||||
|
||||
query = Account.remote.where(protocol: :activitypub)
|
||||
query = query.where(domain: domains) unless domains.empty?
|
||||
|
||||
processed, culled = parallelize_with_progress(query.partitioned) do |account|
|
||||
next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain)
|
||||
|
||||
code = 0
|
||||
|
||||
begin
|
||||
code = Request.new(:head, account.uri).perform(&:code)
|
||||
rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Mastodon::PrivateNetworkAddressError
|
||||
skip_domains << account.domain
|
||||
end
|
||||
|
||||
if [404, 410].include?(code)
|
||||
DeleteAccountService.new.call(account, reserve_username: false) unless dry_run?
|
||||
1
|
||||
else
|
||||
# Touch account even during dry run to avoid getting the account into the window again
|
||||
account.touch
|
||||
end
|
||||
end
|
||||
|
||||
say("Visited #{processed} accounts, removed #{culled}#{dry_run_mode_suffix}", :green)
|
||||
|
||||
unless skip_domains.empty?
|
||||
say('The following domains were not available during the check:', :yellow)
|
||||
skip_domains.each { |domain| say(" #{domain}") }
|
||||
end
|
||||
end
|
||||
|
||||
option :all, type: :boolean
|
||||
option :domain
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
option :dry_run, type: :boolean
|
||||
desc 'refresh [USERNAMES]', 'Fetch remote user data and files'
|
||||
long_desc <<-LONG_DESC
|
||||
Fetch remote user data and files for one or multiple accounts.
|
||||
|
||||
With the --all option, all remote accounts will be processed.
|
||||
Through the --domain option, this can be narrowed down to a
|
||||
specific domain only. Otherwise, remote accounts must be
|
||||
specified with space-separated USERNAMES.
|
||||
LONG_DESC
|
||||
def refresh(*usernames)
|
||||
if options[:domain] || options[:all]
|
||||
scope = Account.remote
|
||||
scope = scope.where(domain: options[:domain]) if options[:domain]
|
||||
|
||||
processed, = parallelize_with_progress(scope) do |account|
|
||||
next if dry_run?
|
||||
|
||||
account.reset_avatar!
|
||||
account.reset_header!
|
||||
account.save
|
||||
end
|
||||
|
||||
say("Refreshed #{processed} accounts#{dry_run_mode_suffix}", :green, true)
|
||||
elsif !usernames.empty?
|
||||
usernames.each do |user|
|
||||
user, domain = user.split('@')
|
||||
account = Account.find_remote(user, domain)
|
||||
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
next if dry_run?
|
||||
|
||||
begin
|
||||
account.reset_avatar!
|
||||
account.reset_header!
|
||||
account.save
|
||||
rescue Mastodon::UnexpectedResponseError
|
||||
say("Account failed: #{user}@#{domain}", :red)
|
||||
end
|
||||
end
|
||||
|
||||
say("OK#{dry_run_mode_suffix}", :green)
|
||||
else
|
||||
say('No account(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
desc 'follow USERNAME', 'Make all local accounts follow account specified by USERNAME'
|
||||
def follow(username)
|
||||
target_account = Account.find_local(username)
|
||||
|
||||
if target_account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
processed, = parallelize_with_progress(Account.local.without_suspended) do |account|
|
||||
FollowService.new.call(account, target_account, bypass_limit: true)
|
||||
end
|
||||
|
||||
say("OK, followed target from #{processed} accounts", :green)
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT'
|
||||
def unfollow(acct)
|
||||
username, domain = acct.split('@')
|
||||
target_account = Account.find_remote(username, domain)
|
||||
|
||||
if target_account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
processed, = parallelize_with_progress(target_account.followers.local) do |account|
|
||||
UnfollowService.new.call(account, target_account)
|
||||
end
|
||||
|
||||
say("OK, unfollowed target from #{processed} accounts", :green)
|
||||
end
|
||||
|
||||
option :follows, type: :boolean, default: false
|
||||
option :followers, type: :boolean, default: false
|
||||
desc 'reset-relationships USERNAME', 'Reset all follows and/or followers for a user'
|
||||
long_desc <<-LONG_DESC
|
||||
Reset all follows and/or followers for a user specified by USERNAME.
|
||||
|
||||
With the --follows option, the command unfollows everyone that the account follows,
|
||||
and then re-follows the users that would be followed by a brand new account.
|
||||
|
||||
With the --followers option, the command removes all followers of the account.
|
||||
LONG_DESC
|
||||
def reset_relationships(username)
|
||||
unless options[:follows] || options[:followers]
|
||||
say('Please specify either --follows or --followers, or both', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
account = Account.find_local(username)
|
||||
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
total = 0
|
||||
total += Account.where(id: ::Follow.where(account: account).select(:target_account_id)).count if options[:follows]
|
||||
total += Account.where(id: ::Follow.where(target_account: account).select(:account_id)).count if options[:followers]
|
||||
progress = create_progress_bar(total)
|
||||
processed = 0
|
||||
|
||||
if options[:follows]
|
||||
scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id))
|
||||
|
||||
scope.find_each do |target_account|
|
||||
UnfollowService.new.call(account, target_account)
|
||||
rescue => e
|
||||
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
|
||||
ensure
|
||||
progress.increment
|
||||
processed += 1
|
||||
end
|
||||
|
||||
BootstrapTimelineWorker.perform_async(account.id)
|
||||
end
|
||||
|
||||
if options[:followers]
|
||||
scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id))
|
||||
|
||||
scope.find_each do |target_account|
|
||||
UnfollowService.new.call(target_account, account)
|
||||
rescue => e
|
||||
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
|
||||
ensure
|
||||
progress.increment
|
||||
processed += 1
|
||||
end
|
||||
end
|
||||
|
||||
progress.finish
|
||||
say("Processed #{processed} relationships", :green, true)
|
||||
end
|
||||
|
||||
option :number, type: :numeric, aliases: [:n]
|
||||
option :all, type: :boolean
|
||||
desc 'approve [USERNAME]', 'Approve pending accounts'
|
||||
long_desc <<~LONG_DESC
|
||||
When registrations require review from staff, approve pending accounts,
|
||||
either all of them with the --all option, or a specific number of them
|
||||
specified with the --number (-n) option, or only a single specific
|
||||
account identified by its username.
|
||||
LONG_DESC
|
||||
def approve(username = nil)
|
||||
if options[:all]
|
||||
User.pending.find_each(&:approve!)
|
||||
say('OK', :green)
|
||||
elsif options[:number]&.positive?
|
||||
User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!)
|
||||
say('OK', :green)
|
||||
elsif username.present?
|
||||
account = Account.find_local(username)
|
||||
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
account.user&.approve!
|
||||
say('OK', :green)
|
||||
else
|
||||
say('Number must be positive', :red) if options[:number]
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :dry_run, type: :boolean
|
||||
desc 'prune', 'Prune remote accounts that never interacted with local users'
|
||||
long_desc <<-LONG_DESC
|
||||
Prune remote account that
|
||||
- follows no local accounts
|
||||
- is not followed by any local accounts
|
||||
- has no statuses on local
|
||||
- has not been mentioned
|
||||
- has not been favourited local posts
|
||||
- not muted/blocked by us
|
||||
LONG_DESC
|
||||
def prune
|
||||
query = Account.remote.where.not(actor_type: %i(Application Service))
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM statuses WHERE account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM follows WHERE account_id = accounts.id OR target_account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM blocks WHERE account_id = accounts.id OR target_account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM mutes WHERE target_account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM reports WHERE target_account_id = accounts.id)')
|
||||
query = query.where('NOT EXISTS (SELECT 1 FROM follow_requests WHERE account_id = accounts.id OR target_account_id = accounts.id)')
|
||||
|
||||
_, deleted = parallelize_with_progress(query) do |account|
|
||||
next if account.bot? || account.group?
|
||||
next if account.suspended?
|
||||
next if account.silenced?
|
||||
|
||||
account.destroy unless dry_run?
|
||||
1
|
||||
end
|
||||
|
||||
say("OK, pruned #{deleted} accounts#{dry_run_mode_suffix}", :green)
|
||||
end
|
||||
|
||||
option :force, type: :boolean
|
||||
option :replay, type: :boolean
|
||||
option :target
|
||||
desc 'migrate USERNAME', 'Migrate a local user to another account'
|
||||
long_desc <<~LONG_DESC
|
||||
With --replay, replay the last migration of the specified account, in
|
||||
case some remote server may not have properly processed the associated
|
||||
`Move` activity.
|
||||
|
||||
With --target, specify another account to migrate to.
|
||||
|
||||
With --force, perform the migration even if the selected account
|
||||
redirects to a different account that the one specified.
|
||||
LONG_DESC
|
||||
def migrate(username)
|
||||
if options[:replay].present? && options[:target].present?
|
||||
say('Use --replay or --target, not both', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if options[:replay].blank? && options[:target].blank?
|
||||
say('Use either --replay or --target', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
account = Account.find_local(username)
|
||||
|
||||
if account.nil?
|
||||
say("No such account: #{username}", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
migration = nil
|
||||
|
||||
if options[:replay]
|
||||
migration = account.migrations.last
|
||||
if migration.nil?
|
||||
say('The specified account has not performed any migration', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless options[:force] || migration.target_account_id == account.moved_to_account_id
|
||||
say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
if options[:target]
|
||||
target_account = ResolveAccountService.new.call(options[:target])
|
||||
|
||||
if target_account.nil?
|
||||
say("The specified target account could not be found: #{options[:target]}", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless options[:force] || account.moved_to_account_id.nil? || account.moved_to_account_id == target_account.id
|
||||
say('The specified account is redirecting to a different target account. Use --force if you want to change the migration target', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
begin
|
||||
migration = account.migrations.create!(acct: target_account.acct)
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
say("Error: #{e.message}", :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
MoveService.new.call(migration)
|
||||
|
||||
say("OK, migrated #{account.acct} to #{migration.target_account.acct}", :green)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def report_errors(errors)
|
||||
errors.each do |error|
|
||||
say('Failure/Error: ', :red)
|
||||
say(error.attribute)
|
||||
say(" #{error.type}", :red)
|
||||
end
|
||||
end
|
||||
|
||||
def rotate_keys_for_account(account, delay = 0)
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
old_key = account.private_key
|
||||
new_key = OpenSSL::PKey::RSA.new(2048)
|
||||
account.update(private_key: new_key.to_pem, public_key: new_key.public_key.to_pem)
|
||||
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, { 'sign_with' => old_key })
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,42 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative '../../../config/boot'
|
||||
require_relative '../../../config/environment'
|
||||
|
||||
require 'thor'
|
||||
require_relative 'progress_helper'
|
||||
|
||||
module Mastodon
|
||||
module CLI
|
||||
class Base < Thor
|
||||
include ProgressHelper
|
||||
|
||||
def self.exit_on_failure?
|
||||
true
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def pastel
|
||||
@pastel ||= Pastel.new
|
||||
end
|
||||
|
||||
def dry_run?
|
||||
options[:dry_run]
|
||||
end
|
||||
|
||||
def dry_run_mode_suffix
|
||||
dry_run? ? ' (DRY RUN)' : ''
|
||||
end
|
||||
|
||||
def reset_connection_pools!
|
||||
ActiveRecord::Base.establish_connection(
|
||||
ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).first.configuration_hash
|
||||
.dup
|
||||
.tap { |config| config['pool'] = options[:concurrency] + 1 }
|
||||
)
|
||||
RedisConfiguration.establish_pool(options[:concurrency])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,72 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Cache < Base
|
||||
desc 'clear', 'Clear out the cache storage'
|
||||
def clear
|
||||
Rails.cache.clear
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
desc 'recount TYPE', 'Update hard-cached counters'
|
||||
long_desc <<~LONG_DESC
|
||||
Update hard-cached counters of TYPE by counting referenced
|
||||
records from scratch. TYPE can be "accounts" or "statuses".
|
||||
|
||||
It may take a very long time to finish, depending on the
|
||||
size of the database.
|
||||
LONG_DESC
|
||||
def recount(type)
|
||||
case type
|
||||
when 'accounts'
|
||||
processed, = parallelize_with_progress(accounts_with_stats) do |account|
|
||||
recount_account_stats(account)
|
||||
end
|
||||
when 'statuses'
|
||||
processed, = parallelize_with_progress(statuses_with_stats) do |status|
|
||||
recount_status_stats(status)
|
||||
end
|
||||
else
|
||||
say("Unknown type: #{type}", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
say
|
||||
say("OK, recounted #{processed} records", :green)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def accounts_with_stats
|
||||
Account.local.includes(:account_stat)
|
||||
end
|
||||
|
||||
def statuses_with_stats
|
||||
Status.includes(:status_stat)
|
||||
end
|
||||
|
||||
def recount_account_stats(account)
|
||||
account.account_stat.tap do |account_stat|
|
||||
account_stat.following_count = account.active_relationships.count
|
||||
account_stat.followers_count = account.passive_relationships.count
|
||||
account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count
|
||||
|
||||
account_stat.save if account_stat.changed?
|
||||
end
|
||||
end
|
||||
|
||||
def recount_status_stats(status)
|
||||
status.status_stat.tap do |status_stat|
|
||||
status_stat.replies_count = status.replies.where.not(visibility: :direct).count
|
||||
status_stat.reblogs_count = status.reblogs.count
|
||||
status_stat.favourites_count = status.favourites.count
|
||||
|
||||
status_stat.save if status_stat.changed?
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,43 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'concurrent'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class CanonicalEmailBlocks < Base
|
||||
desc 'find EMAIL', 'Find a given e-mail address in the canonical e-mail blocks'
|
||||
long_desc <<-LONG_DESC
|
||||
When suspending a local user, a hash of a "canonical" version of their e-mail
|
||||
address is stored to prevent them from signing up again.
|
||||
|
||||
This command can be used to find whether a known email address is blocked.
|
||||
LONG_DESC
|
||||
def find(email)
|
||||
accts = CanonicalEmailBlock.matching_email(email)
|
||||
|
||||
if accts.empty?
|
||||
say("#{email} is not blocked", :green)
|
||||
else
|
||||
say("#{email} is blocked", :red)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'remove EMAIL', 'Remove a canonical e-mail block'
|
||||
long_desc <<-LONG_DESC
|
||||
When suspending a local user, a hash of a "canonical" version of their e-mail
|
||||
address is stored to prevent them from signing up again.
|
||||
|
||||
This command allows removing a canonical email block.
|
||||
LONG_DESC
|
||||
def remove(email)
|
||||
blocks = CanonicalEmailBlock.matching_email(email)
|
||||
|
||||
if blocks.empty?
|
||||
say("#{email} is not blocked", :green)
|
||||
else
|
||||
blocks.destroy_all
|
||||
say("Unblocked #{email}", :green)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,216 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'concurrent'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Domains < Base
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
option :dry_run, type: :boolean
|
||||
option :limited_federation_mode, type: :boolean
|
||||
option :by_uri, type: :boolean
|
||||
option :include_subdomains, type: :boolean
|
||||
option :purge_domain_blocks, type: :boolean
|
||||
desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace'
|
||||
long_desc <<-LONG_DESC
|
||||
Remove all accounts from a given DOMAIN without leaving behind any
|
||||
records. Unlike a suspension, if the DOMAIN still exists in the wild,
|
||||
it means the accounts could return if they are resolved again.
|
||||
|
||||
When the --limited-federation-mode option is given, instead of purging accounts
|
||||
from a single domain, all accounts from domains that have not been explicitly allowed
|
||||
are removed from the database.
|
||||
|
||||
When the --by-uri option is given, DOMAIN is used to match the domain part of actor
|
||||
URIs rather than the domain part of the webfinger handle. For instance, an account
|
||||
that has the handle `foo@bar.com` but whose profile is at the URL
|
||||
`https://mastodon-bar.com/users/foo`, would be purged by either
|
||||
`tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`.
|
||||
|
||||
When the --include-subdomains option is given, not only DOMAIN is deleted, but all
|
||||
subdomains as well. Note that this may be considerably slower.
|
||||
|
||||
When the --purge-domain-blocks option is given, also purge matching domain blocks.
|
||||
LONG_DESC
|
||||
def purge(*domains)
|
||||
domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) }
|
||||
account_scope = Account.none
|
||||
domain_block_scope = DomainBlock.none
|
||||
emoji_scope = CustomEmoji.none
|
||||
|
||||
# Sanity check on command arguments
|
||||
if options[:limited_federation_mode] && !domains.empty?
|
||||
say('DOMAIN parameter not supported with --limited-federation-mode', :red)
|
||||
exit(1)
|
||||
elsif domains.empty? && !options[:limited_federation_mode]
|
||||
say('No domain(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
# Build scopes from command arguments
|
||||
if options[:limited_federation_mode]
|
||||
account_scope = Account.remote.where.not(domain: DomainAllow.select(:domain))
|
||||
emoji_scope = CustomEmoji.remote.where.not(domain: DomainAllow.select(:domain))
|
||||
else
|
||||
# Handle wildcard subdomains
|
||||
subdomain_patterns = domains.filter_map { |domain| "%.#{Account.sanitize_sql_like(domain[2..])}" if domain.start_with?('*.') }
|
||||
domains = domains.filter { |domain| !domain.start_with?('*.') }
|
||||
# Handle --include-subdomains
|
||||
subdomain_patterns += domains.map { |domain| "%.#{Account.sanitize_sql_like(domain)}" } if options[:include_subdomains]
|
||||
uri_patterns = (domains.map { |domain| Account.sanitize_sql_like(domain) } + subdomain_patterns).map { |pattern| "https://#{pattern}/%" }
|
||||
|
||||
if options[:purge_domain_blocks]
|
||||
domain_block_scope = DomainBlock.where(domain: domains)
|
||||
domain_block_scope = domain_block_scope.or(DomainBlock.where(DomainBlock.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
|
||||
end
|
||||
|
||||
if options[:by_uri]
|
||||
account_scope = Account.remote.where(Account.arel_table[:uri].matches_any(uri_patterns, false, true))
|
||||
emoji_scope = CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(uri_patterns, false, true))
|
||||
else
|
||||
account_scope = Account.remote.where(domain: domains)
|
||||
account_scope = account_scope.or(Account.remote.where(Account.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
|
||||
emoji_scope = CustomEmoji.where(domain: domains)
|
||||
emoji_scope = emoji_scope.or(CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
|
||||
end
|
||||
end
|
||||
|
||||
# Actually perform the deletions
|
||||
processed, = parallelize_with_progress(account_scope) do |account|
|
||||
DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless dry_run?
|
||||
end
|
||||
|
||||
say("Removed #{processed} accounts#{dry_run_mode_suffix}", :green)
|
||||
|
||||
if options[:purge_domain_blocks]
|
||||
domain_block_count = domain_block_scope.count
|
||||
domain_block_scope.in_batches.destroy_all unless dry_run?
|
||||
say("Removed #{domain_block_count} domain blocks#{dry_run_mode_suffix}", :green)
|
||||
end
|
||||
|
||||
custom_emojis_count = emoji_scope.count
|
||||
emoji_scope.in_batches.destroy_all unless dry_run?
|
||||
|
||||
Instance.refresh unless dry_run?
|
||||
|
||||
say("Removed #{custom_emojis_count} custom emojis#{dry_run_mode_suffix}", :green)
|
||||
end
|
||||
|
||||
option :concurrency, type: :numeric, default: 50, aliases: [:c]
|
||||
option :format, type: :string, default: 'summary', aliases: [:f]
|
||||
option :exclude_suspended, type: :boolean, default: false, aliases: [:x]
|
||||
desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
|
||||
long_desc <<-LONG_DESC
|
||||
Crawl the fediverse by using the Mastodon REST API endpoints that expose
|
||||
all known peers, and collect statistics from those peers, as long as those
|
||||
peers support those API endpoints. When no START is given, the command uses
|
||||
this server's own database of known peers to seed the crawl.
|
||||
|
||||
The --concurrency (-c) option controls the number of threads performing HTTP
|
||||
requests at the same time. More threads means the crawl may complete faster.
|
||||
|
||||
The --format (-f) option controls how the data is displayed at the end. By
|
||||
default (`summary`), a summary of the statistics is returned. The other options
|
||||
are `domains`, which returns a newline-delimited list of all discovered peers,
|
||||
and `json`, which dumps all the aggregated data raw.
|
||||
|
||||
The --exclude-suspended (-x) option means that domains that are suspended
|
||||
instance-wide do not appear in the output and are not included in summaries.
|
||||
This also excludes subdomains of any of those domains.
|
||||
LONG_DESC
|
||||
def crawl(start = nil)
|
||||
stats = Concurrent::Hash.new
|
||||
processed = Concurrent::AtomicFixnum.new(0)
|
||||
failed = Concurrent::AtomicFixnum.new(0)
|
||||
start_at = Time.now.to_f
|
||||
seed = start ? [start] : Instance.pluck(:domain)
|
||||
blocked_domains = /\.?(#{DomainBlock.where(severity: 1).pluck(:domain).map { |domain| Regexp.escape(domain) }.join('|')})$/
|
||||
progress = create_progress_bar
|
||||
|
||||
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
|
||||
|
||||
work_unit = lambda do |domain|
|
||||
next if stats.key?(domain)
|
||||
next if options[:exclude_suspended] && domain.match?(blocked_domains)
|
||||
|
||||
stats[domain] = nil
|
||||
|
||||
begin
|
||||
Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res|
|
||||
next unless res.code == 200
|
||||
|
||||
stats[domain] = Oj.load(res.to_s)
|
||||
end
|
||||
|
||||
Request.new(:get, "https://#{domain}/api/v1/instance/peers").perform do |res|
|
||||
next unless res.code == 200
|
||||
|
||||
Oj.load(res.to_s).reject { |peer| stats.key?(peer) }.each do |peer|
|
||||
pool.post(peer, &work_unit)
|
||||
end
|
||||
end
|
||||
|
||||
Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res|
|
||||
next unless res.code == 200
|
||||
|
||||
stats[domain]['activity'] = Oj.load(res.to_s)
|
||||
end
|
||||
rescue
|
||||
failed.increment
|
||||
ensure
|
||||
processed.increment
|
||||
progress.increment unless progress.finished?
|
||||
end
|
||||
end
|
||||
|
||||
seed.each do |domain|
|
||||
pool.post(domain, &work_unit)
|
||||
end
|
||||
|
||||
sleep 20
|
||||
sleep 20 until pool.queue_length.zero?
|
||||
|
||||
pool.shutdown
|
||||
pool.wait_for_termination(20)
|
||||
ensure
|
||||
progress.finish
|
||||
pool.shutdown
|
||||
|
||||
case options[:format]
|
||||
when 'summary'
|
||||
stats_to_summary(stats, processed, failed, start_at)
|
||||
when 'domains'
|
||||
stats_to_domains(stats)
|
||||
when 'json'
|
||||
stats_to_json(stats)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def stats_to_summary(stats, processed, failed, start_at)
|
||||
stats.compact!
|
||||
|
||||
total_domains = stats.size
|
||||
total_users = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['stats'].is_a?(Hash) ? sum + val['stats']['user_count'].to_i : sum }
|
||||
total_active = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['logins'].to_i : sum }
|
||||
total_joined = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['registrations'].to_i : sum }
|
||||
|
||||
say("Visited #{processed.value} domains, #{failed.value} failed (#{(Time.now.to_f - start_at).round}s elapsed)", :green)
|
||||
say("Total servers: #{total_domains}", :green)
|
||||
say("Total registered: #{total_users}", :green)
|
||||
say("Total active last week: #{total_active}", :green)
|
||||
say("Total joined last week: #{total_joined}", :green)
|
||||
end
|
||||
|
||||
def stats_to_domains(stats)
|
||||
say(stats.keys.join("\n"))
|
||||
end
|
||||
|
||||
def stats_to_json(stats)
|
||||
stats.compact!
|
||||
say(Oj.dump(stats))
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,123 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'concurrent'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class EmailDomainBlocks < Base
|
||||
desc 'list', 'List blocked e-mail domains'
|
||||
def list
|
||||
EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry|
|
||||
say(entry.domain.to_s, :white)
|
||||
|
||||
EmailDomainBlock.where(parent_id: entry.id).order(id: 'DESC').find_each do |child|
|
||||
say(" #{child.domain}", :cyan)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
option :with_dns_records, type: :boolean
|
||||
desc 'add DOMAIN...', 'Block e-mail domain(s)'
|
||||
long_desc <<-LONG_DESC
|
||||
Blocking an e-mail domain prevents users from signing up
|
||||
with e-mail addresses from that domain. You can provide one or
|
||||
multiple domains to the command.
|
||||
|
||||
When the --with-dns-records option is given, an attempt to resolve the
|
||||
given domains' MX records will be made and the results will also be blocked.
|
||||
This can be helpful if you are blocking an e-mail server that has many
|
||||
different domains pointing to it as it allows you to essentially block
|
||||
it at the root.
|
||||
LONG_DESC
|
||||
def add(*domains)
|
||||
if domains.empty?
|
||||
say('No domain(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
skipped = 0
|
||||
processed = 0
|
||||
|
||||
domains.each do |domain|
|
||||
if EmailDomainBlock.exists?(domain: domain)
|
||||
say("#{domain} is already blocked.", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
other_domains = []
|
||||
if options[:with_dns_records]
|
||||
Resolv::DNS.open do |dns|
|
||||
dns.timeouts = 5
|
||||
other_domains = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a
|
||||
end
|
||||
end
|
||||
|
||||
email_domain_block = EmailDomainBlock.new(domain: domain, other_domains: other_domains)
|
||||
email_domain_block.save!
|
||||
processed += 1
|
||||
|
||||
(email_domain_block.other_domains || []).uniq.each do |hostname|
|
||||
another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block)
|
||||
|
||||
if EmailDomainBlock.exists?(domain: hostname)
|
||||
say("#{hostname} is already blocked.", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
another_email_domain_block.save!
|
||||
processed += 1
|
||||
end
|
||||
end
|
||||
|
||||
say("Added #{processed}, skipped #{skipped}", color(processed, 0))
|
||||
end
|
||||
|
||||
desc 'remove DOMAIN...', 'Remove e-mail domain blocks'
|
||||
def remove(*domains)
|
||||
if domains.empty?
|
||||
say('No domain(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
skipped = 0
|
||||
processed = 0
|
||||
failed = 0
|
||||
|
||||
domains.each do |domain|
|
||||
entry = EmailDomainBlock.find_by(domain: domain)
|
||||
|
||||
if entry.nil?
|
||||
say("#{domain} is not yet blocked.", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
children_count = EmailDomainBlock.where(parent_id: entry.id).count
|
||||
result = entry.destroy
|
||||
|
||||
if result
|
||||
processed += children_count + 1
|
||||
else
|
||||
say("#{domain} could not be unblocked.", :red)
|
||||
failed += 1
|
||||
end
|
||||
end
|
||||
|
||||
say("Removed #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed))
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def color(processed, failed)
|
||||
if !processed.zero? && failed.zero?
|
||||
:green
|
||||
elsif failed.zero?
|
||||
:yellow
|
||||
else
|
||||
:red
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,140 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rubygems/package'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Emoji < Base
|
||||
option :prefix
|
||||
option :suffix
|
||||
option :overwrite, type: :boolean
|
||||
option :unlisted, type: :boolean
|
||||
option :category
|
||||
desc 'import PATH', 'Import emoji from a TAR GZIP archive at PATH'
|
||||
long_desc <<-LONG_DESC
|
||||
Imports custom emoji from a TAR GZIP archive specified by PATH.
|
||||
|
||||
Existing emoji will be skipped unless the --overwrite option
|
||||
is provided, in which case they will be overwritten.
|
||||
|
||||
You can specify a --category under which the emojis will be
|
||||
grouped together.
|
||||
|
||||
With the --prefix option, a prefix can be added to all
|
||||
generated shortcodes. Likewise, the --suffix option controls
|
||||
the suffix of all shortcodes.
|
||||
|
||||
With the --unlisted option, the processed emoji will not be
|
||||
visible in the emoji picker (but still usable via other means)
|
||||
LONG_DESC
|
||||
def import(path)
|
||||
imported = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
category = options[:category] ? CustomEmojiCategory.find_or_create_by(name: options[:category]) : nil
|
||||
|
||||
Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
|
||||
tar.each do |entry|
|
||||
next unless entry.file? && entry.full_name.end_with?('.png', '.gif')
|
||||
|
||||
filename = File.basename(entry.full_name, '.*')
|
||||
|
||||
# Skip macOS shadow files
|
||||
next if filename.start_with?('._')
|
||||
|
||||
shortcode = [options[:prefix], filename, options[:suffix]].compact.join
|
||||
custom_emoji = CustomEmoji.local.find_by('LOWER(shortcode) = ?', shortcode.downcase)
|
||||
|
||||
if custom_emoji && !options[:overwrite]
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
|
||||
custom_emoji.image = StringIO.new(entry.read)
|
||||
custom_emoji.image_file_name = File.basename(entry.full_name)
|
||||
custom_emoji.visible_in_picker = !options[:unlisted]
|
||||
custom_emoji.category = category
|
||||
|
||||
if custom_emoji.save
|
||||
imported += 1
|
||||
else
|
||||
failed += 1
|
||||
say('Failure/Error: ', :red)
|
||||
say(entry.full_name)
|
||||
say(" #{custom_emoji.errors[:image].join(', ')}", :red)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
|
||||
end
|
||||
|
||||
option :category
|
||||
option :overwrite, type: :boolean
|
||||
desc 'export PATH', 'Export emoji to a TAR GZIP archive at PATH'
|
||||
long_desc <<-LONG_DESC
|
||||
Exports custom emoji to 'export.tar.gz' at PATH.
|
||||
|
||||
The --category option dumps only the specified category.
|
||||
If this option is not specified, all emoji will be exported.
|
||||
|
||||
The --overwrite option will overwrite an existing archive.
|
||||
LONG_DESC
|
||||
def export(path)
|
||||
exported = 0
|
||||
category = CustomEmojiCategory.find_by(name: options[:category])
|
||||
export_file_name = File.join(path, 'export.tar.gz')
|
||||
|
||||
if File.file?(export_file_name) && !options[:overwrite]
|
||||
say("Archive already exists! Use '--overwrite' to overwrite it!")
|
||||
exit 1
|
||||
end
|
||||
if category.nil? && options[:category]
|
||||
say("Unable to find category '#{options[:category]}'!")
|
||||
exit 1
|
||||
end
|
||||
|
||||
File.open(export_file_name, 'wb') do |file|
|
||||
Zlib::GzipWriter.wrap(file) do |gzip|
|
||||
Gem::Package::TarWriter.new(gzip) do |tar|
|
||||
scope = !options[:category] || category.nil? ? CustomEmoji.local : category.emojis
|
||||
scope.find_each do |emoji|
|
||||
say("Adding '#{emoji.shortcode}'...")
|
||||
tar.add_file_simple(emoji.shortcode + File.extname(emoji.image_file_name), 0o644, emoji.image_file_size) do |io|
|
||||
io.write Paperclip.io_adapters.for(emoji.image).read
|
||||
exported += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
say("Exported #{exported}")
|
||||
end
|
||||
|
||||
option :remote_only, type: :boolean
|
||||
desc 'purge', 'Remove all custom emoji'
|
||||
long_desc <<-LONG_DESC
|
||||
Removes all custom emoji.
|
||||
|
||||
With the --remote-only option, only remote emoji will be deleted.
|
||||
LONG_DESC
|
||||
def purge
|
||||
scope = options[:remote_only] ? CustomEmoji.remote : CustomEmoji
|
||||
scope.in_batches.destroy_all
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def color(green, _yellow, red)
|
||||
if !green.zero? && red.zero?
|
||||
:green
|
||||
elsif red.zero?
|
||||
:yellow
|
||||
else
|
||||
:red
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,57 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Feeds < Base
|
||||
include Redisable
|
||||
|
||||
option :all, type: :boolean, default: false
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
option :dry_run, type: :boolean, default: false
|
||||
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
|
||||
long_desc <<-LONG_DESC
|
||||
Build home and list feeds that are stored in Redis from the database.
|
||||
|
||||
With the --all option, all active users will be processed.
|
||||
Otherwise, a single user specified by USERNAME.
|
||||
LONG_DESC
|
||||
def build(username = nil)
|
||||
if options[:all] || username.nil?
|
||||
processed, = parallelize_with_progress(active_user_accounts) do |account|
|
||||
PrecomputeFeedService.new.call(account) unless dry_run?
|
||||
end
|
||||
|
||||
say("Regenerated feeds for #{processed} accounts #{dry_run_mode_suffix}", :green, true)
|
||||
elsif username.present?
|
||||
account = Account.find_local(username)
|
||||
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
PrecomputeFeedService.new.call(account) unless dry_run?
|
||||
|
||||
say("OK #{dry_run_mode_suffix}", :green, true)
|
||||
else
|
||||
say('No account(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'clear', 'Remove all home and list feeds from Redis'
|
||||
def clear
|
||||
keys = redis.keys('feed:*')
|
||||
redis.del(keys)
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def active_user_accounts
|
||||
Account.joins(:user).merge(User.active)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,143 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'rubygems/package'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class IpBlocks < Base
|
||||
option :severity, required: true, enum: %w(no_access sign_up_requires_approval sign_up_block), desc: 'Severity of the block'
|
||||
option :comment, aliases: [:c], desc: 'Optional comment'
|
||||
option :duration, aliases: [:d], type: :numeric, desc: 'Duration of the block in seconds'
|
||||
option :force, type: :boolean, aliases: [:f], desc: 'Overwrite existing blocks'
|
||||
desc 'add IP...', 'Add one or more IP blocks'
|
||||
long_desc <<-LONG_DESC
|
||||
Add one or more IP blocks. You can use CIDR syntax to
|
||||
block IP ranges. You must specify --severity of the block. All
|
||||
options will be copied for each IP block you create in one command.
|
||||
|
||||
You can add a --comment. If an IP block already exists for one of
|
||||
the provided IPs, it will be skipped unless you use the --force
|
||||
option to overwrite it.
|
||||
LONG_DESC
|
||||
def add(*addresses)
|
||||
if addresses.empty?
|
||||
say('No IP(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
skipped = 0
|
||||
processed = 0
|
||||
failed = 0
|
||||
|
||||
addresses.each do |address|
|
||||
unless valid_ip_address?(address)
|
||||
say("#{address} is invalid", :red)
|
||||
failed += 1
|
||||
next
|
||||
end
|
||||
|
||||
ip_block = IpBlock.find_by(ip: address)
|
||||
|
||||
if ip_block.present? && !options[:force]
|
||||
say("#{address} is already blocked", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
ip_block ||= IpBlock.new(ip: address)
|
||||
|
||||
ip_block.severity = options[:severity]
|
||||
ip_block.comment = options[:comment] if options[:comment].present?
|
||||
ip_block.expires_in = options[:duration]
|
||||
|
||||
if ip_block.save
|
||||
processed += 1
|
||||
else
|
||||
say("#{address} could not be saved", :red)
|
||||
failed += 1
|
||||
end
|
||||
end
|
||||
|
||||
say("Added #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed))
|
||||
end
|
||||
|
||||
option :force, type: :boolean, aliases: [:f], desc: 'Remove blocks for ranges that cover given IP(s)'
|
||||
desc 'remove IP...', 'Remove one or more IP blocks'
|
||||
long_desc <<-LONG_DESC
|
||||
Remove one or more IP blocks. Normally, only exact matches are removed. If
|
||||
you want to ensure that all of the given IP addresses are unblocked, you
|
||||
can use --force which will also remove any blocks for IP ranges that would
|
||||
cover the given IP(s).
|
||||
LONG_DESC
|
||||
def remove(*addresses)
|
||||
if addresses.empty?
|
||||
say('No IP(s) given', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
|
||||
addresses.each do |address|
|
||||
unless valid_ip_address?(address)
|
||||
say("#{address} is invalid", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
ip_blocks = if options[:force]
|
||||
IpBlock.where('ip >>= ?', address)
|
||||
else
|
||||
IpBlock.where('ip <<= ?', address)
|
||||
end
|
||||
|
||||
if ip_blocks.empty?
|
||||
say("#{address} is not yet blocked", :yellow)
|
||||
skipped += 1
|
||||
next
|
||||
end
|
||||
|
||||
ip_blocks.in_batches.destroy_all
|
||||
processed += 1
|
||||
end
|
||||
|
||||
say("Removed #{processed}, skipped #{skipped}", color(processed, 0))
|
||||
end
|
||||
|
||||
option :format, aliases: [:f], enum: %w(plain nginx), desc: 'Format of the output'
|
||||
desc 'export', 'Export blocked IPs'
|
||||
long_desc <<-LONG_DESC
|
||||
Export blocked IPs. Different formats are supported for usage with other
|
||||
tools. Only blocks with no_access severity are returned.
|
||||
LONG_DESC
|
||||
def export
|
||||
IpBlock.where(severity: :no_access).find_each do |ip_block|
|
||||
case options[:format]
|
||||
when 'nginx'
|
||||
say "deny #{ip_block.ip}/#{ip_block.ip.prefix};"
|
||||
else
|
||||
say "#{ip_block.ip}/#{ip_block.ip.prefix}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def color(processed, failed)
|
||||
if !processed.zero? && failed.zero?
|
||||
:green
|
||||
elsif failed.zero?
|
||||
:yellow
|
||||
else
|
||||
:red
|
||||
end
|
||||
end
|
||||
|
||||
def valid_ip_address?(ip_address)
|
||||
IPAddr.new(ip_address)
|
||||
true
|
||||
rescue IPAddr::InvalidAddressError
|
||||
false
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,155 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
require_relative 'accounts'
|
||||
require_relative 'cache'
|
||||
require_relative 'canonical_email_blocks'
|
||||
require_relative 'domains'
|
||||
require_relative 'email_domain_blocks'
|
||||
require_relative 'emoji'
|
||||
require_relative 'feeds'
|
||||
require_relative 'ip_blocks'
|
||||
require_relative 'maintenance'
|
||||
require_relative 'media'
|
||||
require_relative 'preview_cards'
|
||||
require_relative 'search'
|
||||
require_relative 'settings'
|
||||
require_relative 'statuses'
|
||||
require_relative 'upgrade'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Main < Base
|
||||
desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
|
||||
subcommand 'media', Media
|
||||
|
||||
desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji'
|
||||
subcommand 'emoji', Emoji
|
||||
|
||||
desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts'
|
||||
subcommand 'accounts', Accounts
|
||||
|
||||
desc 'feeds SUBCOMMAND ...ARGS', 'Manage feeds'
|
||||
subcommand 'feeds', Feeds
|
||||
|
||||
desc 'search SUBCOMMAND ...ARGS', 'Manage the search engine'
|
||||
subcommand 'search', Search
|
||||
|
||||
desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings'
|
||||
subcommand 'settings', Settings
|
||||
|
||||
desc 'statuses SUBCOMMAND ...ARGS', 'Manage statuses'
|
||||
subcommand 'statuses', Statuses
|
||||
|
||||
desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
|
||||
subcommand 'domains', Domains
|
||||
|
||||
desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards'
|
||||
subcommand 'preview_cards', PreviewCards
|
||||
|
||||
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
|
||||
subcommand 'cache', Cache
|
||||
|
||||
desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
|
||||
subcommand 'upgrade', Upgrade
|
||||
|
||||
desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks'
|
||||
subcommand 'email_domain_blocks', EmailDomainBlocks
|
||||
|
||||
desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks'
|
||||
subcommand 'ip_blocks', IpBlocks
|
||||
|
||||
desc 'canonical_email_blocks SUBCOMMAND ...ARGS', 'Manage canonical e-mail blocks'
|
||||
subcommand 'canonical_email_blocks', CanonicalEmailBlocks
|
||||
|
||||
desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
|
||||
subcommand 'maintenance', Maintenance
|
||||
|
||||
option :dry_run, type: :boolean
|
||||
desc 'self-destruct', 'Erase the server from the federation'
|
||||
long_desc <<~LONG_DESC
|
||||
Erase the server from the federation by broadcasting account delete
|
||||
activities to all known other servers. This allows a "clean exit" from
|
||||
running a Mastodon server, as it leaves next to no cache behind on
|
||||
other servers.
|
||||
|
||||
This command is always interactive and requires confirmation twice.
|
||||
|
||||
No local data is actually deleted, because emptying the
|
||||
database or removing files is much faster through other, external
|
||||
means, such as e.g. deleting the entire VPS. However, because other
|
||||
servers will delete data about local users, but no local data will be
|
||||
updated (such as e.g. followers), there will be a state mismatch
|
||||
that will lead to glitches and issues if you then continue to run and use
|
||||
the server.
|
||||
|
||||
So either you know exactly what you are doing, or you are starting
|
||||
from a blank slate afterwards by manually clearing out all the local
|
||||
data!
|
||||
LONG_DESC
|
||||
def self_destruct
|
||||
require 'tty-prompt'
|
||||
|
||||
prompt = TTY::Prompt.new
|
||||
|
||||
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
|
||||
|
||||
unless dry_run?
|
||||
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
|
||||
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
|
||||
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
|
||||
|
||||
exit(1) if prompt.no?('Are you sure you want to proceed?')
|
||||
end
|
||||
|
||||
inboxes = Account.inboxes
|
||||
processed = 0
|
||||
|
||||
Setting.registrations_mode = 'none' unless dry_run?
|
||||
|
||||
if inboxes.empty?
|
||||
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run?
|
||||
prompt.ok('It seems like your server has not federated with anything')
|
||||
prompt.ok('You can shut it down and delete it any time')
|
||||
return
|
||||
end
|
||||
|
||||
prompt.warn('Do NOT interrupt this process...')
|
||||
|
||||
delete_account = lambda do |account|
|
||||
payload = ActiveModelSerializers::SerializableResource.new(
|
||||
account,
|
||||
serializer: ActivityPub::DeleteActorSerializer,
|
||||
adapter: ActivityPub::Adapter
|
||||
).as_json
|
||||
|
||||
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
|
||||
|
||||
unless dry_run?
|
||||
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
|
||||
[json, account.id, inbox_url]
|
||||
end
|
||||
|
||||
account.suspend!(block_email: false)
|
||||
end
|
||||
|
||||
processed += 1
|
||||
end
|
||||
|
||||
Account.local.without_suspended.find_each { |account| delete_account.call(account) }
|
||||
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
|
||||
|
||||
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
|
||||
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
|
||||
rescue TTY::Reader::InputInterrupt
|
||||
exit(1)
|
||||
end
|
||||
|
||||
map %w(--version -v) => :version
|
||||
|
||||
desc 'version', 'Show version'
|
||||
def version
|
||||
say(Mastodon::Version.to_s)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,690 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Maintenance < Base
|
||||
MIN_SUPPORTED_VERSION = 2019_10_01_213028
|
||||
MAX_SUPPORTED_VERSION = 2023_09_07_150100
|
||||
|
||||
# Stubs to enjoy ActiveRecord queries while not depending on a particular
|
||||
# version of the code/database
|
||||
|
||||
class Status < ApplicationRecord; end
|
||||
class StatusPin < ApplicationRecord; end
|
||||
class Poll < ApplicationRecord; end
|
||||
class Report < ApplicationRecord; end
|
||||
class Tombstone < ApplicationRecord; end
|
||||
class Favourite < ApplicationRecord; end
|
||||
class Follow < ApplicationRecord; end
|
||||
class FollowRequest < ApplicationRecord; end
|
||||
class Block < ApplicationRecord; end
|
||||
class Mute < ApplicationRecord; end
|
||||
class AccountIdentityProof < ApplicationRecord; end
|
||||
class AccountModerationNote < ApplicationRecord; end
|
||||
class AccountPin < ApplicationRecord; end
|
||||
class ListAccount < ApplicationRecord; end
|
||||
class PollVote < ApplicationRecord; end
|
||||
class Mention < ApplicationRecord; end
|
||||
class AccountDomainBlock < ApplicationRecord; end
|
||||
class AnnouncementReaction < ApplicationRecord; end
|
||||
class FeaturedTag < ApplicationRecord; end
|
||||
class CustomEmoji < ApplicationRecord; end
|
||||
class CustomEmojiCategory < ApplicationRecord; end
|
||||
class Bookmark < ApplicationRecord; end
|
||||
class WebauthnCredential < ApplicationRecord; end
|
||||
class FollowRecommendationSuppression < ApplicationRecord; end
|
||||
class CanonicalEmailBlock < ApplicationRecord; end
|
||||
class Appeal < ApplicationRecord; end
|
||||
class Webhook < ApplicationRecord; end
|
||||
class BulkImport < ApplicationRecord; end
|
||||
class SoftwareUpdate < ApplicationRecord; end
|
||||
|
||||
class PreviewCard < ApplicationRecord
|
||||
self.inheritance_column = false
|
||||
end
|
||||
|
||||
class MediaAttachment < ApplicationRecord
|
||||
self.inheritance_column = nil
|
||||
end
|
||||
|
||||
class AccountStat < ApplicationRecord
|
||||
belongs_to :account, inverse_of: :account_stat
|
||||
end
|
||||
|
||||
# Dummy class, to make migration possible across version changes
|
||||
class Account < ApplicationRecord
|
||||
has_one :user, inverse_of: :account
|
||||
has_one :account_stat, inverse_of: :account
|
||||
|
||||
scope :local, -> { where(domain: nil) }
|
||||
|
||||
def local?
|
||||
domain.nil?
|
||||
end
|
||||
|
||||
def acct
|
||||
local? ? username : "#{username}@#{domain}"
|
||||
end
|
||||
|
||||
# This is a duplicate of the AccountMerging concern because we need it to
|
||||
# be independent from code version.
|
||||
def merge_with!(other_account)
|
||||
# Since it's the same remote resource, the remote resource likely
|
||||
# already believes we are following/blocking, so it's safe to
|
||||
# re-attribute the relationships too. However, during the presence
|
||||
# of the index bug users could have *also* followed the reference
|
||||
# account already, therefore mass update will not work and we need
|
||||
# to check for (and skip past) uniqueness errors
|
||||
|
||||
owned_classes = [
|
||||
Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
|
||||
Follow, FollowRequest, Block, Mute,
|
||||
AccountModerationNote, AccountPin, AccountStat, ListAccount,
|
||||
PollVote, Mention
|
||||
]
|
||||
owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests)
|
||||
owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
|
||||
owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
|
||||
owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
|
||||
owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals)
|
||||
owned_classes << BulkImport if ActiveRecord::Base.connection.table_exists?(:bulk_imports)
|
||||
|
||||
owned_classes.each do |klass|
|
||||
klass.where(account_id: other_account.id).find_each do |record|
|
||||
record.update_attribute(:account_id, id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
|
||||
target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
|
||||
target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
|
||||
|
||||
target_classes.each do |klass|
|
||||
klass.where(target_account_id: other_account.id).find_each do |record|
|
||||
record.update_attribute(:target_account_id, id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
|
||||
if ActiveRecord::Base.connection.table_exists?(:canonical_email_blocks)
|
||||
CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
|
||||
record.update_attribute(:reference_account_id, id)
|
||||
end
|
||||
end
|
||||
|
||||
if ActiveRecord::Base.connection.table_exists?(:appeals)
|
||||
Appeal.where(account_warning_id: other_account.id).find_each do |record|
|
||||
record.update_attribute(:account_warning_id, id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class User < ApplicationRecord
|
||||
belongs_to :account, inverse_of: :user
|
||||
end
|
||||
|
||||
desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes'
|
||||
long_desc <<~LONG_DESC
|
||||
Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes.
|
||||
|
||||
This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes
|
||||
|
||||
Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
|
||||
LONG_DESC
|
||||
def fix_duplicates
|
||||
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
|
||||
say 'Your version of the database schema is too old and is not supported by this script.', :red
|
||||
say 'Please update to at least Mastodon 3.0.0 before running this script.', :red
|
||||
exit(1)
|
||||
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
|
||||
say 'Your version of the database schema is more recent than this script, this may cause unexpected errors.', :yellow
|
||||
exit(1) unless yes?('Continue anyway? (Yes/No)')
|
||||
end
|
||||
|
||||
if Sidekiq::ProcessSet.new.any?
|
||||
say 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.', :red
|
||||
exit(1)
|
||||
end
|
||||
|
||||
say 'This task will take a long time to run and is potentially destructive.', :yellow
|
||||
say 'Please make sure to stop Mastodon and have a backup.', :yellow
|
||||
exit(1) unless yes?('Continue? (Yes/No)')
|
||||
|
||||
deduplicate_users!
|
||||
deduplicate_account_domain_blocks!
|
||||
deduplicate_account_identity_proofs!
|
||||
deduplicate_announcement_reactions!
|
||||
deduplicate_conversations!
|
||||
deduplicate_custom_emojis!
|
||||
deduplicate_custom_emoji_categories!
|
||||
deduplicate_domain_allows!
|
||||
deduplicate_domain_blocks!
|
||||
deduplicate_unavailable_domains!
|
||||
deduplicate_email_domain_blocks!
|
||||
deduplicate_media_attachments!
|
||||
deduplicate_preview_cards!
|
||||
deduplicate_statuses!
|
||||
deduplicate_accounts!
|
||||
deduplicate_tags!
|
||||
deduplicate_webauthn_credentials!
|
||||
deduplicate_webhooks!
|
||||
deduplicate_software_updates!
|
||||
|
||||
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
|
||||
Rails.cache.clear
|
||||
|
||||
say 'Finished!'
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def deduplicate_accounts!
|
||||
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
|
||||
|
||||
say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
|
||||
|
||||
find_duplicate_accounts.each do |row|
|
||||
accounts = Account.where(id: row['ids'].split(',')).to_a
|
||||
|
||||
if accounts.first.local?
|
||||
deduplicate_local_accounts!(accounts)
|
||||
else
|
||||
deduplicate_remote_accounts!(accounts)
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring index_accounts_on_username_and_domain_lower…'
|
||||
if ActiveRecord::Migrator.current_version < 2020_06_20_164023
|
||||
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
|
||||
end
|
||||
|
||||
say 'Reindexing textual indexes on accounts…'
|
||||
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
|
||||
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
|
||||
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
|
||||
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_domain_and_id;') if ActiveRecord::Migrator.current_version >= 2023_05_24_190515
|
||||
end
|
||||
|
||||
def deduplicate_users!
|
||||
remove_index_if_exists!(:users, 'index_users_on_confirmation_token')
|
||||
remove_index_if_exists!(:users, 'index_users_on_email')
|
||||
remove_index_if_exists!(:users, 'index_users_on_remember_token')
|
||||
remove_index_if_exists!(:users, 'index_users_on_reset_password_token')
|
||||
|
||||
say 'Deduplicating user records…'
|
||||
|
||||
# Deduplicating email
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
|
||||
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
|
||||
ref_user = users.shift
|
||||
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
|
||||
say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow
|
||||
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
|
||||
|
||||
users.each_with_index do |user, index|
|
||||
user.update!(email: "#{index} " + user.email)
|
||||
end
|
||||
end
|
||||
|
||||
deduplicate_users_process_confirmation_token
|
||||
deduplicate_users_process_remember_token
|
||||
deduplicate_users_process_password_token
|
||||
|
||||
say 'Restoring users indexes…'
|
||||
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
|
||||
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
|
||||
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010
|
||||
|
||||
if ActiveRecord::Migrator.current_version < 2022_03_10_060641
|
||||
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
|
||||
end
|
||||
|
||||
ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if ActiveRecord::Migrator.current_version >= 2023_07_02_151753
|
||||
end
|
||||
|
||||
def deduplicate_users_process_confirmation_token
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
|
||||
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
|
||||
say "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
|
||||
|
||||
users.each do |user|
|
||||
user.update!(confirmation_token: nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_users_process_remember_token
|
||||
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
|
||||
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
|
||||
say "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
|
||||
|
||||
users.each do |user|
|
||||
user.update!(remember_token: nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_users_process_password_token
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
|
||||
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
|
||||
say "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
|
||||
|
||||
users.each do |user|
|
||||
user.update!(reset_password_token: nil)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_account_domain_blocks!
|
||||
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
|
||||
|
||||
say 'Removing duplicate account domain blocks…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
|
||||
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
|
||||
end
|
||||
|
||||
say 'Restoring account domain blocks indexes…'
|
||||
ActiveRecord::Base.connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_account_identity_proofs!
|
||||
return unless ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
|
||||
|
||||
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
|
||||
|
||||
say 'Removing duplicate account identity proofs…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
|
||||
AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring account identity proofs indexes…'
|
||||
ActiveRecord::Base.connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_announcement_reactions!
|
||||
return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions)
|
||||
|
||||
remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
|
||||
|
||||
say 'Removing duplicate account identity proofs…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
|
||||
AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring announcement_reactions indexes…'
|
||||
ActiveRecord::Base.connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_conversations!
|
||||
remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
|
||||
|
||||
say 'Deduplicating conversations…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
|
||||
conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
||||
|
||||
ref_conversation = conversations.shift
|
||||
|
||||
conversations.each do |other|
|
||||
merge_conversations!(ref_conversation, other)
|
||||
other.destroy
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring conversations indexes…'
|
||||
if ActiveRecord::Migrator.current_version < 2022_03_07_083603
|
||||
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_custom_emojis!
|
||||
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
|
||||
|
||||
say 'Deduplicating custom_emojis…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
|
||||
emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
||||
|
||||
ref_emoji = emojis.shift
|
||||
|
||||
emojis.each do |other|
|
||||
merge_custom_emojis!(ref_emoji, other)
|
||||
other.destroy
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring custom_emojis indexes…'
|
||||
ActiveRecord::Base.connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_custom_emoji_categories!
|
||||
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
|
||||
|
||||
say 'Deduplicating custom_emoji_categories…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
|
||||
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse
|
||||
|
||||
ref_category = categories.shift
|
||||
|
||||
categories.each do |other|
|
||||
merge_custom_emoji_categories!(ref_category, other)
|
||||
other.destroy
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring custom_emoji_categories indexes…'
|
||||
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_domain_allows!
|
||||
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
|
||||
|
||||
say 'Deduplicating domain_allows…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
|
||||
DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring domain_allows indexes…'
|
||||
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_domain_blocks!
|
||||
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
|
||||
|
||||
say 'Deduplicating domain_allows…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
|
||||
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
|
||||
|
||||
reject_media = domain_blocks.any?(&:reject_media?)
|
||||
reject_reports = domain_blocks.any?(&:reject_reports?)
|
||||
|
||||
reference_block = domain_blocks.shift
|
||||
|
||||
private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence }
|
||||
public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence }
|
||||
|
||||
reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment)
|
||||
|
||||
domain_blocks.each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring domain_blocks indexes…'
|
||||
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_unavailable_domains!
|
||||
return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains)
|
||||
|
||||
remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
|
||||
|
||||
say 'Deduplicating unavailable_domains…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
|
||||
UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring domain_allows indexes…'
|
||||
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_email_domain_blocks!
|
||||
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
|
||||
|
||||
say 'Deduplicating email_domain_blocks…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
|
||||
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
|
||||
domain_blocks.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring email_domain_blocks indexes…'
|
||||
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_media_attachments!
|
||||
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
|
||||
|
||||
say 'Deduplicating media_attachments…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
|
||||
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
|
||||
end
|
||||
|
||||
say 'Restoring media_attachments indexes…'
|
||||
if ActiveRecord::Migrator.current_version < 2022_03_10_060626
|
||||
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_preview_cards!
|
||||
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
|
||||
|
||||
say 'Deduplicating preview_cards…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
|
||||
PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring preview_cards indexes…'
|
||||
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_statuses!
|
||||
remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
|
||||
|
||||
say 'Deduplicating statuses…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
|
||||
statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
|
||||
ref_status = statuses.shift
|
||||
statuses.each do |status|
|
||||
merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id
|
||||
status.destroy
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring statuses indexes…'
|
||||
if ActiveRecord::Migrator.current_version < 2022_03_10_060706
|
||||
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_tags!
|
||||
remove_index_if_exists!(:tags, 'index_tags_on_name_lower')
|
||||
remove_index_if_exists!(:tags, 'index_tags_on_name_lower_btree')
|
||||
|
||||
say 'Deduplicating tags…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
|
||||
tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
|
||||
ref_tag = tags.shift
|
||||
tags.each do |tag|
|
||||
merge_tags!(ref_tag, tag)
|
||||
tag.destroy
|
||||
end
|
||||
end
|
||||
|
||||
say 'Restoring tags indexes…'
|
||||
if ActiveRecord::Migrator.current_version < 2021_04_21_121431
|
||||
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
|
||||
else
|
||||
ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_webauthn_credentials!
|
||||
return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials)
|
||||
|
||||
remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
|
||||
|
||||
say 'Deduplicating webauthn_credentials…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
|
||||
WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring webauthn_credentials indexes…'
|
||||
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_webhooks!
|
||||
return unless ActiveRecord::Base.connection.table_exists?(:webhooks)
|
||||
|
||||
remove_index_if_exists!(:webhooks, 'index_webhooks_on_url')
|
||||
|
||||
say 'Deduplicating webhooks…'
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
|
||||
Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
|
||||
end
|
||||
|
||||
say 'Restoring webhooks indexes…'
|
||||
ActiveRecord::Base.connection.add_index :webhooks, ['url'], name: 'index_webhooks_on_url', unique: true
|
||||
end
|
||||
|
||||
def deduplicate_software_updates!
|
||||
# Not bothering with this, it's data that will be recovered with the scheduler
|
||||
SoftwareUpdate.delete_all
|
||||
end
|
||||
|
||||
def deduplicate_local_accounts!(accounts)
|
||||
accounts = accounts.sort_by(&:id).reverse
|
||||
|
||||
say "Multiple local accounts were found for username '#{accounts.first.username}'.", :yellow
|
||||
say 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.', :yellow
|
||||
|
||||
accounts.each_with_index do |account, idx|
|
||||
say format(
|
||||
'%<index>2d. %<username>s: created at: %<created_at>s; updated at: %<updated_at>s; last logged in at: %<last_log_in_at>s; statuses: %<status_count>5d; last status at: %<last_status_at>s',
|
||||
index: idx,
|
||||
username: account.username,
|
||||
created_at: account.created_at,
|
||||
updated_at: account.updated_at,
|
||||
last_log_in_at: account.user&.last_sign_in_at&.to_s || 'N/A',
|
||||
status_count: account.account_stat&.statuses_count || 0,
|
||||
last_status_at: account.account_stat&.last_status_at || 'N/A'
|
||||
)
|
||||
end
|
||||
|
||||
say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
|
||||
|
||||
ref_id = ask('Account to keep unchanged:') do |q|
|
||||
q.required true
|
||||
q.default 0
|
||||
q.convert :int
|
||||
end
|
||||
|
||||
accounts.delete_at(ref_id)
|
||||
|
||||
i = 0
|
||||
accounts.each do |account|
|
||||
i += 1
|
||||
username = account.username + "_#{i}"
|
||||
|
||||
while Account.local.exists?(username: username)
|
||||
i += 1
|
||||
username = account.username + "_#{i}"
|
||||
end
|
||||
|
||||
account.update!(username: username)
|
||||
end
|
||||
end
|
||||
|
||||
def deduplicate_remote_accounts!(accounts)
|
||||
accounts = accounts.sort_by(&:updated_at).reverse
|
||||
|
||||
reference_account = accounts.shift
|
||||
|
||||
accounts.each do |other_account|
|
||||
if other_account.public_key == reference_account.public_key
|
||||
# The accounts definitely point to the same resource, so
|
||||
# it's safe to re-attribute content and relationships
|
||||
reference_account.merge_with!(other_account)
|
||||
end
|
||||
|
||||
other_account.destroy
|
||||
end
|
||||
end
|
||||
|
||||
def merge_conversations!(main_conv, duplicate_conv)
|
||||
owned_classes = [ConversationMute, AccountConversation]
|
||||
owned_classes.each do |klass|
|
||||
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
|
||||
record.update_attribute(:account_id, main_conv.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def merge_custom_emojis!(main_emoji, duplicate_emoji)
|
||||
owned_classes = [AnnouncementReaction]
|
||||
owned_classes.each do |klass|
|
||||
klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id)
|
||||
end
|
||||
end
|
||||
|
||||
def merge_custom_emoji_categories!(main_category, duplicate_category)
|
||||
owned_classes = [CustomEmoji]
|
||||
owned_classes.each do |klass|
|
||||
klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id)
|
||||
end
|
||||
end
|
||||
|
||||
def merge_statuses!(main_status, duplicate_status)
|
||||
owned_classes = [Favourite, Mention, Poll]
|
||||
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
|
||||
owned_classes.each do |klass|
|
||||
klass.where(status_id: duplicate_status.id).find_each do |record|
|
||||
record.update_attribute(:status_id, main_status.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
|
||||
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
|
||||
record.update_attribute(:status_id, main_status.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
|
||||
Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
|
||||
record.update_attribute(:in_reply_to_id, main_status.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
|
||||
Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
|
||||
record.update_attribute(:reblog_of_id, main_status.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
|
||||
def merge_tags!(main_tag, duplicate_tag)
|
||||
[FeaturedTag].each do |klass|
|
||||
klass.where(tag_id: duplicate_tag.id).find_each do |record|
|
||||
record.update_attribute(:tag_id, main_tag.id)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def find_duplicate_accounts
|
||||
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
|
||||
end
|
||||
|
||||
def remove_index_if_exists!(table, name)
|
||||
ActiveRecord::Base.connection.remove_index(table, name: name)
|
||||
rescue ArgumentError, ActiveRecord::StatementInvalid
|
||||
nil
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,376 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Media < Base
|
||||
include ActionView::Helpers::NumberHelper
|
||||
|
||||
VALID_PATH_SEGMENTS_SIZE = [7, 10].freeze
|
||||
|
||||
option :days, type: :numeric, default: 7, aliases: [:d]
|
||||
option :prune_profiles, type: :boolean, default: false
|
||||
option :remove_headers, type: :boolean, default: false
|
||||
option :include_follows, type: :boolean, default: false
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :dry_run, type: :boolean, default: false
|
||||
desc 'remove', 'Remove remote media files, headers or avatars'
|
||||
long_desc <<-DESC
|
||||
Removes locally cached copies of media attachments (and optionally profile
|
||||
headers and avatars) from other servers. By default, only media attachments
|
||||
are removed.
|
||||
The --days option specifies how old media attachments have to be before
|
||||
they are removed. In case of avatars and headers, it specifies how old
|
||||
the last webfinger request and update to the user has to be before they
|
||||
are pruned. It defaults to 7 days.
|
||||
If --prune-profiles is specified, only avatars and headers are removed.
|
||||
If --remove-headers is specified, only headers are removed.
|
||||
If --include-follows is specified along with --prune-profiles or
|
||||
--remove-headers, all non-local profiles will be pruned irrespective of
|
||||
follow status. By default, only accounts that are not followed by or
|
||||
following anyone locally are pruned.
|
||||
DESC
|
||||
def remove
|
||||
if options[:prune_profiles] && options[:remove_headers]
|
||||
say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
|
||||
say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
|
||||
exit(1)
|
||||
end
|
||||
time_ago = options[:days].days.ago
|
||||
|
||||
if options[:prune_profiles] || options[:remove_headers]
|
||||
processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
|
||||
next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists?
|
||||
next if account.avatar.blank? && account.header.blank?
|
||||
next if options[:remove_headers] && account.header.blank?
|
||||
|
||||
size = (account.header_file_size || 0)
|
||||
size += (account.avatar_file_size || 0) if options[:prune_profiles]
|
||||
|
||||
unless dry_run?
|
||||
account.header.destroy
|
||||
account.avatar.destroy if options[:prune_profiles]
|
||||
account.save!
|
||||
end
|
||||
|
||||
size
|
||||
end
|
||||
|
||||
say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
|
||||
unless options[:prune_profiles] || options[:remove_headers]
|
||||
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment|
|
||||
next if media_attachment.file.blank?
|
||||
|
||||
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
|
||||
|
||||
unless dry_run?
|
||||
media_attachment.file.destroy
|
||||
media_attachment.thumbnail.destroy
|
||||
media_attachment.save
|
||||
end
|
||||
|
||||
size
|
||||
end
|
||||
|
||||
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
end
|
||||
|
||||
option :start_after
|
||||
option :prefix
|
||||
option :fix_permissions, type: :boolean, default: false
|
||||
option :dry_run, type: :boolean, default: false
|
||||
desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
|
||||
long_desc <<~LONG_DESC
|
||||
Scans file storage for files that do not belong to existing media attachments. Because this operation
|
||||
requires iterating over every single file individually, it will be slow.
|
||||
|
||||
Please mind that some storage providers charge for the necessary API requests to list objects.
|
||||
LONG_DESC
|
||||
def remove_orphans
|
||||
progress = create_progress_bar(nil)
|
||||
reclaimed_bytes = 0
|
||||
removed = 0
|
||||
prefix = options[:prefix]
|
||||
|
||||
case Paperclip::Attachment.default_options[:storage]
|
||||
when :s3
|
||||
paperclip_instance = MediaAttachment.new.file
|
||||
s3_interface = paperclip_instance.s3_interface
|
||||
s3_permissions = Paperclip::Attachment.default_options[:s3_permissions]
|
||||
bucket = s3_interface.bucket(Paperclip::Attachment.default_options[:s3_credentials][:bucket])
|
||||
last_key = options[:start_after]
|
||||
|
||||
loop do
|
||||
objects = begin
|
||||
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
|
||||
rescue => e
|
||||
progress.log(pastel.red("Error fetching list of files: #{e}"))
|
||||
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
|
||||
break
|
||||
end
|
||||
|
||||
break if objects.empty?
|
||||
|
||||
last_key = objects.last.key
|
||||
record_map = preload_records_from_mixed_objects(objects)
|
||||
|
||||
objects.each do |object|
|
||||
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !dry_run?
|
||||
|
||||
path_segments = object.key.split('/')
|
||||
path_segments.delete('cache')
|
||||
|
||||
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
|
||||
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
|
||||
next
|
||||
end
|
||||
|
||||
model_name = path_segments.first.classify
|
||||
attachment_name = path_segments[1].singularize
|
||||
record_id = path_segments[2..-2].join.to_i
|
||||
file_name = path_segments.last
|
||||
record = record_map.dig(model_name, record_id)
|
||||
attachment = record&.public_send(attachment_name)
|
||||
|
||||
progress.increment
|
||||
|
||||
next unless attachment.blank? || !attachment.variant?(file_name)
|
||||
|
||||
begin
|
||||
object.delete unless dry_run?
|
||||
|
||||
reclaimed_bytes += object.size
|
||||
removed += 1
|
||||
|
||||
progress.log("Found and removed orphan: #{object.key}")
|
||||
rescue => e
|
||||
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
|
||||
end
|
||||
end
|
||||
end
|
||||
when :fog
|
||||
say('The fog storage driver is not supported for this operation at this time', :red)
|
||||
exit(1)
|
||||
when :azure
|
||||
say('The azure storage driver is not supported for this operation at this time', :red)
|
||||
exit(1)
|
||||
when :filesystem
|
||||
require 'find'
|
||||
|
||||
root_path = ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
|
||||
|
||||
Find.find(File.join(*[root_path, prefix].compact)) do |path|
|
||||
next if File.directory?(path)
|
||||
|
||||
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
|
||||
|
||||
path_segments = key.split(File::SEPARATOR)
|
||||
path_segments.delete('cache')
|
||||
|
||||
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
|
||||
progress.log(pastel.yellow("Unrecognized file found: #{key}"))
|
||||
next
|
||||
end
|
||||
|
||||
model_name = path_segments.first.classify
|
||||
record_id = path_segments[2..-2].join.to_i
|
||||
attachment_name = path_segments[1].singularize
|
||||
file_name = path_segments.last
|
||||
|
||||
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
|
||||
|
||||
record = model_name.constantize.find_by(id: record_id)
|
||||
attachment = record&.public_send(attachment_name)
|
||||
|
||||
progress.increment
|
||||
|
||||
next unless attachment.blank? || !attachment.variant?(file_name)
|
||||
|
||||
begin
|
||||
size = File.size(path)
|
||||
|
||||
unless dry_run?
|
||||
File.delete(path)
|
||||
begin
|
||||
FileUtils.rmdir(File.dirname(path), parents: true)
|
||||
rescue Errno::ENOTEMPTY
|
||||
# OK
|
||||
end
|
||||
end
|
||||
|
||||
reclaimed_bytes += size
|
||||
removed += 1
|
||||
|
||||
progress.log("Found and removed orphan: #{key}")
|
||||
rescue => e
|
||||
progress.log(pastel.red("Error processing #{key}: #{e}"))
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
progress.total = progress.progress
|
||||
progress.finish
|
||||
|
||||
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
|
||||
option :account, type: :string
|
||||
option :domain, type: :string
|
||||
option :status, type: :numeric
|
||||
option :days, type: :numeric
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, default: false, aliases: [:v]
|
||||
option :dry_run, type: :boolean, default: false
|
||||
option :force, type: :boolean, default: false
|
||||
desc 'refresh', 'Fetch remote media files'
|
||||
long_desc <<-DESC
|
||||
Re-downloads media attachments from other servers. You must specify the
|
||||
source of media attachments with one of the following options:
|
||||
|
||||
Use the --status option to download attachments from a specific status,
|
||||
using the status local numeric ID.
|
||||
|
||||
Use the --account option to download attachments from a specific account,
|
||||
using username@domain handle of the account.
|
||||
|
||||
Use the --domain option to download attachments from a specific domain.
|
||||
|
||||
Use the --days option to limit attachments created within days.
|
||||
|
||||
By default, attachments that are believed to be already downloaded will
|
||||
not be re-downloaded. To force re-download of every URL, use --force.
|
||||
DESC
|
||||
def refresh
|
||||
if options[:status]
|
||||
scope = MediaAttachment.where(status_id: options[:status])
|
||||
elsif options[:account]
|
||||
username, domain = options[:account].split('@')
|
||||
account = Account.find_remote(username, domain)
|
||||
|
||||
if account.nil?
|
||||
say('No such account', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
scope = MediaAttachment.where(account_id: account.id)
|
||||
elsif options[:domain]
|
||||
scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain]))
|
||||
elsif options[:days].present?
|
||||
scope = MediaAttachment.remote
|
||||
else
|
||||
exit(1)
|
||||
end
|
||||
|
||||
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) if options[:days].present?
|
||||
|
||||
processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
|
||||
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
|
||||
next if DomainBlock.reject_media?(media_attachment.account.domain)
|
||||
|
||||
unless dry_run?
|
||||
media_attachment.reset_file!
|
||||
media_attachment.reset_thumbnail!
|
||||
media_attachment.save
|
||||
end
|
||||
|
||||
media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0)
|
||||
end
|
||||
|
||||
say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
|
||||
desc 'usage', 'Calculate disk space consumed by Mastodon'
|
||||
def usage
|
||||
say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} local)")
|
||||
say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)")
|
||||
say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}")
|
||||
say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")
|
||||
say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)")
|
||||
say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}")
|
||||
say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}")
|
||||
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
|
||||
end
|
||||
|
||||
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
|
||||
def lookup(url)
|
||||
path = Addressable::URI.parse(url).path
|
||||
|
||||
path_segments = path.split('/')[2..]
|
||||
path_segments.delete('cache')
|
||||
|
||||
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
|
||||
say('Not a media URL', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
model_name = path_segments.first.classify
|
||||
record_id = path_segments[2..-2].join.to_i
|
||||
|
||||
unless PRELOAD_MODEL_WHITELIST.include?(model_name)
|
||||
say("Cannot find corresponding model: #{model_name}", :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
record = model_name.constantize.find_by(id: record_id)
|
||||
record = record.status if record.respond_to?(:status)
|
||||
|
||||
unless record
|
||||
say('Cannot find corresponding record', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
display_url = ActivityPub::TagManager.instance.url_for(record)
|
||||
|
||||
if display_url.blank?
|
||||
say('No public URL for this type of record', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
say(display_url, :blue)
|
||||
rescue Addressable::URI::InvalidURIError
|
||||
say('Invalid URL', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
PRELOAD_MODEL_WHITELIST = %w(
|
||||
Account
|
||||
Backup
|
||||
CustomEmoji
|
||||
Import
|
||||
MediaAttachment
|
||||
PreviewCard
|
||||
SiteUpload
|
||||
).freeze
|
||||
|
||||
def preload_records_from_mixed_objects(objects)
|
||||
preload_map = Hash.new { |hash, key| hash[key] = [] }
|
||||
|
||||
objects.map do |object|
|
||||
segments = object.key.split('/')
|
||||
segments.delete('cache')
|
||||
|
||||
next unless VALID_PATH_SEGMENTS_SIZE.include?(segments.size)
|
||||
|
||||
model_name = segments.first.classify
|
||||
record_id = segments[2..-2].join.to_i
|
||||
|
||||
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
|
||||
|
||||
preload_map[model_name] << record_id
|
||||
end
|
||||
|
||||
preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
|
||||
model_map[model_name] = model_name.constantize.where(id: record_ids).index_by(&:id)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,51 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'tty-prompt'
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class PreviewCards < Base
|
||||
include ActionView::Helpers::NumberHelper
|
||||
|
||||
option :days, type: :numeric, default: 180
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c]
|
||||
option :verbose, type: :boolean, aliases: [:v]
|
||||
option :dry_run, type: :boolean, default: false
|
||||
option :link, type: :boolean, default: false
|
||||
desc 'remove', 'Remove preview cards'
|
||||
long_desc <<-DESC
|
||||
Removes local thumbnails for preview cards.
|
||||
|
||||
The --days option specifies how old preview cards have to be before
|
||||
they are removed. It defaults to 180 days. Since preview cards will
|
||||
not be re-fetched unless the link is re-posted after 2 weeks from
|
||||
last time, it is not recommended to delete preview cards within the
|
||||
last 14 days.
|
||||
|
||||
With the --link option, only link-type preview cards will be deleted,
|
||||
leaving video and photo cards untouched.
|
||||
DESC
|
||||
def remove
|
||||
time_ago = options[:days].days.ago
|
||||
link = options[:link] ? 'link-type ' : ''
|
||||
scope = PreviewCard.cached
|
||||
scope = scope.where(type: :link) if options[:link]
|
||||
scope = scope.where('updated_at < ?', time_ago)
|
||||
|
||||
processed, aggregate = parallelize_with_progress(scope) do |preview_card|
|
||||
next if preview_card.image.blank?
|
||||
|
||||
size = preview_card.image_file_size
|
||||
|
||||
unless dry_run?
|
||||
preview_card.image.destroy
|
||||
preview_card.save
|
||||
end
|
||||
|
||||
size
|
||||
end
|
||||
|
||||
say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,87 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
dev_null = Logger.new('/dev/null')
|
||||
|
||||
Rails.logger = dev_null
|
||||
ActiveRecord::Base.logger = dev_null
|
||||
ActiveJob::Base.logger = dev_null
|
||||
HttpLog.configuration.logger = dev_null
|
||||
Paperclip.options[:log] = false
|
||||
Chewy.logger = dev_null
|
||||
|
||||
require 'ruby-progressbar/outputs/null'
|
||||
|
||||
module Mastodon::CLI
|
||||
module ProgressHelper
|
||||
PROGRESS_FORMAT = '%c/%u |%b%i| %e'
|
||||
|
||||
def create_progress_bar(total = nil)
|
||||
ProgressBar.create(
|
||||
{
|
||||
total: total,
|
||||
format: PROGRESS_FORMAT,
|
||||
}.merge(progress_output_options)
|
||||
)
|
||||
end
|
||||
|
||||
def parallelize_with_progress(scope)
|
||||
if options[:concurrency] < 1
|
||||
say('Cannot run with this concurrency setting, must be at least 1', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
reset_connection_pools!
|
||||
|
||||
progress = create_progress_bar(scope.count)
|
||||
pool = Concurrent::FixedThreadPool.new(options[:concurrency])
|
||||
total = Concurrent::AtomicFixnum.new(0)
|
||||
aggregate = Concurrent::AtomicFixnum.new(0)
|
||||
|
||||
scope.reorder(nil).find_in_batches do |items|
|
||||
futures = []
|
||||
|
||||
items.each do |item|
|
||||
futures << Concurrent::Future.execute(executor: pool) do
|
||||
if !progress.total.nil? && progress.progress + 1 > progress.total
|
||||
# The number of items has changed between start and now,
|
||||
# since there is no good way to predict the final count from
|
||||
# here, just change the progress bar to an indeterminate one
|
||||
|
||||
progress.total = nil
|
||||
end
|
||||
|
||||
progress.log("Processing #{item.id}") if options[:verbose]
|
||||
|
||||
Chewy.strategy(:mastodon) do
|
||||
result = ActiveRecord::Base.connection_pool.with_connection do
|
||||
yield(item)
|
||||
ensure
|
||||
RedisConfiguration.pool.checkin if Thread.current[:redis]
|
||||
Thread.current[:redis] = nil
|
||||
end
|
||||
|
||||
aggregate.increment(result) if result.is_a?(Integer)
|
||||
end
|
||||
rescue => e
|
||||
progress.log pastel.red("Error processing #{item.id}: #{e}")
|
||||
ensure
|
||||
progress.increment
|
||||
end
|
||||
end
|
||||
|
||||
total.increment(items.size)
|
||||
futures.map(&:value)
|
||||
end
|
||||
|
||||
progress.stop
|
||||
|
||||
[total.value, aggregate.value]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def progress_output_options
|
||||
Rails.env.test? ? { output: ProgressBar::Outputs::Null } : {}
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,120 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Search < Base
|
||||
# Indices are sorted by amount of data to be expected in each, so that
|
||||
# smaller indices can go online sooner
|
||||
INDICES = [
|
||||
InstancesIndex,
|
||||
AccountsIndex,
|
||||
TagsIndex,
|
||||
PublicStatusesIndex,
|
||||
StatusesIndex,
|
||||
].freeze
|
||||
|
||||
option :concurrency, type: :numeric, default: 5, aliases: [:c], desc: 'Workload will be split between this number of threads'
|
||||
option :batch_size, type: :numeric, default: 100, aliases: [:b], desc: 'Number of records in each batch'
|
||||
option :only, type: :array, enum: %w(instances accounts tags statuses public_statuses), desc: 'Only process these indices'
|
||||
option :import, type: :boolean, default: true, desc: 'Import data from the database to the index'
|
||||
option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index'
|
||||
option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index"
|
||||
desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them'
|
||||
long_desc <<~LONG_DESC
|
||||
If Elasticsearch is empty, this command will create the necessary indices
|
||||
and then import data from the database into those indices.
|
||||
|
||||
This command will also upgrade indices if the underlying schema has been
|
||||
changed since the last run. Index upgrades erase index data.
|
||||
|
||||
Even if creating or upgrading indices is not necessary, data from the
|
||||
database will be imported into the indices, unless overridden with --no-import.
|
||||
LONG_DESC
|
||||
def deploy
|
||||
verify_deploy_options!
|
||||
|
||||
indices = if options[:only]
|
||||
options[:only].map { |str| "#{str.camelize}Index".constantize }
|
||||
else
|
||||
INDICES
|
||||
end
|
||||
|
||||
pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
|
||||
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) }
|
||||
progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false)
|
||||
|
||||
Chewy::Stash::Specification.reset! if options[:reset_chewy]
|
||||
|
||||
# First, ensure all indices are created and have the correct
|
||||
# structure, so that live data can already be written
|
||||
indices.select { |index| index.specification.changed? }.each do |index|
|
||||
progress.title = "Upgrading #{index} "
|
||||
index.purge
|
||||
index.specification.lock!
|
||||
end
|
||||
|
||||
progress.title = 'Estimating workload '
|
||||
progress.total = indices.sum { |index| importers[index].estimate! }
|
||||
|
||||
reset_connection_pools!
|
||||
|
||||
added = 0
|
||||
removed = 0
|
||||
|
||||
indices.each do |index|
|
||||
importer = importers[index]
|
||||
importer.optimize_for_import!
|
||||
|
||||
importer.on_progress do |(indexed, deleted)|
|
||||
progress.total = nil if progress.progress + indexed + deleted > progress.total
|
||||
progress.progress += indexed + deleted
|
||||
added += indexed
|
||||
removed += deleted
|
||||
end
|
||||
|
||||
importer.on_failure do |reason|
|
||||
progress.log(pastel.red("Error while importing #{index}: #{reason}"))
|
||||
end
|
||||
|
||||
if options[:import]
|
||||
progress.title = "Importing #{index} "
|
||||
importer.import!
|
||||
end
|
||||
|
||||
if options[:clean]
|
||||
progress.title = "Cleaning #{index} "
|
||||
importer.clean_up!
|
||||
end
|
||||
ensure
|
||||
importer.optimize_for_search!
|
||||
end
|
||||
|
||||
progress.title = 'Done! '
|
||||
progress.finish
|
||||
|
||||
say("Indexed #{added} records, de-indexed #{removed}", :green, true)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def verify_deploy_options!
|
||||
verify_deploy_concurrency!
|
||||
verify_deploy_batch_size!
|
||||
end
|
||||
|
||||
def verify_deploy_concurrency!
|
||||
return unless options[:concurrency] < 1
|
||||
|
||||
say('Cannot run with this concurrency setting, must be at least 1', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
def verify_deploy_batch_size!
|
||||
return unless options[:batch_size] < 1
|
||||
|
||||
say('Cannot run with this batch_size setting, must be at least 1', :red)
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,38 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Registrations < Base
|
||||
desc 'open', 'Open registrations'
|
||||
def open
|
||||
Setting.registrations_mode = 'open'
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
desc 'approved', 'Open approval-based registrations'
|
||||
option :require_reason, type: :boolean, aliases: [:require_invite_text]
|
||||
long_desc <<~LONG_DESC
|
||||
Set registrations to require review from staff.
|
||||
|
||||
With --require-reason, require users to enter a reason when registering,
|
||||
otherwise this field is optional.
|
||||
LONG_DESC
|
||||
def approved
|
||||
Setting.registrations_mode = 'approved'
|
||||
Setting.require_invite_text = options[:require_reason] unless options[:require_reason].nil?
|
||||
say('OK', :green)
|
||||
end
|
||||
|
||||
desc 'close', 'Close registrations'
|
||||
def close
|
||||
Setting.registrations_mode = 'none'
|
||||
say('OK', :green)
|
||||
end
|
||||
end
|
||||
|
||||
class Settings < Base
|
||||
desc 'registrations SUBCOMMAND ...ARGS', 'Manage state of registrations'
|
||||
subcommand 'registrations', Registrations
|
||||
end
|
||||
end
|
||||
@@ -1,219 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Statuses < Base
|
||||
include ActionView::Helpers::NumberHelper
|
||||
|
||||
option :days, type: :numeric, default: 90
|
||||
option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch'
|
||||
option :continue, type: :boolean, default: false, desc: 'If remove is not completed, execute from the previous continuation'
|
||||
option :clean_followed, type: :boolean, default: false, desc: 'Include the status of remote accounts that are followed by local accounts as candidates for remove'
|
||||
option :skip_status_remove, type: :boolean, default: false, desc: 'Skip status remove (run only cleanup tasks)'
|
||||
option :skip_media_remove, type: :boolean, default: false, desc: 'Skip remove orphaned media attachments'
|
||||
option :compress_database, type: :boolean, default: false, desc: 'Compress database and update the statistics. This option locks the table for a long time, so run it offline'
|
||||
desc 'remove', 'Remove unreferenced statuses'
|
||||
long_desc <<~LONG_DESC
|
||||
Remove statuses that are not referenced by local user activity, such as
|
||||
ones that came from relays, or belonging to users that were once followed
|
||||
by someone locally but no longer are.
|
||||
|
||||
It also removes orphaned records and performs additional cleanup tasks
|
||||
such as updating statistics and recovering disk space.
|
||||
|
||||
This is a computationally heavy procedure that creates extra database
|
||||
indices before commencing, and removes them afterward.
|
||||
LONG_DESC
|
||||
def remove
|
||||
if options[:batch_size] < 1
|
||||
say('Cannot run with this batch_size setting, must be at least 1', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
remove_statuses
|
||||
vacuum_and_analyze_statuses
|
||||
remove_orphans_media_attachments
|
||||
remove_orphans_conversations
|
||||
vacuum_and_analyze_conversations
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def remove_statuses
|
||||
return if options[:skip_status_remove]
|
||||
|
||||
say('Creating temporary database indices...')
|
||||
|
||||
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
|
||||
|
||||
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
|
||||
start_at = Time.now.to_f
|
||||
|
||||
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
|
||||
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true)
|
||||
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true)
|
||||
|
||||
say('Extract the deletion target from statuses... This might take a while...')
|
||||
|
||||
ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true)
|
||||
|
||||
# Skip accounts followed by local accounts
|
||||
clean_followed_sql = 'AND NOT EXISTS (SELECT 1 FROM follows WHERE statuses.account_id = follows.target_account_id)' unless options[:clean_followed]
|
||||
|
||||
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL', [max_id])
|
||||
INSERT INTO statuses_to_be_deleted (id)
|
||||
SELECT statuses.id FROM statuses WHERE deleted_at IS NULL AND NOT local AND uri IS NOT NULL AND (id < $1)
|
||||
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses1.id = statuses.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local))
|
||||
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local OR statuses1.id >= $1))
|
||||
AND NOT EXISTS (SELECT 1 FROM status_pins WHERE statuses.id = status_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
||||
AND NOT EXISTS (SELECT 1 FROM favourites WHERE statuses.id = favourites.status_id AND favourites.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
||||
AND NOT EXISTS (SELECT 1 FROM bookmarks WHERE statuses.id = bookmarks.status_id AND bookmarks.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
||||
#{clean_followed_sql}
|
||||
SQL
|
||||
|
||||
say('Removing temporary database indices to restore write performance...')
|
||||
|
||||
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
|
||||
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
|
||||
end
|
||||
|
||||
say('Beginning statuses removal... This might take a while...')
|
||||
|
||||
klass = Class.new(ApplicationRecord) do |c|
|
||||
c.table_name = 'statuses_to_be_deleted'
|
||||
end
|
||||
|
||||
Object.const_set(:StatusToBeDeleted, klass)
|
||||
|
||||
scope = StatusToBeDeleted
|
||||
processed = 0
|
||||
removed = 0
|
||||
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
|
||||
|
||||
scope.reorder(nil).in_batches(of: options[:batch_size]) do |relation|
|
||||
ids = relation.pluck(:id)
|
||||
processed += ids.count
|
||||
removed += Status.unscoped.where(id: ids).delete_all
|
||||
progress.increment
|
||||
end
|
||||
|
||||
progress.stop
|
||||
|
||||
ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted')
|
||||
|
||||
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green)
|
||||
ensure
|
||||
say('Removing temporary database indices to restore write performance...')
|
||||
|
||||
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
|
||||
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
|
||||
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true)
|
||||
end
|
||||
|
||||
def remove_orphans_media_attachments
|
||||
return if options[:skip_media_remove]
|
||||
|
||||
start_at = Time.now.to_f
|
||||
|
||||
say('Beginning removal of now-orphaned media attachments to free up disk space...')
|
||||
|
||||
scope = MediaAttachment.reorder(nil).unattached.where('created_at < ?', options[:days].pred.days.ago)
|
||||
processed = 0
|
||||
removed = 0
|
||||
progress = create_progress_bar(scope.count)
|
||||
|
||||
scope.find_each do |media_attachment|
|
||||
media_attachment.destroy!
|
||||
|
||||
removed += 1
|
||||
rescue => e
|
||||
progress.log pastel.red("Error processing #{media_attachment.id}: #{e}")
|
||||
ensure
|
||||
progress.increment
|
||||
processed += 1
|
||||
end
|
||||
|
||||
progress.stop
|
||||
|
||||
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} media_attachments.", :green)
|
||||
end
|
||||
|
||||
def remove_orphans_conversations
|
||||
start_at = Time.now.to_f
|
||||
|
||||
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('conversations_to_be_deleted')
|
||||
say('Creating temporary database indices...')
|
||||
|
||||
ActiveRecord::Base.connection.add_index(:statuses, :conversation_id, name: :index_statuses_conversation_id, algorithm: :concurrently, if_not_exists: true)
|
||||
|
||||
say('Extract the deletion target from conversations... This might take a while...')
|
||||
|
||||
ActiveRecord::Base.connection.create_table('conversations_to_be_deleted', force: true)
|
||||
|
||||
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL')
|
||||
INSERT INTO conversations_to_be_deleted (id)
|
||||
SELECT id FROM conversations WHERE NOT EXISTS (SELECT 1 FROM statuses WHERE statuses.conversation_id = conversations.id)
|
||||
SQL
|
||||
|
||||
say('Removing temporary database indices to restore write performance...')
|
||||
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
|
||||
end
|
||||
|
||||
say('Beginning orphans removal... This might take a while...')
|
||||
|
||||
klass = Class.new(ApplicationRecord) do |c|
|
||||
c.table_name = 'conversations_to_be_deleted'
|
||||
end
|
||||
|
||||
Object.const_set(:ConversationsToBeDeleted, klass)
|
||||
|
||||
scope = ConversationsToBeDeleted
|
||||
processed = 0
|
||||
removed = 0
|
||||
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
|
||||
|
||||
scope.in_batches(of: options[:batch_size]) do |relation|
|
||||
ids = relation.pluck(:id)
|
||||
processed += ids.count
|
||||
removed += Conversation.unscoped.where(id: ids).delete_all
|
||||
progress.increment
|
||||
end
|
||||
|
||||
progress.stop
|
||||
|
||||
ActiveRecord::Base.connection.drop_table('conversations_to_be_deleted')
|
||||
|
||||
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} conversations.", :green)
|
||||
ensure
|
||||
say('Removing temporary database indices to restore write performance...')
|
||||
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
|
||||
end
|
||||
|
||||
def vacuum_and_analyze_statuses
|
||||
if options[:compress_database]
|
||||
say('Run VACUUM FULL ANALYZE to statuses...')
|
||||
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE statuses')
|
||||
say('Run REINDEX to statuses...')
|
||||
ActiveRecord::Base.connection.execute('REINDEX TABLE statuses')
|
||||
else
|
||||
say('Run ANALYZE to statuses...')
|
||||
ActiveRecord::Base.connection.execute('ANALYZE statuses')
|
||||
end
|
||||
end
|
||||
|
||||
def vacuum_and_analyze_conversations
|
||||
if options[:compress_database]
|
||||
say('Run VACUUM FULL ANALYZE to conversations...')
|
||||
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE conversations')
|
||||
say('Run REINDEX to conversations...')
|
||||
ActiveRecord::Base.connection.execute('REINDEX TABLE conversations')
|
||||
else
|
||||
say('Run ANALYZE to conversations...')
|
||||
ActiveRecord::Base.connection.execute('ANALYZE conversations')
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,167 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative 'base'
|
||||
|
||||
module Mastodon::CLI
|
||||
class Upgrade < Base
|
||||
CURRENT_STORAGE_SCHEMA_VERSION = 1
|
||||
|
||||
option :dry_run, type: :boolean, default: false
|
||||
option :verbose, type: :boolean, default: false, aliases: [:v]
|
||||
desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
|
||||
long_desc <<~LONG_DESC
|
||||
Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
|
||||
necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
|
||||
|
||||
Will most likely take a long time.
|
||||
LONG_DESC
|
||||
def storage_schema
|
||||
progress = create_progress_bar(nil)
|
||||
records = 0
|
||||
|
||||
klasses = [
|
||||
Account,
|
||||
CustomEmoji,
|
||||
MediaAttachment,
|
||||
PreviewCard,
|
||||
]
|
||||
|
||||
klasses.each do |klass|
|
||||
attachment_names = klass.attachment_definitions.keys
|
||||
|
||||
klass.find_each do |record|
|
||||
attachment_names.each do |attachment_name|
|
||||
attachment = record.public_send(attachment_name)
|
||||
upgraded = false
|
||||
|
||||
next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
|
||||
|
||||
styles = attachment.styles.keys
|
||||
|
||||
styles << :original unless styles.include?(:original)
|
||||
|
||||
styles.each do |style|
|
||||
success = case Paperclip::Attachment.default_options[:storage]
|
||||
when :s3
|
||||
upgrade_storage_s3(progress, attachment, style)
|
||||
when :fog
|
||||
upgrade_storage_fog(progress, attachment, style)
|
||||
when :azure
|
||||
upgrade_storage_azure(progress, attachment, style)
|
||||
when :filesystem
|
||||
upgrade_storage_filesystem(progress, attachment, style)
|
||||
end
|
||||
|
||||
upgraded = true if style == :original && success
|
||||
|
||||
progress.increment
|
||||
end
|
||||
|
||||
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) if upgraded
|
||||
end
|
||||
|
||||
if record.changed?
|
||||
record.save unless dry_run?
|
||||
records += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
progress.total = progress.progress
|
||||
progress.finish
|
||||
|
||||
say("Upgraded storage schema of #{records} records#{dry_run_mode_suffix}", :green, true)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def upgrade_storage_s3(progress, attachment, style)
|
||||
previous_storage_schema_version = attachment.storage_schema_version
|
||||
object = attachment.s3_object(style)
|
||||
success = true
|
||||
|
||||
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
|
||||
|
||||
new_object = attachment.s3_object(style)
|
||||
|
||||
if new_object.key != object.key && object.exists?
|
||||
progress.log("Moving #{object.key} to #{new_object.key}") if options[:verbose]
|
||||
|
||||
begin
|
||||
object.move_to(new_object, acl: attachment.s3_permissions(style)) unless dry_run?
|
||||
rescue => e
|
||||
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
|
||||
success = false
|
||||
end
|
||||
end
|
||||
|
||||
# Because we move files style-by-style, it's important to restore
|
||||
# previous version at the end. The upgrade will be recorded after
|
||||
# all styles are updated
|
||||
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
|
||||
success
|
||||
end
|
||||
|
||||
def upgrade_storage_fog(_progress, _attachment, _style)
|
||||
say('The fog storage driver is not supported for this operation at this time', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
def upgrade_storage_azure(_progress, _attachment, _style)
|
||||
say('The azure storage driver is not supported for this operation at this time', :red)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
def upgrade_storage_filesystem(progress, attachment, style)
|
||||
previous_storage_schema_version = attachment.storage_schema_version
|
||||
previous_path = attachment.path(style)
|
||||
success = true
|
||||
|
||||
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
|
||||
|
||||
upgraded_path = attachment.path(style)
|
||||
|
||||
if upgraded_path != previous_path && File.exist?(previous_path)
|
||||
progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
|
||||
|
||||
begin
|
||||
move_previous_to_upgraded
|
||||
rescue => e
|
||||
progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
|
||||
success = false
|
||||
|
||||
remove_directory
|
||||
end
|
||||
end
|
||||
|
||||
# Because we move files style-by-style, it's important to restore
|
||||
# previous version at the end. The upgrade will be recorded after
|
||||
# all styles are updated
|
||||
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
|
||||
success
|
||||
end
|
||||
|
||||
def move_previous_to_upgraded(previous_path, upgraded_path)
|
||||
return if dry_run?
|
||||
|
||||
FileUtils.mkdir_p(File.dirname(upgraded_path))
|
||||
FileUtils.mv(previous_path, upgraded_path)
|
||||
|
||||
begin
|
||||
FileUtils.rmdir(File.dirname(previous_path), parents: true)
|
||||
rescue Errno::ENOTEMPTY
|
||||
# OK
|
||||
end
|
||||
end
|
||||
|
||||
def remove_directory(path)
|
||||
return if dry_run?
|
||||
|
||||
begin
|
||||
FileUtils.rmdir(File.dirname(path), parents: true)
|
||||
rescue Errno::ENOTEMPTY
|
||||
# OK
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,990 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# This file is copied almost entirely from GitLab, which has done a large
|
||||
# amount of work to ensure that migrations can happen with minimal downtime.
|
||||
# Many thanks to those engineers.
|
||||
|
||||
# Changes have been made to remove dependencies on other GitLab files and to
|
||||
# shorten temporary column names.
|
||||
|
||||
# Documentation on using these functions (and why one might do so):
|
||||
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/what_requires_downtime.md
|
||||
|
||||
# The file itself:
|
||||
# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/database/migration_helpers.rb
|
||||
|
||||
# It is licensed as follows:
|
||||
|
||||
# Copyright (c) 2011-2017 GitLab B.V.
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
|
||||
# This is bad form, but there are enough differences that it's impractical to do
|
||||
# otherwise:
|
||||
|
||||
module Mastodon
|
||||
module MigrationHelpers
|
||||
class CorruptionError < StandardError
|
||||
attr_reader :index_name
|
||||
|
||||
def initialize(index_name)
|
||||
@index_name = index_name
|
||||
|
||||
super "The index `#{index_name}` seems to be corrupted, it contains duplicate rows. " \
|
||||
'For information on how to fix this, see our documentation: ' \
|
||||
'https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/'
|
||||
end
|
||||
|
||||
def cause
|
||||
nil
|
||||
end
|
||||
|
||||
def backtrace
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
# Model that can be used for querying permissions of a SQL user.
|
||||
class Grant < ActiveRecord::Base
|
||||
self.table_name = 'information_schema.role_table_grants'
|
||||
|
||||
def self.scope_to_current_user
|
||||
where('grantee = user')
|
||||
end
|
||||
|
||||
# Returns true if the current user can create and execute triggers on the
|
||||
# given table.
|
||||
def self.create_and_execute_trigger?(table)
|
||||
priv = where(privilege_type: 'TRIGGER', table_name: table)
|
||||
|
||||
priv.scope_to_current_user.any?
|
||||
end
|
||||
end
|
||||
|
||||
BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
|
||||
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
|
||||
|
||||
# Gets an estimated number of rows for a table
|
||||
def estimate_rows_in_table(table_name)
|
||||
exec_query('SELECT reltuples FROM pg_class WHERE relname = ' +
|
||||
"'#{table_name}'").to_a.first['reltuples']
|
||||
end
|
||||
|
||||
# Adds `created_at` and `updated_at` columns with timezone information.
|
||||
#
|
||||
# This method is an improved version of Rails' built-in method `add_timestamps`.
|
||||
#
|
||||
# Available options are:
|
||||
# default - The default value for the column.
|
||||
# null - When set to `true` the column will allow NULL values.
|
||||
# The default is to not allow NULL values.
|
||||
def add_timestamps_with_timezone(table_name, **options)
|
||||
options[:null] = false if options[:null].nil?
|
||||
|
||||
[:created_at, :updated_at].each do |column_name|
|
||||
if options[:default] && transaction_open?
|
||||
raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \
|
||||
'You can disable transactions by calling `disable_ddl_transaction!` ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
# If default value is presented, use `add_column_with_default` method instead.
|
||||
if options[:default]
|
||||
add_column_with_default(
|
||||
table_name,
|
||||
column_name,
|
||||
:datetime_with_timezone,
|
||||
default: options[:default],
|
||||
allow_null: options[:null]
|
||||
)
|
||||
else
|
||||
add_column(table_name, column_name, :datetime_with_timezone, **options)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Creates a new index, concurrently when supported
|
||||
#
|
||||
# On PostgreSQL this method creates an index concurrently, on MySQL this
|
||||
# creates a regular index.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# add_concurrent_index :users, :some_column
|
||||
#
|
||||
# See Rails' `add_index` for more info on the available arguments.
|
||||
def add_concurrent_index(table_name, column_name, **options)
|
||||
if transaction_open?
|
||||
raise 'add_concurrent_index can not be run inside a transaction, ' \
|
||||
'you can disable transactions by calling disable_ddl_transaction! ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
options = options.merge({ algorithm: :concurrently })
|
||||
disable_statement_timeout
|
||||
|
||||
add_index(table_name, column_name, **options)
|
||||
end
|
||||
|
||||
# Removes an existed index, concurrently when supported
|
||||
#
|
||||
# On PostgreSQL this method removes an index concurrently.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# remove_concurrent_index :users, :some_column
|
||||
#
|
||||
# See Rails' `remove_index` for more info on the available arguments.
|
||||
def remove_concurrent_index(table_name, column_name, **options)
|
||||
if transaction_open?
|
||||
raise 'remove_concurrent_index can not be run inside a transaction, ' \
|
||||
'you can disable transactions by calling disable_ddl_transaction! ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
if supports_drop_index_concurrently?
|
||||
options = options.merge({ algorithm: :concurrently })
|
||||
disable_statement_timeout
|
||||
end
|
||||
|
||||
remove_index(table_name, **options.merge({ column: column_name }))
|
||||
end
|
||||
|
||||
# Removes an existing index, concurrently when supported
|
||||
#
|
||||
# On PostgreSQL this method removes an index concurrently.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# remove_concurrent_index :users, "index_X_by_Y"
|
||||
#
|
||||
# See Rails' `remove_index` for more info on the available arguments.
|
||||
def remove_concurrent_index_by_name(table_name, index_name, **options)
|
||||
if transaction_open?
|
||||
raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \
|
||||
'you can disable transactions by calling disable_ddl_transaction! ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
if supports_drop_index_concurrently?
|
||||
options = options.merge({ algorithm: :concurrently })
|
||||
disable_statement_timeout
|
||||
end
|
||||
|
||||
remove_index(table_name, **options.merge({ name: index_name }))
|
||||
end
|
||||
|
||||
# Only available on Postgresql >= 9.2
|
||||
def supports_drop_index_concurrently?
|
||||
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
|
||||
|
||||
version >= 90_200
|
||||
end
|
||||
|
||||
# Only available on Postgresql >= 11
|
||||
def supports_add_column_with_default?
|
||||
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
|
||||
|
||||
version >= 110_000
|
||||
end
|
||||
|
||||
# Adds a foreign key with only minimal locking on the tables involved.
|
||||
#
|
||||
# This method only requires minimal locking when using PostgreSQL. When
|
||||
# using MySQL this method will use Rails' default `add_foreign_key`.
|
||||
#
|
||||
# source - The source table containing the foreign key.
|
||||
# target - The target table the key points to.
|
||||
# column - The name of the column to create the foreign key on.
|
||||
# on_delete - The action to perform when associated data is removed,
|
||||
# defaults to "CASCADE".
|
||||
def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_col: 'id')
|
||||
# Transactions would result in ALTER TABLE locks being held for the
|
||||
# duration of the transaction, defeating the purpose of this method.
|
||||
if transaction_open?
|
||||
raise 'add_concurrent_foreign_key can not be run inside a transaction'
|
||||
end
|
||||
|
||||
# While MySQL does allow disabling of foreign keys it has no equivalent
|
||||
# of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
|
||||
# back to the normal foreign key procedure.
|
||||
on_delete = 'SET NULL' if on_delete == :nullify
|
||||
|
||||
disable_statement_timeout
|
||||
|
||||
key_name = concurrent_foreign_key_name(source, column, target_col)
|
||||
|
||||
# Using NOT VALID allows us to create a key without immediately
|
||||
# validating it. This means we keep the ALTER TABLE lock only for a
|
||||
# short period of time. The key _is_ enforced for any newly created
|
||||
# data.
|
||||
execute <<-EOF.strip_heredoc
|
||||
ALTER TABLE #{source}
|
||||
ADD CONSTRAINT #{key_name}
|
||||
FOREIGN KEY (#{column})
|
||||
REFERENCES #{target} (#{target_col})
|
||||
#{on_delete ? "ON DELETE #{on_delete.upcase}" : ''}
|
||||
NOT VALID;
|
||||
EOF
|
||||
|
||||
# Validate the existing constraint. This can potentially take a very
|
||||
# long time to complete, but fortunately does not lock the source table
|
||||
# while running.
|
||||
execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
|
||||
end
|
||||
|
||||
# Returns the name for a concurrent foreign key.
|
||||
#
|
||||
# PostgreSQL constraint names have a limit of 63 bytes. The logic used
|
||||
# here is based on Rails' foreign_key_name() method, which unfortunately
|
||||
# is private so we can't rely on it directly.
|
||||
def concurrent_foreign_key_name(table, column, target_col)
|
||||
"fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_#{target_col}_fk").first(10)}"
|
||||
end
|
||||
|
||||
# Long-running migrations may take more than the timeout allowed by
|
||||
# the database. Disable the session's statement timeout to ensure
|
||||
# migrations don't get killed prematurely. (PostgreSQL only)
|
||||
def disable_statement_timeout
|
||||
execute('SET statement_timeout TO 0')
|
||||
end
|
||||
|
||||
# Updates the value of a column in batches.
|
||||
#
|
||||
# This method updates the table in batches of 5% of the total row count.
|
||||
# This method will continue updating rows until no rows remain.
|
||||
#
|
||||
# When given a block this method will yield two values to the block:
|
||||
#
|
||||
# 1. An instance of `Arel::Table` for the table that is being updated.
|
||||
# 2. The query to run as an Arel object.
|
||||
#
|
||||
# By supplying a block one can add extra conditions to the queries being
|
||||
# executed. Note that the same block is used for _all_ queries.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# update_column_in_batches(:projects, :foo, 10) do |table, query|
|
||||
# query.where(table[:some_column].eq('hello'))
|
||||
# end
|
||||
#
|
||||
# This would result in this method updating only rows where
|
||||
# `projects.some_column` equals "hello".
|
||||
#
|
||||
# table - The name of the table.
|
||||
# column - The name of the column to update.
|
||||
# value - The value for the column.
|
||||
#
|
||||
# Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop
|
||||
# determines this method to be too complex while there's no way to make it
|
||||
# less "complex" without introducing extra methods (which actually will
|
||||
# make things _more_ complex).
|
||||
def update_column_in_batches(table_name, column, value)
|
||||
if transaction_open?
|
||||
raise 'update_column_in_batches can not be run inside a transaction, ' \
|
||||
'you can disable transactions by calling disable_ddl_transaction! ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
table = Arel::Table.new(table_name)
|
||||
|
||||
total = estimate_rows_in_table(table_name).to_i
|
||||
if total < 1
|
||||
count_arel = table.project(Arel.star.count.as('count'))
|
||||
count_arel = yield table, count_arel if block_given?
|
||||
|
||||
total = exec_query(count_arel.to_sql).to_ary.first['count'].to_i
|
||||
|
||||
return if total == 0
|
||||
end
|
||||
|
||||
# Update in batches of 5% until we run out of any rows to update.
|
||||
batch_size = ((total / 100.0) * 5.0).ceil
|
||||
max_size = 1000
|
||||
|
||||
# The upper limit is 1000 to ensure we don't lock too many rows. For
|
||||
# example, for "merge_requests" even 1% of the table is around 35 000
|
||||
# rows for GitLab.com.
|
||||
batch_size = max_size if batch_size > max_size
|
||||
|
||||
start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
|
||||
start_arel = yield table, start_arel if block_given?
|
||||
first_row = exec_query(start_arel.to_sql).to_ary.first
|
||||
# In case there are no rows but we didn't catch it in the estimated size:
|
||||
return unless first_row
|
||||
start_id = first_row['id'].to_i
|
||||
|
||||
say "Migrating #{table_name}.#{column} (~#{total.to_i} rows)"
|
||||
|
||||
started_time = Time.zone.now
|
||||
last_time = Time.zone.now
|
||||
migrated = 0
|
||||
loop do
|
||||
stop_row = nil
|
||||
|
||||
suppress_messages do
|
||||
stop_arel = table.project(table[:id])
|
||||
.where(table[:id].gteq(start_id))
|
||||
.order(table[:id].asc)
|
||||
.take(1)
|
||||
.skip(batch_size)
|
||||
|
||||
stop_arel = yield table, stop_arel if block_given?
|
||||
stop_row = exec_query(stop_arel.to_sql).to_ary.first
|
||||
|
||||
update_arel = Arel::UpdateManager.new
|
||||
.table(table)
|
||||
.set([[table[column], value]])
|
||||
.where(table[:id].gteq(start_id))
|
||||
|
||||
if stop_row
|
||||
stop_id = stop_row['id'].to_i
|
||||
start_id = stop_id
|
||||
update_arel = update_arel.where(table[:id].lt(stop_id))
|
||||
end
|
||||
|
||||
update_arel = yield table, update_arel if block_given?
|
||||
|
||||
execute(update_arel.to_sql)
|
||||
end
|
||||
|
||||
migrated += batch_size
|
||||
if Time.zone.now - last_time > 1
|
||||
status = "Migrated #{migrated} rows"
|
||||
|
||||
percentage = 100.0 * migrated / total
|
||||
status += " (~#{sprintf('%.2f', percentage)}%, "
|
||||
|
||||
remaining_time = (100.0 - percentage) * (Time.zone.now - started_time) / percentage
|
||||
|
||||
status += "#{(remaining_time / 60).to_i}:"
|
||||
status += sprintf('%02d', remaining_time.to_i % 60)
|
||||
status += ' remaining, '
|
||||
|
||||
# Tell users not to interrupt if we're almost done.
|
||||
if remaining_time > 10
|
||||
status += 'safe to interrupt'
|
||||
else
|
||||
status += 'DO NOT interrupt'
|
||||
end
|
||||
|
||||
status += ')'
|
||||
|
||||
say status, true
|
||||
last_time = Time.zone.now
|
||||
end
|
||||
|
||||
# There are no more rows left to update.
|
||||
break unless stop_row
|
||||
end
|
||||
end
|
||||
|
||||
# Adds a column with a default value without locking an entire table.
|
||||
#
|
||||
# This method runs the following steps:
|
||||
#
|
||||
# 1. Add the column with a default value of NULL.
|
||||
# 2. Change the default value of the column to the specified value.
|
||||
# 3. Update all existing rows in batches.
|
||||
# 4. Set a `NOT NULL` constraint on the column if desired (the default).
|
||||
#
|
||||
# These steps ensure a column can be added to a large and commonly used
|
||||
# table without locking the entire table for the duration of the table
|
||||
# modification.
|
||||
#
|
||||
# table - The name of the table to update.
|
||||
# column - The name of the column to add.
|
||||
# type - The column type (e.g. `:integer`).
|
||||
# default - The default value for the column.
|
||||
# limit - Sets a column limit. For example, for :integer, the default is
|
||||
# 4-bytes. Set `limit: 8` to allow 8-byte integers.
|
||||
# allow_null - When set to `true` the column will allow NULL values, the
|
||||
# default is to not allow NULL values.
|
||||
#
|
||||
# This method can also take a block which is passed directly to the
|
||||
# `update_column_in_batches` method.
|
||||
def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block)
|
||||
if supports_add_column_with_default?
|
||||
add_column(table, column, type, default: default, limit: limit, null: allow_null)
|
||||
return
|
||||
end
|
||||
|
||||
if transaction_open?
|
||||
raise 'add_column_with_default can not be run inside a transaction, ' \
|
||||
'you can disable transactions by calling disable_ddl_transaction! ' \
|
||||
'in the body of your migration class'
|
||||
end
|
||||
|
||||
disable_statement_timeout
|
||||
|
||||
transaction do
|
||||
if limit
|
||||
add_column(table, column, type, default: nil, limit: limit)
|
||||
else
|
||||
add_column(table, column, type, default: nil)
|
||||
end
|
||||
|
||||
# Changing the default before the update ensures any newly inserted
|
||||
# rows already use the proper default value.
|
||||
change_column_default(table, column, default)
|
||||
end
|
||||
|
||||
begin
|
||||
update_column_in_batches(table, column, default, &block)
|
||||
|
||||
change_column_null(table, column, false) unless allow_null
|
||||
# We want to rescue _all_ exceptions here, even those that don't inherit
|
||||
# from StandardError.
|
||||
rescue Exception => error # rubocop: disable all
|
||||
remove_column(table, column)
|
||||
|
||||
raise error
|
||||
end
|
||||
end
|
||||
|
||||
# Renames a column without requiring downtime.
|
||||
#
|
||||
# Concurrent renames work by using database triggers to ensure both the
|
||||
# old and new column are in sync. However, this method will _not_ remove
|
||||
# the triggers or the old column automatically; this needs to be done
|
||||
# manually in a post-deployment migration. This can be done using the
|
||||
# method `cleanup_concurrent_column_rename`.
|
||||
#
|
||||
# table - The name of the database table containing the column.
|
||||
# old - The old column name.
|
||||
# new - The new column name.
|
||||
# type - The type of the new column. If no type is given the old column's
|
||||
# type is used.
|
||||
def rename_column_concurrently(table, old, new, type: nil)
|
||||
if transaction_open?
|
||||
raise 'rename_column_concurrently can not be run inside a transaction'
|
||||
end
|
||||
|
||||
check_trigger_permissions!(table)
|
||||
trigger_name = rename_trigger_name(table, old, new)
|
||||
|
||||
# If we were in the middle of update_column_in_batches, we should remove
|
||||
# the old column and start over, as we have no idea where we were.
|
||||
if column_for(table, new)
|
||||
remove_rename_triggers_for_postgresql(table, trigger_name)
|
||||
|
||||
remove_column(table, new)
|
||||
end
|
||||
|
||||
old_col = column_for(table, old)
|
||||
new_type = type || old_col.type
|
||||
|
||||
col_opts = {
|
||||
precision: old_col.precision,
|
||||
scale: old_col.scale,
|
||||
}
|
||||
|
||||
# We may be trying to reset the limit on an integer column type, so let
|
||||
# Rails handle that.
|
||||
unless [:bigint, :integer].include?(new_type)
|
||||
col_opts[:limit] = old_col.limit
|
||||
end
|
||||
|
||||
add_column(table, new, new_type, **col_opts)
|
||||
|
||||
# We set the default value _after_ adding the column so we don't end up
|
||||
# updating any existing data with the default value. This isn't
|
||||
# necessary since we copy over old values further down.
|
||||
change_column_default(table, new, old_col.default) if old_col.default
|
||||
|
||||
quoted_table = quote_table_name(table)
|
||||
quoted_old = quote_column_name(old)
|
||||
quoted_new = quote_column_name(new)
|
||||
|
||||
install_rename_triggers_for_postgresql(trigger_name, quoted_table,
|
||||
quoted_old, quoted_new)
|
||||
|
||||
update_column_in_batches(table, new, Arel::Table.new(table)[old])
|
||||
|
||||
change_column_null(table, new, false) unless old_col.null
|
||||
|
||||
copy_indexes(table, old, new)
|
||||
copy_foreign_keys(table, old, new)
|
||||
end
|
||||
|
||||
# Changes the type of a column concurrently.
|
||||
#
|
||||
# table - The table containing the column.
|
||||
# column - The name of the column to change.
|
||||
# new_type - The new column type.
|
||||
def change_column_type_concurrently(table, column, new_type)
|
||||
temp_column = rename_column_name(column)
|
||||
|
||||
rename_column_concurrently(table, column, temp_column, type: new_type)
|
||||
|
||||
# Primary keys don't necessarily have an associated index.
|
||||
if ActiveRecord::Base.get_primary_key(table) == column.to_s
|
||||
old_pk_index_name = "index_#{table}_on_#{column}"
|
||||
new_pk_index_name = "index_#{table}_on_#{column}_cm"
|
||||
|
||||
unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
|
||||
add_concurrent_index(table, [temp_column],
|
||||
unique: true,
|
||||
name: new_pk_index_name
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Performs cleanup of a concurrent type change.
|
||||
#
|
||||
# table - The table containing the column.
|
||||
# column - The name of the column to change.
|
||||
# new_type - The new column type.
|
||||
def cleanup_concurrent_column_type_change(table, column)
|
||||
temp_column = rename_column_name(column)
|
||||
|
||||
# Wait for the indices to be built
|
||||
indexes_for(table, column).each do |index|
|
||||
expected_name = index.name + '_cm'
|
||||
|
||||
puts "Waiting for index #{expected_name}"
|
||||
sleep 1 until indexes_for(table, temp_column).find {|i| i.name == expected_name }
|
||||
end
|
||||
|
||||
was_primary = (ActiveRecord::Base.get_primary_key(table) == column.to_s)
|
||||
old_default_fn = column_for(table, column).default_function
|
||||
|
||||
old_fks = []
|
||||
if was_primary
|
||||
# Get any foreign keys pointing at this column we need to recreate, and
|
||||
# remove the old ones.
|
||||
# Based on code from:
|
||||
# http://errorbank.blogspot.com/2011/03/list-all-foreign-keys-references-for.html
|
||||
old_fks_res = execute <<-EOF.strip_heredoc
|
||||
select m.relname as src_table,
|
||||
(select a.attname
|
||||
from pg_attribute a
|
||||
where a.attrelid = m.oid
|
||||
and a.attnum = o.conkey[1]
|
||||
and a.attisdropped = false) as src_col,
|
||||
o.conname as name,
|
||||
o.confdeltype as on_delete
|
||||
from pg_constraint o
|
||||
left join pg_class f on f.oid = o.confrelid
|
||||
left join pg_class c on c.oid = o.conrelid
|
||||
left join pg_class m on m.oid = o.conrelid
|
||||
where o.contype = 'f'
|
||||
and o.conrelid in (
|
||||
select oid from pg_class c where c.relkind = 'r')
|
||||
and f.relname = '#{table}';
|
||||
EOF
|
||||
old_fks = old_fks_res.to_a
|
||||
old_fks.each do |old_fk|
|
||||
add_concurrent_foreign_key(
|
||||
old_fk['src_table'],
|
||||
table,
|
||||
column: old_fk['src_col'],
|
||||
target_col: temp_column,
|
||||
on_delete: extract_foreign_key_action(old_fk['on_delete'])
|
||||
)
|
||||
|
||||
remove_foreign_key(old_fk['src_table'], name: old_fk['name'])
|
||||
end
|
||||
end
|
||||
|
||||
# If there was a sequence owned by the old column, make it owned by the
|
||||
# new column, as it will otherwise be deleted when we get rid of the
|
||||
# old column.
|
||||
if (seq_match = /^nextval\('([^']*)'(::text|::regclass)?\)/.match(old_default_fn))
|
||||
seq_name = seq_match[1]
|
||||
execute("ALTER SEQUENCE #{seq_name} OWNED BY #{table}.#{temp_column}")
|
||||
end
|
||||
|
||||
transaction do
|
||||
# This has to be performed in a transaction as otherwise we might have
|
||||
# inconsistent data.
|
||||
|
||||
cleanup_concurrent_column_rename(table, column, temp_column)
|
||||
rename_column(table, temp_column, column)
|
||||
|
||||
# If there was an old default function, we didn't copy it. Do that now
|
||||
# in the transaction, so we don't miss anything.
|
||||
change_column_default(table, column, -> { old_default_fn }) if old_default_fn
|
||||
end
|
||||
|
||||
# Rename any indices back to what they should be.
|
||||
indexes_for(table, column).each do |index|
|
||||
next unless index.name.end_with?('_cm')
|
||||
|
||||
real_index_name = index.name.sub(/_cm$/, '')
|
||||
rename_index(table, index.name, real_index_name)
|
||||
end
|
||||
|
||||
# Rename any foreign keys back to names based on the real column.
|
||||
foreign_keys_for(table, column).each do |fk|
|
||||
old_fk_name = concurrent_foreign_key_name(fk.from_table, temp_column, 'id')
|
||||
new_fk_name = concurrent_foreign_key_name(fk.from_table, column, 'id')
|
||||
execute("ALTER TABLE #{fk.from_table} RENAME CONSTRAINT " +
|
||||
"#{old_fk_name} TO #{new_fk_name}")
|
||||
end
|
||||
|
||||
# Rename any foreign keys from other tables to names based on the real
|
||||
# column.
|
||||
old_fks.each do |old_fk|
|
||||
old_fk_name = concurrent_foreign_key_name(old_fk['src_table'],
|
||||
old_fk['src_col'], temp_column)
|
||||
new_fk_name = concurrent_foreign_key_name(old_fk['src_table'],
|
||||
old_fk['src_col'], column)
|
||||
execute("ALTER TABLE #{old_fk['src_table']} RENAME CONSTRAINT " +
|
||||
"#{old_fk_name} TO #{new_fk_name}")
|
||||
end
|
||||
|
||||
# If the old column was a primary key, mark the new one as a primary key.
|
||||
if was_primary
|
||||
execute("ALTER TABLE #{table} ADD PRIMARY KEY USING INDEX " +
|
||||
"index_#{table}_on_#{column}")
|
||||
end
|
||||
end
|
||||
|
||||
# Cleans up a concurrent column name.
|
||||
#
|
||||
# This method takes care of removing previously installed triggers as well
|
||||
# as removing the old column.
|
||||
#
|
||||
# table - The name of the database table.
|
||||
# old - The name of the old column.
|
||||
# new - The name of the new column.
|
||||
def cleanup_concurrent_column_rename(table, old, new)
|
||||
trigger_name = rename_trigger_name(table, old, new)
|
||||
|
||||
check_trigger_permissions!(table)
|
||||
|
||||
remove_rename_triggers_for_postgresql(table, trigger_name)
|
||||
|
||||
remove_column(table, old)
|
||||
end
|
||||
|
||||
# Performs a concurrent column rename when using PostgreSQL.
|
||||
def install_rename_triggers_for_postgresql(trigger, table, old, new)
|
||||
execute <<-EOF.strip_heredoc
|
||||
CREATE OR REPLACE FUNCTION #{trigger}()
|
||||
RETURNS trigger AS
|
||||
$BODY$
|
||||
BEGIN
|
||||
NEW.#{new} := NEW.#{old};
|
||||
RETURN NEW;
|
||||
END;
|
||||
$BODY$
|
||||
LANGUAGE 'plpgsql'
|
||||
VOLATILE
|
||||
EOF
|
||||
|
||||
execute <<-EOF.strip_heredoc
|
||||
CREATE TRIGGER #{trigger}
|
||||
BEFORE INSERT OR UPDATE
|
||||
ON #{table}
|
||||
FOR EACH ROW
|
||||
EXECUTE PROCEDURE #{trigger}()
|
||||
EOF
|
||||
end
|
||||
|
||||
# Installs the triggers necessary to perform a concurrent column rename on
|
||||
# MySQL.
|
||||
def install_rename_triggers_for_mysql(trigger, table, old, new)
|
||||
execute <<-EOF.strip_heredoc
|
||||
CREATE TRIGGER #{trigger}_insert
|
||||
BEFORE INSERT
|
||||
ON #{table}
|
||||
FOR EACH ROW
|
||||
SET NEW.#{new} = NEW.#{old}
|
||||
EOF
|
||||
|
||||
execute <<-EOF.strip_heredoc
|
||||
CREATE TRIGGER #{trigger}_update
|
||||
BEFORE UPDATE
|
||||
ON #{table}
|
||||
FOR EACH ROW
|
||||
SET NEW.#{new} = NEW.#{old}
|
||||
EOF
|
||||
end
|
||||
|
||||
# Removes the triggers used for renaming a PostgreSQL column concurrently.
|
||||
def remove_rename_triggers_for_postgresql(table, trigger)
|
||||
execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}")
|
||||
execute("DROP FUNCTION IF EXISTS #{trigger}()")
|
||||
end
|
||||
|
||||
# Removes the triggers used for renaming a MySQL column concurrently.
|
||||
def remove_rename_triggers_for_mysql(trigger)
|
||||
execute("DROP TRIGGER IF EXISTS #{trigger}_insert")
|
||||
execute("DROP TRIGGER IF EXISTS #{trigger}_update")
|
||||
end
|
||||
|
||||
# Returns the (base) name to use for triggers when renaming columns.
|
||||
def rename_trigger_name(table, old, new)
|
||||
'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
|
||||
end
|
||||
|
||||
# Returns the name to use for temporary rename columns.
|
||||
def rename_column_name(base)
|
||||
base.to_s + '_cm'
|
||||
end
|
||||
|
||||
# Returns an Array containing the indexes for the given column
|
||||
def indexes_for(table, column)
|
||||
column = column.to_s
|
||||
|
||||
indexes(table).select { |index| index.columns.include?(column) }
|
||||
end
|
||||
|
||||
# Returns an Array containing the foreign keys for the given column.
|
||||
def foreign_keys_for(table, column)
|
||||
column = column.to_s
|
||||
|
||||
foreign_keys(table).select { |fk| fk.column == column }
|
||||
end
|
||||
|
||||
# Copies all indexes for the old column to a new column.
|
||||
#
|
||||
# table - The table containing the columns and indexes.
|
||||
# old - The old column.
|
||||
# new - The new column.
|
||||
def copy_indexes(table, old, new)
|
||||
old = old.to_s
|
||||
new = new.to_s
|
||||
|
||||
indexes_for(table, old).each do |index|
|
||||
new_columns = index.columns.map do |column|
|
||||
column == old ? new : column
|
||||
end
|
||||
|
||||
# This is necessary as we can't properly rename indexes such as
|
||||
# "ci_taggings_idx".
|
||||
name = index.name + '_cm'
|
||||
|
||||
# If the order contained the old column, map it to the new one.
|
||||
order = index.orders
|
||||
if order.key?(old)
|
||||
order[new] = order.delete(old)
|
||||
end
|
||||
|
||||
options = {
|
||||
unique: index.unique,
|
||||
name: name,
|
||||
length: index.lengths,
|
||||
order: order
|
||||
}
|
||||
|
||||
# These options are not supported by MySQL, so we only add them if
|
||||
# they were previously set.
|
||||
options[:using] = index.using if index.using
|
||||
options[:where] = index.where if index.where
|
||||
|
||||
add_concurrent_index(table, new_columns, **options)
|
||||
end
|
||||
end
|
||||
|
||||
# Copies all foreign keys for the old column to the new column.
|
||||
#
|
||||
# table - The table containing the columns and indexes.
|
||||
# old - The old column.
|
||||
# new - The new column.
|
||||
def copy_foreign_keys(table, old, new)
|
||||
foreign_keys_for(table, old).each do |fk|
|
||||
add_concurrent_foreign_key(fk.from_table,
|
||||
fk.to_table,
|
||||
column: new,
|
||||
on_delete: fk.on_delete)
|
||||
end
|
||||
end
|
||||
|
||||
# Returns the column for the given table and column name.
|
||||
def column_for(table, name)
|
||||
name = name.to_s
|
||||
|
||||
columns(table).find { |column| column.name == name }
|
||||
end
|
||||
|
||||
# Update the configuration of an index by creating a new one and then
|
||||
# removing the old one
|
||||
def update_index(table_name, index_name, columns, **index_options)
|
||||
if index_name_exists?(table_name, "#{index_name}_new") && index_name_exists?(table_name, index_name)
|
||||
remove_index table_name, name: "#{index_name}_new"
|
||||
elsif index_name_exists?(table_name, "#{index_name}_new")
|
||||
# Very unlikely case where the script has been interrupted during/after removal but before renaming
|
||||
rename_index table_name, "#{index_name}_new", index_name
|
||||
end
|
||||
|
||||
begin
|
||||
add_index table_name, columns, **index_options.merge(name: "#{index_name}_new", algorithm: :concurrently)
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
remove_index table_name, name: "#{index_name}_new"
|
||||
raise CorruptionError.new(index_name)
|
||||
end
|
||||
|
||||
remove_index table_name, name: index_name if index_name_exists?(table_name, index_name)
|
||||
rename_index table_name, "#{index_name}_new", index_name
|
||||
end
|
||||
|
||||
# This will replace the first occurrence of a string in a column with
|
||||
# the replacement
|
||||
# On postgresql we can use `regexp_replace` for that.
|
||||
# On mysql we find the location of the pattern, and overwrite it
|
||||
# with the replacement
|
||||
def replace_sql(column, pattern, replacement)
|
||||
quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s)
|
||||
quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s)
|
||||
|
||||
replace = Arel::Nodes::NamedFunction
|
||||
.new("regexp_replace", [column, quoted_pattern, quoted_replacement])
|
||||
Arel::Nodes::SqlLiteral.new(replace.to_sql)
|
||||
end
|
||||
|
||||
def remove_foreign_key_without_error(*args)
|
||||
remove_foreign_key(*args)
|
||||
rescue ArgumentError
|
||||
end
|
||||
|
||||
def sidekiq_queue_migrate(queue_from, to:)
|
||||
while sidekiq_queue_length(queue_from) > 0
|
||||
Sidekiq.redis do |conn|
|
||||
conn.rpoplpush "queue:#{queue_from}", "queue:#{to}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def sidekiq_queue_length(queue_name)
|
||||
Sidekiq.redis do |conn|
|
||||
conn.llen("queue:#{queue_name}")
|
||||
end
|
||||
end
|
||||
|
||||
def check_trigger_permissions!(table)
|
||||
unless Grant.create_and_execute_trigger?(table)
|
||||
dbname = ActiveRecord::Base.configurations[Rails.env]['database']
|
||||
user = ActiveRecord::Base.configurations[Rails.env]['username'] || ENV['USER']
|
||||
|
||||
raise <<-EOF
|
||||
Your database user is not allowed to create, drop, or execute triggers on the
|
||||
table #{table}.
|
||||
|
||||
If you are using PostgreSQL you can solve this by logging in to the Mastodon
|
||||
database (#{dbname}) using a super user and running:
|
||||
|
||||
ALTER USER #{user} WITH SUPERUSER
|
||||
|
||||
The query will grant the user super user permissions, ensuring you don't run
|
||||
into similar problems in the future (e.g. when new tables are created).
|
||||
EOF
|
||||
end
|
||||
end
|
||||
|
||||
# Bulk queues background migration jobs for an entire table, batched by ID range.
|
||||
# "Bulk" meaning many jobs will be pushed at a time for efficiency.
|
||||
# If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`.
|
||||
#
|
||||
# model_class - The table being iterated over
|
||||
# job_class_name - The background migration job class as a string
|
||||
# batch_size - The maximum number of rows per job
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# class Route < ActiveRecord::Base
|
||||
# include EachBatch
|
||||
# self.table_name = 'routes'
|
||||
# end
|
||||
#
|
||||
# bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes')
|
||||
#
|
||||
# Where the model_class includes EachBatch, and the background migration exists:
|
||||
#
|
||||
# class Gitlab::BackgroundMigration::ProcessRoutes
|
||||
# def perform(start_id, end_id)
|
||||
# # do something
|
||||
# end
|
||||
# end
|
||||
def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
|
||||
raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
|
||||
|
||||
jobs = []
|
||||
|
||||
model_class.each_batch(of: batch_size) do |relation|
|
||||
start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
|
||||
|
||||
if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
|
||||
# Note: This code path generally only helps with many millions of rows
|
||||
# We push multiple jobs at a time to reduce the time spent in
|
||||
# Sidekiq/Redis operations. We're using this buffer based approach so we
|
||||
# don't need to run additional queries for every range.
|
||||
BackgroundMigrationWorker.perform_bulk(jobs)
|
||||
jobs.clear
|
||||
end
|
||||
|
||||
jobs << [job_class_name, [start_id, end_id]]
|
||||
end
|
||||
|
||||
BackgroundMigrationWorker.perform_bulk(jobs) unless jobs.empty?
|
||||
end
|
||||
|
||||
# Queues background migration jobs for an entire table, batched by ID range.
|
||||
# Each job is scheduled with a `delay_interval` in between.
|
||||
# If you use a small interval, then some jobs may run at the same time.
|
||||
#
|
||||
# model_class - The table being iterated over
|
||||
# job_class_name - The background migration job class as a string
|
||||
# delay_interval - The duration between each job's scheduled time (must respond to `to_f`)
|
||||
# batch_size - The maximum number of rows per job
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# class Route < ActiveRecord::Base
|
||||
# include EachBatch
|
||||
# self.table_name = 'routes'
|
||||
# end
|
||||
#
|
||||
# queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute)
|
||||
#
|
||||
# Where the model_class includes EachBatch, and the background migration exists:
|
||||
#
|
||||
# class Gitlab::BackgroundMigration::ProcessRoutes
|
||||
# def perform(start_id, end_id)
|
||||
# # do something
|
||||
# end
|
||||
# end
|
||||
def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
|
||||
raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
|
||||
|
||||
model_class.each_batch(of: batch_size) do |relation, index|
|
||||
start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
|
||||
|
||||
# `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
|
||||
# the same time, which is not helpful in most cases where we wish to
|
||||
# spread the work over time.
|
||||
BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id])
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# https://github.com/rails/rails/blob/v5.2.0/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb#L678-L684
|
||||
def extract_foreign_key_action(specifier)
|
||||
case specifier
|
||||
when 'c'; :cascade
|
||||
when 'n'; :nullify
|
||||
when 'r'; :restrict
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,55 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Mastodon
|
||||
module MigrationWarning
|
||||
WARNING_SECONDS = 10
|
||||
|
||||
DEFAULT_WARNING = <<~WARNING_MESSAGE
|
||||
WARNING: This migration may take a *long* time for large instances.
|
||||
It will *not* lock tables for any significant time, but it may run
|
||||
for a very long time. We will pause for #{WARNING_SECONDS} seconds to allow you to
|
||||
interrupt this migration if you are not ready.
|
||||
WARNING_MESSAGE
|
||||
|
||||
def migration_duration_warning(explanation = nil)
|
||||
return unless valid_environment?
|
||||
|
||||
announce_warning(explanation)
|
||||
|
||||
announce_countdown
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def announce_countdown
|
||||
WARNING_SECONDS.downto(1) do |i|
|
||||
say "Continuing in #{i} second#{i == 1 ? '' : 's'}...", true
|
||||
sleep 1
|
||||
end
|
||||
end
|
||||
|
||||
def valid_environment?
|
||||
$stdout.isatty && Rails.env.production?
|
||||
end
|
||||
|
||||
def announce_warning(explanation)
|
||||
announce_message prepare_message(explanation)
|
||||
end
|
||||
|
||||
def announce_message(text)
|
||||
say ''
|
||||
text.each_line do |line|
|
||||
say(line)
|
||||
end
|
||||
say ''
|
||||
end
|
||||
|
||||
def prepare_message(explanation)
|
||||
if explanation.blank?
|
||||
DEFAULT_WARNING
|
||||
else
|
||||
DEFAULT_WARNING + "\n#{explanation}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,23 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module PremailerWebpackStrategy
|
||||
def load(url)
|
||||
asset_host = ENV['CDN_HOST'] || ENV['WEB_DOMAIN'] || ENV['LOCAL_DOMAIN']
|
||||
|
||||
if Webpacker.dev_server.running?
|
||||
asset_host = "#{Webpacker.dev_server.protocol}://#{Webpacker.dev_server.host_with_port}"
|
||||
url = File.join(asset_host, url)
|
||||
end
|
||||
|
||||
css = if url.start_with?('http')
|
||||
HTTP.get(url).to_s
|
||||
else
|
||||
url = url[1..] if url.start_with?('/')
|
||||
Rails.public_path.join(url).read
|
||||
end
|
||||
|
||||
css.gsub(%r{url\(/}, "url(#{asset_host}/")
|
||||
end
|
||||
|
||||
module_function :load
|
||||
end
|
||||
@@ -1,30 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Mastodon::RackMiddleware
|
||||
def initialize(app)
|
||||
@app = app
|
||||
end
|
||||
|
||||
def call(env)
|
||||
@app.call(env)
|
||||
ensure
|
||||
clean_up_sockets!
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def clean_up_sockets!
|
||||
clean_up_redis_socket!
|
||||
clean_up_statsd_socket!
|
||||
end
|
||||
|
||||
def clean_up_redis_socket!
|
||||
RedisConfiguration.pool.checkin if Thread.current[:redis]
|
||||
Thread.current[:redis] = nil
|
||||
end
|
||||
|
||||
def clean_up_statsd_socket!
|
||||
Thread.current[:statsd_socket]&.close
|
||||
Thread.current[:statsd_socket] = nil
|
||||
end
|
||||
end
|
||||
@@ -1,49 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
def setup_redis_env_url(prefix = nil, defaults = true)
|
||||
prefix = "#{prefix.to_s.upcase}_" unless prefix.nil?
|
||||
prefix = '' if prefix.nil?
|
||||
|
||||
return if ENV["#{prefix}REDIS_URL"].present?
|
||||
|
||||
password = ENV.fetch("#{prefix}REDIS_PASSWORD") { '' if defaults }
|
||||
host = ENV.fetch("#{prefix}REDIS_HOST") { 'localhost' if defaults }
|
||||
port = ENV.fetch("#{prefix}REDIS_PORT") { 6379 if defaults }
|
||||
db = ENV.fetch("#{prefix}REDIS_DB") { 0 if defaults }
|
||||
|
||||
ENV["#{prefix}REDIS_URL"] = begin
|
||||
if [password, host, port, db].all?(&:nil?)
|
||||
ENV['REDIS_URL']
|
||||
else
|
||||
Addressable::URI.parse("redis://#{host}:#{port}/#{db}").tap do |uri|
|
||||
uri.password = password if password.present?
|
||||
end.normalize.to_str
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
setup_redis_env_url
|
||||
setup_redis_env_url(:cache, false)
|
||||
setup_redis_env_url(:sidekiq, false)
|
||||
|
||||
namespace = ENV.fetch('REDIS_NAMESPACE', nil)
|
||||
cache_namespace = namespace ? "#{namespace}_cache" : 'cache'
|
||||
sidekiq_namespace = namespace
|
||||
|
||||
REDIS_CACHE_PARAMS = {
|
||||
driver: :hiredis,
|
||||
url: ENV['CACHE_REDIS_URL'],
|
||||
expires_in: 10.minutes,
|
||||
namespace: cache_namespace,
|
||||
pool_size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
|
||||
pool_timeout: 5,
|
||||
connect_timeout: 5,
|
||||
}.freeze
|
||||
|
||||
REDIS_SIDEKIQ_PARAMS = {
|
||||
driver: :hiredis,
|
||||
url: ENV['SIDEKIQ_REDIS_URL'],
|
||||
namespace: sidekiq_namespace,
|
||||
}.freeze
|
||||
|
||||
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?
|
||||
@@ -1,37 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Mastodon::SidekiqMiddleware
|
||||
BACKTRACE_LIMIT = 3
|
||||
|
||||
def call(*, &block)
|
||||
Chewy.strategy(:mastodon, &block)
|
||||
rescue Mastodon::HostValidationError
|
||||
# Do not retry
|
||||
rescue => e
|
||||
limit_backtrace_and_raise(e)
|
||||
ensure
|
||||
clean_up_sockets!
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def limit_backtrace_and_raise(exception)
|
||||
exception.set_backtrace(exception.backtrace.first(BACKTRACE_LIMIT)) unless ENV['BACKTRACE']
|
||||
raise exception
|
||||
end
|
||||
|
||||
def clean_up_sockets!
|
||||
clean_up_redis_socket!
|
||||
clean_up_statsd_socket!
|
||||
end
|
||||
|
||||
def clean_up_redis_socket!
|
||||
RedisConfiguration.pool.checkin if Thread.current[:redis]
|
||||
Thread.current[:redis] = nil
|
||||
end
|
||||
|
||||
def clean_up_statsd_socket!
|
||||
Thread.current[:statsd_socket]&.close
|
||||
Thread.current[:statsd_socket] = nil
|
||||
end
|
||||
end
|
||||
@@ -1,176 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Mastodon::Snowflake
|
||||
DEFAULT_REGEX = /timestamp_id\('(?<seq_prefix>\w+)'/
|
||||
|
||||
class Callbacks
|
||||
def self.around_create(record)
|
||||
now = Time.now.utc
|
||||
|
||||
if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps
|
||||
yield
|
||||
else
|
||||
record.id = Mastodon::Snowflake.id_at(record.created_at)
|
||||
tries = 0
|
||||
|
||||
begin
|
||||
yield
|
||||
rescue ActiveRecord::RecordNotUnique
|
||||
raise if tries > 100
|
||||
|
||||
tries += 1
|
||||
record.id += rand(100)
|
||||
|
||||
retry
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class << self
|
||||
# Our ID will be composed of the following:
|
||||
# 6 bytes (48 bits) of millisecond-level timestamp
|
||||
# 2 bytes (16 bits) of sequence data
|
||||
#
|
||||
# The 'sequence data' is intended to be unique within a
|
||||
# given millisecond, yet obscure the 'serial number' of
|
||||
# this row.
|
||||
#
|
||||
# To do this, we hash the following data:
|
||||
# * Table name (if provided, skipped if not)
|
||||
# * Secret salt (should not be guessable)
|
||||
# * Timestamp (again, millisecond-level granularity)
|
||||
#
|
||||
# We then take the first two bytes of that value, and add
|
||||
# the lowest two bytes of the table ID sequence number
|
||||
# (`table_name`_id_seq). This means that even if we insert
|
||||
# two rows at the same millisecond, they will have
|
||||
# distinct 'sequence data' portions.
|
||||
#
|
||||
# If this happens, and an attacker can see both such IDs,
|
||||
# they can determine which of the two entries was inserted
|
||||
# first, but not the total number of entries in the table
|
||||
# (even mod 2**16).
|
||||
#
|
||||
# The table name is included in the hash to ensure that
|
||||
# different tables derive separate sequence bases so rows
|
||||
# inserted in the same millisecond in different tables do
|
||||
# not reveal the table ID sequence number for one another.
|
||||
#
|
||||
# The secret salt is included in the hash to ensure that
|
||||
# external users cannot derive the sequence base given the
|
||||
# timestamp and table name, which would allow them to
|
||||
# compute the table ID sequence number.
|
||||
def define_timestamp_id
|
||||
return if already_defined?
|
||||
|
||||
connection.execute(sanitized_timestamp_id_sql)
|
||||
end
|
||||
|
||||
def ensure_id_sequences_exist
|
||||
# Find tables using timestamp IDs.
|
||||
connection.tables.each do |table|
|
||||
# We're only concerned with "id" columns.
|
||||
next unless (id_col = connection.columns(table).find { |col| col.name == 'id' })
|
||||
|
||||
# And only those that are using timestamp_id.
|
||||
next unless (data = DEFAULT_REGEX.match(id_col.default_function))
|
||||
|
||||
seq_name = "#{data[:seq_prefix]}_id_seq"
|
||||
|
||||
# If we were on Postgres 9.5+, we could do CREATE SEQUENCE IF
|
||||
# NOT EXISTS, but we can't depend on that. Instead, catch the
|
||||
# possible exception and ignore it.
|
||||
# Note that seq_name isn't a column name, but it's a
|
||||
# relation, like a column, and follows the same quoting rules
|
||||
# in Postgres.
|
||||
connection.execute(<<~SQL)
|
||||
DO $$
|
||||
BEGIN
|
||||
CREATE SEQUENCE #{connection.quote_column_name(seq_name)};
|
||||
EXCEPTION WHEN duplicate_table THEN
|
||||
-- Do nothing, we have the sequence already.
|
||||
END
|
||||
$$ LANGUAGE plpgsql;
|
||||
SQL
|
||||
end
|
||||
end
|
||||
|
||||
def id_at(timestamp, with_random: true)
|
||||
id = timestamp.to_i * 1000
|
||||
id += rand(1000) if with_random
|
||||
id = id << 16
|
||||
id += rand(2**16) if with_random
|
||||
id
|
||||
end
|
||||
|
||||
def to_time(id)
|
||||
Time.at((id >> 16) / 1000).utc
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def already_defined?
|
||||
connection.execute(<<~SQL.squish).values.first.first
|
||||
SELECT EXISTS(
|
||||
SELECT * FROM pg_proc WHERE proname = 'timestamp_id'
|
||||
);
|
||||
SQL
|
||||
end
|
||||
|
||||
def sanitized_timestamp_id_sql
|
||||
ActiveRecord::Base.sanitize_sql_array(timestamp_id_sql_array)
|
||||
end
|
||||
|
||||
def timestamp_id_sql_array
|
||||
[timestamp_id_sql_string, { random_string: SecureRandom.hex(16) }]
|
||||
end
|
||||
|
||||
def timestamp_id_sql_string
|
||||
<<~SQL
|
||||
CREATE OR REPLACE FUNCTION timestamp_id(table_name text)
|
||||
RETURNS bigint AS
|
||||
$$
|
||||
DECLARE
|
||||
time_part bigint;
|
||||
sequence_base bigint;
|
||||
tail bigint;
|
||||
BEGIN
|
||||
time_part := (
|
||||
-- Get the time in milliseconds
|
||||
((date_part('epoch', now()) * 1000))::bigint
|
||||
-- And shift it over two bytes
|
||||
<< 16);
|
||||
|
||||
sequence_base := (
|
||||
'x' ||
|
||||
-- Take the first two bytes (four hex characters)
|
||||
substr(
|
||||
-- Of the MD5 hash of the data we documented
|
||||
md5(table_name || :random_string || time_part::text),
|
||||
1, 4
|
||||
)
|
||||
-- And turn it into a bigint
|
||||
)::bit(16)::bigint;
|
||||
|
||||
-- Finally, add our sequence number to our base, and chop
|
||||
-- it to the last two bytes
|
||||
tail := (
|
||||
(sequence_base + nextval(table_name || '_id_seq'))
|
||||
& 65535);
|
||||
|
||||
-- Return the time part and the sequence part. OR appears
|
||||
-- faster here than addition, but they're equivalent:
|
||||
-- time_part has no trailing two bytes, and tail is only
|
||||
-- the last two bytes.
|
||||
RETURN time_part | tail;
|
||||
END
|
||||
$$ LANGUAGE plpgsql VOLATILE;
|
||||
SQL
|
||||
end
|
||||
|
||||
def connection
|
||||
ActiveRecord::Base.connection
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,71 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Mastodon
|
||||
module Version
|
||||
module_function
|
||||
|
||||
def major
|
||||
4
|
||||
end
|
||||
|
||||
def minor
|
||||
3
|
||||
end
|
||||
|
||||
def patch
|
||||
0
|
||||
end
|
||||
|
||||
def default_prerelease
|
||||
'alpha.0'
|
||||
end
|
||||
|
||||
def prerelease
|
||||
ENV['MASTODON_VERSION_PRERELEASE'].presence || default_prerelease
|
||||
end
|
||||
|
||||
def build_metadata
|
||||
['glitch', ENV.fetch('MASTODON_VERSION_METADATA', nil)].compact_blank.join('.')
|
||||
end
|
||||
|
||||
def to_a
|
||||
[major, minor, patch].compact
|
||||
end
|
||||
|
||||
def to_s
|
||||
components = [to_a.join('.')]
|
||||
components << "-#{prerelease}" if prerelease.present?
|
||||
components << "+#{build_metadata}" if build_metadata.present?
|
||||
components.join
|
||||
end
|
||||
|
||||
def gem_version
|
||||
@gem_version ||= Gem::Version.new(to_s.split('+')[0])
|
||||
end
|
||||
|
||||
def repository
|
||||
ENV.fetch('GITHUB_REPOSITORY', 'glitch-soc/mastodon')
|
||||
end
|
||||
|
||||
def source_base_url
|
||||
ENV.fetch('SOURCE_BASE_URL', "https://github.com/#{repository}")
|
||||
end
|
||||
|
||||
# specify git tag or commit hash here
|
||||
def source_tag
|
||||
ENV.fetch('SOURCE_TAG', nil)
|
||||
end
|
||||
|
||||
def source_url
|
||||
if source_tag
|
||||
"#{source_base_url}/tree/#{source_tag}"
|
||||
else
|
||||
source_base_url
|
||||
end
|
||||
end
|
||||
|
||||
def user_agent
|
||||
@user_agent ||= "#{HTTP::Request::USER_AGENT} (Mastodon/#{Version}; +http#{Rails.configuration.x.use_https ? 's' : ''}://#{Rails.configuration.x.web_domain}/)"
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,98 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
module AttachmentExtensions
|
||||
def meta
|
||||
instance_read(:meta)
|
||||
end
|
||||
|
||||
# monkey-patch to avoid unlinking too avoid unlinking source file too early
|
||||
# see https://github.com/kreeti/kt-paperclip/issues/64
|
||||
def post_process_style(name, style) # :nodoc:
|
||||
raise "Style #{name} has no processors defined." if style.processors.blank?
|
||||
|
||||
intermediate_files = []
|
||||
original = @queued_for_write[:original]
|
||||
# if we're processing the original, close + unlink the source tempfile
|
||||
intermediate_files << original if name == :original
|
||||
|
||||
@queued_for_write[name] = style.processors
|
||||
.inject(original) do |file, processor|
|
||||
file = Paperclip.processor(processor).make(file, style.processor_options, self)
|
||||
intermediate_files << file unless file == original
|
||||
file
|
||||
end
|
||||
|
||||
unadapted_file = @queued_for_write[name]
|
||||
@queued_for_write[name] = Paperclip.io_adapters
|
||||
.for(@queued_for_write[name], @options[:adapter_options])
|
||||
unadapted_file.close if unadapted_file.respond_to?(:close)
|
||||
@queued_for_write[name]
|
||||
rescue Paperclip::Errors::NotIdentifiedByImageMagickError => e
|
||||
log("An error was received while processing: #{e.inspect}")
|
||||
(@errors[:processing] ||= []) << e.message if @options[:whiny]
|
||||
ensure
|
||||
unlink_files(intermediate_files)
|
||||
end
|
||||
|
||||
# We overwrite this method to support delayed processing in
|
||||
# Sidekiq. Since we process the original file to reduce disk
|
||||
# usage, and we still want to generate thumbnails straight
|
||||
# away, it's the only style we need to exclude
|
||||
def process_style?(style_name, style_args)
|
||||
if style_name == :original && instance.respond_to?(:delay_processing_for_attachment?) && instance.delay_processing_for_attachment?(name)
|
||||
false
|
||||
else
|
||||
style_args.empty? || style_args.include?(style_name)
|
||||
end
|
||||
end
|
||||
|
||||
def storage_schema_version
|
||||
instance_read(:storage_schema_version) || 0
|
||||
end
|
||||
|
||||
def assign_attributes
|
||||
super
|
||||
instance_write(:storage_schema_version, 1)
|
||||
end
|
||||
|
||||
def variant?(other_filename)
|
||||
return true if original_filename == other_filename
|
||||
return false if original_filename.nil?
|
||||
|
||||
formats = styles.values.filter_map(&:format)
|
||||
|
||||
return false if formats.empty?
|
||||
|
||||
other_extension = File.extname(other_filename)
|
||||
|
||||
formats.include?(other_extension.delete('.')) && File.basename(other_filename, other_extension) == File.basename(original_filename, File.extname(original_filename))
|
||||
end
|
||||
|
||||
def default_url(style_name = default_style)
|
||||
@url_generator.for_as_default(style_name)
|
||||
end
|
||||
|
||||
STOPLIGHT_THRESHOLD = 10
|
||||
STOPLIGHT_COOLDOWN = 30
|
||||
|
||||
# We overwrite this method to put a circuit breaker around
|
||||
# calls to object storage, to stop hitting APIs that are slow
|
||||
# to respond or don't respond at all and as such minimize the
|
||||
# impact of object storage outages on application throughput
|
||||
def save
|
||||
# Don't go through Stoplight if we don't have anything object-storage-oriented to do
|
||||
return super if @queued_for_delete.empty? && @queued_for_write.empty? && !dirty?
|
||||
|
||||
Stoplight('object-storage') { super }.with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
|
||||
if error.is_a?(Seahorse::Client::NetworkingError)
|
||||
handle.call(error)
|
||||
else
|
||||
raise error
|
||||
end
|
||||
end.run
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Paperclip::Attachment.prepend(Paperclip::AttachmentExtensions)
|
||||
@@ -1,16 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
class BlurhashTranscoder < Paperclip::Processor
|
||||
def make
|
||||
return @file unless options[:style] == :small || options[:blurhash]
|
||||
|
||||
pixels = convert(':source -depth 8 RGB:-', source: File.expand_path(@file.path)).unpack('C*')
|
||||
geometry = options.fetch(:file_geometry_parser).from_file(@file)
|
||||
|
||||
attachment.instance.blurhash = Blurhash.encode(geometry.width, geometry.height, pixels, **(options[:blurhash] || {}))
|
||||
|
||||
@file
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,189 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'mime/types/columnar'
|
||||
|
||||
module Paperclip
|
||||
class ColorExtractor < Paperclip::Processor
|
||||
MIN_CONTRAST = 3.0
|
||||
ACCENT_MIN_CONTRAST = 2.0
|
||||
FREQUENCY_THRESHOLD = 0.01
|
||||
|
||||
def make
|
||||
depth = 8
|
||||
|
||||
# Determine background palette by getting colors close to the image's edge only
|
||||
background_palette = palette_from_histogram(convert(':source -alpha set -gravity Center -region 75%x75% -fill None -colorize 100% -alpha transparent +region -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10)
|
||||
|
||||
# Determine foreground palette from the whole image
|
||||
foreground_palette = palette_from_histogram(convert(':source -format %c -colors :quantity -depth :depth histogram:info:', source: File.expand_path(@file.path), quantity: 10, depth: depth), 10)
|
||||
|
||||
background_color = background_palette.first || foreground_palette.first
|
||||
foreground_colors = []
|
||||
|
||||
return @file if background_color.nil?
|
||||
|
||||
max_distance = 0
|
||||
max_distance_color = nil
|
||||
|
||||
foreground_palette.each do |color|
|
||||
distance = ColorDiff.between(background_color, color)
|
||||
contrast = w3c_contrast(background_color, color)
|
||||
|
||||
if distance > max_distance && contrast >= ACCENT_MIN_CONTRAST
|
||||
max_distance = distance
|
||||
max_distance_color = color
|
||||
end
|
||||
end
|
||||
|
||||
foreground_colors << max_distance_color unless max_distance_color.nil?
|
||||
|
||||
max_distance = 0
|
||||
max_distance_color = nil
|
||||
|
||||
foreground_palette.each do |color|
|
||||
distance = ColorDiff.between(background_color, color)
|
||||
contrast = w3c_contrast(background_color, color)
|
||||
|
||||
if distance > max_distance && contrast >= MIN_CONTRAST && !foreground_colors.include?(color)
|
||||
max_distance = distance
|
||||
max_distance_color = color
|
||||
end
|
||||
end
|
||||
|
||||
foreground_colors << max_distance_color unless max_distance_color.nil?
|
||||
|
||||
# If we don't have enough colors for accent and foreground, generate
|
||||
# new ones by manipulating the background color
|
||||
(2 - foreground_colors.size).times do |i|
|
||||
foreground_colors << lighten_or_darken(background_color, 35 + (i * 15))
|
||||
end
|
||||
|
||||
# We want the color with the highest contrast to background to be the foreground one,
|
||||
# and the one with the highest saturation to be the accent one
|
||||
foreground_color = foreground_colors.max_by { |rgb| w3c_contrast(background_color, rgb) }
|
||||
accent_color = foreground_colors.max_by { |rgb| rgb_to_hsl(rgb.r, rgb.g, rgb.b)[1] }
|
||||
|
||||
meta = {
|
||||
colors: {
|
||||
background: rgb_to_hex(background_color),
|
||||
foreground: rgb_to_hex(foreground_color),
|
||||
accent: rgb_to_hex(accent_color),
|
||||
},
|
||||
}
|
||||
|
||||
attachment.instance.file.instance_write(:meta, (attachment.instance.file.instance_read(:meta) || {}).merge(meta))
|
||||
|
||||
@file
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def w3c_contrast(color1, color2)
|
||||
luminance1 = (color1.to_xyz.y * 0.01) + 0.05
|
||||
luminance2 = (color2.to_xyz.y * 0.01) + 0.05
|
||||
|
||||
if luminance1 > luminance2
|
||||
luminance1 / luminance2
|
||||
else
|
||||
luminance2 / luminance1
|
||||
end
|
||||
end
|
||||
|
||||
# rubocop:disable Naming/MethodParameterName
|
||||
def rgb_to_hsl(r, g, b)
|
||||
r /= 255.0
|
||||
g /= 255.0
|
||||
b /= 255.0
|
||||
max = [r, g, b].max
|
||||
min = [r, g, b].min
|
||||
h = (max + min) / 2.0
|
||||
s = (max + min) / 2.0
|
||||
l = (max + min) / 2.0
|
||||
|
||||
if max == min
|
||||
h = 0
|
||||
s = 0 # achromatic
|
||||
else
|
||||
d = max - min
|
||||
s = l >= 0.5 ? d / (2.0 - max - min) : d / (max + min)
|
||||
|
||||
case max
|
||||
when r
|
||||
h = ((g - b) / d) + (g < b ? 6.0 : 0)
|
||||
when g
|
||||
h = ((b - r) / d) + 2.0
|
||||
when b
|
||||
h = ((r - g) / d) + 4.0
|
||||
end
|
||||
|
||||
h /= 6.0
|
||||
end
|
||||
|
||||
[(h * 360).round, (s * 100).round, (l * 100).round]
|
||||
end
|
||||
|
||||
def hue_to_rgb(p, q, t)
|
||||
t += 1 if t.negative?
|
||||
t -= 1 if t > 1
|
||||
|
||||
return (p + ((q - p) * 6 * t)) if t < 1 / 6.0
|
||||
return q if t < 1 / 2.0
|
||||
return (p + ((q - p) * ((2 / 3.0) - t) * 6)) if t < 2 / 3.0
|
||||
|
||||
p
|
||||
end
|
||||
|
||||
def hsl_to_rgb(h, s, l)
|
||||
h /= 360.0
|
||||
s /= 100.0
|
||||
l /= 100.0
|
||||
|
||||
r = 0.0
|
||||
g = 0.0
|
||||
b = 0.0
|
||||
|
||||
if s.zero?
|
||||
r = l.to_f
|
||||
g = l.to_f
|
||||
b = l.to_f # achromatic
|
||||
else
|
||||
q = l < 0.5 ? l * (s + 1) : l + s - (l * s)
|
||||
p = (2 * l) - q
|
||||
r = hue_to_rgb(p, q, h + (1 / 3.0))
|
||||
g = hue_to_rgb(p, q, h)
|
||||
b = hue_to_rgb(p, q, h - (1 / 3.0))
|
||||
end
|
||||
|
||||
[(r * 255).round, (g * 255).round, (b * 255).round]
|
||||
end
|
||||
# rubocop:enable Naming/MethodParameterName
|
||||
|
||||
def lighten_or_darken(color, by)
|
||||
hue, saturation, light = rgb_to_hsl(color.r, color.g, color.b)
|
||||
|
||||
light = if light < 50
|
||||
[100, light + by].min
|
||||
else
|
||||
[0, light - by].max
|
||||
end
|
||||
|
||||
ColorDiff::Color::RGB.new(*hsl_to_rgb(hue, saturation, light))
|
||||
end
|
||||
|
||||
def palette_from_histogram(result, quantity)
|
||||
frequencies = result.scan(/([0-9]+):/).flatten.map(&:to_f)
|
||||
hex_values = result.scan(/\#([0-9A-Fa-f]{6,8})/).flatten
|
||||
total_frequencies = frequencies.sum.to_f
|
||||
|
||||
frequencies.map.with_index { |f, i| [f / total_frequencies, hex_values[i]] }
|
||||
.sort_by { |r| -r[0] }
|
||||
.reject { |r| r[1].size == 8 && r[1].end_with?('00') }
|
||||
.map { |r| ColorDiff::Color::RGB.new(*r[1][0..5].scan(/../).map { |c| c.to_i(16) }) }
|
||||
.slice(0, quantity)
|
||||
end
|
||||
|
||||
def rgb_to_hex(rgb)
|
||||
format('#%02x%02x%02x', rgb.r, rgb.g, rgb.b)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,126 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class GifReader
|
||||
attr_reader :animated
|
||||
|
||||
EXTENSION_LABELS = [0xf9, 0x01, 0xff].freeze
|
||||
GIF_HEADERS = %w(GIF87a GIF89a).freeze
|
||||
|
||||
class GifReaderException < StandardError; end
|
||||
|
||||
class UnknownImageType < GifReaderException; end
|
||||
|
||||
class CannotParseImage < GifReaderException; end
|
||||
|
||||
def self.animated?(path)
|
||||
new(path).animated
|
||||
rescue GifReaderException
|
||||
false
|
||||
end
|
||||
|
||||
def initialize(path, max_frames = 2)
|
||||
@path = path
|
||||
@nb_frames = 0
|
||||
|
||||
File.open(path, 'rb') do |s|
|
||||
raise UnknownImageType unless GIF_HEADERS.include?(s.read(6))
|
||||
|
||||
# Skip to "packed byte"
|
||||
s.seek(4, IO::SEEK_CUR)
|
||||
|
||||
# "Packed byte" gives us the size of the GIF color table
|
||||
packed_byte, = s.read(1).unpack('C')
|
||||
|
||||
# Skip background color and aspect ratio
|
||||
s.seek(2, IO::SEEK_CUR)
|
||||
|
||||
if packed_byte & 0x80 != 0
|
||||
# GIF uses a global color table, skip it
|
||||
s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
|
||||
end
|
||||
|
||||
# Now read data
|
||||
while @nb_frames < max_frames
|
||||
separator = s.read(1)
|
||||
|
||||
case separator
|
||||
when ',' # Image block
|
||||
@nb_frames += 1
|
||||
|
||||
# Skip to "packed byte"
|
||||
s.seek(8, IO::SEEK_CUR)
|
||||
packed_byte, = s.read(1).unpack('C')
|
||||
|
||||
if packed_byte & 0x80 != 0
|
||||
# Image uses a local color table, skip it
|
||||
s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
|
||||
end
|
||||
|
||||
# Skip lzw min code size
|
||||
raise InvalidValue unless s.read(1).unpack1('C') >= 2
|
||||
|
||||
# Skip image data sub-blocks
|
||||
skip_sub_blocks!(s)
|
||||
when '!' # Extension block
|
||||
skip_extension_block!(s)
|
||||
when ';' # Trailer
|
||||
break
|
||||
else
|
||||
raise CannotParseImage
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@animated = @nb_frames > 1
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def skip_extension_block!(file)
|
||||
if EXTENSION_LABELS.include?(file.read(1).unpack1('C'))
|
||||
block_size, = file.read(1).unpack('C')
|
||||
file.seek(block_size, IO::SEEK_CUR)
|
||||
end
|
||||
|
||||
# Read until extension block end marker
|
||||
skip_sub_blocks!(file)
|
||||
end
|
||||
|
||||
# Skip sub-blocks up until block end marker
|
||||
def skip_sub_blocks!(file)
|
||||
loop do
|
||||
size, = file.read(1).unpack('C')
|
||||
|
||||
break if size.zero?
|
||||
|
||||
file.seek(size, IO::SEEK_CUR)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module Paperclip
|
||||
# This transcoder is only to be used for the MediaAttachment model
|
||||
# to convert animated GIFs to videos
|
||||
|
||||
class GifTranscoder < Paperclip::Processor
|
||||
def make
|
||||
return File.open(@file.path) unless needs_convert?
|
||||
|
||||
final_file = Paperclip::Transcoder.make(file, options, attachment)
|
||||
|
||||
if options[:style] == :original
|
||||
attachment.instance.file_file_name = "#{File.basename(attachment.instance.file_file_name, '.*')}.mp4"
|
||||
attachment.instance.file_content_type = 'video/mp4'
|
||||
attachment.instance.type = MediaAttachment.types[:gifv]
|
||||
end
|
||||
|
||||
final_file
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def needs_convert?
|
||||
GifReader.animated?(file.path)
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,50 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'mime/types/columnar'
|
||||
|
||||
module Paperclip
|
||||
class ImageExtractor < Paperclip::Processor
|
||||
def make
|
||||
return @file unless options[:style] == :original
|
||||
|
||||
image = extract_image_from_file!
|
||||
|
||||
unless image.nil?
|
||||
begin
|
||||
attachment.instance.thumbnail = image if image.size.positive?
|
||||
ensure
|
||||
# Paperclip does not automatically delete the source file of
|
||||
# a new attachment while working on copies of it, so we need
|
||||
# to make sure it's cleaned up
|
||||
|
||||
begin
|
||||
image.close(true)
|
||||
rescue Errno::ENOENT
|
||||
nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@file
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def extract_image_from_file!
|
||||
dst = Tempfile.new([File.basename(@file.path, '.*'), '.png'])
|
||||
dst.binmode
|
||||
|
||||
begin
|
||||
command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
|
||||
command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
|
||||
rescue Terrapin::ExitStatusError
|
||||
dst.close(true)
|
||||
return nil
|
||||
rescue Terrapin::CommandNotFoundError
|
||||
raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
|
||||
end
|
||||
|
||||
dst
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,39 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
class LazyThumbnail < Paperclip::Thumbnail
|
||||
def make
|
||||
return File.open(@file.path) unless needs_convert?
|
||||
|
||||
if options[:geometry]
|
||||
min_side = [@current_geometry.width, @current_geometry.height].min.to_i
|
||||
options[:geometry] = "#{min_side}x#{min_side}#" if @target_geometry.square? && min_side < @target_geometry.width
|
||||
elsif options[:pixels]
|
||||
width = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height)).round.to_i
|
||||
height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width)).round.to_i
|
||||
options[:geometry] = "#{width}x#{height}>"
|
||||
end
|
||||
|
||||
Paperclip::Thumbnail.make(file, options, attachment)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def needs_convert?
|
||||
needs_different_geometry? || needs_different_format? || needs_metadata_stripping?
|
||||
end
|
||||
|
||||
def needs_different_geometry?
|
||||
(options[:geometry] && @current_geometry.width != @target_geometry.width && @current_geometry.height != @target_geometry.height) ||
|
||||
(options[:pixels] && @current_geometry.width * @current_geometry.height > options[:pixels])
|
||||
end
|
||||
|
||||
def needs_different_format?
|
||||
@format.present? && @current_format != @format
|
||||
end
|
||||
|
||||
def needs_metadata_stripping?
|
||||
@attachment.instance.respond_to?(:local?) && @attachment.instance.local?
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,24 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
module MediaTypeSpoofDetectorExtensions
|
||||
MARCEL_MIME_TYPES = %w(audio/mpeg image/avif).freeze
|
||||
|
||||
def calculated_content_type
|
||||
return @calculated_content_type if defined?(@calculated_content_type)
|
||||
|
||||
@calculated_content_type = type_from_file_command.chomp
|
||||
|
||||
# The `file` command fails to recognize some MP3 files as such
|
||||
@calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel.in?(MARCEL_MIME_TYPES)
|
||||
@calculated_content_type
|
||||
end
|
||||
|
||||
def type_from_marcel
|
||||
@type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path),
|
||||
name: @file.path
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions)
|
||||
@@ -1,55 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
class ResponseWithLimitAdapter < AbstractAdapter
|
||||
def self.register
|
||||
Paperclip.io_adapters.register self do |target|
|
||||
target.is_a?(ResponseWithLimit)
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(target, options = {})
|
||||
super
|
||||
cache_current_values
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def cache_current_values
|
||||
@original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data'
|
||||
@tempfile = copy_to_tempfile(@target)
|
||||
@content_type = ContentTypeDetector.new(@tempfile.path).detect
|
||||
@size = File.size(@tempfile)
|
||||
end
|
||||
|
||||
def copy_to_tempfile(source)
|
||||
bytes_read = 0
|
||||
|
||||
source.response.body.each do |chunk|
|
||||
bytes_read += chunk.bytesize
|
||||
|
||||
destination.write(chunk)
|
||||
chunk.clear
|
||||
|
||||
raise Mastodon::LengthValidationError if bytes_read > source.limit
|
||||
end
|
||||
|
||||
destination.rewind
|
||||
destination
|
||||
rescue Mastodon::LengthValidationError
|
||||
destination.close(true)
|
||||
raise
|
||||
ensure
|
||||
source.response.connection.close
|
||||
end
|
||||
|
||||
def filename_from_content_disposition
|
||||
disposition = @target.response.headers['content-disposition']
|
||||
disposition&.match(/filename="([^"]*)"/)&.captures&.first
|
||||
end
|
||||
|
||||
def filename_from_path
|
||||
@target.response.uri.path.split('/').last
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,125 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
# This transcoder is only to be used for the MediaAttachment model
|
||||
# to check when uploaded videos are actually gifv's
|
||||
class Transcoder < Paperclip::Processor
|
||||
# This is the H.264 "High" value taken from https://www.dr-lex.be/info-stuff/videocalc.html
|
||||
BITS_PER_PIXEL = 0.11
|
||||
|
||||
def initialize(file, options = {}, attachment = nil)
|
||||
super
|
||||
|
||||
@current_format = File.extname(@file.path)
|
||||
@basename = File.basename(@file.path, @current_format)
|
||||
@format = options[:format]
|
||||
@time = options[:time] || 3
|
||||
@passthrough_options = options[:passthrough_options]
|
||||
@convert_options = options[:convert_options].dup
|
||||
@vfr_threshold = options[:vfr_frame_rate_threshold]
|
||||
end
|
||||
|
||||
def make
|
||||
metadata = VideoMetadataExtractor.new(@file.path)
|
||||
|
||||
raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid?
|
||||
|
||||
update_attachment_type(metadata)
|
||||
update_options_from_metadata(metadata)
|
||||
|
||||
destination = Tempfile.new([@basename, @format ? ".#{@format}" : ''])
|
||||
destination.binmode
|
||||
|
||||
@output_options = @convert_options[:output]&.dup || {}
|
||||
@input_options = @convert_options[:input]&.dup || {}
|
||||
|
||||
case @format.to_s
|
||||
when /jpg$/, /jpeg$/, /png$/, /gif$/
|
||||
@input_options['ss'] = @time
|
||||
|
||||
@output_options['f'] = 'image2'
|
||||
@output_options['vframes'] = 1
|
||||
when 'mp4'
|
||||
unless eligible_to_passthrough?(metadata)
|
||||
size_limit_in_bits = MediaAttachment::VIDEO_LIMIT * 8
|
||||
desired_bitrate = (metadata.width * metadata.height * 30 * BITS_PER_PIXEL).floor
|
||||
duration = [metadata.duration, 1].max
|
||||
maximum_bitrate = (size_limit_in_bits / duration).floor - 192_000 # Leave some space for the audio stream
|
||||
bitrate = [desired_bitrate, maximum_bitrate].min
|
||||
|
||||
@output_options['b:v'] = bitrate
|
||||
@output_options['maxrate'] = bitrate + 192_000
|
||||
@output_options['bufsize'] = bitrate * 5
|
||||
|
||||
if high_vfr?(metadata)
|
||||
@output_options['vsync'] = 'vfr'
|
||||
@output_options['r'] = @vfr_threshold
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
command_arguments, interpolations = prepare_command(destination)
|
||||
|
||||
begin
|
||||
command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
|
||||
command.run(interpolations)
|
||||
rescue Terrapin::ExitStatusError => e
|
||||
raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"
|
||||
rescue Terrapin::CommandNotFoundError
|
||||
raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
|
||||
end
|
||||
|
||||
destination
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def prepare_command(destination)
|
||||
command_arguments = ['-nostdin']
|
||||
interpolations = {}
|
||||
interpolation_keys = 0
|
||||
|
||||
@input_options.each_pair do |key, value|
|
||||
interpolation_key = interpolation_keys
|
||||
command_arguments << "-#{key} :#{interpolation_key}"
|
||||
interpolations[interpolation_key] = value
|
||||
interpolation_keys += 1
|
||||
end
|
||||
|
||||
command_arguments << '-i :source'
|
||||
interpolations[:source] = @file.path
|
||||
|
||||
@output_options.each_pair do |key, value|
|
||||
interpolation_key = interpolation_keys
|
||||
command_arguments << "-#{key} :#{interpolation_key}"
|
||||
interpolations[interpolation_key] = value
|
||||
interpolation_keys += 1
|
||||
end
|
||||
|
||||
command_arguments << '-y :destination'
|
||||
interpolations[:destination] = destination.path
|
||||
|
||||
[command_arguments, interpolations]
|
||||
end
|
||||
|
||||
def update_options_from_metadata(metadata)
|
||||
return unless eligible_to_passthrough?(metadata)
|
||||
|
||||
@format = @passthrough_options[:options][:format] || @format
|
||||
@time = @passthrough_options[:options][:time] || @time
|
||||
@convert_options = @passthrough_options[:options][:convert_options].dup
|
||||
end
|
||||
|
||||
def high_vfr?(metadata)
|
||||
@vfr_threshold && metadata.r_frame_rate && metadata.r_frame_rate > @vfr_threshold
|
||||
end
|
||||
|
||||
def eligible_to_passthrough?(metadata)
|
||||
@passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
|
||||
end
|
||||
|
||||
def update_attachment_type(metadata)
|
||||
@attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,21 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'mime/types/columnar'
|
||||
|
||||
module Paperclip
|
||||
class TypeCorrector < Paperclip::Processor
|
||||
def make
|
||||
return @file unless options[:format]
|
||||
|
||||
target_extension = ".#{options[:format]}"
|
||||
extension = File.extname(attachment.instance_read(:file_name))
|
||||
|
||||
return @file unless options[:style] == :original && target_extension && extension != target_extension
|
||||
|
||||
attachment.instance_write(:content_type, options[:content_type] || attachment.instance_read(:content_type))
|
||||
attachment.instance_write(:file_name, File.basename(attachment.instance_read(:file_name), '.*') + target_extension)
|
||||
|
||||
@file
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,11 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Paperclip
|
||||
module UrlGeneratorExtensions
|
||||
def for_as_default(style_name)
|
||||
attachment_options[:interpolator].interpolate(default_url, @attachment, style_name)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Paperclip::UrlGenerator.prepend(Paperclip::UrlGeneratorExtensions)
|
||||
@@ -1,48 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'action_dispatch/middleware/static'
|
||||
|
||||
class PublicFileServerMiddleware
|
||||
SERVICE_WORKER_TTL = 7.days.to_i
|
||||
CACHE_TTL = 28.days.to_i
|
||||
|
||||
def initialize(app)
|
||||
@app = app
|
||||
@file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first)
|
||||
end
|
||||
|
||||
def call(env)
|
||||
file = @file_handler.attempt(env)
|
||||
|
||||
# If the request is not a static file, move on!
|
||||
return @app.call(env) if file.nil?
|
||||
|
||||
status, headers, response = file
|
||||
|
||||
# Set cache headers on static files. Some paths require different cache headers
|
||||
headers['Cache-Control'] = begin
|
||||
request_path = env['REQUEST_PATH']
|
||||
|
||||
if request_path.start_with?('/sw.js')
|
||||
"public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate"
|
||||
elsif request_path.start_with?(paperclip_root_url)
|
||||
"public, max-age=#{CACHE_TTL}, immutable"
|
||||
else
|
||||
"public, max-age=#{CACHE_TTL}, must-revalidate"
|
||||
end
|
||||
end
|
||||
|
||||
# Override the default CSP header set by the CSP middleware
|
||||
headers['Content-Security-Policy'] = "default-src 'none'; form-action 'none'" if request_path.start_with?(paperclip_root_url)
|
||||
|
||||
headers['X-Content-Type-Options'] = 'nosniff'
|
||||
|
||||
[status, headers, response]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def paperclip_root_url
|
||||
ENV.fetch('PAPERCLIP_ROOT_URL', '/system')
|
||||
end
|
||||
end
|
||||
@@ -1,13 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Rails
|
||||
module EngineExtensions
|
||||
# Rewrite task loading code to filter digitalocean.rake task
|
||||
def run_tasks_blocks(app)
|
||||
Railtie.instance_method(:run_tasks_blocks).bind_call(self, app)
|
||||
paths['lib/tasks'].existent.reject { |ext| ext.end_with?('digitalocean.rake') }.sort.each { |ext| load(ext) }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Rails::Engine.prepend(Rails::EngineExtensions)
|
||||
@@ -1,12 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Redis
|
||||
module NamespaceExtensions
|
||||
def exists?(...)
|
||||
call_with_namespace('exists?', ...)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Redis::Namespace::COMMANDS['exists?'] = [:first]
|
||||
Redis::Namespace.prepend(Redis::NamespaceExtensions)
|
||||
@@ -1,175 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Sanitize
|
||||
module Config
|
||||
HTTP_PROTOCOLS = %w(
|
||||
http
|
||||
https
|
||||
).freeze
|
||||
|
||||
LINK_PROTOCOLS = %w(
|
||||
http
|
||||
https
|
||||
dat
|
||||
dweb
|
||||
ipfs
|
||||
ipns
|
||||
ssb
|
||||
gopher
|
||||
xmpp
|
||||
magnet
|
||||
gemini
|
||||
).freeze
|
||||
|
||||
CLASS_WHITELIST_TRANSFORMER = lambda do |env|
|
||||
node = env[:node]
|
||||
class_list = node['class']&.split(/[\t\n\f\r ]/)
|
||||
|
||||
return unless class_list
|
||||
|
||||
class_list.keep_if do |e|
|
||||
next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes
|
||||
next true if /^(mention|hashtag)$/.match?(e) # semantic classes
|
||||
next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes
|
||||
end
|
||||
|
||||
node['class'] = class_list.join(' ')
|
||||
end
|
||||
|
||||
IMG_TAG_TRANSFORMER = lambda do |env|
|
||||
node = env[:node]
|
||||
|
||||
return unless env[:node_name] == 'img'
|
||||
|
||||
node.name = 'a'
|
||||
|
||||
node['href'] = node['src']
|
||||
if node['alt'].present?
|
||||
node.content = "[🖼 #{node['alt']}]"
|
||||
else
|
||||
url = node['href']
|
||||
prefix = url.match(%r{\Ahttps?://(www\.)?}).to_s
|
||||
text = url[prefix.length, 30]
|
||||
text += '…' if url.length - prefix.length > 30
|
||||
node.content = "[🖼 #{text}]"
|
||||
end
|
||||
end
|
||||
|
||||
TRANSLATE_TRANSFORMER = lambda do |env|
|
||||
node = env[:node]
|
||||
node.remove_attribute('translate') unless node['translate'] == 'no'
|
||||
end
|
||||
|
||||
UNSUPPORTED_HREF_TRANSFORMER = lambda do |env|
|
||||
return unless env[:node_name] == 'a'
|
||||
|
||||
current_node = env[:node]
|
||||
|
||||
scheme = if current_node['href'] =~ Sanitize::REGEX_PROTOCOL
|
||||
Regexp.last_match(1).downcase
|
||||
else
|
||||
:relative
|
||||
end
|
||||
|
||||
current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme)
|
||||
end
|
||||
|
||||
MASTODON_STRICT ||= freeze_config(
|
||||
elements: %w(p br span a abbr del pre blockquote code b strong u sub sup i em h1 h2 h3 h4 h5 ul ol li),
|
||||
|
||||
attributes: {
|
||||
'a' => %w(href rel class title translate),
|
||||
'abbr' => %w(title),
|
||||
'span' => %w(class translate),
|
||||
'blockquote' => %w(cite),
|
||||
'ol' => %w(start reversed),
|
||||
'li' => %w(value),
|
||||
},
|
||||
|
||||
add_attributes: {
|
||||
'a' => {
|
||||
'rel' => 'nofollow noopener noreferrer',
|
||||
'target' => '_blank',
|
||||
},
|
||||
},
|
||||
|
||||
protocols: {
|
||||
'a' => { 'href' => LINK_PROTOCOLS },
|
||||
'blockquote' => { 'cite' => LINK_PROTOCOLS },
|
||||
},
|
||||
|
||||
transformers: [
|
||||
CLASS_WHITELIST_TRANSFORMER,
|
||||
IMG_TAG_TRANSFORMER,
|
||||
TRANSLATE_TRANSFORMER,
|
||||
UNSUPPORTED_HREF_TRANSFORMER,
|
||||
]
|
||||
)
|
||||
|
||||
MASTODON_OEMBED ||= freeze_config(
|
||||
elements: %w(audio embed iframe source video),
|
||||
|
||||
attributes: {
|
||||
'audio' => %w(controls),
|
||||
'embed' => %w(height src type width),
|
||||
'iframe' => %w(allowfullscreen frameborder height scrolling src width),
|
||||
'source' => %w(src type),
|
||||
'video' => %w(controls height loop width),
|
||||
},
|
||||
|
||||
protocols: {
|
||||
'embed' => { 'src' => HTTP_PROTOCOLS },
|
||||
'iframe' => { 'src' => HTTP_PROTOCOLS },
|
||||
'source' => { 'src' => HTTP_PROTOCOLS },
|
||||
},
|
||||
|
||||
add_attributes: {
|
||||
'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' },
|
||||
}
|
||||
)
|
||||
|
||||
LINK_REL_TRANSFORMER = lambda do |env|
|
||||
return unless env[:node_name] == 'a' && env[:node]['href']
|
||||
|
||||
node = env[:node]
|
||||
|
||||
rel = (node['rel'] || '').split & ['tag']
|
||||
rel += %w(nofollow noopener noreferrer) unless TagManager.instance.local_url?(node['href'])
|
||||
|
||||
if rel.empty?
|
||||
node.remove_attribute('rel')
|
||||
else
|
||||
node['rel'] = rel.join(' ')
|
||||
end
|
||||
end
|
||||
|
||||
LINK_TARGET_TRANSFORMER = lambda do |env|
|
||||
return unless env[:node_name] == 'a' && env[:node]['href']
|
||||
|
||||
node = env[:node]
|
||||
if node['target'] != '_blank' && TagManager.instance.local_url?(node['href'])
|
||||
node.remove_attribute('target')
|
||||
else
|
||||
node['target'] = '_blank'
|
||||
end
|
||||
end
|
||||
|
||||
MASTODON_OUTGOING ||= freeze_config MASTODON_STRICT.merge(
|
||||
attributes: merge(
|
||||
MASTODON_STRICT[:attributes],
|
||||
'a' => %w(href rel class title target translate)
|
||||
),
|
||||
|
||||
add_attributes: {},
|
||||
|
||||
transformers: [
|
||||
CLASS_WHITELIST_TRANSFORMER,
|
||||
IMG_TAG_TRANSFORMER,
|
||||
TRANSLATE_TRANSFORMER,
|
||||
UNSUPPORTED_HREF_TRANSFORMER,
|
||||
LINK_REL_TRANSFORMER,
|
||||
LINK_TARGET_TRANSFORMER,
|
||||
]
|
||||
)
|
||||
end
|
||||
end
|
||||
@@ -1,15 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module SimpleNavigation
|
||||
module ItemExtensions
|
||||
def url
|
||||
if @url.nil? && @sub_navigation
|
||||
@sub_navigation.items.first.url
|
||||
else
|
||||
@url
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
SimpleNavigation::Item.prepend(SimpleNavigation::ItemExtensions)
|
||||
@@ -1,26 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
namespace :assets do
|
||||
desc 'Generate static pages'
|
||||
task generate_static_pages: :environment do
|
||||
def render_static_page(action, dest:, **opts)
|
||||
renderer = Class.new(ApplicationController) do
|
||||
def current_user
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
html = renderer.render(action, opts)
|
||||
File.write(dest, html)
|
||||
end
|
||||
|
||||
render_static_page 'errors/500', layout: 'error', dest: Rails.public_path.join('assets', '500.html')
|
||||
end
|
||||
end
|
||||
|
||||
if Rake::Task.task_defined?('assets:precompile')
|
||||
Rake::Task['assets:precompile'].enhance do
|
||||
Webpacker.manifest.refresh
|
||||
Rake::Task['assets:generate_static_pages'].invoke
|
||||
end
|
||||
end
|
||||
@@ -1,46 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
if Rails.env.development?
|
||||
task :set_annotation_options do
|
||||
Annotate.set_defaults(
|
||||
'routes' => 'false',
|
||||
'models' => 'true',
|
||||
'position_in_routes' => 'before',
|
||||
'position_in_class' => 'before',
|
||||
'position_in_test' => 'before',
|
||||
'position_in_fixture' => 'before',
|
||||
'position_in_factory' => 'before',
|
||||
'position_in_serializer' => 'before',
|
||||
'show_foreign_keys' => 'false',
|
||||
'show_indexes' => 'false',
|
||||
'simple_indexes' => 'false',
|
||||
'model_dir' => 'app/models',
|
||||
'root_dir' => '',
|
||||
'include_version' => 'false',
|
||||
'require' => '',
|
||||
'exclude_tests' => 'true',
|
||||
'exclude_fixtures' => 'true',
|
||||
'exclude_factories' => 'true',
|
||||
'exclude_serializers' => 'true',
|
||||
'exclude_scaffolds' => 'true',
|
||||
'exclude_controllers' => 'true',
|
||||
'exclude_helpers' => 'true',
|
||||
'ignore_model_sub_dir' => 'false',
|
||||
'ignore_columns' => nil,
|
||||
'ignore_routes' => nil,
|
||||
'ignore_unknown_models' => 'false',
|
||||
'hide_limit_column_types' => 'integer,boolean',
|
||||
'skip_on_db_migrate' => 'false',
|
||||
'format_bare' => 'true',
|
||||
'format_rdoc' => 'false',
|
||||
'format_markdown' => 'false',
|
||||
'sort' => 'false',
|
||||
'force' => 'false',
|
||||
'trace' => 'false',
|
||||
'wrapper_open' => nil,
|
||||
'wrapper_close' => nil
|
||||
)
|
||||
end
|
||||
|
||||
Annotate.load_tasks
|
||||
end
|
||||
@@ -1,79 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
namespace :branding do
|
||||
desc 'Generate necessary graphic assets for branding from source SVG files'
|
||||
task generate: :environment do
|
||||
Rake::Task['branding:generate_app_icons'].invoke
|
||||
Rake::Task['branding:generate_app_badge'].invoke
|
||||
Rake::Task['branding:generate_github_assets'].invoke
|
||||
Rake::Task['branding:generate_mailer_assets'].invoke
|
||||
end
|
||||
|
||||
desc 'Generate PNG icons and logos for e-mail templates'
|
||||
task generate_mailer_assets: :environment do
|
||||
rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '-h :size --keep-aspect-ratio :input -o :output')
|
||||
output_dest = Rails.root.join('app', 'javascript', 'images', 'mailer')
|
||||
|
||||
# Displayed size is 64px, at 3x it's 192px
|
||||
Dir[Rails.root.join('app', 'javascript', 'images', 'icons', '*.svg')].each do |path|
|
||||
rsvg_convert.run(input: path, size: 192, output: output_dest.join("#{File.basename(path, '.svg')}.png"))
|
||||
end
|
||||
|
||||
# Displayed size is 34px, at 3x it's 102px
|
||||
rsvg_convert.run(input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.png'))
|
||||
|
||||
# Displayed size is 24px, at 3x it's 72px
|
||||
rsvg_convert.run(input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-icon.svg'), size: 72, output: output_dest.join('logo.png'))
|
||||
end
|
||||
|
||||
desc 'Generate light/dark logotypes for GitHub'
|
||||
task generate_github_assets: :environment do
|
||||
rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '--stylesheet :stylesheet -h :size --keep-aspect-ratio :input -o :output')
|
||||
output_dest = Rails.root.join('lib', 'assets')
|
||||
|
||||
rsvg_convert.run(stylesheet: Rails.root.join('lib', 'assets', 'wordmark.dark.css'), input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.dark.png'))
|
||||
rsvg_convert.run(stylesheet: Rails.root.join('lib', 'assets', 'wordmark.light.css'), input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.light.png'))
|
||||
end
|
||||
|
||||
desc 'Generate favicons and app icons from SVG source files'
|
||||
task generate_app_icons: :environment do
|
||||
favicon_source = Rails.root.join('app', 'javascript', 'images', 'logo.svg')
|
||||
app_icon_source = Rails.root.join('app', 'javascript', 'images', 'app-icon.svg')
|
||||
output_dest = Rails.root.join('app', 'javascript', 'icons')
|
||||
|
||||
rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '-w :size -h :size --keep-aspect-ratio :input -o :output')
|
||||
convert = Terrapin::CommandLine.new('convert', ':input :output', environment: { 'MAGICK_CONFIGURE_PATH' => nil })
|
||||
|
||||
favicon_sizes = [16, 32, 48]
|
||||
apple_icon_sizes = [57, 60, 72, 76, 114, 120, 144, 152, 167, 180, 1024]
|
||||
android_icon_sizes = [36, 48, 72, 96, 144, 192, 256, 384, 512]
|
||||
|
||||
favicons = []
|
||||
|
||||
favicon_sizes.each do |size|
|
||||
output_path = output_dest.join("favicon-#{size}x#{size}.png")
|
||||
favicons << output_path
|
||||
rsvg_convert.run(size: size, input: favicon_source, output: output_path)
|
||||
end
|
||||
|
||||
convert.run(input: favicons, output: Rails.public_path.join('favicon.ico'))
|
||||
|
||||
apple_icon_sizes.each do |size|
|
||||
rsvg_convert.run(size: size, input: app_icon_source, output: output_dest.join("apple-touch-icon-#{size}x#{size}.png"))
|
||||
end
|
||||
|
||||
android_icon_sizes.each do |size|
|
||||
rsvg_convert.run(size: size, input: app_icon_source, output: output_dest.join("android-chrome-#{size}x#{size}.png"))
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Generate badge icon from SVG source files'
|
||||
task generate_app_badge: :environment do
|
||||
rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '--stylesheet :stylesheet -w :size -h :size --keep-aspect-ratio :input -o :output')
|
||||
badge_source = Rails.root.join('app', 'javascript', 'images', 'logo-symbol-icon.svg')
|
||||
output_dest = Rails.public_path
|
||||
stylesheet = Rails.root.join('lib', 'assets', 'wordmark.light.css')
|
||||
|
||||
rsvg_convert.run(stylesheet: stylesheet, input: badge_source, size: 192, output: output_dest.join('badge.png'))
|
||||
end
|
||||
end
|
||||
@@ -1,24 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
namespace :db do
|
||||
namespace :migrate do
|
||||
desc 'Setup the db or migrate depending on state of db'
|
||||
task setup: :environment do
|
||||
if ActiveRecord::Migrator.current_version.zero?
|
||||
Rake::Task['db:migrate'].invoke
|
||||
Rake::Task['db:seed'].invoke
|
||||
end
|
||||
rescue ActiveRecord::NoDatabaseError
|
||||
Rake::Task['db:setup'].invoke
|
||||
else
|
||||
Rake::Task['db:migrate'].invoke
|
||||
end
|
||||
end
|
||||
|
||||
task :pre_migration_check do
|
||||
version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
|
||||
abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500
|
||||
end
|
||||
|
||||
Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
|
||||
end
|
||||
@@ -1,106 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
def gen_border(codepoint, color)
|
||||
input = Rails.public_path.join('emoji', "#{codepoint}.svg")
|
||||
dest = Rails.public_path.join('emoji', "#{codepoint}_border.svg")
|
||||
doc = File.open(input) { |f| Nokogiri::XML(f) }
|
||||
svg = doc.at_css('svg')
|
||||
if svg.key?('viewBox')
|
||||
view_box = svg['viewBox'].split.map(&:to_i)
|
||||
view_box[0] -= 2
|
||||
view_box[1] -= 2
|
||||
view_box[2] += 4
|
||||
view_box[3] += 4
|
||||
svg['viewBox'] = view_box.join(' ')
|
||||
end
|
||||
g = Nokogiri::XML::Node.new 'g', doc
|
||||
doc.css('svg > *').each do |elem|
|
||||
border_elem = elem.dup
|
||||
|
||||
border_elem.delete('fill')
|
||||
|
||||
border_elem['stroke'] = color
|
||||
border_elem['stroke-linejoin'] = 'round'
|
||||
border_elem['stroke-width'] = '4px'
|
||||
|
||||
g.add_child(border_elem)
|
||||
end
|
||||
svg.prepend_child(g)
|
||||
File.write(dest, doc.to_xml)
|
||||
puts "Wrote bordered #{codepoint}.svg to #{dest}!"
|
||||
end
|
||||
|
||||
def codepoints_to_filename(codepoints)
|
||||
codepoints.downcase.gsub(/\A0+/, '').tr(' ', '-')
|
||||
end
|
||||
|
||||
def codepoints_to_unicode(codepoints)
|
||||
if codepoints.include?(' ')
|
||||
codepoints.split.map(&:hex).pack('U*')
|
||||
else
|
||||
[codepoints.hex].pack('U')
|
||||
end
|
||||
end
|
||||
|
||||
namespace :emojis do
|
||||
desc 'Generate a unicode to filename mapping'
|
||||
task :generate do
|
||||
source = 'http://www.unicode.org/Public/emoji/14.0/emoji-test.txt'
|
||||
codes = []
|
||||
dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')
|
||||
|
||||
puts "Downloading emojos from source... (#{source})"
|
||||
|
||||
HTTP.get(source).to_s.split("\n").each do |line|
|
||||
next if line.start_with? '#'
|
||||
|
||||
parts = line.split(';').map(&:strip)
|
||||
next if parts.size < 2
|
||||
|
||||
codes << [parts[0], parts[1].start_with?('fully-qualified')]
|
||||
end
|
||||
|
||||
grouped_codes = codes.reduce([]) do |agg, current|
|
||||
if current[1]
|
||||
agg << [current[0]]
|
||||
else
|
||||
agg.last << current[0]
|
||||
agg
|
||||
end
|
||||
end
|
||||
|
||||
existence_maps = grouped_codes.map { |c| c.index_with { |cc| Rails.public_path.join('emoji', "#{codepoints_to_filename(cc)}.svg").exist? } }
|
||||
map = {}
|
||||
|
||||
existence_maps.each do |group|
|
||||
existing_one = group.key(true)
|
||||
|
||||
next if existing_one.nil?
|
||||
|
||||
group.each_key do |key|
|
||||
map[codepoints_to_unicode(key)] = codepoints_to_filename(existing_one)
|
||||
end
|
||||
end
|
||||
|
||||
map = map.sort { |a, b| a[0].size <=> b[0].size }.to_h
|
||||
|
||||
File.write(dest, Oj.dump(map))
|
||||
puts "Wrote emojo to destination! (#{dest})"
|
||||
end
|
||||
|
||||
desc 'Generate emoji variants with white borders'
|
||||
task :generate_borders do
|
||||
src = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')
|
||||
emojis_light = '👽⚾🐔☁️💨🕊️👀🍥👻🐐❕❔⛸️🌩️🔊🔇📃🌧️🐏🍚🍙🐓🐑💀☠️🌨️🔉🔈💬💭🏐🏳️⚪⬜◽◻️▫️'
|
||||
emojis_dark = '🎱🐜⚫🖤⬛◼️◾◼️✒️▪️💣🎳📷📸♣️🕶️✴️🔌💂♀️📽️🍳🦍💂🔪🕳️🕹️🕋🖊️🖋️💂♂️🎤🎓🎥🎼♠️🎩🦃📼📹🎮🐃🏴🐞🕺📱📲🚲'
|
||||
|
||||
map = Oj.load(File.read(src))
|
||||
|
||||
emojis_light.each_grapheme_cluster do |emoji|
|
||||
gen_border map[emoji], 'black'
|
||||
end
|
||||
emojis_dark.each_grapheme_cluster do |emoji|
|
||||
gen_border map[emoji], 'white'
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,12 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
namespace :glitchsoc do
|
||||
desc 'Backfill local-only flag on statuses table'
|
||||
task backfill_local_only: :environment do
|
||||
Status.local.where(local_only: nil).find_each do |status|
|
||||
ActiveRecord::Base.logger.silence do
|
||||
status.update_attribute(:local_only, status.marked_local_only?) # rubocop:disable Rails/SkipsModelValidations
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,608 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'tty-prompt'
|
||||
|
||||
namespace :mastodon do
|
||||
desc 'Configure the instance for production use'
|
||||
task :setup do
|
||||
prompt = TTY::Prompt.new
|
||||
env = {}
|
||||
|
||||
# When the application code gets loaded, it runs `lib/mastodon/redis_configuration.rb`.
|
||||
# This happens before application environment configuration and sets REDIS_URL etc.
|
||||
# These variables are then used even when REDIS_HOST etc. are changed, so clear them
|
||||
# out so they don't interfere with our new configuration.
|
||||
ENV.delete('REDIS_URL')
|
||||
ENV.delete('CACHE_REDIS_URL')
|
||||
ENV.delete('SIDEKIQ_REDIS_URL')
|
||||
|
||||
begin
|
||||
prompt.say('Your instance is identified by its domain name. Changing it afterward will break things.')
|
||||
env['LOCAL_DOMAIN'] = prompt.ask('Domain name:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
q.validate(/\A[a-z0-9.-]+\z/i)
|
||||
q.messages[:valid?] = 'Invalid domain. If you intend to use unicode characters, enter punycode here'
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
prompt.say('Single user mode disables registrations and redirects the landing page to your public profile.')
|
||||
env['SINGLE_USER_MODE'] = prompt.yes?('Do you want to enable single user mode?', default: false)
|
||||
|
||||
%w(SECRET_KEY_BASE OTP_SECRET).each do |key|
|
||||
env[key] = SecureRandom.hex(64)
|
||||
end
|
||||
|
||||
vapid_key = Webpush.generate_key
|
||||
|
||||
env['VAPID_PRIVATE_KEY'] = vapid_key.private_key
|
||||
env['VAPID_PUBLIC_KEY'] = vapid_key.public_key
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
using_docker = prompt.yes?('Are you using Docker to run Mastodon?')
|
||||
db_connection_works = false
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
loop do
|
||||
env['DB_HOST'] = prompt.ask('PostgreSQL host:') do |q|
|
||||
q.required true
|
||||
q.default using_docker ? 'db' : '/var/run/postgresql'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['DB_PORT'] = prompt.ask('PostgreSQL port:') do |q|
|
||||
q.required true
|
||||
q.default 5432
|
||||
q.convert :int
|
||||
end
|
||||
|
||||
env['DB_NAME'] = prompt.ask('Name of PostgreSQL database:') do |q|
|
||||
q.required true
|
||||
q.default using_docker ? 'postgres' : 'mastodon_production'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['DB_USER'] = prompt.ask('Name of PostgreSQL user:') do |q|
|
||||
q.required true
|
||||
q.default using_docker ? 'postgres' : 'mastodon'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['DB_PASS'] = prompt.ask('Password of PostgreSQL user:') do |q|
|
||||
q.echo false
|
||||
end
|
||||
|
||||
# The chosen database may not exist yet. Connect to default database
|
||||
# to avoid "database does not exist" error.
|
||||
db_options = {
|
||||
adapter: :postgresql,
|
||||
database: 'postgres',
|
||||
host: env['DB_HOST'],
|
||||
port: env['DB_PORT'],
|
||||
user: env['DB_USER'],
|
||||
password: env['DB_PASS'],
|
||||
}
|
||||
|
||||
begin
|
||||
ActiveRecord::Base.establish_connection(db_options)
|
||||
ActiveRecord::Base.connection
|
||||
prompt.ok 'Database configuration works! 🎆'
|
||||
db_connection_works = true
|
||||
break
|
||||
rescue => e
|
||||
prompt.error 'Database connection could not be established with this configuration, try again.'
|
||||
prompt.error e.message
|
||||
break unless prompt.yes?('Try again?')
|
||||
end
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
loop do
|
||||
env['REDIS_HOST'] = prompt.ask('Redis host:') do |q|
|
||||
q.required true
|
||||
q.default using_docker ? 'redis' : 'localhost'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['REDIS_PORT'] = prompt.ask('Redis port:') do |q|
|
||||
q.required true
|
||||
q.default 6379
|
||||
q.convert :int
|
||||
end
|
||||
|
||||
env['REDIS_PASSWORD'] = prompt.ask('Redis password:') do |q|
|
||||
q.required false
|
||||
q.default nil
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
redis_options = {
|
||||
host: env['REDIS_HOST'],
|
||||
port: env['REDIS_PORT'],
|
||||
password: env['REDIS_PASSWORD'],
|
||||
driver: :hiredis,
|
||||
}
|
||||
|
||||
begin
|
||||
redis = Redis.new(redis_options)
|
||||
redis.ping
|
||||
prompt.ok 'Redis configuration works! 🎆'
|
||||
break
|
||||
rescue => e
|
||||
prompt.error 'Redis connection could not be established with this configuration, try again.'
|
||||
prompt.error e.message
|
||||
break unless prompt.yes?('Try again?')
|
||||
end
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
if prompt.yes?('Do you want to store uploaded files on the cloud?', default: false)
|
||||
case prompt.select('Provider', ['DigitalOcean Spaces', 'Amazon S3', 'Wasabi', 'Minio', 'Google Cloud Storage', 'Storj DCS'])
|
||||
when 'DigitalOcean Spaces'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('Space name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_REGION'] = prompt.ask('Space region:') do |q|
|
||||
q.required true
|
||||
q.default 'nyc3'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_HOSTNAME'] = prompt.ask('Space endpoint:') do |q|
|
||||
q.required true
|
||||
q.default 'nyc3.digitaloceanspaces.com'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_ENDPOINT'] = "https://#{env['S3_HOSTNAME']}"
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Space access key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Space secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
when 'Amazon S3'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('S3 bucket name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_REGION'] = prompt.ask('S3 region:') do |q|
|
||||
q.required true
|
||||
q.default 'us-east-1'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_HOSTNAME'] = prompt.ask('S3 hostname:') do |q|
|
||||
q.required true
|
||||
q.default 's3.us-east-1.amazonaws.com'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('S3 access key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('S3 secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
when 'Wasabi'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
env['S3_REGION'] = 'us-east-1'
|
||||
env['S3_HOSTNAME'] = 's3.wasabisys.com'
|
||||
env['S3_ENDPOINT'] = 'https://s3.wasabisys.com/'
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('Wasabi bucket name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Wasabi access key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Wasabi secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
when 'Minio'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
env['S3_REGION'] = 'us-east-1'
|
||||
|
||||
env['S3_ENDPOINT'] = prompt.ask('Minio endpoint URL:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_PROTOCOL'] = env['S3_ENDPOINT'].start_with?('https') ? 'https' : 'http'
|
||||
env['S3_HOSTNAME'] = env['S3_ENDPOINT'].gsub(%r{\Ahttps?://}, '')
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('Minio bucket name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Minio access key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Minio secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
when 'Storj DCS'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
env['S3_REGION'] = 'global'
|
||||
|
||||
env['S3_ENDPOINT'] = prompt.ask('Storj DCS endpoint URL:') do |q|
|
||||
q.required true
|
||||
q.default 'https://gateway.storjshare.io'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_PROTOCOL'] = env['S3_ENDPOINT'].start_with?('https') ? 'https' : 'http'
|
||||
env['S3_HOSTNAME'] = env['S3_ENDPOINT'].gsub(%r{\Ahttps?://}, '')
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('Storj DCS bucket name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('Storj Gateway access key (uplink share --register --readonly=false --not-after=none sj://bucket):') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('Storj Gateway secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
linksharing_access_key = prompt.ask('Storj Linksharing access key (uplink share --register --public --readonly=true --disallow-lists --not-after=none sj://bucket):') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
env['S3_ALIAS_HOST'] = "link.storjshare.io/raw/#{linksharing_access_key}/#{env['S3_BUCKET']}"
|
||||
|
||||
when 'Google Cloud Storage'
|
||||
env['S3_ENABLED'] = 'true'
|
||||
env['S3_PROTOCOL'] = 'https'
|
||||
env['S3_HOSTNAME'] = 'storage.googleapis.com'
|
||||
env['S3_ENDPOINT'] = 'https://storage.googleapis.com'
|
||||
env['S3_MULTIPART_THRESHOLD'] = 50.megabytes
|
||||
|
||||
env['S3_BUCKET'] = prompt.ask('GCS bucket name:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['S3_REGION'] = prompt.ask('GCS region:') do |q|
|
||||
q.required true
|
||||
q.default 'us-west1'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_ACCESS_KEY_ID'] = prompt.ask('GCS access key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('GCS secret key:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
end
|
||||
|
||||
if prompt.yes?('Do you want to access the uploaded files from your own domain?')
|
||||
env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q|
|
||||
q.required true
|
||||
q.default "files.#{env['LOCAL_DOMAIN']}"
|
||||
q.modify :strip
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
loop do
|
||||
if prompt.yes?('Do you want to send e-mails from localhost?', default: false)
|
||||
env['SMTP_SERVER'] = 'localhost'
|
||||
env['SMTP_PORT'] = 25
|
||||
env['SMTP_AUTH_METHOD'] = 'none'
|
||||
env['SMTP_OPENSSL_VERIFY_MODE'] = 'none'
|
||||
env['SMTP_ENABLE_STARTTLS'] = 'auto'
|
||||
else
|
||||
env['SMTP_SERVER'] = prompt.ask('SMTP server:') do |q|
|
||||
q.required true
|
||||
q.default 'smtp.mailgun.org'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['SMTP_PORT'] = prompt.ask('SMTP port:') do |q|
|
||||
q.required true
|
||||
q.default 587
|
||||
q.convert :int
|
||||
end
|
||||
|
||||
env['SMTP_LOGIN'] = prompt.ask('SMTP username:') do |q|
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['SMTP_PASSWORD'] = prompt.ask('SMTP password:') do |q|
|
||||
q.echo false
|
||||
end
|
||||
|
||||
env['SMTP_AUTH_METHOD'] = prompt.ask('SMTP authentication:') do |q|
|
||||
q.required
|
||||
q.default 'plain'
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
env['SMTP_OPENSSL_VERIFY_MODE'] = prompt.select('SMTP OpenSSL verify mode:', %w(none peer client_once fail_if_no_peer_cert))
|
||||
|
||||
env['SMTP_ENABLE_STARTTLS'] = prompt.select('Enable STARTTLS:', %w(auto always never))
|
||||
end
|
||||
|
||||
env['SMTP_FROM_ADDRESS'] = prompt.ask('E-mail address to send e-mails "from":') do |q|
|
||||
q.required true
|
||||
q.default "Mastodon <notifications@#{env['LOCAL_DOMAIN']}>"
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
break unless prompt.yes?('Send a test e-mail with this configuration right now?')
|
||||
|
||||
send_to = prompt.ask('Send test e-mail to:', required: true)
|
||||
|
||||
begin
|
||||
enable_starttls = nil
|
||||
enable_starttls_auto = nil
|
||||
|
||||
case env['SMTP_ENABLE_STARTTLS']
|
||||
when 'always'
|
||||
enable_starttls = true
|
||||
when 'never'
|
||||
enable_starttls = false
|
||||
when 'auto'
|
||||
enable_starttls_auto = true
|
||||
else
|
||||
enable_starttls_auto = env['SMTP_ENABLE_STARTTLS_AUTO'] != 'false'
|
||||
end
|
||||
|
||||
ActionMailer::Base.smtp_settings = {
|
||||
port: env['SMTP_PORT'],
|
||||
address: env['SMTP_SERVER'],
|
||||
user_name: env['SMTP_LOGIN'].presence,
|
||||
password: env['SMTP_PASSWORD'].presence,
|
||||
domain: env['LOCAL_DOMAIN'],
|
||||
authentication: env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain,
|
||||
openssl_verify_mode: env['SMTP_OPENSSL_VERIFY_MODE'],
|
||||
enable_starttls: enable_starttls,
|
||||
enable_starttls_auto: enable_starttls_auto,
|
||||
}
|
||||
|
||||
ActionMailer::Base.default_options = {
|
||||
from: env['SMTP_FROM_ADDRESS'],
|
||||
}
|
||||
|
||||
mail = ActionMailer::Base.new.mail to: send_to, subject: 'Test', body: 'Mastodon SMTP configuration works!'
|
||||
mail.deliver
|
||||
break
|
||||
rescue => e
|
||||
prompt.error 'E-mail could not be sent with this configuration, try again.'
|
||||
prompt.error e.message
|
||||
break unless prompt.yes?('Try again?')
|
||||
end
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
|
||||
env['UPDATE_CHECK_URL'] = '' unless prompt.yes?('Do you want Mastodon to periodically check for important updates and notify you? (Recommended)', default: true)
|
||||
|
||||
prompt.say "\n"
|
||||
prompt.say 'This configuration will be written to .env.production'
|
||||
|
||||
if prompt.yes?('Save configuration?')
|
||||
incompatible_syntax = false
|
||||
|
||||
env_contents = env.each_pair.map do |key, value|
|
||||
value = value.to_s
|
||||
escaped = dotenv_escape(value)
|
||||
incompatible_syntax = true if value != escaped
|
||||
|
||||
"#{key}=#{escaped}"
|
||||
end.join("\n")
|
||||
|
||||
generated_header = generate_header(incompatible_syntax)
|
||||
|
||||
Rails.root.join('.env.production').write("#{generated_header}#{env_contents}\n")
|
||||
|
||||
if using_docker
|
||||
prompt.ok 'Below is your configuration, save it to an .env.production file outside Docker:'
|
||||
prompt.say "\n"
|
||||
prompt.say "#{generated_header}#{env.each_pair.map { |key, value| "#{key}=#{value}" }.join("\n")}"
|
||||
prompt.say "\n"
|
||||
prompt.ok 'It is also saved within this container so you can proceed with this wizard.'
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
prompt.say 'Now that configuration is saved, the database schema must be loaded.'
|
||||
prompt.warn 'If the database already exists, this will erase its contents.'
|
||||
|
||||
if prompt.yes?('Prepare the database now?')
|
||||
prompt.say 'Running `RAILS_ENV=production rails db:setup` ...'
|
||||
prompt.say "\n\n"
|
||||
|
||||
if system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production', 'SAFETY_ASSURED' => '1' }), 'rails db:setup')
|
||||
prompt.ok 'Done!'
|
||||
else
|
||||
prompt.error 'That failed! Perhaps your configuration is not right'
|
||||
end
|
||||
end
|
||||
|
||||
unless using_docker
|
||||
prompt.say "\n"
|
||||
prompt.say 'The final step is compiling CSS/JS assets.'
|
||||
prompt.say 'This may take a while and consume a lot of RAM.'
|
||||
|
||||
if prompt.yes?('Compile the assets now?')
|
||||
prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...'
|
||||
prompt.say "\n\n"
|
||||
|
||||
if system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile')
|
||||
prompt.say 'Done!'
|
||||
else
|
||||
prompt.error 'That failed! Maybe you need swap space?'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
prompt.say "\n"
|
||||
prompt.ok 'All done! You can now power on the Mastodon server 🐘'
|
||||
prompt.say "\n"
|
||||
|
||||
if db_connection_works && prompt.yes?('Do you want to create an admin user straight away?')
|
||||
env.each_pair do |key, value|
|
||||
ENV[key] = value.to_s
|
||||
end
|
||||
|
||||
require_relative '../../config/environment'
|
||||
disable_log_stdout!
|
||||
|
||||
username = prompt.ask('Username:') do |q|
|
||||
q.required true
|
||||
q.default 'admin'
|
||||
q.validate(/\A[a-z0-9_]+\z/i)
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
email = prompt.ask('E-mail:') do |q|
|
||||
q.required true
|
||||
q.modify :strip
|
||||
end
|
||||
|
||||
password = SecureRandom.hex(16)
|
||||
|
||||
owner_role = UserRole.find_by(name: 'Owner')
|
||||
user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
|
||||
user.save(validate: false)
|
||||
|
||||
Setting.site_contact_username = username
|
||||
|
||||
prompt.ok "You can login with the password: #{password}"
|
||||
prompt.warn 'You can change your password once you login.'
|
||||
end
|
||||
else
|
||||
prompt.warn 'Nothing saved. Bye!'
|
||||
end
|
||||
rescue TTY::Reader::InputInterrupt
|
||||
prompt.ok 'Aborting. Bye!'
|
||||
end
|
||||
end
|
||||
|
||||
namespace :webpush do
|
||||
desc 'Generate VAPID key'
|
||||
task :generate_vapid_key do
|
||||
vapid_key = Webpush.generate_key
|
||||
puts "VAPID_PRIVATE_KEY=#{vapid_key.private_key}"
|
||||
puts "VAPID_PUBLIC_KEY=#{vapid_key.public_key}"
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate_header(include_warning)
|
||||
default_message = "# Generated with mastodon:setup on #{Time.now.utc}\n\n"
|
||||
|
||||
default_message.tap do |string|
|
||||
if include_warning
|
||||
string << "# Some variables in this file will be interpreted differently whether you are\n"
|
||||
string << "# using docker-compose or not.\n\n"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def disable_log_stdout!
|
||||
dev_null = Logger.new('/dev/null')
|
||||
|
||||
Rails.logger = dev_null
|
||||
ActiveRecord::Base.logger = dev_null
|
||||
HttpLog.configuration.logger = dev_null
|
||||
Paperclip.options[:log] = false
|
||||
end
|
||||
|
||||
def dotenv_escape(value)
|
||||
# Dotenv has its own parser, which unfortunately deviates somewhat from
|
||||
# what shells actually do.
|
||||
#
|
||||
# In particular, we can't use Shellwords::escape because it outputs a
|
||||
# non-quotable string, while Dotenv requires `#` to always be in quoted
|
||||
# strings.
|
||||
#
|
||||
# Therefore, we need to write our own escape code…
|
||||
# Dotenv's parser has a *lot* of edge cases, and I think not every
|
||||
# ASCII string can even be represented into something Dotenv can parse,
|
||||
# so this is a best effort thing.
|
||||
#
|
||||
# In particular, strings with all the following probably cannot be
|
||||
# escaped:
|
||||
# - `#`, or ends with spaces, which requires some form of quoting (simply escaping won't work)
|
||||
# - `'` (single quote), preventing us from single-quoting
|
||||
# - `\` followed by either `r` or `n`
|
||||
|
||||
# No character that would cause Dotenv trouble
|
||||
return value unless /[\s\#\\"'$]/.match?(value)
|
||||
|
||||
# As long as the value doesn't include single quotes, we can safely
|
||||
# rely on single quotes
|
||||
return "'#{value}'" unless value.include?("'")
|
||||
|
||||
# If the value contains the string '\n' or '\r' we simply can't use
|
||||
# a double-quoted string, because Dotenv will expand \n or \r no
|
||||
# matter how much escaping we add.
|
||||
double_quoting_disallowed = /\\[rn]/.match?(value)
|
||||
|
||||
value = value.gsub(double_quoting_disallowed ? /[\\"'\s]/ : /[\\"']/) { |x| "\\#{x}" }
|
||||
|
||||
# Dotenv is especially tricky with `$` as unbalanced
|
||||
# parenthesis will make it not unescape `\$` as `$`…
|
||||
|
||||
# Variables
|
||||
value = value.gsub(/\$(?!\()/) { |x| "\\#{x}" }
|
||||
# Commands
|
||||
value = value.gsub(/\$(?<cmd>\((?:[^()]|\g<cmd>)+\))/) { |x| "\\#{x}" }
|
||||
|
||||
value = "\"#{value}\"" unless double_quoting_disallowed
|
||||
|
||||
value
|
||||
end
|
||||
@@ -1,136 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
REPOSITORY_NAME = 'mastodon/mastodon'
|
||||
|
||||
namespace :repo do
|
||||
desc 'Generate the AUTHORS.md file'
|
||||
task :authors do
|
||||
file = Rails.root.join('AUTHORS.md').open('w')
|
||||
|
||||
file << <<~HEADER
|
||||
Authors
|
||||
=======
|
||||
|
||||
Mastodon is available on [GitHub](https://github.com/#{REPOSITORY_NAME})
|
||||
and provided thanks to the work of the following contributors:
|
||||
|
||||
HEADER
|
||||
|
||||
url = "https://api.github.com/repos/#{REPOSITORY_NAME}/contributors?anon=1"
|
||||
|
||||
HttpLog.config.compact_log = true
|
||||
|
||||
while url.present?
|
||||
response = HTTP.get(url)
|
||||
contributors = Oj.load(response.body)
|
||||
|
||||
contributors.each do |c|
|
||||
file << "* [#{c['login']}](#{c['html_url']})\n" if c['login']
|
||||
file << "* [#{c['name']}](mailto:#{c['email']})\n" if c['name']
|
||||
end
|
||||
|
||||
url = LinkHeader.parse(response.headers['Link']).find_link(%w(rel next))&.href
|
||||
end
|
||||
|
||||
file << <<~FOOTER
|
||||
|
||||
This document is provided for informational purposes only. Since it is only updated once per release, the version you are looking at may be currently out of date. To see the full list of contributors, consider looking at the [git history](https://github.com/mastodon/mastodon/graphs/contributors) instead.
|
||||
FOOTER
|
||||
end
|
||||
|
||||
desc 'Replace pull requests with authors in the CHANGELOG.md file'
|
||||
task :changelog do
|
||||
path = Rails.root.join('CHANGELOG.md')
|
||||
tmp = Tempfile.new
|
||||
|
||||
HttpLog.config.compact_log = true
|
||||
|
||||
begin
|
||||
File.open(path, 'r') do |file|
|
||||
file.each_line do |line|
|
||||
if line.start_with?('-')
|
||||
new_line = line.gsub(/[(]#([[:digit:]]+)[)]\Z/) do |pull_request_reference|
|
||||
pull_request_number = pull_request_reference[2..-2]
|
||||
response = nil
|
||||
|
||||
loop do
|
||||
response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/#{REPOSITORY_NAME}/pulls/#{pull_request_number}")
|
||||
|
||||
if response.code == 403
|
||||
sleep_for = (response.headers['X-RateLimit-Reset'].to_i - Time.now.to_i).abs
|
||||
puts "Sleeping for #{sleep_for} seconds to get over rate limit"
|
||||
sleep sleep_for
|
||||
else
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
pull_request = Oj.load(response.to_s)
|
||||
"([#{pull_request['user']['login']}](#{pull_request['html_url']}))"
|
||||
end
|
||||
|
||||
tmp.puts new_line
|
||||
else
|
||||
tmp.puts line
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
tmp.close
|
||||
FileUtils.mv(tmp.path, path)
|
||||
ensure
|
||||
tmp.close
|
||||
tmp.unlink
|
||||
end
|
||||
end
|
||||
|
||||
task check_locales_files: :environment do
|
||||
pastel = Pastel.new
|
||||
|
||||
missing_yaml_files = I18n.available_locales.reject { |locale| Rails.root.join('config', 'locales', "#{locale}.yml").exist? }
|
||||
missing_json_files = I18n.available_locales.reject { |locale| Rails.root.join('app', 'javascript', 'mastodon', 'locales', "#{locale}.json").exist? }
|
||||
|
||||
locales_in_files = Dir[Rails.root.join('config', 'locales', '*.yml')].map do |path|
|
||||
file_name = File.basename(path, '.yml')
|
||||
file_name.gsub(/\A(doorkeeper|devise|activerecord|simple_form)\./, '').to_sym
|
||||
end.uniq.compact
|
||||
|
||||
missing_available_locales = locales_in_files - I18n.available_locales
|
||||
supported_locale_codes = Set.new(LanguagesHelper::SUPPORTED_LOCALES.keys + LanguagesHelper::REGIONAL_LOCALE_NAMES.keys)
|
||||
missing_locale_names = I18n.available_locales.reject { |locale| supported_locale_codes.include?(locale) }
|
||||
|
||||
critical = false
|
||||
|
||||
unless missing_json_files.empty?
|
||||
critical = true
|
||||
|
||||
puts pastel.red("You are missing JSON files for these locales: #{pastel.bold(missing_json_files.join(', '))}")
|
||||
puts pastel.red('This will lead to runtime errors for users who have selected those locales')
|
||||
puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
|
||||
end
|
||||
|
||||
unless missing_yaml_files.empty?
|
||||
critical = true
|
||||
|
||||
puts pastel.red("You are missing YAML files for these locales: #{pastel.bold(missing_yaml_files.join(', '))}")
|
||||
puts pastel.red('This will lead to runtime errors for users who have selected those locales')
|
||||
puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
|
||||
end
|
||||
|
||||
unless missing_available_locales.empty?
|
||||
puts pastel.yellow("You have locale files that are not enabled: #{pastel.bold(missing_available_locales.join(', '))}")
|
||||
puts pastel.yellow("Add them to #{pastel.bold('I18n.available_locales')} in config/application.rb or remove them")
|
||||
end
|
||||
|
||||
unless missing_locale_names.empty?
|
||||
puts pastel.yellow("You are missing human-readable names for these locales: #{pastel.bold(missing_locale_names.join(', '))}")
|
||||
puts pastel.yellow("Add them to app/helpers/languages_helper.rb or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
|
||||
end
|
||||
|
||||
if critical
|
||||
exit(1)
|
||||
else
|
||||
puts pastel.green('OK')
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,21 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
if Rake::Task.task_defined?('spec:system')
|
||||
namespace :spec do
|
||||
task :enable_system_specs do # rubocop:disable Rails/RakeEnvironment
|
||||
ENV['RUN_SYSTEM_SPECS'] = 'true'
|
||||
end
|
||||
end
|
||||
|
||||
Rake::Task['spec:system'].enhance ['spec:enable_system_specs']
|
||||
end
|
||||
|
||||
if Rake::Task.task_defined?('spec:search')
|
||||
namespace :spec do
|
||||
task :enable_search_specs do # rubocop:disable Rails/RakeEnvironment
|
||||
ENV['RUN_SEARCH_SPECS'] = 'true'
|
||||
end
|
||||
end
|
||||
|
||||
Rake::Task['spec:search'].enhance ['spec:enable_search_specs']
|
||||
end
|
||||
@@ -1,19 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
task stats: 'mastodon:stats'
|
||||
|
||||
namespace :mastodon do
|
||||
desc 'Report code statistics (KLOCs, etc)'
|
||||
task :stats do
|
||||
require 'rails/code_statistics'
|
||||
[
|
||||
['App Libraries', 'app/lib'],
|
||||
%w(Presenters app/presenters),
|
||||
%w(Services app/services),
|
||||
%w(Validators app/validators),
|
||||
%w(Workers app/workers),
|
||||
].each do |name, dir|
|
||||
STATS_DIRECTORIES << [name, Rails.root.join(dir)]
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,392 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
namespace :tests do
|
||||
namespace :migrations do
|
||||
desc 'Check that database state is consistent with a successful migration from populated data'
|
||||
task check_database: :environment do
|
||||
unless Account.find_by(username: 'admin', domain: nil)&.hide_collections? == false
|
||||
puts 'Unexpected value for Account#hide_collections? for user @admin'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_by(username: 'user', domain: nil)&.hide_collections? == true
|
||||
puts 'Unexpected value for Account#hide_collections? for user @user'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_by(username: 'evil', domain: 'activitypub.com')&.suspended?
|
||||
puts 'Unexpected value for Account#suspended? for user @evil@activitypub.com'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Status.find(6).account_id == Status.find(7).account_id
|
||||
puts 'Users @remote@remote.com and @Remote@remote.com not properly merged'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if Account.where(domain: Rails.configuration.x.local_domain).exists?
|
||||
puts 'Faux remote accounts not properly cleaned up'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless AccountConversation.first&.last_status_id == 11
|
||||
puts 'AccountConversation records not created as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
if Account.find(-99).private_key.blank?
|
||||
puts 'Instance actor does not have a private key'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_by(username: 'user', domain: nil).custom_filters.map { |filter| filter.keywords.pluck(:keyword) } == [['test'], ['take']]
|
||||
puts 'CustomFilterKeyword records not created as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Admin::ActionLog.find_by(target_type: 'DomainBlock', target_id: 1).human_identifier == 'example.org'
|
||||
puts 'Admin::ActionLog domain block records not updated as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Admin::ActionLog.find_by(target_type: 'EmailDomainBlock', target_id: 1).human_identifier == 'example.org'
|
||||
puts 'Admin::ActionLog email domain block records not updated as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless User.find(1).settings['notification_emails.favourite'] == true && User.find(1).settings['notification_emails.mention'] == false
|
||||
puts 'User settings not kept as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless User.find(1).settings['web.trends'] == false
|
||||
puts 'User settings not kept as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_remote('bob', 'ActivityPub.com').domain == 'activitypub.com'
|
||||
puts 'Account domains not properly normalized'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Status.find(12).preview_cards.pluck(:url) == ['https://joinmastodon.org/']
|
||||
puts 'Preview cards not deduplicated as expected'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_local('kmruser').user.chosen_languages == %w(en ku ckb)
|
||||
puts 'Chosen languages not migrated as expected for kmr users'
|
||||
exit(1)
|
||||
end
|
||||
|
||||
unless Account.find_local('kmruser').user.settings['default_language'] == 'ku'
|
||||
puts 'Default posting language not migrated as expected for kmr users'
|
||||
exit(1)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Populate the database with test data for 2.4.3'
|
||||
task populate_v2_4_3: :environment do # rubocop:disable Naming/VariableNumber
|
||||
user_key = OpenSSL::PKey::RSA.new(2048)
|
||||
user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem)
|
||||
user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem)
|
||||
|
||||
ActiveRecord::Base.connection.execute(<<~SQL)
|
||||
INSERT INTO "custom_filters"
|
||||
(id, account_id, phrase, context, whole_word, irreversible, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 2, 'test', '{ "home", "public" }', true, true, now(), now()),
|
||||
(2, 2, 'take', '{ "home" }', false, false, now(), now());
|
||||
|
||||
-- Orphaned admin action logs
|
||||
|
||||
INSERT INTO "admin_action_logs"
|
||||
(account_id, action, target_type, target_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'destroy', 'Account', 1312, now(), now()),
|
||||
(1, 'destroy', 'User', 1312, now(), now()),
|
||||
(1, 'destroy', 'Report', 1312, now(), now()),
|
||||
(1, 'destroy', 'DomainBlock', 1312, now(), now()),
|
||||
(1, 'destroy', 'EmailDomainBlock', 1312, now(), now()),
|
||||
(1, 'destroy', 'Status', 1312, now(), now()),
|
||||
(1, 'destroy', 'CustomEmoji', 1312, now(), now());
|
||||
|
||||
-- Admin action logs with linked objects
|
||||
|
||||
INSERT INTO "domain_blocks"
|
||||
(id, domain, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'example.org', now(), now());
|
||||
|
||||
INSERT INTO "email_domain_blocks"
|
||||
(id, domain, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'example.org', now(), now());
|
||||
|
||||
INSERT INTO "admin_action_logs"
|
||||
(account_id, action, target_type, target_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'destroy', 'Account', 1, now(), now()),
|
||||
(1, 'destroy', 'User', 1, now(), now()),
|
||||
(1, 'destroy', 'DomainBlock', 1, now(), now()),
|
||||
(1, 'destroy', 'EmailDomainBlock', 1, now(), now()),
|
||||
(1, 'destroy', 'Status', 1, now(), now()),
|
||||
(1, 'destroy', 'CustomEmoji', 3, now(), now());
|
||||
|
||||
INSERT INTO "settings"
|
||||
(id, thing_type, thing_id, var, value, created_at, updated_at)
|
||||
VALUES
|
||||
(3, 'User', 1, 'notification_emails', E'--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess\nfollow: false\nreblog: true\nfavourite: true\nmention: false\nfollow_request: true\ndigest: true\nreport: true\npending_account: false\ntrending_tag: true\nappeal: true\n', now(), now()),
|
||||
(4, 'User', 1, 'trends', E'--- false\n', now(), now());
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at)
|
||||
VALUES
|
||||
(10, 'kmruser', NULL, #{user_private_key}, #{user_public_key}, now(), now());
|
||||
|
||||
INSERT INTO "users"
|
||||
(id, account_id, email, created_at, updated_at, admin, locale, chosen_languages)
|
||||
VALUES
|
||||
(4, 10, 'kmruser@localhost', now(), now(), false, 'ku', '{en,kmr,ku,ckb}');
|
||||
|
||||
INSERT INTO "settings"
|
||||
(id, thing_type, thing_id, var, value, created_at, updated_at)
|
||||
VALUES
|
||||
(5, 'User', 4, 'default_language', E'--- kmr\n', now(), now());
|
||||
SQL
|
||||
end
|
||||
|
||||
desc 'Populate the database with test data for 2.4.0'
|
||||
task populate_v2_4: :environment do # rubocop:disable Naming/VariableNumber
|
||||
ActiveRecord::Base.connection.execute(<<~SQL.squish)
|
||||
INSERT INTO "settings"
|
||||
(id, thing_type, thing_id, var, value, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'User', 1, 'hide_network', E'--- false\n', now(), now()),
|
||||
(2, 'User', 2, 'hide_network', E'--- true\n', now(), now());
|
||||
SQL
|
||||
end
|
||||
|
||||
desc 'Populate the database with test data for 2.0.0'
|
||||
task populate_v2: :environment do
|
||||
admin_key = OpenSSL::PKey::RSA.new(2048)
|
||||
user_key = OpenSSL::PKey::RSA.new(2048)
|
||||
remote_key = OpenSSL::PKey::RSA.new(2048)
|
||||
remote_key2 = OpenSSL::PKey::RSA.new(2048)
|
||||
remote_key3 = OpenSSL::PKey::RSA.new(2048)
|
||||
admin_private_key = ActiveRecord::Base.connection.quote(admin_key.to_pem)
|
||||
admin_public_key = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem)
|
||||
user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem)
|
||||
user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem)
|
||||
remote_public_key = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem)
|
||||
remote_public_key2 = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem)
|
||||
remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem)
|
||||
local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain)
|
||||
|
||||
ActiveRecord::Base.connection.execute(<<~SQL)
|
||||
-- accounts
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()),
|
||||
(2, 'user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url)
|
||||
VALUES
|
||||
(3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
|
||||
'https://remote.com/@remote', 'https://remote.com/salmon/1'),
|
||||
(4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
|
||||
'https://remote.com/@Remote', 'https://remote.com/salmon/1'),
|
||||
(5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now() - interval '1 year', now() - interval '1 year',
|
||||
'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1');
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url)
|
||||
VALUES
|
||||
(6, 'bob', 'ActivityPub.com', NULL, #{remote_public_key_ap}, now(), now(),
|
||||
1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers');
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at)
|
||||
VALUES
|
||||
(7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()),
|
||||
(8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
|
||||
|
||||
INSERT INTO "accounts"
|
||||
(id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url, suspended)
|
||||
VALUES
|
||||
(9, 'evil', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(),
|
||||
1, 'https://activitypub.com/users/evil/inbox', 'https://activitypub.com/users/evil/outbox',
|
||||
'https://activitypub.com/users/evil/followers', true);
|
||||
|
||||
-- users
|
||||
|
||||
INSERT INTO "users"
|
||||
(id, account_id, email, created_at, updated_at, admin)
|
||||
VALUES
|
||||
(1, 1, 'admin@localhost', now(), now(), true),
|
||||
(2, 2, 'user@localhost', now(), now(), false);
|
||||
|
||||
INSERT INTO "users"
|
||||
(id, account_id, email, created_at, updated_at, admin, locale)
|
||||
VALUES
|
||||
(3, 8, 'ptuser@localhost', now(), now(), false, 'pt');
|
||||
|
||||
-- conversations
|
||||
INSERT INTO "conversations" (id, created_at, updated_at) VALUES (1, now(), now());
|
||||
|
||||
-- statuses
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 1, 'test', now(), now()),
|
||||
(2, 1, '@remote@remote.com hello', now(), now()),
|
||||
(3, 1, '@Remote@remote.com hello', now(), now()),
|
||||
(4, 1, '@REMOTE@remote.com hello', now(), now());
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at, uri, local)
|
||||
VALUES
|
||||
(5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true);
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at)
|
||||
VALUES
|
||||
(6, 3, 'test', now(), now());
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id)
|
||||
VALUES
|
||||
(7, 4, '@admin hello', now(), now(), 3, 1);
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at)
|
||||
VALUES
|
||||
(8, 5, 'test', now(), now());
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, reblog_of_id, created_at, updated_at)
|
||||
VALUES
|
||||
(9, 1, 2, now(), now());
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, in_reply_to_id, conversation_id, visibility, created_at, updated_at)
|
||||
VALUES
|
||||
(10, 2, '@admin hey!', NULL, 1, 3, now(), now()),
|
||||
(11, 1, '@user hey!', 10, 1, 3, now(), now());
|
||||
|
||||
INSERT INTO "statuses"
|
||||
(id, account_id, text, created_at, updated_at)
|
||||
VALUES
|
||||
(12, 1, 'check out https://joinmastodon.org/', now(), now());
|
||||
|
||||
-- mentions (from previous statuses)
|
||||
|
||||
INSERT INTO "mentions"
|
||||
(id, status_id, account_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 2, 3, now(), now()),
|
||||
(2, 3, 4, now(), now()),
|
||||
(3, 4, 5, now(), now()),
|
||||
(4, 10, 1, now(), now()),
|
||||
(5, 11, 2, now(), now());
|
||||
|
||||
-- stream entries
|
||||
|
||||
INSERT INTO "stream_entries"
|
||||
(activity_id, account_id, activity_type, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 1, 'status', now(), now()),
|
||||
(2, 1, 'status', now(), now()),
|
||||
(3, 1, 'status', now(), now()),
|
||||
(4, 1, 'status', now(), now()),
|
||||
(5, 1, 'status', now(), now()),
|
||||
(6, 3, 'status', now(), now()),
|
||||
(7, 4, 'status', now(), now()),
|
||||
(8, 5, 'status', now(), now()),
|
||||
(9, 1, 'status', now(), now());
|
||||
|
||||
-- custom emoji
|
||||
|
||||
INSERT INTO "custom_emojis"
|
||||
(id, shortcode, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'test', now(), now()),
|
||||
(2, 'Test', now(), now()),
|
||||
(3, 'blobcat', now(), now());
|
||||
|
||||
INSERT INTO "custom_emojis"
|
||||
(id, shortcode, domain, uri, created_at, updated_at)
|
||||
VALUES
|
||||
(4, 'blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
|
||||
(5, 'blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
|
||||
(6, 'Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now());
|
||||
|
||||
-- favourites
|
||||
|
||||
INSERT INTO "favourites"
|
||||
(account_id, status_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 1, now(), now()),
|
||||
(1, 7, now(), now()),
|
||||
(4, 1, now(), now()),
|
||||
(3, 1, now(), now()),
|
||||
(5, 1, now(), now());
|
||||
|
||||
-- pinned statuses
|
||||
|
||||
INSERT INTO "status_pins"
|
||||
(account_id, status_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 1, now(), now()),
|
||||
(3, 6, now(), now()),
|
||||
(4, 7, now(), now());
|
||||
|
||||
-- follows
|
||||
|
||||
INSERT INTO "follows"
|
||||
(id, account_id, target_account_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 1, 5, now(), now()),
|
||||
(2, 6, 2, now(), now()),
|
||||
(3, 5, 2, now(), now()),
|
||||
(4, 6, 1, now(), now());
|
||||
|
||||
-- follow requests
|
||||
|
||||
INSERT INTO "follow_requests"
|
||||
(account_id, target_account_id, created_at, updated_at)
|
||||
VALUES
|
||||
(2, 5, now(), now()),
|
||||
(5, 1, now(), now());
|
||||
|
||||
-- notifications
|
||||
|
||||
INSERT INTO "notifications"
|
||||
(id, from_account_id, account_id, activity_type, activity_id, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 6, 2, 'Follow', 2, now(), now()),
|
||||
(2, 2, 1, 'Mention', 4, now(), now()),
|
||||
(3, 1, 2, 'Mention', 5, now(), now());
|
||||
|
||||
-- preview cards
|
||||
|
||||
INSERT INTO "preview_cards"
|
||||
(id, url, title, created_at, updated_at)
|
||||
VALUES
|
||||
(1, 'https://joinmastodon.org/', 'Mastodon - Decentralized social media', now(), now());
|
||||
|
||||
-- many-to-many association between preview cards and statuses
|
||||
|
||||
INSERT INTO "preview_cards_statuses"
|
||||
(status_id, preview_card_id)
|
||||
VALUES
|
||||
(12, 1),
|
||||
(12, 1);
|
||||
SQL
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -1,10 +0,0 @@
|
||||
= simple_form_for(@<%= singular_table_name %>) do |f|
|
||||
= f.error_notification
|
||||
|
||||
.form-inputs
|
||||
<%- attributes.each do |attribute| -%>
|
||||
= f.<%= attribute.reference? ? :association : :input %> :<%= attribute.name %>
|
||||
<%- end -%>
|
||||
|
||||
.form-actions
|
||||
= f.button :submit
|
||||
@@ -1,66 +0,0 @@
|
||||
# frozen_string_literal: false
|
||||
|
||||
require 'fcntl'
|
||||
|
||||
module Terrapin
|
||||
module MultiPipeExtensions
|
||||
def initialize
|
||||
@stdout_in, @stdout_out = IO.pipe
|
||||
@stderr_in, @stderr_out = IO.pipe
|
||||
|
||||
clear_nonblocking_flags!
|
||||
end
|
||||
|
||||
def pipe_options
|
||||
# Add some flags to explicitly close the other end of the pipes
|
||||
{ :out => @stdout_out, :err => @stderr_out, @stdout_in => :close, @stderr_in => :close }
|
||||
end
|
||||
|
||||
def read
|
||||
# While we are patching Terrapin, fix child process potentially getting stuck on writing
|
||||
# to stderr.
|
||||
|
||||
@stdout_output = +''
|
||||
@stderr_output = +''
|
||||
|
||||
fds_to_read = [@stdout_in, @stderr_in]
|
||||
until fds_to_read.empty?
|
||||
rs, = IO.select(fds_to_read)
|
||||
|
||||
read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in)
|
||||
read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# @param [IO] io IO Stream to read until there is nothing to read
|
||||
# @param [String] result Mutable string to which read values will be appended to
|
||||
# @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF
|
||||
def read_nonblocking!(io, result, fds_to_read)
|
||||
while (partial_result = io.read_nonblock(8192))
|
||||
result << partial_result
|
||||
end
|
||||
rescue IO::WaitReadable
|
||||
# Do nothing
|
||||
rescue EOFError
|
||||
fds_to_read.delete(io)
|
||||
end
|
||||
|
||||
def clear_nonblocking_flags!
|
||||
# Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as
|
||||
# needed when calling fork/exec-related syscalls, but posix-spawn does
|
||||
# not currently do that, so we need to do it manually for the time being
|
||||
# so that the child process do not error out when the buffers are full.
|
||||
stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL)
|
||||
@stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK
|
||||
|
||||
stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL)
|
||||
@stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK
|
||||
rescue NameError, NotImplementedError, Errno::EINVAL
|
||||
# Probably on windows, where pipes are blocking by default
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions)
|
||||
@@ -1,27 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Webpacker::HelperExtensions
|
||||
def javascript_pack_tag(name, **options)
|
||||
src, integrity = current_webpacker_instance.manifest.lookup!(name, type: :javascript, with_integrity: true)
|
||||
javascript_include_tag(src, options.merge(integrity: integrity))
|
||||
end
|
||||
|
||||
def stylesheet_pack_tag(name, **options)
|
||||
src, integrity = current_webpacker_instance.manifest.lookup!(name, type: :stylesheet, with_integrity: true)
|
||||
stylesheet_link_tag(src, options.merge(integrity: integrity))
|
||||
end
|
||||
|
||||
def preload_pack_asset(name, **options)
|
||||
src, integrity = current_webpacker_instance.manifest.lookup!(name, with_integrity: true)
|
||||
|
||||
# This attribute will only work if the assets are on a different domain.
|
||||
# And Webpack will (correctly) only add it in this case, so we need to conditionally set it here
|
||||
# otherwise the preloaded request and the real request will have different crossorigin values
|
||||
# and the preloaded file wont be loaded
|
||||
crossorigin = 'anonymous' if Rails.configuration.action_controller.asset_host.present?
|
||||
|
||||
preload_link_tag(src, options.merge(integrity: integrity, crossorigin: crossorigin))
|
||||
end
|
||||
end
|
||||
|
||||
Webpacker::Helper.prepend(Webpacker::HelperExtensions)
|
||||
@@ -1,17 +0,0 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module Webpacker::ManifestExtensions
|
||||
def lookup(name, pack_type = {})
|
||||
asset = super
|
||||
|
||||
if pack_type[:with_integrity] && asset.respond_to?(:dig)
|
||||
[asset.dig('src'), asset.dig('integrity')]
|
||||
elsif asset.respond_to?(:dig)
|
||||
asset.dig('src')
|
||||
else
|
||||
asset
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Webpacker::Manifest.prepend(Webpacker::ManifestExtensions)
|
||||
Reference in New Issue
Block a user