Browse Source

Add separate cache directory for non-local uploads (#12821)

master^2
Eugen Rochko GitHub 1 year ago
parent
commit
c3ca3801f2
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 319 additions and 105 deletions
  1. +46
    -44
      app/models/account.rb
  2. +15
    -14
      app/models/custom_emoji.rb
  3. +18
    -17
      app/models/media_attachment.rb
  4. +24
    -19
      app/models/preview_card.rb
  5. +19
    -3
      config/initializers/paperclip.rb
  6. +9
    -0
      db/migrate/20200417125749_add_storage_schema_version.rb
  7. +6
    -1
      db/schema.rb
  8. +4
    -0
      lib/cli.rb
  9. +4
    -0
      lib/mastodon/cli_helper.rb
  10. +17
    -7
      lib/mastodon/media_cli.rb
  11. +148
    -0
      lib/mastodon/upgrade_cli.rb
  12. +9
    -0
      lib/paperclip/attachment_extensions.rb

+ 46
- 44
app/models/account.rb View File

@@ -3,50 +3,52 @@
#
# Table name: accounts
#
# id :bigint(8) not null, primary key
# username :string default(""), not null
# domain :string
# secret :string default(""), not null
# private_key :text
# public_key :text default(""), not null
# remote_url :string default(""), not null
# salmon_url :string default(""), not null
# hub_url :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# note :text default(""), not null
# display_name :string default(""), not null
# uri :string default(""), not null
# url :string
# avatar_file_name :string
# avatar_content_type :string
# avatar_file_size :integer
# avatar_updated_at :datetime
# header_file_name :string
# header_content_type :string
# header_file_size :integer
# header_updated_at :datetime
# avatar_remote_url :string
# subscription_expires_at :datetime
# locked :boolean default(FALSE), not null
# header_remote_url :string default(""), not null
# last_webfingered_at :datetime
# inbox_url :string default(""), not null
# outbox_url :string default(""), not null
# shared_inbox_url :string default(""), not null
# followers_url :string default(""), not null
# protocol :integer default("ostatus"), not null
# memorial :boolean default(FALSE), not null
# moved_to_account_id :bigint(8)
# featured_collection_url :string
# fields :jsonb
# actor_type :string
# discoverable :boolean
# also_known_as :string is an Array
# silenced_at :datetime
# suspended_at :datetime
# trust_level :integer
# hide_collections :boolean
# id :bigint(8) not null, primary key
# username :string default(""), not null
# domain :string
# secret :string default(""), not null
# private_key :text
# public_key :text default(""), not null
# remote_url :string default(""), not null
# salmon_url :string default(""), not null
# hub_url :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# note :text default(""), not null
# display_name :string default(""), not null
# uri :string default(""), not null
# url :string
# avatar_file_name :string
# avatar_content_type :string
# avatar_file_size :integer
# avatar_updated_at :datetime
# header_file_name :string
# header_content_type :string
# header_file_size :integer
# header_updated_at :datetime
# avatar_remote_url :string
# subscription_expires_at :datetime
# locked :boolean default(FALSE), not null
# header_remote_url :string default(""), not null
# last_webfingered_at :datetime
# inbox_url :string default(""), not null
# outbox_url :string default(""), not null
# shared_inbox_url :string default(""), not null
# followers_url :string default(""), not null
# protocol :integer default("ostatus"), not null
# memorial :boolean default(FALSE), not null
# moved_to_account_id :bigint(8)
# featured_collection_url :string
# fields :jsonb
# actor_type :string
# discoverable :boolean
# also_known_as :string is an Array
# silenced_at :datetime
# suspended_at :datetime
# trust_level :integer
# hide_collections :boolean
# avatar_storage_schema_version :integer
# header_storage_schema_version :integer
#

class Account < ApplicationRecord


+ 15
- 14
app/models/custom_emoji.rb View File

@@ -3,20 +3,21 @@
#
# Table name: custom_emojis
#
# id :bigint(8) not null, primary key
# shortcode :string default(""), not null
# domain :string
# image_file_name :string
# image_content_type :string
# image_file_size :integer
# image_updated_at :datetime
# created_at :datetime not null
# updated_at :datetime not null
# disabled :boolean default(FALSE), not null
# uri :string
# image_remote_url :string
# visible_in_picker :boolean default(TRUE), not null
# category_id :bigint(8)
# id :bigint(8) not null, primary key
# shortcode :string default(""), not null
# domain :string
# image_file_name :string
# image_content_type :string
# image_file_size :integer
# image_updated_at :datetime
# created_at :datetime not null
# updated_at :datetime not null
# disabled :boolean default(FALSE), not null
# uri :string
# image_remote_url :string
# visible_in_picker :boolean default(TRUE), not null
# category_id :bigint(8)
# image_storage_schema_version :integer
#

class CustomEmoji < ApplicationRecord


+ 18
- 17
app/models/media_attachment.rb View File

@@ -3,23 +3,24 @@
#
# Table name: media_attachments
#
# id :bigint(8) not null, primary key
# status_id :bigint(8)
# file_file_name :string
# file_content_type :string
# file_file_size :integer
# file_updated_at :datetime
# remote_url :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# shortcode :string
# type :integer default("image"), not null
# file_meta :json
# account_id :bigint(8)
# description :text
# scheduled_status_id :bigint(8)
# blurhash :string
# processing :integer
# id :bigint(8) not null, primary key
# status_id :bigint(8)
# file_file_name :string
# file_content_type :string
# file_file_size :integer
# file_updated_at :datetime
# remote_url :string default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
# shortcode :string
# type :integer default("image"), not null
# file_meta :json
# account_id :bigint(8)
# description :text
# scheduled_status_id :bigint(8)
# blurhash :string
# processing :integer
# file_storage_schema_version :integer
#

class MediaAttachment < ApplicationRecord


+ 24
- 19
app/models/preview_card.rb View File

@@ -3,25 +3,26 @@
#
# Table name: preview_cards
#
# id :bigint(8) not null, primary key
# url :string default(""), not null
# title :string default(""), not null
# description :string default(""), not null
# image_file_name :string
# image_content_type :string
# image_file_size :integer
# image_updated_at :datetime
# type :integer default("link"), not null
# html :text default(""), not null
# author_name :string default(""), not null
# author_url :string default(""), not null
# provider_name :string default(""), not null
# provider_url :string default(""), not null
# width :integer default(0), not null
# height :integer default(0), not null
# created_at :datetime not null
# updated_at :datetime not null
# embed_url :string default(""), not null
# id :bigint(8) not null, primary key
# url :string default(""), not null
# title :string default(""), not null
# description :string default(""), not null
# image_file_name :string
# image_content_type :string
# image_file_size :integer
# image_updated_at :datetime
# type :integer default("link"), not null
# html :text default(""), not null
# author_name :string default(""), not null
# author_url :string default(""), not null
# provider_name :string default(""), not null
# provider_url :string default(""), not null
# width :integer default(0), not null
# height :integer default(0), not null
# created_at :datetime not null
# updated_at :datetime not null
# embed_url :string default(""), not null
# image_storage_schema_version :integer
#

class PreviewCard < ApplicationRecord
@@ -47,6 +48,10 @@ class PreviewCard < ApplicationRecord

before_save :extract_dimensions, if: :link?

def local?
false
end

def missing_image?
width.present? && height.present? && image_file_name.blank?
end


+ 19
- 3
config/initializers/paperclip.rb View File

@@ -10,9 +10,25 @@ Paperclip.interpolates :filename do |attachment, style|
end
end

Paperclip.interpolates :path_prefix do |attachment, style|
if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
'cache' + File::SEPARATOR
else
''
end
end

Paperclip.interpolates :url_prefix do |attachment, style|
if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
'cache/'
else
''
end
end

Paperclip::Attachment.default_options.merge!(
use_timestamp: false,
path: ':class/:attachment/:id_partition/:style/:filename',
path: ':url_prefix:class/:attachment/:id_partition/:style/:filename',
storage: :fog
)

@@ -91,7 +107,7 @@ else
Paperclip::Attachment.default_options.merge!(
storage: :filesystem,
use_timestamp: true,
path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':class', ':attachment', ':id_partition', ':style', ':filename'),
url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':path_prefix:class', ':attachment', ':id_partition', ':style', ':filename'),
url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:url_prefix:class/:attachment/:id_partition/:style/:filename',
)
end

+ 9
- 0
db/migrate/20200417125749_add_storage_schema_version.rb View File

@@ -0,0 +1,9 @@
class AddStorageSchemaVersion < ActiveRecord::Migration[5.2]
def change
add_column :preview_cards, :image_storage_schema_version, :integer
add_column :accounts, :avatar_storage_schema_version, :integer
add_column :accounts, :header_storage_schema_version, :integer
add_column :media_attachments, :file_storage_schema_version, :integer
add_column :custom_emojis, :image_storage_schema_version, :integer
end
end

+ 6
- 1
db/schema.rb View File

@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema.define(version: 2020_04_07_202420) do
ActiveRecord::Schema.define(version: 2020_04_17_125749) do

# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@@ -172,6 +172,8 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.datetime "suspended_at"
t.integer "trust_level"
t.boolean "hide_collections"
t.integer "avatar_storage_schema_version"
t.integer "header_storage_schema_version"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
@@ -299,6 +301,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.string "image_remote_url"
t.boolean "visible_in_picker", default: true, null: false
t.bigint "category_id"
t.integer "image_storage_schema_version"
t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
end

@@ -464,6 +467,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.bigint "scheduled_status_id"
t.string "blurhash"
t.integer "processing"
t.integer "file_storage_schema_version"
t.index ["account_id"], name: "index_media_attachments_on_account_id"
t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
@@ -604,6 +608,7 @@ ActiveRecord::Schema.define(version: 2020_04_07_202420) do
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "embed_url", default: "", null: false
t.integer "image_storage_schema_version"
t.index ["url"], name: "index_preview_cards_on_url", unique: true
end



+ 4
- 0
lib/cli.rb View File

@@ -11,6 +11,7 @@ require_relative 'mastodon/statuses_cli'
require_relative 'mastodon/domains_cli'
require_relative 'mastodon/preview_cards_cli'
require_relative 'mastodon/cache_cli'
require_relative 'mastodon/upgrade_cli'
require_relative 'mastodon/version'

module Mastodon
@@ -49,6 +50,9 @@ module Mastodon
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', Mastodon::CacheCLI

desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
subcommand 'upgrade', Mastodon::UpgradeCLI

option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC


+ 4
- 0
lib/mastodon/cli_helper.rb View File

@@ -10,6 +10,10 @@ Paperclip.options[:log] = false

module Mastodon
module CLIHelper
def dry_run?
options[:dry_run]
end

def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end


+ 17
- 7
lib/mastodon/media_cli.rb View File

@@ -85,7 +85,9 @@ module Mastodon
record_map = preload_records_from_mixed_objects(objects)

objects.each do |object|
path_segments = object.key.split('/')
path_segments = object.key.split('/')
path_segments.delete('cache')

model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i
@@ -120,8 +122,11 @@ module Mastodon
Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path)

key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')

path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')

model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize
@@ -229,10 +234,13 @@ module Mastodon

desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url)
path = Addressable::URI.parse(url).path
path = Addressable::URI.parse(url).path

path_segments = path.split('/')[2..-1]
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
path_segments.delete('cache')

model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i

unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red)
@@ -276,7 +284,9 @@ module Mastodon
preload_map = Hash.new { |hash, key| hash[key] = [] }

objects.map do |object|
segments = object.key.split('/')
segments = object.key.split('/')
segments.delete('cache')

model_name = segments.first.classify
record_id = segments[2..-2].join.to_i



+ 148
- 0
lib/mastodon/upgrade_cli.rb View File

@@ -0,0 +1,148 @@
# frozen_string_literal: true

require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'

module Mastodon
class UpgradeCLI < Thor
include CLIHelper

def self.exit_on_failure?
true
end

CURRENT_STORAGE_SCHEMA_VERSION = 1

option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false, aliases: [:v]
desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
long_desc <<~LONG_DESC
Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.

Will most likely take a long time.
LONG_DESC
def storage_schema
progress = create_progress_bar(nil)
dry_run = dry_run? ? ' (DRY RUN)' : ''
records = 0

klasses = [
Account,
CustomEmoji,
MediaAttachment,
PreviewCard,
]

klasses.each do |klass|
attachment_names = klass.attachment_definitions.keys

klass.find_each do |record|
attachment_names.each do |attachment_name|
attachment = record.public_send(attachment_name)

next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION

attachment.styles.each_key do |style|
case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end

progress.increment
end

attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
end

if record.changed?
record.save unless dry_run?
records += 1
end
end
end

progress.total = progress.progress
progress.finish

say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
end

private

def upgrade_storage_s3(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
object = attachment.s3_object(style)

attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)

upgraded_path = attachment.path(style)

if upgraded_path != object.key && object.exists?
progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose]

begin
object.move_to(upgraded_path) unless dry_run?
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
end

# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end

def upgrade_storage_fog(_progress, _attachment, _style)
say('The fog storage driver is not supported for this operation at this time', :red)
exit(1)
end

def upgrade_storage_filesystem(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
previous_path = attachment.path(style)

attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)

upgraded_path = attachment.path(style)

if upgraded_path != previous_path && File.exist?(previous_path)
progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]

begin
unless dry_run?
FileUtils.mkdir_p(File.dirname(upgraded_path))
FileUtils.mv(previous_path, upgraded_path)

begin
FileUtils.rmdir(previous_path, parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
rescue => e
progress.log(pastel.red("Error processing #{previous_path}: #{e}"))

unless dry_run?
begin
FileUtils.rmdir(upgraded_path, parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
end
end

# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end
end
end

+ 9
- 0
lib/paperclip/attachment_extensions.rb View File

@@ -14,6 +14,15 @@ module Paperclip
end
end

def storage_schema_version
instance_read(:storage_schema_version) || 0
end

def assign_attributes
super
instance_write(:storage_schema_version, 1)
end

def variant?(other_filename)
return true if original_filename == other_filename
return false if original_filename.nil?


Loading…
Cancel
Save