Add command to remove avatar and header images of inactive remote accounts from the local database (#22149)

* Add tootctl subcommand media remove-profile-media

* Trigger workflows

* Correcting external linting

* External linting error

* External linting fix

* Merging with remove command

* Linting

* Correct long option names

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Correct long option names

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Correct long option names

Co-authored-by: Claire <claire.github-309c@sitedethib.com>

* Remove saving a list of purged accounts

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
pull/22150/merge
Evan 2022-12-14 19:50:07 +01:00 committed by GitHub
parent a9bd5f65bb
commit 78ef635980
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 61 additions and 17 deletions

View File

@ -14,35 +14,78 @@ module Mastodon
end end
option :days, type: :numeric, default: 7, aliases: [:d] option :days, type: :numeric, default: 7, aliases: [:d]
option :prune_profiles, type: :boolean, default: false
option :remove_headers, type: :boolean, default: false
option :include_follows, type: :boolean, default: false
option :concurrency, type: :numeric, default: 5, aliases: [:c] option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, default: false, aliases: [:v]
option :dry_run, type: :boolean, default: false option :dry_run, type: :boolean, default: false
desc 'remove', 'Remove remote media files' desc 'remove', 'Remove remote media files, headers or avatars'
long_desc <<-DESC long_desc <<-DESC
Removes locally cached copies of media attachments from other servers. Removes locally cached copies of media attachments (and optionally profile
headers and avatars) from other servers. By default, only media attachements
are removed.
The --days option specifies how old media attachments have to be before The --days option specifies how old media attachments have to be before
they are removed. It defaults to 7 days. they are removed. In case of avatars and headers, it specifies how old
the last webfinger request and update to the user has to be before they
are pruned. It defaults to 7 days.
If --prune-profiles is specified, only avatars and headers are removed.
If --remove-headers is specified, only headers are removed.
If --include-follows is specified along with --prune-profiles or
--remove-headers, all non-local profiles will be pruned irrespective of
follow status. By default, only accounts that are not followed by or
following anyone locally are pruned.
DESC DESC
# rubocop:disable Metrics/PerceivedComplexity
def remove def remove
time_ago = options[:days].days.ago if options[:prune_profiles] && options[:remove_headers]
dry_run = options[:dry_run] ? '(DRY RUN)' : '' say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
exit(1)
end
if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
exit(1)
end
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| if options[:prune_profiles] || options[:remove_headers]
next if media_attachment.file.blank? processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists?
next if account.avatar.blank? && account.header.blank?
next if options[:remove_headers] && account.header.blank?
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0) size = (account.header_file_size || 0)
size += (account.avatar_file_size || 0) if options[:prune_profiles]
unless options[:dry_run] unless options[:dry_run]
media_attachment.file.destroy account.header.destroy
media_attachment.thumbnail.destroy account.avatar.destroy if options[:prune_profiles]
media_attachment.save account.save!
end
size
end end
size say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true)
end end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) unless options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment|
next if media_attachment.file.blank?
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
unless options[:dry_run]
media_attachment.file.destroy
media_attachment.thumbnail.destroy
media_attachment.save
end
size
end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
end
end end
option :start_after option :start_after
@ -183,6 +226,7 @@ module Mastodon
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true) say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
end end
# rubocop:enable Metrics/PerceivedComplexity
option :account, type: :string option :account, type: :string
option :domain, type: :string option :domain, type: :string
@ -269,7 +313,7 @@ module Mastodon
def lookup(url) def lookup(url)
path = Addressable::URI.parse(url).path path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..-1] path_segments = path.split('/')[2..]
path_segments.delete('cache') path_segments.delete('cache')
unless [7, 10].include?(path_segments.size) unless [7, 10].include?(path_segments.size)