# frozen_string_literal: true require 'csv' # A non-ActiveRecord helper class for CSV uploads. # Handles saving contents to database. class Form::Import include ActiveModel::Model MODES = %i(merge overwrite).freeze FILE_SIZE_LIMIT = 20.megabytes ROWS_PROCESSING_LIMIT = 20_000 EXPECTED_HEADERS_BY_TYPE = { following: ['Account address', 'Show boosts', 'Notify on new posts', 'Languages'], blocking: ['Account address'], muting: ['Account address', 'Hide notifications'], domain_blocking: ['#domain'], bookmarks: ['#uri'], lists: ['List name', 'Account address'], }.freeze KNOWN_FIRST_HEADERS = EXPECTED_HEADERS_BY_TYPE.values.map(&:first).uniq.freeze ATTRIBUTE_BY_HEADER = { 'Account address' => 'acct', 'Show boosts' => 'show_reblogs', 'Notify on new posts' => 'notify', 'Languages' => 'languages', 'Hide notifications' => 'hide_notifications', '#domain' => 'domain', '#uri' => 'uri', 'List name' => 'list_name', }.freeze class EmptyFileError < StandardError; end attr_accessor :current_account, :data, :type, :overwrite, :bulk_import validates :type, presence: true validates :data, presence: true validate :validate_data def guessed_type return :muting if csv_headers_match?('Hide notifications') return :following if csv_headers_match?('Show boosts') || csv_headers_match?('Notify on new posts') || csv_headers_match?('Languages') return :following if file_name_matches?('follows') || file_name_matches?('following_accounts') return :blocking if file_name_matches?('blocks') || file_name_matches?('blocked_accounts') return :muting if file_name_matches?('mutes') || file_name_matches?('muted_accounts') return :domain_blocking if file_name_matches?('domain_blocks') || file_name_matches?('blocked_domains') return :bookmarks if file_name_matches?('bookmarks') return :lists if file_name_matches?('lists') end # Whether the uploaded CSV file seems to correspond to a different import type than the one selected def likely_mismatched? guessed_type.present? && guessed_type != type.to_sym end def save return false unless valid? ApplicationRecord.transaction do now = Time.now.utc @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?) nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length # rubocop:disable Rails/SkipsModelValidations @bulk_import.update(total_items: nb_items) end end def mode overwrite ? :overwrite : :merge end def mode=(str) self.overwrite = str.to_sym == :overwrite end private def file_name_matches?(string) data.original_filename&.start_with?(string) end def csv_headers_match?(string) csv_data.headers.include?(string) end def default_csv_headers case type.to_sym when :following, :blocking, :muting ['Account address'] when :domain_blocking ['#domain'] when :bookmarks ['#uri'] when :lists ['List name', 'Account address'] end end def csv_data return @csv_data if defined?(@csv_data) csv_converter = lambda do |field, field_info| case field_info.header when 'Show boosts', 'Notify on new posts', 'Hide notifications' ActiveModel::Type::Boolean.new.cast(field) when 'Languages' field&.split(',')&.map(&:strip)&.presence when 'Account address' field.strip.delete_prefix('@') when '#domain', '#uri', 'List name' field.strip else field end end @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: true, converters: csv_converter) @csv_data.take(1) # Ensure the headers are read raise EmptyFileError if @csv_data.headers == true @csv_data = CSV.open(data.path, encoding: 'UTF-8', skip_blanks: true, headers: default_csv_headers, converters: csv_converter) unless KNOWN_FIRST_HEADERS.include?(@csv_data.headers&.first) @csv_data end def csv_row_count return @csv_row_count if defined?(@csv_row_count) csv_data.rewind @csv_row_count = csv_data.take(ROWS_PROCESSING_LIMIT + 2).count end def parsed_rows csv_data.rewind expected_headers = EXPECTED_HEADERS_BY_TYPE[type.to_sym] csv_data.take(ROWS_PROCESSING_LIMIT + 1).map do |row| row.to_h.slice(*expected_headers).transform_keys { |key| ATTRIBUTE_BY_HEADER[key] } end end def validate_data return if data.nil? return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) } errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT if type.to_sym == :following follows_limit = FollowLimitValidator.limit_for_account(current_account) errors.add(:data, I18n.t('users.follow_limit_reached', limit: follows_limit)) if follows_count_after_import > follows_limit end rescue CSV::MalformedCSVError => e errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) rescue EmptyFileError errors.add(:data, I18n.t('imports.errors.empty')) end def follows_count_after_import return csv_row_count if overwrite accts = parsed_rows.pluck('acct').compact.map(&:downcase).uniq.map do |acct| username, domain = acct.split('@') if domain.nil? || TagManager.instance.local_domain?(domain) username.downcase else "#{username.downcase}@#{TagManager.instance.normalize_domain(domain)}" end end current_follows = current_account.following.select(:username, :domain).map { |account| account.acct.downcase } (current_follows + accts).uniq.size end end