From 1ca4e51eb38de6de81cedf3ddcdaa626f1d1c569 Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 9 Jul 2018 02:22:09 +0200 Subject: [PATCH] Add option to not consider word boundaries when processing keyword filtering (#7975) * Add option to not consider word boundaries when filtering phrases * Add a few tests for keyword/phrase filtering --- app/controllers/api/v1/filters_controller.rb | 2 +- app/javascript/mastodon/selectors/index.js | 5 +++- app/lib/feed_manager.rb | 11 +++++++- app/models/custom_filter.rb | 1 + app/serializers/rest/filter_serializer.rb | 2 +- app/views/filters/_fields.html.haml | 3 ++ ...7154237_add_whole_word_to_custom_filter.rb | 17 +++++++++++ db/schema.rb | 3 +- spec/lib/feed_manager_spec.rb | 28 +++++++++++++++---- 9 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20180707154237_add_whole_word_to_custom_filter.rb diff --git a/app/controllers/api/v1/filters_controller.rb b/app/controllers/api/v1/filters_controller.rb index a98080d1d1..e5ebaff4d3 100644 --- a/app/controllers/api/v1/filters_controller.rb +++ b/app/controllers/api/v1/filters_controller.rb @@ -43,6 +43,6 @@ class Api::V1::FiltersController < Api::BaseController end def resource_params - params.permit(:phrase, :expires_in, :irreversible, context: []) + params.permit(:phrase, :expires_in, :irreversible, :whole_word, context: []) end end diff --git a/app/javascript/mastodon/selectors/index.js b/app/javascript/mastodon/selectors/index.js index 7aa7569a05..d0212c379d 100644 --- a/app/javascript/mastodon/selectors/index.js +++ b/app/javascript/mastodon/selectors/index.js @@ -45,7 +45,10 @@ export const regexFromFilters = filters => { return null; } - return new RegExp(filters.map(filter => escapeRegExp(filter.get('phrase'))).map(expr => `\\b${expr}\\b`).join('|'), 'i'); + return new RegExp(filters.map(filter => { + let expr = escapeRegExp(filter.get('phrase')); + return filter.get('whole_word') ? `\\b${expr}\\b` : expr; + }).join('|'), 'i'); }; export const makeGetStatus = () => { diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 55c72d0eac..c247ab21db 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -200,7 +200,16 @@ class FeedManager active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a active_filters.select! { |filter| filter.context.include?(context.to_s) && !filter.expired? } - active_filters.map! { |filter| Regexp.new("\\b#{Regexp.escape(filter.phrase)}\\b", true) } + active_filters.map! do |filter| + if filter.whole_word + sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : '' + eb = filter.phrase =~ /[[:word:]]\Z/ ? '\b' : '' + + /(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/ + else + /#{Regexp.escape(filter.phrase)}/i + end + end return false if active_filters.empty? diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb index 2c1a54375f..342207a55d 100644 --- a/app/models/custom_filter.rb +++ b/app/models/custom_filter.rb @@ -8,6 +8,7 @@ # expires_at :datetime # phrase :text default(""), not null # context :string default([]), not null, is an Array +# whole_word :boolean default(TRUE), not null # irreversible :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/serializers/rest/filter_serializer.rb b/app/serializers/rest/filter_serializer.rb index 51340aa79b..3134be371b 100644 --- a/app/serializers/rest/filter_serializer.rb +++ b/app/serializers/rest/filter_serializer.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true class REST::FilterSerializer < ActiveModel::Serializer - attributes :id, :phrase, :context, :expires_at, + attributes :id, :phrase, :context, :whole_word, :expires_at, :irreversible end diff --git a/app/views/filters/_fields.html.haml b/app/views/filters/_fields.html.haml index af5d648b80..a5a3f03375 100644 --- a/app/views/filters/_fields.html.haml +++ b/app/views/filters/_fields.html.haml @@ -7,5 +7,8 @@ .fields-group = f.input :irreversible, wrapper: :with_label +.fields-group + = f.input :whole_word, wrapper: :with_label + .fields-group = f.input :expires_in, wrapper: :with_label, collection: [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].map(&:to_i), label_method: lambda { |i| I18n.t("invites.expires_in.#{i}") }, prompt: I18n.t('invites.expires_in_prompt') diff --git a/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb new file mode 100644 index 0000000000..63ecb87415 --- /dev/null +++ b/db/migrate/20180707154237_add_whole_word_to_custom_filter.rb @@ -0,0 +1,17 @@ +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class AddWholeWordToCustomFilter < ActiveRecord::Migration[5.2] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def change + safety_assured do + add_column_with_default :custom_filters, :whole_word, :boolean, default: true, allow_null: false + end + end + + def down + remove_column :custom_filters, :whole_word + end +end diff --git a/db/schema.rb b/db/schema.rb index 661fc81793..02032c5482 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2018_06_28_181026) do +ActiveRecord::Schema.define(version: 2018_07_07_154237) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -149,6 +149,7 @@ ActiveRecord::Schema.define(version: 2018_06_28_181026) do t.text "phrase", default: "", null: false t.string "context", default: [], null: false, array: true t.boolean "irreversible", default: false, null: false + t.boolean "whole_word", default: true, null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["account_id"], name: "index_custom_filters_on_account_id" diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb index d1b8476755..7535e144d7 100644 --- a/spec/lib/feed_manager_spec.rb +++ b/spec/lib/feed_manager_spec.rb @@ -127,12 +127,28 @@ RSpec.describe FeedManager do expect(FeedManager.instance.filter?(:home, reblog, alice.id)).to be true end - it 'returns true if status contains irreversibly muted phrase' do - alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) - alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) - alice.follow!(jeff) - status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) - expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + context 'for irreversibly muted phrases' do + it 'considers word boundaries when matching' do + alice.custom_filters.create!(phrase: 'bob', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'bobcats', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be_falsy + end + + it 'returns true if phrase is contained' do + alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true) + alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'i sure like POP TARts', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end + + it 'matches substrings if whole_word is false' do + alice.custom_filters.create!(phrase: 'take', context: %w(home), whole_word: false, irreversible: true) + alice.follow!(jeff) + status = Fabricate(:status, text: 'shiitake', account: jeff) + expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true + end end end