Spam protection tools

app/controllers/categories_controller.rb

@@ -138,12 +138,12 @@ def rss_feed
 
   def post_types
     @post_types = @category.top_level_post_types
-    if @post_types.one?
+    if !user_signed_in?
+      redirect_to_sign_in
+    elsif @post_types.one?
       redirect_to new_category_post_path(post_type: @post_types.first, category: @category)
     elsif @post_types.empty? && current_user&.admin?
       redirect_to edit_category_post_types_path(@category, no_return: '1')
-    elsif !user_signed_in?
-      redirect_to_sign_in
     end
   end
 

app/controllers/posts_controller.rb

@@ -30,7 +30,7 @@ def new
     end
 
     if @post_type.system?
-      check_permissions
+      check_permissions!
       # return # uncomment if you add more code after this
     end
   end
@@ -60,31 +60,8 @@ def create
     @post = Post.new(post_params.merge(user: current_user, body: helpers.rendered_post(:post, :body_markdown),
                                        category: @category, post_type: @post_type, parent: @parent))
 
-    if @post.title? && (@post.title.include? '$$')
-      flash[:danger] = I18n.t 'posts.no_block_mathjax_title'
-      render :new, status: :bad_request
-      return
-    end
-
-    if @post_type.has_parent? && @parent.nil?
-      flash[:danger] = helpers.i18ns('posts.type_requires_parent', type: @post_type.name)
-      redirect_back fallback_location: root_path
-      return
-    end
-
-    if @post_type.has_category? && @category.nil? && @parent.nil?
-      flash[:danger] = helpers.i18ns('posts.type_requires_category', type: @post_type.name)
-      redirect_back fallback_location: root_path
-      return
-    end
-
-    if @category.present? && !current_user.can_post_in?(@category)
-      @post.errors.add(:base, helpers.i18ns('posts.category_low_trust_level', name: @category.name))
-      render :new, status: :forbidden
-      return
-    end
-
-    if @post_type.system? && !check_permissions
+    # Not a validation: check_permissions! calls verify_* methods in turn, which render/redirect.
+    if @post_type.system? && !check_permissions!
       return
     end
 
@@ -738,7 +715,7 @@ def set_scoped_post
     @post = Post.find(params[:id])
   end
 
-  def check_permissions
+  def check_permissions!
     if @post.post_type_id == HelpDoc.post_type_id
       verify_moderator
     elsif @post.post_type_id == PolicyDoc.post_type_id

app/controllers/users/registrations_controller.rb

@@ -8,7 +8,11 @@ class Users::RegistrationsController < Devise::RegistrationsController
   def create
     super do |user|
       unless user.errors.any?
-        rate_limit = AppConfig.server_settings['registration_rate_limit']
+        rate_limit = if Rails.env.development?
+                       0
+                     else
+                       AppConfig.server_settings['registration_rate_limit']
+                     end
         ip_list = [user.current_sign_in_ip, request.remote_ip].compact
         previous_ip_users = User.where(current_sign_in_ip: ip_list).or(User.where(last_sign_in_ip: ip_list))
                                 .where(created_at: rate_limit.seconds.ago..DateTime.now)

app/controllers/users_controller.rb

@@ -342,7 +342,9 @@ def soft_delete
         return
       end
 
-      @user.soft_delete(current_user)
+      max_rep = @user.community_users.maximum(:reputation)
+      check_threshold = AppConfig.spam_protection['deletion_block_max_rep']
+      DeleteUserJob.perform_later(@user, current_user, perform_spam_check: max_rep < check_threshold)
     else
       render json: { status: 'failed', message: 'Unrecognised deletion type.' }, status: 400
       return

app/jobs/check_cidr_job.rb

@@ -0,0 +1,47 @@
+class CheckCIDRJob < ApplicationJob
+  queue_as :default
+
+  def perform(post)
+    @post = post
+    relevant_ips = [post.user.current_sign_in_ip, post.user.last_sign_in_ip]
+    prefixes = BlockedItem.where(item_type: 'ip_prefix')
+                          .where(Arel.sql('expires >= CURRENT_TIMESTAMP'))
+                          .where("? LIKE CONCAT(`value`, '%') OR ? LIKE CONCAT(`value`, '%')", *relevant_ips)
+
+    if prefixes.any?
+      create_flag prefixes[0]
+      return # because prefixes are more performant than CIDR checks, so if we can match there then that'll do
+    end
+
+    cidrs = BlockedItem.where(item_type: 'ip_cidr')
+                       .where(Arel.sql('expires >= CURRENT_TIMESTAMP'))
+    cidrs.each do |cidr|
+      network = IPAddress.parse(cidr.value)
+      relevant_ips.each do |ip|
+        ip = IPAddress.parse(ip)
+        # rubocop:disable Style/Next
+        if network.include?(ip)
+          create_flag cidr
+          # rubocop:disable Lint/NonLocalExitFromIterator
+          return
+          # rubocop:enable Lint/NonLocalExitFromIterator
+        end
+        # rubocop:enable Style/Next
+      end
+    end
+  end
+
+  def create_flag(match)
+    reason = 'Automatically escalated spam flag - please leave this for the community team to handle.'
+    spam_flag_type = PostFlagType.unscoped.where(community: @post.community, name: "it's spam").first
+    flag = @post.flags.create(user: helpers.system_user, community: @post.community, post_flag_type: spam_flag_type,
+                              reason: reason, escalated: true, escalated_at: DateTime.now,
+                              escalated_by: helpers.system_user,
+                              escalation_comment: "Suspicious IP address: user IP matches #{match.value}")
+    FlagMailer.with(flag: flag).flag_escalated.deliver_now
+  end
+
+  def helpers
+    ApplicationController.helpers
+  end
+end

app/jobs/delete_user_job.rb

@@ -0,0 +1,28 @@
+class DeleteUserJob < ApplicationJob
+  queue_as :default
+
+  ##
+  # Perform a network-wide soft-deletion of a user account. Also optionally checks for helpful spam flags against
+  # the target user and applies a spam block if found. The caller is responsible for managing thresholds for this spam
+  # check.
+  # @param user [User] user to soft-delete
+  # @param attribute_to [User] the user performing the deletion
+  # @param perform_spam_check [Boolean] whether to perform the spam check
+  def perform(user, attribute_to, perform_spam_check: true)
+    if perform_spam_check
+      # Can't use model helper methods very easily here, because we want network-wide flags and that doesn't play
+      # nicely with default scopes.
+      flag_query = Post.unscoped
+                       .joins(Arel.sql("INNER JOIN flags ON flags.post_type = 'Post' AND flags.post_id = posts.id"))
+                       .joins(Arel.sql('INNER JOIN post_flag_types ON flags.post_flag_type_id = post_flag_types.id'))
+                       .where(flags: { status: 'helpful' },
+                              post_flag_types: { name: "it's spam" },
+                              posts: { user_id: user.id })
+      if flag_query.any?
+        user.block('automatic block from spam check during deletion')
+      end
+    end
+
+    user.soft_delete(attribute_to)
+  end
+end

app/models/application_record.rb

@@ -26,6 +26,10 @@ def attributes_print(join: ', ')
     end.join(join)
   end
 
+  def helpers
+    ApplicationController.helpers
+  end
+
   def self.sanitize_for_search(term, **cols)
     cols = cols.map do |k, v|
       if v.is_a?(Array)

app/models/concerns/post_creation_validations.rb

@@ -0,0 +1,81 @@
+module PostCreationValidations
+  extend ActiveSupport::Concern
+
+  # rubocop:disable Metrics/BlockLength
+  included do
+    validate :no_mathjax_in_title, on: :create
+    validate :post_type_requires_parent, on: :create
+    validate :post_type_has_category, on: :create
+    validate :can_post_in_category, on: :create
+    validate :identical_post_spam, on: :create
+    validate :no_active_spam_flags, on: :create
+
+    after_create :escalate_suspicious_cidr
+
+    private
+
+    def no_mathjax_in_title
+      if title? && title.include?('$$')
+        errors.add(:base, I18n.t('posts.no_block_mathjax_title'))
+      end
+    end
+
+    def post_type_requires_parent
+      if post_type.has_parent? && parent.nil?
+        errors.add(:base, helpers.i18ns('posts.type_requires_parent', type: post_type.name))
+      end
+    end
+
+    def post_type_has_category
+      if post_type.has_category? && category.nil? && parent.nil?
+        errors.add(:base, helpers.i18ns('posts.type_requires_category', type: post_type.name))
+      end
+    end
+
+    def can_post_in_category
+      if category.present? && !user.can_post_in?(category)
+        errors.add(:base, helpers.i18ns('posts.category_low_trust_level', name: category.name))
+      end
+    end
+
+    def identical_post_spam
+      threshold = AppConfig.spam_protection['identical_post_spam_threshold']
+      prev_non_deleted_count = Post.unscoped.where(user: user, deleted: false).count
+      unless prev_non_deleted_count >= threshold
+        identical_posts = Post.unscoped.where(user: user, body_markdown: body_markdown).where.not(id: id)
+        if identical_posts.any?
+          errors.add(:base, I18n.t('posts.spam_blocked'))
+        end
+      end
+    end
+
+    def no_active_spam_flags
+      posts_threshold = AppConfig.spam_protection['spam_flag_posts_threshold']
+      time_threshold = AppConfig.spam_protection['spam_flag_time_threshold']
+      prev_non_deleted_count = Post.unscoped.where(user: user, deleted: false).count
+      unless prev_non_deleted_count >= posts_threshold
+        active = Post.unscoped
+                     .joins(Arel.sql("INNER JOIN flags ON flags.post_type = 'Post' AND flags.post_id = posts.id"))
+                     .joins(Arel.sql('INNER JOIN post_flag_types ON flags.post_flag_type_id = post_flag_types.id'))
+                     .where(flags: { status: nil },
+                            post_flag_types: { name: "it's spam" },
+                            posts: { user_id: user.id })
+        helpful = Post.unscoped
+                      .joins(Arel.sql("INNER JOIN flags ON flags.post_type = 'Post' AND flags.post_id = posts.id"))
+                      .joins(Arel.sql('INNER JOIN post_flag_types ON flags.post_flag_type_id = post_flag_types.id'))
+                      .where(flags: { status: 'helpful' },
+                             post_flag_types: { name: "it's spam" },
+                             posts: { user_id: user.id })
+                      .where('flags.created_at <= ?', time_threshold.days.ago)
+        if active.any? || helpful.any?
+          errors.add(:base, I18n.t('posts.spam_blocked'))
+        end
+      end
+    end
+
+    def escalate_suspicious_cidr
+      CheckCIDRJob.perform_later(self)
+    end
+  end
+  # rubocop:enable Metrics/BlockLength
+end

app/models/concerns/post_validations.rb

@@ -71,6 +71,8 @@ def maximum_title_length
   end
 
   def tags_in_tag_set
+    return if category.nil?
+
     tag_set = category.tag_set
     unless tags.all? { |t| t.tag_set_id == tag_set.id }
       errors.add(:base, "Not all of this question's tags are in the correct tag set.")

app/models/post.rb

@@ -3,6 +3,7 @@ class Post < ApplicationRecord
   include Lockable
   include PostNormalizations
   include PostValidations
+  include PostCreationValidations
   include SoftDeletable
   include Timestamped
   include UserSortable
@@ -293,6 +294,8 @@ def question?
   ##
   # Before-validation callback. Update the tags association from the tags_cache.
   def update_tag_associations
+    return if category.nil?
+
     tags_cache.each do |tag_name|
       tag, name_used = Tag.find_or_create_synonymized name: tag_name, tag_set: category.tag_set
       unless tags.include? tag

app/models/user.rb

@@ -453,7 +453,7 @@ def send_welcome_tour_message
                         'how this site works.', '/tour')
   end
 
-  def block(reason, length: 180.days, automatic: true)
+  def block(reason, length: 10.years, automatic: true)
     user_email = email
     user_ip = [last_sign_in_ip]
 

config/config/server_settings.yml

@@ -1,4 +1,11 @@
-registration_rate_limit: 300
+# Minimum number of seconds between registration attempts from the same IP.
+registration_rate_limit: 3600
+
+# Number of minutes that sudo mode will last for before asking for password re-entry.
 user_sudo_duration: 30
+
+# Base domain for the network: if you have communities for a.example.com and b.example.com, this should be example.com.
 network_base_domain: codidact.com
+
+# File path on your server to which database backups will be saved while awaiting upload to S3.
 db_backups_path: /var/sql-backups

config/config/spam_protection.yml

@@ -0,0 +1,14 @@
+# When a user is deleted, helpful spam flags against the user's post will additionally cause the user to be fail-banned.
+# Users with equal to or more than this reputation value in any community are exempt from this check.
+deletion_block_max_rep: 100
+
+# Users are blocked from posting something identical to any of their previous posts, unless they have equal to or
+# greater than this number of non-deleted posts anywhere on the network.
+identical_post_spam_threshold: 10
+
+# Users are blocked from posting if there is an active (i.e. not reviewed) spam flag against any of their posts. Users
+# with equal to or greater than this number of non-deleted posts in any community are exempt from this check.
+spam_flag_posts_threshold: 10
+
+# Helpful spam flags will also block posting as above for up to this number of days after the flag was cast.
+spam_flag_time_threshold: 7

config/initializers/inflections.rb

@@ -17,4 +17,5 @@
 
 ActiveSupport::Inflector.inflections(:en) do |inflect|
   inflect.acronym 'SE'
+  inflect.acronym 'CIDR'
 end

config/locales/strings/en.posts.yml

@@ -9,6 +9,8 @@ en:
       You can't create a :type without a category.
     type_requires_parent: >
       You can't create a :type without a parent post.
+    spam_blocked: >
+      Couldn't create your post. Try again later.
     not_public_editable: >
       This type of post can only be edited by its author and the site moderators.
     cant_close_post: >

test/controllers/concerns/users/user_test_helpers.rb

@@ -0,0 +1,40 @@
+module UserTestHelpers
+  extend ActiveSupport::Concern
+
+  included do
+    private
+
+    def create_other_user
+      other_community = Community.create(host: 'other.qpixel.com', name: 'Other')
+      RequestContext.redis.hset('network/community_registrations', 'other@example.com', other_community.id)
+      other_user = User.create!(email: 'other@example.com', password: 'abcdefghijklmnopqrstuvwxyz', username: 'other_user')
+      other_user.community_users.create!(community: other_community)
+      other_user
+    end
+
+    # @param type [String] deletion type (user or profile)
+    # @param user [User] user to soft delete
+    def try_soft_delete_user(type, user)
+      perform_enqueued_jobs do
+        delete :soft_delete, params: { id: user.id,
+                                       type: type,
+                                       format: :json }
+      end
+    end
+
+    def try_save_preference(name, value, community: nil)
+      post :set_preference, params: {
+        community: community,
+        name: name,
+        value: value,
+        format: :json
+      }
+    end
+
+    # @param user [User] user to undelete
+    def try_undelete_user(user)
+      post :undelete, params: { id: user.id,
+                                format: :json }
+    end
+  end
+end