Add crashes_after_fix rule to flag fixed crash bugs still crashing on Nightly

bugbot/rules/crashes_after_fix.py

@@ -0,0 +1,186 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from datetime import timedelta
+
+from libmozdata import utils as lmdutils
+from libmozdata.socorro import SuperSearch
+
+from bugbot import utils
+from bugbot.bzcleaner import BzCleaner
+
+# Marker phrase placed in the needinfo comment so subsequent runs can detect
+# that this rule has already actioned a bug and skip it via the longdesc
+# substring filter in get_bz_params(). Keep it stable across template edits.
+COMMENT_MARKER = "crashes are still being reported against this signature"
+
+
+class CrashesAfterFix(BzCleaner):
+    """Crash bugs whose signature is still crashing on Nightly after the fix
+    landed. Need-infos the assignee to ask whether the fix was incomplete or
+    whether a follow-up is needed."""
+
+    def __init__(self):
+        super().__init__()
+        self.max_days_since_fix = utils.get_config(
+            self.name(), "max_days_since_fix", 10
+        )
+        self.min_crash_count = utils.get_config(self.name(), "min_crash_count", 5)
+        self.extra_ni = {}
+        # bug_id (str) -> per-bug context used to query Socorro and fill the
+        # needinfo template (see bughandler()).
+        self.bug_data = {}
+
+    def description(self):
+        return (
+            "Bugs whose crash signatures keep crashing on Nightly within "
+            "{} days after the fix landed"
+        ).format(self.max_days_since_fix)
+
+    def has_assignee(self):
+        return True
+
+    def get_extra_for_template(self):
+        return {
+            "max_days": self.max_days_since_fix,
+            "min_crashes": self.min_crash_count,
+        }
+
+    def get_extra_for_needinfo_template(self):
+        return self.extra_ni
+
+    def get_bz_params(self, date):
+        today = lmdutils.get_date_ymd(date)
+        oldest_fix = lmdutils.get_date_str(
+            today - timedelta(days=self.max_days_since_fix)
+        )
+
+        fields = [
+            "id",
+            "summary",
+            "assigned_to",
+            "assigned_to_detail",
+            "cf_crash_signature",
+            "cf_last_resolved",
+            "cf_status_firefox_nightly",
+        ]
+
+        params = {
+            "include_fields": fields,
+            "resolution": "FIXED",
+            "bug_status": ["RESOLVED", "VERIFIED"],
+            "f1": "cf_crash_signature",
+            "o1": "isnotempty",
+            "f2": "cf_status_firefox_nightly",
+            "o2": "equals",
+            "v2": "fixed",
+            "f3": "cf_last_resolved",
+            "o3": "greaterthan",
+            "v3": oldest_fix,
+            "f4": "flagtypes.name",
+            "o4": "notsubstring",
+            "v4": "needinfo?",
+            "n5": 1,
+            "f5": "longdesc",
+            "o5": "casesubstring",
+            "v5": COMMENT_MARKER,
+        }
+
+        return params
+
+    def bughandler(self, bug, data):
+        if not bug.get("cf_crash_signature"):
+            return
+
+        sigs = sorted(utils.get_signatures(bug["cf_crash_signature"]))
+        if not sigs:
+            return
+
+        assignee = bug.get("assigned_to") or ""
+        if utils.is_no_assignee(assignee):
+            return
+
+        nickname = ""
+        if bug.get("assigned_to_detail"):
+            nickname = bug["assigned_to_detail"].get("nick", "")
+
+        fix_date = bug.get("cf_last_resolved")
+        if not fix_date:
+            return
+
+        bug_id = str(bug["id"])
+        self.bug_data[bug_id] = {
+            "summary": self.get_summary(bug),
+            "signatures": sigs,
+            "fix_date": fix_date,
+            "assignee_email": assignee,
+            "assignee_nickname": nickname,
+        }
+
+    def _query_socorro(self, info):
+        """Faceted SuperSearch over Nightly crashes on builds shipped after the
+        fix landed. Returns (total_count, per_signature_counts, since_str).
+
+        Filters by build_id rather than crash date so that crashes from Nightly
+        users still running pre-fix builds aren't counted -- those crashes
+        don't mean the fix failed."""
+        fix_dt = lmdutils.get_date_ymd(info["fix_date"])
+        # Nightly build IDs are timestamps in YYYYMMDDHHMMSS format. The first
+        # build that can include the fix is the one created the day after the
+        # bug was resolved, so set the cutoff to midnight of that day.
+        since_day = fix_dt + timedelta(days=1)
+        build_id_min = since_day.strftime("%Y%m%d000000")
+        since = since_day.strftime("%Y-%m-%d")
+
+        counts = {}
+
+        def handler(json, data):
+            if json.get("errors"):
+                return
+            for facet in json.get("facets", {}).get("signature", []):
+                data[facet["term"]] = int(facet["count"])
+
+        params = {
+            "product": "Firefox",
+            "release_channel": "nightly",
+            "build_id": ">=" + build_id_min,
+            "signature": ["=" + s for s in info["signatures"]],
+            "_results_number": 0,
+            "_facets": "signature",
+            "_facets_size": max(len(info["signatures"]), 1),
+        }
+
+        SuperSearch(params=params, handler=handler, handlerdata=counts).wait()
+        return sum(counts.values()), counts, since
+
+    def get_bugs(self, date="today", bug_ids=[]):
+        super().get_bugs(date=date, bug_ids=bug_ids)
+
+        result = {}
+        for bug_id, info in self.bug_data.items():
+            total, per_sig, since = self._query_socorro(info)
+            if total < self.min_crash_count:
+                continue
+
+            self.extra_ni[bug_id] = {
+                "crash_count": total,
+                "since": since,
+                "fix_date": info["fix_date"][:10],
+                "signatures": info["signatures"],
+                "per_signature_counts": per_sig,
+            }
+            self.add_auto_ni(
+                bug_id,
+                {
+                    "mail": info["assignee_email"],
+                    "nickname": info["assignee_nickname"],
+                },
+            )
+            result[bug_id] = {"id": bug_id, "summary": info["summary"]}
+
+        return result
+
+
+if __name__ == "__main__":
+    CrashesAfterFix().run()

configs/rules.json

@@ -112,6 +112,10 @@
     "keyword_exception": ["testcase"],
     "sec": false
   },
+  "crashes_after_fix": {
+    "max_days_since_fix": 10,
+    "min_crash_count": 5
+  },
   "newbie_with_ni": {
     "number_of_days": 7,
     "number_of_comments": 2

scripts/cron_run_weekdays.sh

@@ -179,6 +179,9 @@ python -m bugbot.rules.fuzz_blockers --production
 # Detect bugs with small crash volume
 python -m bugbot.rules.crash_small_volume --production
 
+# Notify assignees when a fix doesn't actually stop the crash on Nightly
+python -m bugbot.rules.crashes_after_fix --production
+
 # Send a list with security bugs that could be un-hidden
 python -m bugbot.rules.security_unhide_dups --production
 

templates/crashes_after_fix.html

@@ -0,0 +1,23 @@
+<p>
+    The following {{ plural('bug', data) }} {{ plural('still has', data, pword='still have') }} reported crashes on Nightly within {{ extra['max_days'] }} days after the fix landed (>= {{ extra['min_crashes'] }} crashes since the day after landing):
+</p>
+<table {{ table_attrs }}>
+    <thead>
+        <tr>
+            <th>Bug</th>
+            <th>Summary</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for i, (bugid, summary) in enumerate(data) -%}
+            <tr {% if i % 2 == 0 %}bgcolor="#E0E0E0"
+            {% endif -%}
+            >
+            <td>
+                <a href="https://bugzilla.mozilla.org/show_bug.cgi?id={{ bugid }}">{{ bugid }}</a>
+            </td>
+            <td>{{ summary | e }}</td>
+        </tr>
+    {% endfor -%}
+</tbody>
+</table>

templates/crashes_after_fix_needinfo.txt

@@ -0,0 +1,4 @@
+{% if nickname %}:{{ nickname }},{% endif %}
+this bug was marked RESOLVED FIXED on {{ extra['fix_date'] }}, but crashes are still being reported against this signature on Nightly. Since the day after the fix landed ({{ extra['since'] }}), {{ extra['crash_count'] }} crash{{ 'es' if extra['crash_count'] != 1 else '' }} {{ 'have' if extra['crash_count'] != 1 else 'has' }} been recorded on Nightly across the {{ extra['signatures']|length }} signature{{ 's' if extra['signatures']|length != 1 else '' }} on this bug.
+Could you take a look at the recent crash reports to determine whether the fix is incomplete, whether the signature is shared with a different underlying crash, or whether a follow-up is needed?
+{{ documentation }}