never_evict events that are a Historic Turning Point

Both for technical (foreign keys) and business reasons (these are events you
care about)

events/migrations/0006_event_never_evict.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.13 on 2024-06-24 14:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('events', '0005_event_events_even_project_abe572_idx'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='event',
+            name='never_evict',
+            field=models.BooleanField(default=False),
+        ),
+    ]

events/migrations/0007_set_never_evict.py

@@ -0,0 +1,18 @@
+from django.db import migrations
+
+
+def set_never_evict(apps, schema_editor):
+    Event = apps.get_model('events', 'Event')
+    Event.objects.filter(turningpoint__isnull=False).update(never_evict=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('events', '0006_event_never_evict'),
+        ('issues', '0003_alter_turningpoint_triggering_event'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_never_evict, migrations.RunPython.noop),
+    ]

events/migrations/0008_remove_event_events_even_project_abe572_idx_and_more.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.13 on 2024-06-24 20:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('events', '0007_set_never_evict'),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name='event',
+            name='events_even_project_abe572_idx',
+        ),
+        migrations.AddIndex(
+            model_name='event',
+            index=models.Index(fields=['project', 'never_evict', 'server_side_timestamp', 'irrelevance_for_retention'], name='events_even_project_adcdee_idx'),
+        ),
+    ]

events/models.py

@@ -152,6 +152,7 @@ class Event(models.Model):
     # irrelevance_for_retention is set on-ingest based on the number of available events for an issue; it is combined
     # with age-based-irrelevance to determine which events will be evicted when retention quota are met.
     irrelevance_for_retention = models.PositiveIntegerField(blank=False, null=False)
+    never_evict = models.BooleanField(blank=False, null=False, default=False)
 
     class Meta:
         unique_together = [
@@ -159,7 +160,7 @@ class Event(models.Model):
             ("issue", "ingest_order"),
         ]
         indexes = [
-            models.Index(fields=["project", "server_side_timestamp", "irrelevance_for_retention"]),
+            models.Index(fields=["project", "never_evict", "server_side_timestamp", "irrelevance_for_retention"]),
         ]
 
     def get_absolute_url(self):

events/retention.py

@@ -107,8 +107,9 @@ def get_age_for_irrelevance(age_based_irrelevance):
 def get_epoch_bounds_with_irrelevance(project, current_timestamp):
     from .models import Event
 
-    # We can safely assume some Event exists when this point is reached because of the conditions in `should_evict`
-    first_epoch = get_epoch(Event.objects.filter(project=project).aggregate(val=Min('server_side_timestamp'))['val'])
+    oldest = Event.objects.filter(project=project, never_evict=False).aggregate(val=Min('server_side_timestamp'))['val']
+    first_epoch = get_epoch(oldest) if oldest is not None else get_epoch(current_timestamp)
+
     current_epoch = get_epoch(current_timestamp)
 
     difference = current_epoch - first_epoch
@@ -121,13 +122,14 @@ def get_epoch_bounds_with_irrelevance(project, current_timestamp):
 
 
 def get_irrelevance_pairs(project, epoch_bounds_with_irrelevance):
-    """tuples of `age_based_irrelevance` and, per associated period, the max observed event irrelevance"""
+    """tuples of `age_based_irrelevance` and, per associated period, the max observed (evictable) event irrelevance"""
     from .models import Event
 
     for (lower_bound, upper_bound), age_based_irrelevance in epoch_bounds_with_irrelevance:
         d = Event.objects.filter(
             get_epoch_bounds(lower_bound, upper_bound),
             project=project,
+            never_evict=False,
         ).aggregate(Max('irrelevance_for_retention'))
         max_event_irrelevance = d["irrelevance_for_retention__max"] or 0
 
@@ -266,7 +268,7 @@ def evict_for_epoch_and_irrelevance(project, max_epoch, max_irrelevance):
     # this call, and only when `B` is cleaned will the points `x` be cleaned. (as-is, they are part of the selection,
     # but will already have been deleted)
 
-    qs = Event.objects.filter(project=project, irrelevance_for_retention__gt=max_irrelevance)
+    qs = Event.objects.filter(project=project, irrelevance_for_retention__gt=max_irrelevance, never_evict=False)
 
     if max_epoch is not None:
         qs = qs.filter(server_side_timestamp__lt=datetime_for_epoch(max_epoch))

ingest/views.py

@@ -245,6 +245,7 @@ class BaseIngestAPIView(View):
             TurningPoint.objects.create(
                 issue=issue, triggering_event=event, timestamp=timestamp,
                 kind=TurningPointKind.FIRST_SEEN)
+            event.never_evict = True
 
             if project.alert_on_new_issue:
                 delay_on_commit(send_new_issue_alert, str(issue.id))
@@ -255,6 +256,7 @@ class BaseIngestAPIView(View):
                 TurningPoint.objects.create(
                     issue=issue, triggering_event=event, timestamp=timestamp,
                     kind=TurningPointKind.REGRESSED)
+                event.never_evict = True
 
                 if project.alert_on_regression:
                     delay_on_commit(send_regression_alert, str(issue.id))
@@ -277,6 +279,14 @@ class BaseIngestAPIView(View):
                     issue, triggering_event=event,
                     unmute_metadata={"mute_for": {"unmute_after": issue.unmute_after}})
 
+        if event.never_evict:
+            # as a sort of poor man's django-dirtyfields (which we haven't adopted for simplicity's sake) we simply do
+            # this manually for a single field; we know that if never_evict has been set, it's always been set after the
+            # .create call, i.e. its results still need to be saved. We accept the cost of the extra .save call, since
+            # TurningPoints are relatively rare (and hence so is this setting of `never_evict` and the associated save
+            # call)
+            event.save()
+
         if release.version + "\n" not in issue.events_at:
             issue.events_at += release.version + "\n"
         issue.save()

issues/models.py

@@ -260,6 +260,7 @@ class IssueStateManager(object):
                 TurningPoint.objects.create(
                     issue=issue, triggering_event=triggering_event, timestamp=triggering_event.server_side_timestamp,
                     kind=TurningPointKind.UNMUTED, metadata=json.dumps(unmute_metadata))
+                triggering_event.never_evict = True  # .save() will be called by the caller of this function
 
     @staticmethod
     def set_unmute_handlers(by_issue, issue, now):

releases/models.py

@@ -108,6 +108,7 @@ def create_release_if_needed(project, version, event):
                     metadata=json.dumps({"actual_release": release.version}), timestamp=event.server_side_timestamp)
                 for issue in resolved_by_next_qs
             ])
+            event.never_evict = True  # .save() will be called by the caller of this function
 
             resolved_by_next_qs.update(
                 fixed_at=Concat("fixed_at", Value(release.version + "\n")),