Migrate cassie.py from deprecated pycassa to cassandra-driver ORM

README.md

@@ -26,13 +26,18 @@ sudo apt install apport-retrace python3-amqp python3-bson python3-cassandra pyth
 sudo apt install python3-django-tastypie python3-numpy
 ```
 
-Then start a local Cassandra, RabbitMQ and swift (`docker` should works fine too):
+Then start a local Cassandra, RabbitMQ and swift (`docker` should work fine too):
 ```
 podman run --name cassandra --network host --rm -d -e HEAP_NEWSIZE=10M -e MAX_HEAP_SIZE=200M docker.io/cassandra
 podman run --name rabbitmq --network host --rm -d docker.io/rabbitmq
 podman run --name swift --network host --rm -d docker.io/openstackswift/saio
 ```
 
+> Note:
+> * Cassandra can take some time (a minute or two?) to fully start.
+> * Also, sometimes, Cassandra can hang and you get some `OperationTimedOut`
+>   issues out of nowhere. Just `podman kill cassandra` and restart it.
+
 You can then then run the tests with `pytest`:
 ```
 cd src

src/daisy/submit.py

@@ -36,22 +36,6 @@
 logger = logging.getLogger("daisy")
 
 
-def update_counters(release, src_package, date, src_version=None):
-    if src_version:
-        key = "%s:%s:%s" % (release, src_package, src_version)
-    else:
-        key = "%s:%s" % (release, src_package)
-    cassandra_schema.Counters(key=key.encode(), column1=date).update(value=1)
-
-
-def update_proposed_counters(release, src_package, date, src_version=None):
-    if src_version:
-        key = "%s:%s:%s" % (release, src_package, src_version)
-    else:
-        key = "%s:%s" % (release, src_package)
-    cassandra_schema.CountersForProposed(key=key.encode(), column1=date).update(value=1)
-
-
 def create_minimal_report_from_bson(data):
     report = Report()
     for key in data:
@@ -181,12 +165,6 @@ def submit(request, system_token):
     pkg_arch = utils.get_package_architecture(data)
     problem_type = data.get("ProblemType", "")
     apport_version = data.get("ApportVersion", "")
-    third_party = False
-    if not utils.retraceable_package(package):
-        third_party = True
-    automated_testing = False
-    if system_token.startswith("deadbeef"):
-        automated_testing = True
 
     if not release:
         metrics.meter("missing.missing_release")
@@ -221,21 +199,6 @@ def submit(request, system_token):
         problem_type, release, package, version, pkg_arch
     )
 
-    # generic counter for crashes about a source package which is used by the
-    # phased-updater and only includes official Ubuntu packages and not those
-    # crahses from systems under auto testing.
-    if not third_party and not automated_testing and problem_type == "Crash":
-        update_counters(release=release, src_package=src_package, date=day_key)
-        if version == "":
-            metrics.meter("missing.missing_package_version")
-        else:
-            update_counters(
-                release=release,
-                src_package=src_package,
-                src_version=version,
-                date=day_key,
-            )
-
     # ProcMaps is useful for creating a crash sig, not after that
     if "Traceback" in data and "ProcMaps" in data:
         data.pop("ProcMaps")
@@ -262,18 +225,6 @@ def submit(request, system_token):
     package_from_proposed = False
     if "package-from-proposed" in tags:
         package_from_proposed = True
-        # generic counter for crashes about a source package which is used by
-        # the phased-updater and only includes official Ubuntu packages and
-        # not those from systems under auto testing.
-        if not third_party and not automated_testing and problem_type == "Crash":
-            update_proposed_counters(release=release, src_package=src_package, date=day_key)
-            if version != "":
-                update_proposed_counters(
-                    release=release,
-                    src_package=src_package,
-                    src_version=version,
-                    date=day_key,
-                )
 
     # A device is manually blocklisted if it has repeatedly failed to have an
     # crash inserted into the OOPS table.
@@ -356,7 +307,7 @@ def bucket(oops_id, data, day_key):
                     key=b"crash_signature_for_stacktrace_address_signature", column1=addr_sig
                 ).value.decode()
             except DoesNotExist:
-                pass
+                metrics.meter("missing.crash_signature")
         failed_to_retrace = False
         if crash_sig.startswith("failed:"):
             failed_to_retrace = True
@@ -401,7 +352,9 @@ def bucket(oops_id, data, day_key):
                     "StacktraceTop",
                 )
                 for unneeded_column in unneeded_columns:
-                    cassandra_schema.OOPS.filter(key=oops_id.encode(), column1=unneeded_column).delete()
+                    cassandra_schema.OOPS.filter(
+                        key=oops_id.encode(), column1=unneeded_column
+                    ).delete()
             # We have already retraced for this address signature, so this
             # crash can be immediately bucketed.
             utils.bucket(oops_id, crash_sig, data)

src/errors/api/resources.py

@@ -1,32 +1,23 @@
 # Treat strings as UTF-8 instead of ASCII
-import importlib
-import sys
-from functools import cmp_to_key
-
-importlib.reload(sys)
-
-from tastypie import fields
-from tastypie.authentication import Authentication, SessionAuthentication
-from tastypie.authorization import Authorization, DjangoAuthorization
-from tastypie.exceptions import NotFound
-from tastypie.resources import Resource
-
-from errors import cassie
-
-TASTYPIE_FULL_DEBUG = True
-
 import datetime
 import json as simplejson
 from collections import OrderedDict
+from functools import cmp_to_key
 from hashlib import sha1
 from operator import itemgetter
 from urllib.error import HTTPError
 from urllib.parse import quote, unquote
 
 import apt
 from django.core.serializers import json
+from tastypie import fields
+from tastypie.authentication import Authentication, SessionAuthentication
+from tastypie.authorization import Authorization, DjangoAuthorization
+from tastypie.exceptions import NotFound
+from tastypie.resources import Resource
 from tastypie.serializers import Serializer
 
+from errors import cassie
 from errortracker import config, launchpad
 
 from ..metrics import measure_view
@@ -313,7 +304,7 @@ def obj_get(self, **kwargs):
         oopses_by_day = set()
         oopses_by_release = set()
         for oops in cassie.get_oopses_by_day(date, limit):
-            oopses_by_day.add(str(oops))
+            oopses_by_day.add(oops)
         oopses = oopses_by_day
 
         if release:
@@ -614,10 +605,6 @@ def __getslice__(klass, start, finish):
                 # TODO: use a cassandra function that does a multiget of the
                 # crashes
                 for crash, ts in crashes:
-                    # cassandra records time in microseconds, convert to
-                    # seconds
-                    ts = (ts["submitted"][1]) * 1e-6
-                    ts = datetime.datetime.utcfromtimestamp(ts)
                     d = cassie.get_crash(str(crash), columns=cols)
                     program = split_package_and_version(d.get("Package", ""))[0]
                     if not program:
@@ -807,7 +794,7 @@ def __getslice__(klass, start, finish):
                             if item[0] in results:
                                 results[item[0]] -= item[1]
                         results = sorted(
-                            list(results.items()), key=cmp_to_key(lambda x, y: cmp(x[0], y[0]))
+                            list(results.items()), key=cmp_to_key(lambda x, y: x[0] <= y[0])
                         )
 
                         res = [{"x": result[0] * 1000, "y": result[1]} for result in results]

src/errors/auth.py

@@ -15,5 +15,5 @@
 def can_see_stacktraces(func):
     def in_groups(u):
         return u.groups.filter(name__in=groups).count() > 0
-    l = "/login-failed"
-    return login_required(user_passes_test(in_groups, login_url=l)(func))
+
+    return login_required(user_passes_test(in_groups, login_url="/login-failed")(func))