diff --git a/.gitignore b/.gitignore index 5c4bfef..950835b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Python +*.pyc +__pycache__ + # Virtual Environments .env .venv @@ -7,3 +11,7 @@ lib/ lib64 pyvenv.cfg share/ + +# sqlite +db.sqlite3 +db.*.sqlite3 diff --git a/ingest/__init__.py b/ingest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingest/admin.py b/ingest/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/ingest/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/ingest/apps.py b/ingest/apps.py new file mode 100644 index 0000000..ab36cdc --- /dev/null +++ b/ingest/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class IngestConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'ingest' diff --git a/ingest/migrations/__init__.py b/ingest/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ingest/models.py b/ingest/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/ingest/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/ingest/negotiation.py b/ingest/negotiation.py new file mode 100644 index 0000000..5a4f798 --- /dev/null +++ b/ingest/negotiation.py @@ -0,0 +1,18 @@ +from rest_framework.negotiation import BaseContentNegotiation + + +class IgnoreClientContentNegotiation(BaseContentNegotiation): + # Sentry clients do not always send the correct Accept headers but expect json anyway. + # Proceed as per https://www.django-rest-framework.org/api-guide/content-negotiation/#example + + def select_parser(self, request, parsers): + """ + Select the first parser in the `.parser_classes` list. + """ + return parsers[0] + + def select_renderer(self, request, renderers, format_suffix): + """ + Select the first renderer in the `.renderer_classes` list. + """ + return (renderers[0], renderers[0].media_type) diff --git a/ingest/parsers.py b/ingest/parsers.py new file mode 100644 index 0000000..109a9d0 --- /dev/null +++ b/ingest/parsers.py @@ -0,0 +1,28 @@ +import codecs + +from django.conf import settings +from rest_framework.exceptions import ParseError +from rest_framework.parsers import JSONParser +from rest_framework.utils import json + + +class EnvelopeParser(JSONParser): + media_type = "application/x-sentry-envelope" + + def parse(self, stream, media_type=None, parser_context=None): + """ + Parses the incoming bytestream as JSON and returns the resulting data. + Supports multiple lines of JSON objects + """ + parser_context = parser_context or {} + encoding = parser_context.get("encoding", settings.DEFAULT_CHARSET) + + try: + decoded_stream = codecs.getreader(encoding)(stream) + parse_constant = json.strict_constant if self.strict else None + messages = [] + for line in decoded_stream: + messages.append(json.loads(line, parse_constant=parse_constant)) + return messages + except ValueError as exc: + raise ParseError("JSON parse error - %s" % str(exc)) diff --git a/ingest/tests.py b/ingest/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/ingest/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/ingest/urls.py b/ingest/urls.py new file mode 100644 index 0000000..1d63bbc --- /dev/null +++ b/ingest/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from .views import IngestEventAPIView, IngestEnvelopeAPIView + +urlpatterns = [ + # project_id has to be an int per Sentry Client expectations. + path("/store/", IngestEventAPIView.as_view()), + path("/envelope/", IngestEnvelopeAPIView.as_view()), +] diff --git a/ingest/views.py b/ingest/views.py new file mode 100644 index 0000000..f46df94 --- /dev/null +++ b/ingest/views.py @@ -0,0 +1,29 @@ +from rest_framework import permissions +from rest_framework.response import Response +from rest_framework.views import APIView + +# from projects.models import Project +# from sentry.utils.auth import parse_auth_header + +from .negotiation import IgnoreClientContentNegotiation +from .parsers import EnvelopeParser + + +class BaseIngestAPIView(APIView): + permission_classes = [permissions.AllowAny] + authentication_classes = [] + content_negotiation_class = IgnoreClientContentNegotiation + http_method_names = ["post"] + + def post(self, request, *args, **kwargs): + import pdb; pdb.set_trace() + # return self.process_event(request.data, request, project) + return Response() + + +class IngestEventAPIView(BaseIngestAPIView): + pass + + +class IngestEnvelopeAPIView(BaseIngestAPIView): + parser_classes = [EnvelopeParser] diff --git a/project/__pycache__/__init__.cpython-310.pyc b/project/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index ccf1e38..0000000 Binary files a/project/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/project/__pycache__/settings.cpython-310.pyc b/project/__pycache__/settings.cpython-310.pyc deleted file mode 100644 index 37b7f6f..0000000 Binary files a/project/__pycache__/settings.cpython-310.pyc and /dev/null differ diff --git a/project/__pycache__/urls.cpython-310.pyc b/project/__pycache__/urls.cpython-310.pyc deleted file mode 100644 index 86000df..0000000 Binary files a/project/__pycache__/urls.cpython-310.pyc and /dev/null differ diff --git a/project/__pycache__/wsgi.cpython-310.pyc b/project/__pycache__/wsgi.cpython-310.pyc deleted file mode 100644 index 23ecf8e..0000000 Binary files a/project/__pycache__/wsgi.cpython-310.pyc and /dev/null differ diff --git a/project/settings.py b/project/settings.py index e214bd6..78ce408 100644 --- a/project/settings.py +++ b/project/settings.py @@ -1,24 +1,14 @@ -""" -Django settings for project project. - -Generated by 'django-admin startproject' using Django 4.2.6. - -For more information on this file, see -https://docs.djangoproject.com/en/4.2/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/4.2/ref/settings/ -""" +import os from pathlib import Path +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ - # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = 'django-insecure-$@clhhieazwnxnha-_zah&(bieq%yux7#^07&xsvhn58t)8@xw' @@ -37,6 +27,8 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + + 'ingest', ] MIDDLEWARE = [ @@ -47,6 +39,8 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + + 'sentry.middleware.proxy.DecompressBodyMiddleware', ] ROOT_URLCONF = 'project.urls' @@ -76,7 +70,7 @@ WSGI_APPLICATION = 'project.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', + 'NAME': BASE_DIR / os.getenv("DATABASE_NAME", 'db.sqlite3'), } } @@ -121,3 +115,16 @@ STATIC_URL = 'static/' # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + +# {PROTOCOL}://{PUBLIC_KEY}:{SECRET_KEY}@{HOST}{PATH}/{PROJECT_ID} +SENTRY_DSN = os.getenv("SENTRY_DSN") # "http://ignored_public_key:ignored_secret_key@127.0.0.1:9000/1" + + +if SENTRY_DSN is not None: + sentry_sdk.init( + dsn=SENTRY_DSN, + integrations=[DjangoIntegration()], + auto_session_tracking=False, + traces_sample_rate=0, + ) diff --git a/project/urls.py b/project/urls.py index cb1bf5d..2fbd3c3 100644 --- a/project/urls.py +++ b/project/urls.py @@ -1,22 +1,18 @@ -""" -URL configuration for project project. +from django.conf import settings -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/4.2/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.contrib import admin -from django.urls import path +from django.urls import include, path + +from .views import trigger_error + urlpatterns = [ + path('api/', include('ingest.urls')), + path('admin/', admin.site.urls), ] + +if settings.DEBUG: + urlpatterns += [ + path('trigger-error/', trigger_error), + ] diff --git a/project/views.py b/project/views.py new file mode 100644 index 0000000..f2fb5fc --- /dev/null +++ b/project/views.py @@ -0,0 +1,2 @@ +def trigger_error(request): + raise Exception("Exception triggered on purpose to debug error handling") diff --git a/requirements.txt b/requirements.txt index e63dac5..69a6ad8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ Django==4.2.* +sentry-sdk==1.32.* +djangorestframework==3.14.* + +six diff --git a/sentry/__init__.py b/sentry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sentry/middleware/__init__.py b/sentry/middleware/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sentry/middleware/proxy.py b/sentry/middleware/proxy.py new file mode 100644 index 0000000..941b7dd --- /dev/null +++ b/sentry/middleware/proxy.py @@ -0,0 +1,206 @@ +import io +import logging +import zlib + +from django.conf import settings +from django.core.exceptions import RequestDataTooBig + +try: + import uwsgi + + has_uwsgi = True +except ImportError: + has_uwsgi = False + + +logger = logging.getLogger(__name__) +Z_CHUNK = 1024 * 8 + + +if has_uwsgi: + + class UWsgiChunkedInput(io.RawIOBase): + def __init__(self): + self._internal_buffer = b"" + + def readable(self): + return True + + def readinto(self, buf): + if not self._internal_buffer: + self._internal_buffer = uwsgi.chunked_read() + + n = min(len(buf), len(self._internal_buffer)) + if n > 0: + buf[:n] = self._internal_buffer[:n] + self._internal_buffer = self._internal_buffer[n:] + + return n + + +class ZDecoder(io.RawIOBase): + """ + Base class for HTTP content decoders based on zlib + See: https://github.com/eBay/wextracto/blob/9c789b1c98d95a1e87dbedfd1541a8688d128f5c/wex/http_decoder.py + """ + + def __init__(self, fp, z=None): + self.fp = fp + self.z = z + self.flushed = None + self.counter = 0 + + def readable(self): + return True + + def readinto(self, buf): + if self.z is None: + self.z = zlib.decompressobj() + retry = True + else: + retry = False + + n = 0 + max_length = len(buf) + # DOS mitigation - block unzipped payloads larger than max allowed size + self.counter += 1 + if self.counter * max_length > 1_000_000_000_000: # TODO I replaced this + raise RequestDataTooBig() + + while max_length > 0: + if self.flushed is None: + chunk = self.fp.read(Z_CHUNK) + compressed = self.z.unconsumed_tail + chunk + try: + decompressed = self.z.decompress(compressed, max_length) + except zlib.error: + if not retry: + raise + self.z = zlib.decompressobj(-zlib.MAX_WBITS) + retry = False + decompressed = self.z.decompress(compressed, max_length) + + if not chunk: + self.flushed = self.z.flush() + else: + if not self.flushed: + return n + + decompressed = self.flushed[:max_length] + self.flushed = self.flushed[max_length:] + + buf[n:n + len(decompressed)] = decompressed + n += len(decompressed) + max_length = len(buf) - n + + return n + + +class DeflateDecoder(ZDecoder): + """ + Decoding for "content-encoding: deflate" + """ + + +class GzipDecoder(ZDecoder): + """ + Decoding for "content-encoding: gzip" + """ + + def __init__(self, fp): + ZDecoder.__init__(self, fp, zlib.decompressobj(16 + zlib.MAX_WBITS)) + + +class SetRemoteAddrFromForwardedFor(object): + def __init__(self): + if not getattr(settings, "SENTRY_USE_X_FORWARDED_FOR", True): + from django.core.exceptions import MiddlewareNotUsed + + raise MiddlewareNotUsed + + def _remove_port_number(self, ip_address): + if "[" in ip_address and "]" in ip_address: + # IPv6 address with brackets, possibly with a port number + return ip_address[ip_address.find("[") + 1:ip_address.find("]")] + if "." in ip_address and ip_address.rfind(":") > ip_address.rfind("."): + # IPv4 address with port number + # the last condition excludes IPv4-mapped IPv6 addresses + return ip_address.rsplit(":", 1)[0] + return ip_address + + def process_request(self, request): + try: + real_ip = request.META["HTTP_X_FORWARDED_FOR"] + except KeyError: + pass + else: + # HTTP_X_FORWARDED_FOR can be a comma-separated list of IPs. + # Take just the first one. + real_ip = real_ip.split(",")[0].strip() + real_ip = self._remove_port_number(real_ip) + request.META["REMOTE_ADDR"] = real_ip + + +class ChunkedMiddleware(object): + def __init__(self): + if not has_uwsgi: + from django.core.exceptions import MiddlewareNotUsed + + raise MiddlewareNotUsed + + def process_request(self, request): + # If we are dealing with chunked data and we have uwsgi we assume + # that we can read to the end of the input stream so we can bypass + # the default limited stream. We set the content length reasonably + # high so that the reads generally succeed. This is ugly but with + # Django 1.6 it seems to be the best we can easily do. + if "HTTP_TRANSFER_ENCODING" not in request.META: + return + + if request.META["HTTP_TRANSFER_ENCODING"].lower() == "chunked": + request._stream = io.BufferedReader(UWsgiChunkedInput()) + request.META["CONTENT_LENGTH"] = "4294967295" # 0xffffffff + + +class DecompressBodyMiddleware(object): + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + decode = False + encoding = request.META.get("HTTP_CONTENT_ENCODING", "").lower() + + if encoding == "gzip": + request._stream = GzipDecoder(request._stream) + decode = True + + if encoding == "deflate": + request._stream = DeflateDecoder(request._stream) + decode = True + + if decode: + # Since we don't know the original content length ahead of time, we + # need to set the content length reasonably high so read generally + # succeeds. This seems to be the only easy way for Django 1.6. + request.META["CONTENT_LENGTH"] = "4294967295" # 0xffffffff + + # The original content encoding is no longer valid, so we have to + # remove the header. Otherwise, LazyData will attempt to re-decode + # the body. + del request.META["HTTP_CONTENT_ENCODING"] + return self.get_response(request) + + +class ContentLengthHeaderMiddleware(object): + """ + Ensure that we have a proper Content-Length/Transfer-Encoding header + """ + + def process_response(self, request, response): + if "Transfer-Encoding" in response or "Content-Length" in response: + return response + + if not response.streaming: + response["Content-Length"] = str(len(response.content)) + + return response diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..dcf6179 --- /dev/null +++ b/tox.ini @@ -0,0 +1,3 @@ +[flake8] +max-line-length=120 +exclude=venv,.venv,migrations