Hi,
I configured helm sentry in kubernetes cluster. It worked fine with fresh setup. But once i restart any pod or sync any config change, nginx starts throwing 502 Bad Gateway.
I looked into logs of worker and web pods, and found out similar errors continuously spamming.
Logs of sentry-web pod
10.23.45.1 - - [02/Jun/2021:12:28:11 +0000] "GET /_health/ HTTP/1.1" 200 200 "-" "kube-probe/1.17+"
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: FATAL: password authentication failed for user "postgres"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/sentry/api/base.py", line 117, in handle_exception
response = super().handle_exception(exc)
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 465, in handle_exception
self.raise_uncaught_exception(exc)
File "/usr/local/lib/python3.6/site-packages/rest_framework/views.py", line 476, in raise_uncaught_exception
raise exc
File "/usr/local/lib/python3.6/site-packages/sentry/api/base.py", line 230, in dispatch
response = handler(request, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry/api/endpoints/relay_register.py", line 99, in post
relay = Relay.objects.get(relay_id=relay_id)
File "/usr/local/lib/python3.6/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 374, in get
num = len(clone)
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 232, in __len__
self._fetch_all()
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 1121, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 53, in __iter__
results = compiler.execute_sql(chunked_fetch=self.chunked_fetch)
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 887, in execute_sql
cursor = self.connection.cursor()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 254, in cursor
return self._cursor()
File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/decorators.py", line 40, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/base.py", line 96, in _cursor
return super()._cursor()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 229, in _cursor
self.ensure_connection()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/usr/local/lib/python3.6/site-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
django.db.utils.OperationalError: FATAL: password authentication failed for user "postgres"
10.23.45.10 - - [02/Jun/2021:12:28:17 +0000] "POST /api/0/relays/register/challenge/ HTTP/1.1" 500 605 "-" "-"
10.23.45.1 - - [02/Jun/2021:12:28:18 +0000] "GET /_health/ HTTP/1.1" 200 200 "-" "kube-probe/1.17+"
10.23.45.1 - - [02/Jun/2021:12:28:21 +0000] "GET /_health/ HTTP/1.1" 200 200 "-" "kube-probe/1.17+"
Logs of sentry-worker pod
12:37:21 [ERROR] celery.app.trace: Task sentry.tasks.check_monitors[e900e484-1916-4fed-9e51-ed167534377b] raised unexpected: OperationalError('FATAL: password authentication failed for user "postgres"\n',) (data={'hostname': 'mowx-sentry-worker-84c69d67f8-5bd7d', 'id': 'e900e484-1916-4fed-9e51-ed167534377b', 'name': 'sentry.tasks.check_monitors', 'exc': 'OperationalError(\'FATAL: password authentication failed for user "postgres"\\n\',)', 'traceback': 'Traceback (most recent call last):\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection\n self.connect()\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect\n self.connection = self.get_new_connection(conn_params)\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection\n connection = Database.connect(**conn_params)\n File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\npsycopg2.OperationalError: FATAL: password authentication failed for user "postgres"\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/usr/local/lib/python3.6/site-packages/celery/app/trace.py", line 412, in trace_task\n R = retval = fun(*args, **kwargs)\n File "/usr/local/lib/python3.6/site-packages/celery/app/trace.py", line 704, in __protected_call__\n return self.run(*args, **kwargs)\n File "/usr/local/lib/python3.6/site-packages/sentry_sdk/integrations/celery.py", line 197, in _inner\n reraise(*exc_info)\n File "/usr/local/lib/python3.6/site-packages/sentry_sdk/_compat.py", line 54, in reraise\n raise value\n File "/usr/local/lib/python3.6/site-packages/sentry_sdk/integrations/celery.py", line 192, in _inner\n return f(*args, **kwargs)\n File "/usr/local/lib/python3.6/site-packages/sentry/tasks/base.py", line 46, in _wrapped\n result = func(*args, **kwargs)\n File "/usr/local/lib/python3.6/site-packages/sentry/tasks/check_monitors.py", line 38, in check_monitors\n for monitor in qs:\n File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 250, in __iter__\n self._fetch_all()\n File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 1121, in _fetch_all\n self._result_cache = list(self._iterable_class(self))\n File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 53, in __iter__\n results = compiler.execute_sql(chunked_fetch=self.chunked_fetch)\n File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 887, in execute_sql\n cursor = self.connection.cursor()\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 254, in cursor\n return self._cursor()\n File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/decorators.py", line 40, in inner\n return func(self, *args, **kwargs)\n File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/base.py", line 96, in _cursor\n return super()._cursor()\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 229, in _cursor\n self.ensure_connection()\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection\n self.connect()\n File "/usr/local/lib/python3.6/site-packages/django/db/utils.py", line 94, in __exit__\n six.reraise(dj_exc_type, dj_exc_value, traceback)\n File "/usr/local/lib/python3.6/site-packages/django/utils/six.py", line 685, in reraise\n raise value.with_traceback(tb)\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection\n self.connect()\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect\n self.connection = self.get_new_connection(conn_params)\n File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection\n connection = Database.connect(**conn_params)\n File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\ndjango.db.utils.OperationalError: FATAL: password authentication failed for user "postgres"\n\n', 'args': '()', 'kwargs': '{}', 'description': 'raised unexpected', 'internal': False})
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
psycopg2.OperationalError: FATAL: password authentication failed for user "postgres"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/celery/app/trace.py", line 412, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/celery/app/trace.py", line 704, in __protected_call__
return self.run(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry_sdk/integrations/celery.py", line 197, in _inner
reraise(*exc_info)
File "/usr/local/lib/python3.6/site-packages/sentry_sdk/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.6/site-packages/sentry_sdk/integrations/celery.py", line 192, in _inner
return f(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry/tasks/base.py", line 46, in _wrapped
result = func(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry/tasks/options.py", line 26, in sync_options
for option in Option.objects.filter(last_updated__gte=cutoff_dt).iterator():
File "/usr/local/lib/python3.6/site-packages/django/db/models/query.py", line 53, in __iter__
results = compiler.execute_sql(chunked_fetch=self.chunked_fetch)
File "/usr/local/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 885, in execute_sql
cursor = self.connection.chunked_cursor()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 230, in chunked_cursor
self._named_cursor_idx,
File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/decorators.py", line 40, in inner
return func(self, *args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/sentry/db/postgres/base.py", line 96, in _cursor
return super()._cursor()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 229, in _cursor
self.ensure_connection()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/usr/local/lib/python3.6/site-packages/django/utils/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 213, in ensure_connection
self.connect()
File "/usr/local/lib/python3.6/site-packages/django/db/backends/base/base.py", line 189, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.6/site-packages/django/db/backends/postgresql/base.py", line 176, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib/python3.6/site-packages/psycopg2/__init__.py", line 130, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
django.db.utils.OperationalError: FATAL: password authentication failed for user "postgres"
Looking into logs, found out that two services are unable to authenticate user “postgres”. I haven’t made any changes to postgres config.
The changes that I made in this default config charts/sentry at develop · sentry-kubernetes/charts · GitHub are:
- switch Nginx to LoadBalancer
- Add google oauth2
Can someone help me debug this?