Getting this time and again on my logs,
May be causing issues in save,
These are my workers
worker1:
<< : *sentry_defaults
command: run worker -Q events.process_event
worker2:
<< : *sentry_defaults
command: run worker -Q events.reprocessing.process_event
worker3:
<< : *sentry_defaults
command: run worker -Q events.reprocess_events
worker4:
<< : *sentry_defaults
command: run worker -Q events.save_event
worker5:
<< : *sentry_defaults
command: run worker -Q subscriptions
worker6:
<< : *sentry_defaults
command: run worker -Q integrations
worker:
<< : *sentry_defaults
command: run worker
celery.worker.request: Hard time limit (65s) exceeded for sentry.tasks.store.save_event
worker4_1 | 06:28:16 [ERROR] celery.worker.request: Hard time limit (65s) exceeded for sentry.tasks.store.save_event[4207f0fc-f8ce-4156-a833-3bb03c0ffb28]
worker4_1 | Traceback (most recent call last):
worker4_1 | File "/usr/local/lib/python2.7/site-packages/celery/app/trace.py", line 375, in trace_task
worker4_1 | R = retval = fun(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/celery/app/trace.py", line 632, in __protected_call__
worker4_1 | return self.run(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/integrations/celery.py", line 171, in _inner
worker4_1 | reraise(*exc_info)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/integrations/celery.py", line 166, in _inner
worker4_1 | return f(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/tasks/base.py", line 48, in _wrapped
worker4_1 | result = func(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/tasks/store.py", line 842, in save_event
worker4_1 | _do_save_event(cache_key, data, start_time, event_id, project_id, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/tasks/store.py", line 763, in _do_save_event
worker4_1 | project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/utils/metrics.py", line 193, in inner
worker4_1 | return f(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/event_manager.py", line 351, in save
worker4_1 | hashes = job["event"].get_hashes()
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/eventstore/models.py", line 344, in get_hashes
worker4_1 | for _f in [x.get_hash() for x in self.get_grouping_variants(force_config).values()]
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/eventstore/models.py", line 382, in get_grouping_variants
worker4_1 | return get_grouping_variants_for_event(self, config)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/api.py", line 204, in get_grouping_variants_for_event
worker4_1 | components = _get_calculated_grouping_variants_for_event(event, config)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/api.py", line 143, in _get_calculated_grouping_variants_for_event
worker4_1 | rv = strategy.get_grouping_component_variants(event, config=config)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 122, in get_grouping_component_variants
worker4_1 | component = self.get_grouping_component(event, variant, config)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 102, in get_grouping_component
worker4_1 | return self(event=event, variant=variant, config=config, *args)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__
worker4_1 | return self._invoke(self.func, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke
worker4_1 | return func(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 515, in chained_exception
worker4_1 | return config.get_grouping_component(exceptions[0], **meta)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component
worker4_1 | return strategy(interface, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__
worker4_1 | return self._invoke(self.func, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke
worker4_1 | return func(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 504, in single_exception
worker4_1 | return single_exception_common(exception, config, meta, with_value=with_value)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 459, in single_exception_common
worker4_1 | stacktrace_component = config.get_grouping_component(exception.stacktrace, **meta)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component
worker4_1 | return strategy(interface, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__
worker4_1 | return self._invoke(self.func, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke
worker4_1 | return func(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 379, in stacktrace
worker4_1 | return get_stacktrace_component(stacktrace, config, variant, meta)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 395, in get_stacktrace_component
worker4_1 | frame_component = config.get_grouping_component(frame, variant=variant, **meta)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component
worker4_1 | return strategy(interface, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__
worker4_1 | return self._invoke(self.func, *args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke
worker4_1 | return func(*args, **kwargs)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 261, in frame
worker4_1 | php_detect_anonymous_classes=php_detect_anonymous_classes,
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 339, in get_frame_component
worker4_1 | php_detect_anonymous_classes=php_detect_anonymous_classes,
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 169, in get_function_component
worker4_1 | id="function", values=[func], similarity_encoder=ident_encoder
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 57, in __init__
worker4_1 | values=values,
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 109, in update
worker4_1 | contributes = _calculate_contributes(values)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 24, in _calculate_contributes
worker4_1 | for value in values or ():
worker4_1 | File "/usr/local/lib/python2.7/site-packages/billiard/pool.py", line 227, in soft_timeout_sighandler
worker4_1 | raise SoftTimeLimitExceeded()
worker4_1 | SoftTimeLimitExceeded: SoftTimeLimitExceeded()
worker4_1 | 06:28:17 [ERROR] celery.app.trace: Task sentry.tasks.store.save_event[4207f0fc-f8ce-4156-a833-3bb03c0ffb28] raised unexpected: SoftTimeLimitExceeded() (data={u'internal': False, u'traceback': u'Traceback (most recent call last):\n File "/usr/local/lib/python2.7/site-packages/celery/app/trace.py", line 375, in trace_task\n R = retval = fun(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/celery/app/trace.py", line 632, in __protected_call__\n return self.run(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry_sdk/integrations/celery.py", line 171, in _inner\n reraise(*exc_info)\n File "/usr/local/lib/python2.7/site-packages/sentry_sdk/integrations/celery.py", line 166, in _inner\n return f(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/tasks/base.py", line 48, in _wrapped\n result = func(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/tasks/store.py", line 842, in save_event\n _do_save_event(cache_key, data, start_time, event_id, project_id, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/tasks/store.py", line 763, in _do_save_event\n project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key\n File "/usr/local/lib/python2.7/site-packages/sentry/utils/metrics.py", line 193, in inner\n return f(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/event_manager.py", line 351, in save\n hashes = job["event"].get_hashes()\n File "/usr/local/lib/python2.7/site-packages/sentry/eventstore/models.py", line 344, in get_hashes\n for _f in [x.get_hash() for x in self.get_grouping_variants(force_config).values()]\n File "/usr/local/lib/python2.7/site-packages/sentry/eventstore/models.py", line 382, in get_grouping_variants\n return get_grouping_variants_for_event(self, config)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/api.py", line 204, in get_grouping_variants_for_event\n components = _get_calculated_grouping_variants_for_event(event, config)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/api.py", line 143, in _get_calculated_grouping_variants_for_event\n rv = strategy.get_grouping_component_variants(event, config=config)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 122, in get_grouping_component_variants\n component = self.get_grouping_component(event, variant, config)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 102, in get_grouping_component\n return self(event=event, variant=variant, config=config, *args)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__\n return self._invoke(self.func, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke\n return func(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 515, in chained_exception\n return config.get_grouping_component(exceptions[0], **meta)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component\n return strategy(interface, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__\n return self._invoke(self.func, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke\n return func(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 504, in single_exception\n return single_exception_common(exception, config, meta, with_value=with_value)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 459, in single_exception_common\n stacktrace_component = config.get_grouping_component(exception.stacktrace, **meta)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component\n return strategy(interface, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__\n return self._invoke(self.func, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke\n return func(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 379, in stacktrace\n return get_stacktrace_component(stacktrace, config, variant, meta)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 395, in get_stacktrace_component\n frame_component = config.get_grouping_component(frame, variant=variant, **meta)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 201, in get_grouping_component\n return strategy(interface, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 84, in __call__\n return self._invoke(self.func, *args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/base.py", line 81, in _invoke\n return func(*args, **kwargs)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 261, in frame\n php_detect_anonymous_classes=php_detect_anonymous_classes,\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 339, in get_frame_component\n php_detect_anonymous_classes=php_detect_anonymous_classes,\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/strategies/newstyle.py", line 169, in get_function_component\n id="function", values=[func], similarity_encoder=ident_encoder\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 57, in __init__\n values=values,\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 109, in update\n contributes = _calculate_contributes(values)\n File "/usr/local/lib/python2.7/site-packages/sentry/grouping/component.py", line 24, in _calculate_contributes\n for value in values or ():\n File "/usr/local/lib/python2.7/site-packages/billiard/pool.py", line 227, in soft_timeout_sighandler\n raise SoftTimeLimitExceeded()\nSoftTimeLimitExceeded: SoftTimeLimitExceeded()\n', u'name': 'sentry.tasks.store.save_event', u'args': '()', u'kwargs': "{'event_id': '1a67a91714204ae7893aa136c224e1a4', 'project_id': 19L, 'start_time': 1612422921.0, 'cache_key': u'e:1a67a91714204ae7893aa136c224e1a4:19', 'data': None}", u'description': u'raised unexpected', u'hostname': u'celery@d32bf612a5dc', u'id': '4207f0fc-f8ce-4156-a833-3bb03c0ffb28', u'exc': 'SoftTimeLimitExceeded()'})
snuba-transactions-consumer_1 | 2021-02-05 06:28:17,228 Completed processing <Batch: 17 messages, open for 0.81 seconds>.
worker4_1 | 06:28:17 [ERROR] multiprocessing: Process 'ForkPoolWorker-1' pid:25 exited with 'signal 9 (SIGKILL)'
snuba-consumer_1 | 2021-02-05 06:28:17,520 Completed processing <Batch: 18 messages, open for 0.79 seconds>.
snuba-outcomes-consumer_1 | 2021-02-05 06:28:17,684 Completed processing <Batch: 18 messages, open for 0.79 seconds>.
snuba-transactions-consumer_1 | 2021-02-05 06:28:18,229 Completed processing <Batch: 25 messages, open for 1.00 seconds>.
snuba-consumer_1 | 2021-02-05 06:28:18,319 Completed processing <Batch: 26 messages, open for 0.80 seconds>.
worker4_1 | Traceback (most recent call last):
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 311, in send_event_wrapper
worker4_1 | self._send_event(event)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 231, in _send_event
worker4_1 | headers={"Content-Type": "application/json", "Content-Encoding": "gzip"},
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 178, in _send_request
worker4_1 | headers=headers,
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/request.py", line 72, in request
worker4_1 | **urlopen_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/request.py", line 150, in request_encode_body
worker4_1 | return self.urlopen(method, url, **extra_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/poolmanager.py", line 324, in urlopen
worker4_1 | response = conn.urlopen(method, u.request_uri, **kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
worker4_1 | _stacktrace=sys.exc_info()[2])
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/util/retry.py", line 399, in increment
worker4_1 | raise MaxRetryError(_pool, url, error or ResponseError(cause))
worker4_1 | MaxRetryError: HTTPSConnectionPool(host='retail-sentry.apple.com', port=443): Max retries exceeded with url: /api/1/store/ (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))
worker4_1 | 06:28:18 [ERROR] sentry_sdk.errors: Internal error in sentry_sdk
worker4_1 | Traceback (most recent call last):
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 311, in send_event_wrapper
worker4_1 | self._send_event(event)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 231, in _send_event
worker4_1 | headers={"Content-Type": "application/json", "Content-Encoding": "gzip"},
worker4_1 | File "/usr/local/lib/python2.7/site-packages/sentry_sdk/transport.py", line 178, in _send_request
worker4_1 | headers=headers,
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/request.py", line 72, in request
worker4_1 | **urlopen_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/request.py", line 150, in request_encode_body
worker4_1 | return self.urlopen(method, url, **extra_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/poolmanager.py", line 324, in urlopen
worker4_1 | response = conn.urlopen(method, u.request_uri, **kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 667, in urlopen
worker4_1 | **response_kw)
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
worker4_1 | _stacktrace=sys.exc_info()[2])
worker4_1 | File "/usr/local/lib/python2.7/site-packages/urllib3/util/retry.py", line 399, in increment
worker4_1 | raise MaxRetryError(_pool, url, error or ResponseError(cause))
worker4_1 | MaxRetryError: HTTPSConnectionPool(host='retail-sentry.apple.com', port=443): Max retries exceeded with url: /api/1/store/ (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))
worker4_1 | 06:28:18 [ERROR] sentry_sdk.errors: Internal error in sentry_sdk