MySQL connection error ?
Le 14/03/24, nous avons constaté des erreurs 500 de façon aléatoire sur app. Dans le dashboard, on a effectivement beaucoup d'erreurs :
On observe deux types d'erreurs :
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 803, in _write_bytes
self._sock.sendall(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 944, in wrapper
ret = func(*func_args, **func_kwargs)
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 929, in wrapper
raise context["exception"]
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 908, in wrapper
context["output"] = func(*args, **kwargs)
File "/usr/share/vidjil/server/py4web/apps/vidjil/modules/vidjil_utils.py", line 73, in wrapper
result = func(*args, **kwargs)
File "/usr/share/vidjil/server/py4web/apps/vidjil/controllers/sample_set.py", line 322, in all
log.info("%s list %s" % (type, search), extra={'user_id': auth.user_id,
File "/usr/lib/python3.8/logging/__init__.py", line 1806, in info
self.log(INFO, msg, *args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1844, in log
self.logger.log(level, msg, *args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1512, in log
self._log(level, msg, args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1589, in _log
self.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1599, in handle
self.callHandlers(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 954, in handle
self.emit(record)
File "/usr/share/vidjil/server/py4web/apps/vidjil/common.py", line 179, in emit
db[self.table].insert(
File "/usr/local/lib/python3.8/dist-packages/pydal/objects.py", line 892, in insert
ret = self._db._adapter.insert(self, row.op_values())
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 546, in insert
raise e
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 541, in insert
self.execute(query)
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/__init__.py", line 70, in wrap
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 468, in execute
rv = self.cursor.execute(command, *args[1:], **kwargs)
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 153, in execute
result = self._query(query)
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 322, in _query
conn.query(q)
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 557, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 861, in _execute_command
self._write_bytes(packet)
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 806, in _write_bytes
raise err.OperationalError(
pymysql.err.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))")
et
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 944, in wrapper
ret = func(*func_args, **func_kwargs)
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 929, in wrapper
raise context["exception"]
File "/usr/local/lib/python3.8/dist-packages/py4web/core.py", line 908, in wrapper
context["output"] = func(*args, **kwargs)
File "/usr/share/vidjil/server/py4web/apps/vidjil/modules/vidjil_utils.py", line 73, in wrapper
result = func(*args, **kwargs)
File "/usr/share/vidjil/server/py4web/apps/vidjil/controllers/sample_set.py", line 322, in all
log.info("%s list %s" % (type, search), extra={'user_id': auth.user_id,
File "/usr/lib/python3.8/logging/__init__.py", line 1806, in info
self.log(INFO, msg, *args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1844, in log
self.logger.log(level, msg, *args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1512, in log
self._log(level, msg, args, **kwargs)
File "/usr/lib/python3.8/logging/__init__.py", line 1589, in _log
self.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1599, in handle
self.callHandlers(record)
File "/usr/lib/python3.8/logging/__init__.py", line 1661, in callHandlers
hdlr.handle(record)
File "/usr/lib/python3.8/logging/__init__.py", line 954, in handle
self.emit(record)
File "/usr/share/vidjil/server/py4web/apps/vidjil/common.py", line 179, in emit
db[self.table].insert(
File "/usr/local/lib/python3.8/dist-packages/pydal/objects.py", line 892, in insert
ret = self._db._adapter.insert(self, row.op_values())
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 546, in insert
raise e
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 541, in insert
self.execute(query)
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/__init__.py", line 70, in wrap
return f(*args, **kwargs)
File "/usr/local/lib/python3.8/dist-packages/pydal/adapters/base.py", line 468, in execute
rv = self.cursor.execute(command, *args[1:], **kwargs)
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 153, in execute
result = self._query(query)
File "/usr/local/lib/python3.8/dist-packages/pymysql/cursors.py", line 322, in _query
conn.query(q)
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 557, in query
self._execute_command(COMMAND.COM_QUERY, sql)
File "/usr/local/lib/python3.8/dist-packages/pymysql/connections.py", line 840, in _execute_command
raise err.InterfaceError(0, "")
pymysql.err.InterfaceError: (0, '')
sur différentes pages.
En jouant avec app, une page peut faire une erreur 500 et au rechargement ne plus en faire. L'hypothèse que nous avons faite avec @fthonier serait qu'au moins un thread aurait perdu la connection mysql et ne la redémarre pas. Nous avons donc testé de redémarrer uwsgi, et pour le moment nous n'avons pas reproduit l'erreur.
cf en PJ:
- les logs docker autour du premier souci identifié (issue de la commande
docker logs -t vidjil_uwsgi --since 2024-03-14T07 --until 2024-03-14T09
)docker-uwsgi-logs.log - un extrait des logs debug vidjil autour des mêmes heurevidjil-debug-extract.log