From 4e13585d632cd9cf91eef934b4506f19ec7bcf15 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen Date: Wed, 27 Oct 2021 17:02:43 +0300 Subject: [PATCH 1/3] add more logging to table trigger setup process. Add lower than default timeout for trigger setup and log warning upon timeout. --- services/data/postgres_async_db.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/services/data/postgres_async_db.py b/services/data/postgres_async_db.py index bf951d90..6cb02617 100644 --- a/services/data/postgres_async_db.py +++ b/services/data/postgres_async_db.py @@ -1,3 +1,4 @@ +import asyncio import psycopg2 import psycopg2.extras import os @@ -366,7 +367,9 @@ async def create_if_missing(db: _AsyncPostgresDB, table_name, command): @staticmethod async def create_trigger_if_missing(db: _AsyncPostgresDB, table_name, trigger_name, commands=[]): "executes the commands only if a trigger with the given name does not already exist on the table" - with (await db.pool.cursor()) as cur: + # NOTE: keep a relatively low timeout for the setup queries, + # as these should not take long to begin with, and we want to release as soon as possible in case of errors + with (await db.pool.cursor(timeout=10)) as cur: try: await cur.execute( """ @@ -378,9 +381,14 @@ async def create_trigger_if_missing(db: _AsyncPostgresDB, table_name, trigger_na (table_name, trigger_name), ) trigger_exist = bool(cur.rowcount) - if not trigger_exist: + if trigger_exist: + db.logger.info("Trigger {} already exists, skipping.".format(trigger_name)) + else: + db.logger.info("Creating trigger for table: {}".format(trigger_name)) for command in commands: await cur.execute(command) + except asyncio.TimeoutError: + db.logger.warning("Trigger creation timed out, no triggers were set up for table: {}".format(table_name)) finally: cur.close() From 1337175d6635a08b1708d0f0cd1d63b370c536c4 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen Date: Wed, 27 Oct 2021 17:47:11 +0300 Subject: [PATCH 2/3] change timeout setup to also include setting serverside statement_timeout --- services/data/postgres_async_db.py | 5 +++-- services/utils/__init__.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/services/data/postgres_async_db.py b/services/data/postgres_async_db.py index 6cb02617..03d5a0a7 100644 --- a/services/data/postgres_async_db.py +++ b/services/data/postgres_async_db.py @@ -80,6 +80,7 @@ async def _init(self, db_conf: DBConfiguration, create_triggers=DB_TRIGGER_CREAT minsize=db_conf.pool_min, maxsize=db_conf.pool_max, timeout=db_conf.timeout, + options=db_conf.options, echo=AIOPG_ECHO) for table in self.tables: @@ -369,7 +370,7 @@ async def create_trigger_if_missing(db: _AsyncPostgresDB, table_name, trigger_na "executes the commands only if a trigger with the given name does not already exist on the table" # NOTE: keep a relatively low timeout for the setup queries, # as these should not take long to begin with, and we want to release as soon as possible in case of errors - with (await db.pool.cursor(timeout=10)) as cur: + with (await db.pool.cursor()) as cur: try: await cur.execute( """ @@ -387,7 +388,7 @@ async def create_trigger_if_missing(db: _AsyncPostgresDB, table_name, trigger_na db.logger.info("Creating trigger for table: {}".format(trigger_name)) for command in commands: await cur.execute(command) - except asyncio.TimeoutError: + except (asyncio.CancelledError, asyncio.TimeoutError): db.logger.warning("Trigger creation timed out, no triggers were set up for table: {}".format(table_name)) finally: cur.close() diff --git a/services/utils/__init__.py b/services/utils/__init__.py index baf2e5e5..baff42a7 100644 --- a/services/utils/__init__.py +++ b/services/utils/__init__.py @@ -169,6 +169,7 @@ class DBConfiguration(object): pool_max: int = None # aiopg default: 10 timeout: int = None # aiopg default: 60 (seconds) + options: str = None # psycopg2 configuration options _dsn: str = None @@ -182,7 +183,9 @@ def __init__(self, prefix="MF_METADATA_DB_", pool_min: int = 1, pool_max: int = 10, - timeout: int = 60): + timeout: int = 60, + statement_timeout: int = None): + self.options = "-c statement_timeout={}".format(statement_timeout * 1000) if statement_timeout else None self._dsn = os.environ.get(prefix + "DSN", dsn) # Check if it is a BAD DSN String. From 4d95f14611c2ce03ff9dc5910d21a2542cc88ac8 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen Date: Wed, 27 Oct 2021 17:54:36 +0300 Subject: [PATCH 3/3] setup serverside query timeout of 60 seconds for ui_server --- services/ui_backend_service/ui_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/ui_backend_service/ui_server.py b/services/ui_backend_service/ui_server.py index 7245f50b..1812d14e 100644 --- a/services/ui_backend_service/ui_server.py +++ b/services/ui_backend_service/ui_server.py @@ -129,7 +129,7 @@ def main(): for sig in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): loop.add_signal_handler(sig, lambda sig=sig: async_loop_signal_handler(sig)) - the_app = app(loop, DBConfiguration()) + the_app = app(loop, DBConfiguration(statement_timeout=60)) handler = web.AppRunner(the_app) loop.run_until_complete(handler.setup()) f = loop.create_server(handler.server, DEFAULT_SERVICE_HOST, DEFAULT_SERVICE_PORT)