Correçẽos para evitar exceptions por causa do redis;

Simplificado o config para carregar dinamicamente via env;
Helper atualizado para ter novas funções compatíveis com o life-cycle;
Biblioteca de logging melhorada como novos métodos;
Melhorias no boot para registrar a vendor de forma automática para o projeto poder ficar no padrão da pep8;

Author: andersoncontreira

examples/lambda_sqs/app.py

@@ -2,17 +2,18 @@
 
 This module contains the handler method
 """
+import boot
 from lambda_app.lambda_flask import LambdaFlask
 from lambda_app import APP_NAME, APP_VERSION
-from lambda_app.logging import get_logger
+from lambda_app.logging import get_logger, set_debug_mode
 from lambda_app.services.v1.carrier_notifier_service import CarrierNotifierService
 from lambda_app.config import get_config
 from lambda_app import helper
-from lambda_app.boot import load_dot_env
+
 
 # load env
 ENV = helper.get_environment()
-load_dot_env(ENV)
+boot.load_dot_env(ENV)
 
 # config
 CONFIG = get_config()
@@ -22,6 +23,12 @@
 LOGGER = get_logger()
 
 APP = LambdaFlask(APP_NAME)
+# override the APP logger
+APP.logger = LOGGER
+# override the log configs
+if DEBUG:
+    # override to the level desired
+    set_debug_mode(LOGGER)
 
 # general vars
 APP_QUEUE = CONFIG.APP_QUEUE

examples/lambda_sqs/lambda_app/boot.py examples/lambda_sqs/boot.pyexamples/lambda_sqs/lambda_app/boot.py renamed to examples/lambda_sqs/boot.py

@@ -14,6 +14,7 @@
 
 ROOT_DIR = current_path
 _LOADED = False
+_ENV_KEYS = []
 
 try:
     import chalicelib
@@ -41,8 +42,14 @@ def is_loaded():
 
 
 def reset():
-    global _LOADED
+    global _LOADED, _ENV_KEYS
     _LOADED = False
+    _ENV_KEYS = []
+
+
+def get_env_keys():
+    global _ENV_KEYS
+    return _ENV_KEYS
 
 
 def get_internal_logger():
@@ -65,12 +72,13 @@ def load_dot_env(env='development', force=False):
 
     logger = get_internal_logger()
 
-    global _LOADED
+    global _LOADED, _ENV_KEYS
     if not _LOADED or force:
         logger.info('Boot - Loading env: {}'.format(env))
 
         # Default
         for k, v in _DEFAULT_ENV_CONFIGS.items():
+            _ENV_KEYS.append(k)
             if k == 'APP_ENV':
                 v = env
             os.environ[k] = v
@@ -80,6 +88,7 @@ def load_dot_env(env='development', force=False):
             env_vars = dotenv_values(config_path)
 
             for k, v in env_vars.items():
+                _ENV_KEYS.append(k)
                 os.environ[k] = v
             _LOADED = True
         else:
@@ -99,6 +108,7 @@ def load_dot_env(env='development', force=False):
 
 
 def load_secrets(env='staging'):
+    global _ENV_KEYS
     from lambda_app.aws.secrets import Secrets
     logger = get_internal_logger()
     result = False
@@ -128,13 +138,14 @@ def load_env(env='dev', force=False):
 
     logger = get_internal_logger()
 
-    global _LOADED
+    global _LOADED, _ENV_KEYS
     if not _LOADED or force:
 
         logger.info('Boot - Loading env: {}'.format(env))
 
         # Default
         for k, v in _DEFAULT_ENV_CONFIGS.items():
+            _ENV_KEYS.append(k)
             if k == 'APP_ENV':
                 v = env
             os.environ[k] = v
@@ -156,6 +167,7 @@ def load_env(env='dev', force=False):
             if env in configs['stages']:
                 env_vars = configs['stages'][env]['environment_variables']
                 for k, v in env_vars.items():
+                    _ENV_KEYS.append(k)
                     os.environ[k] = v
                 _LOADED = True
             else:
@@ -196,3 +208,6 @@ def print_env(app, logger):
     # logger.info('Database: %s' % os.getenv('DB_HOST'))
     logger.info('Log Level: %s' % os.getenv('LOG_LEVEL'))
     logger.info('Debug: %s' % os.getenv('DEBUG'))
+
+
+register_vendor()

examples/lambda_sqs/lambda_app/config.py

@@ -10,55 +10,19 @@
 
 class Configuration:
     # APP
-    APP_ENV = 'development'
     APP_NAME = ''
     APP_VERSION = ''
 
-    # LOG
-    LOG_LEVEL = logging.INFO
-
-    # NEW RELIC
-    NEW_RELIC_DEVELOPER_MODE = 'development'
-    NEW_RELIC_LICENSE_KEY = "license_key"
-    NEW_RELIC_LOG_HOST = "https://log-api.newrelic.com/log/v1"
-
-    DB_HOST = ""
-    DB_USER = ""
-    DB_PASSWORD = ""
-    DB = ""
-
-    APP_QUEUE = ""
-
-    REDIS_HOST = ""
-    REDIS_PORT = 6379
-    REGION_NAME = ""
-
-    SQS_ENDPOINT = None
-    SQS_LOCALSTACK = "http://localhost:4566"
-    SECRET_KEY = 'teste'
-
     def __init__(self):
         # APP
-        self.APP_ENV = os.getenv("APP_ENV") if 'APP_ENV' in os.environ else 'development'
         self.APP_NAME = APP_NAME
         self.APP_VERSION = APP_VERSION
-        self.SECRET_KEY = os.getenv("SECRET_KEY") if 'SECRET_KEY' in os.environ else self.SECRET_KEY
 
-        self.LOG_LEVEL = os.getenv("LOG_LEVEL") if 'LOG_LEVEL' in os.environ else self.LOG_LEVEL
-
-        self.DB_HOST = os.getenv("DB_HOST") if 'DB_HOST' in os.environ else self.DB_HOST
-        self.DB_USER = os.getenv("DB_USER") if 'DB_USER' in os.environ else self.DB_USER
-        self.DB_PASSWORD = os.getenv("DB_PASSWORD") if 'DB_PASSWORD' in os.environ else self.DB_PASSWORD
-        self.DB = os.getenv("DB") if 'DB' in os.environ else self.DB
-
-        self.APP_QUEUE = os.getenv("APP_QUEUE") if 'APP_QUEUE' in os.environ else self.APP_QUEUE
-
-        self.REDIS_HOST = os.getenv("REDIS_HOST") if 'REDIS_HOST' in os.environ else self.REDIS_HOST
-        self.REDIS_PORT = os.getenv("REDIS_PORT") if 'REDIS_PORT' in os.environ else self.REDIS_PORT
-        self.REGION_NAME = os.getenv("REGION_NAME") if 'REGION_NAME' in os.environ else self.REGION_NAME
-
-        self.SQS_ENDPOINT = os.getenv("SQS_ENDPOINT") if 'SQS_ENDPOINT' in os.environ else self.SQS_ENDPOINT
-        self.SQS_LOCALSTACK = os.getenv("SQS_LOCALSTACK") if 'SQS_LOCALSTACK' in os.environ else self.SQS_LOCALSTACK
+        # montagem dinamica de atributos
+        env_keys = Configuration.get_env_keys()
+        for k in env_keys:
+            value = os.getenv(k) if k in os.environ else None
+            setattr(Configuration, k, value)
 
     def __dict__(self):
         attributes = inspect.getmembers(self, lambda a: not (inspect.isroutine(a)))
@@ -67,6 +31,11 @@ def __dict__(self):
     def to_dict(self):
         return self.__dict__()
 
+    @staticmethod
+    def get_env_keys():
+        from boot import get_env_keys
+        return get_env_keys()
+
 
 def reset():
     global _CONFIG

examples/lambda_sqs/lambda_app/helper.py

@@ -7,7 +7,7 @@
 
 import pytz
 
-from lambda_app.logging import get_logger
+from lambda_app.logging import get_logger, get_console_logger
 
 TZ_AMERICA_SAO_PAULO = 'America/Sao_Paulo'
 
@@ -64,7 +64,6 @@ def to_dict(obj, force_str=False):
     if force_str:
         return {k: str(v) for k, v in data.items() if v is not None}
     else:
-        # return {k: v for k, v in data.items() if v is not None}
         _dict = {}
         for k, v in data.items():
             if isinstance(v, Enum):
@@ -146,12 +145,19 @@ def is_https():
     return result
 
 
-def print_routes(app, logger):
+def is_count_request(app):
+    request = app.current_request.query_params
+    return True if request is not None and (request.get('count') == "true" or request.get('count') == "1") else False
+
+
+def print_routes(app, logger=None):
     """
     :param logger:
     :param (chalice.Chalice) app:
     :return:
     """
+    if logger is None:
+        logger = get_console_logger()
     logger.info('List of routes:')
 
     if has_attr(app, 'get_routes'):

examples/lambda_sqs/lambda_app/logging.py

@@ -14,6 +14,25 @@ def get_log_level():
     return log_level
 
 
+def get_console_logger():
+    log_level = get_log_level()
+    log_name = "console-{}".format(APP_NAME)
+    log_filename = None
+    log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    logging.basicConfig(format=log_format, filename=log_filename, level=log_level)
+    logger = logging.getLogger(log_name)
+    return logger
+
+
+def get_stream_handler():
+    return get_console_logger().parent.handlers[0]
+
+
+def set_debug_mode(logger, level=None):
+    logger.addHandler(get_stream_handler())
+    logger.level = level if level is not None else logging.INFO
+
+
 def get_logger():
     global _LOGGER
     if not _LOGGER:

examples/lambda_sqs/lambda_app/repositories/mysql/__init__.py

@@ -28,8 +28,12 @@ def _execute(self, sql, params=None):
             self.connection.connect()
             # with self.connection.cursor() as cursor:
             cursor = self.connection.cursor()
-            cursor.execute(sql, params)
-            result = cursor
+            try:
+                cursor.execute(sql, params)
+            except Exception as err:
+                self.logger.error(err)
+            finally:
+                result = cursor
             # close connection only in read
             # self.connection.close()
         else:

examples/lambda_sqs/lambda_app/services/v1/carrier_notifier_service.py

@@ -31,7 +31,7 @@ def process(self, sqs_event):
 
         event_tracker = EventTracker(self.logger)
         event_hash = None
-
+        # todo extrair essa logica daqui, o servico nao tem que fazer isso
         records = self.get_records_from_sqs_event(sqs_event)
         if records is not None:
             process_counter = 0
@@ -55,7 +55,11 @@ def process(self, sqs_event):
                     self.logger.info('event_vo: {}'.format(event_vo.to_dict()))
                     created = self.repository.create(event_vo)
 
-                    self.redis_repository.create("event_{}".format(event_hash), str(event_vo.to_dict()))
+                    try:
+                        self.redis_repository.create("event_{}".format(event_hash), str(event_vo.to_dict()))
+                    except Exception as err:
+                        self.logger.error("Already in redis")
+                        self.logger.error(err)
                     # self.redis_repository.list(where="*")
 
                     if not created:

examples/lambda_sqs/scripts/boot-lambda.sh

@@ -18,7 +18,6 @@ else
 fi
 
 current_file_path_basename=$(basename -- "$current_file_path")
-#echo "xxxxx current_file_path_basename $current_file_path_basename"
 
 if [ -z "$current_file_path_basename" ] || [ $current_file_path = "./" ]; then
 #  echo 'aq'
@@ -50,14 +49,12 @@ echo 'Installing dependencies...'
 echo "Requirements file: ${current_parent_folder}requirements.txt"
 if test -f ${current_parent_folder}requirements.txt; then
   python3 -m pip install -r ${current_parent_folder}requirements.txt -t ${current_parent_folder}vendor
-#  cat ${current_parent_folder}requirements.txt
   echo "requirements..."
 fi
 
 echo "Requirements file: ${current_parent_folder}requirements-vendor.txt"
 if test -f ${current_parent_folder}requirements-vendor.txt; then
   python3 -m pip install -r ${current_parent_folder}requirements-vendor.txt -t ${current_parent_folder}vendor
-#  cat ${current_parent_folder}requirements-vendor.txt
   echo "requirements vendor..."
 fi
 

examples/lambda_sqs/scripts/migrations/mysql/migrate.py

@@ -36,7 +36,7 @@ def register_paths():
 register_paths()
 
 from lambda_app.logging import get_logger
-from lambda_app.boot import reset, load_dot_env, load_env
+from boot import reset, load_dot_env, load_env
 from lambda_app.config import reset as reset_config, get_config
 from lambda_app.database.mysql import get_connection
 

examples/lambda_sqs/tests/__init__.py

@@ -1,7 +1,7 @@
 import os
 import sys
 
-from lambda_app.boot import load_env, set_root_dir
+from boot import load_env, set_root_dir
 
 if __package__:
     current_path = os.path.abspath(os.path.dirname(__file__)).replace('/' + str(__package__), '', 1)