Develop Reference

Split log file in Spring Boot Gradle App Batchwise

I am working on Spring Boot Gradle App and I am facing issues while generating the log files.
Currently my app is generating 1 big log file and my spring boot uses below logback-spring.xml file.
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE configuration>
<configuration>
<springProperty scope="context" name="app_name" source="spring.application.name"/>
<springProperty scope="context" name="environment" source="param_env" defaultValue="default_env"/>
<springProperty scope="context" name="tenant" source="param_tenant" defaultValue="default_tenant"/>
<property name="GENERIC_PATTERN"
value="[%date{ISO8601}] [%p] [%t] [%c{2}] %4L | %m%n"/>
<property name="CONSOLE_LOG_PATTERN" value="$GENERIC_PATTERN"/>
<property name="CHARSET" value="UTF-8"/>
<property name="LOG_DIR"
value="logs/${tenantname:-${tenant}}/${envname:-${environment}}/${param_gslogname:-${app_name}}"/>
<logger name="org.springframework" level="INFO"/>
<logger name="org.springframework.web" level="INFO"/>
<logger name="org.gs" level="INFO"/>
<logger name="com.gs" level="INFO"/>
<logger name="liquibase" level="FINE"/>
<logger name="org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor"
level="INFO"/>
<logger name="org.springframework.security" level="INFO"/>
<logger name="org.hibernate" level="INFO"/>
<!-- Auto configuration report-->
<logger name="org.springframework.boot.autoconfigure" level="DEBUG"/>
<logger name="org.springframework.boot.autoconfigure.logging.ConditionEvaluationReportLoggingListener"
level="TRACE"/>
<springProfile name="local,component-test,component-test-security,scheduler-test">
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>${CHARSET}</charset>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<logger name="zuul.web.request.logger" level="DEBUG" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
</root>
</springProfile>
<!-- logging configuration for NOT LOCAL profile -->
<springProfile name="dev, test, uat, preprod, prod, perf">
<appender name="FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc/>
<pattern>
<omitEmptyFields>true</omitEmptyFields>
<pattern>
{
"app_name": "${app_name}",
"HOSTNAME":"${HOSTNAME}",
"environment":"${environment}"
}
</pattern>
</pattern>
<logLevel/>
<timestamp/>
<threadName/>
<logstashMarkers/>
<tags/>
<arguments/>
<version/>
<stackTrace/>
<loggerName/>
<threadName/>
<message/>
</providers>
</encoder>
</appender>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>${CHARSET}</charset>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<appender name="CONSOLE_JSON" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>timestamp</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<appender name="API_FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/api-${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/api-${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>[ignore]</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<appender name="APPLICATION_FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/startup-${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/startup-${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>timestamp</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<logger name="api.logger" level="INFO" additivity="false">
<appender-ref ref="API_FILE_APPENDER"/>
</logger>
<logger name="application.startup.logger" level="INFO" additivity="false">
<appender-ref ref="APPLICATION_FILE_APPENDER"/>
</logger>
<!-- Added because of version 1.7.30 and we can't upgrade because our server is not supporting latest version-->
<appender name="SENTRY" class="io.sentry.logback.SentryAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
<encoder>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE_APPENDER"/>
<appender-ref ref="SENTRY"/>
</root>
</springProfile>
</configuration>
My app works in way that whenever new request comes it generates 1 batch id for 1 request and all the processing stuffs or info logs are being written to one log file called gs_log_file.
and likewise when there is another request ie the 2nd request then 2nd batch id is being generated and all the processing stuffs or info logs are being written to the same log file as mentioned in point no. 1 (gs_log_file). and so on for subsequent requests the logs are being written to the same log file (gs_log_file).
So my issue is how can I generate the batchwise log file for each request or the batch id.
e.g
gs_log_file_batchId1 for batch id 1(request1).
gs_log_file_batchId2 for batch id 2(request2).
gs_log_file_batchId3 for batch id 3(request3).
gs_log_file_batchIdn for batch id n(requestn).
my app api is like this with pathVariable batchJobId -> /process-batch-jobs/{batchId}
How can I improve my implementation so that my app will put the log info in particular log file separately according to per request or per batch.
Should I go for multi-threading, again not sure on this part.
Or Should I manually process the batch id and according to that batch id I should generate the files and then log data into that log file
Once I generate the log files like gs_log_file_batchId1 for batch id 1(request1) and gs_log_file_batchId2 for batch id 2(request2), I should be also able to read this log file via UI...so I ll search these log file from UI based on some batch id or request or some customer id, here batchid can be pathvariable or request param or some sort of filter.
Also please let me know how can I externalize the configuration so that I can supply some argument to app so that app will generate the log file smartly based on supplied argument like log file size or the no of lines in log file?
Can anyone please help me on this?

Changing logback.xml file in Jenkins without rebuilding code

I have configured logback.xml to change log level during build time using "scan" property. Using this I can change log level without rebuilding the code. I can change log level by updating logback.xml file
<?xml version="1.0" encoding="UTF-8"?><!--
For more configuration information and examples see
http://logback.qos.ch/manual/configuration.html-->
<configuration scan="true" scanPeriod="10 seconds">
<!--<statusListener class="ch.qos.logback.core.status.OnConsoleStatusListener" />-->
<!-- Debugging appender (duplicates the normal log, PLUS any debug messages) -->
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%date{ISO8601} %level{5} %c{3} --- %message%n</pattern>
</encoder>
</appender>
<!---->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>./log/elk-file.log</file>
<append>true</append>
<!--cleanHistoryOnStart>true</cleanHistoryOnStart-->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>./log/elk-file_%d{yyyyMMdd}-%i.log
</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>200MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>1</maxHistory>
</rollingPolicy>
<encoder>
<pattern>%date{ISO8601} [%thread] %level{5} %c{3} - %message%n</pattern>
</encoder>
</appender>
<!-- Our logger writes to file, console and sends the data to Logstash -->
<logger name="ro.fortsoft.elk.testdata" level="DEBUG" additivity="false">
<appender-ref ref="STASH"/>
</logger>
<logger level="INFO" name="rollingFileLogger">
<appender-ref ref="FILE" />
</logger>
<!-- ROOT logger setup -->
<root level="DEBUG">
<appender-ref ref="CONSOLE"/>
</root>
How can I do this in Jenkins, i-e change logback.xml and change log level without rebuilding the code.

how to print logback internal errors and status logs?

I am using logback api and have a logback.xml in my classpath which looks like this
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration scan="true">
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender class="ch.qos.logback.core.ConsoleAppender" name="STDOUT">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE">
<file>/${path}/logs/application.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>/${path}/logs/application.%i.log</fileNamePattern>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
</encoder>
</appender>
<!-- ~~~ PERFORMANCE TRACKING LOGGER CONFIGURATION (USING PERF4J) || END ||~~~
-->
<logger name="com.nucleus">
<level value="DEBUG"/>
<!-- <appender-ref ref="STDOUT" /> -->
<appender-ref ref="FILE"/>
</logger>
<logger level="DEBUG" name="org.hibernate.transaction.JDBCTransaction"/>
<logger level="DEBUG" name="org.hibernate.jdbc.ConnectionManager"/>
<logger level="DEBUG" name="org.springframework.orm.jpa.JpaTransactionManager"/>
<!-- <root level="info">
<appender-ref ref="FILE" />
</root> -->
</configuration>
Now the rollback that i have implemented is not working in the production environment only. I wish to debug the same and hence want to put a trace for this logback api. Can anyone suggest what i might be doing wrong or what should i do to resolve this issue?

From Logback 1.0.4 you can use system property -Dlogback.debug=true to enable debugging of your Logback setup which will allow you to debug your configuration.

How to have different logging for different packages?

In Log4j it is easy to have different packages in your application being logged at different levels (this is especially useful for debugging). Is there an easy way of doing this other than creating your own filter?
I tried creating an tag within the root tag or after whose name is the package (ie reminiscent of Log4J) and then placed the appenders for it within but, but it didn't work.
So, is there a built in to LogBack way of defining loggers and/or logging levels for certain packages that does not require a custom filter implementation?

I'm using a Play! web app and have different log levels for different packages. You can specify the package in the name of the <logger> element.
My config is:
<configuration>
<conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel"/>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>${application.home}/logs/application.log</file>
<encoder>
<pattern>%date - [%level] - from %logger in %thread %n%message%n%xException%n</pattern>
</encoder>
</appender>
<appender name="TIMESTAMP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${application.home}/logs/batches/current.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy" maxHistory="720">
<fileNamePattern>${application.home}/logs/batches/archived/%d{yyyy/MM/dd}.gz</fileNamePattern>
</rollingPolicy>
<encoder>
<pattern>{mpoplib} %date - [%level] in %thread %n%message%n%xException%n</pattern>
</encoder>
</appender>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%coloredLevel %logger{15} - %message%n%xException{5}</pattern>
</encoder>
</appender>
<logger name="play" level="INFO">
<appender-ref ref="FILE"/>
</logger>
<logger name="application" level="INFO">
<appender-ref ref="FILE"/>
</logger>
<logger name="ru.kupikupon.mpoplib" level="DEBUG">
<appender-ref ref="TIMESTAMP_FILE"/>
</logger>
<root level="DEBUG">
<appender-ref ref="STDOUT"/>
</root>
</configuration>

Logback to log different messages to two files

I am using logback/slf4j to do my logging. I want to parse my log file to analyze some data, so instead of parsing a great big file (mostly consisting of debug statements) I want to have two logger instances which each log to a separate file; one for analytics and one for all purpose logging. Does anyone know if this is possible with Logback, or any other logger for that matter?

It's very possible to do something like this in logback. Here's an example configuration:
<?xml version="1.0"?>
<configuration>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>logfile.log</file>
<append>true</append>
<encoder>
<pattern>%-4relative [%thread] %-5level %logger{35} - %msg %n</pattern>
</encoder>
</appender>
<appender name="ANALYTICS-FILE" class="ch.qos.logback.core.FileAppender">
<file>analytics.log</file>
<append>true</append>
<encoder>
<pattern>%-4relative [%thread] %-5level %logger{35} - %msg %n</pattern>
</encoder>
</appender>
<!-- additivity=false ensures analytics data only goes to the analytics log -->
<logger name="analytics" level="DEBUG" additivity="false">
<appender-ref ref="ANALYTICS-FILE"/>
</logger>
<root>
<appender-ref ref="FILE"/>
</root>
</configuration>
Then you'd setup two separate loggers, one for everything and one to log analytics data like so:
Logger analytics = LoggerFactory.getLogger("analytics");

You can have as many loggers as you wish. But, it's better you have one for each package that you need to log differently. Then all the classes in that package and its sub-packages will get the that specific logger. They all can share the root logger and send their log data to root logger appender using additivity="true". Here's an example:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<property name="pattern" value="%date{HH:mm:ss.SSS} %-5p %logger{36}
%X{akkaSource} [%file:%line] - %m%n" />
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%date{HH:mm:ss.SSS} %-5p %logger{36} %X{akkaSource} [%file:%line] - %m%n</pattern>
</encoder>
</appender>
<appender name="abc" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${catalina.base}/logs/worker.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${catalina.base}/logs/worker-%d{yyyy-MM-dd_HH}.log</fileNamePattern>
<maxHistory>360</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${pattern}</pattern>
</encoder>
</appender>
<appender name="xyz" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${catalina.base}/logs/transformer.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${catalina.base}/logs/transformer-%d{yyyy-MM-dd_HH}.log</fileNamePattern>
<maxHistory>360</maxHistory>
</rollingPolicy>
<encoder>
<pattern>${pattern}</pattern>
</encoder>
</appender>
<logger name="com.xxx.yyy.zzz" level="INFO" additivity="true">
<appender-ref ref="xyz"/>
</logger>
<logger name="com.aaa.bbb.ccc" level="INFO" additivity="true">
<appender-ref ref="abc"/>
</logger>
<root>
<level value="INFO" />
<appender-ref ref="STDOUT" />
</root>

in my case I wanted to leave class names as log name
private static final Logger log = LoggerFactory.getLogger(ScheduledPost.class);
and as I had few such classes, so my logback.xml
<!--additivity=false ensures this log data only goes to the this log, and no one more -->
<logger name="xxx.xxx.xxx.ScheduledPost" level="DEBUG" additivity="false">
<appender-ref ref="ASYNC_SCHEDULE_LOG_FILE"/>
</logger>
<logger name="xxx.xxx.xxx.GcmPost" level="DEBUG" additivity="false">
<appender-ref ref="ASYNC_SCHEDULE_LOG_FILE"/>
</logger>
<logger name="xxx.xxx.xxx.PushUtils" level="DEBUG" additivity="false">
<appender-ref ref="ASYNC_SCHEDULE_LOG_FILE"/>
</logger>