Develop Reference

Split log file in Spring Boot Gradle App Batchwise

I am working on Spring Boot Gradle App and I am facing issues while generating the log files.
Currently my app is generating 1 big log file and my spring boot uses below logback-spring.xml file.
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE configuration>
<configuration>
<springProperty scope="context" name="app_name" source="spring.application.name"/>
<springProperty scope="context" name="environment" source="param_env" defaultValue="default_env"/>
<springProperty scope="context" name="tenant" source="param_tenant" defaultValue="default_tenant"/>
<property name="GENERIC_PATTERN"
value="[%date{ISO8601}] [%p] [%t] [%c{2}] %4L | %m%n"/>
<property name="CONSOLE_LOG_PATTERN" value="$GENERIC_PATTERN"/>
<property name="CHARSET" value="UTF-8"/>
<property name="LOG_DIR"
value="logs/${tenantname:-${tenant}}/${envname:-${environment}}/${param_gslogname:-${app_name}}"/>
<logger name="org.springframework" level="INFO"/>
<logger name="org.springframework.web" level="INFO"/>
<logger name="org.gs" level="INFO"/>
<logger name="com.gs" level="INFO"/>
<logger name="liquibase" level="FINE"/>
<logger name="org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor"
level="INFO"/>
<logger name="org.springframework.security" level="INFO"/>
<logger name="org.hibernate" level="INFO"/>
<!-- Auto configuration report-->
<logger name="org.springframework.boot.autoconfigure" level="DEBUG"/>
<logger name="org.springframework.boot.autoconfigure.logging.ConditionEvaluationReportLoggingListener"
level="TRACE"/>
<springProfile name="local,component-test,component-test-security,scheduler-test">
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>${CHARSET}</charset>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<logger name="zuul.web.request.logger" level="DEBUG" additivity="false">
<appender-ref ref="CONSOLE"/>
</logger>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
</root>
</springProfile>
<!-- logging configuration for NOT LOCAL profile -->
<springProfile name="dev, test, uat, preprod, prod, perf">
<appender name="FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
<providers>
<mdc/>
<pattern>
<omitEmptyFields>true</omitEmptyFields>
<pattern>
{
"app_name": "${app_name}",
"HOSTNAME":"${HOSTNAME}",
"environment":"${environment}"
}
</pattern>
</pattern>
<logLevel/>
<timestamp/>
<threadName/>
<logstashMarkers/>
<tags/>
<arguments/>
<version/>
<stackTrace/>
<loggerName/>
<threadName/>
<message/>
</providers>
</encoder>
</appender>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<charset>${CHARSET}</charset>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<appender name="CONSOLE_JSON" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>timestamp</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<appender name="API_FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/api-${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/api-${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>[ignore]</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<appender name="APPLICATION_FILE_APPENDER" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/startup-${param_gslogname:-${app_name}}-${HOSTNAME}.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>
${LOG_DIR}/startup-${param_gslogname:-${app_name}}-${HOSTNAME}.%d{yyyy-MM-dd}.%i.log
</fileNamePattern>
<maxFileSize>1GB</maxFileSize>
<totalSizeCap>10GB</totalSizeCap>
<!-- keep 30 days worth of history -->
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder class="net.logstash.logback.encoder.LogstashEncoder">
<fieldNames>
<timestamp>timestamp</timestamp>
<message>logdata</message>
<version>[ignore]</version>
<levelValue>[ignore]</levelValue>
</fieldNames>
<timeZone>UTC</timeZone>
</encoder>
</appender>
<logger name="api.logger" level="INFO" additivity="false">
<appender-ref ref="API_FILE_APPENDER"/>
</logger>
<logger name="application.startup.logger" level="INFO" additivity="false">
<appender-ref ref="APPLICATION_FILE_APPENDER"/>
</logger>
<!-- Added because of version 1.7.30 and we can't upgrade because our server is not supporting latest version-->
<appender name="SENTRY" class="io.sentry.logback.SentryAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
<encoder>
<pattern>${GENERIC_PATTERN}</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE_APPENDER"/>
<appender-ref ref="SENTRY"/>
</root>
</springProfile>
</configuration>
My app works in way that whenever new request comes it generates 1 batch id for 1 request and all the processing stuffs or info logs are being written to one log file called gs_log_file.
and likewise when there is another request ie the 2nd request then 2nd batch id is being generated and all the processing stuffs or info logs are being written to the same log file as mentioned in point no. 1 (gs_log_file). and so on for subsequent requests the logs are being written to the same log file (gs_log_file).
So my issue is how can I generate the batchwise log file for each request or the batch id.
e.g
gs_log_file_batchId1 for batch id 1(request1).
gs_log_file_batchId2 for batch id 2(request2).
gs_log_file_batchId3 for batch id 3(request3).
gs_log_file_batchIdn for batch id n(requestn).
my app api is like this with pathVariable batchJobId -> /process-batch-jobs/{batchId}
How can I improve my implementation so that my app will put the log info in particular log file separately according to per request or per batch.
Should I go for multi-threading, again not sure on this part.
Or Should I manually process the batch id and according to that batch id I should generate the files and then log data into that log file
Once I generate the log files like gs_log_file_batchId1 for batch id 1(request1) and gs_log_file_batchId2 for batch id 2(request2), I should be also able to read this log file via UI...so I ll search these log file from UI based on some batch id or request or some customer id, here batchid can be pathvariable or request param or some sort of filter.
Also please let me know how can I externalize the configuration so that I can supply some argument to app so that app will generate the log file smartly based on supplied argument like log file size or the no of lines in log file?
Can anyone please help me on this?

Logback logging maven multi-module

I'm developing an application made of multiple modules that will be deployed on wildfly 13. One of these modules is using another of my project as a jar maven dependency.(included to pom)
Expectation
I want my dependency using its own logback.xml to log in its own file. And I want the application using its own logback.xml file to log in the console and a separate file than the dependency.
What it does now
For the moment both the application module (com.test.app.console.ca.operation)
which include the jar dependency and the dependency use the dependency's logback.xml and everything is logged in the same file. This is very strange to me because of other modules from the main application (which dont have dependecy from the library jar), are correctly logging in the right file.
Could please help me to understand and solve this problem?
Details about the projects
Both use logback as a logger. The dependency is a security implementation that logs communication information in a file that must be in a separate file than application logs. Both the application and the dependency have a classic maven structure with the logback.xml file inside the resource folder.
The main modules logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="true">
<property resource="application.properties" />
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder
by default -->
<encoder>
<pattern>%d{ISO8601} [%thread] %-5level %logger - %msg %n</pattern>
</encoder>
</appender>
<!--Application Log (Daily rolling file appender) -->
<appender name="appCaLog"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>${jboss.server.log.dir}/main_modules_app.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${jboss.server.log.dir}/main_modules_app.log.%d{yyyy-MM-dd}.log
</FileNamePattern>
</rollingPolicy>
<encoder>
<pattern>%d{ISO8601} [%thread] %-5level %logger - %msg %n</pattern>
</encoder>
</appender>
<logger name="com.test.app.console.ca.usermanagement"
level="${cops.usermanagement.log.level}" additivity="false">
<appender-ref ref="appCaLog" />
</logger>
<logger name="com.test.app.console.ca.operation"
level="${cops.operation.log.level}" additivity="false">
<appender-ref ref="appCaLog" />
</logger>
<logger name="com.test.app.console.ca"
level="${cops.main.log.level}" additivity="false">
<appender-ref ref="appCaLog" />
</logger>
<root level="INFO">
<appender-ref ref="appCaLog" />
</root>
</configuration>
The dependency logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="true">
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder
by default -->
<encoder>
<pattern>%d{ISO8601} [%thread] %-5level %logger - %msg %n</pattern>
</encoder>
</appender>
<!--Application Log (Daily rolling file appender) -->
<appender name="dependencyCaCryptoLog"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<File>${jboss.server.log.dir}/dependency_module.log</File>
<rollingPolicy
class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${jboss.server.log.dir}/dependency_module.log.%d{yyyy-MM-dd}.log
</FileNamePattern>
</rollingPolicy>
<encoder>
<pattern>%d{ISO8601} [%thread] %-5level %logger - %msg %n</pattern>
</encoder>
</appender>
<logger name="com.test.app.console.ca.crypto"
level="INFO" additivity="false">
<appender-ref ref="dependencyCaCryptoLog" />
</logger>
<root level="INFO">
<appender-ref ref="dependencyCaCryptoLog" />
</root>
</configuration>

How to log slf4j to a file in cuba Framework

I'm trying to configure cuba framework in order to write the logs in a file, but at the moment I cannot.
I have in the java files:
private static final Logger LOG = LoggerFactory.getLogger(BlisterauftragServiceImpl.class);
LOG.info("This is a, info log");
In the build.gradle file I have:
logbackConfigurationFile = 'etc/war-logback.xml'
Then in the folder etc, I have the file war-logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" packagingData="true">
<property name="logDir" value="${app.home}/logs"/>
<appender name="File" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${logDir}/app.log</file>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>${logDir}/app.%d{yyyy-MM-dd}.log</fileNamePattern>
<!-- keep 30 days' worth of history -->
<maxHistory>30</maxHistory>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread%X{cubaApp}%X{cubaUser}] %logger - %msg%n</pattern>
</encoder>
</appender>
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root>
<appender-ref ref="Console"/>
<appender-ref ref="File"/>
</root>
<!-- Begin CUBA -->
<logger name="com.haulmont.cuba" level="DEBUG"/>
<logger name="com.haulmont.cuba.core.sys" level="INFO"/>
<logger name="com.haulmont.cuba.core.sys.CubaDefaultListableBeanFactory" level="WARN"/>
<logger name="com.haulmont.cuba.core.app.scheduling" level="INFO"/>
<logger name="com.haulmont.cuba.web.sys" level="INFO"/>
<logger name="com.haulmont.cuba.portal" level="INFO"/>
<logger name="com.haulmont.restapi.sys" level="INFO"/>
<logger name="com.haulmont.cuba.core.app.LockManager" level="INFO"/>
<!-- End CUBA -->
<logger name="eclipselink" level="WARN"/>
<logger name="eclipselink.sql" level="INFO"/>
<logger name="org.springframework" level="WARN"/>
<logger name="org.activiti" level="INFO"/>
<logger name="freemarker" level="INFO"/>
<logger name="org.thymeleaf.TemplateEngine" level="INFO"/>
<logger name="org.docx4j" level="WARN"/>
<logger name="org.xlsx4j" level="WARN"/>
<logger name="org.hibernate" level="WARN"/>
<logger name="sun" level="INFO"/>
<logger name="com.sun" level="INFO"/>
<logger name="javax" level="INFO"/>
<logger name="org.apache" level="INFO"/>
<logger name="org.atmosphere" level="INFO"/>
<logger name="org.docx4j.utils.ResourceUtils" level="ERROR"/>
<logger name="org.docx4j.Docx4jProperties" level="ERROR"/>
<logger name="org.xlsx4j.jaxb.Context" level="ERROR"/>
<!-- Begin Perf4J -->
<appender name="PerfStatFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${logDir}/perfstat.log</file>
<append>true</append>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${logDir}/perfstat.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>30</maxHistory>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>%msg%n</pattern>
</encoder>
</appender>
<appender name="CoalescingStatistics" class="org.perf4j.logback.AsyncCoalescingStatisticsAppender">
<param name="TimeSlice" value="60000"/>
<appender-ref ref="PerfStatFile"/>
</appender>
<appender name="UIPerfStatFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${logDir}/perfstat-ui.log</file>
<append>true</append>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${logDir}/perfstat-ui.%d{yyyy-MM-dd}.log</fileNamePattern>
<maxHistory>30</maxHistory>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>%msg%n</pattern>
</encoder>
</appender>
<appender name="UICoalescingStatistics" class="org.perf4j.logback.AsyncCoalescingStatisticsAppender">
<param name="TimeSlice" value="120000"/>
<appender-ref ref="UIPerfStatFile"/>
</appender>
<logger name="org.perf4j.TimingLogger" additivity="false" level="INFO">
<appender-ref ref="CoalescingStatistics"/>
</logger>
<logger name="com.haulmont.cuba.gui.logging.UIPerformanceLogger" additivity="false" level="INFO">
<appender-ref ref="UICoalescingStatistics"/>
</logger>
<!-- End Perf4J -->
</configuration>
If I change anything in this file, it don't do anything. For example I have tried to change:
<file>${logDir}/app.log</file>
to:
<file>${logDir}/app1.log</file>
Then in th folder deploy/tomcat/conf, I find a file logback.xml. In the folder deploy/tomcat/logs, I will find all my logs. I can change the conf files and that will work fine. But my file war-logback is not taken into account.
Then my problem is that deploy folder is created a overrited each time new. Then I have to rewrite le logback.xml file each time the code is regenerated.
Any ideas why is it so?
Thanks
Best regards

logback tries to find configuration as the fllowing steps:
Logback tries to find a file called logback-test.xml in the classpath.
If no such file is found, logback tries to find a file called logback.groovy in the classpath.
If no such file is found, it checks for the file logback.xml in the classpath.
If no such file is found, service-provider loading facility (introduced in JDK 1.6) is used to resolve the implementation of com.qos.logback.classic.spi.Configurator interface by looking up the file META-INF\services\ch.qos.logback.classic.spi.Configurator in the class path. Its contents should specify the fully qualified class name of the desired Configurator implementation.
If none of the above succeeds, logback configures itself automatically using the BasicConfigurator which will cause logging output to be directed to the console.
and you will find it here
so, let's make sure the configuration file in your war is named "logback-test.xml","logback.groovy" or "logback.xml", maybe that's why "war-logback.xml" was ignored.

If you need to permanently override the development logging configuration, then
create a file <project_name>/etc/logback.xml and put your configuration in it.
After running the deploy task, the file will be copied to <project_folder>/deploy/app_home/logback.xml.

Adding timestamp to a log file using Logback-test.xml

Currently my Spring-boot application logs to a file named: myLog.log, this is working as intended, however I would like the log file to have a timestamp at the end of it and create a new file each time it is ran.
I have tried to implement this in my logback-test.xml file shown below, but it is just giving me the filename: myLog.log without a timestamp.
How can I fix this?
Logback-test.xml:
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<include resource="org/springframework/boot/logging/logback/base.xml"/>
<logger name="org.springframework.web" level="INFO"/>
<!-- Send debug messages to System.out -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- By default, encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>path/to/my/file/mylog.log</file>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<Pattern>%d{yyyy-MM-dd_HH:mm:ss.SSS} - %msg%n</Pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<FileNamePattern>mylog.%i{yyyy-MM-dd_HH:mm:ss.SSS}}.log</FileNamePattern>
<MinIndex>1</MinIndex>
<MaxIndex>10</MaxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<MaxFileSize>2MB</MaxFileSize>
</triggeringPolicy>
</appender>
<logger name="com.my.package" level="INFO" additivity="false">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</logger>
<!-- By default, the level of the root level is set to DEBUG -->
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>

You can define a variable like this:
<timestamp key="myTimestamp" datePattern="yyyy-MM-dd'_'HH-mm-ss.SSS"/>
(note: don't use colons in datePattern)
Then use it directly in your appender's file element:
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>path/to/my/file/mylog-${myTimestamp}.log</file>
...
</appender>
Or in a simple FileAppender:
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>path/to/my/file/mylog-${myTimestamp}.log</file>
<encoder>
<Pattern>%d{yyyy-MM-dd_HH:mm:ss.SSS} - %msg%n</Pattern>
</encoder>
</appender>

how to print logback internal errors and status logs?

I am using logback api and have a logback.xml in my classpath which looks like this
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<configuration scan="true">
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<appender class="ch.qos.logback.core.ConsoleAppender" name="STDOUT">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="FILE">
<file>/${path}/logs/application.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>/${path}/logs/application.%i.log</fileNamePattern>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>%date %level [%thread] %logger{10} [%file:%line] %msg%n</pattern>
</encoder>
</appender>
<!-- ~~~ PERFORMANCE TRACKING LOGGER CONFIGURATION (USING PERF4J) || END ||~~~
-->
<logger name="com.nucleus">
<level value="DEBUG"/>
<!-- <appender-ref ref="STDOUT" /> -->
<appender-ref ref="FILE"/>
</logger>
<logger level="DEBUG" name="org.hibernate.transaction.JDBCTransaction"/>
<logger level="DEBUG" name="org.hibernate.jdbc.ConnectionManager"/>
<logger level="DEBUG" name="org.springframework.orm.jpa.JpaTransactionManager"/>
<!-- <root level="info">
<appender-ref ref="FILE" />
</root> -->
</configuration>
Now the rollback that i have implemented is not working in the production environment only. I wish to debug the same and hence want to put a trace for this logback api. Can anyone suggest what i might be doing wrong or what should i do to resolve this issue?

From Logback 1.0.4 you can use system property -Dlogback.debug=true to enable debugging of your Logback setup which will allow you to debug your configuration.