ISO8601 | Ruby parser to work with ISO8601 dateTimes | Date Time Utils library

services:
  tomcat:
    container_name: tomcat
    image: tomcat:10.0-jdk11-openjdk
    restart: always
    
    ports: 
      - "8080:8080"
    networks:
      static:
        ipv4_address: 192.168.20.10
    volumes:
    - '/home/jason/development/projects/docker/java-dev-environment/volumes/tomcat/usr_local_tomcat_conf:/usr/local/tomcat/conf'
    - '/home/jason/development/projects/docker/java-dev-environment/volumes/tomcat/usr_local_tomcat_webapps:/usr/local/tomcat/webapps'
    - '/home/jason/development/projects/docker/java-dev-environment/volumes/tomcat/usr_local_tomcat_logs:/usr/local/tomcat/logs'

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <property name="LOGS" value="/usr/local/tomat/logs/gliese" />

    <include resource="org/springframework/boot/logging/logback/base.xml" />

    <appender name="Console"
              class="ch.qos.logback.core.ConsoleAppender">
        <layout class="ch.qos.logback.classic.PatternLayout">
            <Pattern>
                %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable
            </Pattern>
        </layout>
    </appender>

    <appender name="RollingFile"
              class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>${LOGS}/gliese.log</file>
    <encoder
            class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
        <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
    </encoder>
    <rollingPolicy
            class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
        <!-- rollover daily and when the file reaches 10 MegaBytes -->
        <fileNamePattern>${LOGS}/gliese-%d{yyyy-MM-dd}.%i.log
        </fileNamePattern>
        <timeBasedFileNamingAndTriggeringPolicy
                class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
            <maxFileSize>10MB</maxFileSize>
        </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    </appender>

    <!-- LOG everything at INFO level -->
    <root level="info">
        <appender-ref ref="RollingFile" />
        <appender-ref ref="Console" />
    </root>
</configuration>

x <- as.POSIXct("2015-06-01 12:00:00", tz = "EST")
x
#> [1] "2015-06-01 12:00:00 EST"

lubridate::tz(x)
#> [1] "EST"

tmpfile <- tempfile()

write.csv(data.frame(x = x), tmpfile)
df <- read.csv(tmpfile)

df
#>   X                   x
#> 1 1 2015-06-01 12:00:00

lubridate::tz(df$x)
#> [1] "UTC"

readr::write_csv(tibble::tibble(x = x), tmpfile)
df <- readr::read_csv(tmpfile)
#> Rows: 1 Columns: 1
#> ── Column specification ────────────────────────────────────────────────────────
#> Delimiter: ","
#> dttm (1): x
#> 
#> ℹ Use `spec()` to retrieve the full column specification for this data.
#> ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

df
#> # A tibble: 1 × 1
#>   x                  
#>   <dttm>             
#> 1 2015-06-01 17:00:00

lubridate::tz(df$x)
#> [1] "UTC"

data.table::fwrite(data.table::data.table(x = x), tmpfile)
df <- data.table::fread(tmpfile)

df
#>                      x
#> 1: 2015-06-01 17:00:00

lubridate::tz(df$x)
#> [1] "UTC"

df <- read_csv2(my_file,
                col_types = cols(.default = '?',
                                 my_date = col_date(format = '%d.%m.%Y')))

locale(date_format = '%d.%m.%Y')

df <- read_csv2(my_file,
                col_types = cols(.default = '?',
                                 my_date = 'D'))

expected: <chr> date in ISO8601
actual: <chr> 01.01.2000

input{
  file{
  path => "/path/to/log/file"
  start_position => "beginning"
  }
}
filter{
  grok{
  match => {"message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:log-level} %{(?<function_name>[^.]*)\.(?<elapsed_time>[^.]*)\.(?<input>[^.]*)}"}
  }
    date {
    match => ["timestamp", "ISO8601"]
    }
    function_name {
    match => ["function_name", "DATA"]
    }
    elapsed_time {
    match => ["elapsed_time", "BASE16FLOAT"]
    }
    input {
    match => ["input", "DATA"]
    }
}
output{
  elasticsearch{
  hosts => ["localhost:9200"]
  index => "math_apis"
  }
  stdout{codec => rubydebug}
}

{
          "host" => "hostname",
    "@timestamp" => 2022-02-11T06:27:49.404Z,
       "message" => "2022-02-11 11:57:49 - app - INFO - function_name=add elapsed_time=0.0296 input_params=6_3",
          "path" => "path/to/log/file",
      "@version" => "1",
          "tags" => [
        [0] "_grokparsefailure"
    ]
}

2022-01-21 12:25:01,339 {"category":"runtime","some_id":"order","correlation_id":"OEID_1","servid":"143","provision_id":"898769049","operation_name":"CREATE", "processing_state":"ACTIVE","lifecycle_state":"ACTIVE","created":"2022-01-21 12:25:00,369","changed":"2022-01-21 12:25:00,806","runtime":"0.437"}

filter {

  grok {
    match => { message => "^%{TIMESTAMP_ISO8601:logdate}%{SPACE}*%{DATA:json}$" }
    add_tag => [ "matched", "provisioning_runtime" ]
  }

  json {
    source => "json"
    add_tag => [ "json" ]
  }

  # matcher for the @timestamp
  date {
    match => [ "logdate", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS"]
  }

log4j.rootLogger=INFO, file

log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=/var/log/mylog.out
log4j.appender.file.MaxFileSize=10KB
log4j.appender.file.MaxBackupIndex=2
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %-5p %t %c - %m%n

-rw-r----- 1 root root 7.6K Jan 10 12:27 test.log
-rw-r----- 1 root root 11K Jan 10 12:27 test.log.1
-rw-r----- 1 root root 36K Jan 10 12:27 test.log.2
-rw-r----- 1 root root 11K Jan 10 12:27 test.log2022-01-10

library(lubridate)

w <- paste("2021-W", 46:52, sep = "")
w1 <-c("2022-W01")
ww <-c(w,w1)

df_ww<- format_ISO8601(as.Date(ww,format = "%YYYY-W%w“))

str(ww)
 chr [1:8] "2021-W46" "2021-W47" "2021-W48" "2021-W49" "2021-W50" "2021-W51" "2021-W52" "2022-W01“

df_ww
[1] NA NA NA NA NA NA NA NA

    public class LocalTestAppHost : AppSelfHostBase
    {
        public LocalTestAppHost() : base(nameof(LocalTestAppHost), typeof(MyServices).Assembly, typeof(LocalTestAppHost).Assembly) { }


        public override void Configure(Container container)
        {
            var builder = new ConfigurationBuilder()
                .SetBasePath(Directory.GetCurrentDirectory())
                // .AddJsonFile("appSettings.json")
                .AddEnvironmentVariables()
                .AddUserSecrets(typeof(IntegrationTest).Assembly);
            
            var configuration = builder.Build();
            
            AppSettings = new NetCoreAppSettings(configuration);

            container.AddSingleton<IAppSettings>(AppSettings);

            SetConfig(new HostConfig
            {
                AddRedirectParamsToQueryString = true,
                DebugMode = true
            });
            
            Plugins.Add(new CorsFeature(allowOriginWhitelist: new[] { IntegrationTestBase.BaseUriLocalDev },
                allowedMethods: "GET, PATCH, POST, PUT, DELETE, OPTIONS",
                allowCredentials: true,
                allowedHeaders: "Content-Type, Allow, Authorization"));
            
            JsConfig.DateHandler = DateHandler.ISO8601;
            
            var connectionString = AppSettings.GetString("DefaultConnection");

            OrmLiteConfig.StripUpperInLike = false;

            container.AddSingleton<IDbConnectionFactory>(new OrmLiteConnectionFactory(connectionString, PostgreSqlDialect.Provider));
            
            container.AddSingleton<IAuthRepository>(c =>
                new OrmLiteAuthRepository<UserAuthCustom, UserAuthDetails>(c.Resolve<IDbConnectionFactory>())
                {
                    UseDistinctRoleTables = true
                });
            
            Plugins.Add(new AuthFeature(() => new CustomUserSession(),
                new IAuthProvider[] {
                    new CredentialsAuthProvider(AppSettings)
                    {
                        
                    },     /* Sign In with Username / Password credentials */
                }));
            
            Plugins.Add(new AdminUsersFeature());
            
            
        }
    }

    public class ReportTests : IntegrationTestBase
    {
        private string _adminUser;
        private string _adminPass;
        public ReportTests()
        {
            Licensing.RegisterLicense(Licence);
            this.AppHost = new LocalTestAppHost()
                .Init()
                .Start(BaseUriLocalDev); 
            
            Settings = AppHost.Resolve<IAppSettings>();
            Db = AppHost.Resolve<IDbConnectionFactory>().OpenDbConnection();

            _adminUser = Settings.GetString("adminUser");
            _adminPass = Settings.GetString("adminPass");
        }

        [Test]
        public void TestStats()
        {
            var users = Db.Select<UserAuthCustom>();
            var client = new JsonServiceClient(BaseUriLocalDev);

            var authReq = new Authenticate()
            {
                UserName = _adminUser,
                Password = _adminPass,
                provider = CredentialsAuthProvider.Name
            };

            var resp = client.Post(authReq);
        }

System.Net.WebException: Received an invalid status line: '400'.
 ---> System.Net.Http.HttpRequestException: Received an invalid status line: '400'.
   at System.Net.Http.HttpConnection.ParseStatusLine(Span`1 line, HttpResponseMessage response)
   at System.Net.Http.HttpConnection.SendAsyncCore(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
   at System.Net.Http.HttpConnection.SendAsyncCore(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
   at System.Net.Http.HttpConnectionPool.SendWithRetryAsync(HttpRequestMessage request, Boolean async, Boolean doRequestAuth, CancellationToken cancellationToken)
   at System.Net.Http.RedirectHandler.SendAsync(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
   at System.Net.Http.DecompressionHandler.SendAsync(HttpRequestMessage request, Boolean async, CancellationToken cancellationToken)
   at System.Net.Http.HttpMessageHandlerStage.Send(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.SocketsHttpHandler.Send(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.HttpClientHandler.Send(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.HttpMessageInvoker.Send(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.SendAsyncCore(HttpRequestMessage request, HttpCompletionOption completionOption, Boolean async, Boolean emitTelemetryStartStop, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.Send(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at System.Net.HttpWebRequest.SendRequest(Boolean async)
   at System.Net.HttpWebRequest.GetResponse()
   --- End of inner exception stack trace ---
   at System.Net.HttpWebRequest.GetResponse()
   at ServiceStack.ServiceClientBase.Send[TResponse](String httpMethod, String relativeOrAbsoluteUrl, Object request) in C:\BuildAgent\work\3481147c480f4a2f\src\ServiceStack.Client\ServiceClientBase.cs:line 1416
   at ServiceStack.ServiceClientBase.Post[TResponse](IReturn`1 requestDto) in C:\BuildAgent\work\3481147c480f4a2f\src\ServiceStack.Client\ServiceClientBase.cs:line 1581
   at LeadInput.Tests.IntegrationTests.ReportTests.TestStats() in D:\Clients\LeadInput\LeadInput.Tests\IntegrationTests\ReportTests.cs:line 43

Spring version : 5.3.13
Log4j version : 2.17.0
java : 1.8

<?xml version="1.0" encoding="UTF-8"?>
<Configuration monitorInterval="1">
    <Properties>
        
        <Property name="log-path">${appconfig:log_path}</Property>
        <Property name="log-name">${appconfig:filename}</Property>
        <Property name="archive-days">${appconfig:archive_days}</Property>
        <Property name="file-level">${appconfig:file_level}</Property>
        <Property name="console-level">${appconfig:console_level}</Property>
         
        
    </Properties>
    <Appenders>

        <Routing name="route-log">
            <Routes pattern="${ctx:routingLogFile}">
                
                <Route>
                    <RollingFile name="default-log" fileName="${log-path}/${log-name}.log"
                                 filePattern="${log-path}/${date:yyyy-MM}/${log-name}.%d{MM-dd-yyyy}-%i.log.gz" append="true">
                        <PatternLayout
                                pattern="%d{MM/dd/yyyy HH:mm:ss.SSS z} %X{machine-name} %X{app-name} [%t] %-5level %logger{36}:%-3L - %msg%n" />
                        <Policies>
                            <TimeBasedTriggeringPolicy />
                            <SizeBasedTriggeringPolicy size="150 MB"/>
                        </Policies>
                        <DefaultRolloverStrategy max="1000">
                            <Delete basePath="${log-path}/" maxDepth="2">
                                <IfFileName glob="/${log-name}*.log.gz" />
                                <IfLastModified age="${archive-days}" />
                            </Delete>
                        </DefaultRolloverStrategy>
                    </RollingFile >
                </Route>
        
            </Routes>
                
        </Routing>
        <Console name="STDOUT" target="SYSTEM_OUT">
            <PatternLayout pattern="%d{ISO8601} %-5level %30.30logger{1.}:%-3L - %m%n%throwable" />
        </Console>
    </Appenders>
    
    <Loggers>
        <Logger name="org.springframework" level="ERROR"/>
         <Logger name="org.apache" level="ERROR"/>
        <Root level="${file-level}" additivity="false">
            <AppenderRef ref="route-log" />
            <AppenderRef ref="STDOUT" />
        </Root>
    </Loggers>
    
</Configuration>

import org.apache.logging.log4j.core.LogEvent;
import org.apache.logging.log4j.core.config.plugins.Plugin;
import org.apache.logging.log4j.core.lookup.AbstractLookup;
import org.apache.logging.log4j.core.lookup.StrLookup;
import org.appconfig.properties.ApplicationProperties;
import org.springframework.util.StringUtils;

@Plugin(name = "appconfig", category = StrLookup.CATEGORY)
public class AppLog4JConfigDatabaseLookup extends AbstractLookup {

    public String lookup(final LogEvent event, final String key) {
        
        
        if (key.equalsIgnoreCase("filename")) {
            return ApplicationProperties.getLogFilename();
        }
        if (key.equalsIgnoreCase("log_path")) {
            return ApplicationProperties.getLogPath();
        }
        if (key.equalsIgnoreCase("file_level")) {
            return ApplicationProperties.getFileLogLevel();
        }
        if (key.equalsIgnoreCase("console_level")) {
            return ApplicationProperties.getConsoleLogLevel();
        }
        if (key.equalsIgnoreCase("app_name")) {
            return ApplicationProperties.getAppName();
        }
        if (key.equalsIgnoreCase("archive_days")) {
            return ApplicationProperties.getLogArchiveDays();
        }
        
        return key;
    }
}

public class LoaderJob extends SchedulerAdapterJob {
    @Autowired
    private FileLoaderJob fileLoaderJob;
    
    private static final Logger LOGGER = LogManager.getLogger(LoaderJob.class);
    @Override
    public void executeJob(JobExecutionContext jobExecutionContext) {
        ThreadContext.put("routingLogFile","LOADER_LOGS");
        try {
            fileLoaderJob.execute();        
            
        }
        catch (Exception e) {
            LOGGER.error("Exception in  "+getJobName()+" : "+ e.getMessage());
            throw e;
        }
        finally {
              ThreadContext.remove("routingLogFile");
        }       
        
    }

}