Add local build logs

build.gradle

@@ -78,6 +78,7 @@ dependencies {
     implementation 'org.luaj:luaj-jse:3.0.1'
     //object storage dependency
     implementation 'io.micronaut.objectstorage:micronaut-object-storage-aws'
+    implementation 'io.micronaut.objectstorage:micronaut-object-storage-local'
     // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570
     // this sts dependency is require by micronaut-aws-parameter-store,
     // not directly used by the app, for this reason keeping `runtimeOnly`

src/main/groovy/io/seqera/wave/configuration/LogsConfig.groovy

@@ -0,0 +1,89 @@
+/*
+ *  Wave, containers provisioning service
+ *  Copyright (c) 2023-2024, Seqera Labs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package io.seqera.wave.configuration
+
+import java.nio.file.Files
+import java.nio.file.Path
+
+import groovy.transform.CompileStatic
+import groovy.transform.Memoized
+import groovy.transform.ToString
+import groovy.util.logging.Slf4j
+import io.micronaut.context.annotation.Requires
+import io.micronaut.context.annotation.Value
+import io.micronaut.core.annotation.Nullable
+/**
+ * Model build logs configuration settings
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+@Slf4j
+@Requires(property = 'wave.build.logs')
+@ToString(includePackage = false, includeNames = true)
+@CompileStatic
+class LogsConfig {
+
+    @Nullable
+    @Value('${wave.build.logs.prefix}')
+    private String prefix
+
+    @Nullable
+    @Value('${wave.build.logs.bucket}')
+    private String bucket
+
+    @Value('${wave.build.logs.maxLength:100000}')
+    private long maxLength
+
+    @Nullable
+    @Value('${wave.build.logs.conda-lock-prefix}')
+    private String condaLockPrefix
+
+    @Nullable
+    @Value('${wave.build.logs.local.path}')
+    private String localPath
+
+    String getPrefix() {
+        return prefix
+    }
+
+    String getBucket() {
+        return bucket
+    }
+
+    long getMaxLength() {
+        return maxLength
+    }
+
+    String getCondaLockPrefix() {
+        return condaLockPrefix
+    }
+
+    @Memoized
+    Path getLocalPath() {
+        if( !localPath )
+            return null
+        final result = Path.of(localPath)
+        try {
+            Files.createDirectories(result)
+        } catch (IOException e) {
+            log.warn ("Unable to create logs local storage path: $localPath - cause: ${e.message}")
+        }
+        return result
+    }
+}

src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy → src/main/groovy/io/seqera/wave/service/aws/AwsStorageOperationsFactory.groovy

@@ -22,33 +22,34 @@ import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Factory
 import io.micronaut.context.annotation.Requires
-import io.micronaut.context.annotation.Value
 import io.micronaut.objectstorage.InputStreamMapper
 import io.micronaut.objectstorage.ObjectStorageOperations
 import io.micronaut.objectstorage.aws.AwsS3Configuration
 import io.micronaut.objectstorage.aws.AwsS3Operations
+import io.seqera.wave.configuration.LogsConfig
+import jakarta.inject.Inject
 import jakarta.inject.Named
 import jakarta.inject.Singleton
 import software.amazon.awssdk.services.s3.S3Client
 /**
- * Factory implementation for ObjectStorageOperations
+ * Factory implementation for AWS ObjectStorageOperations
  *
  * @author Munish Chouhan <[email protected]>
  */
 @Factory
 @CompileStatic
 @Slf4j
 @Requires(property = 'wave.build.logs.bucket')
-class ObjectStorageOperationsFactory {
+class AwsStorageOperationsFactory {
 
-    @Value('${wave.build.logs.bucket}')
-    String storageBucket
+    @Inject
+    private LogsConfig logsConfig
 
     @Singleton
     @Named("build-logs")
     ObjectStorageOperations<?, ?, ?> awsStorageOperations(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) {
-        AwsS3Configuration configuration = new AwsS3Configuration('build-logs')
-        configuration.setBucket(storageBucket)
+        final configuration = new AwsS3Configuration('build-logs')
+        configuration.setBucket(logsConfig.bucket)
         return new AwsS3Operations(configuration, s3Client, inputStreamMapper)
     }
 }

src/main/groovy/io/seqera/wave/service/logs/BuildLogServiceImpl.groovy

@@ -21,18 +21,16 @@ package io.seqera.wave.service.logs
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.ExecutorService
 
-import io.micronaut.core.annotation.Nullable
-
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import io.micronaut.context.annotation.Requires
-import io.micronaut.context.annotation.Value
 import io.micronaut.http.server.types.files.StreamedFile
 import io.micronaut.objectstorage.ObjectStorageEntry
 import io.micronaut.objectstorage.ObjectStorageOperations
 import io.micronaut.objectstorage.request.UploadRequest
 import io.micronaut.runtime.event.annotation.EventListener
 import io.micronaut.scheduling.TaskExecutors
+import io.seqera.wave.configuration.LogsConfig
 import io.seqera.wave.service.builder.BuildEvent
 import io.seqera.wave.service.builder.BuildRequest
 import io.seqera.wave.service.persistence.PersistenceService
@@ -46,11 +44,12 @@ import static org.apache.commons.lang3.StringUtils.strip
  * Implements Service  to manage logs from an Object store
  *
  * @author Munish Chouhan <[email protected]>
+ * @author Paolo Di Tommaso <[email protected]>
  */
 @Slf4j
 @Singleton
 @CompileStatic
-@Requires(property = 'wave.build.logs.bucket')
+@Requires(bean = LogsConfig)
 class BuildLogServiceImpl implements BuildLogService {
 
     private static final String CONDA_LOCK_START = ">> CONDA_LOCK_START"
@@ -64,35 +63,24 @@ class BuildLogServiceImpl implements BuildLogService {
     @Inject
     private PersistenceService persistenceService
 
-    @Nullable
-    @Value('${wave.build.logs.prefix}')
-    private String prefix
-
-    @Value('${wave.build.logs.bucket}')
-    private String bucket
-
-    @Value('${wave.build.logs.maxLength:100000}')
-    private long maxLength
-
-    @Nullable
-    @Value('${wave.build.logs.conda-lock-prefix}')
-    private String condaLockPrefix
+    @Inject
+    private LogsConfig config
 
     @Inject
     @Named(TaskExecutors.IO)
     private volatile ExecutorService ioExecutor
 
     @PostConstruct
     private void init() {
-        log.info "Creating Build log service bucket=$bucket; logs prefix=$prefix; maxLength: ${maxLength}; condaLock prefix=$condaLockPrefix"
+        log.info "Creating Build log service - config=${config}"
     }
 
     protected String logKey(String buildId) {
         if( !buildId )
             return null
-        if( !prefix )
+        if( !config.prefix )
             return buildId + '.log'
-        final base = strip(prefix, '/')
+        final base = strip(config.prefix, '/')
         return "${base}/${buildId}.log"
     }
 
@@ -137,8 +125,8 @@ class BuildLogServiceImpl implements BuildLogService {
         final result = fetchLogStream(buildId)
         if( !result )
             return null
-        final logs = new BoundedInputStream(result.getInputStream(), maxLength).getText()
-        return new BuildLog(logs, logs.length()>=maxLength)
+        final logs = new BoundedInputStream(result.getInputStream(), config.maxLength).getText()
+        return new BuildLog(logs, logs.length()>=config.maxLength)
     }
 
     protected static removeCondaLockFile(String logs) {
@@ -166,9 +154,9 @@ class BuildLogServiceImpl implements BuildLogService {
     protected String condaLockKey(String buildId) {
         if( !buildId )
             return null
-        if( !condaLockPrefix )
+        if( !config.condaLockPrefix )
             return buildId + '.lock'
-        final base = strip(condaLockPrefix, '/')
+        final base = strip(config.condaLockPrefix, '/')
         return "${base}/${buildId}.lock"
     }
 
@@ -178,7 +166,6 @@ class BuildLogServiceImpl implements BuildLogService {
         if( !result )
             return null
         return result.getInputStream().getText()
-
     }
 
     @Override

src/main/groovy/io/seqera/wave/service/logs/LocalStorageOperationsFactory.groovy

@@ -0,0 +1,54 @@
+/*
+ *  Wave, containers provisioning service
+ *  Copyright (c) 2023-2024, Seqera Labs
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package io.seqera.wave.service.logs
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import io.micronaut.context.annotation.Factory
+import io.micronaut.context.annotation.Requires
+import io.micronaut.objectstorage.ObjectStorageOperations
+import io.micronaut.objectstorage.local.LocalStorageConfiguration
+import io.micronaut.objectstorage.local.LocalStorageOperations
+import io.seqera.wave.configuration.LogsConfig
+import jakarta.inject.Inject
+import jakarta.inject.Named
+import jakarta.inject.Singleton
+/**
+ * Factory implementation for local ObjectStorageOperations
+ *
+ * @author Paolo Di Tommaso <[email protected]>
+ */
+@Factory
+@CompileStatic
+@Slf4j
+@Requires(property = 'wave.build.logs.local.path')
+class LocalStorageOperationsFactory {
+
+    @Inject
+    private LogsConfig logsConfig
+
+    @Singleton
+    @Named("build-logs")
+    ObjectStorageOperations<?, ?, ?> create() {
+        final configuration = new LocalStorageConfiguration('build-logs')
+        configuration.setEnabled(true)
+        configuration.setPath(logsConfig.localPath)
+        return new LocalStorageOperations(configuration)
+    }
+}

src/main/resources/application-dev.yml

@@ -6,6 +6,11 @@ wave:
       duration: '30s'
   build:
     workspace: 'build-workspace'
+    logs:
+      local:
+        path: "build-workspace"
+      prefix: 'wave-build/logs'
+      conda-lock-prefix: 'wave-build/conda-locks'
   metrics:
     enabled: true
   accounts: