Evict wave usage metrics after 120 days

build.gradle

@@ -63,7 +63,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     implementation 'com.squareup.moshi:moshi:1.15.1'
     implementation 'com.squareup.moshi:moshi-adapters:1.15.1'
-    implementation 'redis.clients:jedis:5.1.3'
+    implementation 'redis.clients:jedis:5.2.0'
     implementation 'io.github.resilience4j:resilience4j-ratelimiter:0.17.0'
     implementation 'io.micronaut:micronaut-retry'
     // caching deps

src/main/groovy/io/seqera/wave/service/counter/AbstractCounterStore.groovy

@@ -55,4 +55,9 @@ abstract class AbstractCounterStore implements CounterStore {
     Map<String, Long> getAllMatchingEntries(String pattern) {
         provider.getAllMatchingEntries(getPrefix(), pattern)
     }
+
+    @Override
+    void deleteAllMatchingEntries(String pattern) {
+        provider.getAllMatchingEntries(getPrefix(), pattern)
+    }
 }

src/main/groovy/io/seqera/wave/service/counter/CounterStore.groovy

@@ -34,4 +34,10 @@ interface CounterStore {
      * @return all the entries whose field matches 'pattern'
      */
     Map<String, Long> getAllMatchingEntries(String pattern)
+
+    /**
+     * @param pattern
+     * @return void
+     */
+    void deleteAllMatchingEntries(String pattern)
 }

src/main/groovy/io/seqera/wave/service/counter/impl/CounterProvider.groovy

@@ -35,4 +35,5 @@ interface CounterProvider {
      * @return all the entries whose field matches 'pattern'
      */
     Map<String, Long> getAllMatchingEntries(String key, String pattern)
+
 }

src/main/groovy/io/seqera/wave/service/counter/impl/LocalCounterProvider.groovy

@@ -61,4 +61,5 @@ class LocalCounterProvider implements CounterProvider {
         }
         return result
     }
+
 }

src/main/groovy/io/seqera/wave/service/counter/impl/RedisCounterProvider.groovy

@@ -18,6 +18,8 @@
 
 package io.seqera.wave.service.counter.impl
 
+import java.time.Duration
+
 import groovy.transform.CompileStatic
 import io.micronaut.context.annotation.Requires
 import io.micronaut.context.annotation.Value
@@ -42,10 +44,21 @@ class RedisCounterProvider implements CounterProvider {
     @Value('${redis.hscan.count:10000}')
     private Integer hscanCount
 
+    @Value('${redis.key.expiry:1s}')
+    private Duration keyExpiry
+
+    final private static String DATE_PATTERN = /\b\d{4}-\d{2}-\d{2}\b/
+
+    static boolean isMetricPerDate(String field) {
+        return field =~ DATE_PATTERN
+    }
     @Override
     long inc(String key, String field, long value) {
         try(Jedis conn=pool.getResource() ) {
-            return conn.hincrBy(key, field, value)
+            long count =  conn.hincrBy(key, field, value)
+            if(isMetricPerDate(field))
+                conn.hexpire(key, keyExpiry.toSeconds(), field)
+            return count
         }
     }
 
@@ -70,4 +83,5 @@ class RedisCounterProvider implements CounterProvider {
             return result
         }
     }
+
 }

src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy

@@ -38,7 +38,8 @@ class RedisCounterProviderTest extends Specification implements RedisTestContain
     def setup() {
         applicationContext = ApplicationContext.run([
                 REDIS_HOST : redisHostName,
-                REDIS_PORT : redisPort
+                REDIS_PORT : redisPort,
+                'redis.key.expiry': '1s'
         ], 'test', 'redis')
         redisCounterProvider = applicationContext.getBean(RedisCounterProvider)
         sleep(500) // workaround to wait for Redis connection
@@ -85,9 +86,22 @@ class RedisCounterProviderTest extends Specification implements RedisTestContain
         then:
         redisCounterProvider.getAllMatchingEntries('metrics/v1', 'pulls/o/*') ==
                 ['pulls/o/abc.in':3, 'pulls/o/bar.es':2, 'pulls/o/foo.it':1, 'pulls/o/abc.com.au/d/2024-05-30':1, 'pulls/o/abc.com.au/d/2024-05-31':1]
+    }
+
+    def 'should expire the hash'(){
+        when:
+        redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au', 1)
+        redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-14', 1)
+        sleep(500)
+        redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-15', 1)
+        sleep(500)
+        then:'this value should be one, because foo should be expired'
+        redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-14') == null
+        sleep(500)
         and:
-        redisCounterProvider.getAllMatchingEntries('metrics/v1', 'pulls/o/*/d/2024-05-30') ==
-                ['pulls/o/abc.com.au/d/2024-05-30':1]
+        redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-15') == null
+        and: 'this value should be one, because org metric should not expire'
+        redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au') == 1
     }
 
     def 'should get correct org count for mirror and scan' () {

src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy

@@ -44,7 +44,7 @@ trait RedisTestContainer {
 
     def setupSpec() {
         log.debug "Starting Redis test container"
-        redisContainer = new GenericContainer(DockerImageName.parse("redis:7.0.4-alpine"))
+        redisContainer = new GenericContainer(DockerImageName.parse("redis:7.4.0-alpine"))
                 .withExposedPorts(6379)
                 .waitingFor(Wait.forLogMessage(".*Ready to accept connections.*\\n", 1))
         redisContainer.start()