Comicvine Cache

README.md

@@ -175,6 +175,8 @@ metadataProviders:
   comicVineApiKey: # required for comicVine provider https://comicvine.gamespot.com/api/ env:KOMF_METADATA_PROVIDERS_COMIC_VINE_API_KEY
   comicVineSearchLimit: # define ComicVine search result Limit, default is 10
   comicVineIssueName: # string that contains "{number}" which will be replaced by the issue number ie. "Issue #{number}". Used when an issue has no name on ComicVine, default is null
+  cacheDatabaseFile: # cache database file location. default is "./cv_cache.db"
+  cacheDatabaseExpiry: # number of days after which an entry in the cache is considered expired. default is 14
   comicVineIdFormat: # string that contains "{id}" which will serve to parse the ComicVine volume of a given book from its title or folder name ie. "[cv-{id}]" which will correctly identify '.../Uncanny X-Men Omnibus (2006) [cv-27512]' as being [4050-27512](https://comicvine.gamespot.com/uncanny-x-men-omnibus/4050-27512/)
   bangumiToken: # bangumi provider require a token to show nsfw items https://next.bgm.tv/demo/access-token  env:KOMF_METADATA_PROVIDERS_BANGUMI_TOKEN
   defaultProviders:

komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt

@@ -25,6 +25,8 @@ data class MetadataProvidersConfig(
     val defaultProviders: ProvidersConfig = ProvidersConfig(),
     val libraryProviders: Map<String, ProvidersConfig> = emptyMap(),
     val mangabakaDatabaseDir: String = "./mangabaka",
+    val cacheDatabaseFile: String = "./cv_cache.db",
+    val cacheDatabaseExpiry: Int = 14,
 )
 
 @Serializable

komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt

@@ -108,6 +108,8 @@ class ProvidersModule(
             comicVineIssueName = config.comicVineIssueName,
             comicVineIdFormat = config.comicVineIdFormat,
             bangumiToken = config.bangumiToken,
+            cacheDatabaseFile = config.cacheDatabaseFile,
+            cacheDatabaseExpiry = config.cacheDatabaseExpiry,
         )
         val libraryProviders = config.libraryProviders
             .map { (libraryId, libraryConfig) ->
@@ -120,6 +122,8 @@ class ProvidersModule(
                     comicVineIssueName = config.comicVineIssueName,
                     comicVineIdFormat = config.comicVineIdFormat,
                     bangumiToken = config.bangumiToken,
+                    cacheDatabaseFile = config.cacheDatabaseFile,
+                    cacheDatabaseExpiry = config.cacheDatabaseExpiry,
                 )
             }
             .toMap()
@@ -333,6 +337,8 @@ class ProvidersModule(
         comicVineIssueName: String?,
         comicVineIdFormat: String?,
         bangumiToken: String?,
+        cacheDatabaseFile: String,
+        cacheDatabaseExpiry: Int,
     ): MetadataProvidersContainer {
         return MetadataProvidersContainer(
             mangaupdates = createMangaUpdatesMetadataProvider(
@@ -403,6 +409,8 @@ class ProvidersModule(
                 comicVineIdFormat = comicVineIdFormat,
                 rateLimiter = comicVineRateLimiter,
                 defaultNameMatcher = defaultNameMatcher,
+                cacheDatabaseFile = cacheDatabaseFile,
+                cacheDatabaseExpiry = cacheDatabaseExpiry,
             ),
             comicVinePriority = config.comicVine.priority,
             hentag = createHentagMetadataProvider(
@@ -706,6 +714,8 @@ class ProvidersModule(
         comicVineIdFormat: String?,
         rateLimiter: ComicVineRateLimiter,
         defaultNameMatcher: NameSimilarityMatcher,
+        cacheDatabaseFile: String,
+        cacheDatabaseExpiry: Int,
     ): ComicVineMetadataProvider? {
         if (config.enabled.not()) return null
         requireNotNull(apiKey) { "Api key is not configured for ComicVine provider" }
@@ -719,7 +729,9 @@ class ProvidersModule(
             },
             apiKey = apiKey,
             comicVineSearchLimit = comicVineSearchLimit,
-            rateLimiter = rateLimiter
+            rateLimiter = rateLimiter,
+            cacheDatabaseFile = cacheDatabaseFile,
+            cacheDatabaseExpiry = cacheDatabaseExpiry,
         )
         val metadataMapper = ComicVineMetadataMapper(
             seriesMetadataConfig = config.seriesMetadata,
@@ -908,4 +920,4 @@ class ProvidersModule(
     }
 
 
-}
+}

komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineCache.kt

@@ -0,0 +1,99 @@
+package snd.komf.providers.comicvine
+
+import org.jetbrains.exposed.v1.jdbc.Database
+import org.jetbrains.exposed.v1.jdbc.SchemaUtils
+import org.jetbrains.exposed.v1.core.Table
+import org.jetbrains.exposed.v1.core.and
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.core.greater
+import org.jetbrains.exposed.v1.datetime.*
+import org.jetbrains.exposed.v1.jdbc.select
+import org.jetbrains.exposed.v1.jdbc.upsert
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import java.nio.file.Path
+import java.io.File
+import java.time.temporal.ChronoUnit
+import kotlin.time.Clock
+import kotlin.time.Duration
+import kotlin.time.Instant
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.toInstant
+import kotlinx.datetime.toLocalDateTime
+import kotlinx.datetime.plus
+import kotlinx.datetime.DateTimeUnit
+
+object CacheTable : Table("cache") {
+    val queryCol = text("query")
+    override val primaryKey = PrimaryKey(queryCol)
+
+    val timestampCol = timestamp("timestamp")
+
+    val responseCol = text("response")
+}
+
+class ComicVineCache(
+    private val databaseFile: String,
+    private val expiry: Int,
+) {
+    private val databasePath = Path.of(databaseFile)
+    private val database = Database.connect("jdbc:sqlite:$databasePath", driver = "org.sqlite.JDBC")
+
+    init {
+        transaction(db = database) {
+            SchemaUtils.create(CacheTable)
+        }
+    }
+
+    private fun getExpiryTimestamp(): Instant {
+        return Clock.System.now()
+            .toLocalDateTime(TimeZone.UTC)
+            .toInstant(TimeZone.UTC)
+            .plus(value = expiry * 24, DateTimeUnit.HOUR)
+    }
+
+    private fun getNowTimestamp(): Instant {
+        return Clock.System.now()
+            .toLocalDateTime(TimeZone.UTC)
+            .toInstant(TimeZone.UTC)
+    }
+
+    private fun maskApiKey(url: String): String {
+        return url.replace(
+            Regex("""api_key=[^&]+"""),
+            "api_key=*****"
+        )
+    }
+
+    fun addEntry(url: String, response: String) {
+        transaction(db = database) {
+            CacheTable.upsert {
+                it[queryCol] = maskApiKey(url)
+                it[responseCol] = response
+                it[timestampCol] = getExpiryTimestamp()
+            }
+        }
+    }
+
+    suspend fun getEntry(url: String): String? {
+        if (expiry == 0) {
+            return transaction(db = database) {
+                CacheTable
+                    .select(CacheTable.responseCol).where {
+                        CacheTable.queryCol eq maskApiKey(url)
+                    }
+                    .firstOrNull()
+                    ?.get(CacheTable.responseCol)
+            }
+        }
+
+        return transaction(db = database) {
+            CacheTable
+                .select(CacheTable.responseCol).where {
+                    (CacheTable.queryCol eq maskApiKey(url)) and
+                    (CacheTable.timestampCol greater getNowTimestamp())
+                }
+                .firstOrNull()
+                ?.get(CacheTable.responseCol)
+        }
+    }
+}

komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineClient.kt

@@ -3,6 +3,12 @@ package snd.komf.providers.comicvine
 import io.ktor.client.*
 import io.ktor.client.call.*
 import io.ktor.client.request.*
+import io.ktor.http.URLBuilder
+import io.ktor.http.ParametersBuilder
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets
+import kotlin.text.Regex
+import kotlinx.serialization.json.Json
 import snd.komf.model.Image
 import snd.komf.providers.comicvine.ComicVineClient.ComicVineTypeId.ISSUE
 import snd.komf.providers.comicvine.ComicVineClient.ComicVineTypeId.VOLUME
@@ -22,7 +28,44 @@ class ComicVineClient(
     private val apiKey: String,
     private val comicVineSearchLimit: Int? = 10,
     private val rateLimiter: ComicVineRateLimiter,
+    private val cacheDatabaseFile: String,
+    private val cacheDatabaseExpiry: Int,
 ) {
+    private val cache = ComicVineCache(cacheDatabaseFile, cacheDatabaseExpiry)
+
+    private fun buildUrlString(
+        url: String,
+        params: Map<String, String> = mapOf(),
+    ): String {
+        val finalParams = sortedMapOf(
+            Pair("api_key", apiKey),
+            Pair("format", "json"),
+        ) + params
+
+        val encodedParams = finalParams.entries.joinToString("&") { (key, value) ->
+            val k = URLEncoder.encode(key, StandardCharsets.UTF_8)
+            val v = URLEncoder.encode(value, StandardCharsets.UTF_8)
+            "$k=$v"
+        }
+
+        return "$url?$encodedParams"
+    }
+
+    private suspend inline fun <reified T> getCachedApi(url: String): ComicVineSearchResult<T> {
+        val fullUrl = buildUrlString(url)
+
+        val cachedResult = cache.getEntry(fullUrl)
+
+        if (cachedResult != null) {
+            return Json.decodeFromString(cachedResult);
+        }
+
+        val response: ComicVineSearchResult<T> = ktor.get(fullUrl).body()
+
+        cache.addEntry(fullUrl, Json.encodeToString(response))
+
+        return response
+    }
 
     suspend fun searchVolume(name: String): ComicVineSearchResult<List<ComicVineVolumeSearch>> {
         rateLimiter.searchAcquire()
@@ -37,27 +80,17 @@ class ComicVineClient(
 
     suspend fun getVolume(id: ComicVineVolumeId): ComicVineSearchResult<ComicVineVolume> {
         rateLimiter.volumeAcquire()
-        return ktor.get("$baseUrl/volume/${VOLUME.id}-${id.value}/") {
-            parameter("format", "json")
-            parameter("api_key", apiKey)
-        }.body()
+        return getCachedApi("$baseUrl/volume/${VOLUME.id}-${id.value}/")
     }
 
     suspend fun getIssue(id: ComicVineIssueId): ComicVineSearchResult<ComicVineIssue> {
         rateLimiter.issueAcquire()
-        return ktor.get("$baseUrl/issue/${ISSUE.id}-${id.value}/") {
-            parameter("format", "json")
-            parameter("api_key", apiKey)
-        }.body()
+        return getCachedApi("$baseUrl/issue/${ISSUE.id}-${id.value}/")
     }
 
     suspend fun getStoryArc(id: ComicVineStoryArcId): ComicVineSearchResult<ComicVineStoryArc> {
         rateLimiter.storyArcAcquire()
-        return ktor.get("$baseUrl/story_arc/${ComicVineTypeId.STORY_ARC.id}-${id.value}/") {
-            parameter("format", "json")
-            parameter("api_key", apiKey)
-        }.body()
-
+        return getCachedApi("$baseUrl/story_arc/${ComicVineTypeId.STORY_ARC.id}-${id.value}/")
     }
 
     suspend fun getCover(url: String): Image {