diff --git a/README.md b/README.md index b78985dc..b6132d9c 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,8 @@ metadataProviders: comicVineApiKey: # required for comicVine provider https://comicvine.gamespot.com/api/ env:KOMF_METADATA_PROVIDERS_COMIC_VINE_API_KEY comicVineSearchLimit: # define ComicVine search result Limit, default is 10 comicVineIssueName: # string that contains "{number}" which will be replaced by the issue number ie. "Issue #{number}". Used when an issue has no name on ComicVine, default is null + cacheDatabaseFile: # cache database file location. default is "./cv_cache.db" + cacheDatabaseExpiry: # number of days after which an entry in the cache is considered expired. default is 14 comicVineIdFormat: # string that contains "{id}" which will serve to parse the ComicVine volume of a given book from its title or folder name ie. "[cv-{id}]" which will correctly identify '.../Uncanny X-Men Omnibus (2006) [cv-27512]' as being [4050-27512](https://comicvine.gamespot.com/uncanny-x-men-omnibus/4050-27512/) bangumiToken: # bangumi provider require a token to show nsfw items https://next.bgm.tv/demo/access-token env:KOMF_METADATA_PROVIDERS_BANGUMI_TOKEN defaultProviders: diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt index 7c229e9a..fdccd661 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/MetadataProvidersConfig.kt @@ -25,6 +25,8 @@ data class MetadataProvidersConfig( val defaultProviders: ProvidersConfig = ProvidersConfig(), val libraryProviders: Map = emptyMap(), val mangabakaDatabaseDir: String = "./mangabaka", + val cacheDatabaseFile: String = "./cv_cache.db", + val cacheDatabaseExpiry: Int = 14, ) @Serializable diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt index a92e54d4..3205a00f 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/ProvidersModule.kt @@ -108,6 +108,8 @@ class ProvidersModule( comicVineIssueName = config.comicVineIssueName, comicVineIdFormat = config.comicVineIdFormat, bangumiToken = config.bangumiToken, + cacheDatabaseFile = config.cacheDatabaseFile, + cacheDatabaseExpiry = config.cacheDatabaseExpiry, ) val libraryProviders = config.libraryProviders .map { (libraryId, libraryConfig) -> @@ -120,6 +122,8 @@ class ProvidersModule( comicVineIssueName = config.comicVineIssueName, comicVineIdFormat = config.comicVineIdFormat, bangumiToken = config.bangumiToken, + cacheDatabaseFile = config.cacheDatabaseFile, + cacheDatabaseExpiry = config.cacheDatabaseExpiry, ) } .toMap() @@ -333,6 +337,8 @@ class ProvidersModule( comicVineIssueName: String?, comicVineIdFormat: String?, bangumiToken: String?, + cacheDatabaseFile: String, + cacheDatabaseExpiry: Int, ): MetadataProvidersContainer { return MetadataProvidersContainer( mangaupdates = createMangaUpdatesMetadataProvider( @@ -403,6 +409,8 @@ class ProvidersModule( comicVineIdFormat = comicVineIdFormat, rateLimiter = comicVineRateLimiter, defaultNameMatcher = defaultNameMatcher, + cacheDatabaseFile = cacheDatabaseFile, + cacheDatabaseExpiry = cacheDatabaseExpiry, ), comicVinePriority = config.comicVine.priority, hentag = createHentagMetadataProvider( @@ -706,6 +714,8 @@ class ProvidersModule( comicVineIdFormat: String?, rateLimiter: ComicVineRateLimiter, defaultNameMatcher: NameSimilarityMatcher, + cacheDatabaseFile: String, + cacheDatabaseExpiry: Int, ): ComicVineMetadataProvider? { if (config.enabled.not()) return null requireNotNull(apiKey) { "Api key is not configured for ComicVine provider" } @@ -719,7 +729,9 @@ class ProvidersModule( }, apiKey = apiKey, comicVineSearchLimit = comicVineSearchLimit, - rateLimiter = rateLimiter + rateLimiter = rateLimiter, + cacheDatabaseFile = cacheDatabaseFile, + cacheDatabaseExpiry = cacheDatabaseExpiry, ) val metadataMapper = ComicVineMetadataMapper( seriesMetadataConfig = config.seriesMetadata, @@ -908,4 +920,4 @@ class ProvidersModule( } -} \ No newline at end of file +} diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineCache.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineCache.kt new file mode 100644 index 00000000..398f193c --- /dev/null +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineCache.kt @@ -0,0 +1,99 @@ +package snd.komf.providers.comicvine + +import org.jetbrains.exposed.v1.jdbc.Database +import org.jetbrains.exposed.v1.jdbc.SchemaUtils +import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.greater +import org.jetbrains.exposed.v1.datetime.* +import org.jetbrains.exposed.v1.jdbc.select +import org.jetbrains.exposed.v1.jdbc.upsert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import java.nio.file.Path +import java.io.File +import java.time.temporal.ChronoUnit +import kotlin.time.Clock +import kotlin.time.Duration +import kotlin.time.Instant +import kotlinx.datetime.TimeZone +import kotlinx.datetime.toInstant +import kotlinx.datetime.toLocalDateTime +import kotlinx.datetime.plus +import kotlinx.datetime.DateTimeUnit + +object CacheTable : Table("cache") { + val queryCol = text("query") + override val primaryKey = PrimaryKey(queryCol) + + val timestampCol = timestamp("timestamp") + + val responseCol = text("response") +} + +class ComicVineCache( + private val databaseFile: String, + private val expiry: Int, +) { + private val databasePath = Path.of(databaseFile) + private val database = Database.connect("jdbc:sqlite:$databasePath", driver = "org.sqlite.JDBC") + + init { + transaction(db = database) { + SchemaUtils.create(CacheTable) + } + } + + private fun getExpiryTimestamp(): Instant { + return Clock.System.now() + .toLocalDateTime(TimeZone.UTC) + .toInstant(TimeZone.UTC) + .plus(value = expiry * 24, DateTimeUnit.HOUR) + } + + private fun getNowTimestamp(): Instant { + return Clock.System.now() + .toLocalDateTime(TimeZone.UTC) + .toInstant(TimeZone.UTC) + } + + private fun maskApiKey(url: String): String { + return url.replace( + Regex("""api_key=[^&]+"""), + "api_key=*****" + ) + } + + fun addEntry(url: String, response: String) { + transaction(db = database) { + CacheTable.upsert { + it[queryCol] = maskApiKey(url) + it[responseCol] = response + it[timestampCol] = getExpiryTimestamp() + } + } + } + + suspend fun getEntry(url: String): String? { + if (expiry == 0) { + return transaction(db = database) { + CacheTable + .select(CacheTable.responseCol).where { + CacheTable.queryCol eq maskApiKey(url) + } + .firstOrNull() + ?.get(CacheTable.responseCol) + } + } + + return transaction(db = database) { + CacheTable + .select(CacheTable.responseCol).where { + (CacheTable.queryCol eq maskApiKey(url)) and + (CacheTable.timestampCol greater getNowTimestamp()) + } + .firstOrNull() + ?.get(CacheTable.responseCol) + } + } +} diff --git a/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineClient.kt b/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineClient.kt index 27ea8833..fe3a0d23 100644 --- a/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineClient.kt +++ b/komf-core/src/commonMain/kotlin/snd/komf/providers/comicvine/ComicVineClient.kt @@ -3,6 +3,12 @@ package snd.komf.providers.comicvine import io.ktor.client.* import io.ktor.client.call.* import io.ktor.client.request.* +import io.ktor.http.URLBuilder +import io.ktor.http.ParametersBuilder +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import kotlin.text.Regex +import kotlinx.serialization.json.Json import snd.komf.model.Image import snd.komf.providers.comicvine.ComicVineClient.ComicVineTypeId.ISSUE import snd.komf.providers.comicvine.ComicVineClient.ComicVineTypeId.VOLUME @@ -22,7 +28,44 @@ class ComicVineClient( private val apiKey: String, private val comicVineSearchLimit: Int? = 10, private val rateLimiter: ComicVineRateLimiter, + private val cacheDatabaseFile: String, + private val cacheDatabaseExpiry: Int, ) { + private val cache = ComicVineCache(cacheDatabaseFile, cacheDatabaseExpiry) + + private fun buildUrlString( + url: String, + params: Map = mapOf(), + ): String { + val finalParams = sortedMapOf( + Pair("api_key", apiKey), + Pair("format", "json"), + ) + params + + val encodedParams = finalParams.entries.joinToString("&") { (key, value) -> + val k = URLEncoder.encode(key, StandardCharsets.UTF_8) + val v = URLEncoder.encode(value, StandardCharsets.UTF_8) + "$k=$v" + } + + return "$url?$encodedParams" + } + + private suspend inline fun getCachedApi(url: String): ComicVineSearchResult { + val fullUrl = buildUrlString(url) + + val cachedResult = cache.getEntry(fullUrl) + + if (cachedResult != null) { + return Json.decodeFromString(cachedResult); + } + + val response: ComicVineSearchResult = ktor.get(fullUrl).body() + + cache.addEntry(fullUrl, Json.encodeToString(response)) + + return response + } suspend fun searchVolume(name: String): ComicVineSearchResult> { rateLimiter.searchAcquire() @@ -37,27 +80,17 @@ class ComicVineClient( suspend fun getVolume(id: ComicVineVolumeId): ComicVineSearchResult { rateLimiter.volumeAcquire() - return ktor.get("$baseUrl/volume/${VOLUME.id}-${id.value}/") { - parameter("format", "json") - parameter("api_key", apiKey) - }.body() + return getCachedApi("$baseUrl/volume/${VOLUME.id}-${id.value}/") } suspend fun getIssue(id: ComicVineIssueId): ComicVineSearchResult { rateLimiter.issueAcquire() - return ktor.get("$baseUrl/issue/${ISSUE.id}-${id.value}/") { - parameter("format", "json") - parameter("api_key", apiKey) - }.body() + return getCachedApi("$baseUrl/issue/${ISSUE.id}-${id.value}/") } suspend fun getStoryArc(id: ComicVineStoryArcId): ComicVineSearchResult { rateLimiter.storyArcAcquire() - return ktor.get("$baseUrl/story_arc/${ComicVineTypeId.STORY_ARC.id}-${id.value}/") { - parameter("format", "json") - parameter("api_key", apiKey) - }.body() - + return getCachedApi("$baseUrl/story_arc/${ComicVineTypeId.STORY_ARC.id}-${id.value}/") } suspend fun getCover(url: String): Image {