Change Cache to Embedded H2 (#113)

* Make clicking "Unknown" language do nothing.

#97

* Implement user cache using embedded H2.

* Separate caching to its own service

#110

* Change UserProfile to be not Serializable

* Fix copy-paste errors.

* Replace Jackson with Gson which reduced boilerplate code by a bit.

* Introduce an extension function

* Remove unnecessary JsonDeserializer class.

Instead, I decided to use a simple TypeAdapter in order to deserialize number into java.util.Date.

* Switch back to Jackson as deserializer.

* Add warning suppression.

* Replace redundant creation of ObjectMapper.

* Replace LocalDateTime with Instant.

* Prevent multiple invocations of generateUserProfile.

* Cache by lazy.

* Remove unnecessary JSON deserialization from lookUpInCache.

* Prevent NPE.

* Update src/main/kotlin/app/CacheService.kt

Co-authored-by: Sidd <[email protected]>

* Revert "Prevent multiple invocations of generateUserProfile."

This reverts commit 8306d92

* Remove unnecessary custom deserializer.

* Rename function.

UserService#lookUpInCache → selectJsonFromDb

* Reduce invocations of CacheService#selectJsonFromDb.

* Extract functions from duplicate code.

* Update src/main/kotlin/app/CacheService.kt

Co-authored-by: Sidd <[email protected]>

* Remove unnecessary class UserNotLoadableException.

* Remove unnecessary lazy initialization.

* Change id values to lowercase in DB.

follow-up change to 80154a0

* Change data type of `timestamp` from 'TIMESTAMP WITH TIME ZONE' to 'TIMESTAMP'.

* Use HikariCP.

* Update src/main/kotlin/app/util/HikariCpDataSource.kt

Co-authored-by: Sidd <[email protected]>

* Replace string concatenation with multi-line string

* Replace uses of magic numbers

* Remove intermediate local variable

* Make CacheService have `HikariCpDataSource.connection` as its field

Co-authored-by: Sidd <[email protected]>

pom.xml

@@ -58,6 +58,17 @@
             <version>${kotlin.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>2.1.212</version>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>4.0.3</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/kotlin/app/CacheService.kt

@@ -0,0 +1,86 @@
+@file:Suppress("SqlResolve")
+
+package app
+
+import app.util.HikariCpDataSource
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import org.slf4j.LoggerFactory
+import java.time.Instant
+import java.time.temporal.ChronoUnit
+
+object CacheService {
+    private val log = LoggerFactory.getLogger(CacheService.javaClass)
+    private val objectMapper = jacksonObjectMapper()
+    private val connection = HikariCpDataSource.connection
+
+    private fun createTableIfAbsent() {
+        val statement = connection.createStatement()
+
+        statement.execute(
+            """
+            CREATE TABLE IF NOT EXISTS userinfo (
+            id VARCHAR2 PRIMARY KEY, 
+            timestamp TIMESTAMP, 
+            data JSON
+            )
+            """.trimIndent()
+        )
+    }
+
+    fun selectJsonFromDb(username: String): String? {
+        createTableIfAbsent()
+
+        val preparedStatement = connection.prepareStatement(
+            """
+            SELECT 
+            timestamp, 
+            data 
+            FROM userinfo 
+            WHERE id = ?
+            """.trimIndent()
+        )
+        preparedStatement.setString(1, username.lowercase())
+
+        val result = preparedStatement.executeQuery()
+        result.use {
+            // guaranteed to be at most one.
+            if (it.next()) {
+                val timestamp = it.getTimestamp("timestamp").toInstant()
+                val diffInHours = ChronoUnit.HOURS.between(timestamp, Instant.now())
+                if (diffInHours <= 6) {
+                    val json: String? = it.getString("data")
+                    if (json != null) {
+                        log.debug("cache hit: {}", json)
+                    }
+
+                    return json
+                }
+            }
+        }
+
+        log.debug("cache miss for username: {}", username)
+
+        return null
+    }
+
+    fun getUserFromJson(json: String) = objectMapper.readValue<UserProfile>(json)
+
+    fun saveInCache(userProfile: UserProfile) {
+        createTableIfAbsent()
+
+        val json = objectMapper.writeValueAsString(userProfile)
+
+        val preparedStatement = connection.prepareStatement(
+            """
+            MERGE INTO userinfo (id, timestamp, data) KEY (id) 
+            VALUES (?, CURRENT_TIMESTAMP(), ? FORMAT JSON)
+            """.trimIndent()
+        )
+
+        preparedStatement.setString(1, userProfile.user.login.lowercase())
+        preparedStatement.setString(2, json)
+
+        preparedStatement.execute()
+    }
+}

src/main/kotlin/app/Main.kt

@@ -46,8 +46,7 @@ fun main() {
         get("/api/user/{user}") { ctx ->
             val user = ctx.pathParam("user")
             if (!UserService.userExists(user)) throw NotFoundResponse()
-            if (!UserService.canLoadUser(user)) throw BadRequestResponse("Can't load user")
-            ctx.json(UserService.getUserProfile(user))
+            UserService.getUserIfCanLoad(user)?.let { ctx.json(it) } ?: throw BadRequestResponse("Can't load user")
         }
         get("/search", VueComponent("search-view"))
         get("/user/{user}", VueComponent("user-view"))

src/main/kotlin/app/UserProfile.kt

@@ -0,0 +1,15 @@
+package app
+
+import org.eclipse.egit.github.core.User
+
+data class UserProfile(
+    val user: User,
+    val quarterCommitCount: Map<String, Int>,
+    val langRepoCount: Map<String, Int>,
+    val langStarCount: Map<String, Int>,
+    val langCommitCount: Map<String, Int>,
+    val repoCommitCount: Map<String, Int>,
+    val repoStarCount: Map<String, Int>,
+    val repoCommitCountDescriptions: Map<String, String?>,
+    val repoStarCountDescriptions: Map<String, String?>
+)

src/main/kotlin/app/UserService.kt

@@ -3,10 +3,7 @@ package app
 import app.util.CommitCountUtil
 import org.eclipse.egit.github.core.Repository
 import org.eclipse.egit.github.core.RepositoryCommit
-import org.eclipse.egit.github.core.User
 import org.slf4j.LoggerFactory
-import java.io.Serializable
-import java.time.Instant
 import java.util.concurrent.ConcurrentHashMap
 import java.util.stream.IntStream
 import kotlin.streams.toList
@@ -25,45 +22,33 @@ object UserService {
         false
     }
 
+    private fun remainingRequests(): Int = GhService.remainingRequests
+    private fun hasFreeRemainingRequests(): Boolean = remainingRequests() > (freeRequestCutoff ?: remainingRequests())
+
     fun canLoadUser(user: String): Boolean {
-        val remainingRequests by lazy { GhService.remainingRequests }
-        val hasFreeRemainingRequests by lazy { remainingRequests > (freeRequestCutoff ?: remainingRequests) }
+        val userCacheJson = CacheService.selectJsonFromDb(user)
         return Config.unrestricted()
-                || Cache.contains(user)
-                || hasFreeRemainingRequests
-                || (remainingRequests > 0 && hasStarredRepo(user))
+            || (userCacheJson != null)
+            || hasFreeRemainingRequests()
+            || (remainingRequests() > 0 && hasStarredRepo(user))
     }
 
-    fun getUserProfile(username: String): UserProfile {
-        if (Cache.invalid(username)) {
-            val user = GhService.users.getUser(username)
-            val repos = GhService.repos.getRepositories(username).filter { !it.isFork && it.size != 0 }
-            val repoCommits = repos.parallelStream().map { it to commitsForRepo(it).filter { it.author?.login.equals(username, ignoreCase = true) } }.toList().toMap()
-            val langRepoGrouping = repos.groupingBy { (it.language ?: "Unknown") }
-
-            val quarterCommitCount = CommitCountUtil.getCommitsForQuarters(user, repoCommits)
-            val langRepoCount = langRepoGrouping.eachCount().toList().sortedBy { (_, v) -> -v }.toMap()
-            val langStarCount = langRepoGrouping.fold(0) { acc, repo -> acc + repo.watchers }.toList().sortedBy { (_, v) -> -v }.toMap()
-            val langCommitCount = langRepoGrouping.fold(0) { acc, repo -> acc + repoCommits[repo]!!.size }.toList().sortedBy { (_, v) -> -v }.toMap()
-            val repoCommitCount = repoCommits.map { it.key.name to it.value.size }.toList().sortedBy { (_, v) -> -v }.take(10).toMap()
-            val repoStarCount = repos.filter { it.watchers > 0 }.map { it.name to it.watchers }.sortedBy { (_, v) -> -v }.take(10).toMap()
-
-            val repoCommitCountDescriptions = repoCommitCount.map { it.key to repos.find { r -> r.name == it.key }?.description }.toMap()
-            val repoStarCountDescriptions = repoStarCount.map { it.key to repos.find { r -> r.name == it.key }?.description }.toMap()
-
-            Cache.putUserProfile(UserProfile(
-                    user,
-                    quarterCommitCount,
-                    langRepoCount,
-                    langStarCount,
-                    langCommitCount,
-                    repoCommitCount,
-                    repoStarCount,
-                    repoCommitCountDescriptions,
-                    repoStarCountDescriptions
-            ))
+    fun getUserIfCanLoad(username: String): UserProfile? {
+        val userCacheJson = CacheService.selectJsonFromDb(username)
+        val canLoadUser = Config.unrestricted()
+            || (userCacheJson != null)
+            || hasFreeRemainingRequests()
+            || remainingRequests() > 0 && hasStarredRepo(username)
+
+        if (canLoadUser) {
+            return if (userCacheJson == null) {
+                generateUserProfile(username)
+            } else {
+                CacheService.getUserFromJson(userCacheJson)
+            }
         }
-        return Cache.getUserProfile(username)!!
+
+        return null
     }
 
     private fun hasStarredRepo(username: String): Boolean {
@@ -97,18 +82,36 @@ object UserService {
         listOf()
     }
 
-}
+    private fun generateUserProfile(username: String): UserProfile {
+        val user = GhService.users.getUser(username)
+        val repos = GhService.repos.getRepositories(username).filter { !it.isFork && it.size != 0 }
+        val repoCommits = repos.parallelStream().map { it to commitsForRepo(it).filter { it.author?.login.equals(username, ignoreCase = true) } }.toList().toMap()
+        val langRepoGrouping = repos.groupingBy { (it.language ?: "Unknown") }
+
+        val quarterCommitCount = CommitCountUtil.getCommitsForQuarters(user, repoCommits)
+        val langRepoCount = langRepoGrouping.eachCount().toList().sortedBy { (_, v) -> -v }.toMap()
+        val langStarCount = langRepoGrouping.fold(0) { acc, repo -> acc + repo.watchers }.toList().sortedBy { (_, v) -> -v }.toMap()
+        val langCommitCount = langRepoGrouping.fold(0) { acc, repo -> acc + repoCommits[repo]!!.size }.toList().sortedBy { (_, v) -> -v }.toMap()
+        val repoCommitCount = repoCommits.map { it.key.name to it.value.size }.toList().sortedBy { (_, v) -> -v }.take(10).toMap()
+        val repoStarCount = repos.filter { it.watchers > 0 }.map { it.name to it.watchers }.sortedBy { (_, v) -> -v }.take(10).toMap()
 
-data class UserProfile(
-        val user: User,
-        val quarterCommitCount: Map<String, Int>,
-        val langRepoCount: Map<String, Int>,
-        val langStarCount: Map<String, Int>,
-        val langCommitCount: Map<String, Int>,
-        val repoCommitCount: Map<String, Int>,
-        val repoStarCount: Map<String, Int>,
-        val repoCommitCountDescriptions: Map<String, String?>,
-        val repoStarCountDescriptions: Map<String, String?>
-) : Serializable {
-    val timeStamp = Instant.now().toEpochMilli()
+        val repoCommitCountDescriptions = repoCommitCount.map { it.key to repos.find { r -> r.name == it.key }?.description }.toMap()
+        val repoStarCountDescriptions = repoStarCount.map { it.key to repos.find { r -> r.name == it.key }?.description }.toMap()
+
+        val userProfile = UserProfile(
+            user,
+            quarterCommitCount,
+            langRepoCount,
+            langStarCount,
+            langCommitCount,
+            repoCommitCount,
+            repoStarCount,
+            repoCommitCountDescriptions,
+            repoStarCountDescriptions
+        )
+
+        CacheService.saveInCache(userProfile)
+
+        return userProfile;
+    }
 }

src/main/kotlin/app/util/HikariCpDataSource.kt

@@ -0,0 +1,20 @@
+package app.util
+
+import com.zaxxer.hikari.HikariConfig
+import com.zaxxer.hikari.HikariDataSource
+import java.sql.Connection
+
+object HikariCpDataSource {
+    private const val urlToDb = "jdbc:h2:mem:userinfo"
+
+    private val config = HikariConfig().apply {
+        jdbcUrl = urlToDb
+        addDataSourceProperty("cachePrepStmts", "true")
+        addDataSourceProperty("prepStmtCacheSize", "250")
+        addDataSourceProperty("prepStmtCacheSqlLimit", "2048")
+    }
+
+    private var hikariDataSource = HikariDataSource(config)
+
+    val connection: Connection get() = hikariDataSource.connection
+}