Update STE UserTriggerDecision and UserModification events

1. add multiple fields in UserTriggerDecision event and UserModification event, 2. Update the latency calculation to reflect the latest definition 3. added some initial a/b testing changes for new auto-trigger UX
aws · Oct 9, 2024 · 68880af · 68880af
1 parent 7e4f6e3
commit 68880af
Show file tree

Hide file tree

Showing 10 changed files with 142 additions and 108 deletions.
diff --git a/...e/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt b/...e/aws/toolkits/jetbrains/services/codewhisperer/credentials/CodeWhispererClientAdaptor.kt
@@ -56,7 +56,6 @@ import software.aws.toolkits.jetbrains.services.telemetry.ClientMetadata
 import software.aws.toolkits.telemetry.CodewhispererCompletionType
 import software.aws.toolkits.telemetry.CodewhispererSuggestionState
 import java.time.Instant
-import java.util.concurrent.TimeUnit
 import kotlin.reflect.KProperty0
 import kotlin.reflect.jvm.isAccessible
 
@@ -114,7 +113,8 @@ interface CodeWhispererClientAdaptor : Disposable {
         requestId: String,
         language: CodeWhispererProgrammingLanguage,
         customizationArn: String,
-        modificationPercentage: Double,
+        acceptedCharacterCount: Int,
+        unmodifiedAcceptedTokenCount: Int,
     ): SendTelemetryEventResponse
 
     fun sendCodeScanTelemetry(
@@ -305,13 +305,14 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         val programmingLanguage = fileContext.programmingLanguage
         var e2eLatency = requestContext.latencyContext.getCodeWhispererEndToEndLatency()
 
-        // When we send a userTriggerDecision of Empty or Discard, we set the time users see the first
-        // suggestion to be now.
-        if (e2eLatency < 0) {
-            e2eLatency = TimeUnit.NANOSECONDS.toMillis(
-                System.nanoTime() - requestContext.latencyContext.codewhispererEndToEndStart
-            ).toDouble()
+        // When we send a userTriggerDecision for neither Accept nor Reject, service side should not use this value
+        // and client side will set this value to 0.0.
+        if (suggestionState != CodewhispererSuggestionState.Accept &&
+            suggestionState != CodewhispererSuggestionState.Reject
+        ) {
+            e2eLatency = 0.0
         }
+
         return bearerClient().sendTelemetryEvent { requestBuilder ->
             requestBuilder.telemetryEvent { telemetryEventBuilder ->
                 telemetryEventBuilder.userTriggerDecisionEvent {
@@ -321,6 +322,9 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                     it.sessionId(responseContext.sessionId)
                     it.recommendationLatencyMilliseconds(e2eLatency)
                     it.triggerToResponseLatencyMilliseconds(requestContext.latencyContext.paginationFirstCompletionTime)
+                    it.perceivedLatencyMilliseconds(
+                        requestContext.latencyContext.getPerceivedLatency(requestContext.triggerTypeInfo.triggerType)
+                    )
                     it.suggestionState(suggestionState.toCodeWhispererSdkType())
                     it.timestamp(Instant.now())
                     it.suggestionReferenceCount(suggestionReferenceCount)
@@ -360,7 +364,8 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
         requestId: String,
         language: CodeWhispererProgrammingLanguage,
         customizationArn: String,
-        modificationPercentage: Double,
+        acceptedCharacterCount: Int,
+        unmodifiedAcceptedTokenCount: Int,
     ): SendTelemetryEventResponse = bearerClient().sendTelemetryEvent { requestBuilder ->
         requestBuilder.telemetryEvent { telemetryEventBuilder ->
             telemetryEventBuilder.userModificationEvent {
@@ -370,8 +375,11 @@ open class CodeWhispererClientAdaptorImpl(override val project: Project) : CodeW
                     languageBuilder.languageName(language.toCodeWhispererRuntimeLanguage().languageId)
                 }
                 it.customizationArn(customizationArn)
-                it.modificationPercentage(modificationPercentage)
+                // deprecated field, service side should not use this % anymore
+                it.modificationPercentage(0.0)
                 it.timestamp(Instant.now())
+                it.acceptedCharacterCount(acceptedCharacterCount)
+                it.unmodifiedAcceptedCharacterCount(unmodifiedAcceptedTokenCount)
             }
         }
         requestBuilder.optOutPreference(getTelemetryOptOutPreference())

diff --git a/...ty/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt b/...ty/src/software/aws/toolkits/jetbrains/services/codewhisperer/model/CodeWhispererModel.kt
@@ -13,6 +13,7 @@ import software.amazon.awssdk.services.codewhispererruntime.model.GenerateComple
 import software.aws.toolkits.jetbrains.core.credentials.ToolkitConnection
 import software.aws.toolkits.jetbrains.services.codewhisperer.codescan.sessionconfig.PayloadContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.language.CodeWhispererProgrammingLanguage
+import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutoTriggerService
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererAutomatedTriggerType
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.RequestContext
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.ResponseContext
@@ -198,6 +199,18 @@ data class LatencyContext(
     fun getCodeWhispererPreprocessingLatency() = TimeUnit.NANOSECONDS.toMillis(
         codewhispererPreprocessingEnd - codewhispererPreprocessingStart
     ).toDouble()
+
+    // For auto-trigger it's from the time when last char typed
+    // for manual-trigger it's from the time when last trigger action happened(alt + c)
+    fun getPerceivedLatency(triggerType: CodewhispererTriggerType) =
+        if (triggerType == CodewhispererTriggerType.OnDemand) {
+            getCodeWhispererEndToEndLatency()
+        } else {
+            (
+                TimeUnit.NANOSECONDS.toMillis(codewhispererEndToEndEnd) -
+                    CodeWhispererAutoTriggerService.getInstance().timeAtLastCharTyped.toEpochMilli()
+                ).toDouble()
+        }
 }
 
 data class TryExampleRowContext(

diff --git a/.../aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt b/.../aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererAutoTriggerService.kt
@@ -42,6 +42,7 @@ class CodeWhispererAutoTriggerService : CodeWhispererAutoTriggerHandler, Disposa
 
     private var lastInvocationTime: Instant? = null
     private var lastInvocationLineNum: Int? = null
+    var timeAtLastCharTyped: Instant = Instant.now()
 
     init {
         scheduleReset()
@@ -54,6 +55,7 @@ class CodeWhispererAutoTriggerService : CodeWhispererAutoTriggerHandler, Disposa
     // a util wrapper
     fun tryInvokeAutoTrigger(editor: Editor, triggerType: CodeWhispererAutomatedTriggerType): Job? {
         // only needed for Classifier group, thus calculate it lazily
+        timeAtLastCharTyped = Instant.now()
         val classifierResult: ClassifierResult by lazy { shouldTriggerClassifier(editor, triggerType.telemetryType) }
         val language = runReadAction {
             FileDocumentManager.getInstance().getFile(editor.document)?.programmingLanguage()

diff --git a/...ws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt b/...ws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt
@@ -32,6 +32,14 @@ class CodeWhispererFeatureConfigService {
                 featureConfigs[it.feature()] = FeatureContext(it.feature(), it.variation(), it.value())
             }
 
+            // Only apply new auto-trigger UX to BID users
+            val isNewAutoTriggerUX = getIsNewAutoTriggerUX()
+            if (isNewAutoTriggerUX) {
+                calculateIfIamIdentityCenterConnection(project) {
+                    featureConfigs.remove(NEW_AUTO_TRIGGER_UX)
+                }
+            }
+
             val customizationArnOverride = featureConfigs[CUSTOMIZATION_ARN_OVERRIDE_NAME]?.value?.stringValue()
             if (customizationArnOverride != null) {
                 // Double check if server-side wrongly returns a customizationArn to BID users
@@ -84,20 +92,24 @@ class CodeWhispererFeatureConfigService {
 
     fun getCustomizationArnOverride(): String = getFeatureValueForKey(CUSTOMIZATION_ARN_OVERRIDE_NAME).stringValue()
 
+    fun getIsNewAutoTriggerUX(): Boolean = getFeatureValueForKey(NEW_AUTO_TRIGGER_UX).boolValue()
+
     // Get the feature value for the given key.
-    // In case of a misconfiguration, it will return a default feature value of Boolean true.
+    // In case of a misconfiguration, it will return a default feature value of Boolean false.
     private fun getFeatureValueForKey(name: String): FeatureValue =
         featureConfigs[name]?.value ?: FEATURE_DEFINITIONS[name]?.value
-            ?: FeatureValue.builder().boolValue(true).build()
+            ?: FeatureValue.builder().boolValue(false).build()
 
     companion object {
         fun getInstance(): CodeWhispererFeatureConfigService = service()
         private const val TEST_FEATURE_NAME = "testFeature"
         private const val DATA_COLLECTION_FEATURE = "IDEProjectContextDataCollection"
         const val CUSTOMIZATION_ARN_OVERRIDE_NAME = "customizationArnOverride"
+        private const val NEW_AUTO_TRIGGER_UX = "newAutoTriggerUX"
         private val LOG = getLogger<CodeWhispererFeatureConfigService>()
 
         // TODO: add real feature later
+        // Also serve as default values in case server-side config isn't there yet
         internal val FEATURE_DEFINITIONS = mapOf(
             TEST_FEATURE_NAME to FeatureContext(
                 TEST_FEATURE_NAME,
@@ -109,7 +121,12 @@ class CodeWhispererFeatureConfigService {
                 CUSTOMIZATION_ARN_OVERRIDE_NAME,
                 "customizationARN",
                 FeatureValue.builder().stringValue("").build()
-            )
+            ),
+            NEW_AUTO_TRIGGER_UX to FeatureContext(
+                NEW_AUTO_TRIGGER_UX,
+                "CONTROL",
+                FeatureValue.builder().boolValue(false).build()
+            ),
         )
     }
 }

diff --git a/...rc/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt b/...rc/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererService.kt
@@ -244,7 +244,8 @@ class CodeWhispererService(private val cs: CoroutineScope) : Disposable {
                     val sessionId = response.sdkHttpResponse().headers().getOrDefault(KET_SESSION_ID, listOf(requestId))[0]
                     if (requestCount == 1) {
                         requestContext.latencyContext.codewhispererPostprocessingStart = System.nanoTime()
-                        requestContext.latencyContext.paginationFirstCompletionTime = latency
+                        requestContext.latencyContext.paginationFirstCompletionTime =
+                            (endTime - requestContext.latencyContext.codewhispererEndToEndStart).toDouble()
                         requestContext.latencyContext.firstRequestId = requestId
                         CodeWhispererInvocationStatus.getInstance().setInvocationSessionId(sessionId)
                     }

diff --git a/...s/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt b/...s/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt
@@ -31,6 +31,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispe
 import software.aws.toolkits.jetbrains.services.codewhisperer.service.CodeWhispererService
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.TOTAL_SECONDS_IN_MINUTE
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getCodeWhispererStartUrl
+import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getUnmodifiedAcceptedCharsCount
 import software.aws.toolkits.jetbrains.services.codewhisperer.util.runIfIdcConnectionOrTelemetryEnabled
 import software.aws.toolkits.telemetry.CodewhispererTelemetry
 import java.time.Duration
@@ -40,6 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger
 import kotlin.math.roundToLong
 
 // TODO: reset code coverage calculator on logging out connection?
+// TODO: rename "Tokens" to "Characters", and many more renames in this file
 abstract class CodeWhispererCodeCoverageTracker(
     private val project: Project,
     private val timeWindowInSec: Long,
@@ -146,19 +148,6 @@ abstract class CodeWhispererCodeCoverageTracker(
         rangeMarker.range?.let { myRange -> rangeMarker.document.getText(myRange) }
     }
 
-    // With edit distance, complicate usermodification can be considered as simple edit(add, delete, replace),
-    // and thus the unmodified part of recommendation length can be deducted/approximated
-    // ex. (modified > original): originalRecom: foo -> modifiedRecom: fobarbarbaro, distance = 9, delta = 12 - 9 = 3
-    // ex. (modified == original): originalRecom: helloworld -> modifiedRecom: HelloWorld, distance = 2, delta = 10 - 2 = 8
-    // ex. (modified < original): originalRecom: CodeWhisperer -> modifiedRecom: CODE, distance = 12, delta = 13 - 12 = 1
-    internal fun getAcceptedTokensDelta(originalRecommendation: String, modifiedRecommendation: String): Int {
-        val editDistance = getEditDistance(modifiedRecommendation, originalRecommendation).toInt()
-        return maxOf(originalRecommendation.length, modifiedRecommendation.length) - editDistance
-    }
-
-    protected open fun getEditDistance(modifiedString: String, originalString: String): Double =
-        levenshteinChecker.distance(modifiedString, originalString)
-
     private fun flush() {
         try {
             if (isTelemetryEnabled()) emitCodeWhispererCodeContribution()
@@ -174,12 +163,12 @@ abstract class CodeWhispererCodeCoverageTracker(
         }
     }
 
-    private fun incrementAcceptedTokens(document: Document, delta: Int) {
+    private fun incrementUnmodifiedAcceptedCharsCount(document: Document, delta: Int) {
         val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() }
         tokens.acceptedTokens.addAndGet(delta)
     }
 
-    private fun incrementRawAcceptedTokens(document: Document, delta: Int) {
+    private fun incrementAcceptedCharsCount(document: Document, delta: Int) {
         val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() }
         tokens.rawAcceptedTokens.addAndGet(delta)
     }
@@ -217,10 +206,10 @@ abstract class CodeWhispererCodeCoverageTracker(
                 }
                 return@forEach
             }
-            val delta = getAcceptedTokensDelta(originalRecommendation, modifiedRecommendation)
+            val unmodifiedRecommendationLength = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
             runReadAction {
-                incrementRawAcceptedTokens(rangeMarker.document, originalRecommendation.length)
-                incrementAcceptedTokens(rangeMarker.document, delta)
+                incrementAcceptedCharsCount(rangeMarker.document, originalRecommendation.length)
+                incrementUnmodifiedAcceptedCharsCount(rangeMarker.document, unmodifiedRecommendationLength)
             }
         }
         val customizationArn: String? = CodeWhispererModelConfigurator.getInstance().activeCustomization(project)?.arn
@@ -277,7 +266,7 @@ abstract class CodeWhispererCodeCoverageTracker(
 
     companion object {
         @JvmStatic
-        protected val levenshteinChecker = Levenshtein()
+        val levenshteinChecker = Levenshtein()
         private const val REMAINING_RECOMMENDATION = "remainingRecommendation"
         private val KEY_REMAINING_RECOMMENDATION = Key<String>(REMAINING_RECOMMENDATION)
         private val LOG = getLogger<CodeWhispererCodeCoverageTracker>()