Skip to content

Commit

Permalink
test fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewyuq
andrewyuq committed Oct 10, 2024
1 parent 68880af commit 5016751
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 92 deletions.
4 changes: 2 additions & 2 deletions ...ws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class CodeWhispererFeatureConfigService {
}

// Only apply new auto-trigger UX to BID users
val isNewAutoTriggerUX = getIsNewAutoTriggerUX()
val isNewAutoTriggerUX = getNewAutoTriggerUX()
if (isNewAutoTriggerUX) {
calculateIfIamIdentityCenterConnection(project) {
featureConfigs.remove(NEW_AUTO_TRIGGER_UX)
Expand Down Expand Up @@ -92,7 +92,7 @@ class CodeWhispererFeatureConfigService {

fun getCustomizationArnOverride(): String = getFeatureValueForKey(CUSTOMIZATION_ARN_OVERRIDE_NAME).stringValue()

fun getIsNewAutoTriggerUX(): Boolean = getFeatureValueForKey(NEW_AUTO_TRIGGER_UX).boolValue()
fun getNewAutoTriggerUX(): Boolean = getFeatureValueForKey(NEW_AUTO_TRIGGER_UX).boolValue()

Check notice on line 95 in plugins/amazonq/codewhisperer/jetbrains-community/src/software/aws/toolkits/jetbrains/services/codewhisperer/service/CodeWhispererFeatureConfigService.kt

View workflow job for this annotation

GitHub Actions / qodana

Class member can have 'private' visibility 

Function 'getNewAutoTriggerUX' could be private

// Get the feature value for the given key.
// In case of a misconfiguration, it will return a default feature value of Boolean false.
Expand Down
58 changes: 26 additions & 32 deletions ...s/toolkits/jetbrains/services/codewhisperer/telemetry/CodeWhispererCodeCoverageTracker.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,22 +51,22 @@ abstract class CodeWhispererCodeCoverageTracker(
private val myServiceInvocationCount: AtomicInteger,
) : Disposable {
val percentage: Long?
get() = if (totalTokensSize != 0L) calculatePercentage(rawAcceptedTokenSize, totalTokensSize) else null
val acceptedTokensSize: Long
get() = if (totalCharsCount != 0L) calculatePercentage(acceptedCharsCount, totalCharsCount) else null
val unmodifiedAcceptedCharsCount: Long
get() = fileToTokens.map {
it.value.acceptedTokens.get()
it.value.unmodifiedAcceptedChars.get()
}.fold(0) { acc, next ->
acc + next
}
val totalTokensSize: Long
val totalCharsCount: Long
get() = fileToTokens.map {
it.value.totalTokens.get()
it.value.totalChars.get()
}.fold(0) { acc, next ->
acc + next
}
private val rawAcceptedTokenSize: Long
private val acceptedCharsCount: Long
get() = fileToTokens.map {
it.value.rawAcceptedTokens.get()
it.value.acceptedChars.get()
}.fold(0) { acc, next ->
acc + next
}
Expand Down Expand Up @@ -95,10 +95,10 @@ abstract class CodeWhispererCodeCoverageTracker(
rangeMarker.putUserData(KEY_REMAINING_RECOMMENDATION, originalRecommendation)
runReadAction {
// also increment total tokens because accepted tokens are part of it
incrementTotalTokens(rangeMarker.document, originalRecommendation.length)
incrementTotalCharsCount(rangeMarker.document, originalRecommendation.length)
// avoid counting CodeWhisperer inserted suggestion twice in total tokens
if (rangeMarker.textRange.length in 2..49 && originalRecommendation.trim().isNotEmpty()) {
incrementTotalTokens(rangeMarker.document, -rangeMarker.textRange.length)
incrementTotalCharsCount(rangeMarker.document, -rangeMarker.textRange.length)
}
}
}
Expand Down Expand Up @@ -135,12 +135,12 @@ abstract class CodeWhispererCodeCoverageTracker(
// when event is auto closing [{(', there will be 2 separated events, both count as 1 char increase in total chars
val text = event.newFragment.toString()
if ((event.newLength == 1 && event.oldLength == 0) || (text.startsWith('\n') && text.trim().isEmpty())) {
incrementTotalTokens(event.document, 1)
incrementTotalCharsCount(event.document, 1)
return
} else if (event.newLength < 50 && text.trim().isNotEmpty()) {
// count doc changes from <50 multi character input as total user written code
// ignore all white space changes, this usually comes from IntelliJ formatting
incrementTotalTokens(event.document, event.newLength)
incrementTotalCharsCount(event.document, event.newLength)
}
}

Expand All @@ -165,19 +165,19 @@ abstract class CodeWhispererCodeCoverageTracker(

private fun incrementUnmodifiedAcceptedCharsCount(document: Document, delta: Int) {
val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() }
tokens.acceptedTokens.addAndGet(delta)
tokens.unmodifiedAcceptedChars.addAndGet(delta)
}

private fun incrementAcceptedCharsCount(document: Document, delta: Int) {
val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() }
tokens.rawAcceptedTokens.addAndGet(delta)
tokens.acceptedChars.addAndGet(delta)
}

private fun incrementTotalTokens(document: Document, delta: Int) {
private fun incrementTotalCharsCount(document: Document, delta: Int) {
val tokens = fileToTokens.getOrPut(document) { CodeCoverageTokens() }
tokens.apply {
totalTokens.addAndGet(delta)
if (totalTokens.get() < 0) totalTokens.set(0)
totalChars.addAndGet(delta)
if (totalChars.get() < 0) totalChars.set(0)
}
}

Expand Down Expand Up @@ -220,9 +220,9 @@ abstract class CodeWhispererCodeCoverageTracker(
val response = CodeWhispererClientAdaptor.getInstance(project).sendCodePercentageTelemetry(
language,
customizationArn,
rawAcceptedTokenSize,
totalTokensSize,
acceptedTokensSize
acceptedCharsCount,
totalCharsCount,
unmodifiedAcceptedCharsCount
)
LOG.debug { "Successfully sent code percentage telemetry. RequestId: ${response.responseMetadata().requestId()}" }
} catch (e: Exception) {
Expand All @@ -237,11 +237,11 @@ abstract class CodeWhispererCodeCoverageTracker(
percentage?.let { percentage ->
CodewhispererTelemetry.codePercentage(
project = null,
codewhispererAcceptedTokens = acceptedTokensSize,
codewhispererSuggestedTokens = rawAcceptedTokenSize,
codewhispererAcceptedTokens = unmodifiedAcceptedCharsCount,
codewhispererSuggestedTokens = acceptedCharsCount,
codewhispererLanguage = language.toTelemetryType(),
codewhispererPercentage = percentage,
codewhispererTotalTokens = totalTokensSize,
codewhispererTotalTokens = totalCharsCount,
successCount = myServiceInvocationCount.get().toLong(),
codewhispererCustomizationArn = customizationArn,
credentialStartUrl = getCodeWhispererStartUrl(project)
Expand Down Expand Up @@ -300,14 +300,8 @@ class DefaultCodeWhispererCodeCoverageTracker(project: Project, language: CodeWh
AtomicInteger(0)
)

class CodeCoverageTokens(totalTokens: Int = 0, acceptedTokens: Int = 0, rawAcceptedTokens: Int = 0) {
val totalTokens: AtomicInteger
val acceptedTokens: AtomicInteger
val rawAcceptedTokens: AtomicInteger

init {
this.totalTokens = AtomicInteger(totalTokens)
this.acceptedTokens = AtomicInteger(acceptedTokens)
this.rawAcceptedTokens = AtomicInteger(rawAcceptedTokens)
}
class CodeCoverageTokens(totalChars: Int = 0, unmodifiedAcceptedChars: Int = 0, acceptedChars: Int = 0) {
val totalChars = AtomicInteger(totalChars)
val unmodifiedAcceptedChars = AtomicInteger(unmodifiedAcceptedChars)
val acceptedChars = AtomicInteger(acceptedChars)
}
2 changes: 1 addition & 1 deletion .../software/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererClientAdaptorTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ class CodeWhispererClientAdaptorTest {
@Test
fun `sendTelemetryEvent for userModification respects telemetry optin status`() {
sendTelemetryEventOptOutCheckHelper {
sut.sendUserModificationTelemetry(aString(), aString(), aProgrammingLanguage(), aString(), 0.0)
sut.sendUserModificationTelemetry(aString(), aString(), aProgrammingLanguage(), aString(), 0, 0)
}
}

Expand Down
90 changes: 33 additions & 57 deletions ...are/aws/toolkits/jetbrains/services/codewhisperer/CodeWhispererCodeCoverageTrackerTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeCove
import software.aws.toolkits.jetbrains.services.codewhisperer.telemetry.CodeWhispererCodeCoverageTracker
import software.aws.toolkits.jetbrains.services.codewhisperer.toolwindow.CodeWhispererCodeReferenceManager
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererConstants.TOTAL_SECONDS_IN_MINUTE
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CodeWhispererUtil.getUnmodifiedAcceptedCharsCount
import software.aws.toolkits.jetbrains.services.codewhisperer.util.CrossFileStrategy
import software.aws.toolkits.jetbrains.services.telemetry.NoOpPublisher
import software.aws.toolkits.jetbrains.services.telemetry.TelemetryService
Expand Down Expand Up @@ -253,7 +254,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
val captor = argumentCaptor<DocumentEvent>()
verify(sut, Times(1)).documentChanged(captor.capture())
assertThat(captor.firstValue.newFragment.toString()).isEqualTo(keystrokeInput)
assertThat(sut.totalTokensSize).isEqualTo(keystrokeInput.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(keystrokeInput.length.toLong())
}

@Test
Expand All @@ -271,15 +272,15 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
val captor = argumentCaptor<DocumentEvent>()
verify(sut, Times(1)).documentChanged(captor.capture())
assertThat(captor.firstValue.newFragment.toString()).isEqualTo(pythonTestLeftContext)
assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong())

val anotherCode = "(x, y):".repeat(8)
runInEdtAndWait {
WriteCommandAction.runWriteCommandAction(project) {
fixture.editor.appendString(anotherCode)
}
}
assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong())
}

@Test
Expand All @@ -293,7 +294,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov

CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut
sut.activateTrackerIfNotActive()
assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong())

runInEdtAndWait {
fixture.editor.caretModel.primaryCaret.moveToOffset(fixture.editor.document.textLength)
Expand All @@ -302,7 +303,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
}
}

assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong())
}

@Test
Expand All @@ -315,15 +316,15 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
)
CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererPython.INSTANCE] = sut
sut.activateTrackerIfNotActive()
assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length.toLong())

runInEdtAndWait {
WriteCommandAction.runWriteCommandAction(project) {
fixture.editor.document.insertString(fixture.editor.caretModel.offset, "\t")
}
}

assertThat(sut.totalTokensSize).isEqualTo(pythonTestLeftContext.length + 1L)
assertThat(sut.totalCharsCount).isEqualTo(pythonTestLeftContext.length + 1L)
}

@Test
Expand Down Expand Up @@ -366,36 +367,14 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov

sut.activateTrackerIfNotActive()
assertThat(sut.activeRequestCount()).isEqualTo(1)
assertThat(sut.acceptedTokensSize).isEqualTo("bar".length.toLong())
assertThat(sut.totalTokensSize).isEqualTo("foobar".length.toLong())
assertThat(sut.unmodifiedAcceptedCharsCount).isEqualTo("bar".length.toLong())
assertThat(sut.totalCharsCount).isEqualTo("foobar".length.toLong())

sut.forceTrackerFlush()

assertThat(sut.activeRequestCount()).isEqualTo(1)
assertThat(sut.acceptedTokensSize).isEqualTo(0)
assertThat(sut.totalTokensSize).isEqualTo(0)
}

@Test
fun `test when rangeMarker is not vaild, acceptedToken will not be updated`() {
// when user delete whole recommendation, rangeMarker will be isValid = false
val rangeMarkerMock: RangeMarker = mock()
whenever(rangeMarkerMock.isValid).thenReturn(false)
sut = spy(
TestCodePercentageTracker(
project,
TOTAL_SECONDS_IN_MINUTE,
CodeWhispererPython.INSTANCE,
mutableListOf(rangeMarkerMock),
)
) {
onGeneric { getUnmodifiedAcceptedCharsCount(any(), any()) } doReturn 100
}

sut.activateTrackerIfNotActive()
sut.forceTrackerFlush()

verify(sut, Times(0)).getUnmodifiedAcceptedCharsCount(any(), any())
assertThat(sut.unmodifiedAcceptedCharsCount).isEqualTo(0)
assertThat(sut.totalCharsCount).isEqualTo(0)
}

@Test
Expand Down Expand Up @@ -430,12 +409,11 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
TOTAL_SECONDS_IN_MINUTE,
CodeWhispererPython.INSTANCE,
mutableListOf(rangeMarkerMock1),
mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = 100, acceptedTokens = 0, rawAcceptedTokens = 0)),
mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalChars = 100, unmodifiedAcceptedChars = 0, acceptedChars = 0)),
1
)
) {
onGeneric { extractRangeMarkerString(any()) } doReturn "fou"
onGeneric { getUnmodifiedAcceptedCharsCount(any(), any()) } doReturn 1
}
sut.emitCodeWhispererCodeContribution()

Expand All @@ -445,8 +423,7 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
metricCaptor.allValues,
CODE_PERCENTAGE,
1,
CWSPR_PERCENTAGE to "3",
CWSPR_ACCEPTED_TOKENS to "1",
CWSPR_ACCEPTED_TOKENS to "2",
CWSPR_RAW_ACCEPTED_TOKENS to "3",
CWSPR_TOTAL_TOKENS to "100",
)
Expand All @@ -465,48 +442,47 @@ internal class CodeWhispererCodeCoverageTrackerTestPython : CodeWhispererCodeCov
}

@Test
fun `test getAcceptedTokensDelta()`() {
val tracker = TestCodePercentageTracker(project, TOTAL_SECONDS_IN_MINUTE, CodeWhispererPython.INSTANCE)
fun `test getUnmodifiedAcceptedCharsCount()`() {
var originalRecommendation = "foo"
var modifiedRecommendation = "fou"
var delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo(2)
var unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo(2)

originalRecommendation = "foo"
modifiedRecommendation = "f11111oo"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo(3)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo(3)

originalRecommendation = "foo"
modifiedRecommendation = "fo"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo(2)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo(2)

originalRecommendation = "helloworld"
modifiedRecommendation = "HelloWorld"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo("helloworld".length - 2)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo("helloworld".length - 2)

originalRecommendation = "helloworld"
modifiedRecommendation = "World"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo("helloworld".length - "hello".length - 1)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo("helloworld".length - "hello".length - 1)

originalRecommendation = "CodeWhisperer"
modifiedRecommendation = "CODE"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo(1)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo(1)

originalRecommendation = "CodeWhisperer"
modifiedRecommendation = "codewhispererISBEST"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo("CodeWhisperer".length - 2)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo("CodeWhisperer".length - 2)

val pythonCommentAddedByUser = "\"\"\"we don't count this comment as generated by CodeWhisperer\"\"\"\n"
originalRecommendation = "x, y):\n\treturn x + y"
modifiedRecommendation = "x, y):\n$pythonCommentAddedByUser\treturn x + y"
delta = tracker.getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(delta).isEqualTo(originalRecommendation.length)
unmodifiedCharsCount = getUnmodifiedAcceptedCharsCount(originalRecommendation, modifiedRecommendation)
assertThat(unmodifiedCharsCount).isEqualTo(originalRecommendation.length)
}

@Test
Expand Down Expand Up @@ -557,7 +533,7 @@ internal class CodeWhispererCodeCoverageTrackerTestJava : CodeWhispererCodeCover
project,
TOTAL_SECONDS_IN_MINUTE,
language = CodeWhispererJava.INSTANCE,
codeCoverageTokens = mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalTokens = codeNeedToBeReformatted.length))
codeCoverageTokens = mutableMapOf(fixture.editor.document to CodeCoverageTokens(totalChars = codeNeedToBeReformatted.length))
)
)
CodeWhispererCodeCoverageTracker.getInstancesMap()[CodeWhispererJava.INSTANCE] = sut
Expand All @@ -568,7 +544,7 @@ internal class CodeWhispererCodeCoverageTrackerTestJava : CodeWhispererCodeCover
}
// reformat should fire documentChanged events, but tracker should not update token from these events
verify(sut, atLeastOnce()).documentChanged(any())
assertThat(sut.totalTokensSize).isEqualTo(codeNeedToBeReformatted.length.toLong())
assertThat(sut.totalCharsCount).isEqualTo(codeNeedToBeReformatted.length.toLong())

val formatted = """
class Answer {
Expand Down

0 comments on commit 5016751

Please sign in to comment.