Autocomplete: Remove trailing whitespace (#542)

I noticed some odd duplications in multi-line completion snippets that
was caused by a difference only in trailing whitespace. Since my IDE
does this on save automatically anyways, I figure it's save to filter
the LLM output for this. I noticed this to be an issue in both Anthropic
and StarCoder, but Anthropic encounters it more often.

## Test plan

- Added a test case


<!-- Required. See
https://docs.sourcegraph.com/dev/background-information/testing_principles.
-->

Author: philipp-spiess

vscode/src/completions/shared-post-process.ts

@@ -1,5 +1,5 @@
 import { truncateMultilineCompletion } from './multiline'
-import { collapseDuplicativeWhitespace, trimUntilSuffix } from './text-processing'
+import { collapseDuplicativeWhitespace, removeTrailingWhitespace, trimUntilSuffix } from './text-processing'
 import { Completion } from './types'
 
 /**
@@ -23,6 +23,7 @@ export function sharedPostProcess({
 
     if (multiline) {
         content = truncateMultilineCompletion(content, prefix, suffix, languageId)
+        content = removeTrailingWhitespace(content)
     }
     content = trimUntilSuffix(content, prefix, suffix, languageId)
     content = collapseDuplicativeWhitespace(prefix, content)

vscode/src/completions/text-processing.ts

@@ -223,3 +223,10 @@ export function collapseDuplicativeWhitespace(prefix: string, completion: string
 export function trimEndOnLastLineIfWhitespaceOnly(text: string): string {
     return text.replace(/(\r?\n)\s+$/, '$1')
 }
+
+export function removeTrailingWhitespace(text: string): string {
+    return text
+        .split('\n')
+        .map(l => l.trimEnd())
+        .join('\n')
+}

vscode/src/completions/vscodeInlineCompletionItemProvider.test.ts

@@ -293,6 +293,28 @@ describe('Cody completions', () => {
     })
 
     describe('multi-line completions', () => {
+        it('removes trailing spaces', async () => {
+            const { completions } = await complete(
+                dedent`
+                    function bubbleSort() {
+                        █
+                    }`,
+                [
+                    completion`
+                            ├console.log('foo')${' '}
+                            console.log('bar')${'    '}
+                            console.log('baz')${'  '}┤
+                        ┴┴┴┴`,
+                ]
+            )
+
+            expect(completions[0].insertText).toMatchInlineSnapshot(`
+              "console.log('foo')
+                  console.log('bar')
+                  console.log('baz')"
+            `)
+        })
+
         it('honors a leading new line in the completion', async () => {
             const { completions } = await complete(
                 dedent`