support multiline tokens by considering source code string when encoding

library/src/main/kotlin/com/strumenta/kolasu/languageserver/KolasuServer.kt

@@ -583,8 +583,9 @@ open class KolasuServer<T : Node>(
 
     override fun semanticTokensFull(params: SemanticTokensParams): CompletableFuture<SemanticTokens> {
         val ast = files[params.textDocument.uri]?.root ?: return CompletableFuture.completedFuture(SemanticTokens())
+        val code = files[params.textDocument.uri]?.code ?: return CompletableFuture.completedFuture(SemanticTokens())
         val tokens = semanticTokens(ast)
-        return CompletableFuture.completedFuture(encode(tokens))
+        return CompletableFuture.completedFuture(encode(tokens, code))
     }
 
     open fun semanticTokens(ast: T): List<SemanticToken> = listOf()

library/src/main/kotlin/com/strumenta/kolasu/languageserver/semanticHighlighting/SemanticToken.kt

@@ -1,24 +1,29 @@
 package com.strumenta.kolasu.languageserver.semanticHighlighting
 
+import com.strumenta.kolasu.model.Point
 import com.strumenta.kolasu.model.Position
 import org.eclipse.lsp4j.SemanticTokens
 
 data class SemanticToken(val position: Position, val type: SemanticTokenType, val modifiers: List<SemanticTokenModifier>)
 
-fun encode(tokens: List<SemanticToken>): SemanticTokens {
-    var lastLine = 1 // ? To offset that Kolasu lines start at 1
-    var lastColumn = 0
+fun encode(tokens: List<SemanticToken>, code: String): SemanticTokens {
+    // Create synthetic token at start to compute relative positions from
+    var lastToken = SemanticToken(Position(Point(1, 0), Point(1, 0)), SemanticTokenType.TYPE, listOf())
+
     val data = mutableListOf<Int>()
     for (token in tokens) {
-        data.addAll(listOf(
-            token.position.start.line - lastLine,
-            if (token.position.start.line == lastLine) token.position.start.column - lastColumn else token.position.start.column,
-            token.position.end.column - token.position.start.column, // ! assumes tokens are in a single line
-            token.type.ordinal,
-            token.modifiers.sumOf { it.bit }
-        ))
-        lastLine = token.position.start.line
-        lastColumn = token.position.start.column
+        val deltaLine = token.position.start.line - (lastToken.position.start.line)
+        val deltaStart = if (token.position.start.line == lastToken.position.start.line) {
+            token.position.start.column - lastToken.position.start.column
+        } else {
+            token.position.start.column
+        }
+        val length = token.position.length(code)
+        val tokenType = token.type.ordinal
+        val tokenModifiers = token.modifiers.sumOf { it.bit }
+
+        data.addAll(listOf(deltaLine, deltaStart, length, tokenType, tokenModifiers))
+        lastToken = token
     }
     return SemanticTokens(data)
 }