ai-app/app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt

package com.example.aiapp

/**
 * A language the highlighter can colour.
 *
 * The names the reader writes after the backticks are aliases onto these; [fenceLanguage] holds
 * that table. A word with no entry there is null, and null is drawn plain, because a fence coloured
 * by another language's rules looks highlighted and is wrong in a way the reader cannot see.
 *
 * Nearly all of them are a row of [RULES], read by one shared scanner. [MARKDOWN] is the one that
 * is not; see [spansOf].
 */
enum class Language {
    C,
    COFFEESCRIPT,
    CPP,
    CSHARP,
    DART,
    FISH,
    GO,
    JAVA,
    JAVASCRIPT,
    JSON,
    KOTLIN,
    MARKDOWN,
    PERL,
    PHP,
    PYTHON,
    RON,
    RUBY,
    RUST,
    SHELL,
    SWIFT,
    TOML,
    TYPESCRIPT,
}

/**
 * What [scan] needs to know about one language -- data, not code, so that adding a language is a
 * row in [RULES] rather than a branch anywhere.
 *
 * The two forms that could not be expressed as data are flags here and a few lines in the scanner:
 * [rawStrings], because the closing delimiter depends on how many hashes the opener had, and
 * [lifetimes], because whether `'` opens anything at all depends on what follows it.
 */
data class Rules(
    /** Words drawn as keywords. Only plain words; the scanner cannot reach anything else. */
    val keywords: Set<String>,
    /** Tokens that open a comment running to the end of the line. */
    val lineComments: List<String> = emptyList(),
    /**
     * Whether [lineComments] count only at the start of a word. The shells need it: `$#`, `${#x}`
     * and `a#b` are not comments, and greying the rest of those lines is one of the mistakes this
     * scanner exists to stop.
     */
    val lineCommentsAtWordStart: Boolean = false,
    val blockComment: BlockComment? = null,
    /** The string forms. The longest opener that matches wins, so `"""` is tried before `"`. */
    val quotes: List<Quote> = emptyList(),
    val attributes: Attributes = Attributes.NONE,
    /** Rust and RON: an optional `b`, `r`, n hashes, `"`, closing at `"` and n hashes. */
    val rawStrings: Boolean = false,
    /**
     * Rust: `'` opens a character literal only when a backslash or one character and a `'` follow.
     * Otherwise it is a lifetime or a label -- without this, `'a` opens a string that runs to the
     * next apostrophe in the block.
     */
    val lifetimes: Boolean = false,
)

data class BlockComment(val open: String, val close: String, val nests: Boolean)

/** One string form. [escapes] is whether a backslash escapes the closer (and itself). */
data class Quote(val open: String, val close: String, val escapes: Boolean)

/** What opens a metadata span, of the shapes that exist across these languages. */
enum class Attributes {
    NONE,
    /** `@` and a word: Kotlin and Java annotations, Python decorators. */
    AT_WORD,
    /** `#[` or `#![` through the matching `]`: Rust and RON attributes. */
    HASH_BRACKET,
    /** `#` at the start of a line, to the end of it: the C preprocessor. */
    HASH_LINE,
    /** `[` at the start of a line through the matching `]`: a TOML table header. */
    LINE_BRACKET,
}

/**
 * The spans [language] colours in [code] -- the one way to ask, whatever the language turns out to
 * be made of.
 *
 * Nearly every language here is tokens, which is a row of [RULES] and the one shared scanner.
 * Markdown has none of those, and what a character means there depends on where on the line it
 * sits, so it brings a scanner of its own. That is the whole extension point -- a new language is a
 * row of rules or an entry in [SCANNERS], and no caller learns which one it got.
 */
fun spansOf(code: String, language: Language): List<Span> = SCANNERS.getValue(language)(code)

// Lazy for the same reason [RULES] is, since it reads it.
private val SCANNERS: Map<Language, (String) -> List<Span>> by lazy {
    RULES.mapValues { (_, rules) -> { code: String -> scan(code, rules) } } +
        mapOf(Language.MARKDOWN to ::scanMarkdown)
}

private val C_STYLE = BlockComment("/*", "*/", nests = false)
private val NESTING = BlockComment("/*", "*/", nests = true)

private val DOUBLE = Quote("\"", "\"", escapes = true)
private val SINGLE = Quote("'", "'", escapes = true)
private val TRIPLE_DOUBLE = Quote("\"\"\"", "\"\"\"", escapes = true)
private val TRIPLE_SINGLE = Quote("'''", "'''", escapes = true)

// Lazy because the keyword sets below are top-level properties too, and a file's properties
// initialize in the order they are written: read eagerly here, every set would be null.
private val RULES: Map<Language, Rules> by lazy {
    mapOf(
        Language.C to
            Rules(
                keywords = KEYWORDS_C,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE),
                attributes = Attributes.HASH_LINE,
            ),
        Language.CPP to
            Rules(
                keywords = KEYWORDS_CPP,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE),
                attributes = Attributes.HASH_LINE,
            ),
        Language.CSHARP to
            Rules(
                keywords = KEYWORDS_CSHARP,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE),
            ),
        // `###` opens and closes a block comment and `#` opens a line one, which is why the scanner
        // tries the block opener first.
        Language.COFFEESCRIPT to
            Rules(
                keywords = KEYWORDS_COFFEESCRIPT,
                lineComments = listOf("#"),
                blockComment = BlockComment("###", "###", nests = false),
                quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE),
            ),
        Language.DART to
            Rules(
                keywords = KEYWORDS_DART,
                lineComments = listOf("//"),
                blockComment = NESTING,
                quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.FISH to
            Rules(
                keywords = KEYWORDS_FISH,
                lineComments = listOf("#"),
                lineCommentsAtWordStart = true,
                // fish's single quotes escape only `\'` and `\\`, which is what "skip the character
                // after a backslash" already does.
                quotes = listOf(DOUBLE, SINGLE),
            ),
        Language.GO to
            Rules(
                keywords = KEYWORDS_GO,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = false)),
            ),
        Language.JAVA to
            Rules(
                keywords = KEYWORDS_JAVA,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.JAVASCRIPT to
            Rules(
                keywords = KEYWORDS_JAVASCRIPT,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = true)),
            ),
        Language.JSON to Rules(keywords = KEYWORDS_JSON, quotes = listOf(DOUBLE)),
        Language.KOTLIN to
            Rules(
                keywords = KEYWORDS_KOTLIN,
                lineComments = listOf("//"),
                blockComment = NESTING,
                quotes = listOf(Quote("\"\"\"", "\"\"\"", escapes = false), DOUBLE, SINGLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.PERL to
            Rules(
                keywords = KEYWORDS_PERL,
                lineComments = listOf("#"),
                quotes = listOf(DOUBLE, SINGLE),
            ),
        Language.PHP to
            Rules(
                keywords = KEYWORDS_PHP,
                lineComments = listOf("//", "#"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.PYTHON to
            Rules(
                keywords = KEYWORDS_PYTHON,
                lineComments = listOf("#"),
                quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.RON to
            Rules(
                keywords = KEYWORDS_RON,
                lineComments = listOf("//"),
                blockComment = NESTING,
                quotes = listOf(DOUBLE, SINGLE),
                attributes = Attributes.HASH_BRACKET,
                rawStrings = true,
            ),
        Language.RUBY to
            Rules(
                keywords = KEYWORDS_RUBY,
                lineComments = listOf("#"),
                quotes = listOf(DOUBLE, SINGLE),
            ),
        Language.RUST to
            Rules(
                keywords = KEYWORDS_RUST,
                lineComments = listOf("//"),
                blockComment = NESTING,
                // No `'` here: [Rules.lifetimes] decides when one opens a character literal.
                quotes = listOf(DOUBLE),
                attributes = Attributes.HASH_BRACKET,
                rawStrings = true,
                lifetimes = true,
            ),
        Language.SHELL to
            Rules(
                keywords = KEYWORDS_SHELL,
                lineComments = listOf("#"),
                lineCommentsAtWordStart = true,
                // A shell's single quotes are literal: `'a\'` is not one string.
                quotes = listOf(DOUBLE, Quote("'", "'", escapes = false)),
            ),
        Language.SWIFT to
            Rules(
                keywords = KEYWORDS_SWIFT,
                lineComments = listOf("//"),
                blockComment = NESTING,
                quotes = listOf(TRIPLE_DOUBLE, DOUBLE),
                attributes = Attributes.AT_WORD,
            ),
        Language.TOML to
            Rules(
                keywords = KEYWORDS_TOML,
                lineComments = listOf("#"),
                quotes =
                    listOf(
                        TRIPLE_DOUBLE,
                        Quote("'''", "'''", escapes = false),
                        DOUBLE,
                        Quote("'", "'", escapes = false),
                    ),
                attributes = Attributes.LINE_BRACKET,
            ),
        Language.TYPESCRIPT to
            Rules(
                keywords = KEYWORDS_TYPESCRIPT,
                lineComments = listOf("//"),
                blockComment = C_STYLE,
                quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = true)),
                attributes = Attributes.AT_WORD,
            ),
    )
}

/**
 * The keyword sets.
 *
 * Every list below other than RON, TOML, fish and JSON came from dev.snipme:highlights 1.1.0
 * (Apache-2.0), the library this scanner replaced, so that no fence which is coloured today turns
 * plain. Entries that are not plain words were dropped -- Kotlin's `as?`, Swift's `#if` family,
 * Ruby's `defined?` -- because the word scanner cannot reach them.
 */
private fun words(list: String): Set<String> =
    list.split(Regex("\\s+")).filterNot(String::isEmpty).toSet()

private val KEYWORDS_C =
    words(
        """auto break case char const continue default do double else enum extern float for goto if
        int long register return short signed sizeof static struct switch typedef union unsigned
        void volatile while"""
    )

private val KEYWORDS_CPP =
    words(
        """asm auto bool break case catch char class const const_cast continue default delete do
        double dynamic_cast else enum explicit export extern false float for friend goto if inline
        int long mutable namespace new operator private protected public register reinterpret_cast
        return short signed sizeof static static_cast struct switch template this throw true try
        typedef typeid typename union unsigned using virtual void volatile wchar_t while"""
    )

private val KEYWORDS_CSHARP =
    words(
        """abstract as base bool break byte case catch char checked class const continue decimal
        default delegate do double else enum event explicit extern false finally fixed float for
        foreach goto if implicit in int interface internal is lock long namespace new null object
        operator out override params private protected public readonly ref return sbyte sealed short
        sizeof stackalloc static string struct switch this throw true try typeof uint ulong unchecked
        unsafe ushort using virtual void volatile while"""
    )

private val KEYWORDS_COFFEESCRIPT =
    words(
        """Infinity NaN and arguments await break by case catch class continue debugger delete defer
        default do else export extends false finally for function if import in instanceof is isnt
        let loop new no not null of on or package return super switch this throw true try typeof
        unless undefined var wait when with yield"""
    )

private val KEYWORDS_DART =
    words(
        """abstract as assert async await base break case catch class const continue covariant
        default deferred do dynamic else enum export extends external factory false final finally
        for get if implements import in interface is late library mixin new null on operator part
        required rethrow return sealed set show static super switch this throw true try var void
        when with while yield"""
    )

/**
 * fish is not in the library at all, so its fences are drawn plain today. The list is the shell's
 * own words, which is what a fish fence is mostly made of.
 */
private val KEYWORDS_FISH =
    words(
        """and begin break builtin case command continue else end exec for function if in not or
        return switch while set echo test string math read source"""
    )

private val KEYWORDS_GO =
    words(
        """break case chan const continue default defer else fallthrough false for func go goto if
        import interface map package range return select struct switch true type var"""
    )

private val KEYWORDS_JAVA =
    words(
        """abstract assert boolean break byte case catch char class const continue default do double
        else enum extends final finally float for goto if implements import instanceof int interface
        long native new null package private protected public return short static strictfp super
        switch synchronized this throw throws transient try void volatile while"""
    )

private val KEYWORDS_JAVASCRIPT =
    words(
        """async await boolean break case catch class const continue debugger default delete do else
        enum export extends false finally for function if implements import in instanceof interface
        let new null package private protected public return super switch this throw true try typeof
        var void while with yield"""
    )

private val KEYWORDS_JSON = words("true false null")

private val KEYWORDS_KOTLIN =
    words(
        """actual abstract annotation as break by catch class companion const constructor continue
        coroutine crossinline data delegate dynamic do else enum expect external false final finally
        for fun get if import in infix inline interface internal is lazy lateinit native null object
        open operator out override package private protected public reified return sealed set super
        suspend tailrec this throw true try typealias typeof val var vararg when while yield"""
    )

private val KEYWORDS_PERL =
    words(
        """__DATA__ __END__ __FILE__ __LINE__ __PACKAGE__ and cmp continue do else elsif eq eval for
        foreach goto gt if last le lt my ne next no not or package redo ref return sub unless until
        use while xor"""
    )

private val KEYWORDS_PHP =
    words(
        """__halt_compiler abstract and array as break callable case catch class clone const continue
        declare default die do echo else elseif empty enddeclare endfor endforeach endif endswitch
        endwhile eval exit extends final finally fn for foreach function global goto if implements
        include include_once instanceof insteadof interface isset list match new or print private
        protected public require require_once return static switch throw trait try unset use var
        while xor yield"""
    )

private val KEYWORDS_PYTHON =
    words(
        """False True and as assert async await break class continue def del elif else except finally
        for from global if import in is lambda nonlocal not or pass raise return try while with
        yield"""
    )

/** RON is not in the library either; these are the words a RON file can hold. */
private val KEYWORDS_RON = words("true false Some None inf NaN")

private val KEYWORDS_RUBY =
    words(
        """__ENCODING__ __END__ __FILE__ __LINE__ BEGIN END alias and begin break case class def do
        else elsif end ensure false for if in module next nil not or redo rescue retry return self
        super then true undef unless until when while yield"""
    )

private val KEYWORDS_RUST =
    words(
        """as async await break const continue crate dyn else enum extern false fn for if impl in
        let loop match mod move mut pub ref return Self self static struct super trait true type
        union unsafe use where while abstract become box do final macro override priv try typeof
        unsized virtual yield"""
    )

private val KEYWORDS_SHELL =
    words(
        """alias bg bind break builtin caller cd command compgen complete compopt continue declare
        dirs disown echo enable eval exec exit export fc fg getopts hash help history jobs kill let
        local logout popd printf pushd pwd read readonly return set shift shopt source suspend
        test"""
    )

private val KEYWORDS_SWIFT =
    words(
        """_ associatedtype class deinit enum extension fileprivate func import init inout internal
        let open operator private precedencegroup protocol public rethrows static struct subscript
        typealias var break case catch continue default defer do else fallthrough for guard if in
        repeat return throw switch where while Any as await false is nil self Self super throws true
        try associativity convenience didSet dynamic final get indirect infix lazy left mutating none
        nonmutating optional override postfix precedence prefix Protocol required right set some Type
        unowned weak willSet"""
    )

/** TOML is not in the library; `inf` and `nan` are values rather than names, like the booleans. */
private val KEYWORDS_TOML = words("true false inf nan")

private val KEYWORDS_TYPESCRIPT =
    words(
        """abstract as asserts await break case catch class const constructor continue debugger
        default delete do else enum export extends false finally for from function get if implements
        import in infer instanceof interface is keyof let module namespace new null number object
        package private protected public readonly require global return set static string super
        switch this throw true try type typeof undefined unique unknown var void while with yield"""
    )