package com.example.aiapp /** * A language the highlighter can colour. * * The names the reader writes after the backticks are aliases onto these; [fenceLanguage] holds * that table. A word with no entry there is null, and null is drawn plain, because a fence coloured * by another language's rules looks highlighted and is wrong in a way the reader cannot see. * * Nearly all of them are a row of [RULES], read by one shared scanner. [MARKDOWN] is the one that * is not; see [spansOf]. */ enum class Language { C, COFFEESCRIPT, CPP, CSHARP, DART, FISH, GO, JAVA, JAVASCRIPT, JSON, KOTLIN, MARKDOWN, PERL, PHP, PYTHON, RON, RUBY, RUST, SHELL, SWIFT, TOML, TYPESCRIPT, } /** * What [scan] needs to know about one language -- data, not code, so that adding a language is a * row in [RULES] rather than a branch anywhere. * * The two forms that could not be expressed as data are flags here and a few lines in the scanner: * [rawStrings], because the closing delimiter depends on how many hashes the opener had, and * [lifetimes], because whether `'` opens anything at all depends on what follows it. */ data class Rules( /** Words drawn as keywords. Only plain words; the scanner cannot reach anything else. */ val keywords: Set, /** Tokens that open a comment running to the end of the line. */ val lineComments: List = emptyList(), /** * Whether [lineComments] count only at the start of a word. The shells need it: `$#`, `${#x}` * and `a#b` are not comments, and greying the rest of those lines is one of the mistakes this * scanner exists to stop. */ val lineCommentsAtWordStart: Boolean = false, val blockComment: BlockComment? = null, /** The string forms. The longest opener that matches wins, so `"""` is tried before `"`. */ val quotes: List = emptyList(), val attributes: Attributes = Attributes.NONE, /** Rust and RON: an optional `b`, `r`, n hashes, `"`, closing at `"` and n hashes. */ val rawStrings: Boolean = false, /** * Rust: `'` opens a character literal only when a backslash or one character and a `'` follow. * Otherwise it is a lifetime or a label -- without this, `'a` opens a string that runs to the * next apostrophe in the block. */ val lifetimes: Boolean = false, ) data class BlockComment(val open: String, val close: String, val nests: Boolean) /** One string form. [escapes] is whether a backslash escapes the closer (and itself). */ data class Quote(val open: String, val close: String, val escapes: Boolean) /** What opens a metadata span, of the shapes that exist across these languages. */ enum class Attributes { NONE, /** `@` and a word: Kotlin and Java annotations, Python decorators. */ AT_WORD, /** `#[` or `#![` through the matching `]`: Rust and RON attributes. */ HASH_BRACKET, /** `#` at the start of a line, to the end of it: the C preprocessor. */ HASH_LINE, /** `[` at the start of a line through the matching `]`: a TOML table header. */ LINE_BRACKET, } /** * The spans [language] colours in [code] -- the one way to ask, whatever the language turns out to * be made of. * * Nearly every language here is tokens, which is a row of [RULES] and the one shared scanner. * Markdown has none of those, and what a character means there depends on where on the line it * sits, so it brings a scanner of its own. That is the whole extension point -- a new language is a * row of rules or an entry in [SCANNERS], and no caller learns which one it got. */ fun spansOf(code: String, language: Language): List = SCANNERS.getValue(language)(code) // Lazy for the same reason [RULES] is, since it reads it. private val SCANNERS: Map List> by lazy { RULES.mapValues { (_, rules) -> { code: String -> scan(code, rules) } } + mapOf(Language.MARKDOWN to ::scanMarkdown) } private val C_STYLE = BlockComment("/*", "*/", nests = false) private val NESTING = BlockComment("/*", "*/", nests = true) private val DOUBLE = Quote("\"", "\"", escapes = true) private val SINGLE = Quote("'", "'", escapes = true) private val TRIPLE_DOUBLE = Quote("\"\"\"", "\"\"\"", escapes = true) private val TRIPLE_SINGLE = Quote("'''", "'''", escapes = true) // Lazy because the keyword sets below are top-level properties too, and a file's properties // initialize in the order they are written: read eagerly here, every set would be null. private val RULES: Map by lazy { mapOf( Language.C to Rules( keywords = KEYWORDS_C, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE), attributes = Attributes.HASH_LINE, ), Language.CPP to Rules( keywords = KEYWORDS_CPP, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE), attributes = Attributes.HASH_LINE, ), Language.CSHARP to Rules( keywords = KEYWORDS_CSHARP, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE), ), // `###` opens and closes a block comment and `#` opens a line one, which is why the scanner // tries the block opener first. Language.COFFEESCRIPT to Rules( keywords = KEYWORDS_COFFEESCRIPT, lineComments = listOf("#"), blockComment = BlockComment("###", "###", nests = false), quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE), ), Language.DART to Rules( keywords = KEYWORDS_DART, lineComments = listOf("//"), blockComment = NESTING, quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE), attributes = Attributes.AT_WORD, ), Language.FISH to Rules( keywords = KEYWORDS_FISH, lineComments = listOf("#"), lineCommentsAtWordStart = true, // fish's single quotes escape only `\'` and `\\`, which is what "skip the character // after a backslash" already does. quotes = listOf(DOUBLE, SINGLE), ), Language.GO to Rules( keywords = KEYWORDS_GO, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = false)), ), Language.JAVA to Rules( keywords = KEYWORDS_JAVA, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE), attributes = Attributes.AT_WORD, ), Language.JAVASCRIPT to Rules( keywords = KEYWORDS_JAVASCRIPT, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = true)), ), Language.JSON to Rules(keywords = KEYWORDS_JSON, quotes = listOf(DOUBLE)), Language.KOTLIN to Rules( keywords = KEYWORDS_KOTLIN, lineComments = listOf("//"), blockComment = NESTING, quotes = listOf(Quote("\"\"\"", "\"\"\"", escapes = false), DOUBLE, SINGLE), attributes = Attributes.AT_WORD, ), Language.PERL to Rules( keywords = KEYWORDS_PERL, lineComments = listOf("#"), quotes = listOf(DOUBLE, SINGLE), ), Language.PHP to Rules( keywords = KEYWORDS_PHP, lineComments = listOf("//", "#"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE), attributes = Attributes.AT_WORD, ), Language.PYTHON to Rules( keywords = KEYWORDS_PYTHON, lineComments = listOf("#"), quotes = listOf(TRIPLE_DOUBLE, TRIPLE_SINGLE, DOUBLE, SINGLE), attributes = Attributes.AT_WORD, ), Language.RON to Rules( keywords = KEYWORDS_RON, lineComments = listOf("//"), blockComment = NESTING, quotes = listOf(DOUBLE, SINGLE), attributes = Attributes.HASH_BRACKET, rawStrings = true, ), Language.RUBY to Rules( keywords = KEYWORDS_RUBY, lineComments = listOf("#"), quotes = listOf(DOUBLE, SINGLE), ), Language.RUST to Rules( keywords = KEYWORDS_RUST, lineComments = listOf("//"), blockComment = NESTING, // No `'` here: [Rules.lifetimes] decides when one opens a character literal. quotes = listOf(DOUBLE), attributes = Attributes.HASH_BRACKET, rawStrings = true, lifetimes = true, ), Language.SHELL to Rules( keywords = KEYWORDS_SHELL, lineComments = listOf("#"), lineCommentsAtWordStart = true, // A shell's single quotes are literal: `'a\'` is not one string. quotes = listOf(DOUBLE, Quote("'", "'", escapes = false)), ), Language.SWIFT to Rules( keywords = KEYWORDS_SWIFT, lineComments = listOf("//"), blockComment = NESTING, quotes = listOf(TRIPLE_DOUBLE, DOUBLE), attributes = Attributes.AT_WORD, ), Language.TOML to Rules( keywords = KEYWORDS_TOML, lineComments = listOf("#"), quotes = listOf( TRIPLE_DOUBLE, Quote("'''", "'''", escapes = false), DOUBLE, Quote("'", "'", escapes = false), ), attributes = Attributes.LINE_BRACKET, ), Language.TYPESCRIPT to Rules( keywords = KEYWORDS_TYPESCRIPT, lineComments = listOf("//"), blockComment = C_STYLE, quotes = listOf(DOUBLE, SINGLE, Quote("`", "`", escapes = true)), attributes = Attributes.AT_WORD, ), ) } /** * The keyword sets. * * Every list below other than RON, TOML, fish and JSON came from dev.snipme:highlights 1.1.0 * (Apache-2.0), the library this scanner replaced, so that no fence which is coloured today turns * plain. Entries that are not plain words were dropped -- Kotlin's `as?`, Swift's `#if` family, * Ruby's `defined?` -- because the word scanner cannot reach them. */ private fun words(list: String): Set = list.split(Regex("\\s+")).filterNot(String::isEmpty).toSet() private val KEYWORDS_C = words( """auto break case char const continue default do double else enum extern float for goto if int long register return short signed sizeof static struct switch typedef union unsigned void volatile while""" ) private val KEYWORDS_CPP = words( """asm auto bool break case catch char class const const_cast continue default delete do double dynamic_cast else enum explicit export extern false float for friend goto if inline int long mutable namespace new operator private protected public register reinterpret_cast return short signed sizeof static static_cast struct switch template this throw true try typedef typeid typename union unsigned using virtual void volatile wchar_t while""" ) private val KEYWORDS_CSHARP = words( """abstract as base bool break byte case catch char checked class const continue decimal default delegate do double else enum event explicit extern false finally fixed float for foreach goto if implicit in int interface internal is lock long namespace new null object operator out override params private protected public readonly ref return sbyte sealed short sizeof stackalloc static string struct switch this throw true try typeof uint ulong unchecked unsafe ushort using virtual void volatile while""" ) private val KEYWORDS_COFFEESCRIPT = words( """Infinity NaN and arguments await break by case catch class continue debugger delete defer default do else export extends false finally for function if import in instanceof is isnt let loop new no not null of on or package return super switch this throw true try typeof unless undefined var wait when with yield""" ) private val KEYWORDS_DART = words( """abstract as assert async await base break case catch class const continue covariant default deferred do dynamic else enum export extends external factory false final finally for get if implements import in interface is late library mixin new null on operator part required rethrow return sealed set show static super switch this throw true try var void when with while yield""" ) /** * fish is not in the library at all, so its fences are drawn plain today. The list is the shell's * own words, which is what a fish fence is mostly made of. */ private val KEYWORDS_FISH = words( """and begin break builtin case command continue else end exec for function if in not or return switch while set echo test string math read source""" ) private val KEYWORDS_GO = words( """break case chan const continue default defer else fallthrough false for func go goto if import interface map package range return select struct switch true type var""" ) private val KEYWORDS_JAVA = words( """abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for goto if implements import instanceof int interface long native new null package private protected public return short static strictfp super switch synchronized this throw throws transient try void volatile while""" ) private val KEYWORDS_JAVASCRIPT = words( """async await boolean break case catch class const continue debugger default delete do else enum export extends false finally for function if implements import in instanceof interface let new null package private protected public return super switch this throw true try typeof var void while with yield""" ) private val KEYWORDS_JSON = words("true false null") private val KEYWORDS_KOTLIN = words( """actual abstract annotation as break by catch class companion const constructor continue coroutine crossinline data delegate dynamic do else enum expect external false final finally for fun get if import in infix inline interface internal is lazy lateinit native null object open operator out override package private protected public reified return sealed set super suspend tailrec this throw true try typealias typeof val var vararg when while yield""" ) private val KEYWORDS_PERL = words( """__DATA__ __END__ __FILE__ __LINE__ __PACKAGE__ and cmp continue do else elsif eq eval for foreach goto gt if last le lt my ne next no not or package redo ref return sub unless until use while xor""" ) private val KEYWORDS_PHP = words( """__halt_compiler abstract and array as break callable case catch class clone const continue declare default die do echo else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval exit extends final finally fn for foreach function global goto if implements include include_once instanceof insteadof interface isset list match new or print private protected public require require_once return static switch throw trait try unset use var while xor yield""" ) private val KEYWORDS_PYTHON = words( """False True and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield""" ) /** RON is not in the library either; these are the words a RON file can hold. */ private val KEYWORDS_RON = words("true false Some None inf NaN") private val KEYWORDS_RUBY = words( """__ENCODING__ __END__ __FILE__ __LINE__ BEGIN END alias and begin break case class def do else elsif end ensure false for if in module next nil not or redo rescue retry return self super then true undef unless until when while yield""" ) private val KEYWORDS_RUST = words( """as async await break const continue crate dyn else enum extern false fn for if impl in let loop match mod move mut pub ref return Self self static struct super trait true type union unsafe use where while abstract become box do final macro override priv try typeof unsized virtual yield""" ) private val KEYWORDS_SHELL = words( """alias bg bind break builtin caller cd command compgen complete compopt continue declare dirs disown echo enable eval exec exit export fc fg getopts hash help history jobs kill let local logout popd printf pushd pwd read readonly return set shift shopt source suspend test""" ) private val KEYWORDS_SWIFT = words( """_ associatedtype class deinit enum extension fileprivate func import init inout internal let open operator private precedencegroup protocol public rethrows static struct subscript typealias var break case catch continue default defer do else fallthrough for guard if in repeat return throw switch where while Any as await false is nil self Self super throws true try associativity convenience didSet dynamic final get indirect infix lazy left mutating none nonmutating optional override postfix precedence prefix Protocol required right set some Type unowned weak willSet""" ) /** TOML is not in the library; `inf` and `nan` are values rather than names, like the booleans. */ private val KEYWORDS_TOML = words("true false inf nan") private val KEYWORDS_TYPESCRIPT = words( """abstract as asserts await break case catch class const constructor continue debugger default delete do else enum export extends false finally for from function get if implements import in infer instanceof interface is keyof let module namespace new null number object package private protected public readonly require global return set static string super switch this throw true try type typeof undefined unique unknown var void while with yield""" )