1package chroma
  2
  3import (
  4	"fmt"
  5	"strings"
  6)
  7
  8var (
  9	defaultOptions = &TokeniseOptions{
 10		State:    "root",
 11		EnsureLF: true,
 12	}
 13)
 14
 15// Config for a lexer.
 16type Config struct {
 17	// Name of the lexer.
 18	Name string `xml:"name,omitempty"`
 19
 20	// Shortcuts for the lexer
 21	Aliases []string `xml:"alias,omitempty"`
 22
 23	// File name globs
 24	Filenames []string `xml:"filename,omitempty"`
 25
 26	// Secondary file name globs
 27	AliasFilenames []string `xml:"alias_filename,omitempty"`
 28
 29	// MIME types
 30	MimeTypes []string `xml:"mime_type,omitempty"`
 31
 32	// Regex matching is case-insensitive.
 33	CaseInsensitive bool `xml:"case_insensitive,omitempty"`
 34
 35	// Regex matches all characters.
 36	DotAll bool `xml:"dot_all,omitempty"`
 37
 38	// Regex does not match across lines ($ matches EOL).
 39	//
 40	// Defaults to multiline.
 41	NotMultiline bool `xml:"not_multiline,omitempty"`
 42
 43	// Don't strip leading and trailing newlines from the input.
 44	// DontStripNL bool
 45
 46	// Strip all leading and trailing whitespace from the input
 47	// StripAll bool
 48
 49	// Make sure that the input ends with a newline. This
 50	// is required for some lexers that consume input linewise.
 51	EnsureNL bool `xml:"ensure_nl,omitempty"`
 52
 53	// If given and greater than 0, expand tabs in the input.
 54	// TabSize int
 55
 56	// Priority of lexer.
 57	//
 58	// If this is 0 it will be treated as a default of 1.
 59	Priority float32 `xml:"priority,omitempty"`
 60}
 61
 62// Token output to formatter.
 63type Token struct {
 64	Type  TokenType `json:"type"`
 65	Value string    `json:"value"`
 66}
 67
 68func (t *Token) String() string   { return t.Value }
 69func (t *Token) GoString() string { return fmt.Sprintf("&Token{%s, %q}", t.Type, t.Value) }
 70
 71// Clone returns a clone of the Token.
 72func (t *Token) Clone() Token {
 73	return *t
 74}
 75
 76// EOF is returned by lexers at the end of input.
 77var EOF Token
 78
 79// TokeniseOptions contains options for tokenisers.
 80type TokeniseOptions struct {
 81	// State to start tokenisation in. Defaults to "root".
 82	State string
 83	// Nested tokenisation.
 84	Nested bool
 85
 86	// If true, all EOLs are converted into LF
 87	// by replacing CRLF and CR
 88	EnsureLF bool
 89}
 90
 91// A Lexer for tokenising source code.
 92type Lexer interface {
 93	// Config describing the features of the Lexer.
 94	Config() *Config
 95	// Tokenise returns an Iterator over tokens in text.
 96	Tokenise(options *TokeniseOptions, text string) (Iterator, error)
 97	// SetRegistry sets the registry this Lexer is associated with.
 98	//
 99	// The registry should be used by the Lexer if it needs to look up other
100	// lexers.
101	SetRegistry(registry *LexerRegistry) Lexer
102	// SetAnalyser sets a function the Lexer should use for scoring how
103	// likely a fragment of text is to match this lexer, between 0.0 and 1.0.
104	// A value of 1 indicates high confidence.
105	//
106	// Lexers may ignore this if they implement their own analysers.
107	SetAnalyser(analyser func(text string) float32) Lexer
108	// AnalyseText scores how likely a fragment of text is to match
109	// this lexer, between 0.0 and 1.0. A value of 1 indicates high confidence.
110	AnalyseText(text string) float32
111}
112
113// Lexers is a slice of lexers sortable by name.
114type Lexers []Lexer
115
116func (l Lexers) Len() int      { return len(l) }
117func (l Lexers) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
118func (l Lexers) Less(i, j int) bool {
119	return strings.ToLower(l[i].Config().Name) < strings.ToLower(l[j].Config().Name)
120}
121
122// PrioritisedLexers is a slice of lexers sortable by priority.
123type PrioritisedLexers []Lexer
124
125func (l PrioritisedLexers) Len() int      { return len(l) }
126func (l PrioritisedLexers) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
127func (l PrioritisedLexers) Less(i, j int) bool {
128	ip := l[i].Config().Priority
129	if ip == 0 {
130		ip = 1
131	}
132	jp := l[j].Config().Priority
133	if jp == 0 {
134		jp = 1
135	}
136	return ip > jp
137}
138
139// Analyser determines how appropriate this lexer is for the given text.
140type Analyser interface {
141	AnalyseText(text string) float32
142}