package core:text/scanner

Source

Position :: struct {
	filename: string,
	// filename, if present
	offset:   int,
	// byte offset, starting @ 0
	line:     int,
	// line number, starting @ 1
	column:   int,
}

Scan_Flag :: enum u32 {
	Scan_Idents, 
	Scan_Ints, 
	Scan_C_Int_Prefixes, 
	Scan_Floats,         // Includes integers and hexadecimal floats
	Scan_Chars, 
	Scan_Strings, 
	Scan_Raw_Strings, 
	Scan_Comments, 
	Skip_Comments,       // if set with .Scan_Comments, comments become white space
}

Scan_Flags :: distinct bit_set[Scan_Flag; u32]

Scanner :: struct {
	src:           string,
	src_pos:       int,
	src_end:       int,
	tok_pos:       int,
	tok_end:       int,
	ch:            rune,
	line:          int,
	column:        int,
	prev_line_len: int,
	prev_char_len: int,
	// error is called for each error encountered
	// If no error procedure is set, the error is reported to os.stderr
	error:         proc(s: ^Scanner, msg: string),
	// error_count is incremented by one for each error encountered
	error_count:   int,
	// flags controls which tokens are recognized
	// e.g. to recognize integers, set the .Scan_Ints flag
	// This field may be changed by the user at any time during scanning
	flags:         Scan_Flags,
	// The whitespace field controls which characters are recognized as white space
	// This field may be changed by the user at any time during scanning
	whitespace:    Whitespace,
	// is_ident_rune is a predicate controlling the characters accepted as the ith rune in an identifier
	// The valid characters must not conflict with the set of white space characters
	// If is_ident_rune is not set, regular Odin-like identifiers are accepted
	// This field may be changed by the user at any time during scanning
	is_ident_rune: proc(ch: rune, i: int) -> bool,
	// Start position of most recently scanned token (set by scan(s))
	// Call init or next invalidates the position
	pos:           Position,
}

Whitespace :: distinct bit_set[rune; u128]

C_Like_Tokens :: Scan_Flags{.Scan_Idents, .Scan_Ints, .Scan_C_Int_Prefixes, .Scan_Floats, .Scan_Chars, .Scan_Strings, .Scan_Raw_Strings, .Scan_Comments, .Skip_Comments}

C_Whitespace :: Whitespace{'\t', '\n', '\r', '\v', '\f', ' '}

Char :: -5

Comment :: -8

EOF :: -1

Float :: -4

Ident :: -2

Int :: -3

Odin_Like_Tokens :: Scan_Flags{.Scan_Idents, .Scan_Ints, .Scan_Floats, .Scan_Chars, .Scan_Strings, .Scan_Raw_Strings, .Scan_Comments, .Skip_Comments}

Odin_Whitespace :: Whitespace{'\t', '\n', '\r', ' '}

Raw_String :: -7

String :: -6

error :: proc(s: ^Scanner, msg: string) {…}

errorf :: proc(s: ^Scanner, format: string, .. args: ..any) {…}

init :: proc(s: ^Scanner, src: string, filename: string = "") -> ^Scanner {…}

next :: proc(s: ^Scanner) -> rune {…}

peek :: proc(s: ^Scanner, n: int = 0) -> (ch: rune) {…}

peek_token :: proc(s: ^Scanner, n: int = 0) -> (tok: rune) {…}

position :: proc(s: ^Scanner) -> Position {…}

position_is_valid :: proc(pos: Position) -> bool {…}

position_to_string :: proc(pos: Position, allocator := context.temp_allocator) -> string {…}

scan :: proc(s: ^Scanner) -> (tok: rune) {…}

token_string :: proc(tok: rune, allocator: runtime.Allocator) -> string {…}

token_text :: proc(s: ^Scanner) -> string {…}