package core:text/regex/tokenizer

Source

Error :: enum int {
	None, 
	Illegal_Null_Character, 
	Illegal_Codepoint, 
	Illegal_Byte_Order_Mark, 
}

Token :: struct {
	kind: Token_Kind,
	text: string,
	pos:  int,
}

Token_Kind :: enum int {
	Invalid, 
	EOF, 
	Rune, 
	Wildcard, 
	Alternate, 
	Concatenate, 
	Repeat_Zero, 
	Repeat_Zero_Non_Greedy, 
	Repeat_One, 
	Repeat_One_Non_Greedy, 
	Repeat_N, 
	Optional, 
	Optional_Non_Greedy, 
	Rune_Class, 
	Open_Paren, 
	Open_Paren_Non_Capture, 
	Close_Paren, 
	Anchor_Start, 
	Anchor_End, 
	Word_Boundary, 
	Non_Word_Boundary, 
}

Tokenizer :: struct {
	flags:           bit_set[regex_common.Flag; u8],
	src:             string,
	ch:              rune,
	offset:          int,
	read_offset:     int,
	last_token_kind: Token_Kind,
	held_token:      Token,
	error_state:     Error,
	paren_depth:     int,
}

advance_rune :: proc(t: ^Tokenizer) -> (err: Error) {…}

init :: proc(t: ^Tokenizer, str: string, flags: bit_set[regex_common.Flag; u8]) {…}

peek_byte :: proc(t: ^Tokenizer, offset: int = 0) -> u8 {…}

scan :: proc(t: ^Tokenizer) -> (token: Token) {…}

scan_class :: proc(t: ^Tokenizer) -> (str: string, ok: bool) {…}

scan_comment :: proc(t: ^Tokenizer) {…}

scan_non_capture_group :: proc(t: ^Tokenizer) -> bool {…}

scan_non_greedy :: proc(t: ^Tokenizer) -> bool {…}

scan_repeat :: proc(t: ^Tokenizer) -> (str: string, ok: bool) {…}