package core:odin/tokenizer

Source

Error_Handler :: proc(pos: Pos, fmt: string, args: ..any)

Flag :: enum int {
	Insert_Semicolon, 
}

Flags :: distinct bit_set[Flag; u32]

Keyword_Hash_Entry :: struct {
	hash: u32,
	kind: Token_Kind,
	name: string,
}

Keyword_LUT :: [512]Keyword_Hash_Entry

Pos :: struct {
	file:   string,
	offset: int,
	// starting at 0
	line:   int,
	// starting at 1
	column: int,
}

Token :: struct {
	kind: Token_Kind,
	text: string,
	pos:  Pos,
}

Token_Kind :: enum u32 {
	Invalid, 
	EOF, 
	Comment, 
	File_Tag, 
	B_Literal_Begin, 
	Ident,                              // main
	Integer,                            // 12345
	Float,                              // 123.45
	Imag,                               // 123.45i
	Rune,                               // 'a'
	String,                             // "abc"
	B_Literal_End, 
	B_Operator_Begin, 
	Eq,                                 // =
	Not,                                // !
	Hash,                               // #
	At,                                 // @
	Dollar,                             // $
	Pointer,                            // ^
	Question,                           // ?
	Add, 
	Sub,                                // -
	Mul,                                // *
	Quo,                                // /
	Mod,                                // %
	Mod_Mod,                            // %%
	And,                                // &
	Or,                                 // |
	Xor,                                // ~
	And_Not,                            // &~
	Shl,                                // <<
	Shr,                                // >>
	Cmp_And,                            // &&
	Cmp_Or,                             // ||
	Mul_Mul,                            // **
	B_Assign_Op_Begin, 
	Add_Eq, 
	Sub_Eq,                             // -=
	Mul_Eq,                             // *=
	Quo_Eq,                             // /=
	Mod_Eq,                             // %=
	Mod_Mod_Eq,                         // %%=
	And_Eq,                             // &=
	Or_Eq,                              // |=
	Xor_Eq,                             // ~=
	And_Not_Eq,                         // &~=
	Shl_Eq,                             // <<=
	Shr_Eq,                             // >>=
	Cmp_And_Eq,                         // &&=
	Cmp_Or_Eq,                          // ||=
	B_Assign_Op_End, 
	Increment, 
	Decrement,                          // --
	Arrow_Right,                        // ->
	Undef,                              // ---
	B_Comparison_Begin, 
	Cmp_Eq,                             // ==
	Not_Eq,                             // !=
	Lt,                                 // <
	Gt,                                 // >
	Lt_Eq,                              // <=
	Gt_Eq,                              // >=
	B_Comparison_End, 
	Open_Paren,                         // (
	Close_Paren,                        // )
	Open_Bracket,                       // [
	Close_Bracket,                      // ]
	Open_Brace,                         // {
	Close_Brace,                        // }
	Colon,                              // :
	Semicolon,                          // ;
	Period,                             // .
	Comma,                              // ,
	Ellipsis,                           // ..
	Range_Half,                         // ..<
	Range_Full,                         // ..=
	B_Operator_End, 
	B_Keyword_Begin, 
	Import,                             // import
	Foreign,                            // foreign
	Package,                            // package
	Typeid,                             // typeid
	When,                               // when
	Where,                              // where
	If,                                 // if
	Else,                               // else
	For,                                // for
	Switch,                             // switch
	In,                                 // in
	Not_In,                             // not_in
	Do,                                 // do
	Case,                               // case
	Break,                              // break
	Continue,                           // continue
	Fallthrough,                        // fallthrough
	Defer,                              // defer
	Return,                             // return
	Proc,                               // proc
	Struct,                             // struct
	Union,                              // union
	Enum,                               // enum
	Bit_Set,                            // bit_set
	Bit_Field,                          // bit_field
	Map,                                // map
	Dynamic,                            // dynamic
	Auto_Cast,                          // auto_cast
	Cast,                               // cast
	Transmute,                          // transmute
	Distinct,                           // distinct
	Using,                              // using
	Context,                            // context
	Or_Else,                            // or_else
	Or_Return,                          // or_return
	Or_Break,                           // or_break
	Or_Continue,                        // or_continue
	Asm,                                // asm
	Matrix,                             // matrix
	B_Keyword_End, 
	COUNT, 
	B_Custom_Keyword_Begin = COUNT + 1, 
}

Tokenizer :: struct {
	// Immutable data
	path:             string,
	src:              string,
	err:              Error_Handler,
	flags:            Flags,
	// Tokenizing state
	ch:               rune,
	offset:           int,
	read_offset:      int,
	line_offset:      int,
	line_count:       int,
	insert_semicolon: bool,
	// Mutable data
	error_count:      int,
}

KEYWORD_LUT_LEN :: 1 << 9

KEYWORD_LUT_MASK :: KEYWORD_LUT_LEN - 1

custom_keyword_tokens: []string

global_keyword_lut: [512]Keyword_Hash_Entry

tokens: [118]string = …

advance_rune :: proc(t: ^Tokenizer) {…}

default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {…}

digit_val :: proc(r: rune) -> int {…}

error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {…}

init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {…}

is_assignment_operator :: proc(kind: Token_Kind) -> bool {…}

is_digit :: proc(r: rune) -> bool {…}

is_keyword :: proc(kind: Token_Kind) -> bool {…}

is_letter :: proc(r: rune) -> bool {…}

is_literal :: proc(kind: Token_Kind) -> bool {…}

is_newline :: proc(tok: Token) -> bool {…}

is_operator :: proc(kind: Token_Kind) -> bool {…}

keyword_hash :: proc(text: string) -> u32 {…}

keyword_lut_init :: proc(lut: ^[512]Keyword_Hash_Entry) -> bool {…}

peek_byte :: proc(t: ^Tokenizer, offset: int = 0) -> u8 {…}

pos_compare :: proc(lhs, rhs: Pos) -> int {…}

scan :: proc(t: ^Tokenizer) -> Token {…}

scan_comment :: proc(t: ^Tokenizer) -> string {…}

scan_escape :: proc(t: ^Tokenizer) -> bool {…}

scan_file_tag :: proc(t: ^Tokenizer) -> string {…}

scan_identifier :: proc(t: ^Tokenizer) -> string {…}

scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {…}

scan_raw_string :: proc(t: ^Tokenizer) -> string {…}

scan_rune :: proc(t: ^Tokenizer) -> string {…}

scan_string :: proc(t: ^Tokenizer) -> string {…}

skip_whitespace :: proc(t: ^Tokenizer) {…}

to_string :: proc(kind: Token_Kind) -> string {…}

token_to_string :: proc(tok: Token) -> string {…}