package core:odin/tokenizer

Source

Error_Handler :: proc(pos: Pos, fmt: string, args: ..any)

Flag :: enum int {
	Insert_Semicolon, 
}

Flags :: distinct bit_set[Flag; u32]

Pos :: struct {
	file:   string,
	offset: int,
	// starting at 0
	line:   int,
	// starting at 1
	column: int,
}

Token :: struct {
	kind: Token_Kind,
	text: string,
	pos:  Pos,
}

Token_Kind :: enum u32 {
	Invalid, 
	EOF, 
	Comment, 
	B_Literal_Begin, 
	Ident,                        // main
	Integer,                      // 12345
	Float,                        // 123.45
	Imag,                         // 123.45i
	Rune,                         // 'a'
	String,                       // "abc"
	B_Literal_End, 
	B_Operator_Begin, 
	Eq,                           // =
	Not,                          // !
	Hash,                         // #
	At,                           // @
	Dollar,                       // $
	Pointer,                      // ^
	Question,                     // ?
	Add, 
	Sub,                          // -
	Mul,                          // *
	Quo,                          // /
	Mod,                          // %
	Mod_Mod,                      // %%
	And,                          // &
	Or,                           // |
	Xor,                          // ~
	And_Not,                      // &~
	Shl,                          // <<
	Shr,                          // >>
	Cmp_And,                      // &&
	Cmp_Or,                       // ||
	B_Assign_Op_Begin, 
	Add_Eq, 
	Sub_Eq,                       // -=
	Mul_Eq,                       // *=
	Quo_Eq,                       // /=
	Mod_Eq,                       // %=
	Mod_Mod_Eq,                   // %%=
	And_Eq,                       // &=
	Or_Eq,                        // |=
	Xor_Eq,                       // ~=
	And_Not_Eq,                   // &~=
	Shl_Eq,                       // <<=
	Shr_Eq,                       // >>=
	Cmp_And_Eq,                   // &&=
	Cmp_Or_Eq,                    // ||=
	B_Assign_Op_End, 
	Increment, 
	Decrement,                    // --
	Arrow_Right,                  // ->
	Undef,                        // ---
	B_Comparison_Begin, 
	Cmp_Eq,                       // ==
	Not_Eq,                       // !=
	Lt,                           // <
	Gt,                           // >
	Lt_Eq,                        // <=
	Gt_Eq,                        // >=
	B_Comparison_End, 
	Open_Paren,                   // (
	Close_Paren,                  // )
	Open_Bracket,                 // [
	Close_Bracket,                // ]
	Open_Brace,                   // {
	Close_Brace,                  // }
	Colon,                        // :
	Semicolon,                    // ;
	Period,                       // .
	Comma,                        // ,
	Ellipsis,                     // ..
	Range_Half,                   // ..<
	Range_Full,                   // ..=
	B_Operator_End, 
	B_Keyword_Begin, 
	Import,                       // import
	Foreign,                      // foreign
	Package,                      // package
	Typeid,                       // typeid
	When,                         // when
	Where,                        // where
	If,                           // if
	Else,                         // else
	For,                          // for
	Switch,                       // switch
	In,                           // in
	Not_In,                       // not_in
	Do,                           // do
	Case,                         // case
	Break,                        // break
	Continue,                     // continue
	Fallthrough,                  // fallthrough
	Defer,                        // defer
	Return,                       // return
	Proc,                         // proc
	Struct,                       // struct
	Union,                        // union
	Enum,                         // enum
	Bit_Set,                      // bit_set
	Map,                          // map
	Dynamic,                      // dynamic
	Auto_Cast,                    // auto_cast
	Cast,                         // cast
	Transmute,                    // transmute
	Distinct,                     // distinct
	Using,                        // using
	Context,                      // context
	Or_Else,                      // or_else
	Or_Return,                    // or_return
	Or_Break,                     // or_break
	Or_Continue,                  // or_continue
	Asm,                          // asm
	Inline,                       // inline
	No_Inline,                    // no_inline
	Matrix,                       // matrix
	B_Keyword_End, 
	COUNT, 
	B_Custom_Keyword_Begin = 118, 
}

Tokenizer :: struct {
	// Immutable data
	path:             string,
	src:              string,
	err:              Error_Handler,
	flags:            Flags,
	// Tokenizing state
	ch:               rune,
	offset:           int,
	read_offset:      int,
	line_offset:      int,
	line_count:       int,
	insert_semicolon: bool,
	// Mutable data
	error_count:      int,
}

custom_keyword_tokens: []string

tokens: [117]string = …

advance_rune :: proc(t: ^Tokenizer) {…}

default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {…}

digit_val :: proc(r: rune) -> int {…}

error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {…}

init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {…}

is_assignment_operator :: proc(kind: Token_Kind) -> bool {…}

is_digit :: proc(r: rune) -> bool {…}

is_keyword :: proc(kind: Token_Kind) -> bool {…}

is_letter :: proc(r: rune) -> bool {…}

is_literal :: proc(kind: Token_Kind) -> bool {…}

is_newline :: proc(tok: Token) -> bool {…}

is_operator :: proc(kind: Token_Kind) -> bool {…}

peek_byte :: proc(t: ^Tokenizer, offset: int = 0) -> u8 {…}

pos_compare :: proc(lhs, rhs: Pos) -> int {…}

scan :: proc(t: ^Tokenizer) -> Token {…}

scan_comment :: proc(t: ^Tokenizer) -> string {…}

scan_escape :: proc(t: ^Tokenizer) -> bool {…}

scan_identifier :: proc(t: ^Tokenizer) -> string {…}

scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {…}

scan_raw_string :: proc(t: ^Tokenizer) -> string {…}

scan_rune :: proc(t: ^Tokenizer) -> string {…}

scan_string :: proc(t: ^Tokenizer) -> string {…}

skip_whitespace :: proc(t: ^Tokenizer) {…}

to_string :: proc(kind: Token_Kind) -> string {…}

token_to_string :: proc(tok: Token) -> string {…}