package core:odin/tokenizer

⌘K
Ctrl+K
or
/

    Types

    Error_Handler ¶

    Error_Handler :: proc(pos: Pos, fmt: string, .. args: ..any)
    Related Procedures With Parameters

    Flag ¶

    Flag :: enum int {
    	Insert_Semicolon, 
    }

    Flags ¶

    Flags :: distinct bit_set[Flag; u32]

    Pos ¶

    Pos :: struct {
    	file:   string,
    	offset: int,
    	// starting at 0
    	line:   int,
    	// starting at 1
    	column: int,
    }
    Related Procedures With Parameters

    Token ¶

    Token :: struct {
    	kind: Token_Kind,
    	text: string,
    	pos:  Pos,
    }
    Related Procedures With Parameters
    Related Procedures With Returns

    Token_Kind ¶

    Token_Kind :: enum u32 {
    	Invalid, 
    	EOF, 
    	Comment, 
    	File_Tag, 
    	B_Literal_Begin, 
    	Ident,                        // main
    	Integer,                      // 12345
    	Float,                        // 123.45
    	Imag,                         // 123.45i
    	Rune,                         // 'a'
    	String,                       // "abc"
    	B_Literal_End, 
    	B_Operator_Begin, 
    	Eq,                           // =
    	Not,                          // !
    	Hash,                         // #
    	At,                           // @
    	Dollar,                       // $
    	Pointer,                      // ^
    	Question,                     // ?
    	Add, 
    	Sub,                          // -
    	Mul,                          // *
    	Quo,                          // /
    	Mod,                          // %
    	Mod_Mod,                      // %%
    	And,                          // &
    	Or,                           // |
    	Xor,                          // ~
    	And_Not,                      // &~
    	Shl,                          // <<
    	Shr,                          // >>
    	Cmp_And,                      // &&
    	Cmp_Or,                       // ||
    	B_Assign_Op_Begin, 
    	Add_Eq, 
    	Sub_Eq,                       // -=
    	Mul_Eq,                       // *=
    	Quo_Eq,                       // /=
    	Mod_Eq,                       // %=
    	Mod_Mod_Eq,                   // %%=
    	And_Eq,                       // &=
    	Or_Eq,                        // |=
    	Xor_Eq,                       // ~=
    	And_Not_Eq,                   // &~=
    	Shl_Eq,                       // <<=
    	Shr_Eq,                       // >>=
    	Cmp_And_Eq,                   // &&=
    	Cmp_Or_Eq,                    // ||=
    	B_Assign_Op_End, 
    	Increment, 
    	Decrement,                    // --
    	Arrow_Right,                  // ->
    	Undef,                        // ---
    	B_Comparison_Begin, 
    	Cmp_Eq,                       // ==
    	Not_Eq,                       // !=
    	Lt,                           // <
    	Gt,                           // >
    	Lt_Eq,                        // <=
    	Gt_Eq,                        // >=
    	B_Comparison_End, 
    	Open_Paren,                   // (
    	Close_Paren,                  // )
    	Open_Bracket,                 // [
    	Close_Bracket,                // ]
    	Open_Brace,                   // {
    	Close_Brace,                  // }
    	Colon,                        // :
    	Semicolon,                    // ;
    	Period,                       // .
    	Comma,                        // ,
    	Ellipsis,                     // ..
    	Range_Half,                   // ..<
    	Range_Full,                   // ..=
    	B_Operator_End, 
    	B_Keyword_Begin, 
    	Import,                       // import
    	Foreign,                      // foreign
    	Package,                      // package
    	Typeid,                       // typeid
    	When,                         // when
    	Where,                        // where
    	If,                           // if
    	Else,                         // else
    	For,                          // for
    	Switch,                       // switch
    	In,                           // in
    	Not_In,                       // not_in
    	Do,                           // do
    	Case,                         // case
    	Break,                        // break
    	Continue,                     // continue
    	Fallthrough,                  // fallthrough
    	Defer,                        // defer
    	Return,                       // return
    	Proc,                         // proc
    	Struct,                       // struct
    	Union,                        // union
    	Enum,                         // enum
    	Bit_Set,                      // bit_set
    	Bit_Field,                    // bit_field
    	Map,                          // map
    	Dynamic,                      // dynamic
    	Auto_Cast,                    // auto_cast
    	Cast,                         // cast
    	Transmute,                    // transmute
    	Distinct,                     // distinct
    	Using,                        // using
    	Context,                      // context
    	Or_Else,                      // or_else
    	Or_Return,                    // or_return
    	Or_Break,                     // or_break
    	Or_Continue,                  // or_continue
    	Asm,                          // asm
    	Inline,                       // inline
    	No_Inline,                    // no_inline
    	Matrix,                       // matrix
    	B_Keyword_End, 
    	COUNT, 
    	B_Custom_Keyword_Begin = 120, 
    }
    Related Procedures With Parameters
    Related Procedures With Returns

    Tokenizer ¶

    Tokenizer :: struct {
    	// Immutable data
    	path:             string,
    	src:              string,
    	err:              Error_Handler,
    	flags:            Flags,
    	// Tokenizing state
    	ch:               rune,
    	offset:           int,
    	read_offset:      int,
    	line_offset:      int,
    	line_count:       int,
    	insert_semicolon: bool,
    	// Mutable data
    	error_count:      int,
    }
    Related Procedures With Parameters

    Constants

    This section is empty.

    Variables

    custom_keyword_tokens ¶

    custom_keyword_tokens: []string

    tokens ¶

    tokens: [119]string = …

    Procedures

    advance_rune ¶

    advance_rune :: proc(t: ^Tokenizer) {…}

    default_error_handler ¶

    default_error_handler :: proc(pos: Pos, msg: string, .. args: ..any) {…}

    digit_val ¶

    digit_val :: proc(r: rune) -> int {…}

    error ¶

    error :: proc(t: ^Tokenizer, offset: int, msg: string, .. args: ..any) {…}

    init ¶

    init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {…}

    is_assignment_operator ¶

    is_assignment_operator :: proc(kind: Token_Kind) -> bool {…}

    is_digit ¶

    is_digit :: proc(r: rune) -> bool {…}

    is_keyword ¶

    is_keyword :: proc(kind: Token_Kind) -> bool {…}

    is_letter ¶

    is_letter :: proc(r: rune) -> bool {…}

    is_literal ¶

    is_literal :: proc(kind: Token_Kind) -> bool {…}

    is_newline ¶

    is_newline :: proc(tok: Token) -> bool {…}

    is_operator ¶

    is_operator :: proc(kind: Token_Kind) -> bool {…}

    peek_byte ¶

    peek_byte :: proc(t: ^Tokenizer, offset: int = 0) -> u8 {…}

    pos_compare ¶

    pos_compare :: proc(lhs, rhs: Pos) -> int {…}

    scan ¶

    scan :: proc(t: ^Tokenizer) -> Token {…}

    scan_comment ¶

    scan_comment :: proc(t: ^Tokenizer) -> string {…}

    scan_escape ¶

    scan_escape :: proc(t: ^Tokenizer) -> bool {…}

    scan_file_tag ¶

    scan_file_tag :: proc(t: ^Tokenizer) -> string {…}

    scan_identifier ¶

    scan_identifier :: proc(t: ^Tokenizer) -> string {…}

    scan_number ¶

    scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {…}

    scan_raw_string ¶

    scan_raw_string :: proc(t: ^Tokenizer) -> string {…}

    scan_rune ¶

    scan_rune :: proc(t: ^Tokenizer) -> string {…}

    scan_string ¶

    scan_string :: proc(t: ^Tokenizer) -> string {…}

    skip_whitespace ¶

    skip_whitespace :: proc(t: ^Tokenizer) {…}

    to_string ¶

    to_string :: proc(kind: Token_Kind) -> string {…}

    token_to_string ¶

    token_to_string :: proc(tok: Token) -> string {…}

    Procedure Groups

    This section is empty.

    Source Files

    Generation Information

    Generated with odin version dev-2024-12 (vendor "odin") Windows_amd64 @ 2024-12-06 21:12:12.670898500 +0000 UTC