package core:c/frontend/tokenizer
Index
Constants (0)
This section is empty.
Variables (2)
Procedures (41)
- add_hide_set
- add_new_file
- advance_rune
- advance_rune_n
- allow_next_to_be_newline
- char_width
- copy_token
- default_error_handler
- default_is_keyword
- default_warn_handler
- digit_val
- display_width
- error
- error_offset
- hide_set_contains
- hide_set_intersection
- hide_set_union
- in_range
- init_defaults
- inline_tokenize
- is_digit
- is_ident0
- is_ident1
- new_eof
- new_hide_set
- peek
- peek_str
- scan
- scan_comment
- scan_escape
- scan_identifier
- scan_literal_prefix
- scan_number
- scan_punct
- scan_rune
- scan_string
- skip_whitespace
- tokenize
- tokenize_file
- warn
- warn_offset
Procedure Groups (0)
This section is empty.
Types
Error_Handler ¶
Error_Handler :: proc "odin" (pos: Pos, fmt: string, args: ..any)
File ¶
File :: struct { name: string, id: int, src: []u8, display_name: string, line_delta: int, }
Hide_Set ¶
Hide_Set :: struct { next: ^Hide_Set, name: string, }
Is_Keyword_Proc ¶
Is_Keyword_Proc :: proc "odin" (tok: ^Token) -> bool
Token ¶
Token :: struct { kind: Token_Kind, next: ^Token, lit: string, pos: Pos, file: ^File, line_delta: int, at_bol: bool, has_space: bool, type_hint: Token_Type_Hint, val: Token_Value, prefix: string, // Preprocessor values hide_set: ^Hide_Set, origin: ^Token, }
Token_Kind ¶
Token_Kind :: enum int { Invalid, Ident, Punct, Keyword, Char, String, Number, PP_Number, Comment, EOF, }
Token_Type_Hint ¶
Token_Type_Hint :: enum u8 { None, Int, Long, Long_Long, Unsigned_Int, Unsigned_Long, Unsigned_Long_Long, Float, Double, Long_Double, UTF_8, UTF_16, UTF_32, UTF_Wide, }
Token_Value ¶
Token_Value :: union { i64, f64, string, []u16, []u32, }
Tokenizer ¶
Tokenizer :: struct { // Immutable data path: string, src: []u8, // Tokenizing state ch: rune, offset: int, read_offset: int, line_offset: int, line_count: int, // Extra information for tokens at_bol: bool, has_space: bool, // Mutable data err: Error_Handler, warn: Error_Handler, error_count: int, warning_count: int, }
Constants
This section is empty.
Variables
default_keyword_set ¶
default_keyword_set: map[string]bool = …
token_name ¶
token_name: [Token_Kind]string = …
Procedures
advance_rune ¶
advance_rune :: proc "odin" (t: ^Tokenizer) {…}
advance_rune_n ¶
advance_rune_n :: proc "odin" (t: ^Tokenizer, n: int) {…}
allow_next_to_be_newline ¶
allow_next_to_be_newline :: proc "odin" (t: ^Tokenizer) -> bool {…}
char_width ¶
char_width :: proc "odin" (c: rune) -> int {…}
Returns the number of columns needed to display a given character in a fixed-width font. Based on https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
default_error_handler ¶
default_error_handler :: proc "odin" (pos: Pos, msg: string, args: ..any) {…}
default_is_keyword ¶
default_is_keyword :: proc "odin" (tok: ^Token) -> bool {…}
default_warn_handler ¶
default_warn_handler :: proc "odin" (pos: Pos, msg: string, args: ..any) {…}
digit_val ¶
digit_val :: proc "odin" (r: rune) -> int {…}
display_width ¶
display_width :: proc "odin" (str: string) -> (w: int) {…}
error_offset ¶
error_offset :: proc "odin" (t: ^Tokenizer, offset: int, msg: string, args: ..any) {…}
hide_set_contains ¶
hide_set_contains :: proc "odin" (hs: ^Hide_Set, name: string) -> bool {…}
in_range ¶
in_range :: proc "odin" (range: []rune, c: rune) -> bool {…}
init_defaults ¶
init_defaults :: proc "odin" (t: ^Tokenizer, err: Error_Handler = default_error_handler, warn: Error_Handler = default_warn_handler) {…}
is_digit ¶
is_digit :: proc "odin" (r: rune) -> bool {…}
is_ident0 ¶
is_ident0 :: proc "odin" (c: rune) -> bool {…}
[https://www.sigbus.info/n1570#D] C11 allows ASCII and some multibyte characters in certan Unicode ranges to be used in an identifier.
is_ident0 returns true if a given character is acceptable as the first character of an identifier.
is_ident1 ¶
is_ident1 :: proc "odin" (c: rune) -> bool {…}
is_ident0 returns true if a given character is acceptable as a non-first character of an identifier.
new_hide_set ¶
new_hide_set :: proc "odin" (name: string) -> ^Hide_Set {…}
peek_str ¶
peek_str :: proc "odin" (t: ^Tokenizer, str: string) -> bool {…}
scan_comment ¶
scan_comment :: proc "odin" (t: ^Tokenizer) -> string {…}
scan_escape ¶
scan_escape :: proc "odin" (t: ^Tokenizer) -> bool {…}
scan_identifier ¶
scan_identifier :: proc "odin" (t: ^Tokenizer) -> string {…}
scan_literal_prefix ¶
scan_literal_prefix :: proc "odin" (t: ^Tokenizer, str: string, prefix: ^string) -> bool {…}
scan_number ¶
scan_number :: proc "odin" (t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {…}
scan_punct ¶
scan_punct :: proc "odin" (t: ^Tokenizer, ch: rune) -> (kind: Token_Kind) {…}
scan_rune ¶
scan_rune :: proc "odin" (t: ^Tokenizer) -> string {…}
scan_string ¶
scan_string :: proc "odin" (t: ^Tokenizer) -> string {…}
skip_whitespace ¶
skip_whitespace :: proc "odin" (t: ^Tokenizer) {…}
tokenize_file ¶
tokenize_file :: proc "odin" (t: ^Tokenizer, path: string, id: int, loc := #caller_location) -> ^Token {…}
warn_offset ¶
warn_offset :: proc "odin" (t: ^Tokenizer, offset: int, msg: string, args: ..any) {…}
Procedure Groups
This section is empty.
Source Files
Generation Information
Generated with odin version dev-2023-03 (vendor "odin") Windows_amd64 @ 2023-03-29 21:09:05.439947600 +0000 UTC