asmgrader::inspection::tokenize Namespace Reference

Recursive descent parser implementation details. More...

Classes
struct	CaseInsensitiveTag
class	Stream

Functions
constexpr std::string_view	substr_to (std::string_view str, auto token)
	A substr of str up to the first occurrence of token, or the entirety of str if token is not found.
constexpr std::string_view	substr_to (std::string_view str, std::invocable< char > auto pred)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. A substr of str up to the first character that satisfies pred, or the entirety of str if pred is never satisfied In essence, performs 'take while not' (a.k.a. 'take until')
constexpr std::string_view	substr_past (std::string_view str, auto what)
	A substr of str past all characters satisfying pred In essence, performs 'drop while'.
template<std::size_t N>
constexpr auto	make_rev_size_sorted (const std::string_view(&arr)[N])
constexpr bool	is_strlike_prefix (std::string_view str)
	Whether the entirety of str is a strlike-prefix See Token::Kind::StringLiteral for details.
constexpr bool	is_int_suffix (std::string_view str)
	Whether the entirety of str is an integer-suffix See Token::Kind::IntLiteral for details.
constexpr auto	is_ident_like ()
	Returns a functor to check for an identifier for a stream of characters Does not verify whether the ident is a reserved token or not. See Token::Kind::Identifier for details.
constexpr auto	digit_or_sep (char c)
	Exactly as named. Sep = '.
constexpr auto	xdigit_or_sep (char c)
	Exactly as named. Sep = '.
template<Token::Kind Kind>
constexpr bool	matches (const Stream &stream)
	Check whether the start of stream matches a token kind Assumes that there is no leading whitespace in stream
template<>
constexpr bool	matches< StringLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a string literal token See Token::Kind::StringLiteral for details.
template<>
constexpr bool	matches< RawStringLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a raw string literal token See Token::Kind::RawStringLiteral for details.
template<>
constexpr bool	matches< CharLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a char literal token See Token::Kind::CharLiteral for details.
template<>
constexpr bool	matches< BoolLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. See Token::Kind::BoolLiteral for details.
template<>
constexpr bool	matches< IntBinLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a binary integer literal token See Token::Kind::IntBinLiteral for details.
template<>
constexpr bool	matches< IntHexLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a hexadecimal integer literal token See Token::Kind::IntHexLiteral for details.
template<>
constexpr bool	matches< IntOctLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an octal integer literal token See Token::Kind::IntOctLiteral for details.
template<>
constexpr bool	matches< IntDecLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a decimal integer literal token Written in terms of other int literals. See Token::Kind::IntDecLiteral for details.
template<>
constexpr bool	matches< FloatHexLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a hexadecimal floating point literal token See Token::Kind::FloatLiteral for details.
template<>
constexpr bool	matches< FloatLiteral > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a floating point literal token See Token::Kind::FloatLiteral for details.
template<>
constexpr bool	matches< Identifier > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an identifier token See Token::Kind::Identifier for details.
template<>
constexpr bool	matches< Grouping > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an identifier token See Token::Kind::Identifier for details.
template<>
constexpr bool	matches< BinaryOperator > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a binary operator token See Token::Kind::BinaryOperator for details.
template<>
constexpr bool	matches< Operator > (const Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an operator token See Token::Kind::Operator for details.
template<Token::Kind Kind>
constexpr std::string_view	test_parse (std::string_view str)
template<Token::Kind Kind>
constexpr std::string_view	parse (Stream &stream)
	Parse the token of Kind from the start of the stream. Assumes that the stream actually starts with a token of Kind. (Check with match)
template<>
constexpr std::string_view	parse< StringLiteral > (Stream &stream)
	This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. See Token::Kind::StringLiteral for details.

Variables
constexpr struct asmgrader::inspection::tokenize::CaseInsensitiveTag	case_insensitive
constexpr auto	binary_operator_tokens
constexpr auto	operator_tokens
constexpr auto	grouping_tokens

Detailed Description

Recursive descent parser implementation details.

Function Documentation

digit_or_sep()

auto asmgrader::inspection::tokenize::digit_or_sep ( char c )

constexpr

Exactly as named. Sep = '.

is_ident_like()

auto asmgrader::inspection::tokenize::is_ident_like ( )

constexpr

Returns a functor to check for an identifier for a stream of characters Does not verify whether the ident is a reserved token or not. See Token::Kind::Identifier for details.

is_int_suffix()

bool asmgrader::inspection::tokenize::is_int_suffix ( std::string_view str )

constexpr

Whether the entirety of str is an integer-suffix See Token::Kind::IntLiteral for details.

is_strlike_prefix()

bool asmgrader::inspection::tokenize::is_strlike_prefix ( std::string_view str )

constexpr

Whether the entirety of str is a strlike-prefix See Token::Kind::StringLiteral for details.

make_rev_size_sorted()

template<std::size_t N>

auto asmgrader::inspection::tokenize::make_rev_size_sorted ( const std::string_view(&) arr[N] )

constexpr

matches()

template<Token::Kind Kind>

bool asmgrader::inspection::tokenize::matches ( const Stream & stream )

constexpr

Check whether the start of stream matches a token kind Assumes that there is no leading whitespace in stream

Parameters

stream

The character stream

Template Parameters

Kind	The kind of token to match to

matches< BinaryOperator >()

template<>

bool asmgrader::inspection::tokenize::matches< BinaryOperator > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a binary operator token See Token::Kind::BinaryOperator for details.

matches< BoolLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< BoolLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. See Token::Kind::BoolLiteral for details.

matches< CharLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< CharLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a char literal token See Token::Kind::CharLiteral for details.

matches< FloatHexLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< FloatHexLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a hexadecimal floating point literal token See Token::Kind::FloatLiteral for details.

matches< FloatLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< FloatLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a floating point literal token See Token::Kind::FloatLiteral for details.

matches< Grouping >()

template<>

bool asmgrader::inspection::tokenize::matches< Grouping > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an identifier token See Token::Kind::Identifier for details.

matches< Identifier >()

template<>

bool asmgrader::inspection::tokenize::matches< Identifier > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an identifier token See Token::Kind::Identifier for details.

matches< IntBinLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< IntBinLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a binary integer literal token See Token::Kind::IntBinLiteral for details.

matches< IntDecLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< IntDecLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a decimal integer literal token Written in terms of other int literals. See Token::Kind::IntDecLiteral for details.

matches< IntHexLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< IntHexLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a hexadecimal integer literal token See Token::Kind::IntHexLiteral for details.

matches< IntOctLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< IntOctLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an octal integer literal token See Token::Kind::IntOctLiteral for details.

matches< Operator >()

template<>

bool asmgrader::inspection::tokenize::matches< Operator > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is an operator token See Token::Kind::Operator for details.

matches< RawStringLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< RawStringLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a raw string literal token See Token::Kind::RawStringLiteral for details.

matches< StringLiteral >()

template<>

bool asmgrader::inspection::tokenize::matches< StringLiteral > ( const Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. Whether the start of stream is a string literal token See Token::Kind::StringLiteral for details.

parse()

template<Token::Kind Kind>

std::string_view asmgrader::inspection::tokenize::parse ( Stream & stream )

constexpr

Parse the token of Kind from the start of the stream. Assumes that the stream actually starts with a token of Kind. (Check with match)

Parameters

stream

The character stream. This is mutated, consuming parsed tokens.

Template Parameters

Kind	The kind of token to parse

Returns

A view of the token content

parse< StringLiteral >()

template<>

std::string_view asmgrader::inspection::tokenize::parse< StringLiteral > ( Stream & stream )

constexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. See Token::Kind::StringLiteral for details.

substr_past()

std::string_view asmgrader::inspection::tokenize::substr_past ( std::string_view str, auto what )

nodiscardconstexpr

A substr of str past all characters satisfying pred In essence, performs 'drop while'.

substr_to() [1/2]

std::string_view asmgrader::inspection::tokenize::substr_to ( std::string_view str, auto token )

nodiscardconstexpr

A substr of str up to the first occurrence of token, or the entirety of str if token is not found.

substr_to() [2/2]

std::string_view asmgrader::inspection::tokenize::substr_to ( std::string_view str, std::invocable< char > auto pred )

nodiscardconstexpr

This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts. A substr of str up to the first character that satisfies pred, or the entirety of str if pred is never satisfied In essence, performs 'take while not' (a.k.a. 'take until')

test_parse()

template<Token::Kind Kind>

std::string_view asmgrader::inspection::tokenize::test_parse ( std::string_view str )

constexpr

xdigit_or_sep()

auto asmgrader::inspection::tokenize::xdigit_or_sep ( char c )

constexpr

Exactly as named. Sep = '.

Variable Documentation

binary_operator_tokens

auto asmgrader::inspection::tokenize::binary_operator_tokens

constexpr

Initial value:

= make_rev_size_sorted({
    "::", ",",                                                           
    ".",  "->",                                                          
    ".*", "->*",                                                         
    "+",  "-",   "*",   "/",  "%",                                       
    "<<", ">>",  "^",   "|",  "&",                                       
    "&&", "||",                                                          
    "==", "!=",  "<=>", "<",  "<=", ">",  ">=",                          
    "=",  "+=",  "-=",  "*=", "/=", "%=", "<<=", ">>=", "&=", "^=", "|=" 
})

case_insensitive

struct asmgrader::inspection::tokenize::CaseInsensitiveTag asmgrader::inspection::tokenize::case_insensitive

grouping_tokens

auto asmgrader::inspection::tokenize::grouping_tokens

constexpr

Initial value:

= std::to_array<char>({
    '{', '}', 
    '(', ')', 
    '<', '>'  
})

operator_tokens

auto asmgrader::inspection::tokenize::operator_tokens

constexpr

Initial value:

= make_rev_size_sorted({
    "++", "--",                                                      
    "(", ")", "[", "]",                                              
    "+", "-",                                                        
    "~", "!",                                                        
    "*", "&",                                                        
    "throw", "sizeof", "alignof", "new", "delete",                   
                                                                     
    "const_cast", "static_cast", "dynamic_cast", "reinterpret_cast", 
    "::", "?", ":"                                                   
})