A single token within a Sentence, or an empty node within a Token. More...

#include <token.hpp>

Public Member Functions
Constructors
constexpr	Token () noexcept=delete
	Default construction is disabled; tokens are created via Sentence::add_token or Token::add_empty_node.
Identity
constexpr Sentence *	sentence () const noexcept
	Pointer to the sentence that owns this token.
constexpr Token *	get_parent () const noexcept
	Pointer to the parent token, or nullptr if this is not an empty node.
constexpr TokenId	get_id () const noexcept
	Token identifier. For an empty node, this is its position among its parent's empty nodes.
constexpr std::optional< Token * >	get_head () const noexcept
	Resolves the HEAD field to the referenced token.
constexpr std::error_code	set_head (TokenId id) noexcept
	Sets the HEAD field, linking this token to its syntactic head.
constexpr bool	is_empty_node () const noexcept
	Whether this token is an empty node.
constexpr std::optional< Word * >	get_word () const noexcept
	The Word this token belongs to, if any.
constexpr bool	is_in_word () const noexcept
	Whether this token belongs to a Word.
Fields
constexpr auto	fields () const noexcept
	View over the field values, in column order.
constexpr std::size_t	num_fields () const noexcept
	Number of fields on this token (normally equal to Treebank::num_columns).
constexpr std::optional< std::string_view >	get_field_value (std::size_t position) const noexcept
	Looks up a field value by column position.
constexpr std::optional< std::string_view >	get_field_value (std::string_view name) const noexcept
	Looks up a field value by column name.
constexpr std::error_code	set_field_value (std::size_t position, std::string_view value) noexcept
	Sets a field value by column position.
constexpr std::error_code	set_field_value (std::string_view name, std::string_view value) noexcept
	Sets a field value by column name.
constexpr std::error_code	unset_field_value (std::size_t position) noexcept
	Unsets a field value by column position.
constexpr std::error_code	unset_field_value (std::string_view name) noexcept
	Unsets a field value by column name.
Empty nodes
constexpr auto	empty_nodes () const noexcept
	View over the empty nodes attached to this token, in order.
constexpr std::size_t	empty_node_count () const noexcept
	Number of empty nodes attached to this token.
constexpr std::expected< Token *, std::error_code >	add_empty_node () noexcept
	Constructs and appends a new empty node.
constexpr std::error_code	remove_empty_node (TokenId id) noexcept
	Removes the empty node with the given identifier.
constexpr std::error_code	remove_empty_node (Token *node) noexcept
	Removes the given empty node.
constexpr void	remove_all_empty_nodes () noexcept
	Removes all empty nodes attached to this token.
State
constexpr std::error_code	validate () const noexcept
	Checks the token for structural inconsistencies.
Destructor
constexpr	~Token () noexcept=default
	Destructor.

Detailed Description

A single token within a Sentence, or an empty node within a Token.

Member Function Documentation

◆ add_empty_node()

std::expected< Token *, std::error_code > tg::conllu::Token::add_empty_node ( )

nodiscardconstexprnoexcept

Constructs and appends a new empty node.

Returns: Pointer to the newly created empty node, or an error_code on failure.

◆ get_field_value() [1/2]

std::optional< std::string_view > tg::conllu::Token::get_field_value ( std::size_t position ) const

nodiscardconstexprnoexcept

Looks up a field value by column position.

Parameters

[in] position Column position to search for.

Returns: Field value, or std::nullopt if the field is unset or position is out of range.

◆ get_field_value() [2/2]

std::optional< std::string_view > tg::conllu::Token::get_field_value ( std::string_view name ) const

nodiscardconstexprnoexcept

Looks up a field value by column name.

Parameters

[in] name Column name to search for.

Returns: Field value, or std::nullopt if the field is unset or no such column exists.

◆ get_head()

std::optional< Token * > tg::conllu::Token::get_head ( ) const

nodiscardconstexprnoexcept

Resolves the HEAD field to the referenced token.

Returns: Pointer to the head token, or std::nullopt if there is none (unset, root, or unresolvable).

◆ get_word()

std::optional< Word * > tg::conllu::Token::get_word ( ) const

nodiscardconstexprnoexcept

The Word this token belongs to, if any.

Returns: Pointer to the owning Word, or std::nullopt if this token is an empty node or is not part of any word.

◆ remove_empty_node() [1/2]

std::error_code tg::conllu::Token::remove_empty_node ( Token * node )

constexprnoexcept

Removes the given empty node.

Parameters

[in] node Pointer to the empty node to remove.

Returns: Empty error_code on success, or an error describing the failure.

◆ remove_empty_node() [2/2]

std::error_code tg::conllu::Token::remove_empty_node ( TokenId id )

constexprnoexcept

Removes the empty node with the given identifier.

Parameters

[in] id Identifier of the empty node to remove.

Returns: Empty error_code on success, or an error describing the failure.

◆ set_field_value() [1/2]

std::error_code tg::conllu::Token::set_field_value	(	std::size_t	position,
		std::string_view	value )

nodiscardconstexprnoexcept

Sets a field value by column position.

Parameters

[in]	position	Column position to set.
[in]	value	New field value.

Returns: Empty error_code on success, or an error describing the failure.

◆ set_field_value() [2/2]

std::error_code tg::conllu::Token::set_field_value	(	std::string_view	name,
		std::string_view	value )

nodiscardconstexprnoexcept

Sets a field value by column name.

Parameters

[in]	name	Column name to set.
[in]	value	New field value.

Returns: Empty error_code on success, or an error describing the failure.

◆ set_head()

std::error_code tg::conllu::Token::set_head ( TokenId id )

constexprnoexcept

Sets the HEAD field, linking this token to its syntactic head.

Parameters

[in] id Identifier of the head token, or 0 to clear the head (root/unset). Must not equal this token's own identifier, and if non-zero must refer to an existing token in the same sentence.

Returns: Empty error_code on success, or an error describing the failure.

◆ unset_field_value() [1/2]

std::error_code tg::conllu::Token::unset_field_value ( std::size_t position )

nodiscardconstexprnoexcept

Unsets a field value by column position.

Parameters

[in] position Column position to unset.

Returns: Empty error_code on success, or an error describing the failure.

◆ unset_field_value() [2/2]

std::error_code tg::conllu::Token::unset_field_value ( std::string_view name )

nodiscardconstexprnoexcept

Unsets a field value by column name.

Parameters

[in] name Column name to unset.

Returns: Empty error_code on success, or an error describing the failure.

◆ validate()

std::error_code tg::conllu::Token::validate ( ) const

nodiscardconstexprnoexcept

Checks the token for structural inconsistencies.

Returns: Empty error_code on success, or an error describing the first issue found.

The documentation for this class was generated from the following file:

include/conllu/token.hpp

Public Member Functions

Detailed Description

Member Function Documentation

◆ add_empty_node()

◆ get_field_value() [1/2]

◆ get_field_value() [2/2]

◆ get_head()

◆ get_word()

◆ remove_empty_node() [1/2]

◆ remove_empty_node() [2/2]

◆ set_field_value() [1/2]

◆ set_field_value() [2/2]

◆ set_head()

◆ unset_field_value() [1/2]

◆ unset_field_value() [2/2]

◆ validate()