text_tokenizer.md

text_tokenizer

Source: emel/text/tokenizer/sm.hpp

Mermaid

stateDiagram-v2
  direction TB
  [*] --> uninitialized
  uninitialized --> binding_preprocessor : bind_runtime [can_bind_] / begin_bind_
  uninitialized --> errored : bind_runtime [always] / reject_bind_
  uninitialized --> errored : tokenize_runtime [always] / reject_invalid_
  idle --> binding_preprocessor : bind_runtime [can_bind_] / begin_bind_
  idle --> errored : bind_runtime [always] / reject_bind_
  idle --> preprocessing : tokenize_runtime [can_tokenize_] / begin_tokenize_
  idle --> errored : tokenize_runtime [always] / reject_invalid_
  done --> binding_preprocessor : bind_runtime [can_bind_] / begin_bind_
  done --> errored : bind_runtime [always] / reject_bind_
  done --> preprocessing : tokenize_runtime [can_tokenize_] / begin_tokenize_
  done --> errored : tokenize_runtime [always] / reject_invalid_
  errored --> binding_preprocessor : bind_runtime [can_bind_] / begin_bind_
  errored --> errored : bind_runtime [always] / reject_bind_
  errored --> preprocessing : tokenize_runtime [can_tokenize_] / begin_tokenize_
  errored --> errored : tokenize_runtime [always] / reject_invalid_
  unexpected --> binding_preprocessor : bind_runtime [can_bind_] / begin_bind_
  unexpected --> unexpected : bind_runtime [always] / reject_bind_
  unexpected --> preprocessing : tokenize_runtime [can_tokenize_] / begin_tokenize_
  unexpected --> unexpected : tokenize_runtime [always] / reject_invalid_
  binding_preprocessor --> binding_preprocessor_decision : completion_bind_runtime_ [always] / bind_preprocessor_
  binding_preprocessor_decision --> binding_encoder : completion_bind_runtime_ [bind_preprocessor_error_none_] / none
  binding_preprocessor_decision --> errored : completion_bind_runtime_ [bind_preprocessor_error_invalid_request_] / none
  binding_preprocessor_decision --> errored : completion_bind_runtime_ [bind_preprocessor_error_model_invalid_] / none
  binding_preprocessor_decision --> errored : completion_bind_runtime_ [bind_preprocessor_error_backend_error_] / none
  binding_preprocessor_decision --> errored : completion_bind_runtime_ [bind_preprocessor_error_unknown_] / none
  binding_encoder --> binding_encoder_decision : completion_bind_runtime_ [always] / bind_encoder_
  binding_encoder_decision --> idle : completion_bind_runtime_ [bind_encoder_error_none_] / mark_bind_success_
  binding_encoder_decision --> errored : completion_bind_runtime_ [bind_encoder_error_invalid_request_] / none
  binding_encoder_decision --> errored : completion_bind_runtime_ [bind_encoder_error_model_invalid_] / none
  binding_encoder_decision --> errored : completion_bind_runtime_ [bind_encoder_error_backend_error_] / none
  binding_encoder_decision --> errored : completion_bind_runtime_ [bind_encoder_error_unknown_] / none
  preprocessing --> preprocess_decision : completion_tokenize_runtime_ [always] / dispatch_preprocess_
  preprocess_decision --> errored : completion_tokenize_runtime_ [preprocess_rejected_no_error_] / set_backend_error_
  preprocess_decision --> errored : completion_tokenize_runtime_ [preprocess_reported_error_] / set_error_from_preprocess_
  preprocess_decision --> errored : completion_tokenize_runtime_ [preprocess_fragment_count_invalid_] / set_invalid_request_error_
  preprocess_decision --> prefix_decision : completion_tokenize_runtime_ [preprocess_success_] / none
  prefix_decision --> encoding_ready : completion_tokenize_runtime_ [bos_ready_] / append_bos_
  prefix_decision --> errored : completion_tokenize_runtime_ [bos_no_capacity_] / set_invalid_request_error_
  prefix_decision --> errored : completion_tokenize_runtime_ [bos_invalid_id_] / set_invalid_id_error_
  prefix_decision --> encoding_ready : completion_tokenize_runtime_ [no_prefix_] / none
  encoding_ready --> suffix_decision : completion_tokenize_runtime_ [no_more_fragments_] / none
  encoding_ready --> errored : completion_tokenize_runtime_ [more_fragments_no_capacity_] / set_invalid_request_error_
  encoding_ready --> errored : completion_tokenize_runtime_ [more_fragments_token_invalid_] / set_invalid_request_error_
  encoding_ready --> encoding_token_fragment : completion_tokenize_runtime_ [more_fragments_token_valid_] / none
  encoding_ready --> encoding_raw_fragment : completion_tokenize_runtime_ [more_fragments_raw_] / none
  encoding_token_fragment --> encoding_ready : completion_tokenize_runtime_ [always] / append_fragment_token_
  encoding_raw_fragment --> encoding_raw_decision : completion_tokenize_runtime_ [always] / dispatch_encode_raw_fragment_
  encoding_raw_decision --> errored : completion_tokenize_runtime_ [encode_rejected_no_error_] / set_invalid_id_error_
  encoding_raw_decision --> errored : completion_tokenize_runtime_ [encode_reported_error_] / set_error_from_encode_
  encoding_raw_decision --> errored : completion_tokenize_runtime_ [encode_count_invalid_] / set_invalid_request_error_
  encoding_raw_decision --> encoding_ready : completion_tokenize_runtime_ [encode_success_] / commit_encoded_fragment_
  suffix_decision --> finalizing : completion_tokenize_runtime_ [sep_ready_] / append_sep_
  suffix_decision --> errored : completion_tokenize_runtime_ [sep_no_capacity_] / set_invalid_request_error_
  suffix_decision --> errored : completion_tokenize_runtime_ [sep_invalid_id_] / set_invalid_id_error_
  suffix_decision --> finalizing : completion_tokenize_runtime_ [eos_ready_] / append_eos_
  suffix_decision --> errored : completion_tokenize_runtime_ [eos_no_capacity_] / set_invalid_request_error_
  suffix_decision --> errored : completion_tokenize_runtime_ [eos_invalid_id_] / set_invalid_id_error_
  suffix_decision --> finalizing : completion_tokenize_runtime_ [no_suffix_] / none
  finalizing --> done : completion_tokenize_runtime_ [always] / finalize_
  uninitialized --> unexpected : _ [always] / on_unexpected_
  binding_preprocessor --> unexpected : _ [always] / on_unexpected_
  binding_preprocessor_decision --> unexpected : _ [always] / on_unexpected_
  binding_encoder --> unexpected : _ [always] / on_unexpected_
  binding_encoder_decision --> unexpected : _ [always] / on_unexpected_
  idle --> unexpected : _ [always] / on_unexpected_
  preprocessing --> unexpected : _ [always] / on_unexpected_
  preprocess_decision --> unexpected : _ [always] / on_unexpected_
  prefix_decision --> unexpected : _ [always] / on_unexpected_
  encoding_ready --> unexpected : _ [always] / on_unexpected_
  encoding_token_fragment --> unexpected : _ [always] / on_unexpected_
  encoding_raw_fragment --> unexpected : _ [always] / on_unexpected_
  encoding_raw_decision --> unexpected : _ [always] / on_unexpected_
  suffix_decision --> unexpected : _ [always] / on_unexpected_
  finalizing --> unexpected : _ [always] / on_unexpected_
  done --> unexpected : _ [always] / on_unexpected_
  errored --> unexpected : _ [always] / on_unexpected_
  unexpected --> unexpected : _ [always] / on_unexpected_

Loading

Transitions

Source	Event	Guard	Action	Target
uninitialized	bind_runtime	can_bind>	begin_bind>	binding_preprocessor
uninitialized	bind_runtime	always	reject_bind>	errored
uninitialized	tokenize_runtime	always	reject_invalid>	errored
idle	bind_runtime	can_bind>	begin_bind>	binding_preprocessor
idle	bind_runtime	always	reject_bind>	errored
idle	tokenize_runtime	can_tokenize>	begin_tokenize>	preprocessing
idle	tokenize_runtime	always	reject_invalid>	errored
done	bind_runtime	can_bind>	begin_bind>	binding_preprocessor
done	bind_runtime	always	reject_bind>	errored
done	tokenize_runtime	can_tokenize>	begin_tokenize>	preprocessing
done	tokenize_runtime	always	reject_invalid>	errored
errored	bind_runtime	can_bind>	begin_bind>	binding_preprocessor
errored	bind_runtime	always	reject_bind>	errored
errored	tokenize_runtime	can_tokenize>	begin_tokenize>	preprocessing
errored	tokenize_runtime	always	reject_invalid>	errored
unexpected	bind_runtime	can_bind>	begin_bind>	binding_preprocessor
unexpected	bind_runtime	always	reject_bind>	unexpected
unexpected	tokenize_runtime	can_tokenize>	begin_tokenize>	preprocessing
unexpected	tokenize_runtime	always	reject_invalid>	unexpected
binding_preprocessor	completion<bind_runtime>	always	bind_preprocessor>	binding_preprocessor_decision
binding_preprocessor_decision	completion<bind_runtime>	bind_preprocessor_error_none>	none	binding_encoder
binding_preprocessor_decision	completion<bind_runtime>	bind_preprocessor_error_invalid_request>	none	errored
binding_preprocessor_decision	completion<bind_runtime>	bind_preprocessor_error_model_invalid>	none	errored
binding_preprocessor_decision	completion<bind_runtime>	bind_preprocessor_error_backend_error>	none	errored
binding_preprocessor_decision	completion<bind_runtime>	bind_preprocessor_error_unknown>	none	errored
binding_encoder	completion<bind_runtime>	always	bind_encoder>	binding_encoder_decision
binding_encoder_decision	completion<bind_runtime>	bind_encoder_error_none>	mark_bind_success>	idle
binding_encoder_decision	completion<bind_runtime>	bind_encoder_error_invalid_request>	none	errored
binding_encoder_decision	completion<bind_runtime>	bind_encoder_error_model_invalid>	none	errored
binding_encoder_decision	completion<bind_runtime>	bind_encoder_error_backend_error>	none	errored
binding_encoder_decision	completion<bind_runtime>	bind_encoder_error_unknown>	none	errored
preprocessing	completion<tokenize_runtime>	always	dispatch_preprocess>	preprocess_decision
preprocess_decision	completion<tokenize_runtime>	preprocess_rejected_no_error>	set_backend_error>	errored
preprocess_decision	completion<tokenize_runtime>	preprocess_reported_error>	set_error_from_preprocess>	errored
preprocess_decision	completion<tokenize_runtime>	preprocess_fragment_count_invalid>	set_invalid_request_error>	errored
preprocess_decision	completion<tokenize_runtime>	preprocess_success>	none	prefix_decision
prefix_decision	completion<tokenize_runtime>	bos_ready>	append_bos>	encoding_ready
prefix_decision	completion<tokenize_runtime>	bos_no_capacity>	set_invalid_request_error>	errored
prefix_decision	completion<tokenize_runtime>	bos_invalid_id>	set_invalid_id_error>	errored
prefix_decision	completion<tokenize_runtime>	no_prefix>	none	encoding_ready
encoding_ready	completion<tokenize_runtime>	no_more_fragments>	none	suffix_decision
encoding_ready	completion<tokenize_runtime>	more_fragments_no_capacity>	set_invalid_request_error>	errored
encoding_ready	completion<tokenize_runtime>	more_fragments_token_invalid>	set_invalid_request_error>	errored
encoding_ready	completion<tokenize_runtime>	more_fragments_token_valid>	none	encoding_token_fragment
encoding_ready	completion<tokenize_runtime>	more_fragments_raw>	none	encoding_raw_fragment
encoding_token_fragment	completion<tokenize_runtime>	always	append_fragment_token>	encoding_ready
encoding_raw_fragment	completion<tokenize_runtime>	always	dispatch_encode_raw_fragment>	encoding_raw_decision
encoding_raw_decision	completion<tokenize_runtime>	encode_rejected_no_error>	set_invalid_id_error>	errored
encoding_raw_decision	completion<tokenize_runtime>	encode_reported_error>	set_error_from_encode>	errored
encoding_raw_decision	completion<tokenize_runtime>	encode_count_invalid>	set_invalid_request_error>	errored
encoding_raw_decision	completion<tokenize_runtime>	encode_success>	commit_encoded_fragment>	encoding_ready
suffix_decision	completion<tokenize_runtime>	sep_ready>	append_sep>	finalizing
suffix_decision	completion<tokenize_runtime>	sep_no_capacity>	set_invalid_request_error>	errored
suffix_decision	completion<tokenize_runtime>	sep_invalid_id>	set_invalid_id_error>	errored
suffix_decision	completion<tokenize_runtime>	eos_ready>	append_eos>	finalizing
suffix_decision	completion<tokenize_runtime>	eos_no_capacity>	set_invalid_request_error>	errored
suffix_decision	completion<tokenize_runtime>	eos_invalid_id>	set_invalid_id_error>	errored
suffix_decision	completion<tokenize_runtime>	no_suffix>	none	finalizing
finalizing	completion<tokenize_runtime>	always	finalize>	done
uninitialized	_	always	on_unexpected>	unexpected
binding_preprocessor	_	always	on_unexpected>	unexpected
binding_preprocessor_decision	_	always	on_unexpected>	unexpected
binding_encoder	_	always	on_unexpected>	unexpected
binding_encoder_decision	_	always	on_unexpected>	unexpected
idle	_	always	on_unexpected>	unexpected
preprocessing	_	always	on_unexpected>	unexpected
preprocess_decision	_	always	on_unexpected>	unexpected
prefix_decision	_	always	on_unexpected>	unexpected
encoding_ready	_	always	on_unexpected>	unexpected
encoding_token_fragment	_	always	on_unexpected>	unexpected
encoding_raw_fragment	_	always	on_unexpected>	unexpected
encoding_raw_decision	_	always	on_unexpected>	unexpected
suffix_decision	_	always	on_unexpected>	unexpected
finalizing	_	always	on_unexpected>	unexpected
done	_	always	on_unexpected>	unexpected
errored	_	always	on_unexpected>	unexpected
unexpected	_	always	on_unexpected>	unexpected