text_encoders_spm.md

text_encoders_spm

Source: emel/text/encoders/spm/sm.hpp

Mermaid

stateDiagram-v2
  direction TB
  [*] --> initialized
  initialized --> encode_validity_decision : encode_runtime [always] / none
  done --> encode_validity_decision : encode_runtime [always] / none
  errored --> encode_validity_decision : encode_runtime [always] / none
  unexpected --> encode_validity_decision : encode_runtime [always] / none
  encode_validity_decision --> encode_vocab_sync_decision : completion_encode_runtime_ [valid_encode_] / none
  encode_validity_decision --> errored : completion_encode_runtime_ [invalid_encode_] / reject_invalid_encode_
  encode_validity_decision --> errored : completion_encode_runtime_ [always] / reject_invalid_encode_
  encode_vocab_sync_decision --> encode_precheck_decision : completion_encode_runtime_ [vocab_changed_] / begin_encode_sync_vocab_
  encode_vocab_sync_decision --> encode_precheck_decision : completion_encode_runtime_ [vocab_unchanged_] / begin_encode_
  encode_vocab_sync_decision --> errored : completion_encode_runtime_ [always] / reject_invalid_encode_
  encode_precheck_decision --> done : completion_encode_runtime_ [text_empty_] / mark_done_
  encode_precheck_decision --> table_policy_decision : completion_encode_runtime_ [text_non_empty_] / none
  encode_precheck_decision --> errored : completion_encode_runtime_ [always] / ensure_last_error_
  table_policy_decision --> table_sync_exec : completion_encode_runtime_ [tables_missing_] / none
  table_policy_decision --> encode_prepare_exec : completion_encode_runtime_ [tables_ready_] / none
  table_policy_decision --> errored : completion_encode_runtime_ [always] / ensure_last_error_
  table_sync_exec --> table_sync_result_decision : completion_encode_runtime_ [always] / sync_tables_
  table_sync_result_decision --> encode_prepare_exec : completion_encode_runtime_ [table_sync_ok_] / none
  table_sync_result_decision --> errored : completion_encode_runtime_ [table_sync_invalid_argument_error_] / ensure_last_error_
  table_sync_result_decision --> errored : completion_encode_runtime_ [table_sync_backend_error_] / ensure_last_error_
  table_sync_result_decision --> errored : completion_encode_runtime_ [table_sync_model_invalid_error_] / ensure_last_error_
  table_sync_result_decision --> errored : completion_encode_runtime_ [table_sync_unclassified_error_code_] / ensure_last_error_
  encode_prepare_exec --> encode_prepare_result_decision : completion_encode_runtime_ [always] / run_prepare_
  encode_prepare_result_decision --> encode_merge_input_capacity_decision : completion_encode_runtime_ [prepare_result_ok_] / none
  encode_prepare_result_decision --> errored : completion_encode_runtime_ [prepare_result_invalid_argument_error_] / ensure_last_error_
  encode_prepare_result_decision --> errored : completion_encode_runtime_ [prepare_result_backend_error_] / ensure_last_error_
  encode_prepare_result_decision --> errored : completion_encode_runtime_ [prepare_result_model_invalid_error_] / ensure_last_error_
  encode_prepare_result_decision --> errored : completion_encode_runtime_ [prepare_result_unclassified_error_code_] / ensure_last_error_
  encode_merge_input_capacity_decision --> encode_merge_exec : completion_encode_runtime_ [merge_symbol_capacity_within_limit_] / none
  encode_merge_input_capacity_decision --> errored : completion_encode_runtime_ [merge_symbol_capacity_exceeded_] / reject_invalid_encode_
  encode_merge_input_capacity_decision --> errored : completion_encode_runtime_ [always] / reject_invalid_encode_
  encode_merge_exec --> encode_merge_result_decision : completion_encode_runtime_ [always] / run_merge_
  encode_merge_result_decision --> encode_emit_input_decision : completion_encode_runtime_ [merge_result_ok_] / none
  encode_merge_result_decision --> errored : completion_encode_runtime_ [merge_result_invalid_argument_error_] / ensure_last_error_
  encode_merge_result_decision --> errored : completion_encode_runtime_ [merge_result_backend_error_] / ensure_last_error_
  encode_merge_result_decision --> errored : completion_encode_runtime_ [merge_result_model_invalid_error_] / ensure_last_error_
  encode_merge_result_decision --> errored : completion_encode_runtime_ [merge_result_unclassified_error_code_] / ensure_last_error_
  encode_emit_input_decision --> encode_exec : completion_encode_runtime_ [symbols_present_] / none
  encode_emit_input_decision --> encode_result_decision : completion_encode_runtime_ [symbols_absent_] / set_emit_result_empty_
  encode_emit_input_decision --> errored : completion_encode_runtime_ [always] / ensure_last_error_
  encode_exec --> emit_result_decision : completion_encode_runtime_ [always] / run_encode_
  emit_result_decision --> encode_result_decision : completion_encode_runtime_ [emit_result_ok_] / apply_emit_result_ok_
  emit_result_decision --> encode_result_decision : completion_encode_runtime_ [emit_result_failed_] / apply_emit_result_failed_
  emit_result_decision --> errored : completion_encode_runtime_ [always] / ensure_last_error_
  encode_result_decision --> done : completion_encode_runtime_ [encode_result_ok_] / mark_done_
  encode_result_decision --> errored : completion_encode_runtime_ [encode_result_invalid_argument_error_] / ensure_last_error_
  encode_result_decision --> errored : completion_encode_runtime_ [encode_result_backend_error_] / ensure_last_error_
  encode_result_decision --> errored : completion_encode_runtime_ [encode_result_model_invalid_error_] / ensure_last_error_
  encode_result_decision --> errored : completion_encode_runtime_ [encode_result_unclassified_error_code_] / ensure_last_error_
  encode_validity_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_vocab_sync_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_precheck_decision --> unexpected : encode_runtime [always] / on_unexpected_
  table_policy_decision --> unexpected : encode_runtime [always] / on_unexpected_
  table_sync_exec --> unexpected : encode_runtime [always] / on_unexpected_
  table_sync_result_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_prepare_exec --> unexpected : encode_runtime [always] / on_unexpected_
  encode_prepare_result_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_merge_input_capacity_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_merge_exec --> unexpected : encode_runtime [always] / on_unexpected_
  encode_merge_result_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_emit_input_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_exec --> unexpected : encode_runtime [always] / on_unexpected_
  emit_result_decision --> unexpected : encode_runtime [always] / on_unexpected_
  encode_result_decision --> unexpected : encode_runtime [always] / on_unexpected_
  initialized --> unexpected : encoding_done [always] / on_unexpected_
  initialized --> unexpected : encoding_error [always] / on_unexpected_
  encode_validity_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_validity_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_vocab_sync_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_vocab_sync_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_precheck_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_precheck_decision --> unexpected : encoding_error [always] / on_unexpected_
  table_policy_decision --> unexpected : encoding_done [always] / on_unexpected_
  table_policy_decision --> unexpected : encoding_error [always] / on_unexpected_
  table_sync_exec --> unexpected : encoding_done [always] / on_unexpected_
  table_sync_exec --> unexpected : encoding_error [always] / on_unexpected_
  table_sync_result_decision --> unexpected : encoding_done [always] / on_unexpected_
  table_sync_result_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_prepare_exec --> unexpected : encoding_done [always] / on_unexpected_
  encode_prepare_exec --> unexpected : encoding_error [always] / on_unexpected_
  encode_prepare_result_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_prepare_result_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_merge_input_capacity_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_merge_input_capacity_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_merge_exec --> unexpected : encoding_done [always] / on_unexpected_
  encode_merge_exec --> unexpected : encoding_error [always] / on_unexpected_
  encode_merge_result_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_merge_result_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_emit_input_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_emit_input_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_exec --> unexpected : encoding_done [always] / on_unexpected_
  encode_exec --> unexpected : encoding_error [always] / on_unexpected_
  emit_result_decision --> unexpected : encoding_done [always] / on_unexpected_
  emit_result_decision --> unexpected : encoding_error [always] / on_unexpected_
  encode_result_decision --> unexpected : encoding_done [always] / on_unexpected_
  encode_result_decision --> unexpected : encoding_error [always] / on_unexpected_
  done --> unexpected : encoding_done [always] / on_unexpected_
  done --> unexpected : encoding_error [always] / on_unexpected_
  errored --> unexpected : encoding_done [always] / on_unexpected_
  errored --> unexpected : encoding_error [always] / on_unexpected_
  unexpected --> unexpected : encoding_done [always] / on_unexpected_
  unexpected --> unexpected : encoding_error [always] / on_unexpected_
  initialized --> unexpected : _ [always] / on_unexpected_
  encode_validity_decision --> unexpected : _ [always] / on_unexpected_
  encode_vocab_sync_decision --> unexpected : _ [always] / on_unexpected_
  encode_precheck_decision --> unexpected : _ [always] / on_unexpected_
  table_policy_decision --> unexpected : _ [always] / on_unexpected_
  table_sync_exec --> unexpected : _ [always] / on_unexpected_
  table_sync_result_decision --> unexpected : _ [always] / on_unexpected_
  encode_prepare_exec --> unexpected : _ [always] / on_unexpected_
  encode_prepare_result_decision --> unexpected : _ [always] / on_unexpected_
  encode_merge_input_capacity_decision --> unexpected : _ [always] / on_unexpected_
  encode_merge_exec --> unexpected : _ [always] / on_unexpected_
  encode_merge_result_decision --> unexpected : _ [always] / on_unexpected_
  encode_emit_input_decision --> unexpected : _ [always] / on_unexpected_
  encode_exec --> unexpected : _ [always] / on_unexpected_
  emit_result_decision --> unexpected : _ [always] / on_unexpected_
  encode_result_decision --> unexpected : _ [always] / on_unexpected_
  done --> unexpected : _ [always] / on_unexpected_
  errored --> unexpected : _ [always] / on_unexpected_
  unexpected --> unexpected : _ [always] / on_unexpected_

Loading

Transitions

Source	Event	Guard	Action	Target
initialized	encode_runtime	always	none	encode_validity_decision
done	encode_runtime	always	none	encode_validity_decision
errored	encode_runtime	always	none	encode_validity_decision
unexpected	encode_runtime	always	none	encode_validity_decision
encode_validity_decision	completion<encode_runtime>	valid_encode>	none	encode_vocab_sync_decision
encode_validity_decision	completion<encode_runtime>	invalid_encode>	reject_invalid_encode>	errored
encode_validity_decision	completion<encode_runtime>	always	reject_invalid_encode>	errored
encode_vocab_sync_decision	completion<encode_runtime>	vocab_changed>	begin_encode_sync_vocab>	encode_precheck_decision
encode_vocab_sync_decision	completion<encode_runtime>	vocab_unchanged>	begin_encode>	encode_precheck_decision
encode_vocab_sync_decision	completion<encode_runtime>	always	reject_invalid_encode>	errored
encode_precheck_decision	completion<encode_runtime>	text_empty>	mark_done>	done
encode_precheck_decision	completion<encode_runtime>	text_non_empty>	none	table_policy_decision
encode_precheck_decision	completion<encode_runtime>	always	ensure_last_error>	errored
table_policy_decision	completion<encode_runtime>	tables_missing>	none	table_sync_exec
table_policy_decision	completion<encode_runtime>	tables_ready>	none	encode_prepare_exec
table_policy_decision	completion<encode_runtime>	always	ensure_last_error>	errored
table_sync_exec	completion<encode_runtime>	always	sync_tables>	table_sync_result_decision
table_sync_result_decision	completion<encode_runtime>	table_sync_ok>	none	encode_prepare_exec
table_sync_result_decision	completion<encode_runtime>	table_sync_invalid_argument_error>	ensure_last_error>	errored
table_sync_result_decision	completion<encode_runtime>	table_sync_backend_error>	ensure_last_error>	errored
table_sync_result_decision	completion<encode_runtime>	table_sync_model_invalid_error>	ensure_last_error>	errored
table_sync_result_decision	completion<encode_runtime>	table_sync_unclassified_error_code>	ensure_last_error>	errored
encode_prepare_exec	completion<encode_runtime>	always	run_prepare>	encode_prepare_result_decision
encode_prepare_result_decision	completion<encode_runtime>	prepare_result_ok>	none	encode_merge_input_capacity_decision
encode_prepare_result_decision	completion<encode_runtime>	prepare_result_invalid_argument_error>	ensure_last_error>	errored
encode_prepare_result_decision	completion<encode_runtime>	prepare_result_backend_error>	ensure_last_error>	errored
encode_prepare_result_decision	completion<encode_runtime>	prepare_result_model_invalid_error>	ensure_last_error>	errored
encode_prepare_result_decision	completion<encode_runtime>	prepare_result_unclassified_error_code>	ensure_last_error>	errored
encode_merge_input_capacity_decision	completion<encode_runtime>	merge_symbol_capacity_within_limit>	none	encode_merge_exec
encode_merge_input_capacity_decision	completion<encode_runtime>	merge_symbol_capacity_exceeded>	reject_invalid_encode>	errored
encode_merge_input_capacity_decision	completion<encode_runtime>	always	reject_invalid_encode>	errored
encode_merge_exec	completion<encode_runtime>	always	run_merge>	encode_merge_result_decision
encode_merge_result_decision	completion<encode_runtime>	merge_result_ok>	none	encode_emit_input_decision
encode_merge_result_decision	completion<encode_runtime>	merge_result_invalid_argument_error>	ensure_last_error>	errored
encode_merge_result_decision	completion<encode_runtime>	merge_result_backend_error>	ensure_last_error>	errored
encode_merge_result_decision	completion<encode_runtime>	merge_result_model_invalid_error>	ensure_last_error>	errored
encode_merge_result_decision	completion<encode_runtime>	merge_result_unclassified_error_code>	ensure_last_error>	errored
encode_emit_input_decision	completion<encode_runtime>	symbols_present>	none	encode_exec
encode_emit_input_decision	completion<encode_runtime>	symbols_absent>	set_emit_result_empty>	encode_result_decision
encode_emit_input_decision	completion<encode_runtime>	always	ensure_last_error>	errored
encode_exec	completion<encode_runtime>	always	run_encode>	emit_result_decision
emit_result_decision	completion<encode_runtime>	emit_result_ok>	apply_emit_result_ok>	encode_result_decision
emit_result_decision	completion<encode_runtime>	emit_result_failed>	apply_emit_result_failed>	encode_result_decision
emit_result_decision	completion<encode_runtime>	always	ensure_last_error>	errored
encode_result_decision	completion<encode_runtime>	encode_result_ok>	mark_done>	done
encode_result_decision	completion<encode_runtime>	encode_result_invalid_argument_error>	ensure_last_error>	errored
encode_result_decision	completion<encode_runtime>	encode_result_backend_error>	ensure_last_error>	errored
encode_result_decision	completion<encode_runtime>	encode_result_model_invalid_error>	ensure_last_error>	errored
encode_result_decision	completion<encode_runtime>	encode_result_unclassified_error_code>	ensure_last_error>	errored
encode_validity_decision	encode_runtime	always	on_unexpected>	unexpected
encode_vocab_sync_decision	encode_runtime	always	on_unexpected>	unexpected
encode_precheck_decision	encode_runtime	always	on_unexpected>	unexpected
table_policy_decision	encode_runtime	always	on_unexpected>	unexpected
table_sync_exec	encode_runtime	always	on_unexpected>	unexpected
table_sync_result_decision	encode_runtime	always	on_unexpected>	unexpected
encode_prepare_exec	encode_runtime	always	on_unexpected>	unexpected
encode_prepare_result_decision	encode_runtime	always	on_unexpected>	unexpected
encode_merge_input_capacity_decision	encode_runtime	always	on_unexpected>	unexpected
encode_merge_exec	encode_runtime	always	on_unexpected>	unexpected
encode_merge_result_decision	encode_runtime	always	on_unexpected>	unexpected
encode_emit_input_decision	encode_runtime	always	on_unexpected>	unexpected
encode_exec	encode_runtime	always	on_unexpected>	unexpected
emit_result_decision	encode_runtime	always	on_unexpected>	unexpected
encode_result_decision	encode_runtime	always	on_unexpected>	unexpected
initialized	encoding_done	always	on_unexpected>	unexpected
initialized	encoding_error	always	on_unexpected>	unexpected
encode_validity_decision	encoding_done	always	on_unexpected>	unexpected
encode_validity_decision	encoding_error	always	on_unexpected>	unexpected
encode_vocab_sync_decision	encoding_done	always	on_unexpected>	unexpected
encode_vocab_sync_decision	encoding_error	always	on_unexpected>	unexpected
encode_precheck_decision	encoding_done	always	on_unexpected>	unexpected
encode_precheck_decision	encoding_error	always	on_unexpected>	unexpected
table_policy_decision	encoding_done	always	on_unexpected>	unexpected
table_policy_decision	encoding_error	always	on_unexpected>	unexpected
table_sync_exec	encoding_done	always	on_unexpected>	unexpected
table_sync_exec	encoding_error	always	on_unexpected>	unexpected
table_sync_result_decision	encoding_done	always	on_unexpected>	unexpected
table_sync_result_decision	encoding_error	always	on_unexpected>	unexpected
encode_prepare_exec	encoding_done	always	on_unexpected>	unexpected
encode_prepare_exec	encoding_error	always	on_unexpected>	unexpected
encode_prepare_result_decision	encoding_done	always	on_unexpected>	unexpected
encode_prepare_result_decision	encoding_error	always	on_unexpected>	unexpected
encode_merge_input_capacity_decision	encoding_done	always	on_unexpected>	unexpected
encode_merge_input_capacity_decision	encoding_error	always	on_unexpected>	unexpected
encode_merge_exec	encoding_done	always	on_unexpected>	unexpected
encode_merge_exec	encoding_error	always	on_unexpected>	unexpected
encode_merge_result_decision	encoding_done	always	on_unexpected>	unexpected
encode_merge_result_decision	encoding_error	always	on_unexpected>	unexpected
encode_emit_input_decision	encoding_done	always	on_unexpected>	unexpected
encode_emit_input_decision	encoding_error	always	on_unexpected>	unexpected
encode_exec	encoding_done	always	on_unexpected>	unexpected
encode_exec	encoding_error	always	on_unexpected>	unexpected
emit_result_decision	encoding_done	always	on_unexpected>	unexpected
emit_result_decision	encoding_error	always	on_unexpected>	unexpected
encode_result_decision	encoding_done	always	on_unexpected>	unexpected
encode_result_decision	encoding_error	always	on_unexpected>	unexpected
done	encoding_done	always	on_unexpected>	unexpected
done	encoding_error	always	on_unexpected>	unexpected
errored	encoding_done	always	on_unexpected>	unexpected
errored	encoding_error	always	on_unexpected>	unexpected
unexpected	encoding_done	always	on_unexpected>	unexpected
unexpected	encoding_error	always	on_unexpected>	unexpected
initialized	_	always	on_unexpected>	unexpected
encode_validity_decision	_	always	on_unexpected>	unexpected
encode_vocab_sync_decision	_	always	on_unexpected>	unexpected
encode_precheck_decision	_	always	on_unexpected>	unexpected
table_policy_decision	_	always	on_unexpected>	unexpected
table_sync_exec	_	always	on_unexpected>	unexpected
table_sync_result_decision	_	always	on_unexpected>	unexpected
encode_prepare_exec	_	always	on_unexpected>	unexpected
encode_prepare_result_decision	_	always	on_unexpected>	unexpected
encode_merge_input_capacity_decision	_	always	on_unexpected>	unexpected
encode_merge_exec	_	always	on_unexpected>	unexpected
encode_merge_result_decision	_	always	on_unexpected>	unexpected
encode_emit_input_decision	_	always	on_unexpected>	unexpected
encode_exec	_	always	on_unexpected>	unexpected
emit_result_decision	_	always	on_unexpected>	unexpected
encode_result_decision	_	always	on_unexpected>	unexpected
done	_	always	on_unexpected>	unexpected
errored	_	always	on_unexpected>	unexpected
unexpected	_	always	on_unexpected>	unexpected