diff --git a/common/chaton.hpp b/common/chaton.hpp index c914cde5d..1cccee95b 100644 --- a/common/chaton.hpp +++ b/common/chaton.hpp @@ -85,7 +85,9 @@ json conMeta; /** - * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence + * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence. + * Inturn gives the flexibility to tokenize with or without parse_special flag, wrt the different parts of the chat msg(s). + * One could use the triplet of str, get_types and get_partslens to achieve the above mentioned flexibility. */ class ChatParts { @@ -100,7 +102,7 @@ public: // Identify no string condition and or ignore string. static const auto X = '?'; - ChatParts() :parts{}, types{""} {} + ChatParts() : parts{}, types{""} {} char last_type() { if (types.length() == 0) { @@ -126,6 +128,18 @@ public: return allin; } + std::string get_types() { + return types; + } + + std::vector get_partslens() { + std::vector lens = {}; + for(auto part: parts) { + lens.push_back(part.length()); + } + return lens; + } + std::string name() { return typeid(*this).name(); }