simple : fix token counting
This commit is contained in:
parent
25bd254089
commit
467e307931
1 changed files with 1 additions and 0 deletions
|
@ -81,6 +81,7 @@ int main(int argc, char ** argv) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
n_cur += tokens_list.size();
|
||||
tokens_list.clear();
|
||||
|
||||
// sample the next token
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue