parent
							
								
									5047dd3546
								
							
						
					
					
						commit
						727368c60f
					
				
					 11 changed files with 53 additions and 54 deletions
				
			
		|  | @ -507,7 +507,7 @@ static std::string tokens_to_str(llama_context * ctx, Iter begin, Iter end) { | |||
| 
 | ||||
| // format incomplete utf-8 multibyte character for output
 | ||||
| static std::string tokens_to_output_formatted_string(const llama_context * ctx, const llama_token token) { | ||||
|     std::string out = token == -1 ? "" : common_token_to_piece(ctx, token); | ||||
|     std::string out = token == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(ctx, token); | ||||
| 
 | ||||
|     // if the size is 1 and first bit is 1, meaning it's a partial character
 | ||||
|     //   (size > 1 meaning it's already a known token)
 | ||||
|  |  | |||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue