Update src/llama.cpp

This commit is contained in:
Georgi Gerganov 2024-07-04 13:50:47 +03:00 committed by GitHub
parent 09a31e2adc
commit 763d4aaf56
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -14346,7 +14346,7 @@ private:
size_t prefix_offset = input_offset;
unicode_cpt_from_utf8(input, prefix_offset);
return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset };
} catch(std::invalid_argument & /*ex*/) {
} catch (std::invalid_argument & /*ex*/) {
// if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER
return { "\xEF\xBF\xBD", 3, 1 };
}