1
0
Fork 0
forked from mirrors/tar-split

common: add a UTF-8 check helper

This commit is contained in:
Vincent Batts 2015-09-23 13:30:00 -04:00
parent 7384cf1827
commit 2865353200
2 changed files with 55 additions and 0 deletions

21
tar/common/utf8.go Normal file
View file

@ -0,0 +1,21 @@
package common
// IsValidUtf8String checks for in valid UTF-8 characters
func IsValidUtf8String(s string) bool {
for _, r := range s {
if int(r) == 0xfffd {
return false
}
}
return true
}
// IsValidUtf8Btyes checks for in valid UTF-8 characters
func IsValidUtf8Btyes(b []byte) bool {
for _, r := range string(b) {
if int(r) == 0xfffd {
return false
}
}
return true
}

34
tar/common/utf8_test.go Normal file
View file

@ -0,0 +1,34 @@
package common
import "testing"
func TestStringValidation(t *testing.T) {
cases := []struct {
value string
result bool
}{
{"aä\uFFFD本☺", false},
{"aä本☺", true},
}
for _, c := range cases {
if got := IsValidUtf8String(c.value); got != c.result {
t.Errorf("string %q - expected %v, got %v", c.value, c.result, got)
}
}
}
func TestBytesValidation(t *testing.T) {
cases := []struct {
value []byte
result bool
}{
{[]byte{0xE4}, false},
{[]byte("aä本☺"), true},
}
for _, c := range cases {
if got := IsValidUtf8Btyes(c.value); got != c.result {
t.Errorf("bytes %q - expected %v, got %v", c.value, c.result, got)
}
}
}