748 lines
14 KiB
Go
748 lines
14 KiB
Go
// Copyright 2010 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package html
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"io/ioutil"
|
|
"reflect"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
type tokenTest struct {
|
|
// A short description of the test case.
|
|
desc string
|
|
// The HTML to parse.
|
|
html string
|
|
// The string representations of the expected tokens, joined by '$'.
|
|
golden string
|
|
}
|
|
|
|
var tokenTests = []tokenTest{
|
|
{
|
|
"empty",
|
|
"",
|
|
"",
|
|
},
|
|
// A single text node. The tokenizer should not break text nodes on whitespace,
|
|
// nor should it normalize whitespace within a text node.
|
|
{
|
|
"text",
|
|
"foo bar",
|
|
"foo bar",
|
|
},
|
|
// An entity.
|
|
{
|
|
"entity",
|
|
"one < two",
|
|
"one < two",
|
|
},
|
|
// A start, self-closing and end tag. The tokenizer does not care if the start
|
|
// and end tokens don't match; that is the job of the parser.
|
|
{
|
|
"tags",
|
|
"<a>b<c/>d</e>",
|
|
"<a>$b$<c/>$d$</e>",
|
|
},
|
|
// Angle brackets that aren't a tag.
|
|
{
|
|
"not a tag #0",
|
|
"<",
|
|
"<",
|
|
},
|
|
{
|
|
"not a tag #1",
|
|
"</",
|
|
"</",
|
|
},
|
|
{
|
|
"not a tag #2",
|
|
"</>",
|
|
"<!---->",
|
|
},
|
|
{
|
|
"not a tag #3",
|
|
"a</>b",
|
|
"a$<!---->$b",
|
|
},
|
|
{
|
|
"not a tag #4",
|
|
"</ >",
|
|
"<!-- -->",
|
|
},
|
|
{
|
|
"not a tag #5",
|
|
"</.",
|
|
"<!--.-->",
|
|
},
|
|
{
|
|
"not a tag #6",
|
|
"</.>",
|
|
"<!--.-->",
|
|
},
|
|
{
|
|
"not a tag #7",
|
|
"a < b",
|
|
"a < b",
|
|
},
|
|
{
|
|
"not a tag #8",
|
|
"<.>",
|
|
"<.>",
|
|
},
|
|
{
|
|
"not a tag #9",
|
|
"a<<<b>>>c",
|
|
"a<<$<b>$>>c",
|
|
},
|
|
{
|
|
"not a tag #10",
|
|
"if x<0 and y < 0 then x*y>0",
|
|
"if x<0 and y < 0 then x*y>0",
|
|
},
|
|
{
|
|
"not a tag #11",
|
|
"<<p>",
|
|
"<$<p>",
|
|
},
|
|
// EOF in a tag name.
|
|
{
|
|
"tag name eof #0",
|
|
"<a",
|
|
"",
|
|
},
|
|
{
|
|
"tag name eof #1",
|
|
"<a ",
|
|
"",
|
|
},
|
|
{
|
|
"tag name eof #2",
|
|
"a<b",
|
|
"a",
|
|
},
|
|
{
|
|
"tag name eof #3",
|
|
"<a><b",
|
|
"<a>",
|
|
},
|
|
{
|
|
"tag name eof #4",
|
|
`<a x`,
|
|
``,
|
|
},
|
|
// Some malformed tags that are missing a '>'.
|
|
{
|
|
"malformed tag #0",
|
|
`<p</p>`,
|
|
`<p< p="">`,
|
|
},
|
|
{
|
|
"malformed tag #1",
|
|
`<p </p>`,
|
|
`<p <="" p="">`,
|
|
},
|
|
{
|
|
"malformed tag #2",
|
|
`<p id`,
|
|
``,
|
|
},
|
|
{
|
|
"malformed tag #3",
|
|
`<p id=`,
|
|
``,
|
|
},
|
|
{
|
|
"malformed tag #4",
|
|
`<p id=>`,
|
|
`<p id="">`,
|
|
},
|
|
{
|
|
"malformed tag #5",
|
|
`<p id=0`,
|
|
``,
|
|
},
|
|
{
|
|
"malformed tag #6",
|
|
`<p id=0</p>`,
|
|
`<p id="0</p">`,
|
|
},
|
|
{
|
|
"malformed tag #7",
|
|
`<p id="0</p>`,
|
|
``,
|
|
},
|
|
{
|
|
"malformed tag #8",
|
|
`<p id="0"</p>`,
|
|
`<p id="0" <="" p="">`,
|
|
},
|
|
{
|
|
"malformed tag #9",
|
|
`<p></p id`,
|
|
`<p>`,
|
|
},
|
|
// Raw text and RCDATA.
|
|
{
|
|
"basic raw text",
|
|
"<script><a></b></script>",
|
|
"<script>$<a></b>$</script>",
|
|
},
|
|
{
|
|
"unfinished script end tag",
|
|
"<SCRIPT>a</SCR",
|
|
"<script>$a</SCR",
|
|
},
|
|
{
|
|
"broken script end tag",
|
|
"<SCRIPT>a</SCR ipt>",
|
|
"<script>$a</SCR ipt>",
|
|
},
|
|
{
|
|
"EOF in script end tag",
|
|
"<SCRIPT>a</SCRipt",
|
|
"<script>$a</SCRipt",
|
|
},
|
|
{
|
|
"scriptx end tag",
|
|
"<SCRIPT>a</SCRiptx",
|
|
"<script>$a</SCRiptx",
|
|
},
|
|
{
|
|
"' ' completes script end tag",
|
|
"<SCRIPT>a</SCRipt ",
|
|
"<script>$a",
|
|
},
|
|
{
|
|
"'>' completes script end tag",
|
|
"<SCRIPT>a</SCRipt>",
|
|
"<script>$a$</script>",
|
|
},
|
|
{
|
|
"self-closing script end tag",
|
|
"<SCRIPT>a</SCRipt/>",
|
|
"<script>$a$</script>",
|
|
},
|
|
{
|
|
"nested script tag",
|
|
"<SCRIPT>a</SCRipt<script>",
|
|
"<script>$a</SCRipt<script>",
|
|
},
|
|
{
|
|
"script end tag after unfinished",
|
|
"<SCRIPT>a</SCRipt</script>",
|
|
"<script>$a</SCRipt$</script>",
|
|
},
|
|
{
|
|
"script/style mismatched tags",
|
|
"<script>a</style>",
|
|
"<script>$a</style>",
|
|
},
|
|
{
|
|
"style element with entity",
|
|
"<style>'",
|
|
"<style>$&apos;",
|
|
},
|
|
{
|
|
"textarea with tag",
|
|
"<textarea><div></textarea>",
|
|
"<textarea>$<div>$</textarea>",
|
|
},
|
|
{
|
|
"title with tag and entity",
|
|
"<title><b>K&R C</b></title>",
|
|
"<title>$<b>K&R C</b>$</title>",
|
|
},
|
|
// DOCTYPE tests.
|
|
{
|
|
"Proper DOCTYPE",
|
|
"<!DOCTYPE html>",
|
|
"<!DOCTYPE html>",
|
|
},
|
|
{
|
|
"DOCTYPE with no space",
|
|
"<!doctypehtml>",
|
|
"<!DOCTYPE html>",
|
|
},
|
|
{
|
|
"DOCTYPE with two spaces",
|
|
"<!doctype html>",
|
|
"<!DOCTYPE html>",
|
|
},
|
|
{
|
|
"looks like DOCTYPE but isn't",
|
|
"<!DOCUMENT html>",
|
|
"<!--DOCUMENT html-->",
|
|
},
|
|
{
|
|
"DOCTYPE at EOF",
|
|
"<!DOCtype",
|
|
"<!DOCTYPE >",
|
|
},
|
|
// XML processing instructions.
|
|
{
|
|
"XML processing instruction",
|
|
"<?xml?>",
|
|
"<!--?xml?-->",
|
|
},
|
|
// Comments.
|
|
{
|
|
"comment0",
|
|
"abc<b><!-- skipme --></b>def",
|
|
"abc$<b>$<!-- skipme -->$</b>$def",
|
|
},
|
|
{
|
|
"comment1",
|
|
"a<!-->z",
|
|
"a$<!---->$z",
|
|
},
|
|
{
|
|
"comment2",
|
|
"a<!--->z",
|
|
"a$<!---->$z",
|
|
},
|
|
{
|
|
"comment3",
|
|
"a<!--x>-->z",
|
|
"a$<!--x>-->$z",
|
|
},
|
|
{
|
|
"comment4",
|
|
"a<!--x->-->z",
|
|
"a$<!--x->-->$z",
|
|
},
|
|
{
|
|
"comment5",
|
|
"a<!>z",
|
|
"a$<!---->$z",
|
|
},
|
|
{
|
|
"comment6",
|
|
"a<!->z",
|
|
"a$<!----->$z",
|
|
},
|
|
{
|
|
"comment7",
|
|
"a<!---<>z",
|
|
"a$<!---<>z-->",
|
|
},
|
|
{
|
|
"comment8",
|
|
"a<!--z",
|
|
"a$<!--z-->",
|
|
},
|
|
{
|
|
"comment9",
|
|
"a<!--z-",
|
|
"a$<!--z-->",
|
|
},
|
|
{
|
|
"comment10",
|
|
"a<!--z--",
|
|
"a$<!--z-->",
|
|
},
|
|
{
|
|
"comment11",
|
|
"a<!--z---",
|
|
"a$<!--z--->",
|
|
},
|
|
{
|
|
"comment12",
|
|
"a<!--z----",
|
|
"a$<!--z---->",
|
|
},
|
|
{
|
|
"comment13",
|
|
"a<!--x--!>z",
|
|
"a$<!--x-->$z",
|
|
},
|
|
// An attribute with a backslash.
|
|
{
|
|
"backslash",
|
|
`<p id="a\"b">`,
|
|
`<p id="a\" b"="">`,
|
|
},
|
|
// Entities, tag name and attribute key lower-casing, and whitespace
|
|
// normalization within a tag.
|
|
{
|
|
"tricky",
|
|
"<p \t\n iD=\"a"B\" foo=\"bar\"><EM>te<&;xt</em></p>",
|
|
`<p id="a"B" foo="bar">$<em>$te<&;xt$</em>$</p>`,
|
|
},
|
|
// A nonexistent entity. Tokenizing and converting back to a string should
|
|
// escape the "&" to become "&".
|
|
{
|
|
"noSuchEntity",
|
|
`<a b="c&noSuchEntity;d"><&alsoDoesntExist;&`,
|
|
`<a b="c&noSuchEntity;d">$<&alsoDoesntExist;&`,
|
|
},
|
|
{
|
|
"entity without semicolon",
|
|
`¬it;∉<a b="q=z&=5¬ice=hello¬=world">`,
|
|
`¬it;∉$<a b="q=z&amp=5&notice=hello¬=world">`,
|
|
},
|
|
{
|
|
"entity with digits",
|
|
"½",
|
|
"½",
|
|
},
|
|
// Attribute tests:
|
|
// http://dev.w3.org/html5/pf-summary/Overview.html#attributes
|
|
{
|
|
"Empty attribute",
|
|
`<input disabled FOO>`,
|
|
`<input disabled="" foo="">`,
|
|
},
|
|
{
|
|
"Empty attribute, whitespace",
|
|
`<input disabled FOO >`,
|
|
`<input disabled="" foo="">`,
|
|
},
|
|
{
|
|
"Unquoted attribute value",
|
|
`<input value=yes FOO=BAR>`,
|
|
`<input value="yes" foo="BAR">`,
|
|
},
|
|
{
|
|
"Unquoted attribute value, spaces",
|
|
`<input value = yes FOO = BAR>`,
|
|
`<input value="yes" foo="BAR">`,
|
|
},
|
|
{
|
|
"Unquoted attribute value, trailing space",
|
|
`<input value=yes FOO=BAR >`,
|
|
`<input value="yes" foo="BAR">`,
|
|
},
|
|
{
|
|
"Single-quoted attribute value",
|
|
`<input value='yes' FOO='BAR'>`,
|
|
`<input value="yes" foo="BAR">`,
|
|
},
|
|
{
|
|
"Single-quoted attribute value, trailing space",
|
|
`<input value='yes' FOO='BAR' >`,
|
|
`<input value="yes" foo="BAR">`,
|
|
},
|
|
{
|
|
"Double-quoted attribute value",
|
|
`<input value="I'm an attribute" FOO="BAR">`,
|
|
`<input value="I'm an attribute" foo="BAR">`,
|
|
},
|
|
{
|
|
"Attribute name characters",
|
|
`<meta http-equiv="content-type">`,
|
|
`<meta http-equiv="content-type">`,
|
|
},
|
|
{
|
|
"Mixed attributes",
|
|
`a<P V="0 1" w='2' X=3 y>z`,
|
|
`a$<p v="0 1" w="2" x="3" y="">$z`,
|
|
},
|
|
{
|
|
"Attributes with a solitary single quote",
|
|
`<p id=can't><p id=won't>`,
|
|
`<p id="can't">$<p id="won't">`,
|
|
},
|
|
}
|
|
|
|
func TestTokenizer(t *testing.T) {
|
|
loop:
|
|
for _, tt := range tokenTests {
|
|
z := NewTokenizer(strings.NewReader(tt.html))
|
|
if tt.golden != "" {
|
|
for i, s := range strings.Split(tt.golden, "$") {
|
|
if z.Next() == ErrorToken {
|
|
t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Err())
|
|
continue loop
|
|
}
|
|
actual := z.Token().String()
|
|
if s != actual {
|
|
t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual)
|
|
continue loop
|
|
}
|
|
}
|
|
}
|
|
z.Next()
|
|
if z.Err() != io.EOF {
|
|
t.Errorf("%s: want EOF got %q", tt.desc, z.Err())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMaxBuffer(t *testing.T) {
|
|
// Exceeding the maximum buffer size generates ErrBufferExceeded.
|
|
z := NewTokenizer(strings.NewReader("<" + strings.Repeat("t", 10)))
|
|
z.SetMaxBuf(5)
|
|
tt := z.Next()
|
|
if got, want := tt, ErrorToken; got != want {
|
|
t.Fatalf("token type: got: %v want: %v", got, want)
|
|
}
|
|
if got, want := z.Err(), ErrBufferExceeded; got != want {
|
|
t.Errorf("error type: got: %v want: %v", got, want)
|
|
}
|
|
if got, want := string(z.Raw()), "<tttt"; got != want {
|
|
t.Fatalf("buffered before overflow: got: %q want: %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestMaxBufferReconstruction(t *testing.T) {
|
|
// Exceeding the maximum buffer size at any point while tokenizing permits
|
|
// reconstructing the original input.
|
|
tests:
|
|
for _, test := range tokenTests {
|
|
for maxBuf := 1; ; maxBuf++ {
|
|
r := strings.NewReader(test.html)
|
|
z := NewTokenizer(r)
|
|
z.SetMaxBuf(maxBuf)
|
|
var tokenized bytes.Buffer
|
|
for {
|
|
tt := z.Next()
|
|
tokenized.Write(z.Raw())
|
|
if tt == ErrorToken {
|
|
if err := z.Err(); err != io.EOF && err != ErrBufferExceeded {
|
|
t.Errorf("%s: unexpected error: %v", test.desc, err)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
// Anything tokenized along with untokenized input or data left in the reader.
|
|
assembled, err := ioutil.ReadAll(io.MultiReader(&tokenized, bytes.NewReader(z.Buffered()), r))
|
|
if err != nil {
|
|
t.Errorf("%s: ReadAll: %v", test.desc, err)
|
|
continue tests
|
|
}
|
|
if got, want := string(assembled), test.html; got != want {
|
|
t.Errorf("%s: reassembled html:\n got: %q\nwant: %q", test.desc, got, want)
|
|
continue tests
|
|
}
|
|
// EOF indicates that we completed tokenization and hence found the max
|
|
// maxBuf that generates ErrBufferExceeded, so continue to the next test.
|
|
if z.Err() == io.EOF {
|
|
break
|
|
}
|
|
} // buffer sizes
|
|
} // tests
|
|
}
|
|
|
|
func TestPassthrough(t *testing.T) {
|
|
// Accumulating the raw output for each parse event should reconstruct the
|
|
// original input.
|
|
for _, test := range tokenTests {
|
|
z := NewTokenizer(strings.NewReader(test.html))
|
|
var parsed bytes.Buffer
|
|
for {
|
|
tt := z.Next()
|
|
parsed.Write(z.Raw())
|
|
if tt == ErrorToken {
|
|
break
|
|
}
|
|
}
|
|
if got, want := parsed.String(), test.html; got != want {
|
|
t.Errorf("%s: parsed output:\n got: %q\nwant: %q", test.desc, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBufAPI(t *testing.T) {
|
|
s := "0<a>1</a>2<b>3<a>4<a>5</a>6</b>7</a>8<a/>9"
|
|
z := NewTokenizer(bytes.NewBufferString(s))
|
|
var result bytes.Buffer
|
|
depth := 0
|
|
loop:
|
|
for {
|
|
tt := z.Next()
|
|
switch tt {
|
|
case ErrorToken:
|
|
if z.Err() != io.EOF {
|
|
t.Error(z.Err())
|
|
}
|
|
break loop
|
|
case TextToken:
|
|
if depth > 0 {
|
|
result.Write(z.Text())
|
|
}
|
|
case StartTagToken, EndTagToken:
|
|
tn, _ := z.TagName()
|
|
if len(tn) == 1 && tn[0] == 'a' {
|
|
if tt == StartTagToken {
|
|
depth++
|
|
} else {
|
|
depth--
|
|
}
|
|
}
|
|
}
|
|
}
|
|
u := "14567"
|
|
v := string(result.Bytes())
|
|
if u != v {
|
|
t.Errorf("TestBufAPI: want %q got %q", u, v)
|
|
}
|
|
}
|
|
|
|
func TestConvertNewlines(t *testing.T) {
|
|
testCases := map[string]string{
|
|
"Mac\rDOS\r\nUnix\n": "Mac\nDOS\nUnix\n",
|
|
"Unix\nMac\rDOS\r\n": "Unix\nMac\nDOS\n",
|
|
"DOS\r\nDOS\r\nDOS\r\n": "DOS\nDOS\nDOS\n",
|
|
"": "",
|
|
"\n": "\n",
|
|
"\n\r": "\n\n",
|
|
"\r": "\n",
|
|
"\r\n": "\n",
|
|
"\r\n\n": "\n\n",
|
|
"\r\n\r": "\n\n",
|
|
"\r\n\r\n": "\n\n",
|
|
"\r\r": "\n\n",
|
|
"\r\r\n": "\n\n",
|
|
"\r\r\n\n": "\n\n\n",
|
|
"\r\r\r\n": "\n\n\n",
|
|
"\r \n": "\n \n",
|
|
"xyz": "xyz",
|
|
}
|
|
for in, want := range testCases {
|
|
if got := string(convertNewlines([]byte(in))); got != want {
|
|
t.Errorf("input %q: got %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReaderEdgeCases(t *testing.T) {
|
|
const s = "<p>An io.Reader can return (0, nil) or (n, io.EOF).</p>"
|
|
testCases := []io.Reader{
|
|
&zeroOneByteReader{s: s},
|
|
&eofStringsReader{s: s},
|
|
&stuckReader{},
|
|
}
|
|
for i, tc := range testCases {
|
|
got := []TokenType{}
|
|
z := NewTokenizer(tc)
|
|
for {
|
|
tt := z.Next()
|
|
if tt == ErrorToken {
|
|
break
|
|
}
|
|
got = append(got, tt)
|
|
}
|
|
if err := z.Err(); err != nil && err != io.EOF {
|
|
if err != io.ErrNoProgress {
|
|
t.Errorf("i=%d: %v", i, err)
|
|
}
|
|
continue
|
|
}
|
|
want := []TokenType{
|
|
StartTagToken,
|
|
TextToken,
|
|
EndTagToken,
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("i=%d: got %v, want %v", i, got, want)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
// zeroOneByteReader is like a strings.Reader that alternates between
|
|
// returning 0 bytes and 1 byte at a time.
|
|
type zeroOneByteReader struct {
|
|
s string
|
|
n int
|
|
}
|
|
|
|
func (r *zeroOneByteReader) Read(p []byte) (int, error) {
|
|
if len(p) == 0 {
|
|
return 0, nil
|
|
}
|
|
if len(r.s) == 0 {
|
|
return 0, io.EOF
|
|
}
|
|
r.n++
|
|
if r.n%2 != 0 {
|
|
return 0, nil
|
|
}
|
|
p[0], r.s = r.s[0], r.s[1:]
|
|
return 1, nil
|
|
}
|
|
|
|
// eofStringsReader is like a strings.Reader but can return an (n, err) where
|
|
// n > 0 && err != nil.
|
|
type eofStringsReader struct {
|
|
s string
|
|
}
|
|
|
|
func (r *eofStringsReader) Read(p []byte) (int, error) {
|
|
n := copy(p, r.s)
|
|
r.s = r.s[n:]
|
|
if r.s != "" {
|
|
return n, nil
|
|
}
|
|
return n, io.EOF
|
|
}
|
|
|
|
// stuckReader is an io.Reader that always returns no data and no error.
|
|
type stuckReader struct{}
|
|
|
|
func (*stuckReader) Read(p []byte) (int, error) {
|
|
return 0, nil
|
|
}
|
|
|
|
const (
|
|
rawLevel = iota
|
|
lowLevel
|
|
highLevel
|
|
)
|
|
|
|
func benchmarkTokenizer(b *testing.B, level int) {
|
|
buf, err := ioutil.ReadFile("testdata/go1.html")
|
|
if err != nil {
|
|
b.Fatalf("could not read testdata/go1.html: %v", err)
|
|
}
|
|
b.SetBytes(int64(len(buf)))
|
|
runtime.GC()
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
z := NewTokenizer(bytes.NewBuffer(buf))
|
|
for {
|
|
tt := z.Next()
|
|
if tt == ErrorToken {
|
|
if err := z.Err(); err != nil && err != io.EOF {
|
|
b.Fatalf("tokenizer error: %v", err)
|
|
}
|
|
break
|
|
}
|
|
switch level {
|
|
case rawLevel:
|
|
// Calling z.Raw just returns the raw bytes of the token. It does
|
|
// not unescape < to <, or lower-case tag names and attribute keys.
|
|
z.Raw()
|
|
case lowLevel:
|
|
// Caling z.Text, z.TagName and z.TagAttr returns []byte values
|
|
// whose contents may change on the next call to z.Next.
|
|
switch tt {
|
|
case TextToken, CommentToken, DoctypeToken:
|
|
z.Text()
|
|
case StartTagToken, SelfClosingTagToken:
|
|
_, more := z.TagName()
|
|
for more {
|
|
_, _, more = z.TagAttr()
|
|
}
|
|
case EndTagToken:
|
|
z.TagName()
|
|
}
|
|
case highLevel:
|
|
// Calling z.Token converts []byte values to strings whose validity
|
|
// extend beyond the next call to z.Next.
|
|
z.Token()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkRawLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, rawLevel) }
|
|
func BenchmarkLowLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, lowLevel) }
|
|
func BenchmarkHighLevelTokenizer(b *testing.B) { benchmarkTokenizer(b, highLevel) }
|