Transfer from dep to go modules.

Also update vendored packages while we're at it.
pull/22/head
Stephen Gregoratto 5 years ago
parent e41559ff17
commit 3ec497b96f
No known key found for this signature in database
GPG Key ID: E65F3C31DBD4FEF2

104
Gopkg.lock generated

@ -1,104 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
digest = "1:289dd4d7abfb3ad2b5f728fbe9b1d5c1bf7d265a3eb9ef92869af1f7baba4c7a"
name = "github.com/BurntSushi/toml"
packages = ["."]
pruneopts = ""
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"
[[projects]]
digest = "1:c742448be99f29f5233ab05bd50eea54ab53f438a3465afa8b247791f0cd885b"
name = "github.com/anaskhan96/soup"
packages = ["."]
pruneopts = ""
revision = "cd07662ec9300d16d2ece1c24b0b29ad89f65ff4"
version = "v1.1"
[[projects]]
digest = "1:d7d69c103aba5ea229451115b28ca06d061e3861b2f44bcd0100cbc65659c4d9"
name = "github.com/fatih/color"
packages = ["."]
pruneopts = ""
revision = "507f6050b8568533fb3f5504de8e5205fa62a114"
version = "v1.6.0"
[[projects]]
digest = "1:9ea83adf8e96d6304f394d40436f2eb44c1dc3250d223b74088cc253a6cd0a1c"
name = "github.com/mattn/go-colorable"
packages = ["."]
pruneopts = ""
revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
version = "v0.0.9"
[[projects]]
digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
name = "github.com/mattn/go-isatty"
packages = ["."]
pruneopts = ""
revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
version = "v0.0.3"
[[projects]]
digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
name = "github.com/mattn/go-runewidth"
packages = ["."]
pruneopts = ""
revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
version = "v0.0.2"
[[projects]]
branch = "master"
digest = "1:80ec738f420f13c23460cc92cd893d3e403588c60682afebbd4515eca5e19578"
name = "golang.org/x/net"
packages = [
"html",
"html/atom",
]
pruneopts = ""
revision = "5f9ae10d9af5b1c89ae6904293b14b064d4ada23"
[[projects]]
branch = "master"
digest = "1:ba4898d0f47bb5a26d7e1842bc1b29799441fbcc76aed65ba45d5b6918e35130"
name = "golang.org/x/sys"
packages = ["unix"]
pruneopts = ""
revision = "bb9c189858d91f42db229b04d45a4c3d23a7662a"
[[projects]]
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text"
packages = [
"feature/plural",
"internal",
"internal/catmsg",
"internal/format",
"internal/gen",
"internal/number",
"internal/stringset",
"internal/tag",
"language",
"message",
"message/catalog",
"unicode/cldr",
]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/BurntSushi/toml",
"github.com/anaskhan96/soup",
"github.com/fatih/color",
"github.com/mattn/go-runewidth",
"golang.org/x/text/language",
"golang.org/x/text/message",
]
solver-name = "gps-cdcl"
solver-version = 1

@ -1,31 +0,0 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
name = "github.com/fatih/color"
version = "1.6.0"

@ -0,0 +1,13 @@
module github.com/miguelmota/cointop
require (
github.com/BurntSushi/toml v0.3.1
github.com/anaskhan96/soup v1.1.1
github.com/fatih/color v1.7.0
github.com/mattn/go-colorable v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.4 // indirect
github.com/mattn/go-runewidth v0.0.4
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 // indirect
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 // indirect
golang.org/x/text v0.3.0
)

@ -0,0 +1,25 @@
github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/anaskhan96/soup v0.0.0-20180328123631-cd07662ec930/go.mod h1:pT5vs4HXDwA5y4KQCsKvnkpQd3D+joP7IqpiGskfWW0=
github.com/anaskhan96/soup v1.1.1 h1:Duux/0htS2Va7XLJ9qIakCSey790hg9OFRm2FwlMTy0=
github.com/anaskhan96/soup v1.1.1/go.mod h1:pT5vs4HXDwA5y4KQCsKvnkpQd3D+joP7IqpiGskfWW0=
github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
golang.org/x/net v0.0.0-20180420171651-5f9ae10d9af5/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sys v0.0.0-20180425194835-bb9c189858d9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 h1:IcgEB62HYgAhX0Nd/QrVgZlxlcyxbGQHElLUhW2X4Fo=
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

@ -1,14 +1,21 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
The MIT License (MIT)
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Copyright (c) 2013 TOML authors
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

@ -1,61 +0,0 @@
package main
import (
"fmt"
"time"
"github.com/BurntSushi/toml"
)
type tomlConfig struct {
Title string
Owner ownerInfo
DB database `toml:"database"`
Servers map[string]server
Clients clients
}
type ownerInfo struct {
Name string
Org string `toml:"organization"`
Bio string
DOB time.Time
}
type database struct {
Server string
Ports []int
ConnMax int `toml:"connection_max"`
Enabled bool
}
type server struct {
IP string
DC string
}
type clients struct {
Data [][]interface{}
Hosts []string
}
func main() {
var config tomlConfig
if _, err := toml.DecodeFile("example.toml", &config); err != nil {
fmt.Println(err)
return
}
fmt.Printf("Title: %s\n", config.Title)
fmt.Printf("Owner: %s (%s, %s), Born: %s\n",
config.Owner.Name, config.Owner.Org, config.Owner.Bio,
config.Owner.DOB)
fmt.Printf("Database: %s %v (Max conn. %d), Enabled? %v\n",
config.DB.Server, config.DB.Ports, config.DB.ConnMax,
config.DB.Enabled)
for serverName, server := range config.Servers {
fmt.Printf("Server: %s (%s, %s)\n", serverName, server.IP, server.DC)
}
fmt.Printf("Client data: %v\n", config.Clients.Data)
fmt.Printf("Client hosts: %v\n", config.Clients.Hosts)
}

@ -1,35 +0,0 @@
# This is a TOML document. Boom.
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
organization = "GitHub"
bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
dob = 1979-05-27T07:32:00Z # First class dates? Why not?
[database]
server = "192.168.1.1"
ports = [ 8001, 8001, 8002 ]
connection_max = 5000
enabled = true
[servers]
# You can indent as you please. Tabs or spaces. TOML don't care.
[servers.alpha]
ip = "10.0.0.1"
dc = "eqdc10"
[servers.beta]
ip = "10.0.0.2"
dc = "eqdc10"
[clients]
data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
# Line breaks are OK when inside arrays
hosts = [
"alpha",
"omega"
]

@ -1,22 +0,0 @@
# Test file for TOML
# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate
# This part you'll really hate
[the]
test_string = "You'll hate me after this - #" # " Annoying, isn't it?
[the.hard]
test_array = [ "] ", " # "] # ] There you go, parse this!
test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ]
# You didn't think it'd as easy as chucking out the last #, did you?
another_test_string = " Same thing, but with a string #"
harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too"
# Things will get harder
[the.hard.bit#]
what? = "You don't think some user won't do that?"
multi_line_array = [
"]",
# ] Oh yes I did
]

@ -1,4 +0,0 @@
# [x] you
# [x.y] don't
# [x.y.z] need these
[x.y.z.w] # for this to work

@ -1,6 +0,0 @@
# DO NOT WANT
[fruit]
type = "apple"
[fruit.type]
apple = "yes"

@ -1,35 +0,0 @@
# This is an INVALID TOML document. Boom.
# Can you spot the error without help?
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
organization = "GitHub"
bio = "GitHub Cofounder & CEO\nLikes tater tots and beer."
dob = 1979-05-27T7:32:00Z # First class dates? Why not?
[database]
server = "192.168.1.1"
ports = [ 8001, 8001, 8002 ]
connection_max = 5000
enabled = true
[servers]
# You can indent as you please. Tabs or spaces. TOML don't care.
[servers.alpha]
ip = "10.0.0.1"
dc = "eqdc10"
[servers.beta]
ip = "10.0.0.2"
dc = "eqdc10"
[clients]
data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it
# Line breaks are OK when inside arrays
hosts = [
"alpha",
"omega"
]

@ -1,5 +0,0 @@
Age = 25
Cats = [ "Cauchy", "Plato" ]
Pi = 3.14
Perfection = [ 6, 28, 496, 8128 ]
DOB = 1987-07-05T05:45:00Z

@ -1 +0,0 @@
some_key_NAME = "wat"

@ -1,14 +0,0 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.

@ -1,13 +0,0 @@
# Implements the TOML test suite interface
This is an implementation of the interface expected by
[toml-test](https://github.com/BurntSushi/toml-test) for my
[toml parser written in Go](https://github.com/BurntSushi/toml).
In particular, it maps TOML data on `stdin` to a JSON format on `stdout`.
Compatible with TOML version
[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
Compatible with `toml-test` version
[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0)

@ -1,90 +0,0 @@
// Command toml-test-decoder satisfies the toml-test interface for testing
// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout.
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path"
"time"
"github.com/BurntSushi/toml"
)
func init() {
log.SetFlags(0)
flag.Usage = usage
flag.Parse()
}
func usage() {
log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0]))
flag.PrintDefaults()
os.Exit(1)
}
func main() {
if flag.NArg() != 0 {
flag.Usage()
}
var tmp interface{}
if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil {
log.Fatalf("Error decoding TOML: %s", err)
}
typedTmp := translate(tmp)
if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil {
log.Fatalf("Error encoding JSON: %s", err)
}
}
func translate(tomlData interface{}) interface{} {
switch orig := tomlData.(type) {
case map[string]interface{}:
typed := make(map[string]interface{}, len(orig))
for k, v := range orig {
typed[k] = translate(v)
}
return typed
case []map[string]interface{}:
typed := make([]map[string]interface{}, len(orig))
for i, v := range orig {
typed[i] = translate(v).(map[string]interface{})
}
return typed
case []interface{}:
typed := make([]interface{}, len(orig))
for i, v := range orig {
typed[i] = translate(v)
}
// We don't really need to tag arrays, but let's be future proof.
// (If TOML ever supports tuples, we'll need this.)
return tag("array", typed)
case time.Time:
return tag("datetime", orig.Format("2006-01-02T15:04:05Z"))
case bool:
return tag("bool", fmt.Sprintf("%v", orig))
case int64:
return tag("integer", fmt.Sprintf("%d", orig))
case float64:
return tag("float", fmt.Sprintf("%v", orig))
case string:
return tag("string", orig)
}
panic(fmt.Sprintf("Unknown type: %T", tomlData))
}
func tag(typeName string, data interface{}) map[string]interface{} {
return map[string]interface{}{
"type": typeName,
"value": data,
}
}

@ -1,14 +0,0 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.

@ -1,13 +0,0 @@
# Implements the TOML test suite interface for TOML encoders
This is an implementation of the interface expected by
[toml-test](https://github.com/BurntSushi/toml-test) for the
[TOML encoder](https://github.com/BurntSushi/toml).
In particular, it maps JSON data on `stdin` to a TOML format on `stdout`.
Compatible with TOML version
[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)
Compatible with `toml-test` version
[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0)

@ -1,131 +0,0 @@
// Command toml-test-encoder satisfies the toml-test interface for testing
// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout.
package main
import (
"encoding/json"
"flag"
"log"
"os"
"path"
"strconv"
"time"
"github.com/BurntSushi/toml"
)
func init() {
log.SetFlags(0)
flag.Usage = usage
flag.Parse()
}
func usage() {
log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0]))
flag.PrintDefaults()
os.Exit(1)
}
func main() {
if flag.NArg() != 0 {
flag.Usage()
}
var tmp interface{}
if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil {
log.Fatalf("Error decoding JSON: %s", err)
}
tomlData := translate(tmp)
if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil {
log.Fatalf("Error encoding TOML: %s", err)
}
}
func translate(typedJson interface{}) interface{} {
switch v := typedJson.(type) {
case map[string]interface{}:
if len(v) == 2 && in("type", v) && in("value", v) {
return untag(v)
}
m := make(map[string]interface{}, len(v))
for k, v2 := range v {
m[k] = translate(v2)
}
return m
case []interface{}:
tabArray := make([]map[string]interface{}, len(v))
for i := range v {
if m, ok := translate(v[i]).(map[string]interface{}); ok {
tabArray[i] = m
} else {
log.Fatalf("JSON arrays may only contain objects. This " +
"corresponds to only tables being allowed in " +
"TOML table arrays.")
}
}
return tabArray
}
log.Fatalf("Unrecognized JSON format '%T'.", typedJson)
panic("unreachable")
}
func untag(typed map[string]interface{}) interface{} {
t := typed["type"].(string)
v := typed["value"]
switch t {
case "string":
return v.(string)
case "integer":
v := v.(string)
n, err := strconv.Atoi(v)
if err != nil {
log.Fatalf("Could not parse '%s' as integer: %s", v, err)
}
return n
case "float":
v := v.(string)
f, err := strconv.ParseFloat(v, 64)
if err != nil {
log.Fatalf("Could not parse '%s' as float64: %s", v, err)
}
return f
case "datetime":
v := v.(string)
t, err := time.Parse("2006-01-02T15:04:05Z", v)
if err != nil {
log.Fatalf("Could not parse '%s' as a datetime: %s", v, err)
}
return t
case "bool":
v := v.(string)
switch v {
case "true":
return true
case "false":
return false
}
log.Fatalf("Could not parse '%s' as a boolean.", v)
case "array":
v := v.([]interface{})
array := make([]interface{}, len(v))
for i := range v {
if m, ok := v[i].(map[string]interface{}); ok {
array[i] = untag(m)
} else {
log.Fatalf("Arrays may only contain other arrays or "+
"primitive values, but found a '%T'.", m)
}
}
return array
}
log.Fatalf("Unrecognized tag type '%s'.", t)
panic("unreachable")
}
func in(key string, m map[string]interface{}) bool {
_, ok := m[key]
return ok
}

@ -1,14 +0,0 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.

@ -1,21 +0,0 @@
# TOML Validator
If Go is installed, it's simple to try it out:
```bash
go get github.com/BurntSushi/toml/cmd/tomlv
tomlv some-toml-file.toml
```
You can see the types of every key in a TOML file with:
```bash
tomlv -types some-toml-file.toml
```
At the moment, only one error message is reported at a time. Error messages
include line numbers. No output means that the files given are valid TOML, or
there is a bug in `tomlv`.
Compatible with TOML version
[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md)

@ -1,61 +0,0 @@
// Command tomlv validates TOML documents and prints each key's type.
package main
import (
"flag"
"fmt"
"log"
"os"
"path"
"strings"
"text/tabwriter"
"github.com/BurntSushi/toml"
)
var (
flagTypes = false
)
func init() {
log.SetFlags(0)
flag.BoolVar(&flagTypes, "types", flagTypes,
"When set, the types of every defined key will be shown.")
flag.Usage = usage
flag.Parse()
}
func usage() {
log.Printf("Usage: %s toml-file [ toml-file ... ]\n",
path.Base(os.Args[0]))
flag.PrintDefaults()
os.Exit(1)
}
func main() {
if flag.NArg() < 1 {
flag.Usage()
}
for _, f := range flag.Args() {
var tmp interface{}
md, err := toml.DecodeFile(f, &tmp)
if err != nil {
log.Fatalf("Error in '%s': %s", f, err)
}
if flagTypes {
printTypes(md)
}
}
}
func printTypes(md toml.MetaData) {
tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
for _, key := range md.Keys() {
fmt.Fprintf(tabw, "%s%s\t%s\n",
strings.Repeat(" ", len(key)-1), key, md.Type(key...))
}
tabw.Flush()
}

File diff suppressed because it is too large Load Diff

@ -1,615 +0,0 @@
package toml
import (
"bytes"
"fmt"
"log"
"net"
"testing"
"time"
)
func TestEncodeRoundTrip(t *testing.T) {
type Config struct {
Age int
Cats []string
Pi float64
Perfection []int
DOB time.Time
Ipaddress net.IP
}
var inputs = Config{
13,
[]string{"one", "two", "three"},
3.145,
[]int{11, 2, 3, 4},
time.Now(),
net.ParseIP("192.168.59.254"),
}
var firstBuffer bytes.Buffer
e := NewEncoder(&firstBuffer)
err := e.Encode(inputs)
if err != nil {
t.Fatal(err)
}
var outputs Config
if _, err := Decode(firstBuffer.String(), &outputs); err != nil {
t.Logf("Could not decode:\n-----\n%s\n-----\n",
firstBuffer.String())
t.Fatal(err)
}
// could test each value individually, but I'm lazy
var secondBuffer bytes.Buffer
e2 := NewEncoder(&secondBuffer)
err = e2.Encode(outputs)
if err != nil {
t.Fatal(err)
}
if firstBuffer.String() != secondBuffer.String() {
t.Error(
firstBuffer.String(),
"\n\n is not identical to\n\n",
secondBuffer.String())
}
}
// XXX(burntsushi)
// I think these tests probably should be removed. They are good, but they
// ought to be obsolete by toml-test.
func TestEncode(t *testing.T) {
type Embedded struct {
Int int `toml:"_int"`
}
type NonStruct int
date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600))
dateStr := "2014-05-11T19:30:40Z"
tests := map[string]struct {
input interface{}
wantOutput string
wantError error
}{
"bool field": {
input: struct {
BoolTrue bool
BoolFalse bool
}{true, false},
wantOutput: "BoolTrue = true\nBoolFalse = false\n",
},
"int fields": {
input: struct {
Int int
Int8 int8
Int16 int16
Int32 int32
Int64 int64
}{1, 2, 3, 4, 5},
wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n",
},
"uint fields": {
input: struct {
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
}{1, 2, 3, 4, 5},
wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" +
"\nUint64 = 5\n",
},
"float fields": {
input: struct {
Float32 float32
Float64 float64
}{1.5, 2.5},
wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n",
},
"string field": {
input: struct{ String string }{"foo"},
wantOutput: "String = \"foo\"\n",
},
"string field and unexported field": {
input: struct {
String string
unexported int
}{"foo", 0},
wantOutput: "String = \"foo\"\n",
},
"datetime field in UTC": {
input: struct{ Date time.Time }{date},
wantOutput: fmt.Sprintf("Date = %s\n", dateStr),
},
"datetime field as primitive": {
// Using a map here to fail if isStructOrMap() returns true for
// time.Time.
input: map[string]interface{}{
"Date": date,
"Int": 1,
},
wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr),
},
"array fields": {
input: struct {
IntArray0 [0]int
IntArray3 [3]int
}{[0]int{}, [3]int{1, 2, 3}},
wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n",
},
"slice fields": {
input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{
nil, []int{}, []int{1, 2, 3},
},
wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n",
},
"datetime slices": {
input: struct{ DatetimeSlice []time.Time }{
[]time.Time{date, date},
},
wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n",
dateStr, dateStr),
},
"nested arrays and slices": {
input: struct {
SliceOfArrays [][2]int
ArrayOfSlices [2][]int
SliceOfArraysOfSlices [][2][]int
ArrayOfSlicesOfArrays [2][][2]int
SliceOfMixedArrays [][2]interface{}
ArrayOfMixedSlices [2][]interface{}
}{
[][2]int{{1, 2}, {3, 4}},
[2][]int{{1, 2}, {3, 4}},
[][2][]int{
{
{1, 2}, {3, 4},
},
{
{5, 6}, {7, 8},
},
},
[2][][2]int{
{
{1, 2}, {3, 4},
},
{
{5, 6}, {7, 8},
},
},
[][2]interface{}{
{1, 2}, {"a", "b"},
},
[2][]interface{}{
{1, 2}, {"a", "b"},
},
},
wantOutput: `SliceOfArrays = [[1, 2], [3, 4]]
ArrayOfSlices = [[1, 2], [3, 4]]
SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
SliceOfMixedArrays = [[1, 2], ["a", "b"]]
ArrayOfMixedSlices = [[1, 2], ["a", "b"]]
`,
},
"empty slice": {
input: struct{ Empty []interface{} }{[]interface{}{}},
wantOutput: "Empty = []\n",
},
"(error) slice with element type mismatch (string and integer)": {
input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}},
wantError: errArrayMixedElementTypes,
},
"(error) slice with element type mismatch (integer and float)": {
input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}},
wantError: errArrayMixedElementTypes,
},
"slice with elems of differing Go types, same TOML types": {
input: struct {
MixedInts []interface{}
MixedFloats []interface{}
}{
[]interface{}{
int(1), int8(2), int16(3), int32(4), int64(5),
uint(1), uint8(2), uint16(3), uint32(4), uint64(5),
},
[]interface{}{float32(1.5), float64(2.5)},
},
wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" +
"MixedFloats = [1.5, 2.5]\n",
},
"(error) slice w/ element type mismatch (one is nested array)": {
input: struct{ Mixed []interface{} }{
[]interface{}{1, []interface{}{2}},
},
wantError: errArrayMixedElementTypes,
},
"(error) slice with 1 nil element": {
input: struct{ NilElement1 []interface{} }{[]interface{}{nil}},
wantError: errArrayNilElement,
},
"(error) slice with 1 nil element (and other non-nil elements)": {
input: struct{ NilElement []interface{} }{
[]interface{}{1, nil},
},
wantError: errArrayNilElement,
},
"simple map": {
input: map[string]int{"a": 1, "b": 2},
wantOutput: "a = 1\nb = 2\n",
},
"map with interface{} value type": {
input: map[string]interface{}{"a": 1, "b": "c"},
wantOutput: "a = 1\nb = \"c\"\n",
},
"map with interface{} value type, some of which are structs": {
input: map[string]interface{}{
"a": struct{ Int int }{2},
"b": 1,
},
wantOutput: "b = 1\n\n[a]\n Int = 2\n",
},
"nested map": {
input: map[string]map[string]int{
"a": {"b": 1},
"c": {"d": 2},
},
wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n",
},
"nested struct": {
input: struct{ Struct struct{ Int int } }{
struct{ Int int }{1},
},
wantOutput: "[Struct]\n Int = 1\n",
},
"nested struct and non-struct field": {
input: struct {
Struct struct{ Int int }
Bool bool
}{struct{ Int int }{1}, true},
wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n",
},
"2 nested structs": {
input: struct{ Struct1, Struct2 struct{ Int int } }{
struct{ Int int }{1}, struct{ Int int }{2},
},
wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n",
},
"deeply nested structs": {
input: struct {
Struct1, Struct2 struct{ Struct3 *struct{ Int int } }
}{
struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}},
struct{ Struct3 *struct{ Int int } }{nil},
},
wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" +
"\n\n[Struct2]\n",
},
"nested struct with nil struct elem": {
input: struct {
Struct struct{ Inner *struct{ Int int } }
}{
struct{ Inner *struct{ Int int } }{nil},
},
wantOutput: "[Struct]\n",
},
"nested struct with no fields": {
input: struct {
Struct struct{ Inner struct{} }
}{
struct{ Inner struct{} }{struct{}{}},
},
wantOutput: "[Struct]\n [Struct.Inner]\n",
},
"struct with tags": {
input: struct {
Struct struct {
Int int `toml:"_int"`
} `toml:"_struct"`
Bool bool `toml:"_bool"`
}{
struct {
Int int `toml:"_int"`
}{1}, true,
},
wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n",
},
"embedded struct": {
input: struct{ Embedded }{Embedded{1}},
wantOutput: "_int = 1\n",
},
"embedded *struct": {
input: struct{ *Embedded }{&Embedded{1}},
wantOutput: "_int = 1\n",
},
"nested embedded struct": {
input: struct {
Struct struct{ Embedded } `toml:"_struct"`
}{struct{ Embedded }{Embedded{1}}},
wantOutput: "[_struct]\n _int = 1\n",
},
"nested embedded *struct": {
input: struct {
Struct struct{ *Embedded } `toml:"_struct"`
}{struct{ *Embedded }{&Embedded{1}}},
wantOutput: "[_struct]\n _int = 1\n",
},
"embedded non-struct": {
input: struct{ NonStruct }{5},
wantOutput: "NonStruct = 5\n",
},
"array of tables": {
input: struct {
Structs []*struct{ Int int } `toml:"struct"`
}{
[]*struct{ Int int }{{1}, {3}},
},
wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n",
},
"array of tables order": {
input: map[string]interface{}{
"map": map[string]interface{}{
"zero": 5,
"arr": []map[string]int{
{
"friend": 5,
},
},
},
},
wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n",
},
"(error) top-level slice": {
input: []struct{ Int int }{{1}, {2}, {3}},
wantError: errNoKey,
},
"(error) slice of slice": {
input: struct {
Slices [][]struct{ Int int }
}{
[][]struct{ Int int }{{{1}}, {{2}}, {{3}}},
},
wantError: errArrayNoTable,
},
"(error) map no string key": {
input: map[int]string{1: ""},
wantError: errNonString,
},
"(error) empty key name": {
input: map[string]int{"": 1},
wantError: errAnything,
},
"(error) empty map name": {
input: map[string]interface{}{
"": map[string]int{"v": 1},
},
wantError: errAnything,
},
}
for label, test := range tests {
encodeExpected(t, label, test.input, test.wantOutput, test.wantError)
}
}
func TestEncodeNestedTableArrays(t *testing.T) {
type song struct {
Name string `toml:"name"`
}
type album struct {
Name string `toml:"name"`
Songs []song `toml:"songs"`
}
type springsteen struct {
Albums []album `toml:"albums"`
}
value := springsteen{
[]album{
{"Born to Run",
[]song{{"Jungleland"}, {"Meeting Across the River"}}},
{"Born in the USA",
[]song{{"Glory Days"}, {"Dancing in the Dark"}}},
},
}
expected := `[[albums]]
name = "Born to Run"
[[albums.songs]]
name = "Jungleland"
[[albums.songs]]
name = "Meeting Across the River"
[[albums]]
name = "Born in the USA"
[[albums.songs]]
name = "Glory Days"
[[albums.songs]]
name = "Dancing in the Dark"
`
encodeExpected(t, "nested table arrays", value, expected, nil)
}
func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) {
type Alpha struct {
V int
}
type Beta struct {
V int
}
type Conf struct {
V int
A Alpha
B []Beta
}
val := Conf{
V: 1,
A: Alpha{2},
B: []Beta{{3}},
}
expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n"
encodeExpected(t, "array hash with normal hash order", val, expected, nil)
}
func TestEncodeWithOmitEmpty(t *testing.T) {
type simple struct {
Bool bool `toml:"bool,omitempty"`
String string `toml:"string,omitempty"`
Array [0]byte `toml:"array,omitempty"`
Slice []int `toml:"slice,omitempty"`
Map map[string]string `toml:"map,omitempty"`
}
var v simple
encodeExpected(t, "fields with omitempty are omitted when empty", v, "", nil)
v = simple{
Bool: true,
String: " ",
Slice: []int{2, 3, 4},
Map: map[string]string{"foo": "bar"},
}
expected := `bool = true
string = " "
slice = [2, 3, 4]
[map]
foo = "bar"
`
encodeExpected(t, "fields with omitempty are not omitted when non-empty",
v, expected, nil)
}
func TestEncodeWithOmitZero(t *testing.T) {
type simple struct {
Number int `toml:"number,omitzero"`
Real float64 `toml:"real,omitzero"`
Unsigned uint `toml:"unsigned,omitzero"`
}
value := simple{0, 0.0, uint(0)}
expected := ""
encodeExpected(t, "simple with omitzero, all zero", value, expected, nil)
value.Number = 10
value.Real = 20
value.Unsigned = 5
expected = `number = 10
real = 20.0
unsigned = 5
`
encodeExpected(t, "simple with omitzero, non-zero", value, expected, nil)
}
func TestEncodeOmitemptyWithEmptyName(t *testing.T) {
type simple struct {
S []int `toml:",omitempty"`
}
v := simple{[]int{1, 2, 3}}
expected := "S = [1, 2, 3]\n"
encodeExpected(t, "simple with omitempty, no name, non-empty field",
v, expected, nil)
}
func TestEncodeAnonymousStruct(t *testing.T) {
type Inner struct{ N int }
type Outer0 struct{ Inner }
type Outer1 struct {
Inner `toml:"inner"`
}
v0 := Outer0{Inner{3}}
expected := "N = 3\n"
encodeExpected(t, "embedded anonymous untagged struct", v0, expected, nil)
v1 := Outer1{Inner{3}}
expected = "[inner]\n N = 3\n"
encodeExpected(t, "embedded anonymous tagged struct", v1, expected, nil)
}
func TestEncodeAnonymousStructPointerField(t *testing.T) {
type Inner struct{ N int }
type Outer0 struct{ *Inner }
type Outer1 struct {
*Inner `toml:"inner"`
}
v0 := Outer0{}
expected := ""
encodeExpected(t, "nil anonymous untagged struct pointer field", v0, expected, nil)
v0 = Outer0{&Inner{3}}
expected = "N = 3\n"
encodeExpected(t, "non-nil anonymous untagged struct pointer field", v0, expected, nil)
v1 := Outer1{}
expected = ""
encodeExpected(t, "nil anonymous tagged struct pointer field", v1, expected, nil)
v1 = Outer1{&Inner{3}}
expected = "[inner]\n N = 3\n"
encodeExpected(t, "non-nil anonymous tagged struct pointer field", v1, expected, nil)
}
func TestEncodeIgnoredFields(t *testing.T) {
type simple struct {
Number int `toml:"-"`
}
value := simple{}
expected := ""
encodeExpected(t, "ignored field", value, expected, nil)
}
func encodeExpected(
t *testing.T, label string, val interface{}, wantStr string, wantErr error,
) {
var buf bytes.Buffer
enc := NewEncoder(&buf)
err := enc.Encode(val)
if err != wantErr {
if wantErr != nil {
if wantErr == errAnything && err != nil {
return
}
t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err)
} else {
t.Errorf("%s: Encode failed: %s", label, err)
}
}
if err != nil {
return
}
if got := buf.String(); wantStr != got {
t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n",
label, wantStr, got)
}
}
func ExampleEncoder_Encode() {
date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC")
var config = map[string]interface{}{
"date": date,
"counts": []int{1, 1, 2, 3, 5, 8},
"hash": map[string]string{
"key1": "val1",
"key2": "val2",
},
}
buf := new(bytes.Buffer)
if err := NewEncoder(buf).Encode(config); err != nil {
log.Fatal(err)
}
fmt.Println(buf.String())
// Output:
// counts = [1, 1, 2, 3, 5, 8]
// date = 2010-03-14T18:00:00Z
//
// [hash]
// key1 = "val1"
// key2 = "val2"
}

@ -775,7 +775,7 @@ func lexDatetime(lx *lexer) stateFn {
return lexDatetime
}
switch r {
case '-', 'T', ':', '.', 'Z':
case '-', 'T', ':', '.', 'Z', '+':
return lexDatetime
}

@ -11,22 +11,24 @@ Exported variables and functions implemented till now :
```go
var Headers map[string]string // Set headers as a map of key-value pairs, an alternative to calling Header() individually
var Cookies map[string]string // Set cookies as a map of key-value pairs, an alternative to calling Cookie() individually
func Get(string) (string,error) // Takes the url as an argument, returns HTML string
func GetWithClient(string, *http.Client) // Takes the url and a custom HTTP client as arguments, returns HTML string
func Header(string, string) // Takes key,value pair to set as headers for the HTTP request made in Get()
func Cookie(string, string) // Takes key, value pair to set as cookies to be sent with the HTTP request in Get()
func HTMLParse(string) Root // Takes the HTML string as an argument, returns a pointer to the DOM constructed
func Find([]string) Root // Element tag,(attribute key-value pair) as argument, pointer to first occurence returned
func FindAll([]string) []Root // Same as Find(), but pointers to all occurrences returned
func FindStrict([]string) Root // Element tag,(attribute key-value pair) as argument, pointer to first occurence returned with exact matching values
func FindAllStrict([]string) []Root // Same as FindStrict(), but pointers to all occurrences returned
func FindNextSibling() Root // Pointer to the next sibling of the Element in the DOM returned
func FindNextElementSibling() Root // Pointer to the next element sibling of the Element in the DOM returned
func FindPrevSibling() Root // Pointer to the previous sibling of the Element in the DOM returned
func FindPrevElementSibling() Root // Pointer to the previous element sibling of the Element in the DOM returned
func Attrs() map[string]string // Map returned with all the attributes of the Element as lookup to their respective values
func Text() string // Full text inside a non-nested tag returned
func SetDebug(bool) // Sets the debug mode to true or false; false by default
func Get(string) (string,error){} // Takes the url as an argument, returns HTML string
func GetWithClient(string, *http.Client){} // Takes the url and a custom HTTP client as arguments, returns HTML string
func Header(string, string){} // Takes key,value pair to set as headers for the HTTP request made in Get()
func Cookie(string, string){} // Takes key, value pair to set as cookies to be sent with the HTTP request in Get()
func HTMLParse(string) Root {} // Takes the HTML string as an argument, returns a pointer to the DOM constructed
func Find([]string) Root {} // Element tag,(attribute key-value pair) as argument, pointer to first occurence returned
func FindAll([]string) []Root {} // Same as Find(), but pointers to all occurrences returned
func FindStrict([]string) Root {} // Element tag,(attribute key-value pair) as argument, pointer to first occurence returned with exact matching values
func FindAllStrict([]string) []Root {} // Same as FindStrict(), but pointers to all occurrences returned
func FindNextSibling() Root {} // Pointer to the next sibling of the Element in the DOM returned
func FindNextElementSibling() Root {} // Pointer to the next element sibling of the Element in the DOM returned
func FindPrevSibling() Root {} // Pointer to the previous sibling of the Element in the DOM returned
func FindPrevElementSibling() Root {} // Pointer to the previous element sibling of the Element in the DOM returned
func Children() []Root {} // Find all direct children of this DOM element
func Attrs() map[string]string {} // Map returned with all the attributes of the Element as lookup to their respective values
func Text() string {} // Full text inside a non-nested tag returned, first half returned in a non-nested one
func FullText() string {} // Full text inside a nested/non-nested tag returned
func SetDebug(bool) {} // Sets the debug mode to true or false; false by default
```
`Root` is a struct, containing three fields :
@ -67,4 +69,4 @@ func main() {
```
## Contributions
This package was developed in my free time. However, contributions from everybody in the community are welcome, to make it a better web scraper. If you think there should be a particular feature or function included in the package, feel free to open up a new issue or pull request.
This package was developed in my free time. However, contributions from everybody in the community are welcome, to make it a better web scraper. If you think there should be a particular feature or function included in the package, feel free to open up a new issue or pull request.

@ -1,38 +0,0 @@
package main
import (
"bufio"
"fmt"
"log"
"os"
"strings"
"github.com/anaskhan96/soup"
)
func main() {
fmt.Printf("Enter the name of the city : ")
city, _ := bufio.NewReader(os.Stdin).ReadString('\n')
city = city[:len(city)-1]
cityInURL := strings.Join(strings.Split(city, " "), "+")
url := "https://www.bing.com/search?q=weather+" + cityInURL
resp, err := soup.Get(url)
if err != nil {
log.Fatal(err)
}
doc := soup.HTMLParse(resp)
grid := doc.Find("div", "class", "b_antiTopBleed b_antiSideBleed b_antiBottomBleed")
heading := grid.Find("div", "class", "wtr_titleCtrn").Find("div").Text()
conditions := grid.Find("div", "class", "wtr_condition")
primaryCondition := conditions.Find("div")
secondaryCondition := primaryCondition.FindNextElementSibling()
temp := primaryCondition.Find("div", "class", "wtr_condiTemp").Find("div").Text()
others := primaryCondition.Find("div", "class", "wtr_condiAttribs").FindAll("div")
caption := secondaryCondition.Find("div").Text()
fmt.Println("City Name : " + heading)
fmt.Println("Temperature : " + temp + "˚C")
for _, i := range others {
fmt.Println(i.Text())
}
fmt.Println(caption)
}

@ -1,29 +0,0 @@
package main
import (
"fmt"
"github.com/anaskhan96/soup"
)
func main() {
fmt.Println("Enter the xkcd comic number :")
var num int
fmt.Scanf("%d", &num)
url := fmt.Sprintf("https://xkcd.com/%d", num)
resp, _ := soup.Get(url)
doc := soup.HTMLParse(resp)
title := doc.Find("div", "id", "ctitle").Text()
fmt.Println("Title of the comic :", title)
comicImg := doc.Find("div", "id", "comic").Find("img")
fmt.Println("Source of the image :", comicImg.Attrs()["src"])
fmt.Println("Underlying text of the image :", comicImg.Attrs()["title"])
}
/* --- Console I/O ---
Enter the xkcd comic number :
353
Title of the comic : Python
Source of the image : //imgs.xkcd.com/comics/python.png
Underlying text of the image : I wrote 20 short programs in Python yesterday. It was wonderful. Perl, I'm leaving you.
*/

@ -5,6 +5,7 @@ keeping it as similar as possible to BeautifulSoup
package soup
import (
"bytes"
"errors"
"io/ioutil"
"net/http"
@ -236,6 +237,17 @@ func (r Root) FindPrevElementSibling() Root {
return p.FindPrevElementSibling()
}
// Children retuns all direct children of this DOME element.
func (r Root) Children() []Root {
child := r.Pointer.FirstChild
var children []Root
for child != nil {
children = append(children, Root{child, child.Data, nil})
child = child.NextSibling
}
return children
}
// Attrs returns a map containing all attributes
func (r Root) Attrs() map[string]string {
if r.Pointer.Type != html.ElementNode {
@ -254,7 +266,7 @@ func (r Root) Attrs() map[string]string {
func (r Root) Text() string {
k := r.Pointer.FirstChild
checkNode:
if k.Type != html.TextNode {
if k != nil && k.Type != html.TextNode {
k = k.NextSibling
if k == nil {
if debug {
@ -281,6 +293,28 @@ checkNode:
return ""
}
// FullText returns the string inside even a nested element
func (r Root) FullText() string {
var buf bytes.Buffer
var f func(*html.Node)
f = func(n *html.Node) {
if n.Type == html.TextNode {
buf.WriteString(n.Data)
}
if n.Type == html.ElementNode {
f(n.FirstChild)
}
if n.NextSibling != nil {
f(n.NextSibling)
}
}
f(r.Pointer.FirstChild)
return buf.String()
}
// Using depth first search to find the first occurrence and return
func findOnce(n *html.Node, args []string, uni bool, strict bool) (*html.Node, bool) {
if uni == true {

@ -1,169 +0,0 @@
package soup
import (
"reflect"
"strconv"
"strings"
"testing"
)
const testHTML = `
<html>
<head>
<title>Sample "Hello, World" Application</title>
</head>
<body bgcolor=white>
<table border="0" cellpadding="10">
<tr>
<td>
<img src="images/springsource.png">
</td>
<td>
<h1>Sample "Hello, World" Application</h1>
</td>
</tr>
</table>
<div id="0">
<div id="1">Just two divs peacing out</div>
</div>
check
<div id="2">One more</div>
<p>This is the home page for the HelloWorld Web application. </p>
<p>To prove that they work, you can execute either of the following links:
<ul>
<li>To a <a href="hello.jsp">JSP page</a></li>
<li>To a <a href="hello">servlet</a></li>
</ul>
</p>
<div id="3">
<div id="4">Last one</div>
</div>
</body>
</html>
`
const multipleClassesHTML = `
<html>
<head>
<title>Sample Application</title>
</head>
<body>
<div class="first second">Multiple classes</div>
<div class="first">Single class</div>
<div class="second first third">Multiple classes inorder</div>
<div>
<div class="first">Inner single class</div>
<div class="first second">Inner multiple classes</div>
<div class="second first">Inner multiple classes inorder</div>
</div>
</body>
</html>
`
var doc = HTMLParse(testHTML)
var multipleClasses = HTMLParse(multipleClassesHTML)
func TestFind(t *testing.T) {
// Find() and Attrs()
actual := doc.Find("img").Attrs()["src"]
if !reflect.DeepEqual(actual, "images/springsource.png") {
t.Error("Instead of `images/springsource.png`, got", actual)
}
// Find(...) and Text()
actual = doc.Find("a", "href", "hello").Text()
if !reflect.DeepEqual(actual, "servlet") {
t.Error("Instead of `servlet`, got", actual)
}
// Nested Find()
actual = doc.Find("div").Find("div").Text()
if !reflect.DeepEqual(actual, "Just two divs peacing out") {
t.Error("Instead of `Just two divs peacing out`, got", actual)
}
}
func TestFindNextPrevElement(t *testing.T) {
// FindNextSibling() and NodeValue field
actual := doc.Find("div", "id", "0").FindNextSibling().NodeValue
if !reflect.DeepEqual(strings.TrimSpace(actual), "check") {
t.Error("Instead of `check`, got", actual)
}
// FindPrevSibling() and NodeValue field
actual = doc.Find("div", "id", "2").FindPrevSibling().NodeValue
if !reflect.DeepEqual(strings.TrimSpace(actual), "check") {
t.Error("Instead of `check`, got", actual)
}
// FindNextElementSibling() and NodeValue field
actual = doc.Find("table").FindNextElementSibling().NodeValue
if !reflect.DeepEqual(actual, "div") {
t.Error("Instead of `div`, got", actual)
}
// FindPrevElementSibling() and NodeValue field
actual = doc.Find("p").FindPrevElementSibling().NodeValue
if !reflect.DeepEqual(actual, "div") {
t.Error("Instead of `div`, got", actual)
}
}
func TestFindAll(t *testing.T) {
// FindAll() and Attrs()
allDivs := doc.FindAll("div")
for i := 0; i < len(allDivs); i++ {
id := allDivs[i].Attrs()["id"]
actual, _ := strconv.Atoi(id)
if !reflect.DeepEqual(actual, i) {
t.Error("Instead of", i, "got", actual)
}
}
}
func TestFindAllBySingleClass(t *testing.T) {
actual := multipleClasses.FindAll("div", "class", "first")
if len(actual) != 6 {
t.Errorf("Expected 6 elements to be returned. Actual: %d", len(actual))
}
actual = multipleClasses.FindAll("div", "class", "third")
if len(actual) != 1 {
t.Errorf("Expected 1 element to be returned. Actual: %d", len(actual))
}
}
func TestFindBySingleClass(t *testing.T) {
actual := multipleClasses.Find("div", "class", "first")
if actual.Text() != "Multiple classes" {
t.Errorf("Wrong element returned: %s", actual.Text())
}
actual = multipleClasses.Find("div", "class", "third")
if actual.Text() != "Multiple classes inorder" {
t.Errorf("Wrong element returned: %s", actual.Text())
}
}
func TestFindAllStrict(t *testing.T) {
actual := multipleClasses.FindAllStrict("div", "class", "first second")
if len(actual) != 2 {
t.Errorf("Expected 2 elements to be returned. Actual: %d", len(actual))
}
actual = multipleClasses.FindAllStrict("div", "class", "first third second")
if len(actual) != 0 {
t.Errorf("0 elements should be returned")
}
actual = multipleClasses.FindAllStrict("div", "class", "second first third")
if len(actual) != 1 {
t.Errorf("Single item should be returned")
}
}
func TestFindStrict(t *testing.T) {
actual := multipleClasses.FindStrict("div", "class", "first")
if actual.Text() != "Single class" {
t.Errorf("Wrong element returned: %s", actual.Text())
}
actual = multipleClasses.FindStrict("div", "class", "third")
if actual.Error == nil {
t.Errorf("Element with class \"third\" should not be returned in strict mode")
}
}

@ -24,6 +24,9 @@ var (
// os.Stdout is used.
Output = colorable.NewColorableStdout()
// Error defines a color supporting writer for os.Stderr.
Error = colorable.NewColorableStderr()
// colorsCache is used to reduce the count of created Color objects and
// allows to reuse already created objects with required Attribute.
colorsCache = make(map[Attribute]*Color)

@ -1,342 +0,0 @@
package color
import (
"bytes"
"fmt"
"os"
"testing"
"github.com/mattn/go-colorable"
)
// Testing colors is kinda different. First we test for given colors and their
// escaped formatted results. Next we create some visual tests to be tested.
// Each visual test includes the color name to be compared.
func TestColor(t *testing.T) {
rb := new(bytes.Buffer)
Output = rb
NoColor = false
testColors := []struct {
text string
code Attribute
}{
{text: "black", code: FgBlack},
{text: "red", code: FgRed},
{text: "green", code: FgGreen},
{text: "yellow", code: FgYellow},
{text: "blue", code: FgBlue},
{text: "magent", code: FgMagenta},
{text: "cyan", code: FgCyan},
{text: "white", code: FgWhite},
{text: "hblack", code: FgHiBlack},
{text: "hred", code: FgHiRed},
{text: "hgreen", code: FgHiGreen},
{text: "hyellow", code: FgHiYellow},
{text: "hblue", code: FgHiBlue},
{text: "hmagent", code: FgHiMagenta},
{text: "hcyan", code: FgHiCyan},
{text: "hwhite", code: FgHiWhite},
}
for _, c := range testColors {
New(c.code).Print(c.text)
line, _ := rb.ReadString('\n')
scannedLine := fmt.Sprintf("%q", line)
colored := fmt.Sprintf("\x1b[%dm%s\x1b[0m", c.code, c.text)
escapedForm := fmt.Sprintf("%q", colored)
fmt.Printf("%s\t: %s\n", c.text, line)
if scannedLine != escapedForm {
t.Errorf("Expecting %s, got '%s'\n", escapedForm, scannedLine)
}
}
for _, c := range testColors {
line := New(c.code).Sprintf("%s", c.text)
scannedLine := fmt.Sprintf("%q", line)
colored := fmt.Sprintf("\x1b[%dm%s\x1b[0m", c.code, c.text)
escapedForm := fmt.Sprintf("%q", colored)
fmt.Printf("%s\t: %s\n", c.text, line)
if scannedLine != escapedForm {
t.Errorf("Expecting %s, got '%s'\n", escapedForm, scannedLine)
}
}
}
func TestColorEquals(t *testing.T) {
fgblack1 := New(FgBlack)
fgblack2 := New(FgBlack)
bgblack := New(BgBlack)
fgbgblack := New(FgBlack, BgBlack)
fgblackbgred := New(FgBlack, BgRed)
fgred := New(FgRed)
bgred := New(BgRed)
if !fgblack1.Equals(fgblack2) {
t.Error("Two black colors are not equal")
}
if fgblack1.Equals(bgblack) {
t.Error("Fg and bg black colors are equal")
}
if fgblack1.Equals(fgbgblack) {
t.Error("Fg black equals fg/bg black color")
}
if fgblack1.Equals(fgred) {
t.Error("Fg black equals Fg red")
}
if fgblack1.Equals(bgred) {
t.Error("Fg black equals Bg red")
}
if fgblack1.Equals(fgblackbgred) {
t.Error("Fg black equals fg black bg red")
}
}
func TestNoColor(t *testing.T) {
rb := new(bytes.Buffer)
Output = rb
testColors := []struct {
text string
code Attribute
}{
{text: "black", code: FgBlack},
{text: "red", code: FgRed},
{text: "green", code: FgGreen},
{text: "yellow", code: FgYellow},
{text: "blue", code: FgBlue},
{text: "magent", code: FgMagenta},
{text: "cyan", code: FgCyan},
{text: "white", code: FgWhite},
{text: "hblack", code: FgHiBlack},
{text: "hred", code: FgHiRed},
{text: "hgreen", code: FgHiGreen},
{text: "hyellow", code: FgHiYellow},
{text: "hblue", code: FgHiBlue},
{text: "hmagent", code: FgHiMagenta},
{text: "hcyan", code: FgHiCyan},
{text: "hwhite", code: FgHiWhite},
}
for _, c := range testColors {
p := New(c.code)
p.DisableColor()
p.Print(c.text)
line, _ := rb.ReadString('\n')
if line != c.text {
t.Errorf("Expecting %s, got '%s'\n", c.text, line)
}
}
// global check
NoColor = true
defer func() {
NoColor = false
}()
for _, c := range testColors {
p := New(c.code)
p.Print(c.text)
line, _ := rb.ReadString('\n')
if line != c.text {
t.Errorf("Expecting %s, got '%s'\n", c.text, line)
}
}
}
func TestColorVisual(t *testing.T) {
// First Visual Test
Output = colorable.NewColorableStdout()
New(FgRed).Printf("red\t")
New(BgRed).Print(" ")
New(FgRed, Bold).Println(" red")
New(FgGreen).Printf("green\t")
New(BgGreen).Print(" ")
New(FgGreen, Bold).Println(" green")
New(FgYellow).Printf("yellow\t")
New(BgYellow).Print(" ")
New(FgYellow, Bold).Println(" yellow")
New(FgBlue).Printf("blue\t")
New(BgBlue).Print(" ")
New(FgBlue, Bold).Println(" blue")
New(FgMagenta).Printf("magenta\t")
New(BgMagenta).Print(" ")
New(FgMagenta, Bold).Println(" magenta")
New(FgCyan).Printf("cyan\t")
New(BgCyan).Print(" ")
New(FgCyan, Bold).Println(" cyan")
New(FgWhite).Printf("white\t")
New(BgWhite).Print(" ")
New(FgWhite, Bold).Println(" white")
fmt.Println("")
// Second Visual test
Black("black")
Red("red")
Green("green")
Yellow("yellow")
Blue("blue")
Magenta("magenta")
Cyan("cyan")
White("white")
HiBlack("hblack")
HiRed("hred")
HiGreen("hgreen")
HiYellow("hyellow")
HiBlue("hblue")
HiMagenta("hmagenta")
HiCyan("hcyan")
HiWhite("hwhite")
// Third visual test
fmt.Println()
Set(FgBlue)
fmt.Println("is this blue?")
Unset()
Set(FgMagenta)
fmt.Println("and this magenta?")
Unset()
// Fourth Visual test
fmt.Println()
blue := New(FgBlue).PrintlnFunc()
blue("blue text with custom print func")
red := New(FgRed).PrintfFunc()
red("red text with a printf func: %d\n", 123)
put := New(FgYellow).SprintFunc()
warn := New(FgRed).SprintFunc()
fmt.Fprintf(Output, "this is a %s and this is %s.\n", put("warning"), warn("error"))
info := New(FgWhite, BgGreen).SprintFunc()
fmt.Fprintf(Output, "this %s rocks!\n", info("package"))
notice := New(FgBlue).FprintFunc()
notice(os.Stderr, "just a blue notice to stderr")
// Fifth Visual Test
fmt.Println()
fmt.Fprintln(Output, BlackString("black"))
fmt.Fprintln(Output, RedString("red"))
fmt.Fprintln(Output, GreenString("green"))
fmt.Fprintln(Output, YellowString("yellow"))
fmt.Fprintln(Output, BlueString("blue"))
fmt.Fprintln(Output, MagentaString("magenta"))
fmt.Fprintln(Output, CyanString("cyan"))
fmt.Fprintln(Output, WhiteString("white"))
fmt.Fprintln(Output, HiBlackString("hblack"))
fmt.Fprintln(Output, HiRedString("hred"))
fmt.Fprintln(Output, HiGreenString("hgreen"))
fmt.Fprintln(Output, HiYellowString("hyellow"))
fmt.Fprintln(Output, HiBlueString("hblue"))
fmt.Fprintln(Output, HiMagentaString("hmagenta"))
fmt.Fprintln(Output, HiCyanString("hcyan"))
fmt.Fprintln(Output, HiWhiteString("hwhite"))
}
func TestNoFormat(t *testing.T) {
fmt.Printf("%s %%s = ", BlackString("Black"))
Black("%s")
fmt.Printf("%s %%s = ", RedString("Red"))
Red("%s")
fmt.Printf("%s %%s = ", GreenString("Green"))
Green("%s")
fmt.Printf("%s %%s = ", YellowString("Yellow"))
Yellow("%s")
fmt.Printf("%s %%s = ", BlueString("Blue"))
Blue("%s")
fmt.Printf("%s %%s = ", MagentaString("Magenta"))
Magenta("%s")
fmt.Printf("%s %%s = ", CyanString("Cyan"))
Cyan("%s")
fmt.Printf("%s %%s = ", WhiteString("White"))
White("%s")
fmt.Printf("%s %%s = ", HiBlackString("HiBlack"))
HiBlack("%s")
fmt.Printf("%s %%s = ", HiRedString("HiRed"))
HiRed("%s")
fmt.Printf("%s %%s = ", HiGreenString("HiGreen"))
HiGreen("%s")
fmt.Printf("%s %%s = ", HiYellowString("HiYellow"))
HiYellow("%s")
fmt.Printf("%s %%s = ", HiBlueString("HiBlue"))
HiBlue("%s")
fmt.Printf("%s %%s = ", HiMagentaString("HiMagenta"))
HiMagenta("%s")
fmt.Printf("%s %%s = ", HiCyanString("HiCyan"))
HiCyan("%s")
fmt.Printf("%s %%s = ", HiWhiteString("HiWhite"))
HiWhite("%s")
}
func TestNoFormatString(t *testing.T) {
tests := []struct {
f func(string, ...interface{}) string
format string
args []interface{}
want string
}{
{BlackString, "%s", nil, "\x1b[30m%s\x1b[0m"},
{RedString, "%s", nil, "\x1b[31m%s\x1b[0m"},
{GreenString, "%s", nil, "\x1b[32m%s\x1b[0m"},
{YellowString, "%s", nil, "\x1b[33m%s\x1b[0m"},
{BlueString, "%s", nil, "\x1b[34m%s\x1b[0m"},
{MagentaString, "%s", nil, "\x1b[35m%s\x1b[0m"},
{CyanString, "%s", nil, "\x1b[36m%s\x1b[0m"},
{WhiteString, "%s", nil, "\x1b[37m%s\x1b[0m"},
{HiBlackString, "%s", nil, "\x1b[90m%s\x1b[0m"},
{HiRedString, "%s", nil, "\x1b[91m%s\x1b[0m"},
{HiGreenString, "%s", nil, "\x1b[92m%s\x1b[0m"},
{HiYellowString, "%s", nil, "\x1b[93m%s\x1b[0m"},
{HiBlueString, "%s", nil, "\x1b[94m%s\x1b[0m"},
{HiMagentaString, "%s", nil, "\x1b[95m%s\x1b[0m"},
{HiCyanString, "%s", nil, "\x1b[96m%s\x1b[0m"},
{HiWhiteString, "%s", nil, "\x1b[97m%s\x1b[0m"},
}
for i, test := range tests {
s := fmt.Sprintf("%s", test.f(test.format, test.args...))
if s != test.want {
t.Errorf("[%d] want: %q, got: %q", i, test.want, s)
}
}
}

@ -1,16 +0,0 @@
package main
import (
"bufio"
"fmt"
"github.com/mattn/go-colorable"
)
func main() {
stdOut := bufio.NewWriter(colorable.NewColorableStdout())
fmt.Fprint(stdOut, "\x1B[3GMove to 3rd Column\n")
fmt.Fprint(stdOut, "\x1B[1;2HMove to 2nd Column on 1st Line\n")
stdOut.Flush()
}

@ -1,16 +0,0 @@
package main
import (
"github.com/mattn/go-colorable"
"github.com/sirupsen/logrus"
)
func main() {
logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true})
logrus.SetOutput(colorable.NewColorableStdout())
logrus.Info("succeeded")
logrus.Warn("not correct")
logrus.Error("something error")
logrus.Fatal("panic")
}

@ -1,14 +0,0 @@
package main
import (
"fmt"
"os"
. "github.com/mattn/go-colorable"
)
func main() {
out := NewColorableStdout()
fmt.Fprint(out, "\x1B]0;TITLE Changed\007(See title and hit any key)")
var c [1]byte
os.Stdin.Read(c[:])
}

@ -1,83 +0,0 @@
package colorable
import (
"bytes"
"os"
"runtime"
"testing"
)
// checkEncoding checks that colorable is output encoding agnostic as long as
// the encoding is a superset of ASCII. This implies that one byte not part of
// an ANSI sequence must give exactly one byte in output
func checkEncoding(t *testing.T, data []byte) {
// Send non-UTF8 data to colorable
b := bytes.NewBuffer(make([]byte, 0, 10))
if b.Len() != 0 {
t.FailNow()
}
// TODO move colorable wrapping outside the test
c := NewNonColorable(b)
c.Write(data)
if b.Len() != len(data) {
t.Fatalf("%d bytes expected, got %d", len(data), b.Len())
}
}
func TestEncoding(t *testing.T) {
checkEncoding(t, []byte{}) // Empty
checkEncoding(t, []byte(`abc`)) // "abc"
checkEncoding(t, []byte(`é`)) // "é" in UTF-8
checkEncoding(t, []byte{233}) // 'é' in Latin-1
}
func TestNonColorable(t *testing.T) {
var buf bytes.Buffer
want := "hello"
NewNonColorable(&buf).Write([]byte("\x1b[0m" + want + "\x1b[2J"))
got := buf.String()
if got != "hello" {
t.Fatalf("want %q but %q", want, got)
}
buf.Reset()
NewNonColorable(&buf).Write([]byte("\x1b["))
got = buf.String()
if got != "" {
t.Fatalf("want %q but %q", "", got)
}
}
func TestNonColorableNil(t *testing.T) {
paniced := false
func() {
defer func() {
recover()
paniced = true
}()
NewNonColorable(nil)
NewColorable(nil)
}()
if !paniced {
t.Fatalf("should panic")
}
}
func TestColorable(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skipf("skip this test on windows")
}
_, ok := NewColorableStdout().(*os.File)
if !ok {
t.Fatalf("should os.Stdout on UNIX")
}
_, ok = NewColorableStderr().(*os.File)
if !ok {
t.Fatalf("should os.Stdout on UNIX")
}
_, ok = NewColorable(os.Stdout).(*os.File)
if !ok {
t.Fatalf("should os.Stdout on UNIX")
}
}

@ -2,6 +2,10 @@ language: go
go:
- tip
os:
- linux
- osx
before_install:
- go get github.com/mattn/goveralls
- go get golang.org/x/tools/cmd/cover

@ -1,18 +0,0 @@
package isatty_test
import (
"fmt"
"os"
"github.com/mattn/go-isatty"
)
func Example() {
if isatty.IsTerminal(os.Stdout.Fd()) {
fmt.Println("Is Terminal")
} else if isatty.IsCygwinTerminal(os.Stdout.Fd()) {
fmt.Println("Is Cygwin/MSYS2 Terminal")
} else {
fmt.Println("Is Not Terminal")
}
}

@ -3,7 +3,7 @@
package isatty
// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2
// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2
// terminal. This is also always false on this environment.
func IsCygwinTerminal(fd uintptr) bool {
return false

@ -1,19 +0,0 @@
// +build !windows
package isatty
import (
"os"
"testing"
)
func TestTerminal(t *testing.T) {
// test for non-panic
IsTerminal(os.Stdout.Fd())
}
func TestCygwinPipeName(t *testing.T) {
if IsCygwinTerminal(os.Stdout.Fd()) {
t.Fatal("should be false always")
}
}

@ -1,35 +0,0 @@
// +build windows
package isatty
import (
"testing"
)
func TestCygwinPipeName(t *testing.T) {
tests := []struct {
name string
result bool
}{
{``, false},
{`\msys-`, false},
{`\cygwin-----`, false},
{`\msys-x-PTY5-pty1-from-master`, false},
{`\cygwin-x-PTY5-from-master`, false},
{`\cygwin-x-pty2-from-toaster`, false},
{`\cygwin--pty2-from-master`, false},
{`\\cygwin-x-pty2-from-master`, false},
{`\cygwin-x-pty2-from-master-`, true}, // for the feature
{`\cygwin-e022582115c10879-pty4-from-master`, true},
{`\msys-e022582115c10879-pty4-to-master`, true},
{`\cygwin-e022582115c10879-pty4-to-master`, true},
}
for _, test := range tests {
want := test.result
got := isCygwinPipeName(test.name)
if want != got {
t.Fatalf("isatty(%q): got %v, want %v:", test.name, got, want)
}
}
}

@ -1,13 +1,36 @@
package runewidth
import (
"os"
)
var (
// EastAsianWidth will be set true if the current locale is CJK
EastAsianWidth = IsEastAsian()
EastAsianWidth bool
// ZeroWidthJoiner is flag to set to use UTR#51 ZWJ
ZeroWidthJoiner bool
// DefaultCondition is a condition in current locale
DefaultCondition = &Condition{EastAsianWidth}
DefaultCondition = &Condition{}
)
func init() {
handleEnv()
}
func handleEnv() {
env := os.Getenv("RUNEWIDTH_EASTASIAN")
if env == "" {
EastAsianWidth = IsEastAsian()
} else {
EastAsianWidth = env == "1"
}
// update DefaultCondition
DefaultCondition.EastAsianWidth = EastAsianWidth
DefaultCondition.ZeroWidthJoiner = ZeroWidthJoiner
}
type interval struct {
first rune
last rune
@ -33,7 +56,7 @@ func inTable(r rune, t table) bool {
bot := 0
top := len(t) - 1
for top >= bot {
mid := (bot + top) / 2
mid := (bot + top) >> 1
switch {
case t[mid].last < r:
@ -55,7 +78,7 @@ var private = table{
var nonprint = table{
{0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD},
{0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F},
{0x202A, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF},
{0x2028, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF},
{0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF},
}
@ -249,19 +272,54 @@ var ambiguous = table{
}
var emoji = table{
{0x1F1E6, 0x1F1FF}, {0x1F321, 0x1F321}, {0x1F324, 0x1F32C},
{0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, {0x1F396, 0x1F397},
{0x1F399, 0x1F39B}, {0x1F39E, 0x1F39F}, {0x1F3CB, 0x1F3CE},
{0x1F3D4, 0x1F3DF}, {0x1F3F3, 0x1F3F5}, {0x1F3F7, 0x1F3F7},
{0x1F43F, 0x1F43F}, {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FD},
{0x1F549, 0x1F54A}, {0x1F56F, 0x1F570}, {0x1F573, 0x1F579},
{0x203C, 0x203C}, {0x2049, 0x2049}, {0x2122, 0x2122},
{0x2139, 0x2139}, {0x2194, 0x2199}, {0x21A9, 0x21AA},
{0x231A, 0x231B}, {0x2328, 0x2328}, {0x23CF, 0x23CF},
{0x23E9, 0x23F3}, {0x23F8, 0x23FA}, {0x24C2, 0x24C2},
{0x25AA, 0x25AB}, {0x25B6, 0x25B6}, {0x25C0, 0x25C0},
{0x25FB, 0x25FE}, {0x2600, 0x2604}, {0x260E, 0x260E},
{0x2611, 0x2611}, {0x2614, 0x2615}, {0x2618, 0x2618},
{0x261D, 0x261D}, {0x2620, 0x2620}, {0x2622, 0x2623},
{0x2626, 0x2626}, {0x262A, 0x262A}, {0x262E, 0x262F},
{0x2638, 0x263A}, {0x2640, 0x2640}, {0x2642, 0x2642},
{0x2648, 0x2653}, {0x265F, 0x2660}, {0x2663, 0x2663},
{0x2665, 0x2666}, {0x2668, 0x2668}, {0x267B, 0x267B},
{0x267E, 0x267F}, {0x2692, 0x2697}, {0x2699, 0x2699},
{0x269B, 0x269C}, {0x26A0, 0x26A1}, {0x26AA, 0x26AB},
{0x26B0, 0x26B1}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5},
{0x26C8, 0x26C8}, {0x26CE, 0x26CF}, {0x26D1, 0x26D1},
{0x26D3, 0x26D4}, {0x26E9, 0x26EA}, {0x26F0, 0x26F5},
{0x26F7, 0x26FA}, {0x26FD, 0x26FD}, {0x2702, 0x2702},
{0x2705, 0x2705}, {0x2708, 0x270D}, {0x270F, 0x270F},
{0x2712, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716},
{0x271D, 0x271D}, {0x2721, 0x2721}, {0x2728, 0x2728},
{0x2733, 0x2734}, {0x2744, 0x2744}, {0x2747, 0x2747},
{0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755},
{0x2757, 0x2757}, {0x2763, 0x2764}, {0x2795, 0x2797},
{0x27A1, 0x27A1}, {0x27B0, 0x27B0}, {0x27BF, 0x27BF},
{0x2934, 0x2935}, {0x2B05, 0x2B07}, {0x2B1B, 0x2B1C},
{0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x3030, 0x3030},
{0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299},
{0x1F004, 0x1F004}, {0x1F0CF, 0x1F0CF}, {0x1F170, 0x1F171},
{0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A},
{0x1F1E6, 0x1F1FF}, {0x1F201, 0x1F202}, {0x1F21A, 0x1F21A},
{0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, {0x1F250, 0x1F251},
{0x1F300, 0x1F321}, {0x1F324, 0x1F393}, {0x1F396, 0x1F397},
{0x1F399, 0x1F39B}, {0x1F39E, 0x1F3F0}, {0x1F3F3, 0x1F3F5},
{0x1F3F7, 0x1F4FD}, {0x1F4FF, 0x1F53D}, {0x1F549, 0x1F54E},
{0x1F550, 0x1F567}, {0x1F56F, 0x1F570}, {0x1F573, 0x1F57A},
{0x1F587, 0x1F587}, {0x1F58A, 0x1F58D}, {0x1F590, 0x1F590},
{0x1F5A5, 0x1F5A5}, {0x1F5A8, 0x1F5A8}, {0x1F5B1, 0x1F5B2},
{0x1F5BC, 0x1F5BC}, {0x1F5C2, 0x1F5C4}, {0x1F5D1, 0x1F5D3},
{0x1F5DC, 0x1F5DE}, {0x1F5E1, 0x1F5E1}, {0x1F5E3, 0x1F5E3},
{0x1F5E8, 0x1F5E8}, {0x1F5EF, 0x1F5EF}, {0x1F5F3, 0x1F5F3},
{0x1F5FA, 0x1F5FA}, {0x1F6CB, 0x1F6CF}, {0x1F6E0, 0x1F6E5},
{0x1F6E9, 0x1F6E9}, {0x1F6F0, 0x1F6F0}, {0x1F6F3, 0x1F6F3},
{0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A5}, {0x1F5A8, 0x1F5A8},
{0x1F5B1, 0x1F5B2}, {0x1F5BC, 0x1F5BC}, {0x1F5C2, 0x1F5C4},
{0x1F5D1, 0x1F5D3}, {0x1F5DC, 0x1F5DE}, {0x1F5E1, 0x1F5E1},
{0x1F5E3, 0x1F5E3}, {0x1F5E8, 0x1F5E8}, {0x1F5EF, 0x1F5EF},
{0x1F5F3, 0x1F5F3}, {0x1F5FA, 0x1F64F}, {0x1F680, 0x1F6C5},
{0x1F6CB, 0x1F6D2}, {0x1F6E0, 0x1F6E5}, {0x1F6E9, 0x1F6E9},
{0x1F6EB, 0x1F6EC}, {0x1F6F0, 0x1F6F0}, {0x1F6F3, 0x1F6F9},
{0x1F910, 0x1F93A}, {0x1F93C, 0x1F93E}, {0x1F940, 0x1F945},
{0x1F947, 0x1F970}, {0x1F973, 0x1F976}, {0x1F97A, 0x1F97A},
{0x1F97C, 0x1F9A2}, {0x1F9B0, 0x1F9B9}, {0x1F9C0, 0x1F9C2},
{0x1F9D0, 0x1F9FF},
}
var notassigned = table{
@ -481,314 +539,141 @@ var notassigned = table{
}
var neutral = table{
{0x0000, 0x001F}, {0x007F, 0x007F}, {0x0080, 0x009F},
{0x00A0, 0x00A0}, {0x00A9, 0x00A9}, {0x00AB, 0x00AB},
{0x00B5, 0x00B5}, {0x00BB, 0x00BB}, {0x00C0, 0x00C5},
{0x00C7, 0x00CF}, {0x00D1, 0x00D6}, {0x00D9, 0x00DD},
{0x00E2, 0x00E5}, {0x00E7, 0x00E7}, {0x00EB, 0x00EB},
{0x00EE, 0x00EF}, {0x00F1, 0x00F1}, {0x00F4, 0x00F6},
{0x00FB, 0x00FB}, {0x00FD, 0x00FD}, {0x00FF, 0x00FF},
{0x0100, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112},
{0x0000, 0x001F}, {0x007F, 0x00A0}, {0x00A9, 0x00A9},
{0x00AB, 0x00AB}, {0x00B5, 0x00B5}, {0x00BB, 0x00BB},
{0x00C0, 0x00C5}, {0x00C7, 0x00CF}, {0x00D1, 0x00D6},
{0x00D9, 0x00DD}, {0x00E2, 0x00E5}, {0x00E7, 0x00E7},
{0x00EB, 0x00EB}, {0x00EE, 0x00EF}, {0x00F1, 0x00F1},
{0x00F4, 0x00F6}, {0x00FB, 0x00FB}, {0x00FD, 0x00FD},
{0x00FF, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112},
{0x0114, 0x011A}, {0x011C, 0x0125}, {0x0128, 0x012A},
{0x012C, 0x0130}, {0x0134, 0x0137}, {0x0139, 0x013E},
{0x0143, 0x0143}, {0x0145, 0x0147}, {0x014C, 0x014C},
{0x014E, 0x0151}, {0x0154, 0x0165}, {0x0168, 0x016A},
{0x016C, 0x017F}, {0x0180, 0x01BA}, {0x01BB, 0x01BB},
{0x01BC, 0x01BF}, {0x01C0, 0x01C3}, {0x01C4, 0x01CD},
{0x01CF, 0x01CF}, {0x01D1, 0x01D1}, {0x01D3, 0x01D3},
{0x01D5, 0x01D5}, {0x01D7, 0x01D7}, {0x01D9, 0x01D9},
{0x01DB, 0x01DB}, {0x01DD, 0x024F}, {0x0250, 0x0250},
{0x0252, 0x0260}, {0x0262, 0x0293}, {0x0294, 0x0294},
{0x0295, 0x02AF}, {0x02B0, 0x02C1}, {0x02C2, 0x02C3},
{0x02C5, 0x02C5}, {0x02C6, 0x02C6}, {0x02C8, 0x02C8},
{0x02CC, 0x02CC}, {0x02CE, 0x02CF}, {0x02D1, 0x02D1},
{0x02D2, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE},
{0x02E0, 0x02E4}, {0x02E5, 0x02EB}, {0x02EC, 0x02EC},
{0x02ED, 0x02ED}, {0x02EE, 0x02EE}, {0x02EF, 0x02FF},
{0x0370, 0x0373}, {0x0374, 0x0374}, {0x0375, 0x0375},
{0x0376, 0x0377}, {0x037A, 0x037A}, {0x037B, 0x037D},
{0x037E, 0x037E}, {0x037F, 0x037F}, {0x0384, 0x0385},
{0x0386, 0x0386}, {0x0387, 0x0387}, {0x0388, 0x038A},
{0x038C, 0x038C}, {0x038E, 0x0390}, {0x03AA, 0x03B0},
{0x03C2, 0x03C2}, {0x03CA, 0x03F5}, {0x03F6, 0x03F6},
{0x03F7, 0x03FF}, {0x0400, 0x0400}, {0x0402, 0x040F},
{0x0450, 0x0450}, {0x0452, 0x0481}, {0x0482, 0x0482},
{0x0483, 0x0487}, {0x0488, 0x0489}, {0x048A, 0x04FF},
{0x0500, 0x052F}, {0x0531, 0x0556}, {0x0559, 0x0559},
{0x055A, 0x055F}, {0x0561, 0x0587}, {0x0589, 0x0589},
{0x058A, 0x058A}, {0x058D, 0x058E}, {0x058F, 0x058F},
{0x0591, 0x05BD}, {0x05BE, 0x05BE}, {0x05BF, 0x05BF},
{0x05C0, 0x05C0}, {0x05C1, 0x05C2}, {0x05C3, 0x05C3},
{0x05C4, 0x05C5}, {0x05C6, 0x05C6}, {0x05C7, 0x05C7},
{0x05D0, 0x05EA}, {0x05F0, 0x05F2}, {0x05F3, 0x05F4},
{0x0600, 0x0605}, {0x0606, 0x0608}, {0x0609, 0x060A},
{0x060B, 0x060B}, {0x060C, 0x060D}, {0x060E, 0x060F},
{0x0610, 0x061A}, {0x061B, 0x061B}, {0x061C, 0x061C},
{0x061E, 0x061F}, {0x0620, 0x063F}, {0x0640, 0x0640},
{0x0641, 0x064A}, {0x064B, 0x065F}, {0x0660, 0x0669},
{0x066A, 0x066D}, {0x066E, 0x066F}, {0x0670, 0x0670},
{0x0671, 0x06D3}, {0x06D4, 0x06D4}, {0x06D5, 0x06D5},
{0x06D6, 0x06DC}, {0x06DD, 0x06DD}, {0x06DE, 0x06DE},
{0x06DF, 0x06E4}, {0x06E5, 0x06E6}, {0x06E7, 0x06E8},
{0x06E9, 0x06E9}, {0x06EA, 0x06ED}, {0x06EE, 0x06EF},
{0x06F0, 0x06F9}, {0x06FA, 0x06FC}, {0x06FD, 0x06FE},
{0x06FF, 0x06FF}, {0x0700, 0x070D}, {0x070F, 0x070F},
{0x0710, 0x0710}, {0x0711, 0x0711}, {0x0712, 0x072F},
{0x0730, 0x074A}, {0x074D, 0x074F}, {0x0750, 0x077F},
{0x0780, 0x07A5}, {0x07A6, 0x07B0}, {0x07B1, 0x07B1},
{0x07C0, 0x07C9}, {0x07CA, 0x07EA}, {0x07EB, 0x07F3},
{0x07F4, 0x07F5}, {0x07F6, 0x07F6}, {0x07F7, 0x07F9},
{0x07FA, 0x07FA}, {0x0800, 0x0815}, {0x0816, 0x0819},
{0x081A, 0x081A}, {0x081B, 0x0823}, {0x0824, 0x0824},
{0x0825, 0x0827}, {0x0828, 0x0828}, {0x0829, 0x082D},
{0x0830, 0x083E}, {0x0840, 0x0858}, {0x0859, 0x085B},
{0x085E, 0x085E}, {0x08A0, 0x08B4}, {0x08B6, 0x08BD},
{0x08D4, 0x08E1}, {0x08E2, 0x08E2}, {0x08E3, 0x08FF},
{0x0900, 0x0902}, {0x0903, 0x0903}, {0x0904, 0x0939},
{0x093A, 0x093A}, {0x093B, 0x093B}, {0x093C, 0x093C},
{0x093D, 0x093D}, {0x093E, 0x0940}, {0x0941, 0x0948},
{0x0949, 0x094C}, {0x094D, 0x094D}, {0x094E, 0x094F},
{0x0950, 0x0950}, {0x0951, 0x0957}, {0x0958, 0x0961},
{0x0962, 0x0963}, {0x0964, 0x0965}, {0x0966, 0x096F},
{0x0970, 0x0970}, {0x0971, 0x0971}, {0x0972, 0x097F},
{0x0980, 0x0980}, {0x0981, 0x0981}, {0x0982, 0x0983},
{0x0985, 0x098C}, {0x098F, 0x0990}, {0x0993, 0x09A8},
{0x09AA, 0x09B0}, {0x09B2, 0x09B2}, {0x09B6, 0x09B9},
{0x09BC, 0x09BC}, {0x09BD, 0x09BD}, {0x09BE, 0x09C0},
{0x09C1, 0x09C4}, {0x09C7, 0x09C8}, {0x09CB, 0x09CC},
{0x09CD, 0x09CD}, {0x09CE, 0x09CE}, {0x09D7, 0x09D7},
{0x09DC, 0x09DD}, {0x09DF, 0x09E1}, {0x09E2, 0x09E3},
{0x09E6, 0x09EF}, {0x09F0, 0x09F1}, {0x09F2, 0x09F3},
{0x09F4, 0x09F9}, {0x09FA, 0x09FA}, {0x09FB, 0x09FB},
{0x0A01, 0x0A02}, {0x0A03, 0x0A03}, {0x0A05, 0x0A0A},
{0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, {0x0A2A, 0x0A30},
{0x0A32, 0x0A33}, {0x0A35, 0x0A36}, {0x0A38, 0x0A39},
{0x0A3C, 0x0A3C}, {0x0A3E, 0x0A40}, {0x0A41, 0x0A42},
{0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51},
{0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, {0x0A66, 0x0A6F},
{0x0A70, 0x0A71}, {0x0A72, 0x0A74}, {0x0A75, 0x0A75},
{0x0A81, 0x0A82}, {0x0A83, 0x0A83}, {0x0A85, 0x0A8D},
{0x016C, 0x01CD}, {0x01CF, 0x01CF}, {0x01D1, 0x01D1},
{0x01D3, 0x01D3}, {0x01D5, 0x01D5}, {0x01D7, 0x01D7},
{0x01D9, 0x01D9}, {0x01DB, 0x01DB}, {0x01DD, 0x0250},
{0x0252, 0x0260}, {0x0262, 0x02C3}, {0x02C5, 0x02C6},
{0x02C8, 0x02C8}, {0x02CC, 0x02CC}, {0x02CE, 0x02CF},
{0x02D1, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE},
{0x02E0, 0x02FF}, {0x0370, 0x0377}, {0x037A, 0x037F},
{0x0384, 0x038A}, {0x038C, 0x038C}, {0x038E, 0x0390},
{0x03AA, 0x03B0}, {0x03C2, 0x03C2}, {0x03CA, 0x0400},
{0x0402, 0x040F}, {0x0450, 0x0450}, {0x0452, 0x052F},
{0x0531, 0x0556}, {0x0559, 0x055F}, {0x0561, 0x0587},
{0x0589, 0x058A}, {0x058D, 0x058F}, {0x0591, 0x05C7},
{0x05D0, 0x05EA}, {0x05F0, 0x05F4}, {0x0600, 0x061C},
{0x061E, 0x070D}, {0x070F, 0x074A}, {0x074D, 0x07B1},
{0x07C0, 0x07FA}, {0x0800, 0x082D}, {0x0830, 0x083E},
{0x0840, 0x085B}, {0x085E, 0x085E}, {0x08A0, 0x08B4},
{0x08B6, 0x08BD}, {0x08D4, 0x0983}, {0x0985, 0x098C},
{0x098F, 0x0990}, {0x0993, 0x09A8}, {0x09AA, 0x09B0},
{0x09B2, 0x09B2}, {0x09B6, 0x09B9}, {0x09BC, 0x09C4},
{0x09C7, 0x09C8}, {0x09CB, 0x09CE}, {0x09D7, 0x09D7},
{0x09DC, 0x09DD}, {0x09DF, 0x09E3}, {0x09E6, 0x09FB},
{0x0A01, 0x0A03}, {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10},
{0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, {0x0A32, 0x0A33},
{0x0A35, 0x0A36}, {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C},
{0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D},
{0x0A51, 0x0A51}, {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E},
{0x0A66, 0x0A75}, {0x0A81, 0x0A83}, {0x0A85, 0x0A8D},
{0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0},
{0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0ABC},
{0x0ABD, 0x0ABD}, {0x0ABE, 0x0AC0}, {0x0AC1, 0x0AC5},
{0x0AC7, 0x0AC8}, {0x0AC9, 0x0AC9}, {0x0ACB, 0x0ACC},
{0x0ACD, 0x0ACD}, {0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE1},
{0x0AE2, 0x0AE3}, {0x0AE6, 0x0AEF}, {0x0AF0, 0x0AF0},
{0x0AF1, 0x0AF1}, {0x0AF9, 0x0AF9}, {0x0B01, 0x0B01},
{0x0B02, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10},
{0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5},
{0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0},
{0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AF9},
{0x0B01, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10},
{0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, {0x0B32, 0x0B33},
{0x0B35, 0x0B39}, {0x0B3C, 0x0B3C}, {0x0B3D, 0x0B3D},
{0x0B3E, 0x0B3E}, {0x0B3F, 0x0B3F}, {0x0B40, 0x0B40},
{0x0B41, 0x0B44}, {0x0B47, 0x0B48}, {0x0B4B, 0x0B4C},
{0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B57, 0x0B57},
{0x0B5C, 0x0B5D}, {0x0B5F, 0x0B61}, {0x0B62, 0x0B63},
{0x0B66, 0x0B6F}, {0x0B70, 0x0B70}, {0x0B71, 0x0B71},
{0x0B72, 0x0B77}, {0x0B82, 0x0B82}, {0x0B83, 0x0B83},
{0x0B35, 0x0B39}, {0x0B3C, 0x0B44}, {0x0B47, 0x0B48},
{0x0B4B, 0x0B4D}, {0x0B56, 0x0B57}, {0x0B5C, 0x0B5D},
{0x0B5F, 0x0B63}, {0x0B66, 0x0B77}, {0x0B82, 0x0B83},
{0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, {0x0B92, 0x0B95},
{0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F},
{0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9},
{0x0BBE, 0x0BBF}, {0x0BC0, 0x0BC0}, {0x0BC1, 0x0BC2},
{0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCC}, {0x0BCD, 0x0BCD},
{0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BEF},
{0x0BF0, 0x0BF2}, {0x0BF3, 0x0BF8}, {0x0BF9, 0x0BF9},
{0x0BFA, 0x0BFA}, {0x0C00, 0x0C00}, {0x0C01, 0x0C03},
{0x0C05, 0x0C0C}, {0x0C0E, 0x0C10}, {0x0C12, 0x0C28},
{0x0C2A, 0x0C39}, {0x0C3D, 0x0C3D}, {0x0C3E, 0x0C40},
{0x0C41, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D},
{0x0C55, 0x0C56}, {0x0C58, 0x0C5A}, {0x0C60, 0x0C61},
{0x0C62, 0x0C63}, {0x0C66, 0x0C6F}, {0x0C78, 0x0C7E},
{0x0C7F, 0x0C7F}, {0x0C80, 0x0C80}, {0x0C81, 0x0C81},
{0x0C82, 0x0C83}, {0x0C85, 0x0C8C}, {0x0C8E, 0x0C90},
{0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD},
{0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA},
{0x0C00, 0x0C03}, {0x0C05, 0x0C0C}, {0x0C0E, 0x0C10},
{0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, {0x0C3D, 0x0C44},
{0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
{0x0C58, 0x0C5A}, {0x0C60, 0x0C63}, {0x0C66, 0x0C6F},
{0x0C78, 0x0C83}, {0x0C85, 0x0C8C}, {0x0C8E, 0x0C90},
{0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9},
{0x0CBC, 0x0CBC}, {0x0CBD, 0x0CBD}, {0x0CBE, 0x0CBE},
{0x0CBF, 0x0CBF}, {0x0CC0, 0x0CC4}, {0x0CC6, 0x0CC6},
{0x0CC7, 0x0CC8}, {0x0CCA, 0x0CCB}, {0x0CCC, 0x0CCD},
{0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE1},
{0x0CE2, 0x0CE3}, {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2},
{0x0D01, 0x0D01}, {0x0D02, 0x0D03}, {0x0D05, 0x0D0C},
{0x0D0E, 0x0D10}, {0x0D12, 0x0D3A}, {0x0D3D, 0x0D3D},
{0x0D3E, 0x0D40}, {0x0D41, 0x0D44}, {0x0D46, 0x0D48},
{0x0D4A, 0x0D4C}, {0x0D4D, 0x0D4D}, {0x0D4E, 0x0D4E},
{0x0D4F, 0x0D4F}, {0x0D54, 0x0D56}, {0x0D57, 0x0D57},
{0x0D58, 0x0D5E}, {0x0D5F, 0x0D61}, {0x0D62, 0x0D63},
{0x0D66, 0x0D6F}, {0x0D70, 0x0D78}, {0x0D79, 0x0D79},
{0x0D7A, 0x0D7F}, {0x0D82, 0x0D83}, {0x0D85, 0x0D96},
{0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD},
{0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD1},
{0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF},
{0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF3}, {0x0DF4, 0x0DF4},
{0x0E01, 0x0E30}, {0x0E31, 0x0E31}, {0x0E32, 0x0E33},
{0x0E34, 0x0E3A}, {0x0E3F, 0x0E3F}, {0x0E40, 0x0E45},
{0x0E46, 0x0E46}, {0x0E47, 0x0E4E}, {0x0E4F, 0x0E4F},
{0x0E50, 0x0E59}, {0x0E5A, 0x0E5B}, {0x0E81, 0x0E82},
{0x0E84, 0x0E84}, {0x0E87, 0x0E88}, {0x0E8A, 0x0E8A},
{0x0E8D, 0x0E8D}, {0x0E94, 0x0E97}, {0x0E99, 0x0E9F},
{0x0EA1, 0x0EA3}, {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EA7},
{0x0EAA, 0x0EAB}, {0x0EAD, 0x0EB0}, {0x0EB1, 0x0EB1},
{0x0EB2, 0x0EB3}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC},
{0x0EBD, 0x0EBD}, {0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6},
{0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF},
{0x0F00, 0x0F00}, {0x0F01, 0x0F03}, {0x0F04, 0x0F12},
{0x0F13, 0x0F13}, {0x0F14, 0x0F14}, {0x0F15, 0x0F17},
{0x0F18, 0x0F19}, {0x0F1A, 0x0F1F}, {0x0F20, 0x0F29},
{0x0F2A, 0x0F33}, {0x0F34, 0x0F34}, {0x0F35, 0x0F35},
{0x0F36, 0x0F36}, {0x0F37, 0x0F37}, {0x0F38, 0x0F38},
{0x0F39, 0x0F39}, {0x0F3A, 0x0F3A}, {0x0F3B, 0x0F3B},
{0x0F3C, 0x0F3C}, {0x0F3D, 0x0F3D}, {0x0F3E, 0x0F3F},
{0x0F40, 0x0F47}, {0x0F49, 0x0F6C}, {0x0F71, 0x0F7E},
{0x0F7F, 0x0F7F}, {0x0F80, 0x0F84}, {0x0F85, 0x0F85},
{0x0F86, 0x0F87}, {0x0F88, 0x0F8C}, {0x0F8D, 0x0F97},
{0x0F99, 0x0FBC}, {0x0FBE, 0x0FC5}, {0x0FC6, 0x0FC6},
{0x0FC7, 0x0FCC}, {0x0FCE, 0x0FCF}, {0x0FD0, 0x0FD4},
{0x0FD5, 0x0FD8}, {0x0FD9, 0x0FDA}, {0x1000, 0x102A},
{0x102B, 0x102C}, {0x102D, 0x1030}, {0x1031, 0x1031},
{0x1032, 0x1037}, {0x1038, 0x1038}, {0x1039, 0x103A},
{0x103B, 0x103C}, {0x103D, 0x103E}, {0x103F, 0x103F},
{0x1040, 0x1049}, {0x104A, 0x104F}, {0x1050, 0x1055},
{0x1056, 0x1057}, {0x1058, 0x1059}, {0x105A, 0x105D},
{0x105E, 0x1060}, {0x1061, 0x1061}, {0x1062, 0x1064},
{0x1065, 0x1066}, {0x1067, 0x106D}, {0x106E, 0x1070},
{0x1071, 0x1074}, {0x1075, 0x1081}, {0x1082, 0x1082},
{0x1083, 0x1084}, {0x1085, 0x1086}, {0x1087, 0x108C},
{0x108D, 0x108D}, {0x108E, 0x108E}, {0x108F, 0x108F},
{0x1090, 0x1099}, {0x109A, 0x109C}, {0x109D, 0x109D},
{0x109E, 0x109F}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7},
{0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FB, 0x10FB},
{0x10FC, 0x10FC}, {0x10FD, 0x10FF}, {0x1160, 0x11FF},
{0x1200, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256},
{0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288},
{0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5},
{0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5},
{0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315},
{0x1318, 0x135A}, {0x135D, 0x135F}, {0x1360, 0x1368},
{0x1369, 0x137C}, {0x1380, 0x138F}, {0x1390, 0x1399},
{0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1400, 0x1400},
{0x1401, 0x166C}, {0x166D, 0x166E}, {0x166F, 0x167F},
{0x1680, 0x1680}, {0x1681, 0x169A}, {0x169B, 0x169B},
{0x169C, 0x169C}, {0x16A0, 0x16EA}, {0x16EB, 0x16ED},
{0x16EE, 0x16F0}, {0x16F1, 0x16F8}, {0x1700, 0x170C},
{0x170E, 0x1711}, {0x1712, 0x1714}, {0x1720, 0x1731},
{0x1732, 0x1734}, {0x1735, 0x1736}, {0x1740, 0x1751},
{0x1752, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770},
{0x1772, 0x1773}, {0x1780, 0x17B3}, {0x17B4, 0x17B5},
{0x17B6, 0x17B6}, {0x17B7, 0x17BD}, {0x17BE, 0x17C5},
{0x17C6, 0x17C6}, {0x17C7, 0x17C8}, {0x17C9, 0x17D3},
{0x17D4, 0x17D6}, {0x17D7, 0x17D7}, {0x17D8, 0x17DA},
{0x17DB, 0x17DB}, {0x17DC, 0x17DC}, {0x17DD, 0x17DD},
{0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1800, 0x1805},
{0x1806, 0x1806}, {0x1807, 0x180A}, {0x180B, 0x180D},
{0x180E, 0x180E}, {0x1810, 0x1819}, {0x1820, 0x1842},
{0x1843, 0x1843}, {0x1844, 0x1877}, {0x1880, 0x1884},
{0x1885, 0x1886}, {0x1887, 0x18A8}, {0x18A9, 0x18A9},
{0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E},
{0x1920, 0x1922}, {0x1923, 0x1926}, {0x1927, 0x1928},
{0x1929, 0x192B}, {0x1930, 0x1931}, {0x1932, 0x1932},
{0x1933, 0x1938}, {0x1939, 0x193B}, {0x1940, 0x1940},
{0x1944, 0x1945}, {0x1946, 0x194F}, {0x1950, 0x196D},
{0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9},
{0x19D0, 0x19D9}, {0x19DA, 0x19DA}, {0x19DE, 0x19DF},
{0x19E0, 0x19FF}, {0x1A00, 0x1A16}, {0x1A17, 0x1A18},
{0x1A19, 0x1A1A}, {0x1A1B, 0x1A1B}, {0x1A1E, 0x1A1F},
{0x1A20, 0x1A54}, {0x1A55, 0x1A55}, {0x1A56, 0x1A56},
{0x1A57, 0x1A57}, {0x1A58, 0x1A5E}, {0x1A60, 0x1A60},
{0x1A61, 0x1A61}, {0x1A62, 0x1A62}, {0x1A63, 0x1A64},
{0x1A65, 0x1A6C}, {0x1A6D, 0x1A72}, {0x1A73, 0x1A7C},
{0x1A7F, 0x1A7F}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99},
{0x1AA0, 0x1AA6}, {0x1AA7, 0x1AA7}, {0x1AA8, 0x1AAD},
{0x1AB0, 0x1ABD}, {0x1ABE, 0x1ABE}, {0x1B00, 0x1B03},
{0x1B04, 0x1B04}, {0x1B05, 0x1B33}, {0x1B34, 0x1B34},
{0x1B35, 0x1B35}, {0x1B36, 0x1B3A}, {0x1B3B, 0x1B3B},
{0x1B3C, 0x1B3C}, {0x1B3D, 0x1B41}, {0x1B42, 0x1B42},
{0x1B43, 0x1B44}, {0x1B45, 0x1B4B}, {0x1B50, 0x1B59},
{0x1B5A, 0x1B60}, {0x1B61, 0x1B6A}, {0x1B6B, 0x1B73},
{0x1B74, 0x1B7C}, {0x1B80, 0x1B81}, {0x1B82, 0x1B82},
{0x1B83, 0x1BA0}, {0x1BA1, 0x1BA1}, {0x1BA2, 0x1BA5},
{0x1BA6, 0x1BA7}, {0x1BA8, 0x1BA9}, {0x1BAA, 0x1BAA},
{0x1BAB, 0x1BAD}, {0x1BAE, 0x1BAF}, {0x1BB0, 0x1BB9},
{0x1BBA, 0x1BBF}, {0x1BC0, 0x1BE5}, {0x1BE6, 0x1BE6},
{0x1BE7, 0x1BE7}, {0x1BE8, 0x1BE9}, {0x1BEA, 0x1BEC},
{0x1BED, 0x1BED}, {0x1BEE, 0x1BEE}, {0x1BEF, 0x1BF1},
{0x1BF2, 0x1BF3}, {0x1BFC, 0x1BFF}, {0x1C00, 0x1C23},
{0x1C24, 0x1C2B}, {0x1C2C, 0x1C33}, {0x1C34, 0x1C35},
{0x1C36, 0x1C37}, {0x1C3B, 0x1C3F}, {0x1C40, 0x1C49},
{0x1C4D, 0x1C4F}, {0x1C50, 0x1C59}, {0x1C5A, 0x1C77},
{0x1C78, 0x1C7D}, {0x1C7E, 0x1C7F}, {0x1C80, 0x1C88},
{0x1CC0, 0x1CC7}, {0x1CD0, 0x1CD2}, {0x1CD3, 0x1CD3},
{0x1CD4, 0x1CE0}, {0x1CE1, 0x1CE1}, {0x1CE2, 0x1CE8},
{0x1CE9, 0x1CEC}, {0x1CED, 0x1CED}, {0x1CEE, 0x1CF1},
{0x1CF2, 0x1CF3}, {0x1CF4, 0x1CF4}, {0x1CF5, 0x1CF6},
{0x1CF8, 0x1CF9}, {0x1D00, 0x1D2B}, {0x1D2C, 0x1D6A},
{0x1D6B, 0x1D77}, {0x1D78, 0x1D78}, {0x1D79, 0x1D7F},
{0x1D80, 0x1D9A}, {0x1D9B, 0x1DBF}, {0x1DC0, 0x1DF5},
{0x1DFB, 0x1DFF}, {0x1E00, 0x1EFF}, {0x1F00, 0x1F15},
{0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD},
{0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE3},
{0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, {0x0D01, 0x0D03},
{0x0D05, 0x0D0C}, {0x0D0E, 0x0D10}, {0x0D12, 0x0D3A},
{0x0D3D, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4F},
{0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, {0x0D82, 0x0D83},
{0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB},
{0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA},
{0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF},
{0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A},
{0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, {0x0E84, 0x0E84},
{0x0E87, 0x0E88}, {0x0E8A, 0x0E8A}, {0x0E8D, 0x0E8D},
{0x0E94, 0x0E97}, {0x0E99, 0x0E9F}, {0x0EA1, 0x0EA3},
{0x0EA5, 0x0EA5}, {0x0EA7, 0x0EA7}, {0x0EAA, 0x0EAB},
{0x0EAD, 0x0EB9}, {0x0EBB, 0x0EBD}, {0x0EC0, 0x0EC4},
{0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9},
{0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C},
{0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC},
{0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7},
{0x10CD, 0x10CD}, {0x10D0, 0x10FF}, {0x1160, 0x1248},
{0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258},
{0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D},
{0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE},
{0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6},
{0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A},
{0x135D, 0x137C}, {0x1380, 0x1399}, {0x13A0, 0x13F5},
{0x13F8, 0x13FD}, {0x1400, 0x169C}, {0x16A0, 0x16F8},
{0x1700, 0x170C}, {0x170E, 0x1714}, {0x1720, 0x1736},
{0x1740, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770},
{0x1772, 0x1773}, {0x1780, 0x17DD}, {0x17E0, 0x17E9},
{0x17F0, 0x17F9}, {0x1800, 0x180E}, {0x1810, 0x1819},
{0x1820, 0x1877}, {0x1880, 0x18AA}, {0x18B0, 0x18F5},
{0x1900, 0x191E}, {0x1920, 0x192B}, {0x1930, 0x193B},
{0x1940, 0x1940}, {0x1944, 0x196D}, {0x1970, 0x1974},
{0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA},
{0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C},
{0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA0, 0x1AAD},
{0x1AB0, 0x1ABE}, {0x1B00, 0x1B4B}, {0x1B50, 0x1B7C},
{0x1B80, 0x1BF3}, {0x1BFC, 0x1C37}, {0x1C3B, 0x1C49},
{0x1C4D, 0x1C88}, {0x1CC0, 0x1CC7}, {0x1CD0, 0x1CF6},
{0x1CF8, 0x1CF9}, {0x1D00, 0x1DF5}, {0x1DFB, 0x1F15},
{0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D},
{0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B},
{0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4},
{0x1FB6, 0x1FBC}, {0x1FBD, 0x1FBD}, {0x1FBE, 0x1FBE},
{0x1FBF, 0x1FC1}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC},
{0x1FCD, 0x1FCF}, {0x1FD0, 0x1FD3}, {0x1FD6, 0x1FDB},
{0x1FDD, 0x1FDF}, {0x1FE0, 0x1FEC}, {0x1FED, 0x1FEF},
{0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x1FFD, 0x1FFE},
{0x2000, 0x200A}, {0x200B, 0x200F}, {0x2011, 0x2012},
{0x2017, 0x2017}, {0x201A, 0x201A}, {0x201B, 0x201B},
{0x201E, 0x201E}, {0x201F, 0x201F}, {0x2023, 0x2023},
{0x2028, 0x2028}, {0x2029, 0x2029}, {0x202A, 0x202E},
{0x202F, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034},
{0x2036, 0x2038}, {0x2039, 0x2039}, {0x203A, 0x203A},
{0x203C, 0x203D}, {0x203F, 0x2040}, {0x2041, 0x2043},
{0x2044, 0x2044}, {0x2045, 0x2045}, {0x2046, 0x2046},
{0x2047, 0x2051}, {0x2052, 0x2052}, {0x2053, 0x2053},
{0x2054, 0x2054}, {0x2055, 0x205E}, {0x205F, 0x205F},
{0x2060, 0x2064}, {0x2066, 0x206F}, {0x2070, 0x2070},
{0x2071, 0x2071}, {0x2075, 0x2079}, {0x207A, 0x207C},
{0x207D, 0x207D}, {0x207E, 0x207E}, {0x2080, 0x2080},
{0x2085, 0x2089}, {0x208A, 0x208C}, {0x208D, 0x208D},
{0x208E, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8},
{0x20AA, 0x20AB}, {0x20AD, 0x20BE}, {0x20D0, 0x20DC},
{0x20DD, 0x20E0}, {0x20E1, 0x20E1}, {0x20E2, 0x20E4},
{0x20E5, 0x20F0}, {0x2100, 0x2101}, {0x2102, 0x2102},
{0x2104, 0x2104}, {0x2106, 0x2106}, {0x2107, 0x2107},
{0x2108, 0x2108}, {0x210A, 0x2112}, {0x2114, 0x2114},
{0x2115, 0x2115}, {0x2117, 0x2117}, {0x2118, 0x2118},
{0x2119, 0x211D}, {0x211E, 0x2120}, {0x2123, 0x2123},
{0x2124, 0x2124}, {0x2125, 0x2125}, {0x2127, 0x2127},
{0x2128, 0x2128}, {0x2129, 0x2129}, {0x212A, 0x212A},
{0x212C, 0x212D}, {0x212E, 0x212E}, {0x212F, 0x2134},
{0x2135, 0x2138}, {0x2139, 0x2139}, {0x213A, 0x213B},
{0x213C, 0x213F}, {0x2140, 0x2144}, {0x2145, 0x2149},
{0x214A, 0x214A}, {0x214B, 0x214B}, {0x214C, 0x214D},
{0x214E, 0x214E}, {0x214F, 0x214F}, {0x2150, 0x2152},
{0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB},
{0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE},
{0x2000, 0x200F}, {0x2011, 0x2012}, {0x2017, 0x2017},
{0x201A, 0x201B}, {0x201E, 0x201F}, {0x2023, 0x2023},
{0x2028, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034},
{0x2036, 0x203A}, {0x203C, 0x203D}, {0x203F, 0x2064},
{0x2066, 0x2071}, {0x2075, 0x207E}, {0x2080, 0x2080},
{0x2085, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8},
{0x20AA, 0x20AB}, {0x20AD, 0x20BE}, {0x20D0, 0x20F0},
{0x2100, 0x2102}, {0x2104, 0x2104}, {0x2106, 0x2108},
{0x210A, 0x2112}, {0x2114, 0x2115}, {0x2117, 0x2120},
{0x2123, 0x2125}, {0x2127, 0x212A}, {0x212C, 0x2152},
{0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F},
{0x217A, 0x2182}, {0x2183, 0x2184}, {0x2185, 0x2188},
{0x218A, 0x218B}, {0x219A, 0x219B}, {0x219C, 0x219F},
{0x21A0, 0x21A0}, {0x21A1, 0x21A2}, {0x21A3, 0x21A3},
{0x21A4, 0x21A5}, {0x21A6, 0x21A6}, {0x21A7, 0x21AD},
{0x21AE, 0x21AE}, {0x21AF, 0x21B7}, {0x21BA, 0x21CD},
{0x21CE, 0x21CF}, {0x21D0, 0x21D1}, {0x21D3, 0x21D3},
{0x21D5, 0x21E6}, {0x21E8, 0x21F3}, {0x21F4, 0x21FF},
{0x2201, 0x2201}, {0x2204, 0x2206}, {0x2209, 0x220A},
{0x220C, 0x220E}, {0x2210, 0x2210}, {0x2212, 0x2214},
{0x2216, 0x2219}, {0x221B, 0x221C}, {0x2221, 0x2222},
{0x2224, 0x2224}, {0x2226, 0x2226}, {0x222D, 0x222D},
{0x222F, 0x2233}, {0x2238, 0x223B}, {0x223E, 0x2247},
{0x2249, 0x224B}, {0x224D, 0x2251}, {0x2253, 0x225F},
{0x2262, 0x2263}, {0x2268, 0x2269}, {0x226C, 0x226D},
{0x2270, 0x2281}, {0x2284, 0x2285}, {0x2288, 0x2294},
{0x2296, 0x2298}, {0x229A, 0x22A4}, {0x22A6, 0x22BE},
{0x22C0, 0x22FF}, {0x2300, 0x2307}, {0x2308, 0x2308},
{0x2309, 0x2309}, {0x230A, 0x230A}, {0x230B, 0x230B},
{0x230C, 0x2311}, {0x2313, 0x2319}, {0x231C, 0x231F},
{0x2320, 0x2321}, {0x2322, 0x2328}, {0x232B, 0x237B},
{0x237C, 0x237C}, {0x237D, 0x239A}, {0x239B, 0x23B3},
{0x23B4, 0x23DB}, {0x23DC, 0x23E1}, {0x23E2, 0x23E8},
{0x23ED, 0x23EF}, {0x23F1, 0x23F2}, {0x23F4, 0x23FE},
{0x2400, 0x2426}, {0x2440, 0x244A}, {0x24EA, 0x24EA},
{0x254C, 0x254F}, {0x2574, 0x257F}, {0x2590, 0x2591},
{0x2596, 0x259F}, {0x25A2, 0x25A2}, {0x25AA, 0x25B1},
{0x25B4, 0x25B5}, {0x25B8, 0x25BB}, {0x25BE, 0x25BF},
{0x25C2, 0x25C5}, {0x25C9, 0x25CA}, {0x25CC, 0x25CD},
{0x25D2, 0x25E1}, {0x25E6, 0x25EE}, {0x25F0, 0x25F7},
{0x25F8, 0x25FC}, {0x25FF, 0x25FF}, {0x2600, 0x2604},
{0x217A, 0x2188}, {0x218A, 0x218B}, {0x219A, 0x21B7},
{0x21BA, 0x21D1}, {0x21D3, 0x21D3}, {0x21D5, 0x21E6},
{0x21E8, 0x21FF}, {0x2201, 0x2201}, {0x2204, 0x2206},
{0x2209, 0x220A}, {0x220C, 0x220E}, {0x2210, 0x2210},
{0x2212, 0x2214}, {0x2216, 0x2219}, {0x221B, 0x221C},
{0x2221, 0x2222}, {0x2224, 0x2224}, {0x2226, 0x2226},
{0x222D, 0x222D}, {0x222F, 0x2233}, {0x2238, 0x223B},
{0x223E, 0x2247}, {0x2249, 0x224B}, {0x224D, 0x2251},
{0x2253, 0x225F}, {0x2262, 0x2263}, {0x2268, 0x2269},
{0x226C, 0x226D}, {0x2270, 0x2281}, {0x2284, 0x2285},
{0x2288, 0x2294}, {0x2296, 0x2298}, {0x229A, 0x22A4},
{0x22A6, 0x22BE}, {0x22C0, 0x2311}, {0x2313, 0x2319},
{0x231C, 0x2328}, {0x232B, 0x23E8}, {0x23ED, 0x23EF},
{0x23F1, 0x23F2}, {0x23F4, 0x23FE}, {0x2400, 0x2426},
{0x2440, 0x244A}, {0x24EA, 0x24EA}, {0x254C, 0x254F},
{0x2574, 0x257F}, {0x2590, 0x2591}, {0x2596, 0x259F},
{0x25A2, 0x25A2}, {0x25AA, 0x25B1}, {0x25B4, 0x25B5},
{0x25B8, 0x25BB}, {0x25BE, 0x25BF}, {0x25C2, 0x25C5},
{0x25C9, 0x25CA}, {0x25CC, 0x25CD}, {0x25D2, 0x25E1},
{0x25E6, 0x25EE}, {0x25F0, 0x25FC}, {0x25FF, 0x2604},
{0x2607, 0x2608}, {0x260A, 0x260D}, {0x2610, 0x2613},
{0x2616, 0x261B}, {0x261D, 0x261D}, {0x261F, 0x263F},
{0x2641, 0x2641}, {0x2643, 0x2647}, {0x2654, 0x265F},
@ -799,256 +684,98 @@ var neutral = table{
{0x26E4, 0x26E7}, {0x2700, 0x2704}, {0x2706, 0x2709},
{0x270C, 0x2727}, {0x2729, 0x273C}, {0x273E, 0x274B},
{0x274D, 0x274D}, {0x274F, 0x2752}, {0x2756, 0x2756},
{0x2758, 0x2767}, {0x2768, 0x2768}, {0x2769, 0x2769},
{0x276A, 0x276A}, {0x276B, 0x276B}, {0x276C, 0x276C},
{0x276D, 0x276D}, {0x276E, 0x276E}, {0x276F, 0x276F},
{0x2770, 0x2770}, {0x2771, 0x2771}, {0x2772, 0x2772},
{0x2773, 0x2773}, {0x2774, 0x2774}, {0x2775, 0x2775},
{0x2780, 0x2793}, {0x2794, 0x2794}, {0x2798, 0x27AF},
{0x27B1, 0x27BE}, {0x27C0, 0x27C4}, {0x27C5, 0x27C5},
{0x27C6, 0x27C6}, {0x27C7, 0x27E5}, {0x27EE, 0x27EE},
{0x27EF, 0x27EF}, {0x27F0, 0x27FF}, {0x2800, 0x28FF},
{0x2900, 0x297F}, {0x2980, 0x2982}, {0x2983, 0x2983},
{0x2984, 0x2984}, {0x2987, 0x2987}, {0x2988, 0x2988},
{0x2989, 0x2989}, {0x298A, 0x298A}, {0x298B, 0x298B},
{0x298C, 0x298C}, {0x298D, 0x298D}, {0x298E, 0x298E},
{0x298F, 0x298F}, {0x2990, 0x2990}, {0x2991, 0x2991},
{0x2992, 0x2992}, {0x2993, 0x2993}, {0x2994, 0x2994},
{0x2995, 0x2995}, {0x2996, 0x2996}, {0x2997, 0x2997},
{0x2998, 0x2998}, {0x2999, 0x29D7}, {0x29D8, 0x29D8},
{0x29D9, 0x29D9}, {0x29DA, 0x29DA}, {0x29DB, 0x29DB},
{0x29DC, 0x29FB}, {0x29FC, 0x29FC}, {0x29FD, 0x29FD},
{0x29FE, 0x29FF}, {0x2A00, 0x2AFF}, {0x2B00, 0x2B1A},
{0x2B1D, 0x2B2F}, {0x2B30, 0x2B44}, {0x2B45, 0x2B46},
{0x2B47, 0x2B4C}, {0x2B4D, 0x2B4F}, {0x2B51, 0x2B54},
{0x2758, 0x2775}, {0x2780, 0x2794}, {0x2798, 0x27AF},
{0x27B1, 0x27BE}, {0x27C0, 0x27E5}, {0x27EE, 0x2984},
{0x2987, 0x2B1A}, {0x2B1D, 0x2B4F}, {0x2B51, 0x2B54},
{0x2B5A, 0x2B73}, {0x2B76, 0x2B95}, {0x2B98, 0x2BB9},
{0x2BBD, 0x2BC8}, {0x2BCA, 0x2BD1}, {0x2BEC, 0x2BEF},
{0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2C7B},
{0x2C7C, 0x2C7D}, {0x2C7E, 0x2C7F}, {0x2C80, 0x2CE4},
{0x2CE5, 0x2CEA}, {0x2CEB, 0x2CEE}, {0x2CEF, 0x2CF1},
{0x2CF2, 0x2CF3}, {0x2CF9, 0x2CFC}, {0x2CFD, 0x2CFD},
{0x2CFE, 0x2CFF}, {0x2D00, 0x2D25}, {0x2D27, 0x2D27},
{0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F},
{0x2D70, 0x2D70}, {0x2D7F, 0x2D7F}, {0x2D80, 0x2D96},
{0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2CF3},
{0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D},
{0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, {0x2D7F, 0x2D96},
{0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6},
{0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE},
{0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2DE0, 0x2DFF},
{0x2E00, 0x2E01}, {0x2E02, 0x2E02}, {0x2E03, 0x2E03},
{0x2E04, 0x2E04}, {0x2E05, 0x2E05}, {0x2E06, 0x2E08},
{0x2E09, 0x2E09}, {0x2E0A, 0x2E0A}, {0x2E0B, 0x2E0B},
{0x2E0C, 0x2E0C}, {0x2E0D, 0x2E0D}, {0x2E0E, 0x2E16},
{0x2E17, 0x2E17}, {0x2E18, 0x2E19}, {0x2E1A, 0x2E1A},
{0x2E1B, 0x2E1B}, {0x2E1C, 0x2E1C}, {0x2E1D, 0x2E1D},
{0x2E1E, 0x2E1F}, {0x2E20, 0x2E20}, {0x2E21, 0x2E21},
{0x2E22, 0x2E22}, {0x2E23, 0x2E23}, {0x2E24, 0x2E24},
{0x2E25, 0x2E25}, {0x2E26, 0x2E26}, {0x2E27, 0x2E27},
{0x2E28, 0x2E28}, {0x2E29, 0x2E29}, {0x2E2A, 0x2E2E},
{0x2E2F, 0x2E2F}, {0x2E30, 0x2E39}, {0x2E3A, 0x2E3B},
{0x2E3C, 0x2E3F}, {0x2E40, 0x2E40}, {0x2E41, 0x2E41},
{0x2E42, 0x2E42}, {0x2E43, 0x2E44}, {0x303F, 0x303F},
{0x4DC0, 0x4DFF}, {0xA4D0, 0xA4F7}, {0xA4F8, 0xA4FD},
{0xA4FE, 0xA4FF}, {0xA500, 0xA60B}, {0xA60C, 0xA60C},
{0xA60D, 0xA60F}, {0xA610, 0xA61F}, {0xA620, 0xA629},
{0xA62A, 0xA62B}, {0xA640, 0xA66D}, {0xA66E, 0xA66E},
{0xA66F, 0xA66F}, {0xA670, 0xA672}, {0xA673, 0xA673},
{0xA674, 0xA67D}, {0xA67E, 0xA67E}, {0xA67F, 0xA67F},
{0xA680, 0xA69B}, {0xA69C, 0xA69D}, {0xA69E, 0xA69F},
{0xA6A0, 0xA6E5}, {0xA6E6, 0xA6EF}, {0xA6F0, 0xA6F1},
{0xA6F2, 0xA6F7}, {0xA700, 0xA716}, {0xA717, 0xA71F},
{0xA720, 0xA721}, {0xA722, 0xA76F}, {0xA770, 0xA770},
{0xA771, 0xA787}, {0xA788, 0xA788}, {0xA789, 0xA78A},
{0xA78B, 0xA78E}, {0xA78F, 0xA78F}, {0xA790, 0xA7AE},
{0xA7B0, 0xA7B7}, {0xA7F7, 0xA7F7}, {0xA7F8, 0xA7F9},
{0xA7FA, 0xA7FA}, {0xA7FB, 0xA7FF}, {0xA800, 0xA801},
{0xA802, 0xA802}, {0xA803, 0xA805}, {0xA806, 0xA806},
{0xA807, 0xA80A}, {0xA80B, 0xA80B}, {0xA80C, 0xA822},
{0xA823, 0xA824}, {0xA825, 0xA826}, {0xA827, 0xA827},
{0xA828, 0xA82B}, {0xA830, 0xA835}, {0xA836, 0xA837},
{0xA838, 0xA838}, {0xA839, 0xA839}, {0xA840, 0xA873},
{0xA874, 0xA877}, {0xA880, 0xA881}, {0xA882, 0xA8B3},
{0xA8B4, 0xA8C3}, {0xA8C4, 0xA8C5}, {0xA8CE, 0xA8CF},
{0xA8D0, 0xA8D9}, {0xA8E0, 0xA8F1}, {0xA8F2, 0xA8F7},
{0xA8F8, 0xA8FA}, {0xA8FB, 0xA8FB}, {0xA8FC, 0xA8FC},
{0xA8FD, 0xA8FD}, {0xA900, 0xA909}, {0xA90A, 0xA925},
{0xA926, 0xA92D}, {0xA92E, 0xA92F}, {0xA930, 0xA946},
{0xA947, 0xA951}, {0xA952, 0xA953}, {0xA95F, 0xA95F},
{0xA980, 0xA982}, {0xA983, 0xA983}, {0xA984, 0xA9B2},
{0xA9B3, 0xA9B3}, {0xA9B4, 0xA9B5}, {0xA9B6, 0xA9B9},
{0xA9BA, 0xA9BB}, {0xA9BC, 0xA9BC}, {0xA9BD, 0xA9C0},
{0xA9C1, 0xA9CD}, {0xA9CF, 0xA9CF}, {0xA9D0, 0xA9D9},
{0xA9DE, 0xA9DF}, {0xA9E0, 0xA9E4}, {0xA9E5, 0xA9E5},
{0xA9E6, 0xA9E6}, {0xA9E7, 0xA9EF}, {0xA9F0, 0xA9F9},
{0xA9FA, 0xA9FE}, {0xAA00, 0xAA28}, {0xAA29, 0xAA2E},
{0xAA2F, 0xAA30}, {0xAA31, 0xAA32}, {0xAA33, 0xAA34},
{0xAA35, 0xAA36}, {0xAA40, 0xAA42}, {0xAA43, 0xAA43},
{0xAA44, 0xAA4B}, {0xAA4C, 0xAA4C}, {0xAA4D, 0xAA4D},
{0xAA50, 0xAA59}, {0xAA5C, 0xAA5F}, {0xAA60, 0xAA6F},
{0xAA70, 0xAA70}, {0xAA71, 0xAA76}, {0xAA77, 0xAA79},
{0xAA7A, 0xAA7A}, {0xAA7B, 0xAA7B}, {0xAA7C, 0xAA7C},
{0xAA7D, 0xAA7D}, {0xAA7E, 0xAA7F}, {0xAA80, 0xAAAF},
{0xAAB0, 0xAAB0}, {0xAAB1, 0xAAB1}, {0xAAB2, 0xAAB4},
{0xAAB5, 0xAAB6}, {0xAAB7, 0xAAB8}, {0xAAB9, 0xAABD},
{0xAABE, 0xAABF}, {0xAAC0, 0xAAC0}, {0xAAC1, 0xAAC1},
{0xAAC2, 0xAAC2}, {0xAADB, 0xAADC}, {0xAADD, 0xAADD},
{0xAADE, 0xAADF}, {0xAAE0, 0xAAEA}, {0xAAEB, 0xAAEB},
{0xAAEC, 0xAAED}, {0xAAEE, 0xAAEF}, {0xAAF0, 0xAAF1},
{0xAAF2, 0xAAF2}, {0xAAF3, 0xAAF4}, {0xAAF5, 0xAAF5},
{0xAAF6, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E},
{0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2DE0, 0x2E44},
{0x303F, 0x303F}, {0x4DC0, 0x4DFF}, {0xA4D0, 0xA62B},
{0xA640, 0xA6F7}, {0xA700, 0xA7AE}, {0xA7B0, 0xA7B7},
{0xA7F7, 0xA82B}, {0xA830, 0xA839}, {0xA840, 0xA877},
{0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, {0xA8E0, 0xA8FD},
{0xA900, 0xA953}, {0xA95F, 0xA95F}, {0xA980, 0xA9CD},
{0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36},
{0xAA40, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2},
{0xAADB, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E},
{0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E},
{0xAB30, 0xAB5A}, {0xAB5B, 0xAB5B}, {0xAB5C, 0xAB5F},
{0xAB60, 0xAB65}, {0xAB70, 0xABBF}, {0xABC0, 0xABE2},
{0xABE3, 0xABE4}, {0xABE5, 0xABE5}, {0xABE6, 0xABE7},
{0xABE8, 0xABE8}, {0xABE9, 0xABEA}, {0xABEB, 0xABEB},
{0xABEC, 0xABEC}, {0xABED, 0xABED}, {0xABF0, 0xABF9},
{0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDB7F},
{0xDB80, 0xDBFF}, {0xDC00, 0xDFFF}, {0xFB00, 0xFB06},
{0xFB13, 0xFB17}, {0xFB1D, 0xFB1D}, {0xFB1E, 0xFB1E},
{0xFB1F, 0xFB28}, {0xFB29, 0xFB29}, {0xFB2A, 0xFB36},
{0xAB30, 0xAB65}, {0xAB70, 0xABED}, {0xABF0, 0xABF9},
{0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDFFF},
{0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB36},
{0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41},
{0xFB43, 0xFB44}, {0xFB46, 0xFB4F}, {0xFB50, 0xFBB1},
{0xFBB2, 0xFBC1}, {0xFBD3, 0xFD3D}, {0xFD3E, 0xFD3E},
{0xFD3F, 0xFD3F}, {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7},
{0xFDF0, 0xFDFB}, {0xFDFC, 0xFDFC}, {0xFDFD, 0xFDFD},
{0xFB43, 0xFB44}, {0xFB46, 0xFBC1}, {0xFBD3, 0xFD3F},
{0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFD},
{0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC},
{0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFC, 0xFFFC},
{0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A},
{0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D},
{0x10080, 0x100FA}, {0x10100, 0x10102}, {0x10107, 0x10133},
{0x10137, 0x1013F}, {0x10140, 0x10174}, {0x10175, 0x10178},
{0x10179, 0x10189}, {0x1018A, 0x1018B}, {0x1018C, 0x1018E},
{0x10190, 0x1019B}, {0x101A0, 0x101A0}, {0x101D0, 0x101FC},
{0x101FD, 0x101FD}, {0x10280, 0x1029C}, {0x102A0, 0x102D0},
{0x102E0, 0x102E0}, {0x102E1, 0x102FB}, {0x10300, 0x1031F},
{0x10320, 0x10323}, {0x10330, 0x10340}, {0x10341, 0x10341},
{0x10342, 0x10349}, {0x1034A, 0x1034A}, {0x10350, 0x10375},
{0x10376, 0x1037A}, {0x10380, 0x1039D}, {0x1039F, 0x1039F},
{0x103A0, 0x103C3}, {0x103C8, 0x103CF}, {0x103D0, 0x103D0},
{0x103D1, 0x103D5}, {0x10400, 0x1044F}, {0x10450, 0x1047F},
{0x10480, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3},
{0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC}, {0x10000, 0x1000B},
{0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D},
{0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA},
{0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E},
{0x10190, 0x1019B}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD},
{0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB},
{0x10300, 0x10323}, {0x10330, 0x1034A}, {0x10350, 0x1037A},
{0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5},
{0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3},
{0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563},
{0x1056F, 0x1056F}, {0x10600, 0x10736}, {0x10740, 0x10755},
{0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808},
{0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C},
{0x1083F, 0x1083F}, {0x10840, 0x10855}, {0x10857, 0x10857},
{0x10858, 0x1085F}, {0x10860, 0x10876}, {0x10877, 0x10878},
{0x10879, 0x1087F}, {0x10880, 0x1089E}, {0x108A7, 0x108AF},
{0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x108FF},
{0x10900, 0x10915}, {0x10916, 0x1091B}, {0x1091F, 0x1091F},
{0x10920, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x1099F},
{0x109A0, 0x109B7}, {0x109BC, 0x109BD}, {0x109BE, 0x109BF},
{0x109C0, 0x109CF}, {0x109D2, 0x109FF}, {0x10A00, 0x10A00},
{0x10A01, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F},
{0x10A10, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A33},
{0x10A38, 0x10A3A}, {0x10A3F, 0x10A3F}, {0x10A40, 0x10A47},
{0x10A50, 0x10A58}, {0x10A60, 0x10A7C}, {0x10A7D, 0x10A7E},
{0x10A7F, 0x10A7F}, {0x10A80, 0x10A9C}, {0x10A9D, 0x10A9F},
{0x10AC0, 0x10AC7}, {0x10AC8, 0x10AC8}, {0x10AC9, 0x10AE4},
{0x10AE5, 0x10AE6}, {0x10AEB, 0x10AEF}, {0x10AF0, 0x10AF6},
{0x10B00, 0x10B35}, {0x10B39, 0x10B3F}, {0x10B40, 0x10B55},
{0x10B58, 0x10B5F}, {0x10B60, 0x10B72}, {0x10B78, 0x10B7F},
{0x10B80, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF},
{0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF},
{0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B},
{0x1091F, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x109B7},
{0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06},
{0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A33},
{0x10A38, 0x10A3A}, {0x10A3F, 0x10A47}, {0x10A50, 0x10A58},
{0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6},
{0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72},
{0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF},
{0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2},
{0x10CFA, 0x10CFF}, {0x10E60, 0x10E7E}, {0x11000, 0x11000},
{0x11001, 0x11001}, {0x11002, 0x11002}, {0x11003, 0x11037},
{0x11038, 0x11046}, {0x11047, 0x1104D}, {0x11052, 0x11065},
{0x11066, 0x1106F}, {0x1107F, 0x1107F}, {0x11080, 0x11081},
{0x11082, 0x11082}, {0x11083, 0x110AF}, {0x110B0, 0x110B2},
{0x110B3, 0x110B6}, {0x110B7, 0x110B8}, {0x110B9, 0x110BA},
{0x110BB, 0x110BC}, {0x110BD, 0x110BD}, {0x110BE, 0x110C1},
{0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11100, 0x11102},
{0x11103, 0x11126}, {0x11127, 0x1112B}, {0x1112C, 0x1112C},
{0x1112D, 0x11134}, {0x11136, 0x1113F}, {0x11140, 0x11143},
{0x11150, 0x11172}, {0x11173, 0x11173}, {0x11174, 0x11175},
{0x11176, 0x11176}, {0x11180, 0x11181}, {0x11182, 0x11182},
{0x11183, 0x111B2}, {0x111B3, 0x111B5}, {0x111B6, 0x111BE},
{0x111BF, 0x111C0}, {0x111C1, 0x111C4}, {0x111C5, 0x111C9},
{0x111CA, 0x111CC}, {0x111CD, 0x111CD}, {0x111D0, 0x111D9},
{0x111DA, 0x111DA}, {0x111DB, 0x111DB}, {0x111DC, 0x111DC},
{0x111DD, 0x111DF}, {0x111E1, 0x111F4}, {0x11200, 0x11211},
{0x11213, 0x1122B}, {0x1122C, 0x1122E}, {0x1122F, 0x11231},
{0x11232, 0x11233}, {0x11234, 0x11234}, {0x11235, 0x11235},
{0x11236, 0x11237}, {0x11238, 0x1123D}, {0x1123E, 0x1123E},
{0x10CFA, 0x10CFF}, {0x10E60, 0x10E7E}, {0x11000, 0x1104D},
{0x11052, 0x1106F}, {0x1107F, 0x110C1}, {0x110D0, 0x110E8},
{0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11143},
{0x11150, 0x11176}, {0x11180, 0x111CD}, {0x111D0, 0x111DF},
{0x111E1, 0x111F4}, {0x11200, 0x11211}, {0x11213, 0x1123E},
{0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D},
{0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112A9, 0x112A9},
{0x112B0, 0x112DE}, {0x112DF, 0x112DF}, {0x112E0, 0x112E2},
{0x112E3, 0x112EA}, {0x112F0, 0x112F9}, {0x11300, 0x11301},
{0x11302, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310},
{0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333},
{0x11335, 0x11339}, {0x1133C, 0x1133C}, {0x1133D, 0x1133D},
{0x1133E, 0x1133F}, {0x11340, 0x11340}, {0x11341, 0x11344},
{0x1128F, 0x1129D}, {0x1129F, 0x112A9}, {0x112B0, 0x112EA},
{0x112F0, 0x112F9}, {0x11300, 0x11303}, {0x11305, 0x1130C},
{0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330},
{0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133C, 0x11344},
{0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11350, 0x11350},
{0x11357, 0x11357}, {0x1135D, 0x11361}, {0x11362, 0x11363},
{0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11400, 0x11434},
{0x11435, 0x11437}, {0x11438, 0x1143F}, {0x11440, 0x11441},
{0x11442, 0x11444}, {0x11445, 0x11445}, {0x11446, 0x11446},
{0x11447, 0x1144A}, {0x1144B, 0x1144F}, {0x11450, 0x11459},
{0x1145B, 0x1145B}, {0x1145D, 0x1145D}, {0x11480, 0x114AF},
{0x114B0, 0x114B2}, {0x114B3, 0x114B8}, {0x114B9, 0x114B9},
{0x114BA, 0x114BA}, {0x114BB, 0x114BE}, {0x114BF, 0x114C0},
{0x114C1, 0x114C1}, {0x114C2, 0x114C3}, {0x114C4, 0x114C5},
{0x114C6, 0x114C6}, {0x114C7, 0x114C7}, {0x114D0, 0x114D9},
{0x11580, 0x115AE}, {0x115AF, 0x115B1}, {0x115B2, 0x115B5},
{0x115B8, 0x115BB}, {0x115BC, 0x115BD}, {0x115BE, 0x115BE},
{0x115BF, 0x115C0}, {0x115C1, 0x115D7}, {0x115D8, 0x115DB},
{0x115DC, 0x115DD}, {0x11600, 0x1162F}, {0x11630, 0x11632},
{0x11633, 0x1163A}, {0x1163B, 0x1163C}, {0x1163D, 0x1163D},
{0x1163E, 0x1163E}, {0x1163F, 0x11640}, {0x11641, 0x11643},
{0x11644, 0x11644}, {0x11650, 0x11659}, {0x11660, 0x1166C},
{0x11680, 0x116AA}, {0x116AB, 0x116AB}, {0x116AC, 0x116AC},
{0x116AD, 0x116AD}, {0x116AE, 0x116AF}, {0x116B0, 0x116B5},
{0x116B6, 0x116B6}, {0x116B7, 0x116B7}, {0x116C0, 0x116C9},
{0x11700, 0x11719}, {0x1171D, 0x1171F}, {0x11720, 0x11721},
{0x11722, 0x11725}, {0x11726, 0x11726}, {0x11727, 0x1172B},
{0x11730, 0x11739}, {0x1173A, 0x1173B}, {0x1173C, 0x1173E},
{0x1173F, 0x1173F}, {0x118A0, 0x118DF}, {0x118E0, 0x118E9},
{0x118EA, 0x118F2}, {0x118FF, 0x118FF}, {0x11AC0, 0x11AF8},
{0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, {0x11C2F, 0x11C2F},
{0x11C30, 0x11C36}, {0x11C38, 0x11C3D}, {0x11C3E, 0x11C3E},
{0x11C3F, 0x11C3F}, {0x11C40, 0x11C40}, {0x11C41, 0x11C45},
{0x11C50, 0x11C59}, {0x11C5A, 0x11C6C}, {0x11C70, 0x11C71},
{0x11C72, 0x11C8F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CA9},
{0x11CAA, 0x11CB0}, {0x11CB1, 0x11CB1}, {0x11CB2, 0x11CB3},
{0x11CB4, 0x11CB4}, {0x11CB5, 0x11CB6}, {0x12000, 0x12399},
{0x11357, 0x11357}, {0x1135D, 0x11363}, {0x11366, 0x1136C},
{0x11370, 0x11374}, {0x11400, 0x11459}, {0x1145B, 0x1145B},
{0x1145D, 0x1145D}, {0x11480, 0x114C7}, {0x114D0, 0x114D9},
{0x11580, 0x115B5}, {0x115B8, 0x115DD}, {0x11600, 0x11644},
{0x11650, 0x11659}, {0x11660, 0x1166C}, {0x11680, 0x116B7},
{0x116C0, 0x116C9}, {0x11700, 0x11719}, {0x1171D, 0x1172B},
{0x11730, 0x1173F}, {0x118A0, 0x118F2}, {0x118FF, 0x118FF},
{0x11AC0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C36},
{0x11C38, 0x11C45}, {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F},
{0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x12000, 0x12399},
{0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543},
{0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38},
{0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16A6F},
{0x16AD0, 0x16AED}, {0x16AF0, 0x16AF4}, {0x16AF5, 0x16AF5},
{0x16B00, 0x16B2F}, {0x16B30, 0x16B36}, {0x16B37, 0x16B3B},
{0x16B3C, 0x16B3F}, {0x16B40, 0x16B43}, {0x16B44, 0x16B44},
{0x16B45, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61},
{0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16F00, 0x16F44},
{0x16F50, 0x16F50}, {0x16F51, 0x16F7E}, {0x16F8F, 0x16F92},
{0x16F93, 0x16F9F}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C},
{0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BC9C},
{0x1BC9D, 0x1BC9E}, {0x1BC9F, 0x1BC9F}, {0x1BCA0, 0x1BCA3},
{0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D164},
{0x1D165, 0x1D166}, {0x1D167, 0x1D169}, {0x1D16A, 0x1D16C},
{0x1D16D, 0x1D172}, {0x1D173, 0x1D17A}, {0x1D17B, 0x1D182},
{0x1D183, 0x1D184}, {0x1D185, 0x1D18B}, {0x1D18C, 0x1D1A9},
{0x1D1AA, 0x1D1AD}, {0x1D1AE, 0x1D1E8}, {0x1D200, 0x1D241},
{0x1D242, 0x1D244}, {0x1D245, 0x1D245}, {0x1D300, 0x1D356},
{0x1D360, 0x1D371}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C},
{0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6},
{0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB},
{0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A},
{0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539},
{0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546},
{0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0},
{0x1D6C1, 0x1D6C1}, {0x1D6C2, 0x1D6DA}, {0x1D6DB, 0x1D6DB},
{0x1D6DC, 0x1D6FA}, {0x1D6FB, 0x1D6FB}, {0x1D6FC, 0x1D714},
{0x1D715, 0x1D715}, {0x1D716, 0x1D734}, {0x1D735, 0x1D735},
{0x1D736, 0x1D74E}, {0x1D74F, 0x1D74F}, {0x1D750, 0x1D76E},
{0x1D76F, 0x1D76F}, {0x1D770, 0x1D788}, {0x1D789, 0x1D789},
{0x1D78A, 0x1D7A8}, {0x1D7A9, 0x1D7A9}, {0x1D7AA, 0x1D7C2},
{0x1D7C3, 0x1D7C3}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF},
{0x1D800, 0x1D9FF}, {0x1DA00, 0x1DA36}, {0x1DA37, 0x1DA3A},
{0x1DA3B, 0x1DA6C}, {0x1DA6D, 0x1DA74}, {0x1DA75, 0x1DA75},
{0x1DA76, 0x1DA83}, {0x1DA84, 0x1DA84}, {0x1DA85, 0x1DA86},
{0x1DA87, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF},
{0x1E000, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021},
{0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E800, 0x1E8C4},
{0x1E8C7, 0x1E8CF}, {0x1E8D0, 0x1E8D6}, {0x1E900, 0x1E943},
{0x1E944, 0x1E94A}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F},
{0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, {0x16B00, 0x16B45},
{0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77},
{0x16B7D, 0x16B8F}, {0x16F00, 0x16F44}, {0x16F50, 0x16F7E},
{0x16F8F, 0x16F9F}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C},
{0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3},
{0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D1E8},
{0x1D200, 0x1D245}, {0x1D300, 0x1D356}, {0x1D360, 0x1D371},
{0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F},
{0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC},
{0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3},
{0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514},
{0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E},
{0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550},
{0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B},
{0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006},
{0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024},
{0x1E026, 0x1E02A}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6},
{0x1E900, 0x1E94A}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F},
{0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22},
{0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32},
{0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B},
@ -1079,12 +806,16 @@ var neutral = table{
// Condition have flag EastAsianWidth whether the current locale is CJK or not.
type Condition struct {
EastAsianWidth bool
EastAsianWidth bool
ZeroWidthJoiner bool
}
// NewCondition return new instance of Condition which is current locale.
func NewCondition() *Condition {
return &Condition{EastAsianWidth}
return &Condition{
EastAsianWidth: EastAsianWidth,
ZeroWidthJoiner: ZeroWidthJoiner,
}
}
// RuneWidth returns the number of cells in r.
@ -1102,14 +833,37 @@ func (c *Condition) RuneWidth(r rune) int {
}
}
// StringWidth return width as you can see
func (c *Condition) StringWidth(s string) (width int) {
func (c *Condition) stringWidth(s string) (width int) {
for _, r := range []rune(s) {
width += c.RuneWidth(r)
}
return width
}
func (c *Condition) stringWidthZeroJoiner(s string) (width int) {
r1, r2 := rune(0), rune(0)
for _, r := range []rune(s) {
if r == 0xFE0E || r == 0xFE0F {
continue
}
w := c.RuneWidth(r)
if r2 == 0x200D && inTables(r, emoji) && inTables(r1, emoji) {
w = 0
}
width += w
r1, r2 = r2, r
}
return width
}
// StringWidth return width as you can see
func (c *Condition) StringWidth(s string) (width int) {
if c.ZeroWidthJoiner {
return c.stringWidthZeroJoiner(s)
}
return c.stringWidth(s)
}
// Truncate return string truncated with w cells
func (c *Condition) Truncate(s string, w int, tail string) string {
if c.StringWidth(s) <= w {

@ -0,0 +1,8 @@
// +build appengine
package runewidth
// IsEastAsian return true if the current locale is CJK
func IsEastAsian() bool {
return false
}

@ -1,4 +1,5 @@
// +build js
// +build !appengine
package runewidth

@ -1,4 +1,6 @@
// +build !windows,!js
// +build !windows
// +build !js
// +build !appengine
package runewidth

@ -1,275 +0,0 @@
package runewidth
import (
"sort"
"testing"
)
var _ sort.Interface = (*table)(nil)
func (t table) Len() int {
return len(t)
}
func (t table) Less(i, j int) bool {
return t[i].first < t[j].first
}
func (t *table) Swap(i, j int) {
(*t)[i], (*t)[j] = (*t)[j], (*t)[i]
}
var tables = []table{
private,
nonprint,
combining,
doublewidth,
ambiguous,
emoji,
notassigned,
neutral,
}
func TestSorted(t *testing.T) {
for _, tbl := range tables {
if !sort.IsSorted(&tbl) {
t.Errorf("not sorted")
}
}
}
var runewidthtests = []struct {
in rune
out int
eaout int
}{
{'世', 2, 2},
{'界', 2, 2},
{'セ', 1, 1},
{'カ', 1, 1},
{'イ', 1, 1},
{'☆', 1, 2}, // double width in ambiguous
{'\x00', 0, 0},
{'\x01', 0, 0},
{'\u0300', 0, 0},
}
func TestRuneWidth(t *testing.T) {
c := NewCondition()
for _, tt := range runewidthtests {
if out := c.RuneWidth(tt.in); out != tt.out {
t.Errorf("RuneWidth(%q) = %d, want %d", tt.in, out, tt.out)
}
}
c.EastAsianWidth = true
for _, tt := range runewidthtests {
if out := c.RuneWidth(tt.in); out != tt.eaout {
t.Errorf("RuneWidth(%q) = %d, want %d", tt.in, out, tt.eaout)
}
}
}
var isambiguouswidthtests = []struct {
in rune
out bool
}{
{'世', false},
{'■', true},
{'界', false},
{'○', true},
{'㈱', false},
{'①', true},
{'②', true},
{'③', true},
{'④', true},
{'⑤', true},
{'⑥', true},
{'⑦', true},
{'⑧', true},
{'⑨', true},
{'⑩', true},
{'⑪', true},
{'⑫', true},
{'⑬', true},
{'⑭', true},
{'⑮', true},
{'⑯', true},
{'⑰', true},
{'⑱', true},
{'⑲', true},
{'⑳', true},
{'☆', true},
}
func TestIsAmbiguousWidth(t *testing.T) {
for _, tt := range isambiguouswidthtests {
if out := IsAmbiguousWidth(tt.in); out != tt.out {
t.Errorf("IsAmbiguousWidth(%q) = %v, want %v", tt.in, out, tt.out)
}
}
}
var stringwidthtests = []struct {
in string
out int
eaout int
}{
{"■㈱の世界①", 10, 12},
{"スター☆", 7, 8},
{"つのだ☆HIRO", 11, 12},
}
func TestStringWidth(t *testing.T) {
c := NewCondition()
for _, tt := range stringwidthtests {
if out := c.StringWidth(tt.in); out != tt.out {
t.Errorf("StringWidth(%q) = %q, want %q", tt.in, out, tt.out)
}
}
c.EastAsianWidth = true
for _, tt := range stringwidthtests {
if out := c.StringWidth(tt.in); out != tt.eaout {
t.Errorf("StringWidth(%q) = %q, want %q", tt.in, out, tt.eaout)
}
}
}
func TestStringWidthInvalid(t *testing.T) {
s := "こんにちわ\x00世界"
if out := StringWidth(s); out != 14 {
t.Errorf("StringWidth(%q) = %q, want %q", s, out, 14)
}
}
func TestTruncateSmaller(t *testing.T) {
s := "あいうえお"
expected := "あいうえお"
if out := Truncate(s, 10, "..."); out != expected {
t.Errorf("Truncate(%q) = %q, want %q", s, out, expected)
}
}
func TestTruncate(t *testing.T) {
s := "あいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおおおおお"
expected := "あいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおお..."
out := Truncate(s, 80, "...")
if out != expected {
t.Errorf("Truncate(%q) = %q, want %q", s, out, expected)
}
width := StringWidth(out)
if width != 79 {
t.Errorf("width of Truncate(%q) should be %d, but %d", s, 79, width)
}
}
func TestTruncateFit(t *testing.T) {
s := "aあいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおおおおお"
expected := "aあいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおお..."
out := Truncate(s, 80, "...")
if out != expected {
t.Errorf("Truncate(%q) = %q, want %q", s, out, expected)
}
width := StringWidth(out)
if width != 80 {
t.Errorf("width of Truncate(%q) should be %d, but %d", s, 80, width)
}
}
func TestTruncateJustFit(t *testing.T) {
s := "あいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおおおお"
expected := "あいうえおあいうえおえおおおおおおおおおおおおおおおおおおおおおおおおおおおおお"
out := Truncate(s, 80, "...")
if out != expected {
t.Errorf("Truncate(%q) = %q, want %q", s, out, expected)
}
width := StringWidth(out)
if width != 80 {
t.Errorf("width of Truncate(%q) should be %d, but %d", s, 80, width)
}
}
func TestWrap(t *testing.T) {
s := `/
123456789012345678901234567890
END`
expected := `
/
123456789012345678901234567890
END`
if out := Wrap(s, 30); out != expected {
t.Errorf("Wrap(%q) = %q, want %q", s, out, expected)
}
}
func TestTruncateNoNeeded(t *testing.T) {
s := "あいうえおあい"
expected := "あいうえおあい"
if out := Truncate(s, 80, "..."); out != expected {
t.Errorf("Truncate(%q) = %q, want %q", s, out, expected)
}
}
var isneutralwidthtests = []struct {
in rune
out bool
}{
{'→', false},
{'┊', false},
{'┈', false},
{'', false},
{'└', false},
{'⣀', true},
{'⣀', true},
}
func TestIsNeutralWidth(t *testing.T) {
for _, tt := range isneutralwidthtests {
if out := IsNeutralWidth(tt.in); out != tt.out {
t.Errorf("IsNeutralWidth(%q) = %v, want %v", tt.in, out, tt.out)
}
}
}
func TestFillLeft(t *testing.T) {
s := "あxいうえお"
expected := " あxいうえお"
if out := FillLeft(s, 15); out != expected {
t.Errorf("FillLeft(%q) = %q, want %q", s, out, expected)
}
}
func TestFillLeftFit(t *testing.T) {
s := "あいうえお"
expected := "あいうえお"
if out := FillLeft(s, 10); out != expected {
t.Errorf("FillLeft(%q) = %q, want %q", s, out, expected)
}
}
func TestFillRight(t *testing.T) {
s := "あxいうえお"
expected := "あxいうえお "
if out := FillRight(s, 15); out != expected {
t.Errorf("FillRight(%q) = %q, want %q", s, out, expected)
}
}
func TestFillRightFit(t *testing.T) {
s := "あいうえお"
expected := "あいうえお"
if out := FillRight(s, 10); out != expected {
t.Errorf("FillRight(%q) = %q, want %q", s, out, expected)
}
}

@ -1,3 +1,6 @@
// +build windows
// +build !appengine
package runewidth
import (

@ -1,10 +0,0 @@
# Treat all files in this repo as binary, with no git magic updating
# line endings. Windows users contributing to Go will need to use a
# modern version of git and editors capable of LF line endings.
#
# We'll prevent accidental CRLF line endings from entering the repo
# via the git-review gofmt checks.
#
# See golang.org/issue/9281
* -text

@ -1,2 +0,0 @@
# Add no patterns to .hgignore except for files generated by the build.
last-change

@ -1,26 +0,0 @@
# Contributing to Go
Go is an open source project.
It is the work of hundreds of contributors. We appreciate your help!
## Filing issues
When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions:
1. What version of Go are you using (`go version`)?
2. What operating system and processor architecture are you using?
3. What did you do?
4. What did you expect to see?
5. What did you see instead?
General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker.
The gophers there will answer or ask you to file an issue if you've tripped over a bug.
## Contributing code
Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html)
before sending patches.
Unless otherwise noted, the Go source files are distributed under
the BSD-style license found in the LICENSE file.

16
vendor/golang.org/x/net/README.md generated vendored

@ -1,16 +0,0 @@
# Go Networking
This repository holds supplementary Go networking libraries.
## Download/Install
The easiest way to install is to run `go get -u golang.org/x/net`. You can
also manually git clone the repository to `$GOPATH/src/golang.org/x/net`.
## Report Issues / Send Patches
This repository uses Gerrit for code changes. To learn how to submit
changes to this repository, see https://golang.org/doc/contribute.html.
The main issue tracker for the net repository is located at
https://github.com/golang/go/issues. Prefix your issue with "x/net:" in the
subject line, so it is easy to find.

@ -1,41 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
import "fmt"
// Assemble converts insts into raw instructions suitable for loading
// into a BPF virtual machine.
//
// Currently, no optimization is attempted, the assembled program flow
// is exactly as provided.
func Assemble(insts []Instruction) ([]RawInstruction, error) {
ret := make([]RawInstruction, len(insts))
var err error
for i, inst := range insts {
ret[i], err = inst.Assemble()
if err != nil {
return nil, fmt.Errorf("assembling instruction %d: %s", i+1, err)
}
}
return ret, nil
}
// Disassemble attempts to parse raw back into
// Instructions. Unrecognized RawInstructions are assumed to be an
// extension not implemented by this package, and are passed through
// unchanged to the output. The allDecoded value reports whether insts
// contains no RawInstructions.
func Disassemble(raw []RawInstruction) (insts []Instruction, allDecoded bool) {
insts = make([]Instruction, len(raw))
allDecoded = true
for i, r := range raw {
insts[i] = r.Disassemble()
if _, ok := insts[i].(RawInstruction); ok {
allDecoded = false
}
}
return insts, allDecoded
}

@ -1,218 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
// A Register is a register of the BPF virtual machine.
type Register uint16
const (
// RegA is the accumulator register. RegA is always the
// destination register of ALU operations.
RegA Register = iota
// RegX is the indirection register, used by LoadIndirect
// operations.
RegX
)
// An ALUOp is an arithmetic or logic operation.
type ALUOp uint16
// ALU binary operation types.
const (
ALUOpAdd ALUOp = iota << 4
ALUOpSub
ALUOpMul
ALUOpDiv
ALUOpOr
ALUOpAnd
ALUOpShiftLeft
ALUOpShiftRight
aluOpNeg // Not exported because it's the only unary ALU operation, and gets its own instruction type.
ALUOpMod
ALUOpXor
)
// A JumpTest is a comparison operator used in conditional jumps.
type JumpTest uint16
// Supported operators for conditional jumps.
const (
// K == A
JumpEqual JumpTest = iota
// K != A
JumpNotEqual
// K > A
JumpGreaterThan
// K < A
JumpLessThan
// K >= A
JumpGreaterOrEqual
// K <= A
JumpLessOrEqual
// K & A != 0
JumpBitsSet
// K & A == 0
JumpBitsNotSet
)
// An Extension is a function call provided by the kernel that
// performs advanced operations that are expensive or impossible
// within the BPF virtual machine.
//
// Extensions are only implemented by the Linux kernel.
//
// TODO: should we prune this list? Some of these extensions seem
// either broken or near-impossible to use correctly, whereas other
// (len, random, ifindex) are quite useful.
type Extension int
// Extension functions available in the Linux kernel.
const (
// extOffset is the negative maximum number of instructions used
// to load instructions by overloading the K argument.
extOffset = -0x1000
// ExtLen returns the length of the packet.
ExtLen Extension = 1
// ExtProto returns the packet's L3 protocol type.
ExtProto Extension = 0
// ExtType returns the packet's type (skb->pkt_type in the kernel)
//
// TODO: better documentation. How nice an API do we want to
// provide for these esoteric extensions?
ExtType Extension = 4
// ExtPayloadOffset returns the offset of the packet payload, or
// the first protocol header that the kernel does not know how to
// parse.
ExtPayloadOffset Extension = 52
// ExtInterfaceIndex returns the index of the interface on which
// the packet was received.
ExtInterfaceIndex Extension = 8
// ExtNetlinkAttr returns the netlink attribute of type X at
// offset A.
ExtNetlinkAttr Extension = 12
// ExtNetlinkAttrNested returns the nested netlink attribute of
// type X at offset A.
ExtNetlinkAttrNested Extension = 16
// ExtMark returns the packet's mark value.
ExtMark Extension = 20
// ExtQueue returns the packet's assigned hardware queue.
ExtQueue Extension = 24
// ExtLinkLayerType returns the packet's hardware address type
// (e.g. Ethernet, Infiniband).
ExtLinkLayerType Extension = 28
// ExtRXHash returns the packets receive hash.
//
// TODO: figure out what this rxhash actually is.
ExtRXHash Extension = 32
// ExtCPUID returns the ID of the CPU processing the current
// packet.
ExtCPUID Extension = 36
// ExtVLANTag returns the packet's VLAN tag.
ExtVLANTag Extension = 44
// ExtVLANTagPresent returns non-zero if the packet has a VLAN
// tag.
//
// TODO: I think this might be a lie: it reads bit 0x1000 of the
// VLAN header, which changed meaning in recent revisions of the
// spec - this extension may now return meaningless information.
ExtVLANTagPresent Extension = 48
// ExtVLANProto returns 0x8100 if the frame has a VLAN header,
// 0x88a8 if the frame has a "Q-in-Q" double VLAN header, or some
// other value if no VLAN information is present.
ExtVLANProto Extension = 60
// ExtRand returns a uniformly random uint32.
ExtRand Extension = 56
)
// The following gives names to various bit patterns used in opcode construction.
const (
opMaskCls uint16 = 0x7
// opClsLoad masks
opMaskLoadDest = 0x01
opMaskLoadWidth = 0x18
opMaskLoadMode = 0xe0
// opClsALU
opMaskOperandSrc = 0x08
opMaskOperator = 0xf0
// opClsJump
opMaskJumpConst = 0x0f
opMaskJumpCond = 0xf0
)
const (
// +---------------+-----------------+---+---+---+
// | AddrMode (3b) | LoadWidth (2b) | 0 | 0 | 0 |
// +---------------+-----------------+---+---+---+
opClsLoadA uint16 = iota
// +---------------+-----------------+---+---+---+
// | AddrMode (3b) | LoadWidth (2b) | 0 | 0 | 1 |
// +---------------+-----------------+---+---+---+
opClsLoadX
// +---+---+---+---+---+---+---+---+
// | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 |
// +---+---+---+---+---+---+---+---+
opClsStoreA
// +---+---+---+---+---+---+---+---+
// | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 |
// +---+---+---+---+---+---+---+---+
opClsStoreX
// +---------------+-----------------+---+---+---+
// | Operator (4b) | OperandSrc (1b) | 1 | 0 | 0 |
// +---------------+-----------------+---+---+---+
opClsALU
// +-----------------------------+---+---+---+---+
// | TestOperator (4b) | 0 | 1 | 0 | 1 |
// +-----------------------------+---+---+---+---+
opClsJump
// +---+-------------------------+---+---+---+---+
// | 0 | 0 | 0 | RetSrc (1b) | 0 | 1 | 1 | 0 |
// +---+-------------------------+---+---+---+---+
opClsReturn
// +---+-------------------------+---+---+---+---+
// | 0 | 0 | 0 | TXAorTAX (1b) | 0 | 1 | 1 | 1 |
// +---+-------------------------+---+---+---+---+
opClsMisc
)
const (
opAddrModeImmediate uint16 = iota << 5
opAddrModeAbsolute
opAddrModeIndirect
opAddrModeScratch
opAddrModePacketLen // actually an extension, not an addressing mode.
opAddrModeMemShift
)
const (
opLoadWidth4 uint16 = iota << 3
opLoadWidth2
opLoadWidth1
)
// Operator defined by ALUOp*
const (
opALUSrcConstant uint16 = iota << 3
opALUSrcX
)
const (
opJumpAlways = iota << 4
opJumpEqual
opJumpGT
opJumpGE
opJumpSet
)
const (
opRetSrcConstant uint16 = iota << 4
opRetSrcA
)
const (
opMiscTAX = 0x00
opMiscTXA = 0x80
)

@ -1,82 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package bpf implements marshaling and unmarshaling of programs for the
Berkeley Packet Filter virtual machine, and provides a Go implementation
of the virtual machine.
BPF's main use is to specify a packet filter for network taps, so that
the kernel doesn't have to expensively copy every packet it sees to
userspace. However, it's been repurposed to other areas where running
user code in-kernel is needed. For example, Linux's seccomp uses BPF
to apply security policies to system calls. For simplicity, this
documentation refers only to packets, but other uses of BPF have their
own data payloads.
BPF programs run in a restricted virtual machine. It has almost no
access to kernel functions, and while conditional branches are
allowed, they can only jump forwards, to guarantee that there are no
infinite loops.
The virtual machine
The BPF VM is an accumulator machine. Its main register, called
register A, is an implicit source and destination in all arithmetic
and logic operations. The machine also has 16 scratch registers for
temporary storage, and an indirection register (register X) for
indirect memory access. All registers are 32 bits wide.
Each run of a BPF program is given one packet, which is placed in the
VM's read-only "main memory". LoadAbsolute and LoadIndirect
instructions can fetch up to 32 bits at a time into register A for
examination.
The goal of a BPF program is to produce and return a verdict (uint32),
which tells the kernel what to do with the packet. In the context of
packet filtering, the returned value is the number of bytes of the
packet to forward to userspace, or 0 to ignore the packet. Other
contexts like seccomp define their own return values.
In order to simplify programs, attempts to read past the end of the
packet terminate the program execution with a verdict of 0 (ignore
packet). This means that the vast majority of BPF programs don't need
to do any explicit bounds checking.
In addition to the bytes of the packet, some BPF programs have access
to extensions, which are essentially calls to kernel utility
functions. Currently, the only extensions supported by this package
are the Linux packet filter extensions.
Examples
This packet filter selects all ARP packets.
bpf.Assemble([]bpf.Instruction{
// Load "EtherType" field from the ethernet header.
bpf.LoadAbsolute{Off: 12, Size: 2},
// Skip over the next instruction if EtherType is not ARP.
bpf.JumpIf{Cond: bpf.JumpNotEqual, Val: 0x0806, SkipTrue: 1},
// Verdict is "send up to 4k of the packet to userspace."
bpf.RetConstant{Val: 4096},
// Verdict is "ignore packet."
bpf.RetConstant{Val: 0},
})
This packet filter captures a random 1% sample of traffic.
bpf.Assemble([]bpf.Instruction{
// Get a 32-bit random number from the Linux kernel.
bpf.LoadExtension{Num: bpf.ExtRand},
// 1% dice roll?
bpf.JumpIf{Cond: bpf.JumpLessThan, Val: 2^32/100, SkipFalse: 1},
// Capture.
bpf.RetConstant{Val: 4096},
// Ignore.
bpf.RetConstant{Val: 0},
})
*/
package bpf // import "golang.org/x/net/bpf"

@ -1,704 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
import "fmt"
// An Instruction is one instruction executed by the BPF virtual
// machine.
type Instruction interface {
// Assemble assembles the Instruction into a RawInstruction.
Assemble() (RawInstruction, error)
}
// A RawInstruction is a raw BPF virtual machine instruction.
type RawInstruction struct {
// Operation to execute.
Op uint16
// For conditional jump instructions, the number of instructions
// to skip if the condition is true/false.
Jt uint8
Jf uint8
// Constant parameter. The meaning depends on the Op.
K uint32
}
// Assemble implements the Instruction Assemble method.
func (ri RawInstruction) Assemble() (RawInstruction, error) { return ri, nil }
// Disassemble parses ri into an Instruction and returns it. If ri is
// not recognized by this package, ri itself is returned.
func (ri RawInstruction) Disassemble() Instruction {
switch ri.Op & opMaskCls {
case opClsLoadA, opClsLoadX:
reg := Register(ri.Op & opMaskLoadDest)
sz := 0
switch ri.Op & opMaskLoadWidth {
case opLoadWidth4:
sz = 4
case opLoadWidth2:
sz = 2
case opLoadWidth1:
sz = 1
default:
return ri
}
switch ri.Op & opMaskLoadMode {
case opAddrModeImmediate:
if sz != 4 {
return ri
}
return LoadConstant{Dst: reg, Val: ri.K}
case opAddrModeScratch:
if sz != 4 || ri.K > 15 {
return ri
}
return LoadScratch{Dst: reg, N: int(ri.K)}
case opAddrModeAbsolute:
if ri.K > extOffset+0xffffffff {
return LoadExtension{Num: Extension(-extOffset + ri.K)}
}
return LoadAbsolute{Size: sz, Off: ri.K}
case opAddrModeIndirect:
return LoadIndirect{Size: sz, Off: ri.K}
case opAddrModePacketLen:
if sz != 4 {
return ri
}
return LoadExtension{Num: ExtLen}
case opAddrModeMemShift:
return LoadMemShift{Off: ri.K}
default:
return ri
}
case opClsStoreA:
if ri.Op != opClsStoreA || ri.K > 15 {
return ri
}
return StoreScratch{Src: RegA, N: int(ri.K)}
case opClsStoreX:
if ri.Op != opClsStoreX || ri.K > 15 {
return ri
}
return StoreScratch{Src: RegX, N: int(ri.K)}
case opClsALU:
switch op := ALUOp(ri.Op & opMaskOperator); op {
case ALUOpAdd, ALUOpSub, ALUOpMul, ALUOpDiv, ALUOpOr, ALUOpAnd, ALUOpShiftLeft, ALUOpShiftRight, ALUOpMod, ALUOpXor:
if ri.Op&opMaskOperandSrc != 0 {
return ALUOpX{Op: op}
}
return ALUOpConstant{Op: op, Val: ri.K}
case aluOpNeg:
return NegateA{}
default:
return ri
}
case opClsJump:
if ri.Op&opMaskJumpConst != opClsJump {
return ri
}
switch ri.Op & opMaskJumpCond {
case opJumpAlways:
return Jump{Skip: ri.K}
case opJumpEqual:
if ri.Jt == 0 {
return JumpIf{
Cond: JumpNotEqual,
Val: ri.K,
SkipTrue: ri.Jf,
SkipFalse: 0,
}
}
return JumpIf{
Cond: JumpEqual,
Val: ri.K,
SkipTrue: ri.Jt,
SkipFalse: ri.Jf,
}
case opJumpGT:
if ri.Jt == 0 {
return JumpIf{
Cond: JumpLessOrEqual,
Val: ri.K,
SkipTrue: ri.Jf,
SkipFalse: 0,
}
}
return JumpIf{
Cond: JumpGreaterThan,
Val: ri.K,
SkipTrue: ri.Jt,
SkipFalse: ri.Jf,
}
case opJumpGE:
if ri.Jt == 0 {
return JumpIf{
Cond: JumpLessThan,
Val: ri.K,
SkipTrue: ri.Jf,
SkipFalse: 0,
}
}
return JumpIf{
Cond: JumpGreaterOrEqual,
Val: ri.K,
SkipTrue: ri.Jt,
SkipFalse: ri.Jf,
}
case opJumpSet:
return JumpIf{
Cond: JumpBitsSet,
Val: ri.K,
SkipTrue: ri.Jt,
SkipFalse: ri.Jf,
}
default:
return ri
}
case opClsReturn:
switch ri.Op {
case opClsReturn | opRetSrcA:
return RetA{}
case opClsReturn | opRetSrcConstant:
return RetConstant{Val: ri.K}
default:
return ri
}
case opClsMisc:
switch ri.Op {
case opClsMisc | opMiscTAX:
return TAX{}
case opClsMisc | opMiscTXA:
return TXA{}
default:
return ri
}
default:
panic("unreachable") // switch is exhaustive on the bit pattern
}
}
// LoadConstant loads Val into register Dst.
type LoadConstant struct {
Dst Register
Val uint32
}
// Assemble implements the Instruction Assemble method.
func (a LoadConstant) Assemble() (RawInstruction, error) {
return assembleLoad(a.Dst, 4, opAddrModeImmediate, a.Val)
}
// String returns the instruction in assembler notation.
func (a LoadConstant) String() string {
switch a.Dst {
case RegA:
return fmt.Sprintf("ld #%d", a.Val)
case RegX:
return fmt.Sprintf("ldx #%d", a.Val)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// LoadScratch loads scratch[N] into register Dst.
type LoadScratch struct {
Dst Register
N int // 0-15
}
// Assemble implements the Instruction Assemble method.
func (a LoadScratch) Assemble() (RawInstruction, error) {
if a.N < 0 || a.N > 15 {
return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N)
}
return assembleLoad(a.Dst, 4, opAddrModeScratch, uint32(a.N))
}
// String returns the instruction in assembler notation.
func (a LoadScratch) String() string {
switch a.Dst {
case RegA:
return fmt.Sprintf("ld M[%d]", a.N)
case RegX:
return fmt.Sprintf("ldx M[%d]", a.N)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// LoadAbsolute loads packet[Off:Off+Size] as an integer value into
// register A.
type LoadAbsolute struct {
Off uint32
Size int // 1, 2 or 4
}
// Assemble implements the Instruction Assemble method.
func (a LoadAbsolute) Assemble() (RawInstruction, error) {
return assembleLoad(RegA, a.Size, opAddrModeAbsolute, a.Off)
}
// String returns the instruction in assembler notation.
func (a LoadAbsolute) String() string {
switch a.Size {
case 1: // byte
return fmt.Sprintf("ldb [%d]", a.Off)
case 2: // half word
return fmt.Sprintf("ldh [%d]", a.Off)
case 4: // word
if a.Off > extOffset+0xffffffff {
return LoadExtension{Num: Extension(a.Off + 0x1000)}.String()
}
return fmt.Sprintf("ld [%d]", a.Off)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// LoadIndirect loads packet[X+Off:X+Off+Size] as an integer value
// into register A.
type LoadIndirect struct {
Off uint32
Size int // 1, 2 or 4
}
// Assemble implements the Instruction Assemble method.
func (a LoadIndirect) Assemble() (RawInstruction, error) {
return assembleLoad(RegA, a.Size, opAddrModeIndirect, a.Off)
}
// String returns the instruction in assembler notation.
func (a LoadIndirect) String() string {
switch a.Size {
case 1: // byte
return fmt.Sprintf("ldb [x + %d]", a.Off)
case 2: // half word
return fmt.Sprintf("ldh [x + %d]", a.Off)
case 4: // word
return fmt.Sprintf("ld [x + %d]", a.Off)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// LoadMemShift multiplies the first 4 bits of the byte at packet[Off]
// by 4 and stores the result in register X.
//
// This instruction is mainly useful to load into X the length of an
// IPv4 packet header in a single instruction, rather than have to do
// the arithmetic on the header's first byte by hand.
type LoadMemShift struct {
Off uint32
}
// Assemble implements the Instruction Assemble method.
func (a LoadMemShift) Assemble() (RawInstruction, error) {
return assembleLoad(RegX, 1, opAddrModeMemShift, a.Off)
}
// String returns the instruction in assembler notation.
func (a LoadMemShift) String() string {
return fmt.Sprintf("ldx 4*([%d]&0xf)", a.Off)
}
// LoadExtension invokes a linux-specific extension and stores the
// result in register A.
type LoadExtension struct {
Num Extension
}
// Assemble implements the Instruction Assemble method.
func (a LoadExtension) Assemble() (RawInstruction, error) {
if a.Num == ExtLen {
return assembleLoad(RegA, 4, opAddrModePacketLen, 0)
}
return assembleLoad(RegA, 4, opAddrModeAbsolute, uint32(extOffset+a.Num))
}
// String returns the instruction in assembler notation.
func (a LoadExtension) String() string {
switch a.Num {
case ExtLen:
return "ld #len"
case ExtProto:
return "ld #proto"
case ExtType:
return "ld #type"
case ExtPayloadOffset:
return "ld #poff"
case ExtInterfaceIndex:
return "ld #ifidx"
case ExtNetlinkAttr:
return "ld #nla"
case ExtNetlinkAttrNested:
return "ld #nlan"
case ExtMark:
return "ld #mark"
case ExtQueue:
return "ld #queue"
case ExtLinkLayerType:
return "ld #hatype"
case ExtRXHash:
return "ld #rxhash"
case ExtCPUID:
return "ld #cpu"
case ExtVLANTag:
return "ld #vlan_tci"
case ExtVLANTagPresent:
return "ld #vlan_avail"
case ExtVLANProto:
return "ld #vlan_tpid"
case ExtRand:
return "ld #rand"
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// StoreScratch stores register Src into scratch[N].
type StoreScratch struct {
Src Register
N int // 0-15
}
// Assemble implements the Instruction Assemble method.
func (a StoreScratch) Assemble() (RawInstruction, error) {
if a.N < 0 || a.N > 15 {
return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N)
}
var op uint16
switch a.Src {
case RegA:
op = opClsStoreA
case RegX:
op = opClsStoreX
default:
return RawInstruction{}, fmt.Errorf("invalid source register %v", a.Src)
}
return RawInstruction{
Op: op,
K: uint32(a.N),
}, nil
}
// String returns the instruction in assembler notation.
func (a StoreScratch) String() string {
switch a.Src {
case RegA:
return fmt.Sprintf("st M[%d]", a.N)
case RegX:
return fmt.Sprintf("stx M[%d]", a.N)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// ALUOpConstant executes A = A <Op> Val.
type ALUOpConstant struct {
Op ALUOp
Val uint32
}
// Assemble implements the Instruction Assemble method.
func (a ALUOpConstant) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsALU | opALUSrcConstant | uint16(a.Op),
K: a.Val,
}, nil
}
// String returns the instruction in assembler notation.
func (a ALUOpConstant) String() string {
switch a.Op {
case ALUOpAdd:
return fmt.Sprintf("add #%d", a.Val)
case ALUOpSub:
return fmt.Sprintf("sub #%d", a.Val)
case ALUOpMul:
return fmt.Sprintf("mul #%d", a.Val)
case ALUOpDiv:
return fmt.Sprintf("div #%d", a.Val)
case ALUOpMod:
return fmt.Sprintf("mod #%d", a.Val)
case ALUOpAnd:
return fmt.Sprintf("and #%d", a.Val)
case ALUOpOr:
return fmt.Sprintf("or #%d", a.Val)
case ALUOpXor:
return fmt.Sprintf("xor #%d", a.Val)
case ALUOpShiftLeft:
return fmt.Sprintf("lsh #%d", a.Val)
case ALUOpShiftRight:
return fmt.Sprintf("rsh #%d", a.Val)
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// ALUOpX executes A = A <Op> X
type ALUOpX struct {
Op ALUOp
}
// Assemble implements the Instruction Assemble method.
func (a ALUOpX) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsALU | opALUSrcX | uint16(a.Op),
}, nil
}
// String returns the instruction in assembler notation.
func (a ALUOpX) String() string {
switch a.Op {
case ALUOpAdd:
return "add x"
case ALUOpSub:
return "sub x"
case ALUOpMul:
return "mul x"
case ALUOpDiv:
return "div x"
case ALUOpMod:
return "mod x"
case ALUOpAnd:
return "and x"
case ALUOpOr:
return "or x"
case ALUOpXor:
return "xor x"
case ALUOpShiftLeft:
return "lsh x"
case ALUOpShiftRight:
return "rsh x"
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
// NegateA executes A = -A.
type NegateA struct{}
// Assemble implements the Instruction Assemble method.
func (a NegateA) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsALU | uint16(aluOpNeg),
}, nil
}
// String returns the instruction in assembler notation.
func (a NegateA) String() string {
return fmt.Sprintf("neg")
}
// Jump skips the following Skip instructions in the program.
type Jump struct {
Skip uint32
}
// Assemble implements the Instruction Assemble method.
func (a Jump) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsJump | opJumpAlways,
K: a.Skip,
}, nil
}
// String returns the instruction in assembler notation.
func (a Jump) String() string {
return fmt.Sprintf("ja %d", a.Skip)
}
// JumpIf skips the following Skip instructions in the program if A
// <Cond> Val is true.
type JumpIf struct {
Cond JumpTest
Val uint32
SkipTrue uint8
SkipFalse uint8
}
// Assemble implements the Instruction Assemble method.
func (a JumpIf) Assemble() (RawInstruction, error) {
var (
cond uint16
flip bool
)
switch a.Cond {
case JumpEqual:
cond = opJumpEqual
case JumpNotEqual:
cond, flip = opJumpEqual, true
case JumpGreaterThan:
cond = opJumpGT
case JumpLessThan:
cond, flip = opJumpGE, true
case JumpGreaterOrEqual:
cond = opJumpGE
case JumpLessOrEqual:
cond, flip = opJumpGT, true
case JumpBitsSet:
cond = opJumpSet
case JumpBitsNotSet:
cond, flip = opJumpSet, true
default:
return RawInstruction{}, fmt.Errorf("unknown JumpTest %v", a.Cond)
}
jt, jf := a.SkipTrue, a.SkipFalse
if flip {
jt, jf = jf, jt
}
return RawInstruction{
Op: opClsJump | cond,
Jt: jt,
Jf: jf,
K: a.Val,
}, nil
}
// String returns the instruction in assembler notation.
func (a JumpIf) String() string {
switch a.Cond {
// K == A
case JumpEqual:
return conditionalJump(a, "jeq", "jneq")
// K != A
case JumpNotEqual:
return fmt.Sprintf("jneq #%d,%d", a.Val, a.SkipTrue)
// K > A
case JumpGreaterThan:
return conditionalJump(a, "jgt", "jle")
// K < A
case JumpLessThan:
return fmt.Sprintf("jlt #%d,%d", a.Val, a.SkipTrue)
// K >= A
case JumpGreaterOrEqual:
return conditionalJump(a, "jge", "jlt")
// K <= A
case JumpLessOrEqual:
return fmt.Sprintf("jle #%d,%d", a.Val, a.SkipTrue)
// K & A != 0
case JumpBitsSet:
if a.SkipFalse > 0 {
return fmt.Sprintf("jset #%d,%d,%d", a.Val, a.SkipTrue, a.SkipFalse)
}
return fmt.Sprintf("jset #%d,%d", a.Val, a.SkipTrue)
// K & A == 0, there is no assembler instruction for JumpBitNotSet, use JumpBitSet and invert skips
case JumpBitsNotSet:
return JumpIf{Cond: JumpBitsSet, SkipTrue: a.SkipFalse, SkipFalse: a.SkipTrue, Val: a.Val}.String()
default:
return fmt.Sprintf("unknown instruction: %#v", a)
}
}
func conditionalJump(inst JumpIf, positiveJump, negativeJump string) string {
if inst.SkipTrue > 0 {
if inst.SkipFalse > 0 {
return fmt.Sprintf("%s #%d,%d,%d", positiveJump, inst.Val, inst.SkipTrue, inst.SkipFalse)
}
return fmt.Sprintf("%s #%d,%d", positiveJump, inst.Val, inst.SkipTrue)
}
return fmt.Sprintf("%s #%d,%d", negativeJump, inst.Val, inst.SkipFalse)
}
// RetA exits the BPF program, returning the value of register A.
type RetA struct{}
// Assemble implements the Instruction Assemble method.
func (a RetA) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsReturn | opRetSrcA,
}, nil
}
// String returns the instruction in assembler notation.
func (a RetA) String() string {
return fmt.Sprintf("ret a")
}
// RetConstant exits the BPF program, returning a constant value.
type RetConstant struct {
Val uint32
}
// Assemble implements the Instruction Assemble method.
func (a RetConstant) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsReturn | opRetSrcConstant,
K: a.Val,
}, nil
}
// String returns the instruction in assembler notation.
func (a RetConstant) String() string {
return fmt.Sprintf("ret #%d", a.Val)
}
// TXA copies the value of register X to register A.
type TXA struct{}
// Assemble implements the Instruction Assemble method.
func (a TXA) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsMisc | opMiscTXA,
}, nil
}
// String returns the instruction in assembler notation.
func (a TXA) String() string {
return fmt.Sprintf("txa")
}
// TAX copies the value of register A to register X.
type TAX struct{}
// Assemble implements the Instruction Assemble method.
func (a TAX) Assemble() (RawInstruction, error) {
return RawInstruction{
Op: opClsMisc | opMiscTAX,
}, nil
}
// String returns the instruction in assembler notation.
func (a TAX) String() string {
return fmt.Sprintf("tax")
}
func assembleLoad(dst Register, loadSize int, mode uint16, k uint32) (RawInstruction, error) {
var (
cls uint16
sz uint16
)
switch dst {
case RegA:
cls = opClsLoadA
case RegX:
cls = opClsLoadX
default:
return RawInstruction{}, fmt.Errorf("invalid target register %v", dst)
}
switch loadSize {
case 1:
sz = opLoadWidth1
case 2:
sz = opLoadWidth2
case 4:
sz = opLoadWidth4
default:
return RawInstruction{}, fmt.Errorf("invalid load byte length %d", sz)
}
return RawInstruction{
Op: cls | sz | mode,
K: k,
}, nil
}

@ -1,525 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
import (
"fmt"
"io/ioutil"
"reflect"
"strconv"
"strings"
"testing"
)
// This is a direct translation of the program in
// testdata/all_instructions.txt.
var allInstructions = []Instruction{
LoadConstant{Dst: RegA, Val: 42},
LoadConstant{Dst: RegX, Val: 42},
LoadScratch{Dst: RegA, N: 3},
LoadScratch{Dst: RegX, N: 3},
LoadAbsolute{Off: 42, Size: 1},
LoadAbsolute{Off: 42, Size: 2},
LoadAbsolute{Off: 42, Size: 4},
LoadIndirect{Off: 42, Size: 1},
LoadIndirect{Off: 42, Size: 2},
LoadIndirect{Off: 42, Size: 4},
LoadMemShift{Off: 42},
LoadExtension{Num: ExtLen},
LoadExtension{Num: ExtProto},
LoadExtension{Num: ExtType},
LoadExtension{Num: ExtRand},
StoreScratch{Src: RegA, N: 3},
StoreScratch{Src: RegX, N: 3},
ALUOpConstant{Op: ALUOpAdd, Val: 42},
ALUOpConstant{Op: ALUOpSub, Val: 42},
ALUOpConstant{Op: ALUOpMul, Val: 42},
ALUOpConstant{Op: ALUOpDiv, Val: 42},
ALUOpConstant{Op: ALUOpOr, Val: 42},
ALUOpConstant{Op: ALUOpAnd, Val: 42},
ALUOpConstant{Op: ALUOpShiftLeft, Val: 42},
ALUOpConstant{Op: ALUOpShiftRight, Val: 42},
ALUOpConstant{Op: ALUOpMod, Val: 42},
ALUOpConstant{Op: ALUOpXor, Val: 42},
ALUOpX{Op: ALUOpAdd},
ALUOpX{Op: ALUOpSub},
ALUOpX{Op: ALUOpMul},
ALUOpX{Op: ALUOpDiv},
ALUOpX{Op: ALUOpOr},
ALUOpX{Op: ALUOpAnd},
ALUOpX{Op: ALUOpShiftLeft},
ALUOpX{Op: ALUOpShiftRight},
ALUOpX{Op: ALUOpMod},
ALUOpX{Op: ALUOpXor},
NegateA{},
Jump{Skip: 10},
JumpIf{Cond: JumpEqual, Val: 42, SkipTrue: 8, SkipFalse: 9},
JumpIf{Cond: JumpNotEqual, Val: 42, SkipTrue: 8},
JumpIf{Cond: JumpLessThan, Val: 42, SkipTrue: 7},
JumpIf{Cond: JumpLessOrEqual, Val: 42, SkipTrue: 6},
JumpIf{Cond: JumpGreaterThan, Val: 42, SkipTrue: 4, SkipFalse: 5},
JumpIf{Cond: JumpGreaterOrEqual, Val: 42, SkipTrue: 3, SkipFalse: 4},
JumpIf{Cond: JumpBitsSet, Val: 42, SkipTrue: 2, SkipFalse: 3},
TAX{},
TXA{},
RetA{},
RetConstant{Val: 42},
}
var allInstructionsExpected = "testdata/all_instructions.bpf"
// Check that we produce the same output as the canonical bpf_asm
// linux kernel tool.
func TestInterop(t *testing.T) {
out, err := Assemble(allInstructions)
if err != nil {
t.Fatalf("assembly of allInstructions program failed: %s", err)
}
t.Logf("Assembled program is %d instructions long", len(out))
bs, err := ioutil.ReadFile(allInstructionsExpected)
if err != nil {
t.Fatalf("reading %s: %s", allInstructionsExpected, err)
}
// First statement is the number of statements, last statement is
// empty. We just ignore both and rely on slice length.
stmts := strings.Split(string(bs), ",")
if len(stmts)-2 != len(out) {
t.Fatalf("test program lengths don't match: %s has %d, Go implementation has %d", allInstructionsExpected, len(stmts)-2, len(allInstructions))
}
for i, stmt := range stmts[1 : len(stmts)-2] {
nums := strings.Split(stmt, " ")
if len(nums) != 4 {
t.Fatalf("malformed instruction %d in %s: %s", i+1, allInstructionsExpected, stmt)
}
actual := out[i]
op, err := strconv.ParseUint(nums[0], 10, 16)
if err != nil {
t.Fatalf("malformed opcode %s in instruction %d of %s", nums[0], i+1, allInstructionsExpected)
}
if actual.Op != uint16(op) {
t.Errorf("opcode mismatch on instruction %d (%#v): got 0x%02x, want 0x%02x", i+1, allInstructions[i], actual.Op, op)
}
jt, err := strconv.ParseUint(nums[1], 10, 8)
if err != nil {
t.Fatalf("malformed jt offset %s in instruction %d of %s", nums[1], i+1, allInstructionsExpected)
}
if actual.Jt != uint8(jt) {
t.Errorf("jt mismatch on instruction %d (%#v): got %d, want %d", i+1, allInstructions[i], actual.Jt, jt)
}
jf, err := strconv.ParseUint(nums[2], 10, 8)
if err != nil {
t.Fatalf("malformed jf offset %s in instruction %d of %s", nums[2], i+1, allInstructionsExpected)
}
if actual.Jf != uint8(jf) {
t.Errorf("jf mismatch on instruction %d (%#v): got %d, want %d", i+1, allInstructions[i], actual.Jf, jf)
}
k, err := strconv.ParseUint(nums[3], 10, 32)
if err != nil {
t.Fatalf("malformed constant %s in instruction %d of %s", nums[3], i+1, allInstructionsExpected)
}
if actual.K != uint32(k) {
t.Errorf("constant mismatch on instruction %d (%#v): got %d, want %d", i+1, allInstructions[i], actual.K, k)
}
}
}
// Check that assembly and disassembly match each other.
func TestAsmDisasm(t *testing.T) {
prog1, err := Assemble(allInstructions)
if err != nil {
t.Fatalf("assembly of allInstructions program failed: %s", err)
}
t.Logf("Assembled program is %d instructions long", len(prog1))
got, allDecoded := Disassemble(prog1)
if !allDecoded {
t.Errorf("Disassemble(Assemble(allInstructions)) produced unrecognized instructions:")
for i, inst := range got {
if r, ok := inst.(RawInstruction); ok {
t.Logf(" insn %d, %#v --> %#v", i+1, allInstructions[i], r)
}
}
}
if len(allInstructions) != len(got) {
t.Fatalf("disassembly changed program size: %d insns before, %d insns after", len(allInstructions), len(got))
}
if !reflect.DeepEqual(allInstructions, got) {
t.Errorf("program mutated by disassembly:")
for i := range got {
if !reflect.DeepEqual(allInstructions[i], got[i]) {
t.Logf(" insn %d, s: %#v, p1: %#v, got: %#v", i+1, allInstructions[i], prog1[i], got[i])
}
}
}
}
type InvalidInstruction struct{}
func (a InvalidInstruction) Assemble() (RawInstruction, error) {
return RawInstruction{}, fmt.Errorf("Invalid Instruction")
}
func (a InvalidInstruction) String() string {
return fmt.Sprintf("unknown instruction: %#v", a)
}
func TestString(t *testing.T) {
testCases := []struct {
instruction Instruction
assembler string
}{
{
instruction: LoadConstant{Dst: RegA, Val: 42},
assembler: "ld #42",
},
{
instruction: LoadConstant{Dst: RegX, Val: 42},
assembler: "ldx #42",
},
{
instruction: LoadConstant{Dst: 0xffff, Val: 42},
assembler: "unknown instruction: bpf.LoadConstant{Dst:0xffff, Val:0x2a}",
},
{
instruction: LoadScratch{Dst: RegA, N: 3},
assembler: "ld M[3]",
},
{
instruction: LoadScratch{Dst: RegX, N: 3},
assembler: "ldx M[3]",
},
{
instruction: LoadScratch{Dst: 0xffff, N: 3},
assembler: "unknown instruction: bpf.LoadScratch{Dst:0xffff, N:3}",
},
{
instruction: LoadAbsolute{Off: 42, Size: 1},
assembler: "ldb [42]",
},
{
instruction: LoadAbsolute{Off: 42, Size: 2},
assembler: "ldh [42]",
},
{
instruction: LoadAbsolute{Off: 42, Size: 4},
assembler: "ld [42]",
},
{
instruction: LoadAbsolute{Off: 42, Size: -1},
assembler: "unknown instruction: bpf.LoadAbsolute{Off:0x2a, Size:-1}",
},
{
instruction: LoadIndirect{Off: 42, Size: 1},
assembler: "ldb [x + 42]",
},
{
instruction: LoadIndirect{Off: 42, Size: 2},
assembler: "ldh [x + 42]",
},
{
instruction: LoadIndirect{Off: 42, Size: 4},
assembler: "ld [x + 42]",
},
{
instruction: LoadIndirect{Off: 42, Size: -1},
assembler: "unknown instruction: bpf.LoadIndirect{Off:0x2a, Size:-1}",
},
{
instruction: LoadMemShift{Off: 42},
assembler: "ldx 4*([42]&0xf)",
},
{
instruction: LoadExtension{Num: ExtLen},
assembler: "ld #len",
},
{
instruction: LoadExtension{Num: ExtProto},
assembler: "ld #proto",
},
{
instruction: LoadExtension{Num: ExtType},
assembler: "ld #type",
},
{
instruction: LoadExtension{Num: ExtPayloadOffset},
assembler: "ld #poff",
},
{
instruction: LoadExtension{Num: ExtInterfaceIndex},
assembler: "ld #ifidx",
},
{
instruction: LoadExtension{Num: ExtNetlinkAttr},
assembler: "ld #nla",
},
{
instruction: LoadExtension{Num: ExtNetlinkAttrNested},
assembler: "ld #nlan",
},
{
instruction: LoadExtension{Num: ExtMark},
assembler: "ld #mark",
},
{
instruction: LoadExtension{Num: ExtQueue},
assembler: "ld #queue",
},
{
instruction: LoadExtension{Num: ExtLinkLayerType},
assembler: "ld #hatype",
},
{
instruction: LoadExtension{Num: ExtRXHash},
assembler: "ld #rxhash",
},
{
instruction: LoadExtension{Num: ExtCPUID},
assembler: "ld #cpu",
},
{
instruction: LoadExtension{Num: ExtVLANTag},
assembler: "ld #vlan_tci",
},
{
instruction: LoadExtension{Num: ExtVLANTagPresent},
assembler: "ld #vlan_avail",
},
{
instruction: LoadExtension{Num: ExtVLANProto},
assembler: "ld #vlan_tpid",
},
{
instruction: LoadExtension{Num: ExtRand},
assembler: "ld #rand",
},
{
instruction: LoadAbsolute{Off: 0xfffff038, Size: 4},
assembler: "ld #rand",
},
{
instruction: LoadExtension{Num: 0xfff},
assembler: "unknown instruction: bpf.LoadExtension{Num:4095}",
},
{
instruction: StoreScratch{Src: RegA, N: 3},
assembler: "st M[3]",
},
{
instruction: StoreScratch{Src: RegX, N: 3},
assembler: "stx M[3]",
},
{
instruction: StoreScratch{Src: 0xffff, N: 3},
assembler: "unknown instruction: bpf.StoreScratch{Src:0xffff, N:3}",
},
{
instruction: ALUOpConstant{Op: ALUOpAdd, Val: 42},
assembler: "add #42",
},
{
instruction: ALUOpConstant{Op: ALUOpSub, Val: 42},
assembler: "sub #42",
},
{
instruction: ALUOpConstant{Op: ALUOpMul, Val: 42},
assembler: "mul #42",
},
{
instruction: ALUOpConstant{Op: ALUOpDiv, Val: 42},
assembler: "div #42",
},
{
instruction: ALUOpConstant{Op: ALUOpOr, Val: 42},
assembler: "or #42",
},
{
instruction: ALUOpConstant{Op: ALUOpAnd, Val: 42},
assembler: "and #42",
},
{
instruction: ALUOpConstant{Op: ALUOpShiftLeft, Val: 42},
assembler: "lsh #42",
},
{
instruction: ALUOpConstant{Op: ALUOpShiftRight, Val: 42},
assembler: "rsh #42",
},
{
instruction: ALUOpConstant{Op: ALUOpMod, Val: 42},
assembler: "mod #42",
},
{
instruction: ALUOpConstant{Op: ALUOpXor, Val: 42},
assembler: "xor #42",
},
{
instruction: ALUOpConstant{Op: 0xffff, Val: 42},
assembler: "unknown instruction: bpf.ALUOpConstant{Op:0xffff, Val:0x2a}",
},
{
instruction: ALUOpX{Op: ALUOpAdd},
assembler: "add x",
},
{
instruction: ALUOpX{Op: ALUOpSub},
assembler: "sub x",
},
{
instruction: ALUOpX{Op: ALUOpMul},
assembler: "mul x",
},
{
instruction: ALUOpX{Op: ALUOpDiv},
assembler: "div x",
},
{
instruction: ALUOpX{Op: ALUOpOr},
assembler: "or x",
},
{
instruction: ALUOpX{Op: ALUOpAnd},
assembler: "and x",
},
{
instruction: ALUOpX{Op: ALUOpShiftLeft},
assembler: "lsh x",
},
{
instruction: ALUOpX{Op: ALUOpShiftRight},
assembler: "rsh x",
},
{
instruction: ALUOpX{Op: ALUOpMod},
assembler: "mod x",
},
{
instruction: ALUOpX{Op: ALUOpXor},
assembler: "xor x",
},
{
instruction: ALUOpX{Op: 0xffff},
assembler: "unknown instruction: bpf.ALUOpX{Op:0xffff}",
},
{
instruction: NegateA{},
assembler: "neg",
},
{
instruction: Jump{Skip: 10},
assembler: "ja 10",
},
{
instruction: JumpIf{Cond: JumpEqual, Val: 42, SkipTrue: 8, SkipFalse: 9},
assembler: "jeq #42,8,9",
},
{
instruction: JumpIf{Cond: JumpEqual, Val: 42, SkipTrue: 8},
assembler: "jeq #42,8",
},
{
instruction: JumpIf{Cond: JumpEqual, Val: 42, SkipFalse: 8},
assembler: "jneq #42,8",
},
{
instruction: JumpIf{Cond: JumpNotEqual, Val: 42, SkipTrue: 8},
assembler: "jneq #42,8",
},
{
instruction: JumpIf{Cond: JumpLessThan, Val: 42, SkipTrue: 7},
assembler: "jlt #42,7",
},
{
instruction: JumpIf{Cond: JumpLessOrEqual, Val: 42, SkipTrue: 6},
assembler: "jle #42,6",
},
{
instruction: JumpIf{Cond: JumpGreaterThan, Val: 42, SkipTrue: 4, SkipFalse: 5},
assembler: "jgt #42,4,5",
},
{
instruction: JumpIf{Cond: JumpGreaterThan, Val: 42, SkipTrue: 4},
assembler: "jgt #42,4",
},
{
instruction: JumpIf{Cond: JumpGreaterOrEqual, Val: 42, SkipTrue: 3, SkipFalse: 4},
assembler: "jge #42,3,4",
},
{
instruction: JumpIf{Cond: JumpGreaterOrEqual, Val: 42, SkipTrue: 3},
assembler: "jge #42,3",
},
{
instruction: JumpIf{Cond: JumpBitsSet, Val: 42, SkipTrue: 2, SkipFalse: 3},
assembler: "jset #42,2,3",
},
{
instruction: JumpIf{Cond: JumpBitsSet, Val: 42, SkipTrue: 2},
assembler: "jset #42,2",
},
{
instruction: JumpIf{Cond: JumpBitsNotSet, Val: 42, SkipTrue: 2, SkipFalse: 3},
assembler: "jset #42,3,2",
},
{
instruction: JumpIf{Cond: JumpBitsNotSet, Val: 42, SkipTrue: 2},
assembler: "jset #42,0,2",
},
{
instruction: JumpIf{Cond: 0xffff, Val: 42, SkipTrue: 1, SkipFalse: 2},
assembler: "unknown instruction: bpf.JumpIf{Cond:0xffff, Val:0x2a, SkipTrue:0x1, SkipFalse:0x2}",
},
{
instruction: TAX{},
assembler: "tax",
},
{
instruction: TXA{},
assembler: "txa",
},
{
instruction: RetA{},
assembler: "ret a",
},
{
instruction: RetConstant{Val: 42},
assembler: "ret #42",
},
// Invalid instruction
{
instruction: InvalidInstruction{},
assembler: "unknown instruction: bpf.InvalidInstruction{}",
},
}
for _, testCase := range testCases {
if input, ok := testCase.instruction.(fmt.Stringer); ok {
got := input.String()
if got != testCase.assembler {
t.Errorf("String did not return expected assembler notation, expected: %s, got: %s", testCase.assembler, got)
}
} else {
t.Errorf("Instruction %#v is not a fmt.Stringer", testCase.instruction)
}
}
}

@ -1,10 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
// A Setter is a type which can attach a compiled BPF filter to itself.
type Setter interface {
SetBPF(filter []RawInstruction) error
}

@ -1 +0,0 @@
50,0 0 0 42,1 0 0 42,96 0 0 3,97 0 0 3,48 0 0 42,40 0 0 42,32 0 0 42,80 0 0 42,72 0 0 42,64 0 0 42,177 0 0 42,128 0 0 0,32 0 0 4294963200,32 0 0 4294963204,32 0 0 4294963256,2 0 0 3,3 0 0 3,4 0 0 42,20 0 0 42,36 0 0 42,52 0 0 42,68 0 0 42,84 0 0 42,100 0 0 42,116 0 0 42,148 0 0 42,164 0 0 42,12 0 0 0,28 0 0 0,44 0 0 0,60 0 0 0,76 0 0 0,92 0 0 0,108 0 0 0,124 0 0 0,156 0 0 0,172 0 0 0,132 0 0 0,5 0 0 10,21 8 9 42,21 0 8 42,53 0 7 42,37 0 6 42,37 4 5 42,53 3 4 42,69 2 3 42,7 0 0 0,135 0 0 0,22 0 0 0,6 0 0 0,

@ -1,79 +0,0 @@
# This filter is compiled to all_instructions.bpf by the `bpf_asm`
# tool, which can be found in the linux kernel source tree under
# tools/net.
# Load immediate
ld #42
ldx #42
# Load scratch
ld M[3]
ldx M[3]
# Load absolute
ldb [42]
ldh [42]
ld [42]
# Load indirect
ldb [x + 42]
ldh [x + 42]
ld [x + 42]
# Load IPv4 header length
ldx 4*([42]&0xf)
# Run extension function
ld #len
ld #proto
ld #type
ld #rand
# Store scratch
st M[3]
stx M[3]
# A <op> constant
add #42
sub #42
mul #42
div #42
or #42
and #42
lsh #42
rsh #42
mod #42
xor #42
# A <op> X
add x
sub x
mul x
div x
or x
and x
lsh x
rsh x
mod x
xor x
# !A
neg
# Jumps
ja end
jeq #42,prev,end
jne #42,end
jlt #42,end
jle #42,end
jgt #42,prev,end
jge #42,prev,end
jset #42,prev,end
# Register transfers
tax
txa
# Returns
prev: ret a
end: ret #42

140
vendor/golang.org/x/net/bpf/vm.go generated vendored

@ -1,140 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
import (
"errors"
"fmt"
)
// A VM is an emulated BPF virtual machine.
type VM struct {
filter []Instruction
}
// NewVM returns a new VM using the input BPF program.
func NewVM(filter []Instruction) (*VM, error) {
if len(filter) == 0 {
return nil, errors.New("one or more Instructions must be specified")
}
for i, ins := range filter {
check := len(filter) - (i + 1)
switch ins := ins.(type) {
// Check for out-of-bounds jumps in instructions
case Jump:
if check <= int(ins.Skip) {
return nil, fmt.Errorf("cannot jump %d instructions; jumping past program bounds", ins.Skip)
}
case JumpIf:
if check <= int(ins.SkipTrue) {
return nil, fmt.Errorf("cannot jump %d instructions in true case; jumping past program bounds", ins.SkipTrue)
}
if check <= int(ins.SkipFalse) {
return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse)
}
// Check for division or modulus by zero
case ALUOpConstant:
if ins.Val != 0 {
break
}
switch ins.Op {
case ALUOpDiv, ALUOpMod:
return nil, errors.New("cannot divide by zero using ALUOpConstant")
}
// Check for unknown extensions
case LoadExtension:
switch ins.Num {
case ExtLen:
default:
return nil, fmt.Errorf("extension %d not implemented", ins.Num)
}
}
}
// Make sure last instruction is a return instruction
switch filter[len(filter)-1].(type) {
case RetA, RetConstant:
default:
return nil, errors.New("BPF program must end with RetA or RetConstant")
}
// Though our VM works using disassembled instructions, we
// attempt to assemble the input filter anyway to ensure it is compatible
// with an operating system VM.
_, err := Assemble(filter)
return &VM{
filter: filter,
}, err
}
// Run runs the VM's BPF program against the input bytes.
// Run returns the number of bytes accepted by the BPF program, and any errors
// which occurred while processing the program.
func (v *VM) Run(in []byte) (int, error) {
var (
// Registers of the virtual machine
regA uint32
regX uint32
regScratch [16]uint32
// OK is true if the program should continue processing the next
// instruction, or false if not, causing the loop to break
ok = true
)
// TODO(mdlayher): implement:
// - NegateA:
// - would require a change from uint32 registers to int32
// registers
// TODO(mdlayher): add interop tests that check signedness of ALU
// operations against kernel implementation, and make sure Go
// implementation matches behavior
for i := 0; i < len(v.filter) && ok; i++ {
ins := v.filter[i]
switch ins := ins.(type) {
case ALUOpConstant:
regA = aluOpConstant(ins, regA)
case ALUOpX:
regA, ok = aluOpX(ins, regA, regX)
case Jump:
i += int(ins.Skip)
case JumpIf:
jump := jumpIf(ins, regA)
i += jump
case LoadAbsolute:
regA, ok = loadAbsolute(ins, in)
case LoadConstant:
regA, regX = loadConstant(ins, regA, regX)
case LoadExtension:
regA = loadExtension(ins, in)
case LoadIndirect:
regA, ok = loadIndirect(ins, in, regX)
case LoadMemShift:
regX, ok = loadMemShift(ins, in)
case LoadScratch:
regA, regX = loadScratch(ins, regScratch, regA, regX)
case RetA:
return int(regA), nil
case RetConstant:
return int(ins.Val), nil
case StoreScratch:
regScratch = storeScratch(ins, regScratch, regA, regX)
case TAX:
regX = regA
case TXA:
regA = regX
default:
return 0, fmt.Errorf("unknown Instruction at index %d: %T", i, ins)
}
}
return 0, nil
}

@ -1,512 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"testing"
"golang.org/x/net/bpf"
)
func TestVMALUOpAdd(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpAdd,
Val: 3,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
8, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 3, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpSub(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.TAX{},
bpf.ALUOpX{
Op: bpf.ALUOpSub,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpMul(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpMul,
Val: 2,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
6, 2, 3, 4,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpDiv(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpDiv,
Val: 2,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
20, 2, 3, 4,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpDivByZeroALUOpConstant(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.ALUOpConstant{
Op: bpf.ALUOpDiv,
Val: 0,
},
bpf.RetA{},
})
if errStr(err) != "cannot divide by zero using ALUOpConstant" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMALUOpDivByZeroALUOpX(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
// Load byte 0 into X
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.TAX{},
// Load byte 1 into A
bpf.LoadAbsolute{
Off: 9,
Size: 1,
},
// Attempt to perform 1/0
bpf.ALUOpX{
Op: bpf.ALUOpDiv,
},
// Return 4 bytes if program does not terminate
bpf.LoadConstant{
Val: 12,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 3, 4,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpOr(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpOr,
Val: 0x01,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x00, 0x10, 0x03, 0x04,
0x05, 0x06, 0x07, 0x08,
0x09, 0xff,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 9, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpAnd(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpAnd,
Val: 0x0019,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0xaa, 0x09,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpShiftLeft(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpShiftLeft,
Val: 0x01,
},
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 0x02,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x01, 0xaa,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpShiftRight(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpShiftRight,
Val: 0x01,
},
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 0x04,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x08, 0xff, 0xff,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpMod(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpMod,
Val: 20,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
30, 0, 0,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpModByZeroALUOpConstant(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpMod,
Val: 0,
},
bpf.RetA{},
})
if errStr(err) != "cannot divide by zero using ALUOpConstant" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMALUOpModByZeroALUOpX(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
// Load byte 0 into X
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.TAX{},
// Load byte 1 into A
bpf.LoadAbsolute{
Off: 9,
Size: 1,
},
// Attempt to perform 1%0
bpf.ALUOpX{
Op: bpf.ALUOpMod,
},
// Return 4 bytes if program does not terminate
bpf.LoadConstant{
Val: 12,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 3, 4,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpXor(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpXor,
Val: 0x0a,
},
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 0x01,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x0b, 0x00, 0x00, 0x00,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMALUOpUnknown(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.ALUOpConstant{
Op: bpf.ALUOpAdd,
Val: 1,
},
// Verify that an unknown operation is a no-op
bpf.ALUOpConstant{
Op: 100,
},
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 0x02,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}

@ -1,192 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"net"
"runtime"
"testing"
"time"
"golang.org/x/net/bpf"
"golang.org/x/net/ipv4"
)
// A virtualMachine is a BPF virtual machine which can process an
// input packet against a BPF program and render a verdict.
type virtualMachine interface {
Run(in []byte) (int, error)
}
// canUseOSVM indicates if the OS BPF VM is available on this platform.
func canUseOSVM() bool {
// OS BPF VM can only be used on platforms where x/net/ipv4 supports
// attaching a BPF program to a socket.
switch runtime.GOOS {
case "linux":
return true
}
return false
}
// All BPF tests against both the Go VM and OS VM are assumed to
// be used with a UDP socket. As a result, the entire contents
// of a UDP datagram is sent through the BPF program, but only
// the body after the UDP header will ever be returned in output.
// testVM sets up a Go BPF VM, and if available, a native OS BPF VM
// for integration testing.
func testVM(t *testing.T, filter []bpf.Instruction) (virtualMachine, func(), error) {
goVM, err := bpf.NewVM(filter)
if err != nil {
// Some tests expect an error, so this error must be returned
// instead of fatally exiting the test
return nil, nil, err
}
mvm := &multiVirtualMachine{
goVM: goVM,
t: t,
}
// If available, add the OS VM for tests which verify that both the Go
// VM and OS VM have exactly the same output for the same input program
// and packet.
done := func() {}
if canUseOSVM() {
osVM, osVMDone := testOSVM(t, filter)
done = func() { osVMDone() }
mvm.osVM = osVM
}
return mvm, done, nil
}
// udpHeaderLen is the length of a UDP header.
const udpHeaderLen = 8
// A multiVirtualMachine is a virtualMachine which can call out to both the Go VM
// and the native OS VM, if the OS VM is available.
type multiVirtualMachine struct {
goVM virtualMachine
osVM virtualMachine
t *testing.T
}
func (mvm *multiVirtualMachine) Run(in []byte) (int, error) {
if len(in) < udpHeaderLen {
mvm.t.Fatalf("input must be at least length of UDP header (%d), got: %d",
udpHeaderLen, len(in))
}
// All tests have a UDP header as part of input, because the OS VM
// packets always will. For the Go VM, this output is trimmed before
// being sent back to tests.
goOut, goErr := mvm.goVM.Run(in)
if goOut >= udpHeaderLen {
goOut -= udpHeaderLen
}
// If Go output is larger than the size of the packet, packet filtering
// interop tests must trim the output bytes to the length of the packet.
// The BPF VM should not do this on its own, as other uses of it do
// not trim the output byte count.
trim := len(in) - udpHeaderLen
if goOut > trim {
goOut = trim
}
// When the OS VM is not available, process using the Go VM alone
if mvm.osVM == nil {
return goOut, goErr
}
// The OS VM will apply its own UDP header, so remove the pseudo header
// that the Go VM needs.
osOut, err := mvm.osVM.Run(in[udpHeaderLen:])
if err != nil {
mvm.t.Fatalf("error while running OS VM: %v", err)
}
// Verify both VMs return same number of bytes
var mismatch bool
if goOut != osOut {
mismatch = true
mvm.t.Logf("output byte count does not match:\n- go: %v\n- os: %v", goOut, osOut)
}
if mismatch {
mvm.t.Fatal("Go BPF and OS BPF packet outputs do not match")
}
return goOut, goErr
}
// An osVirtualMachine is a virtualMachine which uses the OS's BPF VM for
// processing BPF programs.
type osVirtualMachine struct {
l net.PacketConn
s net.Conn
}
// testOSVM creates a virtualMachine which uses the OS's BPF VM by injecting
// packets into a UDP listener with a BPF program attached to it.
func testOSVM(t *testing.T, filter []bpf.Instruction) (virtualMachine, func()) {
l, err := net.ListenPacket("udp4", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to open OS VM UDP listener: %v", err)
}
prog, err := bpf.Assemble(filter)
if err != nil {
t.Fatalf("failed to compile BPF program: %v", err)
}
p := ipv4.NewPacketConn(l)
if err = p.SetBPF(prog); err != nil {
t.Fatalf("failed to attach BPF program to listener: %v", err)
}
s, err := net.Dial("udp4", l.LocalAddr().String())
if err != nil {
t.Fatalf("failed to dial connection to listener: %v", err)
}
done := func() {
_ = s.Close()
_ = l.Close()
}
return &osVirtualMachine{
l: l,
s: s,
}, done
}
// Run sends the input bytes into the OS's BPF VM and returns its verdict.
func (vm *osVirtualMachine) Run(in []byte) (int, error) {
go func() {
_, _ = vm.s.Write(in)
}()
vm.l.SetDeadline(time.Now().Add(50 * time.Millisecond))
var b [512]byte
n, _, err := vm.l.ReadFrom(b[:])
if err != nil {
// A timeout indicates that BPF filtered out the packet, and thus,
// no input should be returned.
if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
return n, nil
}
return n, err
}
return n, nil
}

@ -1,49 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"testing"
"golang.org/x/net/bpf"
)
func TestVMLoadExtensionNotImplemented(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadExtension{
Num: 100,
},
bpf.RetA{},
})
if errStr(err) != "extension 100 not implemented" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMLoadExtensionExtLen(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadExtension{
Num: bpf.ExtLen,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}

@ -1,174 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf
import (
"encoding/binary"
"fmt"
)
func aluOpConstant(ins ALUOpConstant, regA uint32) uint32 {
return aluOpCommon(ins.Op, regA, ins.Val)
}
func aluOpX(ins ALUOpX, regA uint32, regX uint32) (uint32, bool) {
// Guard against division or modulus by zero by terminating
// the program, as the OS BPF VM does
if regX == 0 {
switch ins.Op {
case ALUOpDiv, ALUOpMod:
return 0, false
}
}
return aluOpCommon(ins.Op, regA, regX), true
}
func aluOpCommon(op ALUOp, regA uint32, value uint32) uint32 {
switch op {
case ALUOpAdd:
return regA + value
case ALUOpSub:
return regA - value
case ALUOpMul:
return regA * value
case ALUOpDiv:
// Division by zero not permitted by NewVM and aluOpX checks
return regA / value
case ALUOpOr:
return regA | value
case ALUOpAnd:
return regA & value
case ALUOpShiftLeft:
return regA << value
case ALUOpShiftRight:
return regA >> value
case ALUOpMod:
// Modulus by zero not permitted by NewVM and aluOpX checks
return regA % value
case ALUOpXor:
return regA ^ value
default:
return regA
}
}
func jumpIf(ins JumpIf, value uint32) int {
var ok bool
inV := uint32(ins.Val)
switch ins.Cond {
case JumpEqual:
ok = value == inV
case JumpNotEqual:
ok = value != inV
case JumpGreaterThan:
ok = value > inV
case JumpLessThan:
ok = value < inV
case JumpGreaterOrEqual:
ok = value >= inV
case JumpLessOrEqual:
ok = value <= inV
case JumpBitsSet:
ok = (value & inV) != 0
case JumpBitsNotSet:
ok = (value & inV) == 0
}
if ok {
return int(ins.SkipTrue)
}
return int(ins.SkipFalse)
}
func loadAbsolute(ins LoadAbsolute, in []byte) (uint32, bool) {
offset := int(ins.Off)
size := int(ins.Size)
return loadCommon(in, offset, size)
}
func loadConstant(ins LoadConstant, regA uint32, regX uint32) (uint32, uint32) {
switch ins.Dst {
case RegA:
regA = ins.Val
case RegX:
regX = ins.Val
}
return regA, regX
}
func loadExtension(ins LoadExtension, in []byte) uint32 {
switch ins.Num {
case ExtLen:
return uint32(len(in))
default:
panic(fmt.Sprintf("unimplemented extension: %d", ins.Num))
}
}
func loadIndirect(ins LoadIndirect, in []byte, regX uint32) (uint32, bool) {
offset := int(ins.Off) + int(regX)
size := int(ins.Size)
return loadCommon(in, offset, size)
}
func loadMemShift(ins LoadMemShift, in []byte) (uint32, bool) {
offset := int(ins.Off)
if !inBounds(len(in), offset, 0) {
return 0, false
}
// Mask off high 4 bits and multiply low 4 bits by 4
return uint32(in[offset]&0x0f) * 4, true
}
func inBounds(inLen int, offset int, size int) bool {
return offset+size <= inLen
}
func loadCommon(in []byte, offset int, size int) (uint32, bool) {
if !inBounds(len(in), offset, size) {
return 0, false
}
switch size {
case 1:
return uint32(in[offset]), true
case 2:
return uint32(binary.BigEndian.Uint16(in[offset : offset+size])), true
case 4:
return uint32(binary.BigEndian.Uint32(in[offset : offset+size])), true
default:
panic(fmt.Sprintf("invalid load size: %d", size))
}
}
func loadScratch(ins LoadScratch, regScratch [16]uint32, regA uint32, regX uint32) (uint32, uint32) {
switch ins.Dst {
case RegA:
regA = regScratch[ins.N]
case RegX:
regX = regScratch[ins.N]
}
return regA, regX
}
func storeScratch(ins StoreScratch, regScratch [16]uint32, regA uint32, regX uint32) [16]uint32 {
switch ins.Src {
case RegA:
regScratch[ins.N] = regA
case RegX:
regScratch[ins.N] = regX
}
return regScratch
}

@ -1,380 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"testing"
"golang.org/x/net/bpf"
)
func TestVMJumpOne(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.Jump{
Skip: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpOutOfProgram(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.Jump{
Skip: 1,
},
bpf.RetA{},
})
if errStr(err) != "cannot jump 1 instructions; jumping past program bounds" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMJumpIfTrueOutOfProgram(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.JumpIf{
Cond: bpf.JumpEqual,
SkipTrue: 2,
},
bpf.RetA{},
})
if errStr(err) != "cannot jump 2 instructions in true case; jumping past program bounds" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMJumpIfFalseOutOfProgram(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.JumpIf{
Cond: bpf.JumpEqual,
SkipFalse: 3,
},
bpf.RetA{},
})
if errStr(err) != "cannot jump 3 instructions in false case; jumping past program bounds" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMJumpIfEqual(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 1,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfNotEqual(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.JumpIf{
Cond: bpf.JumpNotEqual,
Val: 1,
SkipFalse: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfGreaterThan(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 4,
},
bpf.JumpIf{
Cond: bpf.JumpGreaterThan,
Val: 0x00010202,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 12,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfLessThan(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 4,
},
bpf.JumpIf{
Cond: bpf.JumpLessThan,
Val: 0xff010203,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 12,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfGreaterOrEqual(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 4,
},
bpf.JumpIf{
Cond: bpf.JumpGreaterOrEqual,
Val: 0x00010203,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 12,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfLessOrEqual(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 4,
},
bpf.JumpIf{
Cond: bpf.JumpLessOrEqual,
Val: 0xff010203,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 12,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 4, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfBitsSet(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.JumpIf{
Cond: bpf.JumpBitsSet,
Val: 0x1122,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 10,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x01, 0x02,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMJumpIfBitsNotSet(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.JumpIf{
Cond: bpf.JumpBitsNotSet,
Val: 0x1221,
SkipTrue: 1,
},
bpf.RetConstant{
Val: 0,
},
bpf.RetConstant{
Val: 10,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x01, 0x02,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}

@ -1,246 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"net"
"testing"
"golang.org/x/net/bpf"
"golang.org/x/net/ipv4"
)
func TestVMLoadAbsoluteOffsetOutOfBounds(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 100,
Size: 2,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1, 2, 3,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMLoadAbsoluteOffsetPlusSizeOutOfBounds(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMLoadAbsoluteBadInstructionSize(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Size: 5,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid load byte length 0" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMLoadConstantOK(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadConstant{
Dst: bpf.RegX,
Val: 9,
},
bpf.TXA{},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMLoadIndirectOutOfBounds(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadIndirect{
Off: 100,
Size: 1,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMLoadMemShiftOutOfBounds(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadMemShift{
Off: 100,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
const (
dhcp4Port = 53
)
func TestVMLoadMemShiftLoadIndirectNoResult(t *testing.T) {
vm, in, done := testDHCPv4(t)
defer done()
// Append mostly empty UDP header with incorrect DHCPv4 port
in = append(in, []byte{
0, 0,
0, dhcp4Port + 1,
0, 0,
0, 0,
}...)
out, err := vm.Run(in)
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 0, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMLoadMemShiftLoadIndirectOK(t *testing.T) {
vm, in, done := testDHCPv4(t)
defer done()
// Append mostly empty UDP header with correct DHCPv4 port
in = append(in, []byte{
0, 0,
0, dhcp4Port,
0, 0,
0, 0,
}...)
out, err := vm.Run(in)
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := len(in)-8, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func testDHCPv4(t *testing.T) (virtualMachine, []byte, func()) {
// DHCPv4 test data courtesy of David Anderson:
// https://github.com/google/netboot/blob/master/dhcp4/conn_linux.go#L59-L70
vm, done, err := testVM(t, []bpf.Instruction{
// Load IPv4 packet length
bpf.LoadMemShift{Off: 8},
// Get UDP dport
bpf.LoadIndirect{Off: 8 + 2, Size: 2},
// Correct dport?
bpf.JumpIf{Cond: bpf.JumpEqual, Val: dhcp4Port, SkipFalse: 1},
// Accept
bpf.RetConstant{Val: 1500},
// Ignore
bpf.RetConstant{Val: 0},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
// Minimal requirements to make a valid IPv4 header
h := &ipv4.Header{
Len: ipv4.HeaderLen,
Src: net.IPv4(192, 168, 1, 1),
Dst: net.IPv4(192, 168, 1, 2),
}
hb, err := h.Marshal()
if err != nil {
t.Fatalf("failed to marshal IPv4 header: %v", err)
}
hb = append([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
}, hb...)
return vm, hb, done
}

@ -1,115 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"testing"
"golang.org/x/net/bpf"
)
func TestVMRetA(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
9,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMRetALargerThanInput(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadAbsolute{
Off: 8,
Size: 2,
},
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 255,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMRetConstant(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.RetConstant{
Val: 9,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 1, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMRetConstantLargerThanInput(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.RetConstant{
Val: 16,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0, 1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}

@ -1,247 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"testing"
"golang.org/x/net/bpf"
)
func TestVMStoreScratchInvalidScratchRegisterTooSmall(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.StoreScratch{
Src: bpf.RegA,
N: -1,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid scratch slot -1" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMStoreScratchInvalidScratchRegisterTooLarge(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.StoreScratch{
Src: bpf.RegA,
N: 16,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid scratch slot 16" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMStoreScratchUnknownSourceRegister(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.StoreScratch{
Src: 100,
N: 0,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid source register 100" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMLoadScratchInvalidScratchRegisterTooSmall(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadScratch{
Dst: bpf.RegX,
N: -1,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid scratch slot -1" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMLoadScratchInvalidScratchRegisterTooLarge(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadScratch{
Dst: bpf.RegX,
N: 16,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid scratch slot 16" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMLoadScratchUnknownDestinationRegister(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadScratch{
Dst: 100,
N: 0,
},
bpf.RetA{},
})
if errStr(err) != "assembling instruction 1: invalid target register 100" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMStoreScratchLoadScratchOneValue(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
// Load byte 255
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
// Copy to X and store in scratch[0]
bpf.TAX{},
bpf.StoreScratch{
Src: bpf.RegX,
N: 0,
},
// Load byte 1
bpf.LoadAbsolute{
Off: 9,
Size: 1,
},
// Overwrite 1 with 255 from scratch[0]
bpf.LoadScratch{
Dst: bpf.RegA,
N: 0,
},
// Return 255
bpf.RetA{},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
255, 1, 2,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 3, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}
func TestVMStoreScratchLoadScratchMultipleValues(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
// Load byte 10
bpf.LoadAbsolute{
Off: 8,
Size: 1,
},
// Store in scratch[0]
bpf.StoreScratch{
Src: bpf.RegA,
N: 0,
},
// Load byte 20
bpf.LoadAbsolute{
Off: 9,
Size: 1,
},
// Store in scratch[1]
bpf.StoreScratch{
Src: bpf.RegA,
N: 1,
},
// Load byte 30
bpf.LoadAbsolute{
Off: 10,
Size: 1,
},
// Store in scratch[2]
bpf.StoreScratch{
Src: bpf.RegA,
N: 2,
},
// Load byte 1
bpf.LoadAbsolute{
Off: 11,
Size: 1,
},
// Store in scratch[3]
bpf.StoreScratch{
Src: bpf.RegA,
N: 3,
},
// Load in byte 10 to X
bpf.LoadScratch{
Dst: bpf.RegX,
N: 0,
},
// Copy X -> A
bpf.TXA{},
// Verify value is 10
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 10,
SkipTrue: 1,
},
// Fail test if incorrect
bpf.RetConstant{
Val: 0,
},
// Load in byte 20 to A
bpf.LoadScratch{
Dst: bpf.RegA,
N: 1,
},
// Verify value is 20
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 20,
SkipTrue: 1,
},
// Fail test if incorrect
bpf.RetConstant{
Val: 0,
},
// Load in byte 30 to A
bpf.LoadScratch{
Dst: bpf.RegA,
N: 2,
},
// Verify value is 30
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: 30,
SkipTrue: 1,
},
// Fail test if incorrect
bpf.RetConstant{
Val: 0,
},
// Return first two bytes on success
bpf.RetConstant{
Val: 10,
},
})
if err != nil {
t.Fatalf("failed to load BPF program: %v", err)
}
defer done()
out, err := vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
10, 20, 30, 1,
})
if err != nil {
t.Fatalf("unexpected error while running program: %v", err)
}
if want, got := 2, out; want != got {
t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d",
want, got)
}
}

@ -1,144 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package bpf_test
import (
"fmt"
"testing"
"golang.org/x/net/bpf"
)
var _ bpf.Instruction = unknown{}
type unknown struct{}
func (unknown) Assemble() (bpf.RawInstruction, error) {
return bpf.RawInstruction{}, nil
}
func TestVMUnknownInstruction(t *testing.T) {
vm, done, err := testVM(t, []bpf.Instruction{
bpf.LoadConstant{
Dst: bpf.RegA,
Val: 100,
},
// Should terminate the program with an error immediately
unknown{},
bpf.RetA{},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer done()
_, err = vm.Run([]byte{
0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff,
0x00, 0x00,
})
if errStr(err) != "unknown Instruction at index 1: bpf_test.unknown" {
t.Fatalf("unexpected error while running program: %v", err)
}
}
func TestVMNoReturnInstruction(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{
bpf.LoadConstant{
Dst: bpf.RegA,
Val: 1,
},
})
if errStr(err) != "BPF program must end with RetA or RetConstant" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestVMNoInputInstructions(t *testing.T) {
_, _, err := testVM(t, []bpf.Instruction{})
if errStr(err) != "one or more Instructions must be specified" {
t.Fatalf("unexpected error: %v", err)
}
}
// ExampleNewVM demonstrates usage of a VM, using an Ethernet frame
// as input and checking its EtherType to determine if it should be accepted.
func ExampleNewVM() {
// Offset | Length | Comment
// -------------------------
// 00 | 06 | Ethernet destination MAC address
// 06 | 06 | Ethernet source MAC address
// 12 | 02 | Ethernet EtherType
const (
etOff = 12
etLen = 2
etARP = 0x0806
)
// Set up a VM to filter traffic based on if its EtherType
// matches the ARP EtherType.
vm, err := bpf.NewVM([]bpf.Instruction{
// Load EtherType value from Ethernet header
bpf.LoadAbsolute{
Off: etOff,
Size: etLen,
},
// If EtherType is equal to the ARP EtherType, jump to allow
// packet to be accepted
bpf.JumpIf{
Cond: bpf.JumpEqual,
Val: etARP,
SkipTrue: 1,
},
// EtherType does not match the ARP EtherType
bpf.RetConstant{
Val: 0,
},
// EtherType matches the ARP EtherType, accept up to 1500
// bytes of packet
bpf.RetConstant{
Val: 1500,
},
})
if err != nil {
panic(fmt.Sprintf("failed to load BPF program: %v", err))
}
// Create an Ethernet frame with the ARP EtherType for testing
frame := []byte{
0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55,
0x08, 0x06,
// Payload omitted for brevity
}
// Run our VM's BPF program using the Ethernet frame as input
out, err := vm.Run(frame)
if err != nil {
panic(fmt.Sprintf("failed to accept Ethernet frame: %v", err))
}
// BPF VM can return a byte count greater than the number of input
// bytes, so trim the output to match the input byte length
if out > len(frame) {
out = len(frame)
}
fmt.Printf("out: %d bytes", out)
// Output:
// out: 14 bytes
}
// errStr returns the string representation of an error, or
// "<nil>" if it is nil.
func errStr(err error) string {
if err == nil {
return "<nil>"
}
return err.Error()
}

@ -1 +0,0 @@
issuerepo: golang/go

@ -1,56 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package context defines the Context type, which carries deadlines,
// cancelation signals, and other request-scoped values across API boundaries
// and between processes.
// As of Go 1.7 this package is available in the standard library under the
// name context. https://golang.org/pkg/context.
//
// Incoming requests to a server should create a Context, and outgoing calls to
// servers should accept a Context. The chain of function calls between must
// propagate the Context, optionally replacing it with a modified copy created
// using WithDeadline, WithTimeout, WithCancel, or WithValue.
//
// Programs that use Contexts should follow these rules to keep interfaces
// consistent across packages and enable static analysis tools to check context
// propagation:
//
// Do not store Contexts inside a struct type; instead, pass a Context
// explicitly to each function that needs it. The Context should be the first
// parameter, typically named ctx:
//
// func DoSomething(ctx context.Context, arg Arg) error {
// // ... use ctx ...
// }
//
// Do not pass a nil Context, even if a function permits it. Pass context.TODO
// if you are unsure about which Context to use.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
//
// The same Context may be passed to functions running in different goroutines;
// Contexts are safe for simultaneous use by multiple goroutines.
//
// See http://blog.golang.org/context for example code for a server that uses
// Contexts.
package context // import "golang.org/x/net/context"
// Background returns a non-nil, empty Context. It is never canceled, has no
// values, and has no deadline. It is typically used by the main function,
// initialization, and tests, and as the top-level Context for incoming
// requests.
func Background() Context {
return background
}
// TODO returns a non-nil, empty Context. Code should use context.TODO when
// it's unclear which Context to use or it is not yet available (because the
// surrounding function has not yet been extended to accept a Context
// parameter). TODO is recognized by static analysis tools that determine
// whether Contexts are propagated correctly in a program.
func TODO() Context {
return todo
}

@ -1,583 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.7
package context
import (
"fmt"
"math/rand"
"runtime"
"strings"
"sync"
"testing"
"time"
)
// otherContext is a Context that's not one of the types defined in context.go.
// This lets us test code paths that differ based on the underlying type of the
// Context.
type otherContext struct {
Context
}
func TestBackground(t *testing.T) {
c := Background()
if c == nil {
t.Fatalf("Background returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.Background"; got != want {
t.Errorf("Background().String() = %q want %q", got, want)
}
}
func TestTODO(t *testing.T) {
c := TODO()
if c == nil {
t.Fatalf("TODO returned nil")
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
if got, want := fmt.Sprint(c), "context.TODO"; got != want {
t.Errorf("TODO().String() = %q want %q", got, want)
}
}
func TestWithCancel(t *testing.T) {
c1, cancel := WithCancel(Background())
if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want {
t.Errorf("c1.String() = %q want %q", got, want)
}
o := otherContext{c1}
c2, _ := WithCancel(o)
contexts := []Context{c1, o, c2}
for i, c := range contexts {
if d := c.Done(); d == nil {
t.Errorf("c[%d].Done() == %v want non-nil", i, d)
}
if e := c.Err(); e != nil {
t.Errorf("c[%d].Err() == %v want nil", i, e)
}
select {
case x := <-c.Done():
t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
default:
}
}
cancel()
time.Sleep(100 * time.Millisecond) // let cancelation propagate
for i, c := range contexts {
select {
case <-c.Done():
default:
t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i)
}
if e := c.Err(); e != Canceled {
t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled)
}
}
}
func TestParentFinishesChild(t *testing.T) {
// Context tree:
// parent -> cancelChild
// parent -> valueChild -> timerChild
parent, cancel := WithCancel(Background())
cancelChild, stop := WithCancel(parent)
defer stop()
valueChild := WithValue(parent, "key", "value")
timerChild, stop := WithTimeout(valueChild, 10000*time.Hour)
defer stop()
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-cancelChild.Done():
t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x)
case x := <-timerChild.Done():
t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x)
case x := <-valueChild.Done():
t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x)
default:
}
// The parent's children should contain the two cancelable children.
pc := parent.(*cancelCtx)
cc := cancelChild.(*cancelCtx)
tc := timerChild.(*timerCtx)
pc.mu.Lock()
if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] {
t.Errorf("bad linkage: pc.children = %v, want %v and %v",
pc.children, cc, tc)
}
pc.mu.Unlock()
if p, ok := parentCancelCtx(cc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc)
}
if p, ok := parentCancelCtx(tc.Context); !ok || p != pc {
t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc)
}
cancel()
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children)
}
pc.mu.Unlock()
// parent and children should all be finished.
check := func(ctx Context, name string) {
select {
case <-ctx.Done():
default:
t.Errorf("<-%s.Done() blocked, but shouldn't have", name)
}
if e := ctx.Err(); e != Canceled {
t.Errorf("%s.Err() == %v want %v", name, e, Canceled)
}
}
check(parent, "parent")
check(cancelChild, "cancelChild")
check(valueChild, "valueChild")
check(timerChild, "timerChild")
// WithCancel should return a canceled context on a canceled parent.
precanceledChild := WithValue(parent, "key", "value")
select {
case <-precanceledChild.Done():
default:
t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have")
}
if e := precanceledChild.Err(); e != Canceled {
t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled)
}
}
func TestChildFinishesFirst(t *testing.T) {
cancelable, stop := WithCancel(Background())
defer stop()
for _, parent := range []Context{Background(), cancelable} {
child, cancel := WithCancel(parent)
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
case x := <-child.Done():
t.Errorf("<-child.Done() == %v want nothing (it should block)", x)
default:
}
cc := child.(*cancelCtx)
pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background()
if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) {
t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok)
}
if pcok {
pc.mu.Lock()
if len(pc.children) != 1 || !pc.children[cc] {
t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc)
}
pc.mu.Unlock()
}
cancel()
if pcok {
pc.mu.Lock()
if len(pc.children) != 0 {
t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children)
}
pc.mu.Unlock()
}
// child should be finished.
select {
case <-child.Done():
default:
t.Errorf("<-child.Done() blocked, but shouldn't have")
}
if e := child.Err(); e != Canceled {
t.Errorf("child.Err() == %v want %v", e, Canceled)
}
// parent should not be finished.
select {
case x := <-parent.Done():
t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
default:
}
if e := parent.Err(); e != nil {
t.Errorf("parent.Err() == %v want nil", e)
}
}
}
func testDeadline(c Context, wait time.Duration, t *testing.T) {
select {
case <-time.After(wait):
t.Fatalf("context should have timed out")
case <-c.Done():
}
if e := c.Err(); e != DeadlineExceeded {
t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded)
}
}
func TestDeadline(t *testing.T) {
t.Parallel()
const timeUnit = 500 * time.Millisecond
c, _ := WithDeadline(Background(), time.Now().Add(1*timeUnit))
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 2*timeUnit, t)
c, _ = WithDeadline(Background(), time.Now().Add(1*timeUnit))
o := otherContext{c}
testDeadline(o, 2*timeUnit, t)
c, _ = WithDeadline(Background(), time.Now().Add(1*timeUnit))
o = otherContext{c}
c, _ = WithDeadline(o, time.Now().Add(3*timeUnit))
testDeadline(c, 2*timeUnit, t)
}
func TestTimeout(t *testing.T) {
t.Parallel()
const timeUnit = 500 * time.Millisecond
c, _ := WithTimeout(Background(), 1*timeUnit)
if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
t.Errorf("c.String() = %q want prefix %q", got, prefix)
}
testDeadline(c, 2*timeUnit, t)
c, _ = WithTimeout(Background(), 1*timeUnit)
o := otherContext{c}
testDeadline(o, 2*timeUnit, t)
c, _ = WithTimeout(Background(), 1*timeUnit)
o = otherContext{c}
c, _ = WithTimeout(o, 3*timeUnit)
testDeadline(c, 2*timeUnit, t)
}
func TestCanceledTimeout(t *testing.T) {
t.Parallel()
const timeUnit = 500 * time.Millisecond
c, _ := WithTimeout(Background(), 2*timeUnit)
o := otherContext{c}
c, cancel := WithTimeout(o, 4*timeUnit)
cancel()
time.Sleep(1 * timeUnit) // let cancelation propagate
select {
case <-c.Done():
default:
t.Errorf("<-c.Done() blocked, but shouldn't have")
}
if e := c.Err(); e != Canceled {
t.Errorf("c.Err() == %v want %v", e, Canceled)
}
}
type key1 int
type key2 int
var k1 = key1(1)
var k2 = key2(1) // same int as k1, different type
var k3 = key2(3) // same type as k2, different int
func TestValues(t *testing.T) {
check := func(c Context, nm, v1, v2, v3 string) {
if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 {
t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0)
}
if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 {
t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0)
}
if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 {
t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0)
}
}
c0 := Background()
check(c0, "c0", "", "", "")
c1 := WithValue(Background(), k1, "c1k1")
check(c1, "c1", "c1k1", "", "")
if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want {
t.Errorf("c.String() = %q want %q", got, want)
}
c2 := WithValue(c1, k2, "c2k2")
check(c2, "c2", "c1k1", "c2k2", "")
c3 := WithValue(c2, k3, "c3k3")
check(c3, "c2", "c1k1", "c2k2", "c3k3")
c4 := WithValue(c3, k1, nil)
check(c4, "c4", "", "c2k2", "c3k3")
o0 := otherContext{Background()}
check(o0, "o0", "", "", "")
o1 := otherContext{WithValue(Background(), k1, "c1k1")}
check(o1, "o1", "c1k1", "", "")
o2 := WithValue(o1, k2, "o2k2")
check(o2, "o2", "c1k1", "o2k2", "")
o3 := otherContext{c4}
check(o3, "o3", "", "c2k2", "c3k3")
o4 := WithValue(o3, k3, nil)
check(o4, "o4", "", "c2k2", "")
}
func TestAllocs(t *testing.T) {
bg := Background()
for _, test := range []struct {
desc string
f func()
limit float64
gccgoLimit float64
}{
{
desc: "Background()",
f: func() { Background() },
limit: 0,
gccgoLimit: 0,
},
{
desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1),
f: func() {
c := WithValue(bg, k1, nil)
c.Value(k1)
},
limit: 3,
gccgoLimit: 3,
},
{
desc: "WithTimeout(bg, 15*time.Millisecond)",
f: func() {
c, _ := WithTimeout(bg, 15*time.Millisecond)
<-c.Done()
},
limit: 8,
gccgoLimit: 16,
},
{
desc: "WithCancel(bg)",
f: func() {
c, cancel := WithCancel(bg)
cancel()
<-c.Done()
},
limit: 5,
gccgoLimit: 8,
},
{
desc: "WithTimeout(bg, 100*time.Millisecond)",
f: func() {
c, cancel := WithTimeout(bg, 100*time.Millisecond)
cancel()
<-c.Done()
},
limit: 8,
gccgoLimit: 25,
},
} {
limit := test.limit
if runtime.Compiler == "gccgo" {
// gccgo does not yet do escape analysis.
// TODO(iant): Remove this when gccgo does do escape analysis.
limit = test.gccgoLimit
}
if n := testing.AllocsPerRun(100, test.f); n > limit {
t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit))
}
}
}
func TestSimultaneousCancels(t *testing.T) {
root, cancel := WithCancel(Background())
m := map[Context]CancelFunc{root: cancel}
q := []Context{root}
// Create a tree of contexts.
for len(q) != 0 && len(m) < 100 {
parent := q[0]
q = q[1:]
for i := 0; i < 4; i++ {
ctx, cancel := WithCancel(parent)
m[ctx] = cancel
q = append(q, ctx)
}
}
// Start all the cancels in a random order.
var wg sync.WaitGroup
wg.Add(len(m))
for _, cancel := range m {
go func(cancel CancelFunc) {
cancel()
wg.Done()
}(cancel)
}
// Wait on all the contexts in a random order.
for ctx := range m {
select {
case <-ctx.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n])
}
}
// Wait for all the cancel functions to return.
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n])
}
}
func TestInterlockedCancels(t *testing.T) {
parent, cancelParent := WithCancel(Background())
child, cancelChild := WithCancel(parent)
go func() {
parent.Done()
cancelChild()
}()
cancelParent()
select {
case <-child.Done():
case <-time.After(1 * time.Second):
buf := make([]byte, 10<<10)
n := runtime.Stack(buf, true)
t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n])
}
}
func TestLayersCancel(t *testing.T) {
testLayers(t, time.Now().UnixNano(), false)
}
func TestLayersTimeout(t *testing.T) {
testLayers(t, time.Now().UnixNano(), true)
}
func testLayers(t *testing.T, seed int64, testTimeout bool) {
rand.Seed(seed)
errorf := func(format string, a ...interface{}) {
t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...)
}
const (
timeout = 200 * time.Millisecond
minLayers = 30
)
type value int
var (
vals []*value
cancels []CancelFunc
numTimers int
ctx = Background()
)
for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ {
switch rand.Intn(3) {
case 0:
v := new(value)
ctx = WithValue(ctx, v, v)
vals = append(vals, v)
case 1:
var cancel CancelFunc
ctx, cancel = WithCancel(ctx)
cancels = append(cancels, cancel)
case 2:
var cancel CancelFunc
ctx, cancel = WithTimeout(ctx, timeout)
cancels = append(cancels, cancel)
numTimers++
}
}
checkValues := func(when string) {
for _, key := range vals {
if val := ctx.Value(key).(*value); key != val {
errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key)
}
}
}
select {
case <-ctx.Done():
errorf("ctx should not be canceled yet")
default:
}
if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) {
t.Errorf("ctx.String() = %q want prefix %q", s, prefix)
}
t.Log(ctx)
checkValues("before cancel")
if testTimeout {
select {
case <-ctx.Done():
case <-time.After(timeout + 100*time.Millisecond):
errorf("ctx should have timed out")
}
checkValues("after timeout")
} else {
cancel := cancels[rand.Intn(len(cancels))]
cancel()
select {
case <-ctx.Done():
default:
errorf("ctx should be canceled")
}
checkValues("after cancel")
}
}
func TestCancelRemoves(t *testing.T) {
checkChildren := func(when string, ctx Context, want int) {
if got := len(ctx.(*cancelCtx).children); got != want {
t.Errorf("%s: context has %d children, want %d", when, got, want)
}
}
ctx, _ := WithCancel(Background())
checkChildren("after creation", ctx, 0)
_, cancel := WithCancel(ctx)
checkChildren("with WithCancel child ", ctx, 1)
cancel()
checkChildren("after cancelling WithCancel child", ctx, 0)
ctx, _ = WithCancel(Background())
checkChildren("after creation", ctx, 0)
_, cancel = WithTimeout(ctx, 60*time.Minute)
checkChildren("with WithTimeout child ", ctx, 1)
cancel()
checkChildren("after cancelling WithTimeout child", ctx, 0)
}

@ -1,74 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.7
// Package ctxhttp provides helper functions for performing context-aware HTTP requests.
package ctxhttp // import "golang.org/x/net/context/ctxhttp"
import (
"io"
"net/http"
"net/url"
"strings"
"golang.org/x/net/context"
)
// Do sends an HTTP request with the provided http.Client and returns
// an HTTP response.
//
// If the client is nil, http.DefaultClient is used.
//
// The provided ctx must be non-nil. If it is canceled or times out,
// ctx.Err() will be returned.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req.WithContext(ctx))
// If we got an error, and the context has been canceled,
// the context's error is probably more useful.
if err != nil {
select {
case <-ctx.Done():
err = ctx.Err()
default:
}
}
return resp, err
}
// Get issues a GET request via the Do function.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Head issues a HEAD request via the Do function.
func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Post issues a POST request via the Do function.
func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return Do(ctx, client, req)
}
// PostForm issues a POST request via the Do function.
func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) {
return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}

@ -1,29 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !plan9,go1.7
package ctxhttp
import (
"io"
"net/http"
"net/http/httptest"
"testing"
"context"
)
func TestGo17Context(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.WriteString(w, "ok")
}))
defer ts.Close()
ctx := context.Background()
resp, err := Get(ctx, http.DefaultClient, ts.URL)
if resp == nil || err != nil {
t.Fatalf("error received from client: %v %v", err, resp)
}
resp.Body.Close()
}

@ -1,147 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.7
package ctxhttp // import "golang.org/x/net/context/ctxhttp"
import (
"io"
"net/http"
"net/url"
"strings"
"golang.org/x/net/context"
)
func nop() {}
var (
testHookContextDoneBeforeHeaders = nop
testHookDoReturned = nop
testHookDidBodyClose = nop
)
// Do sends an HTTP request with the provided http.Client and returns an HTTP response.
// If the client is nil, http.DefaultClient is used.
// If the context is canceled or times out, ctx.Err() will be returned.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if client == nil {
client = http.DefaultClient
}
// TODO(djd): Respect any existing value of req.Cancel.
cancel := make(chan struct{})
req.Cancel = cancel
type responseAndError struct {
resp *http.Response
err error
}
result := make(chan responseAndError, 1)
// Make local copies of test hooks closed over by goroutines below.
// Prevents data races in tests.
testHookDoReturned := testHookDoReturned
testHookDidBodyClose := testHookDidBodyClose
go func() {
resp, err := client.Do(req)
testHookDoReturned()
result <- responseAndError{resp, err}
}()
var resp *http.Response
select {
case <-ctx.Done():
testHookContextDoneBeforeHeaders()
close(cancel)
// Clean up after the goroutine calling client.Do:
go func() {
if r := <-result; r.resp != nil {
testHookDidBodyClose()
r.resp.Body.Close()
}
}()
return nil, ctx.Err()
case r := <-result:
var err error
resp, err = r.resp, r.err
if err != nil {
return resp, err
}
}
c := make(chan struct{})
go func() {
select {
case <-ctx.Done():
close(cancel)
case <-c:
// The response's Body is closed.
}
}()
resp.Body = &notifyingReader{resp.Body, c}
return resp, nil
}
// Get issues a GET request via the Do function.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Head issues a HEAD request via the Do function.
func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return nil, err
}
return Do(ctx, client, req)
}
// Post issues a POST request via the Do function.
func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", bodyType)
return Do(ctx, client, req)
}
// PostForm issues a POST request via the Do function.
func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) {
return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
}
// notifyingReader is an io.ReadCloser that closes the notify channel after
// Close is called or a Read fails on the underlying ReadCloser.
type notifyingReader struct {
io.ReadCloser
notify chan<- struct{}
}
func (r *notifyingReader) Read(p []byte) (int, error) {
n, err := r.ReadCloser.Read(p)
if err != nil && r.notify != nil {
close(r.notify)
r.notify = nil
}
return n, err
}
func (r *notifyingReader) Close() error {
err := r.ReadCloser.Close()
if r.notify != nil {
close(r.notify)
r.notify = nil
}
return err
}

@ -1,79 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !plan9,!go1.7
package ctxhttp
import (
"net"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"golang.org/x/net/context"
)
// golang.org/issue/14065
func TestClosesResponseBodyOnCancel(t *testing.T) {
defer func() { testHookContextDoneBeforeHeaders = nop }()
defer func() { testHookDoReturned = nop }()
defer func() { testHookDidBodyClose = nop }()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
defer ts.Close()
ctx, cancel := context.WithCancel(context.Background())
// closed when Do enters select case <-ctx.Done()
enteredDonePath := make(chan struct{})
testHookContextDoneBeforeHeaders = func() {
close(enteredDonePath)
}
testHookDoReturned = func() {
// We now have the result (the Flush'd headers) at least,
// so we can cancel the request.
cancel()
// But block the client.Do goroutine from sending
// until Do enters into the <-ctx.Done() path, since
// otherwise if both channels are readable, select
// picks a random one.
<-enteredDonePath
}
sawBodyClose := make(chan struct{})
testHookDidBodyClose = func() { close(sawBodyClose) }
tr := &http.Transport{}
defer tr.CloseIdleConnections()
c := &http.Client{Transport: tr}
req, _ := http.NewRequest("GET", ts.URL, nil)
_, doErr := Do(ctx, c, req)
select {
case <-sawBodyClose:
case <-time.After(5 * time.Second):
t.Fatal("timeout waiting for body to close")
}
if doErr != ctx.Err() {
t.Errorf("Do error = %v; want %v", doErr, ctx.Err())
}
}
type noteCloseConn struct {
net.Conn
onceClose sync.Once
closefn func()
}
func (c *noteCloseConn) Close() error {
c.onceClose.Do(c.closefn)
return c.Conn.Close()
}

@ -1,105 +0,0 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !plan9
package ctxhttp
import (
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"time"
"golang.org/x/net/context"
)
const (
requestDuration = 100 * time.Millisecond
requestBody = "ok"
)
func okHandler(w http.ResponseWriter, r *http.Request) {
time.Sleep(requestDuration)
io.WriteString(w, requestBody)
}
func TestNoTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(okHandler))
defer ts.Close()
ctx := context.Background()
res, err := Get(ctx, nil, ts.URL)
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
slurp, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
if string(slurp) != requestBody {
t.Errorf("body = %q; want %q", slurp, requestBody)
}
}
func TestCancelBeforeHeaders(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
blockServer := make(chan struct{})
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cancel()
<-blockServer
io.WriteString(w, requestBody)
}))
defer ts.Close()
defer close(blockServer)
res, err := Get(ctx, nil, ts.URL)
if err == nil {
res.Body.Close()
t.Fatal("Get returned unexpected nil error")
}
if err != context.Canceled {
t.Errorf("err = %v; want %v", err, context.Canceled)
}
}
func TestCancelAfterHangingRequest(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.(http.Flusher).Flush()
<-w.(http.CloseNotifier).CloseNotify()
}))
defer ts.Close()
ctx, cancel := context.WithCancel(context.Background())
resp, err := Get(ctx, nil, ts.URL)
if err != nil {
t.Fatalf("unexpected error in Get: %v", err)
}
// Cancel befer reading the body.
// Reading Request.Body should fail, since the request was
// canceled before anything was written.
cancel()
done := make(chan struct{})
go func() {
b, err := ioutil.ReadAll(resp.Body)
if len(b) != 0 || err == nil {
t.Errorf(`Read got (%q, %v); want ("", error)`, b, err)
}
close(done)
}()
select {
case <-time.After(1 * time.Second):
t.Errorf("Test timed out")
case <-done:
}
}

@ -1,72 +0,0 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.7
package context
import (
"context" // standard library's context, as of Go 1.7
"time"
)
var (
todo = context.TODO()
background = context.Background()
)
// Canceled is the error returned by Context.Err when the context is canceled.
var Canceled = context.Canceled
// DeadlineExceeded is the error returned by Context.Err when the context's
// deadline passes.
var DeadlineExceeded = context.DeadlineExceeded
// WithCancel returns a copy of parent with a new Done channel. The returned
// context's Done channel is closed when the returned cancel function is called
// or when the parent context's Done channel is closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
ctx, f := context.WithCancel(parent)
return ctx, CancelFunc(f)
}
// WithDeadline returns a copy of the parent context with the deadline adjusted
// to be no later than d. If the parent's deadline is already earlier than d,
// WithDeadline(parent, d) is semantically equivalent to parent. The returned
// context's Done channel is closed when the deadline expires, when the returned
// cancel function is called, or when the parent context's Done channel is
// closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
ctx, f := context.WithDeadline(parent, deadline)
return ctx, CancelFunc(f)
}
// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete:
//
// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
// defer cancel() // releases resources if slowOperation completes before timeout elapses
// return slowOperation(ctx)
// }
func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
return WithDeadline(parent, time.Now().Add(timeout))
}
// WithValue returns a copy of parent in which the value associated with key is
// val.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
func WithValue(parent Context, key interface{}, val interface{}) Context {
return context.WithValue(parent, key, val)
}

@ -1,20 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.9
package context
import "context" // standard library's context, as of Go 1.7
// A Context carries a deadline, a cancelation signal, and other values across
// API boundaries.
//
// Context's methods may be called by multiple goroutines simultaneously.
type Context = context.Context
// A CancelFunc tells an operation to abandon its work.
// A CancelFunc does not wait for the work to stop.
// After the first call, subsequent calls to a CancelFunc do nothing.
type CancelFunc = context.CancelFunc

@ -1,300 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.7
package context
import (
"errors"
"fmt"
"sync"
"time"
)
// An emptyCtx is never canceled, has no values, and has no deadline. It is not
// struct{}, since vars of this type must have distinct addresses.
type emptyCtx int
func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
return
}
func (*emptyCtx) Done() <-chan struct{} {
return nil
}
func (*emptyCtx) Err() error {
return nil
}
func (*emptyCtx) Value(key interface{}) interface{} {
return nil
}
func (e *emptyCtx) String() string {
switch e {
case background:
return "context.Background"
case todo:
return "context.TODO"
}
return "unknown empty Context"
}
var (
background = new(emptyCtx)
todo = new(emptyCtx)
)
// Canceled is the error returned by Context.Err when the context is canceled.
var Canceled = errors.New("context canceled")
// DeadlineExceeded is the error returned by Context.Err when the context's
// deadline passes.
var DeadlineExceeded = errors.New("context deadline exceeded")
// WithCancel returns a copy of parent with a new Done channel. The returned
// context's Done channel is closed when the returned cancel function is called
// or when the parent context's Done channel is closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
c := newCancelCtx(parent)
propagateCancel(parent, c)
return c, func() { c.cancel(true, Canceled) }
}
// newCancelCtx returns an initialized cancelCtx.
func newCancelCtx(parent Context) *cancelCtx {
return &cancelCtx{
Context: parent,
done: make(chan struct{}),
}
}
// propagateCancel arranges for child to be canceled when parent is.
func propagateCancel(parent Context, child canceler) {
if parent.Done() == nil {
return // parent is never canceled
}
if p, ok := parentCancelCtx(parent); ok {
p.mu.Lock()
if p.err != nil {
// parent has already been canceled
child.cancel(false, p.err)
} else {
if p.children == nil {
p.children = make(map[canceler]bool)
}
p.children[child] = true
}
p.mu.Unlock()
} else {
go func() {
select {
case <-parent.Done():
child.cancel(false, parent.Err())
case <-child.Done():
}
}()
}
}
// parentCancelCtx follows a chain of parent references until it finds a
// *cancelCtx. This function understands how each of the concrete types in this
// package represents its parent.
func parentCancelCtx(parent Context) (*cancelCtx, bool) {
for {
switch c := parent.(type) {
case *cancelCtx:
return c, true
case *timerCtx:
return c.cancelCtx, true
case *valueCtx:
parent = c.Context
default:
return nil, false
}
}
}
// removeChild removes a context from its parent.
func removeChild(parent Context, child canceler) {
p, ok := parentCancelCtx(parent)
if !ok {
return
}
p.mu.Lock()
if p.children != nil {
delete(p.children, child)
}
p.mu.Unlock()
}
// A canceler is a context type that can be canceled directly. The
// implementations are *cancelCtx and *timerCtx.
type canceler interface {
cancel(removeFromParent bool, err error)
Done() <-chan struct{}
}
// A cancelCtx can be canceled. When canceled, it also cancels any children
// that implement canceler.
type cancelCtx struct {
Context
done chan struct{} // closed by the first cancel call.
mu sync.Mutex
children map[canceler]bool // set to nil by the first cancel call
err error // set to non-nil by the first cancel call
}
func (c *cancelCtx) Done() <-chan struct{} {
return c.done
}
func (c *cancelCtx) Err() error {
c.mu.Lock()
defer c.mu.Unlock()
return c.err
}
func (c *cancelCtx) String() string {
return fmt.Sprintf("%v.WithCancel", c.Context)
}
// cancel closes c.done, cancels each of c's children, and, if
// removeFromParent is true, removes c from its parent's children.
func (c *cancelCtx) cancel(removeFromParent bool, err error) {
if err == nil {
panic("context: internal error: missing cancel error")
}
c.mu.Lock()
if c.err != nil {
c.mu.Unlock()
return // already canceled
}
c.err = err
close(c.done)
for child := range c.children {
// NOTE: acquiring the child's lock while holding parent's lock.
child.cancel(false, err)
}
c.children = nil
c.mu.Unlock()
if removeFromParent {
removeChild(c.Context, c)
}
}
// WithDeadline returns a copy of the parent context with the deadline adjusted
// to be no later than d. If the parent's deadline is already earlier than d,
// WithDeadline(parent, d) is semantically equivalent to parent. The returned
// context's Done channel is closed when the deadline expires, when the returned
// cancel function is called, or when the parent context's Done channel is
// closed, whichever happens first.
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete.
func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
// The current deadline is already sooner than the new one.
return WithCancel(parent)
}
c := &timerCtx{
cancelCtx: newCancelCtx(parent),
deadline: deadline,
}
propagateCancel(parent, c)
d := deadline.Sub(time.Now())
if d <= 0 {
c.cancel(true, DeadlineExceeded) // deadline has already passed
return c, func() { c.cancel(true, Canceled) }
}
c.mu.Lock()
defer c.mu.Unlock()
if c.err == nil {
c.timer = time.AfterFunc(d, func() {
c.cancel(true, DeadlineExceeded)
})
}
return c, func() { c.cancel(true, Canceled) }
}
// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to
// implement Done and Err. It implements cancel by stopping its timer then
// delegating to cancelCtx.cancel.
type timerCtx struct {
*cancelCtx
timer *time.Timer // Under cancelCtx.mu.
deadline time.Time
}
func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
return c.deadline, true
}
func (c *timerCtx) String() string {
return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
}
func (c *timerCtx) cancel(removeFromParent bool, err error) {
c.cancelCtx.cancel(false, err)
if removeFromParent {
// Remove this timerCtx from its parent cancelCtx's children.
removeChild(c.cancelCtx.Context, c)
}
c.mu.Lock()
if c.timer != nil {
c.timer.Stop()
c.timer = nil
}
c.mu.Unlock()
}
// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
//
// Canceling this context releases resources associated with it, so code should
// call cancel as soon as the operations running in this Context complete:
//
// func slowOperationWithTimeout(ctx context.Context) (Result, error) {
// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
// defer cancel() // releases resources if slowOperation completes before timeout elapses
// return slowOperation(ctx)
// }
func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
return WithDeadline(parent, time.Now().Add(timeout))
}
// WithValue returns a copy of parent in which the value associated with key is
// val.
//
// Use context Values only for request-scoped data that transits processes and
// APIs, not for passing optional parameters to functions.
func WithValue(parent Context, key interface{}, val interface{}) Context {
return &valueCtx{parent, key, val}
}
// A valueCtx carries a key-value pair. It implements Value for that key and
// delegates all other calls to the embedded Context.
type valueCtx struct {
Context
key, val interface{}
}
func (c *valueCtx) String() string {
return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
}
func (c *valueCtx) Value(key interface{}) interface{} {
if c.key == key {
return c.val
}
return c.Context.Value(key)
}

@ -1,109 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !go1.9
package context
import "time"
// A Context carries a deadline, a cancelation signal, and other values across
// API boundaries.
//
// Context's methods may be called by multiple goroutines simultaneously.
type Context interface {
// Deadline returns the time when work done on behalf of this context
// should be canceled. Deadline returns ok==false when no deadline is
// set. Successive calls to Deadline return the same results.
Deadline() (deadline time.Time, ok bool)
// Done returns a channel that's closed when work done on behalf of this
// context should be canceled. Done may return nil if this context can
// never be canceled. Successive calls to Done return the same value.
//
// WithCancel arranges for Done to be closed when cancel is called;
// WithDeadline arranges for Done to be closed when the deadline
// expires; WithTimeout arranges for Done to be closed when the timeout
// elapses.
//
// Done is provided for use in select statements:
//
// // Stream generates values with DoSomething and sends them to out
// // until DoSomething returns an error or ctx.Done is closed.
// func Stream(ctx context.Context, out chan<- Value) error {
// for {
// v, err := DoSomething(ctx)
// if err != nil {
// return err
// }
// select {
// case <-ctx.Done():
// return ctx.Err()
// case out <- v:
// }
// }
// }
//
// See http://blog.golang.org/pipelines for more examples of how to use
// a Done channel for cancelation.
Done() <-chan struct{}
// Err returns a non-nil error value after Done is closed. Err returns
// Canceled if the context was canceled or DeadlineExceeded if the
// context's deadline passed. No other values for Err are defined.
// After Done is closed, successive calls to Err return the same value.
Err() error
// Value returns the value associated with this context for key, or nil
// if no value is associated with key. Successive calls to Value with
// the same key returns the same result.
//
// Use context values only for request-scoped data that transits
// processes and API boundaries, not for passing optional parameters to
// functions.
//
// A key identifies a specific value in a Context. Functions that wish
// to store values in Context typically allocate a key in a global
// variable then use that key as the argument to context.WithValue and
// Context.Value. A key can be any type that supports equality;
// packages should define keys as an unexported type to avoid
// collisions.
//
// Packages that define a Context key should provide type-safe accessors
// for the values stores using that key:
//
// // Package user defines a User type that's stored in Contexts.
// package user
//
// import "golang.org/x/net/context"
//
// // User is the type of value stored in the Contexts.
// type User struct {...}
//
// // key is an unexported type for keys defined in this package.
// // This prevents collisions with keys defined in other packages.
// type key int
//
// // userKey is the key for user.User values in Contexts. It is
// // unexported; clients use user.NewContext and user.FromContext
// // instead of using this key directly.
// var userKey key = 0
//
// // NewContext returns a new Context that carries value u.
// func NewContext(ctx context.Context, u *User) context.Context {
// return context.WithValue(ctx, userKey, u)
// }
//
// // FromContext returns the User value stored in ctx, if any.
// func FromContext(ctx context.Context) (*User, bool) {
// u, ok := ctx.Value(userKey).(*User)
// return u, ok
// }
Value(key interface{}) interface{}
}
// A CancelFunc tells an operation to abandon its work.
// A CancelFunc does not wait for the work to stop.
// After the first call, subsequent calls to a CancelFunc do nothing.
type CancelFunc func()

@ -1,31 +0,0 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package context_test
import (
"fmt"
"time"
"golang.org/x/net/context"
)
// This example passes a context with a timeout to tell a blocking function that
// it should abandon its work after the timeout elapses.
func ExampleWithTimeout() {
// Pass a context with a timeout to tell a blocking function that it
// should abandon its work after the timeout elapses.
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
select {
case <-time.After(1 * time.Second):
fmt.Println("overslept")
case <-ctx.Done():
fmt.Println(ctx.Err()) // prints "context deadline exceeded"
}
// Output:
// context deadline exceeded
}

@ -1,210 +0,0 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package dict implements the Dictionary Server Protocol
// as defined in RFC 2229.
package dict // import "golang.org/x/net/dict"
import (
"net/textproto"
"strconv"
"strings"
)
// A Client represents a client connection to a dictionary server.
type Client struct {
text *textproto.Conn
}
// Dial returns a new client connected to a dictionary server at
// addr on the given network.
func Dial(network, addr string) (*Client, error) {
text, err := textproto.Dial(network, addr)
if err != nil {
return nil, err
}
_, _, err = text.ReadCodeLine(220)
if err != nil {
text.Close()
return nil, err
}
return &Client{text: text}, nil
}
// Close closes the connection to the dictionary server.
func (c *Client) Close() error {
return c.text.Close()
}
// A Dict represents a dictionary available on the server.
type Dict struct {
Name string // short name of dictionary
Desc string // long description
}
// Dicts returns a list of the dictionaries available on the server.
func (c *Client) Dicts() ([]Dict, error) {
id, err := c.text.Cmd("SHOW DB")
if err != nil {
return nil, err
}
c.text.StartResponse(id)
defer c.text.EndResponse(id)
_, _, err = c.text.ReadCodeLine(110)
if err != nil {
return nil, err
}
lines, err := c.text.ReadDotLines()
if err != nil {
return nil, err
}
_, _, err = c.text.ReadCodeLine(250)
dicts := make([]Dict, len(lines))
for i := range dicts {
d := &dicts[i]
a, _ := fields(lines[i])
if len(a) < 2 {
return nil, textproto.ProtocolError("invalid dictionary: " + lines[i])
}
d.Name = a[0]
d.Desc = a[1]
}
return dicts, err
}
// A Defn represents a definition.
type Defn struct {
Dict Dict // Dict where definition was found
Word string // Word being defined
Text []byte // Definition text, typically multiple lines
}
// Define requests the definition of the given word.
// The argument dict names the dictionary to use,
// the Name field of a Dict returned by Dicts.
//
// The special dictionary name "*" means to look in all the
// server's dictionaries.
// The special dictionary name "!" means to look in all the
// server's dictionaries in turn, stopping after finding the word
// in one of them.
func (c *Client) Define(dict, word string) ([]*Defn, error) {
id, err := c.text.Cmd("DEFINE %s %q", dict, word)
if err != nil {
return nil, err
}
c.text.StartResponse(id)
defer c.text.EndResponse(id)
_, line, err := c.text.ReadCodeLine(150)
if err != nil {
return nil, err
}
a, _ := fields(line)
if len(a) < 1 {
return nil, textproto.ProtocolError("malformed response: " + line)
}
n, err := strconv.Atoi(a[0])
if err != nil {
return nil, textproto.ProtocolError("invalid definition count: " + a[0])
}
def := make([]*Defn, n)
for i := 0; i < n; i++ {
_, line, err = c.text.ReadCodeLine(151)
if err != nil {
return nil, err
}
a, _ := fields(line)
if len(a) < 3 {
// skip it, to keep protocol in sync
i--
n--
def = def[0:n]
continue
}
d := &Defn{Word: a[0], Dict: Dict{a[1], a[2]}}
d.Text, err = c.text.ReadDotBytes()
if err != nil {
return nil, err
}
def[i] = d
}
_, _, err = c.text.ReadCodeLine(250)
return def, err
}
// Fields returns the fields in s.
// Fields are space separated unquoted words
// or quoted with single or double quote.
func fields(s string) ([]string, error) {
var v []string
i := 0
for {
for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
i++
}
if i >= len(s) {
break
}
if s[i] == '"' || s[i] == '\'' {
q := s[i]
// quoted string
var j int
for j = i + 1; ; j++ {
if j >= len(s) {
return nil, textproto.ProtocolError("malformed quoted string")
}
if s[j] == '\\' {
j++
continue
}
if s[j] == q {
j++
break
}
}
v = append(v, unquote(s[i+1:j-1]))
i = j
} else {
// atom
var j int
for j = i; j < len(s); j++ {
if s[j] == ' ' || s[j] == '\t' || s[j] == '\\' || s[j] == '"' || s[j] == '\'' {
break
}
}
v = append(v, s[i:j])
i = j
}
if i < len(s) {
c := s[i]
if c != ' ' && c != '\t' {
return nil, textproto.ProtocolError("quotes not on word boundaries")
}
}
}
return v, nil
}
func unquote(s string) string {
if strings.Index(s, "\\") < 0 {
return s
}
b := []byte(s)
w := 0
for r := 0; r < len(b); r++ {
c := b[r]
if c == '\\' {
r++
c = b[r]
}
b[w] = c
w++
}
return string(b[0:w])
}

@ -1,132 +0,0 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package dnsmessage_test
import (
"fmt"
"net"
"strings"
"golang.org/x/net/dns/dnsmessage"
)
func mustNewName(name string) dnsmessage.Name {
n, err := dnsmessage.NewName(name)
if err != nil {
panic(err)
}
return n
}
func ExampleParser() {
msg := dnsmessage.Message{
Header: dnsmessage.Header{Response: true, Authoritative: true},
Questions: []dnsmessage.Question{
{
Name: mustNewName("foo.bar.example.com."),
Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET,
},
{
Name: mustNewName("bar.example.com."),
Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET,
},
},
Answers: []dnsmessage.Resource{
{
Header: dnsmessage.ResourceHeader{
Name: mustNewName("foo.bar.example.com."),
Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET,
},
Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 1}},
},
{
Header: dnsmessage.ResourceHeader{
Name: mustNewName("bar.example.com."),
Type: dnsmessage.TypeA,
Class: dnsmessage.ClassINET,
},
Body: &dnsmessage.AResource{A: [4]byte{127, 0, 0, 2}},
},
},
}
buf, err := msg.Pack()
if err != nil {
panic(err)
}
wantName := "bar.example.com."
var p dnsmessage.Parser
if _, err := p.Start(buf); err != nil {
panic(err)
}
for {
q, err := p.Question()
if err == dnsmessage.ErrSectionDone {
break
}
if err != nil {
panic(err)
}
if q.Name.String() != wantName {
continue
}
fmt.Println("Found question for name", wantName)
if err := p.SkipAllQuestions(); err != nil {
panic(err)
}
break
}
var gotIPs []net.IP
for {
h, err := p.AnswerHeader()
if err == dnsmessage.ErrSectionDone {
break
}
if err != nil {
panic(err)
}
if (h.Type != dnsmessage.TypeA && h.Type != dnsmessage.TypeAAAA) || h.Class != dnsmessage.ClassINET {
continue
}
if !strings.EqualFold(h.Name.String(), wantName) {
if err := p.SkipAnswer(); err != nil {
panic(err)
}
continue
}
switch h.Type {
case dnsmessage.TypeA:
r, err := p.AResource()
if err != nil {
panic(err)
}
gotIPs = append(gotIPs, r.A[:])
case dnsmessage.TypeAAAA:
r, err := p.AAAAResource()
if err != nil {
panic(err)
}
gotIPs = append(gotIPs, r.AAAA[:])
}
}
fmt.Printf("Found A/AAAA records for name %s: %v\n", wantName, gotIPs)
// Output:
// Found question for name bar.example.com.
// Found A/AAAA records for name bar.example.com.: [127.0.0.2]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -1,109 +0,0 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package atom
import (
"sort"
"testing"
)
func TestKnown(t *testing.T) {
for _, s := range testAtomList {
if atom := Lookup([]byte(s)); atom.String() != s {
t.Errorf("Lookup(%q) = %#x (%q)", s, uint32(atom), atom.String())
}
}
}
func TestHits(t *testing.T) {
for _, a := range table {
if a == 0 {
continue
}
got := Lookup([]byte(a.String()))
if got != a {
t.Errorf("Lookup(%q) = %#x, want %#x", a.String(), uint32(got), uint32(a))
}
}
}
func TestMisses(t *testing.T) {
testCases := []string{
"",
"\x00",
"\xff",
"A",
"DIV",
"Div",
"dIV",
"aa",
"a\x00",
"ab",
"abb",
"abbr0",
"abbr ",
" abbr",
" a",
"acceptcharset",
"acceptCharset",
"accept_charset",
"h0",
"h1h2",
"h7",
"onClick",
"λ",
// The following string has the same hash (0xa1d7fab7) as "onmouseover".
"\x00\x00\x00\x00\x00\x50\x18\xae\x38\xd0\xb7",
}
for _, tc := range testCases {
got := Lookup([]byte(tc))
if got != 0 {
t.Errorf("Lookup(%q): got %d, want 0", tc, got)
}
}
}
func TestForeignObject(t *testing.T) {
const (
afo = Foreignobject
afO = ForeignObject
sfo = "foreignobject"
sfO = "foreignObject"
)
if got := Lookup([]byte(sfo)); got != afo {
t.Errorf("Lookup(%q): got %#v, want %#v", sfo, got, afo)
}
if got := Lookup([]byte(sfO)); got != afO {
t.Errorf("Lookup(%q): got %#v, want %#v", sfO, got, afO)
}
if got := afo.String(); got != sfo {
t.Errorf("Atom(%#v).String(): got %q, want %q", afo, got, sfo)
}
if got := afO.String(); got != sfO {
t.Errorf("Atom(%#v).String(): got %q, want %q", afO, got, sfO)
}
}
func BenchmarkLookup(b *testing.B) {
sortedTable := make([]string, 0, len(table))
for _, a := range table {
if a != 0 {
sortedTable = append(sortedTable, a.String())
}
}
sort.Strings(sortedTable)
x := make([][]byte, 1000)
for i := range x {
x[i] = []byte(sortedTable[i%len(sortedTable)])
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, s := range x {
Lookup(s)
}
}
}

@ -1,376 +0,0 @@
// Code generated by go generate gen.go; DO NOT EDIT.
//go:generate go run gen.go -test
package atom
var testAtomList = []string{
"a",
"abbr",
"accept",
"accept-charset",
"accesskey",
"acronym",
"action",
"address",
"align",
"allowfullscreen",
"allowpaymentrequest",
"allowusermedia",
"alt",
"annotation",
"annotation-xml",
"applet",
"area",
"article",
"as",
"aside",
"async",
"audio",
"autocomplete",
"autofocus",
"autoplay",
"b",
"base",
"basefont",
"bdi",
"bdo",
"bgsound",
"big",
"blink",
"blockquote",
"body",
"br",
"button",
"canvas",
"caption",
"center",
"challenge",
"charset",
"checked",
"cite",
"class",
"code",
"col",
"colgroup",
"color",
"cols",
"colspan",
"command",
"content",
"contenteditable",
"contextmenu",
"controls",
"coords",
"crossorigin",
"data",
"datalist",
"datetime",
"dd",
"default",
"defer",
"del",
"desc",
"details",
"dfn",
"dialog",
"dir",
"dirname",
"disabled",
"div",
"dl",
"download",
"draggable",
"dropzone",
"dt",
"em",
"embed",
"enctype",
"face",
"fieldset",
"figcaption",
"figure",
"font",
"footer",
"for",
"foreignObject",
"foreignobject",
"form",
"formaction",
"formenctype",
"formmethod",
"formnovalidate",
"formtarget",
"frame",
"frameset",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"head",
"header",
"headers",
"height",
"hgroup",
"hidden",
"high",
"hr",
"href",
"hreflang",
"html",
"http-equiv",
"i",
"icon",
"id",
"iframe",
"image",
"img",
"input",
"inputmode",
"ins",
"integrity",
"is",
"isindex",
"ismap",
"itemid",
"itemprop",
"itemref",
"itemscope",
"itemtype",
"kbd",
"keygen",
"keytype",
"kind",
"label",
"lang",
"legend",
"li",
"link",
"list",
"listing",
"loop",
"low",
"main",
"malignmark",
"manifest",
"map",
"mark",
"marquee",
"math",
"max",
"maxlength",
"media",
"mediagroup",
"menu",
"menuitem",
"meta",
"meter",
"method",
"mglyph",
"mi",
"min",
"minlength",
"mn",
"mo",
"ms",
"mtext",
"multiple",
"muted",
"name",
"nav",
"nobr",
"noembed",
"noframes",
"nomodule",
"nonce",
"noscript",
"novalidate",
"object",
"ol",
"onabort",
"onafterprint",
"onautocomplete",
"onautocompleteerror",
"onauxclick",
"onbeforeprint",
"onbeforeunload",
"onblur",
"oncancel",
"oncanplay",
"oncanplaythrough",
"onchange",
"onclick",
"onclose",
"oncontextmenu",
"oncopy",
"oncuechange",
"oncut",
"ondblclick",
"ondrag",
"ondragend",
"ondragenter",
"ondragexit",
"ondragleave",
"ondragover",
"ondragstart",
"ondrop",
"ondurationchange",
"onemptied",
"onended",
"onerror",
"onfocus",
"onhashchange",
"oninput",
"oninvalid",
"onkeydown",
"onkeypress",
"onkeyup",
"onlanguagechange",
"onload",
"onloadeddata",
"onloadedmetadata",
"onloadend",
"onloadstart",
"onmessage",
"onmessageerror",
"onmousedown",
"onmouseenter",
"onmouseleave",
"onmousemove",
"onmouseout",
"onmouseover",
"onmouseup",
"onmousewheel",
"onoffline",
"ononline",
"onpagehide",
"onpageshow",
"onpaste",
"onpause",
"onplay",
"onplaying",
"onpopstate",
"onprogress",
"onratechange",
"onrejectionhandled",
"onreset",
"onresize",
"onscroll",
"onsecuritypolicyviolation",
"onseeked",
"onseeking",
"onselect",
"onshow",
"onsort",
"onstalled",
"onstorage",
"onsubmit",
"onsuspend",
"ontimeupdate",
"ontoggle",
"onunhandledrejection",
"onunload",
"onvolumechange",
"onwaiting",
"onwheel",
"open",
"optgroup",
"optimum",
"option",
"output",
"p",
"param",
"pattern",
"picture",
"ping",
"placeholder",
"plaintext",
"playsinline",
"poster",
"pre",
"preload",
"progress",
"prompt",
"public",
"q",
"radiogroup",
"rb",
"readonly",
"referrerpolicy",
"rel",
"required",
"reversed",
"rows",
"rowspan",
"rp",
"rt",
"rtc",
"ruby",
"s",
"samp",
"sandbox",
"scope",
"scoped",
"script",
"seamless",
"section",
"select",
"selected",
"shape",
"size",
"sizes",
"slot",
"small",
"sortable",
"sorted",
"source",
"spacer",
"span",
"spellcheck",
"src",
"srcdoc",
"srclang",
"srcset",
"start",
"step",
"strike",
"strong",
"style",
"sub",
"summary",
"sup",
"svg",
"system",
"tabindex",
"table",
"target",
"tbody",
"td",
"template",
"textarea",
"tfoot",
"th",
"thead",
"time",
"title",
"tr",
"track",
"translate",
"tt",
"type",
"typemustmatch",
"u",
"ul",
"updateviacache",
"usemap",
"value",
"var",
"video",
"wbr",
"width",
"workertype",
"wrap",
"xmp",
}

@ -1,257 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package charset provides common text encodings for HTML documents.
//
// The mapping from encoding labels to encodings is defined at
// https://encoding.spec.whatwg.org/.
package charset // import "golang.org/x/net/html/charset"
import (
"bytes"
"fmt"
"io"
"mime"
"strings"
"unicode/utf8"
"golang.org/x/net/html"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/encoding/htmlindex"
"golang.org/x/text/transform"
)
// Lookup returns the encoding with the specified label, and its canonical
// name. It returns nil and the empty string if label is not one of the
// standard encodings for HTML. Matching is case-insensitive and ignores
// leading and trailing whitespace. Encoders will use HTML escape sequences for
// runes that are not supported by the character set.
func Lookup(label string) (e encoding.Encoding, name string) {
e, err := htmlindex.Get(label)
if err != nil {
return nil, ""
}
name, _ = htmlindex.Name(e)
return &htmlEncoding{e}, name
}
type htmlEncoding struct{ encoding.Encoding }
func (h *htmlEncoding) NewEncoder() *encoding.Encoder {
// HTML requires a non-terminating legacy encoder. We use HTML escapes to
// substitute unsupported code points.
return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder())
}
// DetermineEncoding determines the encoding of an HTML document by examining
// up to the first 1024 bytes of content and the declared Content-Type.
//
// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) {
if len(content) > 1024 {
content = content[:1024]
}
for _, b := range boms {
if bytes.HasPrefix(content, b.bom) {
e, name = Lookup(b.enc)
return e, name, true
}
}
if _, params, err := mime.ParseMediaType(contentType); err == nil {
if cs, ok := params["charset"]; ok {
if e, name = Lookup(cs); e != nil {
return e, name, true
}
}
}
if len(content) > 0 {
e, name = prescan(content)
if e != nil {
return e, name, false
}
}
// Try to detect UTF-8.
// First eliminate any partial rune at the end.
for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
b := content[i]
if b < 0x80 {
break
}
if utf8.RuneStart(b) {
content = content[:i]
break
}
}
hasHighBit := false
for _, c := range content {
if c >= 0x80 {
hasHighBit = true
break
}
}
if hasHighBit && utf8.Valid(content) {
return encoding.Nop, "utf-8", false
}
// TODO: change default depending on user's locale?
return charmap.Windows1252, "windows-1252", false
}
// NewReader returns an io.Reader that converts the content of r to UTF-8.
// It calls DetermineEncoding to find out what r's encoding is.
func NewReader(r io.Reader, contentType string) (io.Reader, error) {
preview := make([]byte, 1024)
n, err := io.ReadFull(r, preview)
switch {
case err == io.ErrUnexpectedEOF:
preview = preview[:n]
r = bytes.NewReader(preview)
case err != nil:
return nil, err
default:
r = io.MultiReader(bytes.NewReader(preview), r)
}
if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop {
r = transform.NewReader(r, e.NewDecoder())
}
return r, nil
}
// NewReaderLabel returns a reader that converts from the specified charset to
// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
// returns an error if Lookup returns nil. It is suitable for use as
// encoding/xml.Decoder's CharsetReader function.
func NewReaderLabel(label string, input io.Reader) (io.Reader, error) {
e, _ := Lookup(label)
if e == nil {
return nil, fmt.Errorf("unsupported charset: %q", label)
}
return transform.NewReader(input, e.NewDecoder()), nil
}
func prescan(content []byte) (e encoding.Encoding, name string) {
z := html.NewTokenizer(bytes.NewReader(content))
for {
switch z.Next() {
case html.ErrorToken:
return nil, ""
case html.StartTagToken, html.SelfClosingTagToken:
tagName, hasAttr := z.TagName()
if !bytes.Equal(tagName, []byte("meta")) {
continue
}
attrList := make(map[string]bool)
gotPragma := false
const (
dontKnow = iota
doNeedPragma
doNotNeedPragma
)
needPragma := dontKnow
name = ""
e = nil
for hasAttr {
var key, val []byte
key, val, hasAttr = z.TagAttr()
ks := string(key)
if attrList[ks] {
continue
}
attrList[ks] = true
for i, c := range val {
if 'A' <= c && c <= 'Z' {
val[i] = c + 0x20
}
}
switch ks {
case "http-equiv":
if bytes.Equal(val, []byte("content-type")) {
gotPragma = true
}
case "content":
if e == nil {
name = fromMetaElement(string(val))
if name != "" {
e, name = Lookup(name)
if e != nil {
needPragma = doNeedPragma
}
}
}
case "charset":
e, name = Lookup(string(val))
needPragma = doNotNeedPragma
}
}
if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
continue
}
if strings.HasPrefix(name, "utf-16") {
name = "utf-8"
e = encoding.Nop
}
if e != nil {
return e, name
}
}
}
}
func fromMetaElement(s string) string {
for s != "" {
csLoc := strings.Index(s, "charset")
if csLoc == -1 {
return ""
}
s = s[csLoc+len("charset"):]
s = strings.TrimLeft(s, " \t\n\f\r")
if !strings.HasPrefix(s, "=") {
continue
}
s = s[1:]
s = strings.TrimLeft(s, " \t\n\f\r")
if s == "" {
return ""
}
if q := s[0]; q == '"' || q == '\'' {
s = s[1:]
closeQuote := strings.IndexRune(s, rune(q))
if closeQuote == -1 {
return ""
}
return s[:closeQuote]
}
end := strings.IndexAny(s, "; \t\n\f\r")
if end == -1 {
end = len(s)
}
return s[:end]
}
return ""
}
var boms = []struct {
bom []byte
enc string
}{
{[]byte{0xfe, 0xff}, "utf-16be"},
{[]byte{0xff, 0xfe}, "utf-16le"},
{[]byte{0xef, 0xbb, 0xbf}, "utf-8"},
}

@ -1,237 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package charset
import (
"bytes"
"encoding/xml"
"io/ioutil"
"runtime"
"strings"
"testing"
"golang.org/x/text/transform"
)
func transformString(t transform.Transformer, s string) (string, error) {
r := transform.NewReader(strings.NewReader(s), t)
b, err := ioutil.ReadAll(r)
return string(b), err
}
type testCase struct {
utf8, other, otherEncoding string
}
// testCases for encoding and decoding.
var testCases = []testCase{
{"Résumé", "Résumé", "utf8"},
{"Résumé", "R\xe9sum\xe9", "latin1"},
{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
{"Hello, world", "Hello, world", "ASCII"},
{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
{"latviešu", "latvie\xf0u", "ISO-8859-13"},
{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
{"Kağan", "Ka\xf0an", "ISO-8859-9"},
{"Résumé", "R\x8esum\x8e", "macintosh"},
{"Gdańsk", "Gda\xf1sk", "windows-1250"},
{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
{"Résumé", "R\xe9sum\xe9", "windows-1252"},
{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
{"Kağan", "Ka\xf0an", "windows-1254"},
{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
{"latviešu", "latvie\xf0u", "windows-1257"},
{"Việt", "Vi\xea\xf2t", "windows-1258"},
{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
{"㧯", "\x82\x31\x89\x38", "gb18030"},
{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
{"イウエオカ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
{"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
}
func TestDecode(t *testing.T) {
testCases := append(testCases, []testCase{
// Replace multi-byte maximum subpart of ill-formed subsequence with
// single replacement character (WhatWG requirement).
{"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
}...)
for _, tc := range testCases {
e, _ := Lookup(tc.otherEncoding)
if e == nil {
t.Errorf("%s: not found", tc.otherEncoding)
continue
}
s, err := transformString(e.NewDecoder(), tc.other)
if err != nil {
t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
continue
}
if s != tc.utf8 {
t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
}
}
}
func TestEncode(t *testing.T) {
testCases := append(testCases, []testCase{
// Use Go-style replacement.
{"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
// U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
{"Gdańsk", "Gda&#324;sk", "ISO-8859-11"},
{"\ufffd", "&#65533;", "ISO-8859-11"},
{"a\xe1\x80b", "a&#65533;&#65533;b", "ISO-8859-11"},
}...)
for _, tc := range testCases {
e, _ := Lookup(tc.otherEncoding)
if e == nil {
t.Errorf("%s: not found", tc.otherEncoding)
continue
}
s, err := transformString(e.NewEncoder(), tc.utf8)
if err != nil {
t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
continue
}
if s != tc.other {
t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
}
}
}
var sniffTestCases = []struct {
filename, declared, want string
}{
{"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
{"UTF-16LE-BOM.html", "", "utf-16le"},
{"UTF-16BE-BOM.html", "", "utf-16be"},
{"meta-content-attribute.html", "text/html", "iso-8859-15"},
{"meta-charset-attribute.html", "text/html", "iso-8859-15"},
{"No-encoding-declaration.html", "text/html", "utf-8"},
{"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
{"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
{"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
{"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
{"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
}
func TestSniff(t *testing.T) {
switch runtime.GOOS {
case "nacl": // platforms that don't permit direct file system access
t.Skipf("not supported on %q", runtime.GOOS)
}
for _, tc := range sniffTestCases {
content, err := ioutil.ReadFile("testdata/" + tc.filename)
if err != nil {
t.Errorf("%s: error reading file: %v", tc.filename, err)
continue
}
_, name, _ := DetermineEncoding(content, tc.declared)
if name != tc.want {
t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
continue
}
}
}
func TestReader(t *testing.T) {
switch runtime.GOOS {
case "nacl": // platforms that don't permit direct file system access
t.Skipf("not supported on %q", runtime.GOOS)
}
for _, tc := range sniffTestCases {
content, err := ioutil.ReadFile("testdata/" + tc.filename)
if err != nil {
t.Errorf("%s: error reading file: %v", tc.filename, err)
continue
}
r, err := NewReader(bytes.NewReader(content), tc.declared)
if err != nil {
t.Errorf("%s: error creating reader: %v", tc.filename, err)
continue
}
got, err := ioutil.ReadAll(r)
if err != nil {
t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
continue
}
e, _ := Lookup(tc.want)
want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
if err != nil {
t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
continue
}
if !bytes.Equal(got, want) {
t.Errorf("%s: got %q, want %q", tc.filename, got, want)
continue
}
}
}
var metaTestCases = []struct {
meta, want string
}{
{"", ""},
{"text/html", ""},
{"text/html; charset utf-8", ""},
{"text/html; charset=latin-2", "latin-2"},
{"text/html; charset; charset = utf-8", "utf-8"},
{`charset="big5"`, "big5"},
{"charset='shift_jis'", "shift_jis"},
}
func TestFromMeta(t *testing.T) {
for _, tc := range metaTestCases {
got := fromMetaElement(tc.meta)
if got != tc.want {
t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
}
}
}
func TestXML(t *testing.T) {
const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><a><Word>r\xe9sum\xe9</Word></a>"
d := xml.NewDecoder(strings.NewReader(s))
d.CharsetReader = NewReaderLabel
var a struct {
Word string
}
err := d.Decode(&a)
if err != nil {
t.Fatalf("Decode: %v", err)
}
want := "résumé"
if a.Word != want {
t.Errorf("got %q, want %q", a.Word, want)
}
}

@ -1,48 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<title>HTTP charset</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="The character encoding of a page can be set using the HTTP header charset declaration.">
<style type='text/css'>
.test div { width: 50px; }</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-15.css">
</head>
<body>
<p class='title'>HTTP charset</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">The character encoding of a page can be set using the HTTP header charset declaration.</p>
<div class="notes"><p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00C3;&#x0153;&#x00C3;&#x20AC;&#x00C3;&#x0161;</code>. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.</p><p>The only character encoding declaration for this HTML file is in the HTTP header, which sets the encoding to ISO 8859-15.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-003">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-001<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#basics" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-001" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,48 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<title>HTTP vs UTF-8 BOM</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.">
<style type='text/css'>
.test div { width: 50px; }</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-utf8.css">
</head>
<body>
<p class='title'>HTTP vs UTF-8 BOM</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.</p>
<div class="notes"><p><p>The HTTP header attempts to set the character encoding to ISO 8859-15. The page starts with a UTF-8 signature.</p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00FD;&#x00E4;&#x00E8;</code>. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.</p><p>If the test is unsuccessful, the characters &#x00EF;&#x00BB;&#x00BF; should appear at the top of the page. These represent the bytes that make up the UTF-8 signature when encountered in the ISO 8859-15 encoding.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-022">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-034<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#precedence" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-034" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<meta charset="iso-8859-1" > <title>HTTP vs meta charset</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.">
<style type='text/css'>
.test div { width: 50px; }.test div { width: 90px; }
</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-15.css">
</head>
<body>
<p class='title'>HTTP vs meta charset</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.</p>
<div class="notes"><p><p>The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-1.</p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00C3;&#x0153;&#x00C3;&#x20AC;&#x00C3;&#x0161;</code>. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-037">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-018<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#precedence" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-018" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<meta http-equiv="content-type" content="text/html;charset=iso-8859-1" > <title>HTTP vs meta content</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.">
<style type='text/css'>
.test div { width: 50px; }.test div { width: 90px; }
</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-15.css">
</head>
<body>
<p class='title'>HTTP vs meta content</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.</p>
<div class="notes"><p><p>The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-1.</p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00C3;&#x0153;&#x00C3;&#x20AC;&#x00C3;&#x0161;</code>. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-018">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-016<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#precedence" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-016" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,47 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<title>No encoding declaration</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.">
<style type='text/css'>
.test div { width: 50px; }</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-utf8.css">
</head>
<body>
<p class='title'>No encoding declaration</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.</p>
<div class="notes"><p><p>The test on this page contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00FD;&#x00E4;&#x00E8;</code>. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-034">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-015<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#basics" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-015" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,9 +0,0 @@
These test cases come from
http://www.w3.org/International/tests/repository/html5/the-input-byte-stream/results-basics
Distributed under both the W3C Test Suite License
(http://www.w3.org/Consortium/Legal/2008/04-testsuite-license)
and the W3C 3-clause BSD License
(http://www.w3.org/Consortium/Legal/2008/03-bsd-license).
To contribute to a W3C Test Suite, see the policies and contribution
forms (http://www.w3.org/2004/10/27-testcases).

@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<meta charset="iso-8859-15"> <title>UTF-8 BOM vs meta charset</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.">
<style type='text/css'>
.test div { width: 50px; }.test div { width: 90px; }
</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-utf8.css">
</head>
<body>
<p class='title'>UTF-8 BOM vs meta charset</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.</p>
<div class="notes"><p><p>The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.</p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00FD;&#x00E4;&#x00E8;</code>. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-024">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-038<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#precedence" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-038" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

@ -1,48 +0,0 @@
<!DOCTYPE html>
<html lang="en" >
<head>
<meta http-equiv="content-type" content="text/html; charset=iso-8859-15"> <title>UTF-8 BOM vs meta content</title>
<link rel='author' title='Richard Ishida' href='mailto:ishida@w3.org'>
<link rel='help' href='http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream'>
<link rel="stylesheet" type="text/css" href="./generatedtests.css">
<script src="http://w3c-test.org/resources/testharness.js"></script>
<script src="http://w3c-test.org/resources/testharnessreport.js"></script>
<meta name='flags' content='http'>
<meta name="assert" content="A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.">
<style type='text/css'>
.test div { width: 50px; }</style>
<link rel="stylesheet" type="text/css" href="the-input-byte-stream/support/encodingtests-utf8.css">
</head>
<body>
<p class='title'>UTF-8 BOM vs meta content</p>
<div id='log'></div>
<div class='test'><div id='box' class='ýäè'>&#xA0;</div></div>
<div class='description'>
<p class="assertion" title="Assertion">A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.</p>
<div class="notes"><p><p>The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.</p><p>The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector <code>.test div.&#x00FD;&#x00E4;&#x00E8;</code>. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.</p></p>
</div>
</div>
<div class="nexttest"><div><a href="generate?test=the-input-byte-stream-038">Next test</a></div><div class="doctype">HTML5</div>
<p class="jump">the-input-byte-stream-037<br /><a href="/International/tests/html5/the-input-byte-stream/results-basics#precedence" target="_blank">Result summary &amp; related tests</a><br /><a href="http://w3c-test.org/framework/details/i18n-html5/the-input-byte-stream-037" target="_blank">Detailed results for this test</a><br/> <a href="http://www.w3.org/TR/html5/syntax.html#the-input-byte-stream" target="_blank">Link to spec</a></p>
<div class='prereq'>Assumptions: <ul><li>The default encoding for the browser you are testing is not set to ISO 8859-15.</li>
<li>The test is read from a server that supports HTTP.</li></ul></div>
</div>
<script>
test(function() {
assert_equals(document.getElementById('box').offsetWidth, 100);
}, " ");
</script>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save