Skip to content

encoding/mvt: use protoscan for unmarshalling #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulmach merged 1 commit into from
Jan 16, 2021
Merged

encoding/mvt: use protoscan for unmarshalling #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 0 additions & 164 deletions encoding/mvt/geometry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,170 +140,6 @@ type keyValueEncoder struct {
valueMap map[interface{}]uint32
}

// A geomDecoder holds state for geometry decoding.
type geomDecoder struct {
geom []uint32
i int

prev orb.Point
}

func decodeGeometry(geomType vectortile.Tile_GeomType, geom []uint32) (orb.Geometry, error) {
if len(geom) < 2 {
return nil, errors.Errorf("geom is not long enough: %v", geom)
}

gd := &geomDecoder{geom: geom}

switch geomType {
case vectortile.Tile_POINT:
return gd.decodePoint()
case vectortile.Tile_LINESTRING:
return gd.decodeLineString()
case vectortile.Tile_POLYGON:
return gd.decodePolygon()
}

return nil, errors.Errorf("unknown geometry type: %v", geomType)
}

func (gd *geomDecoder) decodePoint() (orb.Geometry, error) {
_, count, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if count == 1 {
return gd.NextPoint(), nil
}

mp := make(orb.MultiPoint, 0, count)
for i := uint32(0); i < count; i++ {
mp = append(mp, gd.NextPoint())
}

return mp, nil
}

func (gd *geomDecoder) decodeLine() (orb.LineString, error) {
cmd, count, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd != moveTo || count != 1 {
return nil, errors.New("first command not one moveTo")
}

first := gd.NextPoint()
cmd, count, err = gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd != lineTo {
return nil, errors.New("second command not a lineTo")
}

ls := make(orb.LineString, 0, count+1)
ls = append(ls, first)

for i := uint32(0); i < count; i++ {
ls = append(ls, gd.NextPoint())
}

return ls, nil
}

func (gd *geomDecoder) decodeLineString() (orb.Geometry, error) {
var mls orb.MultiLineString
for !gd.done() {
ls, err := gd.decodeLine()
if err != nil {
return nil, err
}

if gd.done() && len(mls) == 0 {
return ls, nil
}

mls = append(mls, ls)
}

return mls, nil
}

func (gd *geomDecoder) decodePolygon() (orb.Geometry, error) {
var mp orb.MultiPolygon
var p orb.Polygon
for !gd.done() {
ls, err := gd.decodeLine()
if err != nil {
return nil, err
}

r := orb.Ring(ls)

cmd, _, err := gd.cmdAndCount()
if err != nil {
return nil, err
}

if cmd == closePath && !r.Closed() {
r = append(r, r[0])
}

// figure out if new polygon
if len(mp) == 0 && len(p) == 0 {
p = append(p, r)
} else {
if r.Orientation() == orb.CCW {
mp = append(mp, p)
p = orb.Polygon{r}
} else {
p = append(p, r)
}
}
}

if len(mp) == 0 {
return p, nil
}

return append(mp, p), nil
}

func (gd *geomDecoder) cmdAndCount() (uint32, uint32, error) {
if gd.i >= len(gd.geom) {
return 0, 0, errors.New("no more data")
}

v := gd.geom[gd.i]

cmd := v & 0x07
count := v >> 3
gd.i++

if cmd != closePath {
if v := gd.i + int(2*count); len(gd.geom) < v {
return 0, 0, errors.Errorf("data cut short: needed %d, have %d", v, len(gd.geom))
}
}

return cmd, count, nil
}

func (gd *geomDecoder) NextPoint() orb.Point {
gd.i += 2
gd.prev[0] += unzigzag(gd.geom[gd.i-2])
gd.prev[1] += unzigzag(gd.geom[gd.i-1])
return gd.prev
}

func (gd *geomDecoder) done() bool {
return gd.i >= len(gd.geom)
}

func newKeyValueEncoder() *keyValueEncoder {
return &keyValueEncoder{
keyMap: make(map[string]uint32),
Expand Down
32 changes: 31 additions & 1 deletion encoding/mvt/geometry_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/paulmach/orb"
"github.com/paulmach/orb/encoding/mvt/vectortile"
"github.com/paulmach/protoscan"
)

func TestGeometry_Point(t *testing.T) {
Expand Down Expand Up @@ -322,7 +323,8 @@ func compareGeometry(
t.Errorf("different encoding")
}

result, err := decodeGeometry(geomType, input)
d := &decoder{geom: sliceToIterator(input)}
result, err := d.Geometry(geomType)
if err != nil {
t.Fatalf("decode error: %v", err)
}
Expand All @@ -337,3 +339,31 @@ func compareGeometry(
t.Errorf("geometry not equal")
}
}

func sliceToIterator(vals []uint32) *protoscan.Iterator {
feature := &vectortile.Tile_Feature{
Geometry: vals,
}

data, err := feature.Marshal()
if err != nil {
panic(err)
}

msg := protoscan.New(data)
for msg.Next() {
switch msg.FieldNumber() {
case 4:
iter, err := msg.Iterator(nil)
if err != nil {
panic(err)
}

return iter
default:
msg.Skip()
}
}

panic("unreachable")
}
97 changes: 0 additions & 97 deletions encoding/mvt/marshal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"strconv"

"github.com/paulmach/orb/encoding/mvt/vectortile"
Expand Down Expand Up @@ -84,72 +83,6 @@ func Marshal(layers Layers) ([]byte, error) {
return proto.Marshal(vt)
}

// UnmarshalGzipped takes gzipped Mapbox Vector Tile (MVT) data and unzips it
// before decoding it into a set of layers, It does not project the coordinates.
func UnmarshalGzipped(data []byte) (Layers, error) {
gzreader, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, errors.WithMessage(err, "failed to create gzreader")
}

decoded, err := ioutil.ReadAll(gzreader)
if err != nil {
return nil, errors.WithMessage(err, "failed to unzip")
}

return Unmarshal(decoded)
}

// Unmarshal takes Mapbox Vector Tile (MVT) data and converts into a
// set of layers, It does not project the coordinates.
func Unmarshal(data []byte) (Layers, error) {
vt := &vectortile.Tile{}
err := vt.Unmarshal(data)
if err != nil {
return nil, err
}

return decode(vt)
}

func decode(vt *vectortile.Tile) (Layers, error) {
result := make(Layers, 0, len(vt.Layers))
for i, l := range vt.Layers {
layer := &Layer{
Name: l.GetName(),
Version: l.GetVersion(),
Extent: l.GetExtent(),
Features: make([]*geojson.Feature, 0, len(l.Features)),
}

for j, f := range l.Features {
geom, err := decodeGeometry(f.GetType(), f.Geometry)
if err != nil {
return nil, errors.WithMessage(err, fmt.Sprintf("layer %d: feature %d", i, j))
}

properties := decodeFeatureProperties(l.Keys, l.Values, f.Tags)

if geom != nil {
gjf := &geojson.Feature{
Geometry: geom,
Properties: properties,
}

if f.Id != nil {
gjf.ID = float64(*f.Id)
}

layer.Features = append(layer.Features, gjf)
}
}

result = append(result, layer)
}

return result, nil
}

func encodeProperties(kve *keyValueEncoder, properties geojson.Properties) ([]uint32, error) {
tags := make([]uint32, 0, 2*len(properties))
for k, v := range properties {
Expand All @@ -165,36 +98,6 @@ func encodeProperties(kve *keyValueEncoder, properties geojson.Properties) ([]ui
return tags, nil
}

func decodeFeatureProperties(
keys []string,
values []*vectortile.Tile_Value,
tags []uint32,
) geojson.Properties {
result := make(geojson.Properties, len(tags)/2)
if len(tags) == 0 {
return result
}

for i := 2; i <= len(tags); i += 2 {
vi := tags[i-1]
if int(vi) >= len(values) {
continue
}

v := decodeValue(values[vi])
if v != nil {
ti := tags[i-2]
if int(ti) >= len(keys) {
continue
}

result[keys[ti]] = v
}
}

return result
}

func convertID(id interface{}) *uint64 {
if id == nil {
return nil
Expand Down
22 changes: 0 additions & 22 deletions encoding/mvt/marshal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"testing"

"github.com/paulmach/orb"
"github.com/paulmach/orb/encoding/mvt/vectortile"
"github.com/paulmach/orb/geojson"
"github.com/paulmach/orb/maptile"
)
Expand Down Expand Up @@ -484,27 +483,6 @@ func BenchmarkUnmarshal(b *testing.B) {
}
}

func BenchmarkDecode(b *testing.B) {
layers := NewLayers(loadGeoJSON(b, maptile.New(17896, 24449, 16)))
data, err := Marshal(layers)
if err != nil {
b.Fatalf("marshal error: %v", err)
}

vt := &vectortile.Tile{}
err = vt.Unmarshal(data)
if err != nil {
b.Fatalf("unmarshal error: %v", err)
}

b.ReportAllocs()
b.ResetTimer()

for i := 0; i < b.N; i++ {
decode(vt)
}
}

func BenchmarkProjectToTile(b *testing.B) {
tile := maptile.New(17896, 24449, 16)
layers := NewLayers(loadGeoJSON(b, maptile.New(17896, 24449, 16)))
Expand Down
Loading
⚡