Unmarshal JSON with some known, and some unknown field names

I have the following JSON

{"a":1, "b":2, "?":1, "??":1}

I know that it has the "a" and "b" fields, but I don't know the names of other fields. So I want to unmarshal it in the following type:

type Foo struct {
  // Known fields
  A int `json:"a"`
  B int `json:"b"`
  // Unknown fields
  X map[string]interface{} `json:???` // Rest of the fields should go here.
}

How do I do that?

Unmarshal twice

One option is to unmarshal twice: once into a value of type Foo and once into a value of type map[string]interface{} and removing the keys "a" and "b":

type Foo struct {
	A int                    `json:"a"`
	B int                    `json:"b"`
	X map[string]interface{} `json:"-"` // Rest of the fields should go here.
}

func main() {
	s := `{"a":1, "b":2, "x":1, "y":1}`
	f := Foo{}
	if err := json.Unmarshal([]byte(s), &f); err != nil {
		panic(err)
	}

	if err := json.Unmarshal([]byte(s), &f.X); err != nil {
		panic(err)
	}
	delete(f.X, "a")
	delete(f.X, "b")

	fmt.Printf("%+v", f)
}

Output (try it on the Go Playground):

{A:1 B:2 X:map[x:1 y:1]}

Unmarshal once and manual handling

Another option is to unmarshal once into an map[string]interface{} and handle the Foo.A and Foo.B fields manually:

type Foo struct {
	A int                    `json:"a"`
	B int                    `json:"b"`
	X map[string]interface{} `json:"-"` // Rest of the fields should go here.
}

func main() {
	s := `{"a":1, "b":2, "x":1, "y":1}`
	f := Foo{}
	if err := json.Unmarshal([]byte(s), &f.X); err != nil {
		panic(err)
	}
	if n, ok := f.X["a"].(float64); ok {
		f.A = int(n)
	}
	if n, ok := f.X["b"].(float64); ok {
		f.B = int(n)
	}
	delete(f.X, "a")
	delete(f.X, "b")

	fmt.Printf("%+v", f)
}

Output is the same (Go Playground):

{A:1 B:2 X:map[x:1 y:1]}

It's not nice, but you could to it by implementing Unmarshaler:

type _Foo Foo

func (f *Foo) UnmarshalJSON(bs []byte) (err error) {
	foo := _Foo{}

	if err = json.Unmarshal(bs, &foo); err == nil {
		*f = Foo(foo)
	}

	m := make(map[string]interface{})

	if err = json.Unmarshal(bs, &m); err == nil {
		delete(m, "a")
		delete(m, "b")
		f.X = m
	}

	return err
}

The type _Foo is necessary to avoid recursion while decoding.

Simplest way is to use an interface like this:

var f interface{}
s := `{"a":1, "b":2, "x":1, "y":1}`

if err := json.Unmarshal([]byte(s), &f); err != nil {
	panic(err)
}

Go Playground example

I use interface to unmarshal uncertain type json.

bytes := []byte(`{"name":"Liam","gender":1, "salary": 1}`)
var p2 interface{}
json.Unmarshal(bytes, &p2)
m := p2.(map[string]interface{})
fmt.Println(m)

Almost single pass, uses json.RawMessage

We can unmarshal into map[string]json.RawMessage, and then unmarshal each field separately.

JSON will be tokenized twice, but that's quite cheap.

The following helper function can be used:

func UnmarshalJsonObject(jsonStr []byte, obj interface{}, otherFields map[string]json.RawMessage) (err error) {
	objValue := reflect.ValueOf(obj).Elem()
	knownFields := map[string]reflect.Value{}
	for i := 0; i != objValue.NumField(); i++ {
		jsonName := strings.Split(objValue.Type().Field(i).Tag.Get("json"), ",")[0]
		knownFields[jsonName] = objValue.Field(i)
	}

	err = json.Unmarshal(jsonStr, &otherFields)
	if err != nil {
		return
	}

	for key, chunk := range otherFields {
		if field, found := knownFields[key]; found {
			err = json.Unmarshal(chunk, field.Addr().Interface())
			if err != nil {
				return
			}
			delete(otherFields, key)
		}
	}
	return
}

Here is the complete code on Go Playground - http://play.golang.org/p/EtkJUzMmKt

Single pass, use github.com/ugorji/go/codec

When unmarshaling into a map, encoding/json empties the map, but ugorji/go/codec doesn't. It also attempts to fill existing values, so we can put pointers to foo.A, foo.B into foo.X:

package main

import (
	"fmt"
	"github.com/ugorji/go/codec"
)

type Foo struct {
	A int
	B int
	X map[string]interface{}
}

func (this *Foo) UnmarshalJSON(jsonStr []byte) (err error) {
	this.X = make(map[string]interface{})
	this.X["a"] = &this.A
	this.X["b"] = &this.B
	return codec.NewDecoderBytes(jsonStr, &codec.JsonHandle{}).Decode(&this.X)
}

func main() {
	s := `{"a":1, "b":2, "x":3, "y":[]}`
	f := &Foo{}
	err := codec.NewDecoderBytes([]byte(s), &codec.JsonHandle{}).Decode(f)
	fmt.Printf("err = %v\n", err)
	fmt.Printf("%+v\n", f)
}

Use Hashicorp's map-to-struct decoder, which keeps track of unused fields: https://godoc.org/github.com/mitchellh/mapstructure#example-Decode--Metadata

It's two-pass, but you don't have to use known field names anywhere.

func UnmarshalJson(input []byte, result interface{}) (map[string]interface{}, error) {
	// unmarshal json to a map
	foomap := make(map[string]interface{})
	json.Unmarshal(input, &foomap)

	// create a mapstructure decoder
	var md mapstructure.Metadata
	decoder, err := mapstructure.NewDecoder(
		&mapstructure.DecoderConfig{
			Metadata: &md,
			Result:   result,
		})
	if err != nil {
		return nil, err
	}

	// decode the unmarshalled map into the given struct
	if err := decoder.Decode(foomap); err != nil {
		return nil, err
	}

	// copy and return unused fields
	unused := map[string]interface{}{}
	for _, k := range md.Unused {
		unused[k] = foomap[k]
	}
	return unused, nil
}

type Foo struct {
	// Known fields
	A int
	B int
	// Unknown fields
	X map[string]interface{} // Rest of the fields should go here.
}

func main() {
	s := []byte(`{"a":1, "b":2, "?":3, "??":4}`)

	var foo Foo
	unused, err := UnmarshalJson(s, &foo)
	if err != nil {
		panic(err)
	}

	foo.X = unused
	fmt.Println(foo) // prints {1 2 map[?:3 ??:4]}
}