Go- Copy all common fields between structs

I have a database that stores JSON, and a server that provides an external API to whereby through an HTTP post, values in this database can be changed. The database is used by different processes internally, and as such have a common naming scheme.

The keys the customer sees are different, but map 1:1 with the keys in the database (there are unexposed keys). For example:

This is in the database:

{ "bit_size": 8, "secret_key": false }

And this is presented to the client:

{ "num_bits": 8 }

The API can change with respect to field names, but the database always has consistent keys.

I have named the fields the same in the struct, with different flags to the json encoder:

type DB struct {
    NumBits int  `json:"bit_size"`
    Secret  bool `json:"secret_key"`
}
type User struct {
    NumBits int `json:"num_bits"`
}

I'm using encoding/json to do the Marshal/Unmarshal.

Is reflect the right tool for this? Is there an easier way since all of the keys are the same? I was thinking some kind of memcpy (if I kept the user fields in the same order).

Couldn't struct embedding be useful here?

package main

import (
	"fmt"
)

type DB struct {
	User
	Secret bool `json:"secret_key"`
}

type User struct {
	NumBits int `json:"num_bits"`
}

func main() {
	db := DB{User{10}, true}
	fmt.Printf("Hello, DB: %+v\n", db)
	fmt.Printf("Hello, DB.NumBits: %+v\n", db.NumBits)
	fmt.Printf("Hello, User: %+v\n", db.User)
}

http://play.golang.org/p/9s4bii3tQ2

buf := bytes.Buffer{}
err := gob.NewEncoder(&buf).Encode(&DbVar)
if err != nil {
    return err
}
u := User{}
err = gob.NewDecoder(&buf).Decode(&u)
if err != nil {
    return err
}

Here's a solution using reflection. You have to further develop it if you need more complex structures with embedded struct fields and such.

http://play.golang.org/p/iTaDgsdSaI

package main

import (
	"encoding/json"
	"fmt"
	"reflect"
)

type M map[string]interface{} // just an alias

var Record = []byte(`{ "bit_size": 8, "secret_key": false }`)

type DB struct {
	NumBits int  `json:"bit_size"`
	Secret  bool `json:"secret_key"`
}

type User struct {
	NumBits int `json:"num_bits"`
}

func main() {
	d := new(DB)
	e := json.Unmarshal(Record, d)
	if e != nil {
		panic(e)
	}
	m := mapFields(d)
	fmt.Println("Mapped fields: ", m)
	u := new(User)
	o := applyMap(u, m)
	fmt.Println("Applied map: ", o)
	j, e := json.Marshal(o)
	if e != nil {
		panic(e)
	}
	fmt.Println("Output JSON: ", string(j))
}

func applyMap(u *User, m M) M {
	t := reflect.TypeOf(u).Elem()
	o := make(M)
	for i := 0; i < t.NumField(); i++ {
		f := t.FieldByIndex([]int{i})
		// skip unexported fields
		if f.PkgPath != "" {
			continue
		}
		if x, ok := m[f.Name]; ok {
			k := f.Tag.Get("json")
			o[k] = x
		}
	}
	return o
}

func mapFields(x *DB) M {
	o := make(M)
	v := reflect.ValueOf(x).Elem()
	t := v.Type()
	for i := 0; i < v.NumField(); i++ {
		f := t.FieldByIndex([]int{i})
		// skip unexported fields
		if f.PkgPath != "" {
			continue
		}
		o[f.Name] = v.FieldByIndex([]int{i}).Interface()
	}
	return o
}

Using struct tags, the following would sure be nice,

package main

import (
    "fmt"
    "log"

    "hacked/json"
)

var dbj = `{ "bit_size": 8, "secret_key": false }`

type User struct {
    NumBits int `json:"bit_size" api:"num_bits"`
}

func main() {
    fmt.Println(dbj)
    // unmarshal from full db record to User struct
    var u User
    if err := json.Unmarshal([]byte(dbj), &u); err != nil {
        log.Fatal(err)
    }
    // remarshal User struct using api field names 
    api, err := json.MarshalTag(u, "api")
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(string(api))
}

Adding MarshalTag requires just a small patch to encode.go:

106c106,112
<       e := &encodeState{}
---
>       return MarshalTag(v, "json")
> }
> 
> // MarshalTag is like Marshal but marshalls fields with
> // the specified tag key instead of the default "json".
> func MarshalTag(v interface{}, tag string) ([]byte, error) {
>       e := &encodeState{tagKey: tag}
201a208
>       tagKey       string
328c335
<               for _, ef := range encodeFields(v.Type()) {
---
>               for _, ef := range encodeFields(v.Type(), e.tagKey) {
509c516
< func encodeFields(t reflect.Type) []encodeField {
---
> func encodeFields(t reflect.Type, tagKey string) []encodeField {
540c547
<               tv := f.Tag.Get("json")
---
>               tv := f.Tag.Get(tagKey)

The following function use reflect to copy fields between two structs. A src field is copied to a dest field if they have the same field name.

// CopyCommonFields copies src fields into dest fields. A src field is copied 
// to a dest field if they have the same field name.
// Dest and src must be pointers to structs.
func CopyCommonFields(dest, src interface{}) {
	srcType := reflect.TypeOf(src).Elem()
	destType := reflect.TypeOf(dest).Elem()
	destFieldsMap := map[string]int{}

	for i := 0; i < destType.NumField(); i++ {
		destFieldsMap[destType.Field(i).Name] = i
	}

	for i := 0; i < srcType.NumField(); i++ {
		if j, ok := destFieldsMap[srcType.Field(i).Name]; ok {
			reflect.ValueOf(dest).Elem().Field(j).Set(
				reflect.ValueOf(src).Elem().Field(i),
			)
		}
	}
}

Usage:

func main() {
	type T struct {
		A string
		B int
	}

	type U struct {
		A string
	}

	src := T{
		A: "foo",
		B: 5,
	}

	dest := U{}
	CopyCommonFields(&dest, &src)
	fmt.Printf("%+v\n", dest)
	// output: {A:foo}
}

You can cast structures if they have same field names and types, effectively reassigning field tags:

package main

import "encoding/json"

type DB struct {
	dbNumBits
	Secret bool `json:"secret_key"`
}

type dbNumBits struct {
	NumBits int `json:"bit_size"`
}

type User struct {
	NumBits int `json:"num_bits"`
}

var Record = []byte(`{ "bit_size": 8, "secret_key": false }`)

func main() {
	d := new(DB)
	e := json.Unmarshal(Record, d)
	if e != nil {
		panic(e)
	}
	
	var u User = User(d.dbNumBits)
	println(u.NumBits)
}

https://play.golang.org/p/uX-IIgL-rjc

Here's a solution without reflection, unsafe, or a function per struct. The example is a little convoluted, and maybe you wouldn't need to do it just like this, but the key is using a map[string]interface{} to get away from a struct with field tags. You might be able to use the idea in a similar solution.

package main

import (
    "encoding/json"
    "fmt"
    "log"
)

// example full database record
var dbj = `{ "bit_size": 8, "secret_key": false }`

// User type has only the fields going to the API
type User struct {
    // tag still specifies internal name, not API name
    NumBits int `json:"bit_size"`
}

// mapping from internal field names to API field names.
// (you could have more than one mapping, or even construct this
// at run time)
var ApiField = map[string]string{
    // internal: API
    "bit_size": "num_bits",
    // ...
}

func main() {
    fmt.Println(dbj)
    // select user fields from full db record by unmarshalling
    var u User
    if err := json.Unmarshal([]byte(dbj), &u); err != nil {
        log.Fatal(err)
    }
    // remarshal from User struct back to json
    exportable, err := json.Marshal(u)
    if err != nil {
        log.Fatal(err)
    }
    // unmarshal into a map this time, to shrug field tags.
    type jmap map[string]interface{}
    mInternal := jmap{}
    if err := json.Unmarshal(exportable, &mInternal); err != nil {
        log.Fatal(err)
    }
    // translate field names
    mExportable := jmap{}
    for internalField, v := range mInternal {
        mExportable[ApiField[internalField]] = v
    }
    // marshal final result with API field names
    if exportable, err = json.Marshal(mExportable); err != nil {
        log.Fatal(err)
    }
    fmt.Println(string(exportable))
}

Output:

> { "bit_size": 8, "secret_key": false }
> {"num_bits":8}

Edit: More explanation. As Tom notes in a comment, there's reflection going on behind the code. The goal here is to keep the code simple by using the available capabilities of the library. Package json currently offers two ways to work with data, struct tags and maps of [string]interface{}. The struct tags let you select fields, but force you to statically pick a single json field name. The maps let you pick field names at run time, but not which fields to Marshal. It would be nice if the json package let you do both at once, but it doesn't. The answer here just shows the two techniques and how they can be composed in a solution to the example problem in the OP.

"Is reflect the right tool for this?" A better question might be, "Are struct tags the right tool for this?" and the answer might be no.

package main

import (
    "encoding/json"
    "fmt"
    "log"
)

var dbj = `{ "bit_size": 8, "secret_key": false }`

// translation from internal field name to api field name
type apiTrans struct {
    db, api string
}

var User = []apiTrans{
    {db: "bit_size", api: "num_bits"},
}

func main() {
    fmt.Println(dbj)
    type jmap map[string]interface{}
    // unmarshal full db record
    mdb := jmap{}
    if err := json.Unmarshal([]byte(dbj), &mdb); err != nil {
        log.Fatal(err)
    }
    // build result
    mres := jmap{}
    for _, t := range User {
        if v, ok := mdb[t.db]; ok {
            mres[t.api] = v
        }
    }
    // marshal result
    exportable, err := json.Marshal(mres)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(string(exportable))
}

An efficient way to achieve your goal is to use the gob package.

Here an example with the playground:

package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
)

type DB struct {
	NumBits int
	Secret  bool
}

type User struct {
	NumBits int
}

func main() {
	db := DB{10, true}
	user := User{}

	buf := bytes.Buffer{}
	err := gob.NewEncoder(&buf).Encode(&db)
	if err != nil {
		panic(err)
	}

	err = gob.NewDecoder(&buf).Decode(&user)
	if err != nil {
		panic(err)
	}
	fmt.Println(user)
}

Here the official blog post: https://blog.golang.org/gob