Size of Object in Go

Just thinking of building an LRU based caching mechanism which is memory consumption aware as I can't find a ready one after some searching. The cached item is native Go object which can be basic type, struct, slice, array or any valid combination, but without recursive reference, and we can assign a upper limited of memory usage to the pool and once the total memory consumption reach a threshold, a cleanup based on Latest Recently Used would be triggered.

I understand accurate memory size calculation is not practical but I think a rough estimation may do lot help here. At least it's better then item number counting like what's done in GroupCache ignoring size of the cached object.

So what's a proper way to calculate/estimate the bytes used by given value?

I wrote this function a long time ago, it's recursive and haven't been tested much, but it gives you an idea on how to implement it:

var (
sliceSize  = uint64(reflect.TypeOf(reflect.SliceHeader{}).Size())
stringSize = uint64(reflect.TypeOf(reflect.StringHeader{}).Size())
)
func isNativeType(k reflect.Kind) bool {
switch k {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
return true
}
return false
}
func sizeofInternal(val reflect.Value, fromStruct bool, depth int) (sz uint64) {
if depth++; depth > 1000 {
panic("sizeOf recursed more than 1000 times.")
}
typ := val.Type()
if !fromStruct {
sz = uint64(typ.Size())
}
switch val.Kind() {
case reflect.Ptr:
if val.IsNil() {
break
}
sz += sizeofInternal(val.Elem(), false, depth)
case reflect.Struct:
for i := 0; i < val.NumField(); i++ {
sz += sizeofInternal(val.Field(i), true, depth)
}
case reflect.Array:
if isNativeType(typ.Elem().Kind()) {
break
}
sz = 0
for i := 0; i < val.Len(); i++ {
sz += sizeofInternal(val.Index(i), false, depth)
}
case reflect.Slice:
if !fromStruct {
sz = sliceSize
}
el := typ.Elem()
if isNativeType(el.Kind()) {
sz += uint64(val.Len()) * uint64(el.Size())
break
}
for i := 0; i < val.Len(); i++ {
sz += sizeofInternal(val.Index(i), false, depth)
}
case reflect.Map:
if val.IsNil() {
break
}
kel, vel := typ.Key(), typ.Elem()
if isNativeType(kel.Kind()) && isNativeType(vel.Kind()) {
sz += uint64(kel.Size()+vel.Size()) * uint64(val.Len())
break
}
keys := val.MapKeys()
for i := 0; i < len(keys); i++ {
sz += sizeofInternal(keys[i], false, depth) + sizeofInternal(val.MapIndex(keys[i]), false, depth)
}
case reflect.String:
if !fromStruct {
sz = stringSize
}
sz += uint64(val.Len())
}
return
}
// Sizeof returns the estimated memory usage of object(s) not just the size of the type.
// On 64bit Sizeof("test") == 12 (8 = sizeof(StringHeader) + 4 bytes).
func Sizeof(objs ...interface{}) (sz uint64) {
for i := range objs {
sz += sizeofInternal(reflect.ValueOf(objs[i]), false, 0)
}
return
}

<kbd>playground</kbd>

<strike>The math might be a tiny bit off.</strike>

// edit

Fixed the math and pushed to github for future references.

I'm not exactly sure if there's a difference between the two, but here's
two ways, one using reflection and another using the unsafe package.

package main 
import (
"fmt"
"reflect"
"unsafe"
)
type T struct {
a byte
b int32
c [1234]byte
d float32	
}
func main(){
var a T
s1 := reflect.TypeOf(a)
s2 := unsafe.Sizeof(T{})
fmt.Printf("reflect.Sizeof(T) = %d, unsafe.Sizeof(T{} = %d)", s1.Size(), s2)
}

This may be a bit late but may be of help. The Sizeof() would return the size.

package util
import (
"math/cmplx"
"reflect"
"unsafe"
)
var Size []uint64
//Sizeof Function Will Find Approximate Size of Object in Bytes
func Sizeof(i interface{}) (size uint64) {
size = 0
sizeof(reflect.ValueOf(i), &size)
return
}
//sizeof  private function which used to calculate size of object
func sizeof(val reflect.Value, sizeptr *uint64) {
if val.Kind() >= reflect.Bool && val.Kind() <= reflect.Complex128 {
(*sizeptr) += Size[val.Kind()]
return
}
switch val.Kind() {
case reflect.String:
(*sizeptr) += Size[reflect.String] * uint64(val.Len())
case reflect.Array:
/*
Then iterate through the array and get recursively call the size method.
If all element hold the same value calculate size for one and multiply it with
length of array. Wont wonk correctly if the array holds slices or any dynamic
elements
*/
for i := 0; i < val.Len(); i++ {
sizeof(val.Index(i), (sizeptr))
}
case reflect.Interface:
/*
First we need to get the underlying object of Interface in golang using the Elem()
And then we need to recursively call this function
*/
temp := val.Elem()
sizeof(temp, (sizeptr))
case reflect.Map:
for _, key := range val.MapKeys() {
/*
get the size of key by calling the size function
*/
sizeof(key, (sizeptr))
/*
get the value pointed by the key and then recursively compute its size
*/
mapVal := val.MapIndex(key)
sizeof(mapVal, sizeptr)
}
case reflect.Ptr:
prtVal := val.Elem()
/*
If the pointer is invalid or the pointer is nil then return without updating the size
*/
if !prtVal.IsValid() {
return
}
sizeof(val.Elem(), sizeptr)
case reflect.Slice:
for i := 0; i < val.Len(); i++ {
sizeof(val.Index(i), sizeptr)
}
case reflect.Struct:
/*
Didn't consider the NumMethod here. Don't this that this is necessary or is it??
Need to verify this...
*/
for i := 0; i < val.NumField(); i++ {
sizeof(val.Field(i), sizeptr)
}
case reflect.UnsafePointer:
/*
This allows conversion between elements. Dont think this should this is used in calculating
size
*/
case reflect.Func:
// How to handle function pointers
case reflect.Chan:
// Don't think this case has to be handled as it will be produced and consumed
default:
return
}
return
}
func init() {
Size = make([]uint64, 32)
bool_val := true
Size[reflect.Bool] = uint64(unsafe.Sizeof(bool(bool_val)))
int_val := int(0)
Size[reflect.Int] = uint64(unsafe.Sizeof(int_val))
int8_val := int8(0)
Size[reflect.Int8] = uint64(unsafe.Sizeof(int8_val))
int16_val := int16(0)
Size[reflect.Int16] = uint64(unsafe.Sizeof(int16_val))
int32_val := int32(0)
Size[reflect.Int32] = uint64(unsafe.Sizeof(int32_val))
int64_val := int64(0)
Size[reflect.Int64] = uint64(unsafe.Sizeof(int64_val))
uint_val := uint(0)
Size[reflect.Uint] = uint64(unsafe.Sizeof(uint_val))
uint8_val := uint8(0)
Size[reflect.Uint8] = uint64(unsafe.Sizeof(uint8_val))
uint16_val := uint16(0)
Size[reflect.Uint16] = uint64(unsafe.Sizeof(uint16_val))
uint32_val := uint32(0)
Size[reflect.Uint32] = uint64(unsafe.Sizeof(uint32_val))
uint64_val := uint64(0)
Size[reflect.Uint64] = uint64(unsafe.Sizeof(uint64_val))
uintptr_val := uint64(0)
Size[reflect.Uintptr] = uint64(unsafe.Sizeof(uintptr_val))
float32_val := float32(0.0)
Size[reflect.Float32] = uint64(unsafe.Sizeof(float32_val))
float64_val := float64(0.0)
Size[reflect.Float64] = uint64(unsafe.Sizeof(float64_val))
complex64_val := complex64(cmplx.Sqrt(0 + 0i))
Size[reflect.Complex64] = uint64(unsafe.Sizeof(complex64_val))
complex128_val := complex128(cmplx.Sqrt(0 + 0i))
Size[reflect.Complex128] = uint64(unsafe.Sizeof(complex128_val))
string_val := string("0")
Size[reflect.String] = uint64(unsafe.Sizeof(string_val))
}