Unmarshaling a json http response

I have very recently started playing with GO and I am trying to unmarshal a JSON response from http://www.oref.org.il/WarningMessages/alerts.json.
For some reason that I can't understand the unmarshaling failed, the unmarshalled struct is empty (my guess is that it is somehow related to encoding).

Below is the code, any help is appreciated.

Thanks,
Itay

package main

import (
  "fmt"
  "io/ioutil"
  "net/http"
  "encoding/json"
)

const alertsUrl = "http://www.oref.org.il/WarningMessages/alerts.json"

type Record struct {
  Id string `json:id`
  Title string `json:title`
  Data []string `json:data`
}

func main() {
  client := &http.Client{}
  req, err := http.NewRequest("GET", alertsUrl, nil)
  perror(err)

  req.Header.Add("Content-Type", "application/json; charset=utf-8")
  res, err := client.Do(req)
  perror(err)

  defer res.Body.Close()
  body, err := ioutil.ReadAll(res.Body)
  perror(err)

  var record Record
  json.Unmarshal(body, &record)
  fmt.Println(record)
}

func perror(err error) {
  if err != nil {
    panic(err)
  }
}

You're ignoring the error on JSON Unmarshal:

func Unmarshal(data []byte, v interface{}) error

See that it returns an error. Adding that in,

err = json.Unmarshal(body, &record)
perror(err)

It looks like this is a Unicode error — you need to decode the UTF-16 data.

How should you do this? Take a look at this answer. Basically once you read the body like body, err := ioutil.ReadAll(res.Body), you want to decode the UTF-16 bytes to a string. There is a lot going on there, but we can take some liberties: for instance, pulling up the URL in Chrome, the browser tells us that it is UTF-16 LE. So we can skip the ByteOrder detection. So the key here is this function:

func UTF16BytesToString(b []byte, o binary.ByteOrder) string {
utf := make([]uint16, (len(b)+(2-1))/2)
for i := 0; i+(2-1) < len(b); i += 2 {
utf[i/2] = o.Uint16(b[i:])
}
if len(b)/2 < len(utf) {
utf[len(utf)-1] = utf8.RuneError
}
return string(utf16.Decode(utf))
}

Knowing our byte order and passing it in, this will convert the naïve byte array to a string of UTF-16 characters. Thanks to user OneOfOne's comment, we can also detect the BOM easily.

The result:

package main
import (
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"unicode/utf16"
"unicode/utf8"
)
const alertsUrl = "http://www.oref.org.il/WarningMessages/alerts.json"
type Record struct {
Id    string   `json:id`
Title string   `json:title`
Data  []string `json:data`
}
// LazyUTF16BytesToString converts UTF-16 encoded bytes, in big or little endian byte order,
// to a UTF-8 encoded string.
func LazyUTF16BytesToString(b []byte) string {
if len(b)%2 != 0 {
panic("len(b) % 2 != 0")
}
var codec binary.ByteOrder = binary.LittleEndian
if b[0] == 0xFE && b[1] == 0xFF { //check and strip the BOM
b = b[2:]
codec = binary.BigEndian
} else if b[0] == 0xFF && b[1] == 0xFE {
b = b[2:]
}
utf := make([]uint16, (len(b)+(2-1))/2)
for i := 0; i+(2-1) < len(b); i += 2 {
utf[i/2] = codec.Uint16(b[i:])
}
if len(b)/2 < len(utf) {
utf[len(utf)-1] = utf8.RuneError
}
return string(utf16.Decode(utf))
}
func main() {
client := &http.Client{}
req, err := http.NewRequest("GET", alertsUrl, nil)
perror(err)
req.Header.Add("Content-Type", "application/json; charset=utf-8")
res, err := client.Do(req)
perror(err)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
perror(err)
bodyString := LazyUTF16BytesToString(body)
var record Record
err = json.Unmarshal([]byte(bodyString), &record)
perror(err)
fmt.Println(record)
}
func perror(err error) {
if err != nil {
panic(err)
}
}