Golang program memory leak?

My golang program (url monitor) has a memory leak, it finially killed by kernel (oom).
the env:

$ go version
go version go1.0.3
$ go env
GOARCH="amd64"
GOBIN=""
GOCHAR="6"
GOEXE=""
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread"
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/data/apps/go"
GOROOT="/usr/local/go"
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
CGO_ENABLED="1"

code:

package main
import (
"bytes"
"database/sql"
"flag"
"fmt"
_ "github.com/Go-SQL-Driver/MySQL"
"ijinshan.com/cfg"
"log"
"net"
"net/http"
"net/smtp"
"os"
"strconv"
"strings"
"sync"
"time"
)
var (
Log           *log.Logger
Conf          cfg.KVConfig
Debug         bool
CpuCore       int
HttpTransport = &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
deadline := time.Now().Add(30 * time.Second)
c, err := net.DialTimeout(netw, addr, 20*time.Second)
if err != nil {
return nil, err
}
c.SetDeadline(deadline)
return c, nil
},
DisableKeepAlives: true,
}
HttpClient = &http.Client{
Transport: HttpTransport,
}
WG            sync.WaitGroup
)
const (
LogFileFlag   = os.O_WRONLY | os.O_CREATE | os.O_APPEND
LogFileMode   = 0644
LogFlag       = log.LstdFlags | log.Lshortfile
GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum, 
IFNULL(A.Website, ''), IFNULL(A.Descr, ''), 
IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''), 
IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '')
FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200`
HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名
</td><td>tsid</td><td>章节</td><td>章节号</td><td>描述
</td><td>videoid</td><td>网站</td><td>地址</td></tr>`
HtmlTail = "</table>"
)
type videoInfo struct {
name          string
tsid          uint
chapter       string
chapterNum    uint
descr         string
videoId       string
website       string
androidWebUrl string
iosWebUrl     string
androidUrl    string
androidUrl2   string
iosUrl        string
}
func init() {
var (
confFile string
err      error
)
// parse command argument:w
flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path")
flag.Parse()
// read config
if Conf, err = cfg.Read(confFile); err != nil {
panic(fmt.Sprintf("Read config file \"%s\" failed (%s)",
confFile, err.Error()))
}
// open log file
file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode)
if err != nil {
panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"],
err.Error()))
}
// init LOG
Log = log.New(file, "", LogFlag)
Debug = false
i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32)
if err != nil {
panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"],
err.Error()))
}
CpuCore = int(i)
}
func getHttpStatusCode(url string) int {
if url == "" {
return 200
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return 0
}
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
req.Header.Add("Connection", "close")
resp, err := HttpClient.Do(req)
if err != nil {
return 0
}
defer resp.Body.Close()
return resp.StatusCode
}
func sendMail(host, user, pwd, from, to, subject, body, mailType string) error {
auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0])
cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType)
msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s",
to, from, user, subject, cntType, body)
return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg))
}
func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error {
defer HttpTransport.CloseIdleConnections()
db, err := sql.Open("mysql", Conf["weikan.mysql"])
if err != nil {
return err
}
rows, err := db.Query(GET_VIDEO_SQL)
if err != nil {
db.Close()
return err
}
for rows.Next() {
video := &videoInfo{}
err = rows.Scan(&video.name, &video.tsid, &video.chapter,
&video.chapterNum,
&video.website, &video.descr, &video.videoId, &video.androidWebUrl,
&video.iosWebUrl, &video.androidUrl, &video.androidUrl2,
&video.iosUrl)
if err != nil {
db.Close()
return err
}
videoChan <- video
WG.Add(1)
}
db.Close()
// wait check url finish
WG.Wait()
// send mail
for {
if htmlBuf.Len() == 0 {
Log.Print("no error found!!!!!!!!")
break
}
Log.Print("found error !!!!!!!!")
/*
err := sendMail("smtp.gmail.com:587", "xxxx",
"xxx", "xxx <xxx>",
Conf["mail.to"], "xxxxx",
HtmlHead+htmlBuf.String()+HtmlTail, "html")
if err != nil {
Log.Printf("sendMail failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
*/
Log.Print("send mail")
break
}
Log.Print("reset buf")
htmlBuf.Reset()
return nil
}
func checkUrl(videoChan chan *videoInfo, errChan chan string) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
video := <-videoChan
ok := true
errUrl := ""
if code := getHttpStatusCode(video.androidWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.iosWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl2); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl2, code)
ok = false
}
if code := getHttpStatusCode(video.iosUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosUrl, code)
ok = false
}
if !ok {
errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td>
<td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
video.name, video.tsid, video.chapter, video.chapterNum,
video.descr, video.videoId,
video.website, errUrl)
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name,
video.chapter, video.descr)
} else {
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name,
video.chapter, video.descr)
WG.Done()
}
}
}
func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
html := <-errChan
_, err := htmlBuf.WriteString(html)
if err != nil {
Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html,
err.Error())
panic(err)
}
WG.Done()
}
}
func main() {
videoChan := make(chan *videoInfo, 100000)
errChan := make(chan string, 100000)
htmlBuf := &bytes.Buffer{}
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
// check url
for i := 0; i < CpuCore; i++ {
go checkUrl(videoChan, errChan)
}
// merge error string then send mail
go mergeErr(errChan, htmlBuf)
for {
// get Video and LiveSrc video source
if err := getVideos(videoChan, htmlBuf); err != nil {
Log.Printf("getVideos failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
// time.Sleep(1 * time.Hour)
}
Log.Print("exit...")
}

the code has four funcs:

> getHttpStatusCode

free resource use resp.Body.Close()
> sendMail

I don't need to free the resource manually

> mergeErr

concat the err string by using a htmlBuf(*bytes.Buffer)
> getVideos

First it gets the Video urls and then sends them to videoChan then it waits all the routines finish their check jobs.
then sendmail and reset htmlBuf.

I don't find any resource that needs free, but.

> $ top

shows:

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                                                                      
6451 root      20   0 3946m 115m 2808 S  0.7  0.2   6:11.20 vsmonitor

the VIRT and RES will grow ...

memory profiling:

(pprof) top
Total: 10.8 MB
2.3  21.2%  21.2%      2.3  21.2% main.main
2.0  18.5%  39.8%      2.0  18.5% bufio.NewWriterSize
1.5  13.9%  53.7%      1.5  13.9% bufio.NewReaderSize
1.5  13.9%  67.6%      1.5  13.9% compress/flate.NewReader
0.5   4.6%  72.2%      0.5   4.6% net.newFD
0.5   4.6%  76.8%      0.5   4.6% net.sockaddrToTCP
0.5   4.6%  81.5%      4.5  41.7% net/http.(*Transport).getConn
0.5   4.6%  86.1%      2.5  23.2% net/http.(*persistConn).readLoop
0.5   4.6%  90.7%      0.5   4.6% net/textproto.(*Reader).ReadMIMEHeader
0.5   4.6%  95.4%      0.5   4.6% net/url.(*URL).ResolveReference

It's pretty easy to add an option to your program so it'll record where the memory is being used. Nothing stood out to me in your program as terribly wrong. Are the files you download very large? Could you do a HEAD request instead? I've no idea if that'd help; if you have a high volume of requests maybe it would.

There is an (old-ish) article on the Go blog about the memory profiling at http://blog.golang.org/2011/06/profiling-go-programs.html and documentation at http://golang.org/pkg/runtime/pprof/ and http://golang.org/pkg/net/http/pprof/