英文:
Memory leak in go lang http Handler
问题
我有以下关于HTTP处理程序的代码，当后续请求下载来自Amazon S3的原始图像并将其转换为所需的纵横比后，将其保存回S3。这段代码存在内存泄漏问题，一段时间后会崩溃。我已经从我的一面处理了所有问题，并对代码进行了分析。但是，仍然无法找出问题所在。如果有人能在这里找出问题，我将不胜感激。顺便说一句，我正在使用go version go1.5.3 linux/amd64版本。
分析结果：
总共3302.42kB中的3259.27kB（98.69%）
丢弃了258个节点（累计小于等于16.51kB）
显示了前91个节点中的前30个（累计大于等于27.76kB）
flat flat% sum% cum cum%
1552.59kB 47.01% 47.01% 1552.59kB 47.01% bytes.makeSlice
584kB 17.68% 64.70% 584kB 17.68% imagick._Cfunc_GoBytes
257.38kB 7.79% 72.49% 257.38kB 7.79% encoding/pem.Decode
168.11kB 5.09% 77.58% 168.11kB 5.09% crypto/tls.(*block).reserve
165.09kB 5.00% 82.58% 389.49kB 11.79% crypto/x509.parseCertificate
105.32kB 3.19% 85.77% 105.32kB 3.19% reflect.unsafe_NewArray
83.64kB 2.53% 88.30% 83.64kB 2.53% math/big.nat.make
75.55kB 2.29% 90.59% 75.55kB 2.29% net/http.(*Transport).dialConn
64.02kB 1.94% 92.53% 64.02kB 1.94% regexp.(*bitState).reset
43.77kB 1.33% 93.85% 43.77kB 1.33% crypto/x509.(*CertPool).AddCert
40.44kB 1.22% 95.08% 40.44kB 1.22% crypto/x509/pkix.(*Name).FillFromRDNSequence
40.16kB 1.22% 96.29% 40.16kB 1.22% encoding/asn1.parsePrintableString
24.07kB 0.73% 97.02% 24.07kB 0.73% net/http.newBufioWriterSize
18.98kB 0.57% 97.60% 18.98kB 0.57% net/http.newBufioReader
16.14kB 0.49% 98.09% 64.77kB 1.96% crypto/tls.(*Conn).readHandshake
12.01kB 0.36% 98.45% 237.09kB 7.18% encoding/asn1.parseField
8.01kB 0.24% 98.69% 91.65kB 2.78% crypto/x509.parsePublicKey
0 0% 98.69% 112.33kB 3.40% bufio.(*Reader).Read
0 0% 98.69% 80.32kB 2.43% bufio.(*Reader).fill
0 0% 98.69% 27.76kB 0.84% bufio.(*Writer).ReadFrom
0 0% 98.69% 27.76kB 0.84% bufio.(*Writer).flush
0 0% 98.69% 1648.33kB 49.91% bytes.(*Buffer).ReadFrom
0 0% 98.69% 16.59kB 0.5% bytes.(*Buffer).Write
0 0% 98.69% 16.59kB 0.5% bytes.(*Buffer).grow
0 0% 98.69% 843.06kB 25.53% crypto/tls.(*Conn).Handshake
0 0% 98.69% 112.33kB 3.40% crypto/tls.(*Conn).Read
0 0% 98.69% 27.76kB 0.84% crypto/tls.(*Conn).Write
0 0% 98.69% 843.06kB 25.53% crypto/tls.(*Conn).clientHandshake
0 0% 98.69% 160.96kB 4.87% crypto/tls.(*Conn).readRecord
0 0% 98.69% 27.76kB 0.84% crypto/tls.(*Conn).writeRecord
代码：
func main() {
imagick.Initialize()
defer imagick.Terminate()
myMux := http.NewServeMux()
myMux.HandleFunc("/", serveHTTP)
if err := http.ListenAndServe(":8082", myMux); err != nil {
logFatal("Error when starting or running http server: %v", err)
}
}
func serveHTTP(w http.ResponseWriter, r *http.Request) {
var isMaster bool = true
var desiredAspectRatio float64 = 1.77
if r.RequestURI == "/favicon.ico" {
w.WriteHeader(http.StatusNotFound)
return
}
if len(strings.TrimSpace(r.URL.Query().Get("ar"))) != 0 {
desiredAspectRatio, _ = strconv.ParseFloat(r.URL.Query().Get("ar"), 64)
}
if len(strings.TrimSpace(r.URL.Query().Get("ism"))) != 0 {
isMaster, _ = strconv.ParseBool(r.URL.Query().Get("ism"))
}
imageUrl := strings.ToLower(r.URL.Path[1:])
isProcessed := CreateMaster(imageUrl, desiredAspectRatio, isMaster)
if isProcessed == false {
w.WriteHeader(http.StatusNotFound)
return
}
if !sendResponse(w, r, imageUrl) {
// w.WriteHeader() is skipped intentionally here, since the response may be already partially created.
return
}
logRequestMessage(r, "SUCCESS")
}
func sendResponse(w http.ResponseWriter, r *http.Request, imageUrl string) bool {
w.Header().Set("Content-Type", "text/plain")
if _, err := w.Write([]byte("master created")); err != nil {
logRequestError(r, "Cannot send image from imageUrl=%v to client: %v", imageUrl, err)
return false
}
return true
}
func CreateMaster(keyName string, desiredAspectRatio float64, isMaster bool) bool {
s3Client := s3.New(session.New(), &aws.Config{Region: aws.String(region)})
params := &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(keyName),
}
fmt.Println(" Master creation request for key : " + keyName)
out, err := s3Client.GetObject(params)
if err != nil {
return false
}
defer out.Body.Close()
img, err := ioutil.ReadAll(out.Body)
if err != nil {
return false
}
mw := imagick.NewMagickWand()
defer mw.Destroy()
err = mw.ReadImageBlob(img)
if err != nil {
return false
}
if isMaster == false {
paramsPut := &s3.PutObjectInput{
Bucket: aws.String(masterBucketName),
Key: aws.String(keyName),
Body: bytes.NewReader(mw.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf("Couldn't put the image on s3 : " + keyName + "%s\n", err)
}
return true
}
originalWidth := float64(mw.GetImageWidth())
originalHeight := float64(mw.GetImageHeight())
imageAspectRatio := originalWidth / originalHeight
masterWidth := cwMasterWidth
masterHeight := cwMasterHeight
masterAspectRatio := math.Trunc((cwMasterWidth / cwMasterHeight) * 100)/100
if masterAspectRatio != desiredAspectRatio {
masterAspectRatio = desiredAspectRatio
}
pwm := imagick.NewPixelWand()
defer pwm.Destroy()
tx := imagick.NewMagickWand()
defer tx.Destroy()
if isMaster == true {
var w, h uint = 0, 0
size := fmt.Sprintf("%dx%d^+0+0", w, h)
if imageAspectRatio <= masterAspectRatio {
// trim the height
w = uint(originalWidth)
h = (uint(originalWidth / masterAspectRatio))
size = fmt.Sprintf("%dx%d^+0+0", w, h)
} else {
// trim the width
w = uint(originalHeight * masterAspectRatio)
h = uint(originalHeight)
size = fmt.Sprintf("%dx%d^+0+0", w, h)
}
tx = mw.TransformImage("", size)
tx.SetImageGravity(imagick.GRAVITY_CENTER)
offsetX := -(int(w) - int(tx.GetImageWidth())) / 2
offsetY := -(int(h) - int(tx.GetImageHeight())) / 2
err := tx.ExtentImage(w, h, offsetX, offsetY)
if float64(tx.GetImageWidth()) > masterWidth && float64(tx.GetImageHeight()) > masterHeight {
err = tx.ResizeImage(uint(masterWidth), uint(masterHeight), imagick.FILTER_BOX, 1)
if err != nil {
log.Printf("Inside CreateMaster function Couldn't resize the image : " + keyName + "%s\n", err)
return false
}
}
}
paramsPut := &s3.PutObjectInput{
Bucket: aws.String(masterBucketName),
Key: aws.String(keyName),
Body: bytes.NewReader(tx.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf("Inside CreateMaster function Couldn't put the image on s3 : " + keyName + "%s\n", err)
return false
}
return true
}
希望对你有帮助！
英文:
I have the following code of http handler which on subsequent request download the original image from Amazon S3 and convert it in the desired aspect ratio and save it back on s3. This code leaks memory and after sometime it crashes.I have handled everything from my side and also did the profiling on the code. But, still couldn't figure out the issue. Would appreciate if anyone can figure out here. FYI, I am using the go version go1.5.3 linux/amd64 version .
Output from profiling :
3259.27kB of 3302.42kB total (98.69%)
Dropped 258 nodes (cum <= 16.51kB)
Showing top 30 nodes out of 91 (cum >= 27.76kB)
flat flat% sum% cum cum%
1552.59kB 47.01% 47.01% 1552.59kB 47.01% bytes.makeSlice
584kB 17.68% 64.70% 584kB 17.68% imagick._Cfunc_GoBytes
257.38kB 7.79% 72.49% 257.38kB 7.79% encoding/pem.Decode
168.11kB 5.09% 77.58% 168.11kB 5.09% crypto/tls.(*block).reserve
165.09kB 5.00% 82.58% 389.49kB 11.79% crypto/x509.parseCertificate
105.32kB 3.19% 85.77% 105.32kB 3.19% reflect.unsafe_NewArray
83.64kB 2.53% 88.30% 83.64kB 2.53% math/big.nat.make
75.55kB 2.29% 90.59% 75.55kB 2.29% net/http.(*Transport).dialConn
64.02kB 1.94% 92.53% 64.02kB 1.94% regexp.(*bitState).reset
43.77kB 1.33% 93.85% 43.77kB 1.33% crypto/x509.(*CertPool).AddCert
40.44kB 1.22% 95.08% 40.44kB 1.22% crypto/x509/pkix.(*Name).FillFromRDNSequence
40.16kB 1.22% 96.29% 40.16kB 1.22% encoding/asn1.parsePrintableString
24.07kB 0.73% 97.02% 24.07kB 0.73% net/http.newBufioWriterSize
18.98kB 0.57% 97.60% 18.98kB 0.57% net/http.newBufioReader
16.14kB 0.49% 98.09% 64.77kB 1.96% crypto/tls.(*Conn).readHandshake
12.01kB 0.36% 98.45% 237.09kB 7.18% encoding/asn1.parseField
8.01kB 0.24% 98.69% 91.65kB 2.78% crypto/x509.parsePublicKey
0 0% 98.69% 112.33kB 3.40% bufio.(*Reader).Read
0 0% 98.69% 80.32kB 2.43% bufio.(*Reader).fill
0 0% 98.69% 27.76kB 0.84% bufio.(*Writer).ReadFrom
0 0% 98.69% 27.76kB 0.84% bufio.(*Writer).flush
0 0% 98.69% 1648.33kB 49.91% bytes.(*Buffer).ReadFrom
0 0% 98.69% 16.59kB 0.5% bytes.(*Buffer).Write
0 0% 98.69% 16.59kB 0.5% bytes.(*Buffer).grow
0 0% 98.69% 843.06kB 25.53% crypto/tls.(*Conn).Handshake
0 0% 98.69% 112.33kB 3.40% crypto/tls.(*Conn).Read
0 0% 98.69% 27.76kB 0.84% crypto/tls.(*Conn).Write
0 0% 98.69% 843.06kB 25.53% crypto/tls.(*Conn).clientHandshake
0 0% 98.69% 160.96kB 4.87% crypto/tls.(*Conn).readRecord
0 0% 98.69% 27.76kB 0.84% crypto/tls.(*Conn).writeRecord
Code :
func main() {
imagick.Initialize()
defer imagick.Terminate()
myMux := http.NewServeMux()
myMux.HandleFunc("/", serveHTTP)
if err := http.ListenAndServe(":8082", myMux); err != nil {
logFatal("Error when starting or running http server: %v", err)
}
}
func serveHTTP(w http.ResponseWriter, r *http.Request) {
var isMaster bool = true
var desiredAspectRatio float64 = 1.77
if r.RequestURI == "/favicon.ico" {
w.WriteHeader(http.StatusNotFound)
return
}
if len(strings.TrimSpace(r.URL.Query().Get("ar"))) != 0 {
desiredAspectRatio, _ = strconv.ParseFloat(r.URL.Query().Get("ar"), 64)
}
if len(strings.TrimSpace(r.URL.Query().Get("ism"))) != 0 {
isMaster, _ = strconv.ParseBool(r.URL.Query().Get("ism"))
}
imageUrl := strings.ToLower(r.URL.Path[1:])
isProcessed := CreateMaster(imageUrl, desiredAspectRatio, isMaster)
if isProcessed == false {
w.WriteHeader(http.StatusNotFound)
return
}
if !sendResponse(w, r, imageUrl) {
// w.WriteHeader() is skipped intentionally here, since the response may be already partially created.
return
}
logRequestMessage(r, "SUCCESS")
}
func sendResponse(w http.ResponseWriter, r *http.Request, imageUrl string) bool {
w.Header().Set("Content-Type", "text/plain")
if _, err := w.Write([]byte("master created")); err != nil {
logRequestError(r, "Cannot send image from imageUrl=%v to client: %v", imageUrl, err)
return false
}
return true
}
func CreateMaster(keyName string, desiredAspectRatio float64, isMaster bool) bool {
s3Client := s3.New(session.New(), &aws.Config{Region: aws.String(region)})
params := &s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(keyName),
}
fmt.Println(" Master creation request for key : " + keyName)
out, err := s3Client.GetObject(params)
if err != nil {
return false
}
defer out.Body.Close()
img, err := ioutil.ReadAll(out.Body)
if err != nil {
return false
}
mw := imagick.NewMagickWand()
defer mw.Destroy()
err = mw.ReadImageBlob(img)
if err != nil {
return false
}
if isMaster == false {
paramsPut := &s3.PutObjectInput{
Bucket: aws.String(masterBucketName),
Key: aws.String(keyName),
Body: bytes.NewReader(mw.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf("Couldn't put the image on s3 : " + keyName + "%s\n", err)
}
return true
}
originalWidth := float64(mw.GetImageWidth())
originalHeight := float64(mw.GetImageHeight())
imageAspectRatio := originalWidth / originalHeight
masterWidth := cwMasterWidth
masterHeight := cwMasterHeight
masterAspectRatio := math.Trunc((cwMasterWidth / cwMasterHeight) * 100)/100
if masterAspectRatio != desiredAspectRatio {
masterAspectRatio = desiredAspectRatio
}
pwm := imagick.NewPixelWand()
defer pwm.Destroy()
tx := imagick.NewMagickWand()
defer tx.Destroy()
if isMaster == true {
var w, h uint = 0, 0
size := fmt.Sprintf("%dx%d^+0+0", w, h)
if imageAspectRatio <= masterAspectRatio {
// trim the height
w = uint(originalWidth)
h = (uint(originalWidth / masterAspectRatio))
size = fmt.Sprintf("%dx%d^+0+0", w, h)
} else {
// trim the width
w = uint(originalHeight * masterAspectRatio)
h = uint(originalHeight)
size = fmt.Sprintf("%dx%d^+0+0", w, h)
}
tx = mw.TransformImage("", size)
tx.SetImageGravity(imagick.GRAVITY_CENTER)
offsetX := -(int(w) - int(tx.GetImageWidth())) / 2
offsetY := -(int(h) - int(tx.GetImageHeight())) / 2
err := tx.ExtentImage(w, h, offsetX, offsetY)
if float64(tx.GetImageWidth()) > masterWidth && float64(tx.GetImageHeight()) > masterHeight {
err = tx.ResizeImage(uint(masterWidth), uint(masterHeight), imagick.FILTER_BOX, 1)
if err != nil {
log.Printf("Inside CreateMaster function Couldn't resize the image : " + keyName + "%s\n", err)
return false
}
}
}
paramsPut := &s3.PutObjectInput{
Bucket: aws.String(masterBucketName),
Key: aws.String(keyName),
Body: bytes.NewReader(tx.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf("Inside CreateMaster function Couldn't put the image on s3 : " + keyName + "%s\n", err)
return false
}
return true
}
答案1
得分: 4
你正在泄漏一个魔杖。
在这里，你分配了一个新的魔杖，并将其延迟到销毁：
tx := imagick.NewMagickWand()
defer tx.Destroy()
但是在稍后的一个“if”块中，你用从调用TransformImage()返回的魔杖替换了它：
tx = mw.TransformImage("", size)
tx.SetImageGravity(imagick.GRAVITY_CENTER)
如果你完全摆脱了对新的魔杖的第一次分配，并确保销毁从TransformImage()返回的新魔杖，泄漏问题就会消失。
有关详细信息，请参阅问题跟踪器，＃72。
英文:
You are leaking a wand.
Here, you allocate a new wand and defer it to destroy:
tx := imagick.NewMagickWand()
defer tx.Destroy()
But then further down, in an "if" block, you replace it with the wand that is returned from the call to TransformImage():
tx = mw.TransformImage("", size)
tx.SetImageGravity(imagick.GRAVITY_CENTER)
If you completely get rid of the first allocation of that new magick wand, and simply make sure to Destroy() that new wand returned from TransformImage(), the leak goes away.
Ref the issue tracker, #72, for details
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库，让每个人都能够通过互相帮助和分享经验来进步。
评论