2016年3月3日 17:36:21go评论92阅读模式

英文:

Memory leak in go lang http Handler

问题

我有以下关于HTTP处理程序的代码，当后续请求下载来自Amazon S3的原始图像并将其转换为所需的纵横比后，将其保存回S3。这段代码存在内存泄漏问题，一段时间后会崩溃。我已经从我的一面处理了所有问题，并对代码进行了分析。但是，仍然无法找出问题所在。如果有人能在这里找出问题，我将不胜感激。顺便说一句，我正在使用go version go1.5.3 linux/amd64版本。

分析结果：

总共3302.42kB中的3259.27kB（98.69%）
丢弃了258个节点（累计小于等于16.51kB）
显示了前91个节点中的前30个（累计大于等于27.76kB）
      flat  flat%   sum%        cum   cum%
 1552.59kB 47.01% 47.01%  1552.59kB 47.01%  bytes.makeSlice
     584kB 17.68% 64.70%      584kB 17.68%  imagick._Cfunc_GoBytes
  257.38kB  7.79% 72.49%   257.38kB  7.79%  encoding/pem.Decode
  168.11kB  5.09% 77.58%   168.11kB  5.09%  crypto/tls.(*block).reserve
  165.09kB  5.00% 82.58%   389.49kB 11.79%  crypto/x509.parseCertificate
  105.32kB  3.19% 85.77%   105.32kB  3.19%  reflect.unsafe_NewArray
   83.64kB  2.53% 88.30%    83.64kB  2.53%  math/big.nat.make
   75.55kB  2.29% 90.59%    75.55kB  2.29%  net/http.(*Transport).dialConn
   64.02kB  1.94% 92.53%    64.02kB  1.94%  regexp.(*bitState).reset
   43.77kB  1.33% 93.85%    43.77kB  1.33%  crypto/x509.(*CertPool).AddCert
   40.44kB  1.22% 95.08%    40.44kB  1.22%  crypto/x509/pkix.(*Name).FillFromRDNSequence
   40.16kB  1.22% 96.29%    40.16kB  1.22%  encoding/asn1.parsePrintableString
   24.07kB  0.73% 97.02%    24.07kB  0.73%  net/http.newBufioWriterSize
   18.98kB  0.57% 97.60%    18.98kB  0.57%  net/http.newBufioReader
   16.14kB  0.49% 98.09%    64.77kB  1.96%  crypto/tls.(*Conn).readHandshake
   12.01kB  0.36% 98.45%   237.09kB  7.18%  encoding/asn1.parseField
    8.01kB  0.24% 98.69%    91.65kB  2.78%  crypto/x509.parsePublicKey
         0     0% 98.69%   112.33kB  3.40%  bufio.(*Reader).Read
         0     0% 98.69%    80.32kB  2.43%  bufio.(*Reader).fill
         0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).ReadFrom
         0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).flush
         0     0% 98.69%  1648.33kB 49.91%  bytes.(*Buffer).ReadFrom
         0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).Write
         0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).grow
         0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).Handshake
         0     0% 98.69%   112.33kB  3.40%  crypto/tls.(*Conn).Read
         0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).Write
         0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).clientHandshake
         0     0% 98.69%   160.96kB  4.87%  crypto/tls.(*Conn).readRecord
         0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).writeRecord

代码：

func main() {       
    imagick.Initialize() 
    defer imagick.Terminate()	
          
    myMux := http.NewServeMux()
    myMux.HandleFunc("/", serveHTTP)       

    if err := http.ListenAndServe(":8082", myMux); err != nil {
        logFatal("Error when starting or running http server: %v", err)
    }      
}

func serveHTTP(w http.ResponseWriter, r *http.Request) {
    var isMaster bool = true		
    var desiredAspectRatio float64 = 1.77	

    if r.RequestURI == "/favicon.ico" {
        w.WriteHeader(http.StatusNotFound)
        return
    }
    		
    if len(strings.TrimSpace(r.URL.Query().Get("ar"))) != 0 {
        desiredAspectRatio, _ = strconv.ParseFloat(r.URL.Query().Get("ar"), 64)
    }
    			
    if len(strings.TrimSpace(r.URL.Query().Get("ism"))) != 0 {		
        isMaster, _ = strconv.ParseBool(r.URL.Query().Get("ism"))
    }	
    	
    imageUrl := strings.ToLower(r.URL.Path[1:])		
    	
    isProcessed := CreateMaster(imageUrl, desiredAspectRatio, isMaster)		

    if isProcessed == false {
        w.WriteHeader(http.StatusNotFound) 
        return
    }

    if !sendResponse(w, r, imageUrl) {
        // w.WriteHeader() is skipped intentionally here, since the response may be already partially created.
        return
    }

    logRequestMessage(r, "SUCCESS")  	
}

func sendResponse(w http.ResponseWriter, r *http.Request, imageUrl string) bool {
    w.Header().Set("Content-Type", "text/plain")		

    if _, err := w.Write([]byte("master created")); err != nil {
        logRequestError(r, "Cannot send image from imageUrl=%v to client: %v", imageUrl, err)
        return false
    }
    return true
}

func CreateMaster(keyName string, desiredAspectRatio float64, isMaster bool) bool {       		
    s3Client := s3.New(session.New(), &aws.Config{Region: aws.String(region)})
    params := &s3.GetObjectInput{
        Bucket: aws.String(bucketName),
        Key: aws.String(keyName),
    }
    	
    fmt.Println(" Master creation request for key : " + keyName)
    out, err := s3Client.GetObject(params)
    	
    if err != nil { 
        return false 		      	    	    		   
    }
    	
    defer out.Body.Close()	
    img, err := ioutil.ReadAll(out.Body)
    	
    if err != nil {	
        return false      
    }      				

    mw := imagick.NewMagickWand()
    defer mw.Destroy()
    	
    err = mw.ReadImageBlob(img)
    if err != nil {	 
        return false 	               
    }

    if isMaster == false {
        paramsPut := &s3.PutObjectInput{
            Bucket:         aws.String(masterBucketName),
            Key:            aws.String(keyName),
            Body:         bytes.NewReader(mw.GetImageBlob()),
        }
        	
        _, err = s3Client.PutObject(paramsPut)
        if err != nil {
            log.Printf("Couldn't put the image on s3 : " + keyName + "%s\n", err) 	    
        }
    	
        return true
    }

    originalWidth := float64(mw.GetImageWidth())
    originalHeight := float64(mw.GetImageHeight())

    imageAspectRatio  := originalWidth / originalHeight
    masterWidth := cwMasterWidth
    masterHeight := cwMasterHeight
    masterAspectRatio := math.Trunc((cwMasterWidth / cwMasterHeight) * 100)/100
    			
    if masterAspectRatio != desiredAspectRatio {  		   
        masterAspectRatio = desiredAspectRatio
    }

    pwm := imagick.NewPixelWand()
    defer pwm.Destroy()

    tx := imagick.NewMagickWand()
    defer tx.Destroy()	
    	
    if isMaster == true {               
        var w, h uint = 0, 0
        size := fmt.Sprintf("%dx%d^+0+0", w, h)	
        if imageAspectRatio <= masterAspectRatio {                
            // trim the height
            w = uint(originalWidth)
            h = (uint(originalWidth / masterAspectRatio))
            size = fmt.Sprintf("%dx%d^+0+0", w, h)
        } else { 
            // trim the width
            w = uint(originalHeight * masterAspectRatio)
            h = uint(originalHeight)
            size = fmt.Sprintf("%dx%d^+0+0", w, h)
        }

        tx = mw.TransformImage("", size)		
        tx.SetImageGravity(imagick.GRAVITY_CENTER)
        offsetX := -(int(w) - int(tx.GetImageWidth())) / 2
        offsetY := -(int(h) - int(tx.GetImageHeight())) / 2
        err := tx.ExtentImage(w, h, offsetX, offsetY)

        if float64(tx.GetImageWidth()) > masterWidth && float64(tx.GetImageHeight()) > masterHeight  {                                        
            err = tx.ResizeImage(uint(masterWidth), uint(masterHeight), imagick.FILTER_BOX, 1)
            if err != nil {
                log.Printf("Inside CreateMaster function Couldn't resize the image : " + keyName + "%s\n", err) 
                return false				 
            }                                                           
        }                                       
    } 		

    paramsPut := &s3.PutObjectInput{
        Bucket:         aws.String(masterBucketName),
        Key:            aws.String(keyName),
        Body:         bytes.NewReader(tx.GetImageBlob()),
    }
        	
    _, err = s3Client.PutObject(paramsPut)
    if err != nil {
        log.Printf("Inside CreateMaster function Couldn't put the image on s3 : " + keyName + "%s\n", err) 	
        return false	    
    }

    return true
}

希望对你有帮助！

英文:

I have the following code of http handler which on subsequent request download the original image from Amazon S3 and convert it in the desired aspect ratio and save it back on s3. This code leaks memory and after sometime it crashes.I have handled everything from my side and also did the profiling on the code. But, still couldn't figure out the issue. Would appreciate if anyone can figure out here. FYI, I am using the go version go1.5.3 linux/amd64 version .

Output from profiling :

3259.27kB of 3302.42kB total (98.69%)
Dropped 258 nodes (cum &lt;= 16.51kB)
Showing top 30 nodes out of 91 (cum &gt;= 27.76kB)
flat  flat%   sum%        cum   cum%
1552.59kB 47.01% 47.01%  1552.59kB 47.01%  bytes.makeSlice
584kB 17.68% 64.70%      584kB 17.68%  imagick._Cfunc_GoBytes
257.38kB  7.79% 72.49%   257.38kB  7.79%  encoding/pem.Decode
168.11kB  5.09% 77.58%   168.11kB  5.09%  crypto/tls.(*block).reserve
165.09kB  5.00% 82.58%   389.49kB 11.79%  crypto/x509.parseCertificate
105.32kB  3.19% 85.77%   105.32kB  3.19%  reflect.unsafe_NewArray
83.64kB  2.53% 88.30%    83.64kB  2.53%  math/big.nat.make
75.55kB  2.29% 90.59%    75.55kB  2.29%  net/http.(*Transport).dialConn
64.02kB  1.94% 92.53%    64.02kB  1.94%  regexp.(*bitState).reset
43.77kB  1.33% 93.85%    43.77kB  1.33%  crypto/x509.(*CertPool).AddCert
40.44kB  1.22% 95.08%    40.44kB  1.22%  crypto/x509/pkix.(*Name).FillFromRDNSequence
40.16kB  1.22% 96.29%    40.16kB  1.22%  encoding/asn1.parsePrintableString
24.07kB  0.73% 97.02%    24.07kB  0.73%  net/http.newBufioWriterSize
18.98kB  0.57% 97.60%    18.98kB  0.57%  net/http.newBufioReader
16.14kB  0.49% 98.09%    64.77kB  1.96%  crypto/tls.(*Conn).readHandshake
12.01kB  0.36% 98.45%   237.09kB  7.18%  encoding/asn1.parseField
8.01kB  0.24% 98.69%    91.65kB  2.78%  crypto/x509.parsePublicKey
0     0% 98.69%   112.33kB  3.40%  bufio.(*Reader).Read
0     0% 98.69%    80.32kB  2.43%  bufio.(*Reader).fill
0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).ReadFrom
0     0% 98.69%    27.76kB  0.84%  bufio.(*Writer).flush
0     0% 98.69%  1648.33kB 49.91%  bytes.(*Buffer).ReadFrom
0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).Write
0     0% 98.69%    16.59kB   0.5%  bytes.(*Buffer).grow
0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).Handshake
0     0% 98.69%   112.33kB  3.40%  crypto/tls.(*Conn).Read
0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).Write
0     0% 98.69%   843.06kB 25.53%  crypto/tls.(*Conn).clientHandshake
0     0% 98.69%   160.96kB  4.87%  crypto/tls.(*Conn).readRecord
0     0% 98.69%    27.76kB  0.84%  crypto/tls.(*Conn).writeRecord

Code :

func main() {       
imagick.Initialize() 
defer imagick.Terminate()	
myMux := http.NewServeMux()
myMux.HandleFunc(&quot;/&quot;, serveHTTP)       
if err := http.ListenAndServe(&quot;:8082&quot;, myMux); err != nil {
logFatal(&quot;Error when starting or running http server: %v&quot;, err)
}      
}
func serveHTTP(w http.ResponseWriter, r *http.Request) {
var isMaster bool = true		
var desiredAspectRatio float64 = 1.77	
if r.RequestURI == &quot;/favicon.ico&quot; {
w.WriteHeader(http.StatusNotFound)
return
}
if len(strings.TrimSpace(r.URL.Query().Get(&quot;ar&quot;))) != 0 {
desiredAspectRatio, _ = strconv.ParseFloat(r.URL.Query().Get(&quot;ar&quot;), 64)
}
if len(strings.TrimSpace(r.URL.Query().Get(&quot;ism&quot;))) != 0 {		
isMaster, _ = strconv.ParseBool(r.URL.Query().Get(&quot;ism&quot;))
}	
imageUrl := strings.ToLower(r.URL.Path[1:])		
isProcessed := CreateMaster(imageUrl, desiredAspectRatio, isMaster)		
if isProcessed == false {
w.WriteHeader(http.StatusNotFound) 
return
}
if !sendResponse(w, r, imageUrl) {
// w.WriteHeader() is skipped intentionally here, since the response may be already partially created.
return
}
logRequestMessage(r, &quot;SUCCESS&quot;)  	
}
func sendResponse(w http.ResponseWriter, r *http.Request, imageUrl string) bool {
w.Header().Set(&quot;Content-Type&quot;, &quot;text/plain&quot;)		
if _, err := w.Write([]byte(&quot;master created&quot;)); err != nil {
logRequestError(r, &quot;Cannot send image from imageUrl=%v to client: %v&quot;, imageUrl, err)
return false
}
return true
}
func CreateMaster(keyName string, desiredAspectRatio float64, isMaster bool) bool {       		
s3Client := s3.New(session.New(), &amp;aws.Config{Region: aws.String(region)})
params := &amp;s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(keyName),
}
fmt.Println(&quot; Master creation request for key : &quot; + keyName)
out, err := s3Client.GetObject(params)
if err != nil { 
return false 		      	    	    		   
}
defer out.Body.Close()	
img, err := ioutil.ReadAll(out.Body)
if err != nil {	
return false      
}      				
mw := imagick.NewMagickWand()
defer mw.Destroy()
err = mw.ReadImageBlob(img)
if err != nil {	 
return false 	               
}
if isMaster == false {
paramsPut := &amp;s3.PutObjectInput{
Bucket:         aws.String(masterBucketName),
Key:            aws.String(keyName),
Body:         bytes.NewReader(mw.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf(&quot;Couldn&#39;t put the image on s3 : &quot; + keyName + &quot;%s\n&quot;, err) 	    
}
return true
}
originalWidth := float64(mw.GetImageWidth())
originalHeight := float64(mw.GetImageHeight())
imageAspectRatio  := originalWidth / originalHeight
masterWidth := cwMasterWidth
masterHeight := cwMasterHeight
masterAspectRatio := math.Trunc((cwMasterWidth / cwMasterHeight) * 100)/100
if masterAspectRatio != desiredAspectRatio {  		   
masterAspectRatio = desiredAspectRatio
}
pwm := imagick.NewPixelWand()
defer pwm.Destroy()
tx := imagick.NewMagickWand()
defer tx.Destroy()	
if isMaster == true {               
var w, h uint = 0, 0
size := fmt.Sprintf(&quot;%dx%d^+0+0&quot;, w, h)	
if imageAspectRatio &lt;= masterAspectRatio {                
// trim the height
w = uint(originalWidth)
h = (uint(originalWidth / masterAspectRatio))
size = fmt.Sprintf(&quot;%dx%d^+0+0&quot;, w, h)
} else { 
// trim the width
w = uint(originalHeight * masterAspectRatio)
h = uint(originalHeight)
size = fmt.Sprintf(&quot;%dx%d^+0+0&quot;, w, h)
}
tx = mw.TransformImage(&quot;&quot;, size)		
tx.SetImageGravity(imagick.GRAVITY_CENTER)
offsetX := -(int(w) - int(tx.GetImageWidth())) / 2
offsetY := -(int(h) - int(tx.GetImageHeight())) / 2
err := tx.ExtentImage(w, h, offsetX, offsetY)
if float64(tx.GetImageWidth()) &gt; masterWidth &amp;&amp; float64(tx.GetImageHeight()) &gt; masterHeight  {                                        
err = tx.ResizeImage(uint(masterWidth), uint(masterHeight), imagick.FILTER_BOX, 1)
if err != nil {
log.Printf(&quot;Inside CreateMaster function Couldn&#39;t resize the image : &quot; + keyName + &quot;%s\n&quot;, err) 
return false				 
}                                                           
}                                       
} 		
paramsPut := &amp;s3.PutObjectInput{
Bucket:         aws.String(masterBucketName),
Key:            aws.String(keyName),
Body:         bytes.NewReader(tx.GetImageBlob()),
}
_, err = s3Client.PutObject(paramsPut)
if err != nil {
log.Printf(&quot;Inside CreateMaster function Couldn&#39;t put the image on s3 : &quot; + keyName + &quot;%s\n&quot;, err) 	
return false	    
}
return true
}

答案1

得分: 4

你正在泄漏一个魔杖。

在这里，你分配了一个新的魔杖，并将其延迟到销毁：

tx := imagick.NewMagickWand()
defer tx.Destroy()

但是在稍后的一个“if”块中，你用从调用TransformImage()返回的魔杖替换了它：

        tx = mw.TransformImage("", size)        
tx.SetImageGravity(imagick.GRAVITY_CENTER)

如果你完全摆脱了对新的魔杖的第一次分配，并确保销毁从TransformImage()返回的新魔杖，泄漏问题就会消失。

有关详细信息，请参阅问题跟踪器，＃72。

英文:

You are leaking a wand.

Here, you allocate a new wand and defer it to destroy:

tx := imagick.NewMagickWand()
defer tx.Destroy()

But then further down, in an "if" block, you replace it with the wand that is returned from the call to TransformImage():

        tx = mw.TransformImage(&quot;&quot;, size)        
tx.SetImageGravity(imagick.GRAVITY_CENTER)

If you completely get rid of the first allocation of that new magick wand, and simply make sure to Destroy() that new wand returned from TransformImage(), the leak goes away.

Ref the issue tracker, #72, for details

通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库，让每个人都能够通过互相帮助和分享经验来进步。

在Go语言的HTTP处理程序中存在内存泄漏问题。

问题

答案1

GAE GO – HTML模板和数据安全

是否可以捕获Ctrl+C信号（SIGINT）并以“defer”方式运行清理函数？

Is a bug: flag.Parse() could not get values after bool arg?

Golang 包括本地文件

What's the correct way to type hint an empty list as a literal in python?

如何在Highcharts Gantt中更改本地化的星期名称

如何在同一个流中使用多个过滤器和映射函数？

如何使用Map/Set来将代码优化到O(n)？

.NET MAUI Android在GitHub Actions上构建失败，错误代码为1。

如何在Playwright视觉比较中屏蔽多个定位器？

在C++中，可以使用可变模板参数来检索类型的内部类型。

selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: stale element not found

Creating and opening a URL to log in to Website via Basic Auth with Robot Framework/Selenium (Python)

AG Grid 在上下文菜单中以大文本形式打开

发表评论