Golang如何更快地上传图片到Amazon S3？

huangapple
go评论157阅读模式
英文:

Golang amazon s3 how can I upload images faster

问题

我有一些图片，我正在使用官方的Amazon S3包将它们上传到我的S3账户。我的图片通常大小在250-350 KB左右，所以是小图片，但是它们需要大约8或9秒才能上传，这似乎太长了。有没有什么建议可以提高上传速度？以下是我的代码，即使我将调整图像大小的代码去掉，上传仍然需要8或9秒。

func UploadStreamImage(w http.ResponseWriter, r *http.Request) {
    r.ParseForm()
    var buff bytes.Buffer
    wg := sync.WaitGroup{}
    wg.Add(1)
    go func() {
        defer wg.Done()
        var buf bytes.Buffer
        sess, _ := session.NewSession(&aws.Config{
            Region:      aws.String("us-west-2"),
            Credentials: credentials.NewStaticCredentials(aws_access_key_id, aws_secret_access_key, ""),
        })
        svc := s3.New(sess)
        file, handler, err := r.FormFile("file")
        if err != nil {
            log_errors(" error on upload",err.Error(),w)
            fmt.Println("Error Uploading Image")
            return
        }
        defer file.Close()
        img,err := imaging.Decode(file)
        if err != nil {
            print("Imaging Open error")
            log_errors("Error decoding",err.Error(),w)
            return
        }
        imgSize,err := strconv.Atoi(r.FormValue("imgsize"))
        if err != nil {
            println("Error converting to integer")
            log_errors("Error converting to integer",err.Error(),w)
            return
        }
        b := img.Bounds()
        heightImg := b.Max.Y
        widthImg := b.Max.X
        // resize image
        height := int(float64(heightImg) * .23)
        width := int(float64(widthImg) * .23)
        if imgSize < 401 {
            height = int(float64(heightImg) * 1)
            width = int(float64(widthImg) * 1)
        } else if imgSize >= 401 && imgSize < 900 {
            height = int(float64(heightImg) * .68)
            width = int(float64(widthImg) * .68)
            println("Middle Image")
        } else if  imgSize >= 900 && imgSize < 1300 {
            height = int(float64(heightImg) * .45)
            width = int(float64(widthImg) * .45)
        } else if  imgSize >= 1301 && imgSize < 1700 {
            height = int(float64(heightImg) * .40)
            width = int(float64(widthImg) * .40)
        }
        new_img := imaging.Resize(img,width,height, imaging.Lanczos)
        // end resize
        err = imaging.Encode(&buf,new_img, imaging.JPEG)
        if err != nil {
            log.Println(err)
            log_errors(" error encoding file",err.Error(),w)
            return
        }
        r := bytes.NewReader(buf.Bytes())
        read_file,err := ioutil.ReadAll(r)
        if err != nil {
            fmt.Println("Error Reading file")
            log_errors(" error reading file",err.Error(),w)
            return
        }
        file.Read(read_file)
        fileBytes := bytes.NewReader(read_file)
        fileSize, err := buff.ReadFrom(fileBytes)
        if err != nil {
            log_errors(" error fileSize",err.Error(),w)
            return
        }
        fileType := http.DetectContentType(read_file)
        path := handler.Filename
        params := &s3.PutObjectInput{
            Bucket: aws.String("bucket name"),
            Key: aws.String(path),
            Body: fileBytes,
            ContentLength: aws.Int64(fileSize),
            ContentType: aws.String(fileType),
        }
        resp, err := svc.PutObject(params)
        if err != nil {
            fmt.Printf("bad response: %s", err)
            log_errors("error in putObject",err.Error(),w)
            return
        }
        fmt.Println(w,"Done")
    }()
    wg.Wait()
}

请注意，我只会翻译代码部分，不会回答关于翻译的问题。

英文:

I have some images that I am uploading to my s3 account using the official amazon s3 package . My images are typically around 250 - 350 KB so small images, however they take around 8 or 9 seconds to upload which seems excessive any suggestions on improving speed would be great. This is my code and if I take the resizing image code off it still takes a good 8 or 9 seconds still .

      func UploadStreamImage(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
var buff bytes.Buffer
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
var buf bytes.Buffer
sess, _ := session.NewSession(&amp;aws.Config{
Region:      aws.String(&quot;us-west-2&quot;),
Credentials: credentials.NewStaticCredentials(aws_access_key_id, aws_secret_access_key, &quot;&quot;),
})
svc := s3.New(sess)
file, handler, err := r.FormFile(&quot;file&quot;)
if err != nil {
log_errors(&quot; error on upload&quot;,err.Error(),w)
fmt.Println(&quot;Error Uploading Image&quot;)
return
}
defer file.Close()
img,err := imaging.Decode(file)
if err != nil {
print(&quot;Imaging Open error&quot;)
log_errors(&quot;Error decoding&quot;,err.Error(),w)
return
}
imgSize,err := strconv.Atoi(r.FormValue(&quot;imgsize&quot;))
if err != nil {
println(&quot;Error converting to integer&quot;)
log_errors(&quot;Error converting to integer&quot;,err.Error(),w)
return
}
b := img.Bounds()
heightImg := b.Max.Y
widthImg := b.Max.X
// resize image
height := int(float64(heightImg) * .23)
width := int(float64(widthImg) * .23)
if imgSize &lt; 401 {
height = int(float64(heightImg) * 1)
width = int(float64(widthImg) * 1)
} else if imgSize  &gt;= 401 &amp;&amp; imgSize &lt; 900 {
height = int(float64(heightImg) * .68)
width = int(float64(widthImg) * .68)
println(&quot;Middle Image&quot;)
} else if  imgSize  &gt;= 900 &amp;&amp; imgSize &lt; 1300 {
height = int(float64(heightImg) * .45)
width = int(float64(widthImg) * .45)
} else if  imgSize  &gt;= 1301 &amp;&amp; imgSize &lt; 1700 {
height = int(float64(heightImg) * .40)
width = int(float64(widthImg) * .40)
} 
new_img := imaging.Resize(img,width,height, imaging.Lanczos)
// end resize
err = imaging.Encode(&amp;buf,new_img, imaging.JPEG)
if err != nil {
log.Println(err)
log_errors(&quot; error encoding file&quot;,err.Error(),w)
return
}
r := bytes.NewReader(buf.Bytes())
read_file,err := ioutil.ReadAll(r)
if err != nil {
fmt.Println(&quot;Error Reading file&quot;)
log_errors(&quot; error reading file&quot;,err.Error(),w)
return
}
file.Read(read_file)
fileBytes := bytes.NewReader(read_file)
fileSize, err := buff.ReadFrom(fileBytes)
if err != nil {
log_errors(&quot; error fileSize&quot;,err.Error(),w)
return
}
fileType := http.DetectContentType(read_file)
path := handler.Filename
params := &amp;s3.PutObjectInput{
Bucket: aws.String(&quot;bucket name&quot;),
Key: aws.String(path),
Body: fileBytes,
ContentLength: aws.Int64(fileSize),
ContentType: aws.String(fileType),
}
resp, err := svc.PutObject(params)
if err != nil {
fmt.Printf(&quot;bad response: %s&quot;, err)
log_errors(&quot;error in putObject&quot;,err.Error(),w)
return
}
fmt.Println(w,&quot;Done&quot;)
}()
wg.Wait()
}

答案1

得分: 4

"提高速度？"这是一个主观的问题，取决于许多因素，如服务器上行链路、客户端上行链路等。

相反，我将提供改进你的代码片段的建议：

  • 让我们从sync.WaitGroup开始 - 根据你的代码流程，我没有看到它的好处，你只是创建并等待goroutine完成。不要为了使用而使用WaitGroup。提示：不要为了追求新功能而使用它，而是在需要时使用它。
  • 当你可以不使用bytes.Buffer和Reader时，不要创建多个。例如，在调整大小后获取文件大小，只需使用buf.Len()。
  • 如果你想将AWS上传分担出去，将上传代码部分提取到单独的func中，并作为goroutine调用。

我已经更新了你的代码片段（我没有测试你的代码，所以请根据需要进行修复/改进）：

func UploadStreamImage(w http.ResponseWriter, r *http.Request) {
    file, handler, err := r.FormFile("file")
    if err != nil {
        log_errors(" error on upload", err.Error(), w)
        fmt.Println("Error Uploading Image")
        return
    }
    defer file.Close()

    // 建议：你可以通过字节计算文件大小，而不是从表单中获取；因为调整大小后图像会改变
    imgSize, err := strconv.Atoi(r.FormValue("imgsize"))
    if err != nil {
        println("Error converting to integer")
        log_errors("Error converting to integer", err.Error(), w)
        return
    }

    img, err := imaging.Decode(file)
    if err != nil {
        print("Imaging Open error")
        log_errors("Error decoding", err.Error(), w)
        return
    }

    b := img.Bounds()
    heightImg := b.Max.Y
    widthImg := b.Max.X

    // 调整大小
    height := int(float64(heightImg) * .23)
    width := int(float64(widthImg) * .23)

    if imgSize < 401 {
        height = int(float64(heightImg) * 1)
        width = int(float64(widthImg) * 1)

    } else if imgSize >= 401 && imgSize < 900 {
        height = int(float64(heightImg) * .68)
        width = int(float64(widthImg) * .68)
        println("Middle Image")
    } else if imgSize >= 900 && imgSize < 1300 {
        height = int(float64(heightImg) * .45)
        width = int(float64(widthImg) * .45)
    } else if imgSize >= 1301 && imgSize < 1700 {
        height = int(float64(heightImg) * .40)
        width = int(float64(widthImg) * .40)
    }

    new_img := imaging.Resize(img, width, height, imaging.Lanczos)
    // 结束调整大小

    var buf bytes.Buffer
    err = imaging.Encode(&buf, new_img, imaging.JPEG)
    if err != nil {
        log.Println(err)
        log_errors(" error encoding file", err.Error(), w)
        return
    }

    fileType := http.DetectContentType(buf.Bytes())
    fileSize := buf.Len()
    path := handler.Filename
    params := &s3.PutObjectInput{
        Bucket: aws.String("bucket name"),
        Key:    aws.String(path),
        Body:   bytes.NewReader(buf.Bytes()),

        ContentLength: aws.Int64(fileSize),
        ContentType:   aws.String(fileType),
    }

    resp, err := svc.PutObject(params)
    if err != nil {
        fmt.Printf("bad response: %s", err)
        log_errors("error in putObject", err.Error(), w)
        return
    }

    fmt.Println("Done", resp)
}
英文:

Improving speed? this is subjective one; depends on many factors such as server uplink, client uplink, etc.

Instead, I will provide my inputs to improve your code snippets:

  • Let's start with sync.WaitGroup - I do not see a benefit as per your code flow, you just create and wait for goroutine to complete. Instead use without WaitGroup. Tip: Do not use feature for sake/buzz; use it when needed.
  • Do not create multiple bytes.Buffer and Reader, when you can accomplish without it. For e.g. getting fileSize after the resize, just do buf.Len().
  • If you want to offload the AWS upload then extract upload code part into separate func and call it as goroutine.

I have updated your code snippet (I have not tested your code, so please fix/improve it as required):

func UploadStreamImage(w http.ResponseWriter, r *http.Request) {
file, handler, err := r.FormFile(&quot;file&quot;)
if err != nil {
log_errors(&quot; error on upload&quot;, err.Error(), w)
fmt.Println(&quot;Error Uploading Image&quot;)
return
}
defer file.Close()
// Suggestion: You can calculate the file size from bytes.
// instead getting it from form; since after resize image will change
imgSize, err := strconv.Atoi(r.FormValue(&quot;imgsize&quot;))
if err != nil {
println(&quot;Error converting to integer&quot;)
log_errors(&quot;Error converting to integer&quot;, err.Error(), w)
return
}
img, err := imaging.Decode(file)
if err != nil {
print(&quot;Imaging Open error&quot;)
log_errors(&quot;Error decoding&quot;, err.Error(), w)
return
}
b := img.Bounds()
heightImg := b.Max.Y
widthImg := b.Max.X
// resize image
height := int(float64(heightImg) * .23)
width := int(float64(widthImg) * .23)
if imgSize &lt; 401 {
height = int(float64(heightImg) * 1)
width = int(float64(widthImg) * 1)
} else if imgSize &gt;= 401 &amp;&amp; imgSize &lt; 900 {
height = int(float64(heightImg) * .68)
width = int(float64(widthImg) * .68)
println(&quot;Middle Image&quot;)
} else if imgSize &gt;= 900 &amp;&amp; imgSize &lt; 1300 {
height = int(float64(heightImg) * .45)
width = int(float64(widthImg) * .45)
} else if imgSize &gt;= 1301 &amp;&amp; imgSize &lt; 1700 {
height = int(float64(heightImg) * .40)
width = int(float64(widthImg) * .40)
}
new_img := imaging.Resize(img, width, height, imaging.Lanczos)
// end resize
var buf bytes.Buffer
err = imaging.Encode(&amp;buf, new_img, imaging.JPEG)
if err != nil {
log.Println(err)
log_errors(&quot; error encoding file&quot;, err.Error(), w)
return
}
fileType := http.DetectContentType(buf.Bytes())
fileSize := buf.Len()
path := handler.Filename
params := &amp;s3.PutObjectInput{
Bucket: aws.String(&quot;bucket name&quot;),
Key:    aws.String(path),
Body:   bytes.NewReader(buf.Bytes()),
ContentLength: aws.Int64(fileSize),
ContentType:   aws.String(fileType),
}
resp, err := svc.PutObject(params)
if err != nil {
fmt.Printf(&quot;bad response: %s&quot;, err)
log_errors(&quot;error in putObject&quot;, err.Error(), w)
return
}
fmt.Println(&quot;Done&quot;, resp)
}

huangapple
  • 本文由 发表于 2017年7月8日 07:14:06
  • 转载请务必保留本文链接：https://go.coder-hub.com/44980906.html
  • amazon-s3
  • go
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定