为什么使用Golang并发上传文件到S3存储桶会导致被取消和上下文超时?

huangapple go评论117阅读模式
英文:

Why does golang concurrent uploading of files to S3 bucket result in canceled, context deadline exceeded

问题

我已经写了一小段用于递归遍历目录并上传目录中文件的 Golang 代码。目录中大约有93K+个项目。一段时间后,我遇到了以下错误:

上传文件时出错:/Users/randolphhill/Fat-Tree-Business/SandBox/DDD/heydoc/ios/Pods/gRPC-Core/src/core/ext/transport/chttp2/alpn/alpn.h
操作错误 S3: PutObject,https 响应错误 StatusCode: 0,RequestID: ,HostID: ,已取消,上下文截止时间已过。

以下是代码片段:

  1. func PutFile(c context.Context, api S3PutObjectAPI, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
  2. return api.PutObject(c, input)
  3. }
  4. func PutFileS3(dir, filename, bucket, reg string) error {
  5. var cfg aws.Config
  6. st, err := fthash.Filehash(dir + filename)
  7. if err != nil {
  8. panic("configuration error, " + err.Error())
  9. return err
  10. }
  11. m := make(map[string]string)
  12. m["hashcode"] = st
  13. cfg, err = config.LoadDefaultConfig(context.TODO(), config.WithRegion(reg))
  14. if err != nil {
  15. panic("configuration error, " + err.Error())
  16. }
  17. client := s3.NewFromConfig(cfg)
  18. tmp := "backup" + dir + filename
  19. uri := strings.Replace(tmp, " ", "##,##", -1)
  20. if checkFileOnS3(client, bucket, uri, st) {
  21. fmt.Println(" FILE EXIST")
  22. return nil
  23. }
  24. file, err2 := os.Open(dir + filename)
  25. defer file.Close()
  26. if err2 != nil {
  27. fmt.Println("Unable to open file " + filename)
  28. return err2
  29. }
  30. tmp = "backup" + dir + filename
  31. //uri := "backup" + dir + filename
  32. uri = strings.Replace(tmp, " ", "##,##", -1)
  33. input := &s3.PutObjectInput{
  34. Bucket: &bucket,
  35. Key: aws.String(uri),
  36. Body: file,
  37. Metadata: m,
  38. }
  39. ctx, cancelFn := context.WithTimeout(context.TODO(), 10*time.Second)
  40. defer cancelFn()
  41. _, err2 = PutFile(ctx, client, input)
  42. if err2 != nil {
  43. fmt.Println("Got error uploading file:", dir+filename)
  44. fmt.Println(err2)
  45. return err2
  46. }
  47. return nil
  48. }

希望对你有帮助!

英文:

I have written a small golang piece of code to recursive traverse a directory and upload the files in the director. There are approximately 93K+ items in the directory.
After a while I get the following error:

Got error uploading file: /Users/randolphhill/Fat-Tree-Business/SandBox/DDD/heydoc/ios/Pods/gRPC-Core/src/core/ext/transport/chttp2/alpn/alpn.h
operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , canceled, context deadline exceeded.

Below is the code snippet

  1. func PutFile(c context.Context, api S3PutObjectAPI, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
  2. return api.PutObject(c, input)
  3. }
  4. func PutFileS3(dir, filename, bucket, reg string) error {
  5. var cfg aws.Config
  6. st, err := fthash.Filehash(dir + filename)
  7. if err != nil {
  8. panic("configuration error, " + err.Error())
  9. return err
  10. }
  11. m := make(map[string]string)
  12. m["hashcode"] = st
  13. cfg, err = config.LoadDefaultConfig(context.TODO(), config.WithRegion(reg))
  14. if err != nil {
  15. panic("configuration error, " + err.Error())
  16. }
  17. client := s3.NewFromConfig(cfg)
  18. tmp := "backup" + dir + filename
  19. uri := strings.Replace(tmp, " ", "##,##", -1)
  20. if checkFileOnS3(client, bucket, uri, st) {
  21. fmt.Println(" FILE EXIST")
  22. return nil
  23. }
  24. file, err2 := os.Open(dir + filename)
  25. defer file.Close()
  26. if err2 != nil {
  27. fmt.Println("Unable to open file " + filename)
  28. return err2
  29. }
  30. tmp = "backup" + dir + filename
  31. //uri := "backup" + dir + filename
  32. uri = strings.Replace(tmp, " ", "##,##", -1)
  33. input := &s3.PutObjectInput{
  34. Bucket: &bucket,
  35. Key: aws.String(uri),
  36. //Key: &filename,
  37. Body: file,
  38. Metadata: m,
  39. }
  40. ctx, cancelFn := context.WithTimeout(context.TODO(), 10*time.Second)
  41. defer cancelFn()
  42. _, err2 = PutFile(ctx, client, input)
  43. if err2 != nil {
  44. fmt.Println("Got error uploading file:", dir+filename)
  45. fmt.Println(err2)
  46. return err2
  47. }
  48. return nil
  49. }

答案1

得分: 1

你在这里添加了一个10秒的超时:

  1. ctx, cancelFn := context.WithTimeout(context.TODO(), 10*time.Second)
  2. defer cancelFn()
  3. _, err2 = PutFile(ctx, client, input)
  4. if err2 != nil {
  5. fmt.Println("Got error uploading file:", dir+filename)
  6. fmt.Println(err2)
  7. return err2
  8. }

10秒后,调用PutFile将以上下文错误退出。如果你有需要更长时间上传的文件,你可能只需要增加超时时间。

英文:

You've added a 10 second timeout here:

  1. ctx, cancelFn := context.WithTimeout(context.TODO(), 10*time.Second)
  2. defer cancelFn()
  3. _, err2 = PutFile(ctx, client, input)
  4. if err2 != nil {
  5. fmt.Println("Got error uploading file:", dir+filename)
  6. fmt.Println(err2)
  7. return err2
  8. }

After 10 seconds, the call to PutFile will exit with a context error. You likely just need to increase the timeout if you have files that take longer to upload.

答案2

得分: 0

  1. package main
  2. import (
  3. // "html/template"
  4. "log"
  5. "net/http"
  6. "os"
  7. "github.com/gin-gonic/gin"
  8. "github.com/joho/godotenv"
  9. "github.com/aws/aws-sdk-go/aws"
  10. "github.com/aws/aws-sdk-go/aws/credentials"
  11. "github.com/aws/aws-sdk-go/aws/session"
  12. "github.com/aws/aws-sdk-go/service/s3/s3manager"
  13. )
  14. var AccessKeyID string
  15. var SecretAccessKey string
  16. var MyRegion string
  17. var MyBucket string
  18. var filepath string
  19. //GetEnvWithKey : get env value
  20. func GetEnvWithKey(key string) string {
  21. return os.Getenv(key)
  22. }
  23. func LoadEnv() {
  24. err := godotenv.Load(".env")
  25. if err != nil {
  26. log.Fatalf("Error loading .env file")
  27. os.Exit(1)
  28. }
  29. }
  30. func ConnectAws() *session.Session {
  31. AccessKeyID = GetEnvWithKey("AWS_ACCESS_KEY_ID")
  32. SecretAccessKey = GetEnvWithKey("AWS_SECRET_ACCESS_KEY")
  33. MyRegion = GetEnvWithKey("AWS_REGION")
  34. sess, err := session.NewSession(
  35. &aws.Config{
  36. Region: aws.String(MyRegion),
  37. Credentials: credentials.NewStaticCredentials(
  38. AccessKeyID,
  39. SecretAccessKey,
  40. "", // a token will be created when the session it's used.
  41. ),
  42. })
  43. if err != nil {
  44. panic(err)
  45. }
  46. return sess
  47. }
  48. func SetupRouter(sess *session.Session) {
  49. router := gin.Default()
  50. router.Use(func(c *gin.Context) {
  51. c.Set("sess", sess)
  52. c.Next()
  53. })
  54. // router.Get("/upload", Form)
  55. router.POST("/upload", UploadImage)
  56. // router.GET("/image", controllers.DisplayImage)
  57. _ = router.Run(":4000")
  58. }
  59. func UploadImage(c *gin.Context) {
  60. sess := c.MustGet("sess").(*session.Session)
  61. uploader := s3manager.NewUploader(sess)
  62. MyBucket = GetEnvWithKey("BUCKET_NAME")
  63. file, header, err := c.Request.FormFile("photo")
  64. filename := header.Filename
  65. //upload to the s3 bucket
  66. up, err := uploader.Upload(&s3manager.UploadInput{
  67. Bucket: aws.String(MyBucket),
  68. //ACL: aws.String("public-read"),
  69. Key: aws.String(filename),
  70. Body: file,
  71. })
  72. if err != nil {
  73. c.JSON(http.StatusInternalServerError, gin.H{
  74. "error": "Failed to upload file",
  75. "uploader": up,
  76. })
  77. return
  78. }
  79. filepath = "https://" + MyBucket + "." + "s3-" + MyRegion + ".amazonaws.com/" + filename
  80. c.JSON(http.StatusOK, gin.H{
  81. "filepath": filepath,
  82. })
  83. }
  84. func main() {
  85. LoadEnv()
  86. sess := ConnectAws()
  87. router := gin.Default()
  88. router.Use(func(c *gin.Context) {
  89. c.Set("sess", sess)
  90. c.Next()
  91. })
  92. router.POST("/upload", UploadImage)
  93. //router.LoadHTMLGlob("templates/*")
  94. //router.GET("/image", func(c *gin.Context) {
  95. //c.HTML(http.StatusOK, "index.tmpl", gin.H{
  96. // "title": "Main website",
  97. //})
  98. //})
  99. _ = router.Run(":4000")
  100. }
英文:

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-html -->

  1. package main
  2. import (
  3. // &quot;html/template&quot;
  4. &quot;log&quot;
  5. &quot;net/http&quot;
  6. &quot;os&quot;
  7. &quot;github.com/gin-gonic/gin&quot;
  8. &quot;github.com/joho/godotenv&quot;
  9. &quot;github.com/aws/aws-sdk-go/aws&quot;
  10. &quot;github.com/aws/aws-sdk-go/aws/credentials&quot;
  11. &quot;github.com/aws/aws-sdk-go/aws/session&quot;
  12. &quot;github.com/aws/aws-sdk-go/service/s3/s3manager&quot;
  13. )
  14. var AccessKeyID string
  15. var SecretAccessKey string
  16. var MyRegion string
  17. var MyBucket string
  18. var filepath string
  19. //GetEnvWithKey : get env value
  20. func GetEnvWithKey(key string) string {
  21. return os.Getenv(key)
  22. }
  23. func LoadEnv() {
  24. err := godotenv.Load(&quot;.env&quot;)
  25. if err != nil {
  26. log.Fatalf(&quot;Error loading .env file&quot;)
  27. os.Exit(1)
  28. }
  29. }
  30. func ConnectAws() *session.Session {
  31. AccessKeyID = GetEnvWithKey(&quot;AWS_ACCESS_KEY_ID&quot;)
  32. SecretAccessKey = GetEnvWithKey(&quot;AWS_SECRET_ACCESS_KEY&quot;)
  33. MyRegion = GetEnvWithKey(&quot;AWS_REGION&quot;)
  34. sess, err := session.NewSession(
  35. &amp;aws.Config{
  36. Region: aws.String(MyRegion),
  37. Credentials: credentials.NewStaticCredentials(
  38. AccessKeyID,
  39. SecretAccessKey,
  40. &quot;&quot;, // a token will be created when the session it&#39;s used.
  41. ),
  42. })
  43. if err != nil {
  44. panic(err)
  45. }
  46. return sess
  47. }
  48. func SetupRouter(sess *session.Session) {
  49. router := gin.Default()
  50. router.Use(func(c *gin.Context) {
  51. c.Set(&quot;sess&quot;, sess)
  52. c.Next()
  53. })
  54. // router.Get(&quot;/upload&quot;, Form)
  55. router.POST(&quot;/upload&quot;, UploadImage)
  56. // router.GET(&quot;/image&quot;, controllers.DisplayImage)
  57. _ = router.Run(&quot;:4000&quot;)
  58. }
  59. func UploadImage(c *gin.Context) {
  60. sess := c.MustGet(&quot;sess&quot;).(*session.Session)
  61. uploader := s3manager.NewUploader(sess)
  62. MyBucket = GetEnvWithKey(&quot;BUCKET_NAME&quot;)
  63. file, header, err := c.Request.FormFile(&quot;photo&quot;)
  64. filename := header.Filename
  65. //upload to the s3 bucket
  66. up, err := uploader.Upload(&amp;s3manager.UploadInput{
  67. Bucket: aws.String(MyBucket),
  68. //ACL: aws.String(&quot;public-read&quot;),
  69. Key: aws.String(filename),
  70. Body: file,
  71. })
  72. if err != nil {
  73. c.JSON(http.StatusInternalServerError, gin.H{
  74. &quot;error&quot;: &quot;Failed to upload file&quot;,
  75. &quot;uploader&quot;: up,
  76. })
  77. return
  78. }
  79. filepath = &quot;https://&quot; + MyBucket + &quot;.&quot; + &quot;s3-&quot; + MyRegion + &quot;.amazonaws.com/&quot; + filename
  80. c.JSON(http.StatusOK, gin.H{
  81. &quot;filepath&quot;: filepath,
  82. })
  83. }
  84. func main() {
  85. LoadEnv()
  86. sess := ConnectAws()
  87. router := gin.Default()
  88. router.Use(func(c *gin.Context) {
  89. c.Set(&quot;sess&quot;, sess)
  90. c.Next()
  91. })
  92. router.POST(&quot;/upload&quot;, UploadImage)
  93. //router.LoadHTMLGlob(&quot;templates/*&quot;)
  94. //router.GET(&quot;/image&quot;, func(c *gin.Context) {
  95. //c.HTML(http.StatusOK, &quot;index.tmpl&quot;, gin.H{
  96. // &quot;title&quot;: &quot;Main website&quot;,
  97. //})
  98. //})
  99. _ = router.Run(&quot;:4000&quot;)
  100. }

huangapple
  • 本文由 发表于 2022年2月6日 01:22:03
  • 转载请务必保留本文链接:https://go.coder-hub.com/71000371.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定