从api并发请求数据时,发生了TLS握手超时。

huangapple go评论110阅读模式
英文:

TLS handshake timeout on requesting data concurrently from api

问题

这是我第一个使用并发的程序,所以我可能会错过一些非常简单的东西。

  1. package main
  2. import (
  3. "net/http"
  4. "net/url"
  5. "log"
  6. "flag"
  7. "io/ioutil"
  8. "fmt"
  9. "golang.org/x/net/html"
  10. "strings"
  11. "encoding/json"
  12. "os"
  13. "html/template"
  14. )
  15. type fileInfo struct{
  16. Title string `json:"Title"`
  17. Year string `json:"Year"`
  18. Runtime string `json:"Runtime"`
  19. Genre string `json:"Genre"`
  20. Rating string `json:"imdbRating"`
  21. Description string `json:"Plot"`
  22. Image string `json:"Poster"`
  23. Awards string `json:"Awards"`
  24. }
  25. var movie struct{
  26. Name string
  27. Year string
  28. }
  29. var Movies []fileInfo
  30. func main() {
  31. flag.Parse()
  32. files, _ := ioutil.ReadDir(flag.Args()[0])
  33. var queryNames []string
  34. for _, f := range files {
  35. go func(){
  36. queryNames= append(queryNames,url.QueryEscape(f.Name()))
  37. }()
  38. }
  39. fmt.Println("Preparing data")
  40. for _, f := range queryNames {
  41. go GetTitleAndYear("https://opensubtitles.co/search?q=" + f)
  42. }
  43. fmt.Println("Done")
  44. http.HandleFunc("/",ShowRatings)
  45. http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
  46. log.Fatal(http.ListenAndServe(":8080",nil))
  47. }
  48. func ShowRatings(w http.ResponseWriter,r *http.Request){
  49. t,err := template.ParseFiles(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
  50. if(err!=nil){
  51. log.Fatal(err)
  52. }
  53. t.Execute(w,Movies)
  54. }
  55. func GetTitleAndYear(url string){
  56. resp,err := http.Get(url)
  57. if err!=nil{
  58. log.Fatal(err)
  59. }
  60. var movieData string
  61. if resp.StatusCode != 200 {
  62. fmt.Println("statuscode",err)
  63. }
  64. z := html.NewTokenizer(resp.Body)
  65. for{
  66. tt := z.Next()
  67. if tt == html.ErrorToken{
  68. return
  69. }else if tt==html.StartTagToken{
  70. t:= z.Token()
  71. if t.Data=="h4"{
  72. tt = z.Next()
  73. tt = z.Next()
  74. tt = z.Next()
  75. t = z.Token()
  76. movieData = strings.TrimSpace(t.Data)
  77. break
  78. }
  79. }
  80. }
  81. movie.Name = movieData[:len(movieData)-6]
  82. movie.Year = movieData[len(movieData)-5:len(movieData)-1]
  83. movie.Name = strings.Replace(movie.Name, " ", "+", -1)
  84. url = "http://www.omdbapi.com/?t=" + movie.Name + "&y=" + movie.Year + "&plot=short&r=json"
  85. req,err := http.Get(url)
  86. if err!=nil{
  87. log.Fatal(err)
  88. }
  89. var x fileInfo
  90. jsonParser := json.NewDecoder(req.Body)
  91. if err := jsonParser.Decode(&x); err != nil {
  92. log.Fatal("parsing config file", err)
  93. }
  94. Movies = append(Movies,x)
  95. fmt.Println(x.Title,x.Year)
  96. }

这个程序第一次运行得很完美。但之后,它在随机的文件名上一直报 net/http:TLS Handshake timeout 错误。我不确定是什么原因导致的。可能的解决方案是什么?这个错误到底是什么?

编辑2:为了解决并发中的竞争问题,我使用了通道,但现在我的程序比以前慢得多。我更新的 main 函数如下:

  1. func main() {
  2. flag.Parse()
  3. files, _ := ioutil.ReadDir(flag.Args()[0])
  4. var queryNames []string
  5. for _, f := range files {
  6. queryNames= append(queryNames,url.QueryEscape(f.Name()))
  7. }
  8. fmt.Println("Preparing data")
  9. ch := make(chan string)
  10. done := make(chan bool)
  11. go func(){
  12. for{
  13. name,more := <-ch
  14. if more{
  15. GetTitleAndYear("https://opensubtitles.co/search?q=" + name)
  16. }else{
  17. done <- true
  18. }
  19. }
  20. }()
  21. for i:=0;i<len(queryNames);i++{
  22. ch <- queryNames[i]
  23. }
  24. <- done
  25. fmt.Println("Preparation DONE")
  26. http.HandleFunc("/",ShowRatings)
  27. http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
  28. log.Fatal(http.ListenAndServe(":8080",nil))
  29. }

请给我提供一种方法,既能保持与以前相同的速度,又能避免竞争问题。

编辑1:我已经添加了完整的程序。在stackoverflow上无法显示行号,我只在 main 函数中使用了并发。如果你觉得有必要对我编写Go程序的方式提出建议,请这样做,我是一个初学者,我希望能做对事情。

英文:

This is my first program in which I have used concurrency so I might be missing something fairly simple.

  1. package main
  2. import(
  3. &quot;net/http&quot;
  4. &quot;net/url&quot;
  5. &quot;log&quot;
  6. &quot;flag&quot;
  7. &quot;io/ioutil&quot;
  8. &quot;fmt&quot;
  9. &quot;golang.org/x/net/html&quot;
  10. &quot;strings&quot;
  11. &quot;encoding/json&quot;
  12. &quot;os&quot;
  13. &quot;html/template&quot;
  14. )
  15. type fileInfo struct{
  16. Title string `json:&quot;Title&quot;`;
  17. Year string `json:&quot;Year&quot;`;
  18. Runtime string `json:&quot;Runtime&quot;`
  19. Genre string `json:&quot;Genre&quot;`
  20. Rating string `json:&quot;imdbRating&quot;`;
  21. Description string `json:&quot;Plot&quot;`;
  22. Image string `json:&quot;Poster&quot;`;
  23. Awards string `json:&quot;Awards&quot;`;
  24. }
  25. var movie struct{
  26. Name string;
  27. Year string;
  28. }
  29. var Movies []fileInfo
  30. func main() {
  31. flag.Parse()
  32. files, _ := ioutil.ReadDir(flag.Args()[0])
  33. var queryNames []string
  34. for _, f := range files {
  35. go func(){
  36. queryNames= append(queryNames,url.QueryEscape(f.Name()))
  37. }()
  38. }
  39. //fmt.Println(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
  40. fmt.Println(&quot;Preparing data&quot;)
  41. for _, f := range queryNames {
  42. go GetTitleAndYear(&quot;https://opensubtitles.co/search?q=&quot; + f)
  43. }
  44. fmt.Println(&quot;Done&quot;)
  45. http.HandleFunc(&quot;/&quot;,ShowRatings)
  46. http.Handle(&quot;/static/&quot;,http.StripPrefix(&quot;/static/&quot;,http.FileServer(http.Dir(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/static&quot;))))
  47. log.Fatal(http.ListenAndServe(&quot;:8080&quot;,nil))
  48. }
  49. func ShowRatings(w http.ResponseWriter,r *http.Request){
  50. t,err := template.ParseFiles(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
  51. if(err!=nil){
  52. log.Fatal(err)
  53. }
  54. t.Execute(w,Movies)
  55. }
  56. func GetTitleAndYear(url string){
  57. resp,err := http.Get(url)
  58. if err!=nil{
  59. log.Fatal(err)
  60. }
  61. var movieData string
  62. if resp.StatusCode != 200 {
  63. fmt.Println(&quot;statuscode&quot;,err)
  64. }
  65. z := html.NewTokenizer(resp.Body)
  66. for{
  67. tt := z.Next()
  68. if tt == html.ErrorToken{
  69. return
  70. }else if tt==html.StartTagToken{
  71. t:= z.Token()
  72. if t.Data==&quot;h4&quot;{
  73. tt = z.Next()
  74. tt = z.Next()
  75. tt = z.Next()
  76. t = z.Token()
  77. movieData = strings.TrimSpace(t.Data)
  78. break
  79. }
  80. }
  81. }
  82. movie.Name = movieData[:len(movieData)-6]
  83. movie.Year = movieData[len(movieData)-5:len(movieData)-1]
  84. movie.Name = strings.Replace(movie.Name, &quot; &quot;, &quot;+&quot;, -1)
  85. url = &quot;http://www.omdbapi.com/?t=&quot; + movie.Name + &quot;&amp;y=&quot; + movie.Year + &quot;&amp;plot=short&amp;r=json&quot;
  86. req,err := http.Get(url)
  87. if err!=nil{
  88. log.Fatal(err)
  89. }
  90. var x fileInfo
  91. jsonParser := json.NewDecoder(req.Body)
  92. if err := jsonParser.Decode(&amp;x); err != nil {
  93. log.Fatal(&quot;parsing config file&quot;, err)
  94. }
  95. Movies = append(Movies,x)
  96. fmt.Println(x.Title,x.Year)
  97. }

This program ran perfectly for the first time. But after that it keeps on giving net/http:TLS Handshake timeout on random filenames. I am not sure what is causing this. What might be a possible solution? Also what exactly is this error?

EDIT 2: To solve the racing problem in concurrency, I used channels but now my program is very very slow compared to before. My updated main function :

  1. func main() {
  2. flag.Parse()
  3. files, _ := ioutil.ReadDir(flag.Args()[0])
  4. var queryNames []string
  5. for _, f := range files {
  6. queryNames= append(queryNames,url.QueryEscape(f.Name()))
  7. }
  8. //fmt.Println(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
  9. fmt.Println(&quot;Preparing data&quot;)
  10. ch := make(chan string)
  11. done := make(chan bool)
  12. go func(){
  13. for{
  14. name,more := &lt;-ch
  15. if more{
  16. GetTitleAndYear(&quot;https://opensubtitles.co/search?q=&quot; + name)
  17. }else{
  18. done &lt;-true
  19. }
  20. }
  21. }()
  22. for i:=0;i&lt;len(queryNames);i++{
  23. ch &lt;- queryNames[i]
  24. }
  25. &lt;- done
  26. fmt.Println(&quot;Preparation DONE&quot;)
  27. http.HandleFunc(&quot;/&quot;,ShowRatings)
  28. http.Handle(&quot;/static/&quot;,http.StripPrefix(&quot;/static/&quot;,http.FileServer(http.Dir(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/static&quot;))))
  29. log.Fatal(http.ListenAndServe(&quot;:8080&quot;,nil))
  30. }

Please suggest me a way to have the same speed as before but also avoid racing around problems.

EDIT 1: I have added my complete program. There is no way to depict line no.s on stack overflow , I have used concurrency only in the main function. If you feel the need to advice me on my way of writing go programs, please do so, I am a beginner and I would love to do things right.

答案1

得分: 8

net/http在客户端发起大量http.Get()请求时会出现。为了避免TLS握手超时错误,可以使用以下代码:

  1. t := &http.Transport{
  2. Dial: (&net.Dialer{
  3. Timeout: 60 * time.Second,
  4. KeepAlive: 30 * time.Second,
  5. }).Dial,
  6. // We use ABSURDLY large keys, and should probably not.
  7. TLSHandshakeTimeout: 60 * time.Second,
  8. }
  9. c := &http.Client{
  10. Transport: t,
  11. }
  12. resp, err := c.Get("https://internal.lan/")

在这里找到了相关信息:链接

英文:

The net/http occurs when there is a large amount of http.Get() requests from client side. To avoid the TLS handshake timeout error,

  1. t := &amp;http.Transport{
  2. Dial: (&amp;net.Dialer{
  3. Timeout: 60 * time.Second,
  4. KeepAlive: 30 * time.Second,
  5. }).Dial,
  6. // We use ABSURDLY large keys, and should probably not.
  7. TLSHandshakeTimeout: 60 * time.Second,
  8. }
  9. c := &amp;http.Client{
  10. Transport: t,
  11. }
  12. resp, err := c.Get(&quot;https://internal.lan/&quot;)

Found it here

huangapple
  • 本文由 发表于 2017年1月18日 20:45:10
  • 转载请务必保留本文链接:https://go.coder-hub.com/41719797.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定