从api并发请求数据时,发生了TLS握手超时。

huangapple go评论82阅读模式
英文:

TLS handshake timeout on requesting data concurrently from api

问题

这是我第一个使用并发的程序,所以我可能会错过一些非常简单的东西。

package main

import (
	"net/http"
	"net/url"
	"log"
	"flag"
	"io/ioutil"
	"fmt"
	"golang.org/x/net/html"
	"strings"
	"encoding/json"
	"os"
	"html/template"
)

type fileInfo struct{
	Title string `json:"Title"`
	Year string `json:"Year"`
	Runtime string `json:"Runtime"`
	Genre string `json:"Genre"`
	Rating string `json:"imdbRating"`
	Description string `json:"Plot"`
	Image string `json:"Poster"`
	Awards string `json:"Awards"`
}

var movie struct{
	Name string
	Year string
}
var Movies []fileInfo

func main() {
	flag.Parse()
	files, _ := ioutil.ReadDir(flag.Args()[0])
	var queryNames []string
	for _, f := range files {
		go func(){
			queryNames= append(queryNames,url.QueryEscape(f.Name()))
		}()
	}
	fmt.Println("Preparing data")

	for _, f := range queryNames {
		go GetTitleAndYear("https://opensubtitles.co/search?q=" + f)
	}

	fmt.Println("Done")
	http.HandleFunc("/",ShowRatings)
	http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))

	log.Fatal(http.ListenAndServe(":8080",nil))
}

func ShowRatings(w http.ResponseWriter,r *http.Request){
	t,err := template.ParseFiles(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
	if(err!=nil){
		log.Fatal(err)
	}
	t.Execute(w,Movies)
}

func GetTitleAndYear(url string){
	resp,err := http.Get(url)
	if err!=nil{
		log.Fatal(err)
	}
	var movieData string
	if resp.StatusCode != 200 {
		fmt.Println("statuscode",err)
	}
	z := html.NewTokenizer(resp.Body)
	for{
		tt := z.Next()

		if tt == html.ErrorToken{
			return
		}else if tt==html.StartTagToken{
			t:= z.Token()
			if t.Data=="h4"{
				tt = z.Next()
				tt = z.Next()
				tt = z.Next()
				t = z.Token()
				movieData = strings.TrimSpace(t.Data)				
				break
			}
		}
	}

	movie.Name = movieData[:len(movieData)-6]
	movie.Year = movieData[len(movieData)-5:len(movieData)-1]
	movie.Name = strings.Replace(movie.Name, " ", "+", -1)
	url = "http://www.omdbapi.com/?t=" + movie.Name + "&y=" + movie.Year + "&plot=short&r=json"  
	req,err := http.Get(url)

	if err!=nil{
		log.Fatal(err)
	}
	var x fileInfo
	jsonParser := json.NewDecoder(req.Body)
	if err := jsonParser.Decode(&x); err != nil {
		log.Fatal("parsing config file", err)
	}
	Movies = append(Movies,x)
	fmt.Println(x.Title,x.Year)     
}

这个程序第一次运行得很完美。但之后,它在随机的文件名上一直报 net/http:TLS Handshake timeout 错误。我不确定是什么原因导致的。可能的解决方案是什么?这个错误到底是什么?

编辑2:为了解决并发中的竞争问题,我使用了通道,但现在我的程序比以前慢得多。我更新的 main 函数如下:

func main() {
	flag.Parse()
	files, _ := ioutil.ReadDir(flag.Args()[0])
	var queryNames []string
	for _, f := range files {
		queryNames= append(queryNames,url.QueryEscape(f.Name()))
	}
	fmt.Println("Preparing data")
	ch := make(chan string)
	done := make(chan bool)
	go func(){
		for{
			name,more := <-ch
			if more{
				GetTitleAndYear("https://opensubtitles.co/search?q=" + name)
			}else{
				done <- true
			}
		}
	}()

	for i:=0;i<len(queryNames);i++{
		ch <- queryNames[i]
	}
	<- done
	fmt.Println("Preparation DONE")
	http.HandleFunc("/",ShowRatings)
	http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))

	log.Fatal(http.ListenAndServe(":8080",nil))
}

请给我提供一种方法,既能保持与以前相同的速度,又能避免竞争问题。

编辑1:我已经添加了完整的程序。在stackoverflow上无法显示行号,我只在 main 函数中使用了并发。如果你觉得有必要对我编写Go程序的方式提出建议,请这样做,我是一个初学者,我希望能做对事情。

英文:

This is my first program in which I have used concurrency so I might be missing something fairly simple.

package main
import(
&quot;net/http&quot;
&quot;net/url&quot;
&quot;log&quot;
&quot;flag&quot;
&quot;io/ioutil&quot;
&quot;fmt&quot;
&quot;golang.org/x/net/html&quot;
&quot;strings&quot;
&quot;encoding/json&quot;
&quot;os&quot;
&quot;html/template&quot;
)
type fileInfo struct{
Title string `json:&quot;Title&quot;`;
Year string `json:&quot;Year&quot;`;
Runtime string `json:&quot;Runtime&quot;`
Genre string `json:&quot;Genre&quot;` 
Rating string `json:&quot;imdbRating&quot;`;
Description string `json:&quot;Plot&quot;`;
Image string `json:&quot;Poster&quot;`;
Awards string `json:&quot;Awards&quot;`;
}
var movie struct{
Name string;
Year string;
}
var Movies []fileInfo
func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
go func(){
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}()
}
//fmt.Println(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
fmt.Println(&quot;Preparing data&quot;)
for _, f := range queryNames {
go GetTitleAndYear(&quot;https://opensubtitles.co/search?q=&quot; + f)
}
fmt.Println(&quot;Done&quot;)
http.HandleFunc(&quot;/&quot;,ShowRatings)
http.Handle(&quot;/static/&quot;,http.StripPrefix(&quot;/static/&quot;,http.FileServer(http.Dir(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/static&quot;))))
log.Fatal(http.ListenAndServe(&quot;:8080&quot;,nil))
}
func ShowRatings(w http.ResponseWriter,r *http.Request){
t,err := template.ParseFiles(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
if(err!=nil){
log.Fatal(err)
}
t.Execute(w,Movies)
}
func GetTitleAndYear(url string){
resp,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var movieData string
if resp.StatusCode != 200 {
fmt.Println(&quot;statuscode&quot;,err)
}
z := html.NewTokenizer(resp.Body)
for{
tt := z.Next()
if tt == html.ErrorToken{
return
}else if tt==html.StartTagToken{
t:= z.Token()
if t.Data==&quot;h4&quot;{
tt = z.Next()
tt = z.Next()
tt = z.Next()
t = z.Token()
movieData = strings.TrimSpace(t.Data)				
break
}
}
}
movie.Name = movieData[:len(movieData)-6]
movie.Year = movieData[len(movieData)-5:len(movieData)-1]
movie.Name = strings.Replace(movie.Name, &quot; &quot;, &quot;+&quot;, -1)
url = &quot;http://www.omdbapi.com/?t=&quot; + movie.Name + &quot;&amp;y=&quot; + movie.Year + &quot;&amp;plot=short&amp;r=json&quot;  
req,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var x fileInfo
jsonParser := json.NewDecoder(req.Body)
if err := jsonParser.Decode(&amp;x); err != nil {
log.Fatal(&quot;parsing config file&quot;, err)
}
Movies = append(Movies,x)
fmt.Println(x.Title,x.Year)     
}

This program ran perfectly for the first time. But after that it keeps on giving net/http:TLS Handshake timeout on random filenames. I am not sure what is causing this. What might be a possible solution? Also what exactly is this error?

EDIT 2: To solve the racing problem in concurrency, I used channels but now my program is very very slow compared to before. My updated main function :

func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}
//fmt.Println(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/template/index.tpl&quot;)
fmt.Println(&quot;Preparing data&quot;)
ch := make(chan string)
done := make(chan bool)
go func(){
for{
name,more := &lt;-ch
if more{
GetTitleAndYear(&quot;https://opensubtitles.co/search?q=&quot; + name)
}else{
done &lt;-true
}
}
}()
for i:=0;i&lt;len(queryNames);i++{
ch &lt;- queryNames[i] 
}
&lt;- done
fmt.Println(&quot;Preparation DONE&quot;)
http.HandleFunc(&quot;/&quot;,ShowRatings)
http.Handle(&quot;/static/&quot;,http.StripPrefix(&quot;/static/&quot;,http.FileServer(http.Dir(os.Getenv(&quot;GOPATH&quot;) + &quot;/src/github.com/krashcan/review/static&quot;))))
log.Fatal(http.ListenAndServe(&quot;:8080&quot;,nil))
}

Please suggest me a way to have the same speed as before but also avoid racing around problems.

EDIT 1: I have added my complete program. There is no way to depict line no.s on stack overflow , I have used concurrency only in the main function. If you feel the need to advice me on my way of writing go programs, please do so, I am a beginner and I would love to do things right.

答案1

得分: 8

net/http在客户端发起大量http.Get()请求时会出现。为了避免TLS握手超时错误,可以使用以下代码:

t := &http.Transport{
    Dial: (&net.Dialer{
        Timeout:   60 * time.Second,
        KeepAlive: 30 * time.Second,
    }).Dial,
    // We use ABSURDLY large keys, and should probably not.
    TLSHandshakeTimeout: 60 * time.Second,
}
c := &http.Client{
    Transport: t,
}
resp, err := c.Get("https://internal.lan/")

在这里找到了相关信息:链接

英文:

The net/http occurs when there is a large amount of http.Get() requests from client side. To avoid the TLS handshake timeout error,

t := &amp;http.Transport{
Dial: (&amp;net.Dialer{
Timeout: 60 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
// We use ABSURDLY large keys, and should probably not.
TLSHandshakeTimeout: 60 * time.Second,
}
c := &amp;http.Client{
Transport: t,
}
resp, err := c.Get(&quot;https://internal.lan/&quot;)

Found it here

huangapple
  • 本文由 发表于 2017年1月18日 20:45:10
  • 转载请务必保留本文链接:https://go.coder-hub.com/41719797.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定