英文:
TLS handshake timeout on requesting data concurrently from api
问题
这是我第一个使用并发的程序,所以我可能会错过一些非常简单的东西。
package main
import (
"net/http"
"net/url"
"log"
"flag"
"io/ioutil"
"fmt"
"golang.org/x/net/html"
"strings"
"encoding/json"
"os"
"html/template"
)
type fileInfo struct{
Title string `json:"Title"`
Year string `json:"Year"`
Runtime string `json:"Runtime"`
Genre string `json:"Genre"`
Rating string `json:"imdbRating"`
Description string `json:"Plot"`
Image string `json:"Poster"`
Awards string `json:"Awards"`
}
var movie struct{
Name string
Year string
}
var Movies []fileInfo
func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
go func(){
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}()
}
fmt.Println("Preparing data")
for _, f := range queryNames {
go GetTitleAndYear("https://opensubtitles.co/search?q=" + f)
}
fmt.Println("Done")
http.HandleFunc("/",ShowRatings)
http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
log.Fatal(http.ListenAndServe(":8080",nil))
}
func ShowRatings(w http.ResponseWriter,r *http.Request){
t,err := template.ParseFiles(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
if(err!=nil){
log.Fatal(err)
}
t.Execute(w,Movies)
}
func GetTitleAndYear(url string){
resp,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var movieData string
if resp.StatusCode != 200 {
fmt.Println("statuscode",err)
}
z := html.NewTokenizer(resp.Body)
for{
tt := z.Next()
if tt == html.ErrorToken{
return
}else if tt==html.StartTagToken{
t:= z.Token()
if t.Data=="h4"{
tt = z.Next()
tt = z.Next()
tt = z.Next()
t = z.Token()
movieData = strings.TrimSpace(t.Data)
break
}
}
}
movie.Name = movieData[:len(movieData)-6]
movie.Year = movieData[len(movieData)-5:len(movieData)-1]
movie.Name = strings.Replace(movie.Name, " ", "+", -1)
url = "http://www.omdbapi.com/?t=" + movie.Name + "&y=" + movie.Year + "&plot=short&r=json"
req,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var x fileInfo
jsonParser := json.NewDecoder(req.Body)
if err := jsonParser.Decode(&x); err != nil {
log.Fatal("parsing config file", err)
}
Movies = append(Movies,x)
fmt.Println(x.Title,x.Year)
}
这个程序第一次运行得很完美。但之后,它在随机的文件名上一直报 net/http:TLS Handshake timeout
错误。我不确定是什么原因导致的。可能的解决方案是什么?这个错误到底是什么?
编辑2:为了解决并发中的竞争问题,我使用了通道,但现在我的程序比以前慢得多。我更新的 main
函数如下:
func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}
fmt.Println("Preparing data")
ch := make(chan string)
done := make(chan bool)
go func(){
for{
name,more := <-ch
if more{
GetTitleAndYear("https://opensubtitles.co/search?q=" + name)
}else{
done <- true
}
}
}()
for i:=0;i<len(queryNames);i++{
ch <- queryNames[i]
}
<- done
fmt.Println("Preparation DONE")
http.HandleFunc("/",ShowRatings)
http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
log.Fatal(http.ListenAndServe(":8080",nil))
}
请给我提供一种方法,既能保持与以前相同的速度,又能避免竞争问题。
编辑1:我已经添加了完整的程序。在stackoverflow上无法显示行号,我只在 main
函数中使用了并发。如果你觉得有必要对我编写Go程序的方式提出建议,请这样做,我是一个初学者,我希望能做对事情。
英文:
This is my first program in which I have used concurrency so I might be missing something fairly simple.
package main
import(
"net/http"
"net/url"
"log"
"flag"
"io/ioutil"
"fmt"
"golang.org/x/net/html"
"strings"
"encoding/json"
"os"
"html/template"
)
type fileInfo struct{
Title string `json:"Title"`;
Year string `json:"Year"`;
Runtime string `json:"Runtime"`
Genre string `json:"Genre"`
Rating string `json:"imdbRating"`;
Description string `json:"Plot"`;
Image string `json:"Poster"`;
Awards string `json:"Awards"`;
}
var movie struct{
Name string;
Year string;
}
var Movies []fileInfo
func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
go func(){
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}()
}
//fmt.Println(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
fmt.Println("Preparing data")
for _, f := range queryNames {
go GetTitleAndYear("https://opensubtitles.co/search?q=" + f)
}
fmt.Println("Done")
http.HandleFunc("/",ShowRatings)
http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
log.Fatal(http.ListenAndServe(":8080",nil))
}
func ShowRatings(w http.ResponseWriter,r *http.Request){
t,err := template.ParseFiles(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
if(err!=nil){
log.Fatal(err)
}
t.Execute(w,Movies)
}
func GetTitleAndYear(url string){
resp,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var movieData string
if resp.StatusCode != 200 {
fmt.Println("statuscode",err)
}
z := html.NewTokenizer(resp.Body)
for{
tt := z.Next()
if tt == html.ErrorToken{
return
}else if tt==html.StartTagToken{
t:= z.Token()
if t.Data=="h4"{
tt = z.Next()
tt = z.Next()
tt = z.Next()
t = z.Token()
movieData = strings.TrimSpace(t.Data)
break
}
}
}
movie.Name = movieData[:len(movieData)-6]
movie.Year = movieData[len(movieData)-5:len(movieData)-1]
movie.Name = strings.Replace(movie.Name, " ", "+", -1)
url = "http://www.omdbapi.com/?t=" + movie.Name + "&y=" + movie.Year + "&plot=short&r=json"
req,err := http.Get(url)
if err!=nil{
log.Fatal(err)
}
var x fileInfo
jsonParser := json.NewDecoder(req.Body)
if err := jsonParser.Decode(&x); err != nil {
log.Fatal("parsing config file", err)
}
Movies = append(Movies,x)
fmt.Println(x.Title,x.Year)
}
This program ran perfectly for the first time. But after that it keeps on giving net/http:TLS Handshake timeout
on random filenames. I am not sure what is causing this. What might be a possible solution? Also what exactly is this error?
EDIT 2: To solve the racing problem in concurrency, I used channels but now my program is very very slow compared to before. My updated main function :
func main() {
flag.Parse()
files, _ := ioutil.ReadDir(flag.Args()[0])
var queryNames []string
for _, f := range files {
queryNames= append(queryNames,url.QueryEscape(f.Name()))
}
//fmt.Println(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/template/index.tpl")
fmt.Println("Preparing data")
ch := make(chan string)
done := make(chan bool)
go func(){
for{
name,more := <-ch
if more{
GetTitleAndYear("https://opensubtitles.co/search?q=" + name)
}else{
done <-true
}
}
}()
for i:=0;i<len(queryNames);i++{
ch <- queryNames[i]
}
<- done
fmt.Println("Preparation DONE")
http.HandleFunc("/",ShowRatings)
http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir(os.Getenv("GOPATH") + "/src/github.com/krashcan/review/static"))))
log.Fatal(http.ListenAndServe(":8080",nil))
}
Please suggest me a way to have the same speed as before but also avoid racing around problems.
EDIT 1: I have added my complete program. There is no way to depict line no.s on stack overflow , I have used concurrency only in the main function. If you feel the need to advice me on my way of writing go programs, please do so, I am a beginner and I would love to do things right.
答案1
得分: 8
net/http
在客户端发起大量http.Get()
请求时会出现。为了避免TLS握手超时错误
,可以使用以下代码:
t := &http.Transport{
Dial: (&net.Dialer{
Timeout: 60 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
// We use ABSURDLY large keys, and should probably not.
TLSHandshakeTimeout: 60 * time.Second,
}
c := &http.Client{
Transport: t,
}
resp, err := c.Get("https://internal.lan/")
在这里找到了相关信息:链接
英文:
The net/http
occurs when there is a large amount of http.Get() requests from client side. To avoid the TLS handshake timeout error
,
t := &http.Transport{
Dial: (&net.Dialer{
Timeout: 60 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
// We use ABSURDLY large keys, and should probably not.
TLSHandshakeTimeout: 60 * time.Second,
}
c := &http.Client{
Transport: t,
}
resp, err := c.Get("https://internal.lan/")
Found it here
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论