英文:
Colly Go package: how to check if the error is a Timeout error?
问题
除了我想在超时错误时仅重试之外,一切都正常,但我不知道如何与生成的特定Client.Timeout
错误进行比较。
缺失的部分在这里的注释中:if errors.Is(err, colly.Client.Timeout)...
:
package main
import (
"crypto/tls"
"fmt"
"github.com/gocolly/colly"
"net/http"
"os"
"strings"
)
func main() {
crawl()
}
func crawl() {
httpMethod := "https"
domains := []string{
"www.myweb1.com",
"www.myweb2.com",
}
for _, domain := range domains {
// 实例化默认的收集器
c := colly.NewCollector(
colly.Async(true),
colly.AllowedDomains(domain),
)
c.Limit(&colly.LimitRule{Parallelism: 100})
/* 默认 = 10s = 1000000000 纳秒:*/
c.SetRequestTimeout(10 * 1e9)
c.WithTransport(&http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
})
// 对于每个具有 href 属性的 a 元素调用回调函数
added := false
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
link := e.Request.AbsoluteURL(e.Attr("href"))
// 只访问 AllowedDomains 中的链接
// 创建一个新的上下文以记住引用页(以防出错)
for _, domain_ok := range domains {
if strings.Contains(link, domain_ok) {
ctx := colly.NewContext()
ctx.Put("Referrer", url)
c.Request(http.MethodGet, link, nil, ctx, nil)
// 访问在页面上找到的链接
c.Visit(link)
added = true
break
}
}
if !added {
fmt.Fprintf(os.Stdout, "Ignoring %s (%s)\n", link, url)
}
})
// 在发出请求之前打印“正在访问...”
c.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
c.OnError(func(resp *colly.Response, err error) {
url := resp.Request.URL.String()
fmt.Fprintf(
os.Stdout, "ERR on URL: %s (from: %s), error: %s\n", url,
resp.Request.Ctx.Get("Referrer"), err,
)
//if errors.Is(err, colly.Client.Timeout) {
// fmt.Fprintf(os.Stdout, "Retry: '%s'\n", url)
// r.Retry()
//}
})
urlBase := fmt.Sprintf("%s://%s", httpMethod, domain)
fmt.Println("Scraping:", urlBase)
c.Visit(urlBase)
c.Wait()
}
}
英文:
Everything is working except that I would like to retry only on Timeout errors, but I dont know how to compare with the specific Client.Timeout
error that is generated.
The missing part is in comment here: if errors.Is(err, colly.Client.Timeout)...
:
package main
import (
"crypto/tls"
"fmt"
"github.com/gocolly/colly"
"net/http"
"os"
"strings"
)
func main() {
crawl()
}
func crawl() {
httpMethod := "https"
domains := []string{
"www.myweb1.com",
"www.myweb2.com",
}
for _, domain := range domains {
// Instantiate default collector
c := colly.NewCollector(
colly.Async(true),
colly.AllowedDomains(domain),
)
c.Limit(&colly.LimitRule{Parallelism: 100})
/* default = 10s = 1000000000 nanoseconds: */
c.SetRequestTimeout(10 * 1e9)
c.WithTransport(&http.Transport{
TLSClientConfig:&tls.Config{InsecureSkipVerify: true},
})
// On every a element which has href attribute call callback
added := false
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
link := e.Request.AbsoluteURL(e.Attr("href"))
// Only those links are visited which are in AllowedDomains
// create a new context to remember the referer (in case of error)
for _, domain_ok := range domains {
if (strings.Contains(link, domain_ok)) {
ctx := colly.NewContext()
ctx.Put("Referrer", url)
c.Request(http.MethodGet, link, nil, ctx, nil)
// Visit link found on page
c.Visit(link)
added = true
break
}
}
if !added {
fmt.Fprintf( os.Stdout, "Ignoring %s (%s)\n", link, url)
}
})
// Before making a request print "Visiting ..."
c.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
c.OnError(func(resp *colly.Response, err error) {
url := resp.Request.URL.String()
fmt.Fprintf(
os.Stdout, "ERR on URL: %s (from: %s), error: %s\n", url,
resp.Request.Ctx.Get("Referrer"), err,
)
//if errors.Is(err, colly.Client.Timeout) {
// fmt.Fprintf(os.Stdout, "Retry: '%s'\n", url)
// r.Retry()
//}
})
urlBase := fmt.Sprintf("%s://%s", httpMethod, domain)
fmt.Println("Scraping: ", urlBase)
c.Visit(urlBase)
c.Wait()
}
}
答案1
得分: 2
我猜你可以使用这段代码片段。最有可能的超时错误是因为上下文中的截止时间已经超过了。可以试一试。
import (
"context"
"os"
"github.com/cockroachdb/errors"
)
func IsTimeoutError(err error) bool {
if errors.Is(err, context.DeadlineExceeded) {
return true
}
if errors.Is(errors.Cause(err), context.DeadlineExceeded) {
return true
}
if os.IsTimeout(err) {
return true
}
if os.IsTimeout(errors.Cause(err)) {
return true
}
return false
}
英文:
I guess you could use this snippet. Most likely the timeout error is threw because the deadline in context is exceeded. Could give a try.
import (
"context"
"os"
"github.com/cockroachdb/errors"
)
func IsTimeoutError(err error) bool {
if errors.Is(err, context.DeadlineExceeded) {
return true
}
if errors.Is(errors.Cause(err), context.DeadlineExceeded) {
return true
}
if os.IsTimeout(err) {
return true
}
if os.IsTimeout(errors.Cause(err)) {
return true
}
return false
}
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论