Colly Go包:如何检查错误是否为超时错误?

huangapple go评论78阅读模式
英文:

Colly Go package: how to check if the error is a Timeout error?

问题

除了我想在超时错误时重试之外,一切都正常,但我不知道如何与生成的特定Client.Timeout错误进行比较。

缺失的部分在这里的注释中:if errors.Is(err, colly.Client.Timeout)...

package main

import (
    "crypto/tls"
    "fmt"
    "github.com/gocolly/colly"
    "net/http"
    "os"
    "strings"
)

func main() {
    crawl()
}

func crawl() {
    httpMethod := "https"
    domains := []string{
        "www.myweb1.com",
        "www.myweb2.com",
    }
    for _, domain := range domains {
        // 实例化默认的收集器
        c := colly.NewCollector(
            colly.Async(true),
            colly.AllowedDomains(domain),
        )
        c.Limit(&colly.LimitRule{Parallelism: 100})
        /* 默认 = 10s = 1000000000 纳秒:*/
        c.SetRequestTimeout(10 * 1e9)
        c.WithTransport(&http.Transport{
            TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
        })

        // 对于每个具有 href 属性的 a 元素调用回调函数
        added := false
        c.OnHTML("a[href]", func(e *colly.HTMLElement) {
            url := e.Request.URL.String()
            link := e.Request.AbsoluteURL(e.Attr("href"))
            // 只访问 AllowedDomains 中的链接
            // 创建一个新的上下文以记住引用页(以防出错)
            for _, domain_ok := range domains {
                if strings.Contains(link, domain_ok) {
                    ctx := colly.NewContext()
                    ctx.Put("Referrer", url)
                    c.Request(http.MethodGet, link, nil, ctx, nil)
                    // 访问在页面上找到的链接
                    c.Visit(link)
                    added = true
                    break
                }
            }
            if !added {
                fmt.Fprintf(os.Stdout, "Ignoring %s (%s)\n", link, url)
            }
        })

        // 在发出请求之前打印“正在访问...”
        c.OnRequest(func(r *colly.Request) {
            fmt.Println("Visiting", r.URL.String())
        })
        c.OnError(func(resp *colly.Response, err error) {
            url := resp.Request.URL.String()
            fmt.Fprintf(
                os.Stdout, "ERR on URL: %s (from: %s), error: %s\n", url,
                resp.Request.Ctx.Get("Referrer"), err,
            )
            //if errors.Is(err, colly.Client.Timeout) {
            //    fmt.Fprintf(os.Stdout, "Retry: '%s'\n", url)
            //    r.Retry()
            //}
        })
        urlBase := fmt.Sprintf("%s://%s", httpMethod, domain)
        fmt.Println("Scraping:", urlBase)
        c.Visit(urlBase)
        c.Wait()
    }
}
英文:

Everything is working except that I would like to retry only on Timeout errors, but I dont know how to compare with the specific Client.Timeout error that is generated.

The missing part is in comment here: if errors.Is(err, colly.Client.Timeout)...:

package main
import (
"crypto/tls"
"fmt"
"github.com/gocolly/colly"
"net/http"
"os"
"strings"
)
func main() {
crawl()
}
func crawl() {
httpMethod := "https"
domains := []string{
"www.myweb1.com",
"www.myweb2.com",
}
for _, domain := range domains {
// Instantiate default collector
c := colly.NewCollector(
colly.Async(true),
colly.AllowedDomains(domain),
)
c.Limit(&colly.LimitRule{Parallelism: 100})
/* default = 10s = 1000000000 nanoseconds: */
c.SetRequestTimeout(10 * 1e9)
c.WithTransport(&http.Transport{
TLSClientConfig:&tls.Config{InsecureSkipVerify: true},
})
// On every a element which has href attribute call callback
added := false
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
url := e.Request.URL.String()
link := e.Request.AbsoluteURL(e.Attr("href"))
// Only those links are visited which are in AllowedDomains
// create a new context to remember the referer (in case of error)
for _, domain_ok := range domains {
if (strings.Contains(link, domain_ok)) {
ctx := colly.NewContext()
ctx.Put("Referrer", url)
c.Request(http.MethodGet, link, nil, ctx, nil)
// Visit link found on page
c.Visit(link)
added = true
break
}
}
if !added {
fmt.Fprintf( os.Stdout, "Ignoring %s (%s)\n", link, url)
}
})
// Before making a request print "Visiting ..."
c.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting", r.URL.String())
})
c.OnError(func(resp *colly.Response, err error) {
url := resp.Request.URL.String()
fmt.Fprintf(
os.Stdout, "ERR on URL: %s (from: %s), error: %s\n", url,
resp.Request.Ctx.Get("Referrer"), err,
)
//if errors.Is(err, colly.Client.Timeout) {
//    fmt.Fprintf(os.Stdout, "Retry: '%s'\n", url)
//    r.Retry()
//}
})
urlBase := fmt.Sprintf("%s://%s", httpMethod, domain)
fmt.Println("Scraping: ", urlBase)
c.Visit(urlBase)
c.Wait()
}
}

答案1

得分: 2

我猜你可以使用这段代码片段。最有可能的超时错误是因为上下文中的截止时间已经超过了。可以试一试。

import (
    "context"
    "os"

    "github.com/cockroachdb/errors"
)

func IsTimeoutError(err error) bool {
    if errors.Is(err, context.DeadlineExceeded) {
        return true
    }
    if errors.Is(errors.Cause(err), context.DeadlineExceeded) {
        return true
    }
    if os.IsTimeout(err) {
        return true
    }
    if os.IsTimeout(errors.Cause(err)) {
        return true
    }
    return false
}
英文:

I guess you could use this snippet. Most likely the timeout error is threw because the deadline in context is exceeded. Could give a try.

import (
    "context"
    "os"

    "github.com/cockroachdb/errors"
)

func IsTimeoutError(err error) bool {
    if errors.Is(err, context.DeadlineExceeded) {
        return true
    }
    if errors.Is(errors.Cause(err), context.DeadlineExceeded) {
        return true
    }
    if os.IsTimeout(err) {
        return true
    }
    if os.IsTimeout(errors.Cause(err)) {
        return true
    }
    return false
}

huangapple
  • 本文由 发表于 2023年2月20日 19:26:03
  • 转载请务必保留本文链接:https://go.coder-hub.com/75508664.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定