多线程用于HTTP的GET请求。

huangapple go评论111阅读模式
英文:

Multiple threads Go for HTTP get

问题

我正在使用Go语言编写一个用于查询我所在城市公交车频率的API,但是在尝试同时向多个URL发起HTTP Get请求时遇到了一些困难。
如果没有并发处理,程序需要超过16分钟才能完成对1500个URL的调用,以获取HTTP状态码。我尝试使用并发处理,但是在阅读了许多帖子后,我还是不理解goroutines(Go语言中的协程)是如何工作的...

我的想法是创建一个函数,并更改请求的数量,就像这样:

 go getBusPostStatus(600, 800)

但是我完全被卡住了...

以下是代码:

package main

import (
    "fmt"
    "net/http"
    "strconv"
    "time"
)

var i int = 0
var convStr string
var message = make(chan string)

// 返回200个帖子
func returnTH(c chan string) {
    for i = 0; i < 200; i++ {
        convStr = strconv.Itoa(i)
        url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println("Houston, we've got problems")
        } else {
            if resp.StatusCode == 200 {
                c <- "OK: 公交车站 " + convStr + " 存在"
            } else {
                c <- "WARN: 公交车站 " + convStr + " 不存在"
            }
        }
    }
}

func returnFH(z chan string) {
    for i = 201; i < 400; i++ {
        convStr = strconv.Itoa(i)
        url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println("Houston, we've got problems")
        } else {
            if resp.StatusCode == 200 {
                z <- "OK: 公交车站 " + convStr + " 存在"
            } else {
                z <- "WARN: 公交车站 " + convStr + " 不存在"
            }
        }
    }
}

func threadPrint(c, z chan string) {
    for {
        threadOne := <-c
        threadTwo := <-z
        fmt.Println(threadOne)
        fmt.Println(threadTwo)
    }
}

func main() {
    start := time.Now()
    var c chan string = make(chan string)
    var z chan string = make(chan string)
    go returnTH(c)
    go returnFH(z)
    go threadPrint(c, z)
    timeExec := time.Since(start)
    fmt.Println("代码执行时间 = ", timeExec)
}

非常感谢!

英文:

I'm working with Go in an API for the bus frequency of my city, but i'm a little bit stuck on the threads when i try to make HTTP Get to many urls.
Without concurrency, the programs takes over 16 minutes to complete the 1500 url calls to take the HTTP status code, and i was trying to use the concurrency, but after reading many posts i don't understand how goroutines work...

The idea is to make ONE function and change the number of requests, like here:

 go getBusPostStatus(600, 800)

But i'm completely stucked on that...

Here is the code:

package main
import (
&quot;fmt&quot;
&quot;net/http&quot;
&quot;strconv&quot;
&quot;time&quot;
)
var i int = 0
var convStr string
var message = make(chan string)
/*func main(){
for i = 0; i &lt; 1500; i++ {
z = strconv.Itoa(i)
url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + z
resp, err := http.Get(url)
if err != nil {
fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
}else{
if resp.StatusCode == 200{
fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
}else{
fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
}
}
}
}*/
//Return 2 houndred posts
func returnTH(c chan string){
for i = 0; i &lt; 200; i++ {	
convStr = strconv.Itoa(i)
url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + convStr
resp, err := http.Get(url)
if err != nil {
fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
}else{
if resp.StatusCode == 200{
//fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
c &lt;- &quot;OK: The bus post &quot;+convStr+&quot; exists&quot;
}else{
//fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
c &lt;- &quot;WARN: The bus post &quot;+convStr+&quot; does not exist&quot;
}   
}
}
}
func returnFH(z chan string){
for i = 201; i &lt; 400; i++ {	
convStr = strconv.Itoa(i)
url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + convStr
resp, err := http.Get(url)
if err != nil {
fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
}else{
if resp.StatusCode == 200{
//fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
z &lt;- &quot;OK: The bus post &quot;+convStr+&quot; exists&quot;
}else{
//fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
z &lt;- &quot;WARN: The bus post &quot;+convStr+&quot; does not exist&quot;
}   
} 
}
}
func threadPrint(c, z chan string){
for {
threadOne := &lt;- c
threadTwo := &lt;- z
fmt.Println(threadOne)
fmt.Println(threadTwo)
}
}
func main(){
start := time.Now()
var c chan string = make(chan string)
var z chan string = make(chan string)
//for i = 0; i &lt; 1500; i++{
go returnTH(c)
go returnFH(z)
go threadPrint(c,z)
/*go getBusPostStatus(400, 600)
go getBusPostStatus(600, 800)
go getBusPostStatus(800, 1000)
go getBusPostStatus(1000, 1200)
go getBusPostStatus(1200, 1400)
go getBusPostStatus(1400, 1500)*/
//}
timeExec:= time.Since(start)
fmt.Println(&quot;Time to exec code = &quot;, timeExec)
/*var input string
fmt.Scanln(&amp;input)
fmt.Println(&quot;done&quot;)*/
}

Many thanks in advance!!

答案1

得分: 4

以下是一个简化的示例代码,它使用goroutine和channel并发地请求100次,并打印结果。希望这段代码对你有帮助。

package main

import (
	"fmt"
	"math/rand"
	"time"
)

func main() {
	rep := 100
	results := make(chan string)

	// 使用goroutine将多个耗时的任务发送到通道中
	for i := 0; i < rep; i++ {
		go func(num int) {
			results <- mockHTTPRequest(num)
		}(i)
	}

	// 从通道接收结果并使用它们
	for i := 0; i < rep; i++ {
		fmt.Println(<-results)
	}
}

func mockHTTPRequest(num int) string {
	timeDelay := rand.Intn(5000)
	time.Sleep(time.Duration(timeDelay) * time.Millisecond)
	if timeDelay%2 == 0 {
		return fmt.Sprintf("OK: The bus post %v exists", num)
	}
	return fmt.Sprintf("WARN: The bus post %v does not exist", num)
}

你可以在 https://play.golang.org/p/RR34roRIl4 上运行这段代码。

英文:

Following is a simplified example code which requests 100 times concurrently and prints results, using goroutine and channel. Hope this code helps.

package main
import (
&quot;fmt&quot;
&quot;math/rand&quot;
&quot;time&quot;
)
func main() {
rep := 100
results := make(chan string)
// Use goroutine to send multiple time-consuming jobs to the channel.
for i := 0; i &lt; rep; i++ {
go func(num int) {
results &lt;- mockHTTPRequest(num)
}(i)
}
// Receive results from the channel and use them.
for i := 0; i &lt; rep; i++ {
fmt.Println(&lt;-results)
}
}
func mockHTTPRequest(num int) string {
timeDelay := rand.Intn(5000)
time.Sleep(time.Duration(timeDelay) * time.Millisecond)
if timeDelay%2 == 0 {
return fmt.Sprintf(&quot;OK: The bus post %v exists&quot;, num)
}
return fmt.Sprintf(&quot;WARN: The bus post %v does not exist&quot;, num)
}

You can run this code on https://play.golang.org/p/RR34roRIl4 .

huangapple
  • 本文由 发表于 2016年11月18日 23:23:42
  • 转载请务必保留本文链接:https://go.coder-hub.com/40680642.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定