多线程用于HTTP的GET请求。

huangapple go评论239阅读模式
英文:

Multiple threads Go for HTTP get

问题

我正在使用Go语言编写一个用于查询我所在城市公交车频率的API,但是在尝试同时向多个URL发起HTTP Get请求时遇到了一些困难。
如果没有并发处理,程序需要超过16分钟才能完成对1500个URL的调用,以获取HTTP状态码。我尝试使用并发处理,但是在阅读了许多帖子后,我还是不理解goroutines(Go语言中的协程)是如何工作的...

我的想法是创建一个函数,并更改请求的数量,就像这样:

 go getBusPostStatus(600, 800)

但是我完全被卡住了...

以下是代码:

package main

import (
    "fmt"
    "net/http"
    "strconv"
    "time"
)

var i int = 0
var convStr string
var message = make(chan string)

// 返回200个帖子
func returnTH(c chan string) {
    for i = 0; i < 200; i++ {
        convStr = strconv.Itoa(i)
        url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println("Houston, we've got problems")
        } else {
            if resp.StatusCode == 200 {
                c <- "OK: 公交车站 " + convStr + " 存在"
            } else {
                c <- "WARN: 公交车站 " + convStr + " 不存在"
            }
        }
    }
}

func returnFH(z chan string) {
    for i = 201; i < 400; i++ {
        convStr = strconv.Itoa(i)
        url := "http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=" + convStr
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println("Houston, we've got problems")
        } else {
            if resp.StatusCode == 200 {
                z <- "OK: 公交车站 " + convStr + " 存在"
            } else {
                z <- "WARN: 公交车站 " + convStr + " 不存在"
            }
        }
    }
}

func threadPrint(c, z chan string) {
    for {
        threadOne := <-c
        threadTwo := <-z
        fmt.Println(threadOne)
        fmt.Println(threadTwo)
    }
}

func main() {
    start := time.Now()
    var c chan string = make(chan string)
    var z chan string = make(chan string)
    go returnTH(c)
    go returnFH(z)
    go threadPrint(c, z)
    timeExec := time.Since(start)
    fmt.Println("代码执行时间 = ", timeExec)
}

非常感谢!

英文:

I'm working with Go in an API for the bus frequency of my city, but i'm a little bit stuck on the threads when i try to make HTTP Get to many urls.
Without concurrency, the programs takes over 16 minutes to complete the 1500 url calls to take the HTTP status code, and i was trying to use the concurrency, but after reading many posts i don't understand how goroutines work...

The idea is to make ONE function and change the number of requests, like here:

 go getBusPostStatus(600, 800)

But i'm completely stucked on that...

Here is the code:

package main

import (
&quot;fmt&quot;
&quot;net/http&quot;
&quot;strconv&quot;
&quot;time&quot;
)
var i int = 0
var convStr string
var message = make(chan string)

/*func main(){
    for i = 0; i &lt; 1500; i++ {
        z = strconv.Itoa(i)
        url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + z
        resp, err := http.Get(url)
        if err != nil {
            fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
        }else{
            if resp.StatusCode == 200{
                fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
            }else{
                fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
            }
        }
    }
}*/

//Return 2 houndred posts
func returnTH(c chan string){
 for i = 0; i &lt; 200; i++ {	
    convStr = strconv.Itoa(i)
    url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + convStr
    resp, err := http.Get(url)
    if err != nil {
        fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
    }else{
        if resp.StatusCode == 200{
            //fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
            c &lt;- &quot;OK: The bus post &quot;+convStr+&quot; exists&quot;
        }else{
            //fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
            c &lt;- &quot;WARN: The bus post &quot;+convStr+&quot; does not exist&quot;
        }   
    }
  }
}
func returnFH(z chan string){
  for i = 201; i &lt; 400; i++ {	
   convStr = strconv.Itoa(i)
   url := &quot;http://www.urbanosdezaragoza.es/frm_esquemaparadatime.php?poste=&quot; + convStr
    resp, err := http.Get(url)
    if err != nil {
        fmt.Println(&quot;Houston, we&#39;ve got problems&quot;)
    }else{
        if resp.StatusCode == 200{
            //fmt.Println(&quot;OK: El poste &quot;+z+&quot; existe&quot;)
            z &lt;- &quot;OK: The bus post &quot;+convStr+&quot; exists&quot;
        }else{
            //fmt.Println(&quot;WARN: El poste &quot;+z+&quot; NO existe&quot;)
            z &lt;- &quot;WARN: The bus post &quot;+convStr+&quot; does not exist&quot;
        }   
    } 
  }
}

func threadPrint(c, z chan string){
   for {
       threadOne := &lt;- c
       threadTwo := &lt;- z
       fmt.Println(threadOne)
       fmt.Println(threadTwo)
   }
}
func main(){
    start := time.Now()
    var c chan string = make(chan string)
    var z chan string = make(chan string)
    //for i = 0; i &lt; 1500; i++{
    go returnTH(c)
    go returnFH(z)
    go threadPrint(c,z)
    /*go getBusPostStatus(400, 600)
    go getBusPostStatus(600, 800)
    go getBusPostStatus(800, 1000)
    go getBusPostStatus(1000, 1200)
    go getBusPostStatus(1200, 1400)
    go getBusPostStatus(1400, 1500)*/
    //}
    timeExec:= time.Since(start)
    fmt.Println(&quot;Time to exec code = &quot;, timeExec)

    /*var input string
    fmt.Scanln(&amp;input)
    fmt.Println(&quot;done&quot;)*/
}

Many thanks in advance!!

答案1

得分: 4

以下是一个简化的示例代码,它使用goroutine和channel并发地请求100次,并打印结果。希望这段代码对你有帮助。

package main

import (
	"fmt"
	"math/rand"
	"time"
)

func main() {
	rep := 100
	results := make(chan string)

	// 使用goroutine将多个耗时的任务发送到通道中
	for i := 0; i < rep; i++ {
		go func(num int) {
			results <- mockHTTPRequest(num)
		}(i)
	}

	// 从通道接收结果并使用它们
	for i := 0; i < rep; i++ {
		fmt.Println(<-results)
	}
}

func mockHTTPRequest(num int) string {
	timeDelay := rand.Intn(5000)
	time.Sleep(time.Duration(timeDelay) * time.Millisecond)
	if timeDelay%2 == 0 {
		return fmt.Sprintf("OK: The bus post %v exists", num)
	}
	return fmt.Sprintf("WARN: The bus post %v does not exist", num)
}

你可以在 https://play.golang.org/p/RR34roRIl4 上运行这段代码。

英文:

Following is a simplified example code which requests 100 times concurrently and prints results, using goroutine and channel. Hope this code helps.

package main

import (
	&quot;fmt&quot;
	&quot;math/rand&quot;
	&quot;time&quot;
)

func main() {
	rep := 100
	results := make(chan string)

	// Use goroutine to send multiple time-consuming jobs to the channel.
	for i := 0; i &lt; rep; i++ {
		go func(num int) {
			results &lt;- mockHTTPRequest(num)
		}(i)
	}

	// Receive results from the channel and use them.
	for i := 0; i &lt; rep; i++ {
		fmt.Println(&lt;-results)
	}
}

func mockHTTPRequest(num int) string {
	timeDelay := rand.Intn(5000)
	time.Sleep(time.Duration(timeDelay) * time.Millisecond)
	if timeDelay%2 == 0 {
		return fmt.Sprintf(&quot;OK: The bus post %v exists&quot;, num)
	}
	return fmt.Sprintf(&quot;WARN: The bus post %v does not exist&quot;, num)
}

You can run this code on https://play.golang.org/p/RR34roRIl4 .

huangapple
  • 本文由 发表于 2016年11月18日 23:23:42
  • 转载请务必保留本文链接:https://go.coder-hub.com/40680642.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定