如果通道已满或经过特定时间后,从Go缓冲通道中读取数据。

huangapple go评论85阅读模式
英文:

Read from Go buffered channel if channel is full or after specific time

问题

我想要在通道满了或经过一定时间后进行批处理。我的用例类似于现有的问题,我已经尝试修改现有的答案

我的代码在https://go.dev/play/p/HaGZ9HHqj0i,

package main

import (
	"fmt"
	"sync"
	"time"
)

type Audit struct {
	ID int
}

const batchSize = 5

var batch = make([]Audit, 0, batchSize)

func upsertBigQueryAudits(audits []Audit) {
	fmt.Printf("处理批次:%d\n", len(audits))
	for _, a := range audits {
		fmt.Printf("%d ", a.ID)
	}
	fmt.Println()
	batch = []Audit{}
}

func process(full <-chan struct{}) {
	ticker := time.NewTicker(1 * time.Nanosecond)

	for {
		select {
		case <-full:
			fmt.Println("来自满通道")
			upsertBigQueryAudits(batch)
		case <-ticker.C:
			fmt.Println("来自定时器")
			if len(batch) > 0 {
				fmt.Println("来自定时器发送批次")
				upsertBigQueryAudits(batch)
			}
		}
	}
}

func processAudits(audits <-chan Audit, full chan<- struct{}, batchSize int) {
	for audit := range audits {
		batch = append(batch, audit)
		if len(batch) == cap(batch) {
			// upsertBigQueryAudits(batch)
			fmt.Println("发送满通道")
			full <- struct{}{}
		}
	}
}

func produceAudits(x int, to chan Audit) {
	for i := 0; i < x; i++ {
		to <- Audit{
			ID: i,
		}
	}
}

func main() {
	var wg sync.WaitGroup
	audits := make(chan Audit)
	full := make(chan struct{})
	wg.Add(1)
	go func() {
		defer wg.Done()
		process(full)
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		processAudits(audits, full, batchSize)
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		produceAudits(25, audits)
		close(audits)
	}()
	wg.Wait()
	fmt.Println("完成")
}

在这里,我希望在批次填满batchSize个元素或经过一定时间(例如1纳秒)后进行处理。

编辑:内联添加了我的问题。
这是示例输出。

运行程序超时
发送满通道 <--- 这一行出现两次
发送满通道
来自满通道
处理批次:10 <--- 它处理了10个项目而不是5个
0 1 2 3 4 5 6 7 8 9
来自满通道
处理批次:1
10
发送满通道
发送满通道
来自满通道
处理批次:1 <--- 它只处理了1个项目,而不是5个。它不会处理超过1个项目。
11
来自满通道
处理批次:0
...
...
发送满通道
发送满通道
来自满通道
处理批次:1 <--- 它只处理了1个项目,而不是5个
24
来自满通道
处理批次:0
来自定时器 <--- 这一行只在消耗25个项目后出现。
来自定时器
来自定时器
来自定时器
英文:

I want to process from channel if channel is full or certain time is elapsed. My use case is similar to existing question and I have tried to modify the existing ans

My code is at https://go.dev/play/p/HaGZ9HHqj0i,

package main
import (
&quot;fmt&quot;
&quot;sync&quot;
&quot;time&quot;
)
type Audit struct {
ID int
}
const batchSize = 5
var batch = make([]Audit, 0, batchSize)
func upsertBigQueryAudits(audits []Audit) {
fmt.Printf(&quot;Processing batch of %d\n&quot;, len(audits))
for _, a := range audits {
fmt.Printf(&quot;%d &quot;, a.ID)
}
fmt.Println()
batch = []Audit{}
}
func process(full &lt;-chan struct{}) {
ticker := time.NewTicker(1 * time.Nanosecond)
for {
select {
case &lt;-full:
fmt.Println(&quot;From full&quot;)
upsertBigQueryAudits(batch)
case &lt;-ticker.C:
fmt.Println(&quot;From ticker&quot;)
if len(batch) &gt; 0 {
fmt.Println(&quot;From ticker sending batch&quot;)
upsertBigQueryAudits(batch)
}
}
}
}
func processAudits(audits &lt;-chan Audit, full chan&lt;- struct{}, batchSize int) {
for audit := range audits {
batch = append(batch, audit)
if len(batch) == cap(batch) {
// upsertBigQueryAudits(batch)
fmt.Println(&quot;Sending full&quot;)
full &lt;- struct{}{}
}
}
}
func produceAudits(x int, to chan Audit) {
for i := 0; i &lt; x; i++ {
to &lt;- Audit{
ID: i,
}
}
}
func main() {
var wg sync.WaitGroup
audits := make(chan Audit)
full := make(chan struct{})
wg.Add(1)
go func() {
defer wg.Done()
process(full)
}()
wg.Add(1)
go func() {
defer wg.Done()
processAudits(audits, full, batchSize)
}()
wg.Add(1)
go func() {
defer wg.Done()
produceAudits(25, audits)
close(audits)
}()
wg.Wait()
fmt.Println(&quot;Complete&quot;)
}

Here I want to process the batch when it is filled with batchSize elements Or when certain time (for eg 1 nano second) is elapsed.

Edit: Added my concern inline.
This is sample output.

timeout running program
Sending full &lt;--- this line comes twice
Sending full
From full
Processing batch of 10 &lt;--- instead of 5 it is processing 10 items
0 1 2 3 4 5 6 7 8 9 
From full
Processing batch of 1
10 
Sending full
Sending full
From full
Processing batch of 1 &lt;-- instead of 5 it is processing 1 items. It does not process more than 1 item.
11 
From full
Processing batch of 0
...
...
Sending full
Sending full
From full
Processing batch of 1 &lt;-- instead of 5 it is processing 1 items
24 
From full
Processing batch of 0
From ticker &lt;-- this line comes only after consuming 25 items.  
From ticker
From ticker
From ticker

答案1

得分: 0

我在https://elliotchance.medium.com/batch-a-channel-by-size-or-time-in-go-92fa3098f65找到了答案。

package main

import (
	"context"
	"fmt"
	"time"
)

func BatchStringsCtx[T any](ctx context.Context, values <-chan T, maxItems int, maxTimeout time.Duration) chan []T {
	batches := make(chan []T)

	go func() {
		defer close(batches)

		for keepGoing := true; keepGoing; {
			var batch []T
			expire := time.After(maxTimeout)
			for {
				select {
				case <-ctx.Done():
					keepGoing = false
					goto done

				case value, ok := <-values:
					if !ok {
						keepGoing = false
						goto done
					}

					batch = append(batch, value)
					if len(batch) == maxItems {
						goto done
					}

				case <-expire:
					goto done
				}
			}

		done:
			if len(batch) > 0 {
				batches <- batch
			}
		}
	}()

	return batches
}

func main() {
	strings := make(chan string)
	go func() {
		strings <- "hello"
		strings <- "there" // hit limit of 2

		strings <- "how"
		time.Sleep(15 * time.Millisecond) // hit timeout

		strings <- "are"
		strings <- "you" // hit limit of 2

		// A really long time without any new values.
		// The context was cancelled around 300ms,
		// before this sleep finished.
		time.Sleep(500 * time.Millisecond)

		strings <- "doing?" // never read

		close(strings)
	}()

	//ctx, cancel := context.WithTimeout(context.Background(), 300 * time.Millisecond)
	ctx := context.Background()

	start := time.Now()
	batches := BatchStringsCtx[string](ctx, strings, 2, 10*time.Millisecond)
	for batch := range batches {
		fmt.Println(time.Now().Sub(start), batch)
	}
}
英文:

I found answer at https://elliotchance.medium.com/batch-a-channel-by-size-or-time-in-go-92fa3098f65

package main
import (
&quot;context&quot;
&quot;fmt&quot;
&quot;time&quot;
)
func BatchStringsCtx[T any](ctx context.Context, values &lt;-chan T, maxItems int, maxTimeout time.Duration) chan []T {
batches := make(chan []T)
go func() {
defer close(batches)
for keepGoing := true; keepGoing; {
var batch []T
expire := time.After(maxTimeout)
for {
select {
case &lt;-ctx.Done():
keepGoing = false
goto done
case value, ok := &lt;-values:
if !ok {
keepGoing = false
goto done
}
batch = append(batch, value)
if len(batch) == maxItems {
goto done
}
case &lt;-expire:
goto done
}
}
done:
if len(batch) &gt; 0 {
batches &lt;- batch
}
}
}()
return batches
}
func main() {
strings := make(chan string)
go func() {
strings &lt;- &quot;hello&quot;
strings &lt;- &quot;there&quot; // hit limit of 2
strings &lt;- &quot;how&quot;
time.Sleep(15 * time.Millisecond) // hit timeout
strings &lt;- &quot;are&quot;
strings &lt;- &quot;you&quot; // hit limit of 2
// A really long time without any new values.
// The context was cancelled around 300ms,
// before this sleep finished.
time.Sleep(500 * time.Millisecond)
strings &lt;- &quot;doing?&quot; // never read
close(strings)
}()
//ctx, cancel := context.WithTimeout(context.Background(), 300 * time.Millisecond)
ctx := context.Background()
start := time.Now()
batches := BatchStringsCtx[string](ctx, strings, 2, 10*time.Millisecond)
for batch := range batches {
fmt.Println(time.Now().Sub(start), batch)
}
}

huangapple
  • 本文由 发表于 2022年11月8日 20:03:41
  • 转载请务必保留本文链接:https://go.coder-hub.com/74360432.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定