将Golang中的标准输入流分段写入文件。

huangapple go评论79阅读模式
英文:

Golang segment stdin stream to file

问题

代替将数据写入一个大文件,我想在接收到USR1信号时将流分段处理。我认为基本功能已经实现,但应用程序却一直挂起,什么都没有发生。在处理无法控制的输入流和字节完美分段时,有什么提示或最佳实践吗?

package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var done bool

func handle(c chan os.Signal) {
	for {
		sig := <-c
		switch sig {
		case syscall.SIGUSR1:
			fmt.Println("###Sink temporarily_closed###")
			done = true
		case syscall.SIGUSR2:
			fmt.Println("###Sink closed###")
			done = true
		case syscall.SIGHUP:
			fmt.Println("###Sink running###")
		}
	}
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
	go handle(c)

	reader := bufio.NewReaderSize(os.Stdin, 1024*10)

	for true {
		if done {
			file, err := os.Create("./temp.file")
			check(err)
			writer := bufio.NewWriter(file)
			written, err := io.Copy(writer, reader)
			check(err)
			fmt.Println(written)
			writer.Flush()
			file.Close()
			reader.Reset(os.Stdin)
			done = false
		}
		time.Sleep(time.Millisecond)
	}
}

希望这可以帮助到你!

英文:

instead of writing a pipe to a huge file i want to segment the stream in chunks on signal USR1. i think i got the basics working but the app just hangs and nothing happens, any clues or best practices when handling with an uncontrollable input stream and byte perfect segmentation?

package main
import (
&quot;bufio&quot;
&quot;fmt&quot;
&quot;io&quot;
&quot;os&quot;
&quot;os/signal&quot;
&quot;syscall&quot;
&quot;time&quot;
)
var done bool
func handle(c chan os.Signal) {
for {
sig := &lt;-c
switch sig {
case syscall.SIGUSR1:
fmt.Println(&quot;###Sink temporarily_closed###&quot;)
done = true
case syscall.SIGUSR2:
fmt.Println(&quot;###Sink closed###&quot;)
done = true
case syscall.SIGHUP:
fmt.Println(&quot;###Sink running###&quot;)
}
}
}
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
go handle(c)
reader := bufio.NewReaderSize(os.Stdin,1024*10)
for true {
if done {
file, err := os.Create(&quot;./temp.file&quot;)
check(err)
writer := bufio.NewWriter(file)
written, err := io.Copy(writer,reader)
check(err)
fmt.Println(written)
writer.Flush()
file.Close()
reader.Reset(os.Stdin)
done = false
}
time.Sleep(time.Millisecond)
}
}

答案1

得分: 1

所以你需要在循环中使用io.CopyN(dst, src, 4096)并定期旋转文件。请参考以下示例。我通过文件大小进行了旋转,但添加信号处理很容易。

package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

var count int
var f *os.File

func rotate() *os.File {
	if f != nil {
		if err := f.Close(); err != nil {
			log.Fatal(err)
		}
	}

	fname := fmt.Sprintf("./dump-%d.bin", count)
	count++
	f, err := os.Create(fname)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("rotated:", fname)

	return f
}

func main() {
	var n, written int
	reader := os.Stdin

	for {
		if written == 0 || written >= 4096*10 {
			f = rotate()
			written = 0
		}

		n, err := io.CopyN(f, reader, 4096)
		if err != nil {
			log.Fatal(err)
		}
		written += n
		log.Println("written:", written)
		time.Sleep(time.Millisecond * 500)
	}
}
英文:

So you need to io.CopyN(dst, src, 4096) in the loop and rotate the file once in a while. See example. I made rotation by size but it is easy to add signal handling.

package main
import (
&quot;fmt&quot;
&quot;io&quot;
&quot;log&quot;
&quot;os&quot;
&quot;time&quot;
)
var count int
var f *os.File
func rotate() *os.File {
if f != nil {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
fname := fmt.Sprintf(&quot;./dump-%d.bin&quot;, count)
count++
f, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
log.Println(&quot;rotated:&quot;, fname)
return f
}
func main() {
var n, written int
reader := os.Stdin
for {
if written == 0 || written &gt;= 4096*10 {
f = rotate()
written = 0
}
n, err := io.CopyN(f, reader, 4096)
if err != nil {
log.Fatal(err)
}
written += n
log.Println(&quot;written:&quot;, written)
time.Sleep(time.Millisecond * 500)
}
}

huangapple
  • 本文由 发表于 2022年1月20日 06:32:20
  • 转载请务必保留本文链接:https://go.coder-hub.com/70778595.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定