将Golang中的标准输入流分段写入文件。

huangapple
go评论110阅读模式
英文:

Golang segment stdin stream to file

问题

代替将数据写入一个大文件，我想在接收到USR1信号时将流分段处理。我认为基本功能已经实现，但应用程序却一直挂起，什么都没有发生。在处理无法控制的输入流和字节完美分段时，有什么提示或最佳实践吗？

  1. package main
  3. import (
  4. 	"bufio"
  5. 	"fmt"
  6. 	"io"
  7. 	"os"
  8. 	"os/signal"
  9. 	"syscall"
  10. 	"time"
  11. )
  13. var done bool
  15. func handle(c chan os.Signal) {
  16. 	for {
  17. 		sig := <-c
  18. 		switch sig {
  19. 		case syscall.SIGUSR1:
  20. 			fmt.Println("###Sink temporarily_closed###")
  21. 			done = true
  22. 		case syscall.SIGUSR2:
  23. 			fmt.Println("###Sink closed###")
  24. 			done = true
  25. 		case syscall.SIGHUP:
  26. 			fmt.Println("###Sink running###")
  27. 		}
  28. 	}
  29. }
  31. func check(e error) {
  32. 	if e != nil {
  33. 		panic(e)
  34. 	}
  35. }
  37. func main() {
  38. 	c := make(chan os.Signal, 1)
  39. 	signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
  40. 	go handle(c)
  42. 	reader := bufio.NewReaderSize(os.Stdin, 1024*10)
  44. 	for true {
  45. 		if done {
  46. 			file, err := os.Create("./temp.file")
  47. 			check(err)
  48. 			writer := bufio.NewWriter(file)
  49. 			written, err := io.Copy(writer, reader)
  50. 			check(err)
  51. 			fmt.Println(written)
  52. 			writer.Flush()
  53. 			file.Close()
  54. 			reader.Reset(os.Stdin)
  55. 			done = false
  56. 		}
  57. 		time.Sleep(time.Millisecond)
  58. 	}
  59. }

希望这可以帮助到你！

英文:

instead of writing a pipe to a huge file i want to segment the stream in chunks on signal USR1. i think i got the basics working but the app just hangs and nothing happens, any clues or best practices when handling with an uncontrollable input stream and byte perfect segmentation?

  1. package main
  2. import (
  3. &quot;bufio&quot;
  4. &quot;fmt&quot;
  5. &quot;io&quot;
  6. &quot;os&quot;
  7. &quot;os/signal&quot;
  8. &quot;syscall&quot;
  9. &quot;time&quot;
  10. )
  11. var done bool
  12. func handle(c chan os.Signal) {
  13. for {
  14. sig := &lt;-c
  15. switch sig {
  16. case syscall.SIGUSR1:
  17. fmt.Println(&quot;###Sink temporarily_closed###&quot;)
  18. done = true
  19. case syscall.SIGUSR2:
  20. fmt.Println(&quot;###Sink closed###&quot;)
  21. done = true
  22. case syscall.SIGHUP:
  23. fmt.Println(&quot;###Sink running###&quot;)
  24. }
  25. }
  26. }
  27. func check(e error) {
  28. if e != nil {
  29. panic(e)
  30. }
  31. }
  32. func main() {
  33. c := make(chan os.Signal, 1)
  34. signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
  35. go handle(c)
  36. reader := bufio.NewReaderSize(os.Stdin,1024*10)
  37. for true {
  38. if done {
  39. file, err := os.Create(&quot;./temp.file&quot;)
  40. check(err)
  41. writer := bufio.NewWriter(file)
  42. written, err := io.Copy(writer,reader)
  43. check(err)
  44. fmt.Println(written)
  45. writer.Flush()
  46. file.Close()
  47. reader.Reset(os.Stdin)
  48. done = false
  49. }
  50. time.Sleep(time.Millisecond)
  51. }
  52. }

答案1

得分: 1

所以你需要在循环中使用io.CopyN(dst, src, 4096)并定期旋转文件。请参考以下示例。我通过文件大小进行了旋转，但添加信号处理很容易。

  1. package main
  3. import (
  4. 	"fmt"
  5. 	"io"
  6. 	"log"
  7. 	"os"
  8. 	"time"
  9. )
  11. var count int
  12. var f *os.File
  14. func rotate() *os.File {
  15. 	if f != nil {
  16. 		if err := f.Close(); err != nil {
  17. 			log.Fatal(err)
  18. 		}
  19. 	}
  21. 	fname := fmt.Sprintf("./dump-%d.bin", count)
  22. 	count++
  23. 	f, err := os.Create(fname)
  24. 	if err != nil {
  25. 		log.Fatal(err)
  26. 	}
  27. 	log.Println("rotated:", fname)
  29. 	return f
  30. }
  32. func main() {
  33. 	var n, written int
  34. 	reader := os.Stdin
  36. 	for {
  37. 		if written == 0 || written >= 4096*10 {
  38. 			f = rotate()
  39. 			written = 0
  40. 		}
  42. 		n, err := io.CopyN(f, reader, 4096)
  43. 		if err != nil {
  44. 			log.Fatal(err)
  45. 		}
  46. 		written += n
  47. 		log.Println("written:", written)
  48. 		time.Sleep(time.Millisecond * 500)
  49. 	}
  50. }
英文:

So you need to io.CopyN(dst, src, 4096) in the loop and rotate the file once in a while. See example. I made rotation by size but it is easy to add signal handling.

  1. package main
  2. import (
  3. &quot;fmt&quot;
  4. &quot;io&quot;
  5. &quot;log&quot;
  6. &quot;os&quot;
  7. &quot;time&quot;
  8. )
  9. var count int
  10. var f *os.File
  11. func rotate() *os.File {
  12. if f != nil {
  13. if err := f.Close(); err != nil {
  14. log.Fatal(err)
  15. }
  16. }
  17. fname := fmt.Sprintf(&quot;./dump-%d.bin&quot;, count)
  18. count++
  19. f, err := os.Create(fname)
  20. if err != nil {
  21. log.Fatal(err)
  22. }
  23. log.Println(&quot;rotated:&quot;, fname)
  24. return f
  25. }
  26. func main() {
  27. var n, written int
  28. reader := os.Stdin
  29. for {
  30. if written == 0 || written &gt;= 4096*10 {
  31. f = rotate()
  32. written = 0
  33. }
  34. n, err := io.CopyN(f, reader, 4096)
  35. if err != nil {
  36. log.Fatal(err)
  37. }
  38. written += n
  39. log.Println(&quot;written:&quot;, written)
  40. time.Sleep(time.Millisecond * 500)
  41. }
  42. }

huangapple
  • 本文由 发表于 2022年1月20日 06:32:20
  • 转载请务必保留本文链接：https://go.coder-hub.com/70778595.html
  • go
  • stdin
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定