如何在Golang中优化文件轮转？

huangapple
go评论112阅读模式
英文:

how should I optimize file rotation in golang?

问题

我已经设计并实现了在Go语言中对文件进行轮转的功能。根据设计，我会根据文件大小是否大于等于FileSizeThreshold（50000字节）或文件持续时间是否大于等于FileDurationThreshold（1分钟）（以先到者为准）来进行文件轮转。

以下是Go语言中的实现代码：

  1. package main
  3. import (
  4. 	"os"
  5. 	"path/filepath"
  6. 	"time"
  7. 	"log"
  8. 	"strings"
  9. 	"flag"
  10. 	"os/exec"
  11. )
  12. type FileStruct struct{
  13. 	Filename string
  14. 	CreatedAt time.Time
  15. }
  16. type FileRotate struct {
  17. 	Dir string
  18. 	File chan FileStruct
  19. }
  21. const(
  22. 	MAX_FILE_SIZE  = 50000
  23. 	MAX_FILE_DURATION = time.Minute * 1
  24. 	filename_time_format = "20060102150405000"
  25. 	MAX_TRY = 5
  26. )
  27. var blockingChan chan int
  28. func main(){
  29. 	path := flag.String("dir", "", "absolute path of dir ")
  30. 	flag.Parse()
  31. 	if strings.Contains(*path, "./") {
  32. 		log.Fatalln("ERROR: please give absolute path")
  33. 	}
  34. 	if info, err := os.Stat(*path); err == nil{
  35. 		if ! info.IsDir(){
  36. 			log.Fatalln(*path," is not a directory")
  37. 		}
  38. 		log.Println("directory found..")
  39. 	} else {
  40. 		if os.IsNotExist(err){
  41. 			log.Println("directory not found..")
  42. 			log.Println("creating the directory..",*path)
  43. 			if err := exec.Command("mkdir","-p",*path).Run(); err != nil{
  44. 				log.Fatalln("failed to create the directory ERROR:",err)
  45. 			}
  46. 			log.Println("directory created successfully")
  47. 		}
  48. 	}
  49. 	filerotate := &FileRotate{*path,make(chan FileStruct,1)}
  50. 	go filerotate.FileOperationsRoutine()
  52. 	log.Println("generating file name struct..")
  53. 	filerotate.File <- GetFileStruct()
  54. 	<- blockingChan
  55. }
  57. func (rotate *FileRotate) FileOperationsRoutine(){
  58. 	try := 0
  59. 	var f *os.File
  60. 	for{
  61. 		if file, ok := <- rotate.File; ok{
  62. 			if f == nil {
  63. 				log.Println("WARN: file ptr is nil")
  64. 			}
  65. 			filePath := filepath.Join(rotate.Dir, file.Filename)
  66. 			fileInfo, err := os.Stat(filePath)
  67. 			if err != nil && os.IsNotExist(err) {
  68. 				log.Println("file:", filePath, " does not exist...creating file")
  69. 				_, err = os.Create(filePath)
  70. 				if err != nil {
  71. 					log.Println("failed to create the file ERROR:",err)
  72. 					try++
  73. 					if try == MAX_TRY {
  74. 						log.Println("tried creating the file ",MAX_TRY," times. No luck")
  75. 						time.Sleep(time.Second * 3)
  76. 						continue
  77. 					}
  78. 					rotate.File <- file
  79. 					continue
  80. 				}
  81. 				log.Println("file:", filePath, " created successfully")
  82. 				fileInfo,err = os.Stat(filePath)
  83. 			}
  84. 			sizeCheck := fileInfo.Size() >= MAX_FILE_SIZE
  85. 			durationCheck := time.Now().After(file.CreatedAt.Add(MAX_FILE_DURATION))
  86. 			if sizeCheck || durationCheck {
  87. 				log.Println("filesize of ",filePath," is ",fileInfo.Size(),"..filesizeCheck=",sizeCheck)
  88. 				log.Println("fileDurationCheck=",durationCheck)
  89. 				log.Println("rotating the file..")
  90. 				f.Close()
  91. 				f = nil
  92. 				go ZipAndSendRoutine(filePath)
  93. 				rotate.File <- GetFileStruct()
  94. 			}else{
  95. 				if f == nil {
  96. 					f, err = os.OpenFile(filePath, os.O_RDWR | os.O_APPEND, 0644)
  97. 					if err != nil {
  98. 						log.Println("failed to open the file ERROR:", err)
  99. 						try++
  100. 						if try == MAX_TRY {
  101. 							log.Println("tried opening the file ", MAX_TRY, " times. No luck")
  102. 							time.Sleep(time.Second * 3)
  103. 							continue
  104. 						}
  105. 						rotate.File <- file
  106. 						continue
  107. 					}
  108. 					log.Println("file opened in append mode")
  109. 				}
  110. 				rotate.File <- file
  111. 			}
  112. 		}
  113. 	}
  114. }
  116. func GetFileStruct() FileStruct{
  117. 	current_time := time.Now()
  118. 	log.Println("returning the filestruct..")
  119. 	return FileStruct{"example_" + current_time.Format(filename_time_format),current_time}
  120. }
  122. func ZipAndSendRoutine(file string){
  123. 	log.Println("zipping and sending the file:",file,"to remote server")
  124. }

执行日志：

  1. root@workstation:/media/sf_golang# ./bin/file_rotation -dir "/tmp/file_rotaion"
  2. 2017/01/16 15:05:03 directory found..
  3. 2017/01/16 15:05:03 starting file operations routine...
  4. 2017/01/16 15:05:03 generating file name struct..
  5. 2017/01/16 15:05:03 returning the filestruct..
  6. 2017/01/16 15:05:03 WARN: file ptr is nil
  7. 2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  does not exist...creating file
  8. 2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  created successfully
  9. 2017/01/16 15:05:03 file opened in append mode
  10. 2017/01/16 15:06:03 filesize of  /tmp/file_rotaion/example_20170116150503000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
  11. 2017/01/16 15:06:03 rotating the file..
  12. 2017/01/16 15:06:03 returning the filestruct..
  13. 2017/01/16 15:06:03 WARN: file ptr is nil
  14. 2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  does not exist...creating file
  15. 2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  created successfully
  16. 2017/01/16 15:06:03 file opened in append mode
  17. 2017/01/16 15:06:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150503000 to remote server
  18. 2017/01/16 15:07:03 filesize of  /tmp/file_rotaion/example_20170116150603000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
  19. 2017/01/16 15:07:03 rotating the file..
  20. 2017/01/16 15:07:03 returning the filestruct..
  21. 2017/01/16 15:07:03 WARN: file ptr is nil
  22. 2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  does not exist...creating file
  23. 2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  created successfully
  24. 2017/01/16 15:07:03 file opened in append mode
  25. 2017/01/16 15:07:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150603000 to remote server

从日志中可以看出，该工具按预期工作。但在执行此工具时，CPU使用率几乎达到100%。

[![CPU utilization during process execution][1]][1]
[1]: https://i.stack.imgur.com/Uen4F.png

停止工具后的CPU使用率如下所示：

[![CPU utilization after process stopped][2]][2]
[2]: https://i.stack.imgur.com/X3glG.png

我已经确定了原因：
FileOperations goroutine无限运行，并且在该例程中我正在将文件指针发送到rotate.File通道上。

我陷入了困境，不确定如何进一步优化。有人能告诉我如何优化此工具的CPU利用率吗？

英文:

I had designed and implemented the file rotation of file in golang.
As per design I am rotating the file based on filesize &gt;= FileSizeThreshold(50000bytes) or file duration &gt;= FileDurationThreshold(1 minute) (whichever is first).

Following is the implementation in golang.

  1. package main
  2. import (
  3. &quot;os&quot;
  4. &quot;path/filepath&quot;
  5. &quot;time&quot;
  6. &quot;log&quot;
  7. &quot;strings&quot;
  8. &quot;flag&quot;
  9. &quot;os/exec&quot;
  10. )
  11. type FileStruct struct{
  12. Filename string
  13. CreatedAt time.Time
  14. }
  15. type FileRotate struct {
  16. Dir string
  17. File chan FileStruct
  18. }
  19. const(
  20. MAX_FILE_SIZE  = 50000
  21. MAX_FILE_DURATION = time.Minute * 1
  22. filename_time_format = &quot;20060102150405000&quot;
  23. MAX_TRY = 5
  24. )
  25. var blockingChan chan int
  26. func main(){
  27. path := flag.String(&quot;dir&quot;, &quot;&quot;, &quot;absolute path of dir &quot;)
  28. flag.Parse()
  29. if strings.Contains(*path, &quot;./&quot;) {
  30. log.Fatalln(&quot;ERROR: please give absolute path&quot;)
  31. }
  32. if info, err := os.Stat(*path); err == nil{
  33. if ! info.IsDir(){
  34. log.Fatalln(*path,&quot; is not a directory&quot;)
  35. }
  36. log.Println(&quot;directory found..&quot;)
  37. } else {
  38. if os.IsNotExist(err){
  39. log.Println(&quot;directory not found..&quot;)
  40. log.Println(&quot;creating the directory..&quot;,*path)
  41. if err := exec.Command(&quot;mkdir&quot;,&quot;-p&quot;,*path).Run(); err != nil{
  42. log.Fatalln(&quot;failed to create the directory ERROR:&quot;,err)
  43. }
  44. log.Println(&quot;directory created successfully&quot;)
  45. }
  46. }
  47. filerotate := &amp;FileRotate{*path,make(chan FileStruct,1)}
  48. go filerotate.FileOperationsRoutine()
  49. log.Println(&quot;generating file name struct..&quot;)
  50. filerotate.File &lt;- GetFileStruct()
  51. &lt;- blockingChan
  52. }
  53. func (rotate *FileRotate) FileOperationsRoutine(){
  54. try := 0
  55. var f *os.File
  56. for{
  57. if file, ok := &lt;- rotate.File; ok{
  58. if f == nil {
  59. log.Println(&quot;WARN: file ptr is nil&quot;)
  60. }
  61. filePath := filepath.Join(rotate.Dir, file.Filename)
  62. fileInfo, err := os.Stat(filePath)
  63. if err != nil &amp;&amp; os.IsNotExist(err) {
  64. log.Println(&quot;file:&quot;, filePath, &quot; does not exist...creating file&quot;)
  65. _, err = os.Create(filePath)
  66. if err != nil {
  67. log.Println(&quot;failed to create the file ERROR:&quot;,err)
  68. try++
  69. if try == MAX_TRY {
  70. log.Println(&quot;tried creating the file &quot;,MAX_TRY,&quot; times. No luck&quot;)
  71. time.Sleep(time.Second * 3)
  72. continue
  73. }
  74. rotate.File &lt;- file
  75. continue
  76. }
  77. log.Println(&quot;file:&quot;, filePath, &quot; created successfully&quot;)
  78. fileInfo,err = os.Stat(filePath)
  79. }
  80. sizeCheck := fileInfo.Size() &gt;= MAX_FILE_SIZE
  81. durationCheck := time.Now().After(file.CreatedAt.Add(MAX_FILE_DURATION))
  82. if sizeCheck || durationCheck {
  83. log.Println(&quot;filesize of &quot;,filePath,&quot; is &quot;,fileInfo.Size(),&quot;..filesizeCheck=&quot;,sizeCheck)
  84. log.Println(&quot;fileDurationCheck=&quot;,durationCheck)
  85. log.Println(&quot;rotating the file..&quot;)
  86. f.Close()
  87. f = nil
  88. go ZipAndSendRoutine(filePath)
  89. rotate.File &lt;- GetFileStruct()
  90. }else{
  91. if f == nil {
  92. f, err = os.OpenFile(filePath, os.O_RDWR | os.O_APPEND, 0644)
  93. if err != nil {
  94. log.Println(&quot;failed to open the file ERROR:&quot;, err)
  95. try++
  96. if try == MAX_TRY {
  97. log.Println(&quot;tried opening the file &quot;, MAX_TRY, &quot; times. No luck&quot;)
  98. time.Sleep(time.Second * 3)
  99. continue
  100. }
  101. rotate.File &lt;- file
  102. continue
  103. }
  104. log.Println(&quot;file opened in append mode&quot;)
  105. }
  106. rotate.File &lt;- file
  107. }
  108. }
  109. }
  110. }
  111. func GetFileStruct() FileStruct{
  112. current_time := time.Now()
  113. log.Println(&quot;returning the filestruct..&quot;)
  114. return FileStruct{&quot;example_&quot; + current_time.Format(filename_time_format),current_time}
  115. }
  116. func ZipAndSendRoutine(file string){
  117. log.Println(&quot;zipping and sending the file:&quot;,file,&quot;to remote server&quot;)
  118. }

Execution log :

  1. root@workstation:/media/sf_golang# ./bin/file_rotation -dir &quot;/tmp/file_rotaion&quot;
  2. 2017/01/16 15:05:03 directory found..
  3. 2017/01/16 15:05:03 starting file operations routine...
  4. 2017/01/16 15:05:03 generating file name struct..
  5. 2017/01/16 15:05:03 returning the filestruct..
  6. 2017/01/16 15:05:03 WARN: file ptr is nil
  7. 2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  does not exist...creating file
  8. 2017/01/16 15:05:03 file: /tmp/file_rotaion/example_20170116150503000  created successfully
  9. 2017/01/16 15:05:03 file opened in append mode
  10. 2017/01/16 15:06:03 filesize of  /tmp/file_rotaion/example_20170116150503000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
  11. 2017/01/16 15:06:03 rotating the file..
  12. 2017/01/16 15:06:03 returning the filestruct..
  13. 2017/01/16 15:06:03 WARN: file ptr is nil
  14. 2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  does not exist...creating file
  15. 2017/01/16 15:06:03 file: /tmp/file_rotaion/example_20170116150603000  created successfully
  16. 2017/01/16 15:06:03 file opened in append mode
  17. 2017/01/16 15:06:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150503000 to remote server
  18. 2017/01/16 15:07:03 filesize of  /tmp/file_rotaion/example_20170116150603000  is  0 ..filesizeCheck= false ...fileDurationCheck= true
  19. 2017/01/16 15:07:03 rotating the file..
  20. 2017/01/16 15:07:03 returning the filestruct..
  21. 2017/01/16 15:07:03 WARN: file ptr is nil
  22. 2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  does not exist...creating file
  23. 2017/01/16 15:07:03 file: /tmp/file_rotaion/example_20170116150703000  created successfully
  24. 2017/01/16 15:07:03 file opened in append mode
  25. 2017/01/16 15:07:03 zipping and sending the file: /tmp/file_rotaion/example_20170116150603000 to remote server

As seen from the logs, the utility is working as expected.
But during the execution of this utility, CPU usage was almost 100%

[![CPU utilization during process execution][1]][1]
[1]: https://i.stack.imgur.com/Uen4F.png

After stopping the utility..
[![CPU utilization after process stopped][2]][2]
[2]: https://i.stack.imgur.com/X3glG.png

I have identified the cause of this:
FileOperations goroutine is running indefinitely and within this routine I am sending the file pointer on rotate.File channel

I am stuck at this point and not sure how to optimize this further.
Could anyone tell me how should I optimize CPU utilization for this utility?

答案1

得分: 3

你的代码主要问题在于for循环中，你一直在将FileStruct传递给通道，无论是old还是new。因此，通道接收数据时没有等待时间，在if循环内部，你正在对文件进行stat操作以获取其数据，而这个操作你可能已经完成了。

以下是你程序的strace输出：

  1. % time     seconds  usecs/call     calls    errors syscall
  2. ------ ----------- ----------- --------- --------- ----------------
  3. 93.58   11.835475        3793      3120       227 futex
  4. 6.38    0.807279           4    192048         1 stat
  5. 0.03    0.003284           9       366           sched_yield
  6. 0.01    0.000759           7       114           rt_sigaction
  7. 0.00    0.000271          90         3           openat
  8. 0.00    0.000197          10        19           mmap
  9. 0.00    0.000143          20         7           write
  10. 0.00    0.000071          24         3           clone
  11. 0.00    0.000064           8         8           rt_sigprocmask
  12. 0.00    0.000034          17         2           select
  13. 0.00    0.000021          11         2           read
  14. 0.00    0.000016          16         1           sched_getaffinity
  15. 0.00    0.000014          14         1           munmap
  16. 0.00    0.000014          14         1           execve
  17. 0.00    0.000013          13         1           arch_prctl
  18. 0.00    0.000011          11         1           close
  19. 0.00    0.000000           0         2           sigaltstack
  20. 0.00    0.000000           0         1           gettid
  21. ------ ----------- ----------- --------- --------- ----------------
  22. 100.00   12.647666                195700       228 total

在大约40秒内，有195,000个系统调用。

你可以在for循环后添加一个等待时间：

  1. for {
  2.     <-time.After(time.Second)
  3.     if file, ok := <-rotate.File; ok {
  4.         // 进行其他操作
  5.     }
  6. }

并且你可以在FileStruct中添加fileinfo，在每次循环中首先检查结构体中的fileinfo，然后再执行stat操作。

添加了<-time.After(time.Second)后的strace输出如下：

  1. % time     seconds  usecs/call     calls    errors syscall
  2. ------ ----------- ----------- --------- --------- ----------------
  3. 65.65    0.001512          35        43         1 futex
  4. 23.71    0.000546           5       114           rt_sigaction
  5. 3.04    0.000070           9         8           mmap
  6. 2.43    0.000056          19         3           clone
  7. 2.26    0.000052           7         8           rt_sigprocmask
  8. 0.56    0.000013           7         2           stat
  9. 0.48    0.000011          11         1           munmap
  10. 0.48    0.000011           6         2           sigaltstack
  11. 0.43    0.000010          10         1           execve
  12. 0.39    0.000009           9         1           sched_getaffinity
  13. 0.35    0.000008           8         1           arch_prctl
  14. 0.22    0.000005           5         1           gettid
  15. 0.00    0.000000           0         2           read
  16. 0.00    0.000000           0         3           write
  17. 0.00    0.000000           0         1           close
  18. 0.00    0.000000           0         1           openat
  19. ------ ----------- ----------- --------- --------- ----------------
  20. 100.00    0.002303                   192         1 total

结论：
在相同的时间段内，没有使用time.After()的代码进行了195,000次系统调用，而使用了time.After(time.Second)的代码只进行了192次系统调用。你可以通过将已获取的文件信息作为FileStruct的一部分来进一步改进代码。

英文:

The primary problem with your code is in the for loop all the time you are passing a FileStruct to the channel either old or new . So there is no wait time for the channel receive for data and inside the if loop you are doing stat on the file for getting its data which mostly you must have already done

Here is strace on your program

  1. % time     seconds  usecs/call     calls    errors syscall
  2. ------ ----------- ----------- --------- --------- ----------------
  3. 93.58   11.835475        3793      3120       227 futex
  4. 6.38    0.807279           4    192048         1 stat
  5. 0.03    0.003284           9       366           sched_yield
  6. 0.01    0.000759           7       114           rt_sigaction
  7. 0.00    0.000271          90         3           openat
  8. 0.00    0.000197          10        19           mmap
  9. 0.00    0.000143          20         7           write
  10. 0.00    0.000071          24         3           clone
  11. 0.00    0.000064           8         8           rt_sigprocmask
  12. 0.00    0.000034          17         2           select
  13. 0.00    0.000021          11         2           read
  14. 0.00    0.000016          16         1           sched_getaffinity
  15. 0.00    0.000014          14         1           munmap
  16. 0.00    0.000014          14         1           execve
  17. 0.00    0.000013          13         1           arch_prctl
  18. 0.00    0.000011          11         1           close
  19. 0.00    0.000000           0         2           sigaltstack
  20. 0.00    0.000000           0         1           gettid
  21. ------ ----------- ----------- --------- --------- ----------------
  22. 100.00   12.647666                195700       228 total

Here with in around 40 seconds there are 195k system calls

What you may do is add a wait time just after for

  1. for { 
  2. &lt;- time.After(time.Second)
  3. if file, ok := &lt;- rotate.File; ok{

And you may add fileinfo in the FileStruct and on every looping you may check for that in the struct first then only do the stat

Here is the strace after adding &lt;- time.After(time.Second)

  1. % time     seconds  usecs/call     calls    errors syscall
  2. ------ ----------- ----------- --------- --------- ----------------
  3. 65.65    0.001512          35        43         1 futex
  4. 23.71    0.000546           5       114           rt_sigaction
  5. 3.04    0.000070           9         8           mmap
  6. 2.43    0.000056          19         3           clone
  7. 2.26    0.000052           7         8           rt_sigprocmask
  8. 0.56    0.000013           7         2           stat
  9. 0.48    0.000011          11         1           munmap
  10. 0.48    0.000011           6         2           sigaltstack
  11. 0.43    0.000010          10         1           execve
  12. 0.39    0.000009           9         1           sched_getaffinity
  13. 0.35    0.000008           8         1           arch_prctl
  14. 0.22    0.000005           5         1           gettid
  15. 0.00    0.000000           0         2           read
  16. 0.00    0.000000           0         3           write
  17. 0.00    0.000000           0         1           close
  18. 0.00    0.000000           0         1           openat
  19. ------ ----------- ----------- --------- --------- ----------------
  20. 100.00    0.002303                   192         1 total

Conclusion

For same time duration code without time.After() made 195K system calls where the one with time.After(time.Second) made only 192 system calls. You can further improve it by adding already fetched file info as a part of FileStruct

huangapple
  • 本文由 发表于 2017年1月16日 17:53:02
  • 转载请务必保留本文链接：https://go.coder-hub.com/41673576.html
  • go
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定