创建一个用于测试文件访问的大型csv文件

huangapple
go评论108阅读模式
英文:

Creating a large csv file for testing file access

问题

我想创建一个大小为10GB的文件，格式如下：

  1. 前缀:用户名:时间戳, 数字

例如：

  1. login:jbill:2013/3/25, 1

我想通过创建类似上面的随机行来创建一个10GB的文件。

在Go语言中，我该如何实现这个功能？

我可以有一个前缀的数组，如下：

  1. login, logout, register

还可以有一个用户名的数组：

  1. jbill, dkennedy
英文:

I want to create a 10 GB file that looks like:

  1. prefix:username:timestamp, number

So an example is like:

  1. login:jbill:2013/3/25, 1

I want to create a 10GB file, by creating random rows like the one above.

How could I do this in Go?

I can have an array of prefixes like:

  1. login, logout, register

And also an array of usernames:

  1. jbill, dkennedy

答案1

得分: 5

例如，

  1. package main
  3. import (
  4. 	"bufio"
  5. 	"fmt"
  6. 	"math/rand"
  7. 	"os"
  8. 	"strconv"
  9. 	"time"
  10. )
  12. func main() {
  13. 	fileSize := int64(10e9) // 10GB
  14. 	f, err := os.Create("/tmp/largefile")
  15. 	if err != nil {
  16. 		fmt.Println(err)
  17. 		return
  18. 	}
  19. 	w := bufio.NewWriter(f)
  20. 	prefixes := []string{"login", "logout", "register"}
  21. 	names := []string{"jbill", "dkennedy"}
  22. 	timeStart := time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)
  23. 	timeDur := timeStart.AddDate(1, 0, 0).Sub(timeStart)
  24. 	rand.Seed(time.Now().UnixNano())
  25. 	size := int64(0)
  26. 	for size < fileSize {
  27. 		// prefix:username:timestamp, number
  28. 		// login:jbill:2012/3/25, 1
  29. 		prefix := prefixes[int(rand.Int31n(int32(len(prefixes))))]
  30. 		name := names[int(rand.Int31n(int32(len(names))))]
  31. 		time := timeStart.Add(time.Duration(rand.Int63n(int64(timeDur)))).Format("2006/1/2")
  32. 		number := strconv.Itoa(int(rand.Int31n(100) + 1))
  33. 		line := prefix + ":" + name + ":" + time + ", " + number + "\n"
  34. 		n, err := w.WriteString(line)
  35. 		if err != nil {
  36. 			fmt.Println(n, err)
  37. 			return
  38. 		}
  39. 		size += int64(len(line))
  40. 	}
  41. 	err = w.Flush()
  42. 	if err != nil {
  43. 		fmt.Println(err)
  44. 		return
  45. 	}
  46. 	err = f.Close()
  47. 	if err != nil {
  48. 		fmt.Println(err)
  49. 		return
  50. 	}
  51. 	fmt.Println("Size:", size)
  52. }

输出：

  1. register:jbill:2012/8/24, 15
  2. login:jbill:2012/10/7, 98
  3. register:dkennedy:2012/8/29, 70
  4. register:jbill:2012/6/1, 89
  5. register:jbill:2012/5/24, 63
  6. login:dkennedy:2012/3/29, 48
  7. logout:jbill:2012/7/8, 93
  8. logout:dkennedy:2012/1/12, 74
  9. login:jbill:2012/4/12, 14
  10. login:jbill:2012/2/5, 83
英文:

For example,

  1. package main
  3. import (
  4. 	"bufio"
  5. 	"fmt"
  6. 	"math/rand"
  7. 	"os"
  8. 	"strconv"
  9. 	"time"
  10. )
  12. func main() {
  13. 	fileSize := int64(10e9) // 10GB
  14. 	f, err := os.Create("/tmp/largefile")
  15. 	if err != nil {
  16. 		fmt.Println(err)
  17. 		return
  18. 	}
  19. 	w := bufio.NewWriter(f)
  20. 	prefixes := []string{"login", "logout", "register"}
  21. 	names := []string{"jbill", "dkennedy"}
  22. 	timeStart := time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)
  23. 	timeDur := timeStart.AddDate(1, 0, 0).Sub(timeStart)
  24. 	rand.Seed(time.Now().UnixNano())
  25. 	size := int64(0)
  26. 	for size < fileSize {
  27. 		// prefix:username:timestamp, number
  28. 		// login:jbill:2012/3/25, 1
  29. 		prefix := prefixes[int(rand.Int31n(int32(len(prefixes))))]
  30. 		name := names[int(rand.Int31n(int32(len(names))))]
  31. 		time := timeStart.Add(time.Duration(rand.Int63n(int64(timeDur)))).Format("2006/1/2")
  32. 		number := strconv.Itoa(int(rand.Int31n(100) + 1))
  33. 		line := prefix + ":" + name + ":" + time + ", " + number + "\n"
  34. 		n, err := w.WriteString(line)
  35. 		if err != nil {
  36. 			fmt.Println(n, err)
  37. 			return
  38. 		}
  39. 		size += int64(len(line))
  40. 	}
  41. 	err = w.Flush()
  42. 	if err != nil {
  43. 		fmt.Println(err)
  44. 		return
  45. 	}
  46. 	err = f.Close()
  47. 	if err != nil {
  48. 		fmt.Println(err)
  49. 		return
  50. 	}
  51. 	fmt.Println("Size:", size)
  52. }

Output:

  1. register:jbill:2012/8/24, 15
  2. login:jbill:2012/10/7, 98
  3. register:dkennedy:2012/8/29, 70
  4. register:jbill:2012/6/1, 89
  5. register:jbill:2012/5/24, 63
  6. login:dkennedy:2012/3/29, 48
  7. logout:jbill:2012/7/8, 93
  8. logout:dkennedy:2012/1/12, 74
  9. login:jbill:2012/4/12, 14
  10. login:jbill:2012/2/5, 83

答案2

得分: 4

这是一个天真的方法（1GB）：

  1. package main
  3. import (
  4.     "fmt"
  5.     "log"
  6.     "os"
  7. )
  9. func main() {
  10.     myfile, err := os.OpenFile("myfile", os.O_WRONLY|os.O_CREATE, 0644)
  11.     if err != nil {
  12.         log.Fatal(err)
  13.     }
  14.     defer myfile.Close()
  15.     var pos int
  16.     var line string
  17.     // sample: login:jbill:2013/3/25, 1
  18.     line = fmt.Sprintf("%s:%s:%s, %d\n", "login", "jbill", "2013/3/25", 1)
  19.     for pos < 1024*1024*1024 {
  20.         bytes, err := myfile.Write([]byte(line))
  21.         if err != nil {
  22.             log.Fatal(err)
  23.         }
  24.         pos = pos + bytes
  25.     }
  26. }

这需要很长时间（1:16），因为输出没有缓冲。通过添加bufio，您可以大大减少时间

  1. package main
  3. import (
  4.     "bufio"
  5.     "fmt"
  6.     "log"
  7.     "os"
  8. )
  10. func main() {
  11.     myfile, err := os.OpenFile("myfile", os.O_WRONLY|os.O_CREATE, 0644)
  12.     if err != nil {
  13.         log.Fatal(err)
  14.     }
  15.     defer myfile.Close()
  16.     mybufferedfile := bufio.NewWriter(myfile)
  17.     var pos int
  18.     var line string
  19.     // sample: login:jbill:2013/3/25, 1
  20.     line = fmt.Sprintf("%s:%s:%s, %d\n", "login", "jbill", "2013/3/25", 1)
  21.     for pos < 1024*1024*1024 {
  22.         bytes, err := mybufferedfile.WriteString(line)
  23.         if err != nil {
  24.             log.Fatal(err)
  25.         }
  26.         pos = pos + bytes
  27.     }
  28.     err = mybufferedfile.Flush()
  29.     if err != nil {
  30.         log.Fatal(err)
  31.     }
  32. }

在我的机器上仍然需要26秒，我希望看到一个更快的解决方案。

顺便说一下：您需要做随机字段，但这留给读者作为练习 创建一个用于测试文件访问的大型csv文件

英文:

This is a naive approach (1GB):

  1. package main
  3. import (
  4. 	&quot;fmt&quot;
  5. 	&quot;log&quot;
  6. 	&quot;os&quot;
  7. )
  9. func main() {
  10. 	myfile, err := os.OpenFile(&quot;myfile&quot;, os.O_WRONLY|os.O_CREATE, 0644)
  11. 	if err != nil {
  12. 		log.Fatal(err)
  13. 	}
  14. 	defer myfile.Close()
  15. 	var pos int
  16. 	var line string
  17. 	// sample: login:jbill:2013/3/25, 1
  18. 	line = fmt.Sprintf(&quot;%s:%s:%s, %d\n&quot;, &quot;login&quot;, &quot;jbill&quot;, &quot;2013/3/25&quot;, 1)
  19. 	for pos &lt; 1024*1024*1024 {
  20. 		bytes, err := myfile.Write([]byte(line))
  21. 		if err != nil {
  22. 			log.Fatal(err)
  23. 		}
  24. 		pos = pos + bytes
  25. 	}
  26. }

which takes forever (1:16), because the output is not buffered. By adding bufio you can decrease the time dramatically

  1. package main
  3. import (
  4. 	&quot;bufio&quot;
  5. 	&quot;fmt&quot;
  6. 	&quot;log&quot;
  7. 	&quot;os&quot;
  8. )
  10. func main() {
  11. 	myfile, err := os.OpenFile(&quot;myfile&quot;, os.O_WRONLY|os.O_CREATE, 0644)
  12. 	if err != nil {
  13. 		log.Fatal(err)
  14. 	}
  15. 	defer myfile.Close()
  16. 	mybufferedfile := bufio.NewWriter(myfile)
  17. 	var pos int
  18. 	var line string
  19. 	// sample: login:jbill:2013/3/25, 1
  20. 	line = fmt.Sprintf(&quot;%s:%s:%s, %d\n&quot;, &quot;login&quot;, &quot;jbill&quot;, &quot;2013/3/25&quot;, 1)
  21. 	for pos &lt; 1024*1024*1024 {
  22. 		bytes, err := mybufferedfile.WriteString(line)
  23. 		if err != nil {
  24. 			log.Fatal(err)
  25. 		}
  26. 		pos = pos + bytes
  27. 	}
  28. 	err = mybufferedfile.Flush()
  29. 	if err != nil {
  30. 		log.Fatal(err)
  31. 	}
  32. }

Still 26 sec on my machine, I'd like to see a faster solution.

BTW: you need to do the random fileds, but that is left as an exercise to the reader 创建一个用于测试文件访问的大型csv文件

huangapple
  • 本文由 发表于 2013年3月25日 23:55:06
  • 转载请务必保留本文链接：https://go.coder-hub.com/15619225.html
  • file-io
  • go
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定