英文:
Build Zip with goroutines panic with slice bounds out of range with capacity 4096
问题
我正在尝试改进一个函数,通过添加goroutines来处理每个需要归档的文件,以构建一个Zip文件。但是最终出现了一个panic错误:panic: runtime error: slice bounds out of range [4126:4096]
。
目标目录包含190个文件(500 MB)。
我不太明白出了什么问题,
非常感谢您的帮助。
这是函数的代码:
func BuildArchive() error {
var files []string
err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
if err != nil {
fmt.Println(err)
return err
}
files = append(files, filePath)
return nil
})
if err != nil {
return err
}
bundle, err := os.Create("/tmp/archive.zip")
if err != nil {
return err
}
bundleWriter := zip.NewWriter(bundle)
var wg sync.WaitGroup
wg.Add(len(files))
for _, filePath := range files {
go func(filePath string) {
defer wg.Done()
relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
bundleFile, err := bundleWriter.Create(relPath)
if err != nil {
fmt.Println(err)
}
fsFile, err := os.Open(filePath)
if err != nil {
fmt.Println(err)
}
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
}(filePath)
}
wg.Wait()
err = bundleWriter.Close()
if err != nil {
return err
}
return nil
}
错误发生在这里:
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
堆栈跟踪:
goroutine 48 [running]:
bufio.(*Writer).Write(0xc00002a100, {0xc00041a400?, 0x3d?, 0xc00041a400?})
/usr/local/go/src/bufio/bufio.go:670 +0x1c8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc00041a400?, 0x3d?, 0x4afa20?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
io.WriteString({0x4e7538, 0xc00000c138}, {0xc0000212c9, 0x3d})
/usr/local/go/src/io/io.go:314 +0x91
archive/zip.writeHeader({0x4e7538, 0xc00000c138}, 0xc000220090)
/usr/local/go/src/archive/zip/writer.go:422 +0x5ec
archive/zip.(*Writer).CreateHeader(0xc0000760a0, 0xc00021e1b0)
/usr/local/go/src/archive/zip/writer.go:378 +0x797
archive/zip.(*Writer).Create(0x4e7698?, {0xc0000212c9, 0x3d})
/usr/local/go/src/archive/zip/writer.go:223 +0x6c
main.BuildArchive.func2({0xc0000212c0, 0x46})
/home/simba/go/src/foobar/main.go:79 +0x1c5
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
panic: runtime error: slice bounds out of range [:4126] with capacity 4096
goroutine 6 [running]:
bufio.(*Writer).Flush(0xc00002a100)
/usr/local/go/src/bufio/bufio.go:634 +0x171
bufio.(*Writer).Write(0xc00002a100, {0xc0001b4200?, 0xc000199b20?, 0xc000199b20?})
/usr/local/go/src/bufio/bufio.go:672 +0xd8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc0001b4200?, 0x0?, 0xc000199b40?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
archive/zip.(*countWriter).Write(0xc000220018, {0xc0001b4200?, 0xc0001b02c0?, 0xc0001b02f0?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
compress/flate.(*huffmanBitWriter).write(...)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:136
compress/flate.(*huffmanBitWriter).writeCode(0xc0001b41e0?, {0x6000?, 0x22?})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:347 +0xe5
compress/flate.(*huffmanBitWriter).writeTokens(0xc0001b41e0, {0xc002558000, 0x4001, 0x403f800000403f?}, {0xc0001aa900, 0x11e, 0x108129000000000f?}, {0xc0001ac100, 0x1e, 0x1e})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:583 +0xb9
compress/flate.(*huffmanBitWriter).writeBlock(0xc0001b41e0, {0xc002558000?, 0x20?, 0xd79?}, 0x0, {0x0, 0x0, 0x0})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:495 +0x490
compress/flate.(*compressor).writeBlock(0xc0005a2000, {0xc002558000?, 0xc000032f00?, 0xc000199d28?}, 0x47739b?)
/usr/local/go/src/compress/flate/deflate.go:170 +0x9c
compress/flate.(*compressor).deflate(0xc0005a2000)
/usr/local/go/src/compress/flate/deflate.go:509 +0x59b
compress/flate.(*compressor).write(0xc0005a2000, {0xc00256a000?, 0x8000, 0xf311b6fd?})
/usr/local/go/src/compress/flate/deflate.go:554 +0x82
compress/flate.(*Writer).Write(...)
/usr/local/go/src/compress/flate/deflate.go:712
archive/zip.(*pooledFlateWriter).Write(0xc00020c040?, {0xc00256a000?, 0x8000?, 0x4af140?})
/usr/local/go/src/archive/zip/register.go:51 +0xc5
archive/zip.(*countWriter).Write(...)
/usr/local/go/src/archive/zip/writer.go:601
archive/zip.(*fileWriter).Write(0xc000222000, {0xc00256a000, 0x8000, 0x8000})
/usr/local/go/src/archive/zip/writer.go:533 +0x97
io.copyBuffer({0x4e7558, 0xc000222000}, {0x4e7678, 0xc0001f8008}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:428 +0x204
io.Copy(...)
/usr/local/go/src/io/io.go:385
main.BuildArchive.func2({0xc00001c0c0, 0x35})
/home/simba/go/src/foobar/main.go:89 +0x385
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
exit status 2
希望这能帮到你!
英文:
I'm trying to improve a function to build a Zip by adding goroutines to handle each file that has to be archived.
But it ends up with a panic
panic: runtime error: slice bounds out of range [4126:4096
]
The target directory contains 190 files (500 Mo).
I don't really understand what's wrong,
Thank you in advance for your help
The function :
func BuildArchive() error {
var files []string
err := filepath.Walk("/tmp/dir-to-zip", func(filePath string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
if err != nil {
fmt.Println(err)
return err
}
files = append(files, filePath)
return nil
})
if err != nil {
return err
}
bundle, err := os.Create("/tmp/archive.zip")
if err != nil {
return err
}
bundleWriter := zip.NewWriter(bundle)
var wg sync.WaitGroup
wg.Add(len(files))
for _, filePath := range files {
go func(filePath string) {
defer wg.Done()
relPath := strings.TrimPrefix(filePath, fmt.Sprintf("%v/", filepath.Dir("/tmp/dir-to-zip")))
bundleFile, err := bundleWriter.Create(relPath)
if err != nil {
fmt.Println(err)
}
fsFile, err := os.Open(filePath)
if err != nil {
fmt.Println(err)
}
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
}(filePath)
}
wg.Wait()
err = bundleWriter.Close()
if err != nil {
return err
}
return nil
}
The error occurs here :
_, err = io.Copy(bundleFile, fsFile)
if err != nil {
fmt.Println(err)
}
The stack trace :
goroutine 48 [running]:
bufio.(*Writer).Write(0xc00002a100, {0xc00041a400?, 0x3d?, 0xc00041a400?})
/usr/local/go/src/bufio/bufio.go:670 +0x1c8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc00041a400?, 0x3d?, 0x4afa20?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
io.WriteString({0x4e7538, 0xc00000c138}, {0xc0000212c9, 0x3d})
/usr/local/go/src/io/io.go:314 +0x91
archive/zip.writeHeader({0x4e7538, 0xc00000c138}, 0xc000220090)
/usr/local/go/src/archive/zip/writer.go:422 +0x5ec
archive/zip.(*Writer).CreateHeader(0xc0000760a0, 0xc00021e1b0)
/usr/local/go/src/archive/zip/writer.go:378 +0x797
archive/zip.(*Writer).Create(0x4e7698?, {0xc0000212c9, 0x3d})
/usr/local/go/src/archive/zip/writer.go:223 +0x6c
main.BuildArchive.func2({0xc0000212c0, 0x46})
/home/simba/go/src/foobar/main.go:79 +0x1c5
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
panic: runtime error: slice bounds out of range [:4126] with capacity 4096
goroutine 6 [running]:
bufio.(*Writer).Flush(0xc00002a100)
/usr/local/go/src/bufio/bufio.go:634 +0x171
bufio.(*Writer).Write(0xc00002a100, {0xc0001b4200?, 0xc000199b20?, 0xc000199b20?})
/usr/local/go/src/bufio/bufio.go:672 +0xd8
archive/zip.(*countWriter).Write(0xc00000c138, {0xc0001b4200?, 0x0?, 0xc000199b40?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
archive/zip.(*countWriter).Write(0xc000220018, {0xc0001b4200?, 0xc0001b02c0?, 0xc0001b02f0?})
/usr/local/go/src/archive/zip/writer.go:601 +0x2e
compress/flate.(*huffmanBitWriter).write(...)
/usr/local/go/src/compress/flate/huffman_bit_writer.go:136
compress/flate.(*huffmanBitWriter).writeCode(0xc0001b41e0?, {0x6000?, 0x22?})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:347 +0xe5
compress/flate.(*huffmanBitWriter).writeTokens(0xc0001b41e0, {0xc002558000, 0x4001, 0x403f800000403f?}, {0xc0001aa900, 0x11e, 0x108129000000000f?}, {0xc0001ac100, 0x1e, 0x1e})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:583 +0xb9
compress/flate.(*huffmanBitWriter).writeBlock(0xc0001b41e0, {0xc002558000?, 0x20?, 0xd79?}, 0x0, {0x0, 0x0, 0x0})
/usr/local/go/src/compress/flate/huffman_bit_writer.go:495 +0x490
compress/flate.(*compressor).writeBlock(0xc0005a2000, {0xc002558000?, 0xc000032f00?, 0xc000199d28?}, 0x47739b?)
/usr/local/go/src/compress/flate/deflate.go:170 +0x9c
compress/flate.(*compressor).deflate(0xc0005a2000)
/usr/local/go/src/compress/flate/deflate.go:509 +0x59b
compress/flate.(*compressor).write(0xc0005a2000, {0xc00256a000?, 0x8000, 0xf311b6fd?})
/usr/local/go/src/compress/flate/deflate.go:554 +0x82
compress/flate.(*Writer).Write(...)
/usr/local/go/src/compress/flate/deflate.go:712
archive/zip.(*pooledFlateWriter).Write(0xc00020c040?, {0xc00256a000?, 0x8000?, 0x4af140?})
/usr/local/go/src/archive/zip/register.go:51 +0xc5
archive/zip.(*countWriter).Write(...)
/usr/local/go/src/archive/zip/writer.go:601
archive/zip.(*fileWriter).Write(0xc000222000, {0xc00256a000, 0x8000, 0x8000})
/usr/local/go/src/archive/zip/writer.go:533 +0x97
io.copyBuffer({0x4e7558, 0xc000222000}, {0x4e7678, 0xc0001f8008}, {0x0, 0x0, 0x0})
/usr/local/go/src/io/io.go:428 +0x204
io.Copy(...)
/usr/local/go/src/io/io.go:385
main.BuildArchive.func2({0xc00001c0c0, 0x35})
/home/simba/go/src/foobar/main.go:89 +0x385
created by main.BuildArchive
/home/simba/go/src/foobar/main.go:73 +0x5aa
exit status 2
答案1
得分: 3
zip.Writer
不支持并发使用。你启动了多个 goroutine,每个 goroutine 创建并写入 zip 条目(文件)。
Writer.Create()
的文档说明如下:
> Create 使用提供的名称将文件添加到 zip 文件中。它返回一个 Writer,用于写入文件内容。
>
> [...] 在下一次调用 Create、CreateHeader 或 Close 之前,必须将文件内容写入 io.Writer。
你不能并发地创建 zip 文件。每个 zip 条目的 io.Writer
都写入同一个底层文件(或者通常情况下写入同一个 io.Writer
),即使你的应用程序没有崩溃或崩溃,生成的 zip 归档文件很可能是无效的。
英文:
zip.Writer
is not safe for concurrent use. You launch multiple goroutines, each creating and writing to zip entries (files).
Writer.Create()
documents that:
> Create adds a file to the zip file using the provided name. It returns a Writer to which the file contents should be written.
>
> [...] The file's contents must be written to the io.Writer before the next call to Create, CreateHeader, or Close.
You can't create a zip concurrently. io.Writer
s of each zip entry write to the same underlying file (or to the same io.Writer
in general), even if your app doesn't panic or crash, the resulting zip archive will likely be invalid.
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论