英文:
loading docker image fails
问题
我正在使用golang
和docker client
来加载一个以.tar
格式的Docker镜像。
func loadImageFromTar(cli *client.Client, tarFilePath string) (string, error) {
// 读取tar文件
tarFile, err := os.Open(tarFilePath)
if err != nil {
return "", fmt.Errorf("打开tar文件失败: %w", err)
}
defer tarFile.Close()
// 创建一个管道来在tar读取器和Docker客户端之间传输数据
pr, pw := io.Pipe()
// 设置一个WaitGroup用于同步
var wg sync.WaitGroup
wg.Add(2)
// 在单独的goroutine中加载Docker镜像
var imageLoadResponse types.ImageLoadResponse
go func() {
defer wg.Done()
imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
if err != nil {
err = fmt.Errorf("加载Docker镜像失败: %w", err)
}
}()
// 在单独的goroutine中读取tar文件的元数据并将tar文件复制到管道写入器中
var repoTag string
go func() {
defer wg.Done()
defer pw.Close()
tarReader := tar.NewReader(tarFile)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
err = fmt.Errorf("读取tar头部失败: %w", err)
fmt.Printf("错误: %v", err)
return
}
// 从manifest文件中提取仓库和标签
if header.Name == "manifest.json" {
data, err := io.ReadAll(tarReader)
if err != nil {
err = fmt.Errorf("读取manifest文件失败: %w", err)
fmt.Printf("错误: %v", err)
return
}
var manifest []map[string]interface{}
err = json.Unmarshal(data, &manifest)
if err != nil {
err = fmt.Errorf("解析manifest失败: %w", err)
fmt.Printf("错误: %v", err)
return
}
repoTag = manifest[0]["RepoTags"].([]interface{})[0].(string)
}
// 将tar文件数据复制到管道写入器中
_, err = io.Copy(pw, tarReader)
if err != nil {
err = fmt.Errorf("复制tar数据失败: %w", err)
fmt.Printf("错误: %v", err)
return
}
}
}()
// 等待两个goroutine完成
wg.Wait()
// 检查是否有任何错误发生在goroutine中
if err != nil {
return "", err
}
// 关闭镜像加载响应体
defer imageLoadResponse.Body.Close()
// 获取镜像ID
imageID, err := getImageIDByRepoTag(cli, repoTag)
if err != nil {
return "", fmt.Errorf("获取镜像ID失败: %w", err)
}
return imageID, nil
}
// func: getImageIDByRepoTag
func getImageIDByRepoTag(cli *client.Client, repoTag string) (string, error) {
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
if err != nil {
return "", fmt.Errorf("列出镜像失败: %w", err)
}
for _, image := range images {
for _, tag := range image.RepoTags {
if tag == repoTag {
return image.ID, nil
}
}
}
return "", fmt.Errorf("未找到镜像ID: %s", repoTag)
}
getImageIDByRepoTag
函数总是返回fmt.Errorf("未找到镜像ID: %s", repoTag)
。当我运行docker images
命令时,我看不到加载的镜像。看起来镜像加载没有完成。
在我的其他代码中,Docker镜像加载通常需要一些时间,尽管Docker客户端cli.ImageLoad
立即返回。我通常在检查getImageIDByRepoTag
之前添加大约30秒的等待时间。在这种情况下,添加等待时间也没有帮助。
谢谢。
英文:
I am using golang
, docker client
to load a docker image in .tar
format.
func loadImageFromTar(cli *client.Client, tarFilePath string) (string, error) {
// Read tar file
tarFile, err := os.Open(tarFilePath)
if err != nil {
return "", fmt.Errorf("failed to open tar file: %w", err)
}
defer tarFile.Close()
// Create a pipe to stream data between tar reader and Docker client
pr, pw := io.Pipe()
// Set up a WaitGroup for synchronization
var wg sync.WaitGroup
wg.Add(2)
// Load the Docker image in a separate goroutine
var imageLoadResponse types.ImageLoadResponse
go func() {
defer wg.Done()
imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
if err != nil {
err = fmt.Errorf("failed to load Docker image: %w", err)
}
}()
// Read tar file metadata and copy the tar file to the pipe writer in a separate goroutine
var repoTag string
go func() {
defer wg.Done()
defer pw.Close()
tarReader := tar.NewReader(tarFile)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
err = fmt.Errorf("failed to read tar header: %w", err)
fmt.Printf("Error: %v", err)
return
}
// Extract the repository and tag from the manifest file
if header.Name == "manifest.json" {
data, err := io.ReadAll(tarReader)
if err != nil {
err = fmt.Errorf("failed to read manifest file: %w", err)
fmt.Printf("Error: %v", err)
return
}
var manifest []map[string]interface{}
err = json.Unmarshal(data, &manifest)
if err != nil {
err = fmt.Errorf("failed to unmarshal manifest: %w", err)
fmt.Printf("Error: %v", err)
return
}
repoTag = manifest[0]["RepoTags"].([]interface{})[0].(string)
}
// Copy the tar file data to the pipe writer
_, err = io.Copy(pw, tarReader)
if err != nil {
err = fmt.Errorf("failed to copy tar data: %w", err)
fmt.Printf("Error: %v", err)
return
}
}
}()
// Wait for both goroutines to finish
wg.Wait()
// Check if any error occurred in the goroutines
if err != nil {
return "", err
}
// Close the image load response body
defer imageLoadResponse.Body.Close()
// Get the image ID
imageID, err := getImageIDByRepoTag(cli, repoTag)
if err != nil {
return "", fmt.Errorf("failed to get image ID: %w", err)
}
return imageID, nil
}
// func: getImageIDByRepoTag
func getImageIDByRepoTag(cli *client.Client, repoTag string) (string, error) {
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
if err != nil {
return "", fmt.Errorf("failed to list images: %w", err)
}
for _, image := range images {
for _, tag := range image.RepoTags {
if tag == repoTag {
return image.ID, nil
}
}
}
return "", fmt.Errorf("image ID not found for repo tag: %s", repoTag)
}
The getImageIDByRepoTag
always returns fmt.Errorf("image ID not found for repo tag: %s", repoTag)
.
Also when I run docker images
I do not see the image being loaded. Looks like the image load is not completing .
In my other code, The docker images load usually takes time although the docker client cli.ImageLoad
returns immediately. I usually add some 30 seconds wait time before checking for getImageIDByRepoTag
. Adding a wait time also did not help in this case.
Thanks
答案1
得分: 1
有几个问题:
- 两个goroutine共享了
err
变量,因此可能会丢失一些错误处理。- 在每个goroutine中应该使用唯一的错误变量,并在
wg.Wait()
之后检查两个错误。
- 在每个goroutine中应该使用唯一的错误变量,并在
- 主要问题是:你正在从
tar
读取器中读取以查找清单文件并提取标签信息 - 这是可以的 - 但是在找到后,你将剩余的字节流复制到了管道中。因此,你丢失了一部分字节流,这部分字节流永远不会到达docker
客户端。
为了避免两次读取tar字节流,你可以使用io.TeeReader。
这允许你读取tar归档文件 - 以扫描manifest
文件 - 但同时将完整的流写入其他地方(即docker
客户端)。
创建TeeReader
:
tr := io.TeeReader(tarFile, pw) // 读取`tr`将读取tarFile - 但同时写入`pw`
现在,图像加载将从这里读取(而不是管道):
//imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
imageLoadResponse, err = cli.ImageLoad(context.Background(), tr, false)
然后将你的archive/tar
读取器更改为从管道中读取:
//tarReader := tar.NewReader(tarFile) // 直接从文件读取
tarReader := tar.NewReader(pr) // 间接从管道中读取(从文件中读取)
然后你可以删除io.Copy
块:
// 不再需要:
//
// _, err = io.Copy(pw, tarReader)
//
因为tar检查代码将读取整个流直到EOF。
P.S. 你可能希望将io.EOF
重置为nil
,以避免在检查任何潜在的错误时将EOF
视为更严重的错误:
header, err = tarReader.Next()
if err == io.EOF {
err = nil // 在这里,EOF是一个非致命错误
break
}
英文:
There's a couple of issues:
- the two goroutines share
err
so some error handling may get lost- you should use unique error variables here for each goroutine & inspect both errors after the
wg.Wait()
- you should use unique error variables here for each goroutine & inspect both errors after the
- the main issue: you are reading from the
tar
reader to find the manifest file and extract the tag info - which is fine - but upon finding this, you copy the rest of the byte stream to your pipe. You are thus losing a chunk of the byte stream which never reaches thedocker
client
To avoid reading the tar byte stream twice, you could use io.TeeReader.
This allows you to read the tar archive - to scan for the manifest
file - but also have this stream written in full elsewhere (i.e. to the docker
client).
Create the TeeReader
:
tr := io.TeeReader(tarFile, pw) // reading `tr` will read the tarFile - but simultaneously write to `pw`
the image load will now read from this (instead of the pipe):
//imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
imageLoadResponse, err = cli.ImageLoad(context.Background(), tr, false)
then change your archive/tar
reader to read from the pipe:
//tarReader := tar.NewReader(tarFile) // direct from file
tarReader := tar.NewReader(pr) // read from pipe (indirectly from the file)
you can then drop the your io.Copy
block:
// no longer needed:
//
// _, err = io.Copy(pw, tarReader)
//
since the tar-inspection code will read the entire stream to EOF.
P.S. you may want to reset the io.EOF
to nil
to avoid thinking a EOF
is a more critical error, when you inspect any potential errors from either of the goroutines:
header, err = tarReader.Next()
if err == io.EOF {
err = nil // EOF is a non-fatal error here
break
}
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论