加载 Docker 镜像失败

huangapple go评论91阅读模式
英文:

loading docker image fails

问题

我正在使用golangdocker client来加载一个以.tar格式的Docker镜像。

func loadImageFromTar(cli *client.Client, tarFilePath string) (string, error) {
    // 读取tar文件
    tarFile, err := os.Open(tarFilePath)
    if err != nil {
        return "", fmt.Errorf("打开tar文件失败: %w", err)
    }
    defer tarFile.Close()

    // 创建一个管道来在tar读取器和Docker客户端之间传输数据
    pr, pw := io.Pipe()

    // 设置一个WaitGroup用于同步
    var wg sync.WaitGroup
    wg.Add(2)

    // 在单独的goroutine中加载Docker镜像
    var imageLoadResponse types.ImageLoadResponse
    go func() {
        defer wg.Done()
        imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
        if err != nil {
            err = fmt.Errorf("加载Docker镜像失败: %w", err)
        }
    }()

    // 在单独的goroutine中读取tar文件的元数据并将tar文件复制到管道写入器中
    var repoTag string
    go func() {
        defer wg.Done()
        defer pw.Close()

        tarReader := tar.NewReader(tarFile)

        for {
            header, err := tarReader.Next()
            if err == io.EOF {
                break
            }
            if err != nil {
                err = fmt.Errorf("读取tar头部失败: %w", err)
                fmt.Printf("错误: %v", err)
                return
            }

            // 从manifest文件中提取仓库和标签
            if header.Name == "manifest.json" {
                data, err := io.ReadAll(tarReader)
                if err != nil {
                    err = fmt.Errorf("读取manifest文件失败: %w", err)
                    fmt.Printf("错误: %v", err)
                    return
                }

                var manifest []map[string]interface{}
                err = json.Unmarshal(data, &manifest)
                if err != nil {
                    err = fmt.Errorf("解析manifest失败: %w", err)
                    fmt.Printf("错误: %v", err)
                    return
                }

                repoTag = manifest[0]["RepoTags"].([]interface{})[0].(string)
            }

            // 将tar文件数据复制到管道写入器中
            _, err = io.Copy(pw, tarReader)
            if err != nil {
                err = fmt.Errorf("复制tar数据失败: %w", err)
                fmt.Printf("错误: %v", err)
                return
            }
        }
    }()

    // 等待两个goroutine完成
    wg.Wait()

    // 检查是否有任何错误发生在goroutine中
    if err != nil {
        return "", err
    }

    // 关闭镜像加载响应体
    defer imageLoadResponse.Body.Close()

    // 获取镜像ID
    imageID, err := getImageIDByRepoTag(cli, repoTag)
    if err != nil {
        return "", fmt.Errorf("获取镜像ID失败: %w", err)
    }

    return imageID, nil
}

// func: getImageIDByRepoTag

func getImageIDByRepoTag(cli *client.Client, repoTag string) (string, error) {
    images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
    if err != nil {
        return "", fmt.Errorf("列出镜像失败: %w", err)
    }

    for _, image := range images {
        for _, tag := range image.RepoTags {
            if tag == repoTag {
                return image.ID, nil
            }
        }
    }

    return "", fmt.Errorf("未找到镜像ID: %s", repoTag)
}

getImageIDByRepoTag函数总是返回fmt.Errorf("未找到镜像ID: %s", repoTag)。当我运行docker images命令时,我看不到加载的镜像。看起来镜像加载没有完成。

在我的其他代码中,Docker镜像加载通常需要一些时间,尽管Docker客户端cli.ImageLoad立即返回。我通常在检查getImageIDByRepoTag之前添加大约30秒的等待时间。在这种情况下,添加等待时间也没有帮助。

谢谢。

英文:

I am using golang, docker client to load a docker image in .tar format.

func loadImageFromTar(cli *client.Client, tarFilePath string) (string, error) {
// Read tar file
tarFile, err := os.Open(tarFilePath)
if err != nil {
return "", fmt.Errorf("failed to open tar file: %w", err)
}
defer tarFile.Close()
// Create a pipe to stream data between tar reader and Docker client
pr, pw := io.Pipe()
// Set up a WaitGroup for synchronization
var wg sync.WaitGroup
wg.Add(2)
// Load the Docker image in a separate goroutine
var imageLoadResponse types.ImageLoadResponse
go func() {
defer wg.Done()
imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
if err != nil {
err = fmt.Errorf("failed to load Docker image: %w", err)
}
}()
// Read tar file metadata and copy the tar file to the pipe writer in a separate goroutine
var repoTag string
go func() {
defer wg.Done()
defer pw.Close()
tarReader := tar.NewReader(tarFile)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
err = fmt.Errorf("failed to read tar header: %w", err)
fmt.Printf("Error: %v", err)
return
}
// Extract the repository and tag from the manifest file
if header.Name == "manifest.json" {
data, err := io.ReadAll(tarReader)
if err != nil {
err = fmt.Errorf("failed to read manifest file: %w", err)
fmt.Printf("Error: %v", err)
return
}
var manifest []map[string]interface{}
err = json.Unmarshal(data, &manifest)
if err != nil {
err = fmt.Errorf("failed to unmarshal manifest: %w", err)
fmt.Printf("Error: %v", err)
return
}
repoTag = manifest[0]["RepoTags"].([]interface{})[0].(string)
}
// Copy the tar file data to the pipe writer
_, err = io.Copy(pw, tarReader)
if err != nil {
err = fmt.Errorf("failed to copy tar data: %w", err)
fmt.Printf("Error: %v", err)
return
}
}
}()
// Wait for both goroutines to finish
wg.Wait()
// Check if any error occurred in the goroutines
if err != nil {
return "", err
}
// Close the image load response body
defer imageLoadResponse.Body.Close()
// Get the image ID
imageID, err := getImageIDByRepoTag(cli, repoTag)
if err != nil {
return "", fmt.Errorf("failed to get image ID: %w", err)
}
return imageID, nil
}

// func: getImageIDByRepoTag

func getImageIDByRepoTag(cli *client.Client, repoTag string) (string, error) {
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
if err != nil {
return "", fmt.Errorf("failed to list images: %w", err)
}
for _, image := range images {
for _, tag := range image.RepoTags {
if tag == repoTag {
return image.ID, nil
}
}
}
return "", fmt.Errorf("image ID not found for repo tag: %s", repoTag)
}

The getImageIDByRepoTag always returns fmt.Errorf("image ID not found for repo tag: %s", repoTag).
Also when I run docker images I do not see the image being loaded. Looks like the image load is not completing .

In my other code, The docker images load usually takes time although the docker client cli.ImageLoad returns immediately. I usually add some 30 seconds wait time before checking for getImageIDByRepoTag. Adding a wait time also did not help in this case.

Thanks

答案1

得分: 1

有几个问题:

  • 两个goroutine共享了err变量,因此可能会丢失一些错误处理。
    • 在每个goroutine中应该使用唯一的错误变量,并在wg.Wait()之后检查两个错误。
  • 主要问题是:你正在从tar读取器中读取以查找清单文件并提取标签信息 - 这是可以的 - 但是在找到后,你将剩余的字节流复制到了管道中。因此,你丢失了一部分字节流,这部分字节流永远不会到达docker客户端。

为了避免两次读取tar字节流,你可以使用io.TeeReader
这允许你读取tar归档文件 - 以扫描manifest文件 - 但同时将完整的流写入其他地方(即docker客户端)。

创建TeeReader

tr := io.TeeReader(tarFile, pw)  // 读取`tr`将读取tarFile - 但同时写入`pw`

现在,图像加载将从这里读取(而不是管道):

//imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
imageLoadResponse, err = cli.ImageLoad(context.Background(), tr, false)

然后将你的archive/tar读取器更改为从管道中读取:

//tarReader := tar.NewReader(tarFile) // 直接从文件读取
tarReader := tar.NewReader(pr) // 间接从管道中读取(从文件中读取)

然后你可以删除io.Copy块:

// 不再需要:
//
// _, err = io.Copy(pw, tarReader)
//

因为tar检查代码将读取整个流直到EOF。

P.S. 你可能希望将io.EOF重置为nil,以避免在检查任何潜在的错误时将EOF视为更严重的错误:

header, err = tarReader.Next()
if err == io.EOF {
err = nil  //  在这里,EOF是一个非致命错误
break
}
英文:

There's a couple of issues:

  • the two goroutines share err so some error handling may get lost
    • you should use unique error variables here for each goroutine & inspect both errors after the wg.Wait()
  • the main issue: you are reading from the tar reader to find the manifest file and extract the tag info - which is fine - but upon finding this, you copy the rest of the byte stream to your pipe. You are thus losing a chunk of the byte stream which never reaches the docker client

To avoid reading the tar byte stream twice, you could use io.TeeReader.
This allows you to read the tar archive - to scan for the manifest file - but also have this stream written in full elsewhere (i.e. to the docker client).

Create the TeeReader:

tr := io.TeeReader(tarFile, pw)  // reading `tr` will read the tarFile - but simultaneously write to `pw`

the image load will now read from this (instead of the pipe):

//imageLoadResponse, err = cli.ImageLoad(context.Background(), pr, false)
imageLoadResponse, err = cli.ImageLoad(context.Background(), tr, false)

then change your archive/tar reader to read from the pipe:

//tarReader := tar.NewReader(tarFile) // direct from file
tarReader := tar.NewReader(pr) // read from pipe (indirectly from the file)

you can then drop the your io.Copy block:

// no longer needed:
//
// _, err = io.Copy(pw, tarReader)
//

since the tar-inspection code will read the entire stream to EOF.

P.S. you may want to reset the io.EOF to nil to avoid thinking a EOF is a more critical error, when you inspect any potential errors from either of the goroutines:

header, err = tarReader.Next()
if err == io.EOF {
err = nil  //  EOF is a non-fatal error here
break
}

huangapple
  • 本文由 发表于 2023年4月8日 22:29:40
  • 转载请务必保留本文链接:https://go.coder-hub.com/75965806.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定