无法使用Go客户端从Docker中访问标准输出

huangapple
go评论191阅读模式
英文:

Can't reach stdout from Docker using Go client

问题

我有一个小项目，我的Go服务器会将通过HTTP发送的C文件复制到Docker容器中，然后在容器中进行编译和运行。
然而，我无法获取发送到容器标准输出(stdout)的任何数据。

我已经确定文件被发送到Docker容器中，而且任何编译问题都会显示在错误流(error stream)中。然而，即使我在C程序中通过stderr发送数据，也没有显示任何结果，直到我在Dockerfile中使用了'>&2 echo ""'的方式，这样才能通过流传递数据并且我能够读取它。

现在，正如上面提到的，我只能读取stderr，并且只能通过一个变通方法。有什么想法为什么我不能使用标准方法来做到这一点吗？

Go服务器

  1. package main
  3. import (
  4.     "fmt"
  5.     "net/http"
  6.     "io"
  7.     "os"
  8.     "os/exec"
  9.     "log"
  10.     "encoding/json"
  12.     "github.com/docker/docker/client"
  13.     dockertypes "github.com/docker/docker/api/types"
  14.     "github.com/docker/docker/api/types/container"
  15.     "golang.org/x/net/context"
  16.     "time"
  17.     "bytes"
  18. )
  20. type Result struct {
  21.     CompilationCode int
  22.     RunCode int
  23.     TestsPositive int
  24.     TestsTotal int
  25. }
  27. func upload(w http.ResponseWriter, r *http.Request) {
  28.     log.Println("method:", r.Method)
  29.     if r.Method == "POST" {
  30.         log.Println("Processing new SUBMISSION.")
  31.         // https://github.com/astaxie/build-web-application-with-golang/blob/master/de/04.5.md
  32.         r.ParseMultipartForm(32 << 20)
  33.         file, handler, err := r.FormFile("file")
  34.         if err != nil {
  35.             fmt.Println(err)
  36.             return
  37.         }
  39.         defer file.Close()
  40.         baseName:= os.Args[1]
  41.         f, err := os.OpenFile(baseName+handler.Filename, os.O_WRONLY|os.O_CREATE, 777)
  42.         if err != nil {
  43.             fmt.Println(err)
  44.             return
  45.         }
  46.         defer f.Close()
  47.         io.Copy(f, file)
  48.         if err != nil {
  49.             fmt.Println(err)
  50.             return
  51.         }
  53.         compilationCode, runCode, testsPositive, testsTotal := processWithDocker(baseName + handler.Filename, handler.Filename)
  55.         result := Result{
  56.             CompilationCode: compilationCode,
  57.             RunCode: runCode,
  58.             TestsPositive:testsPositive,
  59.             TestsTotal:testsTotal,
  60.         }
  61.         resultMarshaled, _ := json.Marshal(result)
  62.         w.Write(resultMarshaled)
  63.     } else {
  64.         w.Write([]byte("GO server is active. Use POST to submit your solution."))
  65.     }
  66. }
  68. // there is assumption that docker is installed where server.go is running
  69. // and the container is already pulled
  70. // TODO: handle situation when container is not pulled
  71. // TODO: somehow capture if compilation wasn't successful and
  72. // TODO: distinguish it from possible execution / time limit / memory limit error
  73. // http://stackoverflow.com/questions/18986943/in-golang-how-can-i-write-the-stdout-of-an-exec-cmd-to-a-file
  75. func processWithDocker(filenameWithDir string, filenameWithoutDir string) (int, int, int, int) {
  77.     ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  78.     defer cancel()
  79.     cli, err := client.NewEnvClient()
  80.     if err != nil {
  81.         panic(err)
  82.     }
  84.     var hostVolumeString = filenameWithDir
  85.     var hostConfigBindString = hostVolumeString  + ":/WORKING_FOLDER/" + filenameWithoutDir
  87.     var hostConfig = &container.HostConfig{
  88.         Binds: []string{hostConfigBindString},
  89.     }
  91.     resp, err := cli.ContainerCreate(ctx, &container.Config{
  92.         Image: "tusty53/ubuntu_c_runner:twelfth",
  93.         Env: []string{"F00=" + filenameWithoutDir},
  94.         Volumes: map[string]struct{}{
  95.             hostVolumeString: struct{}{},
  96.         },
  97.     }, hostConfig, nil, "")
  98.     if err != nil {
  99.         panic(err)
  100.     }
  102.     if err := cli.ContainerStart(ctx, resp.ID, dockertypes.ContainerStartOptions{}); err != nil {
  103.         panic(err)
  104.     }
  106.     fmt.Println(resp.ID)
  108.     var exited = false
  110.     for !exited {
  112.         json, err := cli.ContainerInspect(ctx, resp.ID)
  113.         if err != nil {
  114.             panic(err)
  115.         }
  117.         exited = json.State.Running
  119.         fmt.Println(json.State.Status)
  120.     }
  122.     normalOut, err := cli.ContainerLogs(ctx, resp.ID, dockertypes.ContainerLogsOptions{ShowStdout: true, ShowStderr: false})
  123.     if err != nil {
  124.         panic(err)
  125.     }
  127.     errorOut, err := cli.ContainerLogs(ctx, resp.ID, dockertypes.ContainerLogsOptions{ShowStdout: false, ShowStderr: true})
  128.     if err != nil {
  129.         panic(err)
  130.     }
  132.     buf := new(bytes.Buffer)
  133.     buf.ReadFrom(normalOut)
  134.     sOut := buf.String()
  136.     buf2 := new(bytes.Buffer)
  137.     buf2.ReadFrom(errorOut)
  138.     sErr := buf2.String()
  140.     log.Printf("start\n")
  141.     log.Printf(sOut)
  142.     log.Printf("end\n")
  144.     log.Printf("start error\n")
  145.     log.Printf(sErr)
  146.     log.Printf("end error\n")
  149.     var testsPositive=0
  150.     var testsTotal=0
  152.     if(sErr!=""){
  153.         return 0,0,0,0
  154.     }
  156.     if(sOut!=""){
  157.         fmt.Sscanf(sOut, "%d %d", &testsPositive, &testsTotal)
  158.         return 1,1,testsPositive,testsTotal
  159.     }
  160.     return 1,0,0,0
  162. }
  165. // Creates examine directory if it doesn't exist.
  166. // If examine directory already exists, then comes an error.
  167. func prepareDir() {
  168.     cmdMkdir := exec.Command("mkdir", os.Args[1])
  169.     errMkdir := cmdMkdir.Run()
  170.     if errMkdir != nil {
  171.         log.Println(errMkdir)
  172.     }
  173. }
  175. func main() {
  176.     prepareDir()
  177.     go http.HandleFunc("/submission", upload)
  178.     http.ListenAndServe(":8123", nil)
  179. }

Dockerfile

  1. FROM ubuntu
  2. ENV DEBIAN_FRONTEND noninteractive
  3. RUN apt-get update && \
  4.     apt-get -y install gcc
  5. COPY . /WORKING_FOLDER
  6. WORKDIR /WORKING_FOLDER
  7. CMD ["./chain"]

Chain文件

  1. #!/bin/bash
  2. gcc -Wall $F00  -o hello
  3. ./hello
  4. >&2 echo ""
英文:

I have a small project in which my go server copies the C files send by http into Docker containers, where they are compiled and run.
However, I am not able to obtain any data send to stdout in the container.

I have determined that file is sent into Docker container, what's more - any problems with compilation are shown on the error stream. However, sending data through stderr in C program also didn't show any results until I have, playing with Dockerfile, used '>&2 echo ""' which somehow pushed data through the stream and I was able to read it.

Right now, as mentioned above, I can only read stderr and solely thanks to a workaround. Any idea why I can't do it using standard methods?

Go server

  1. package main
  2. import (
  3. &quot;fmt&quot;
  4. &quot;net/http&quot;
  5. &quot;io&quot;
  6. &quot;os&quot;
  7. &quot;os/exec&quot;
  8. &quot;log&quot;
  9. &quot;encoding/json&quot;
  10. &quot;github.com/docker/docker/client&quot;
  11. dockertypes &quot;github.com/docker/docker/api/types&quot;
  12. &quot;github.com/docker/docker/api/types/container&quot;
  13. &quot;golang.org/x/net/context&quot;
  14. &quot;time&quot;
  15. &quot;bytes&quot;
  16. )
  17. type Result struct {
  18. CompilationCode int
  19. RunCode int
  20. TestsPositive int
  21. TestsTotal int
  22. }
  23. func upload(w http.ResponseWriter, r *http.Request) {
  24. log.Println(&quot;method:&quot;, r.Method)
  25. if r.Method == &quot;POST&quot; {
  26. log.Println(&quot;Processing new SUBMISSION.&quot;)
  27. // https://github.com/astaxie/build-web-application-with-golang/blob/master/de/04.5.md
  28. r.ParseMultipartForm(32 &lt;&lt; 20)
  29. file, handler, err := r.FormFile(&quot;file&quot;)
  30. if err != nil {
  31. fmt.Println(err)
  32. return
  33. }
  34. defer file.Close()
  35. baseName:= os.Args[1]
  36. f, err := os.OpenFile(baseName+handler.Filename, os.O_WRONLY|os.O_CREATE, 777)
  37. if err != nil {
  38. fmt.Println(err)
  39. return
  40. }
  41. defer f.Close()
  42. io.Copy(f, file)
  43. if err != nil {
  44. fmt.Println(err)
  45. return
  46. }
  47. compilationCode, runCode, testsPositive, testsTotal := processWithDocker(baseName + handler.Filename, handler.Filename)
  48. result := Result{
  49. CompilationCode: compilationCode,
  50. RunCode: runCode,
  51. TestsPositive:testsPositive,
  52. TestsTotal:testsTotal,
  53. }
  54. resultMarshaled, _ := json.Marshal(result)
  55. w.Write(resultMarshaled)
  56. } else {
  57. w.Write([]byte(&quot;GO server is active. Use POST to submit your solution.&quot;))
  58. }
  59. }
  60. // there is assumption that docker is installed where server.go is running
  61. // and the container is already pulled
  62. // TODO: handle situation when container is not pulled
  63. // TODO: somehow capture if compilation wasn&#39;t successful and
  64. // TODO: distinguish it from possible execution / time limit / memory limit error
  65. // http://stackoverflow.com/questions/18986943/in-golang-how-can-i-write-the-stdout-of-an-exec-cmd-to-a-file
  66. func processWithDocker(filenameWithDir string, filenameWithoutDir string) (int, int, int, int) {
  67. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  68. defer cancel()
  69. cli, err := client.NewEnvClient()
  70. if err != nil {
  71. panic(err)
  72. }
  73. var hostVolumeString = filenameWithDir
  74. var hostConfigBindString = hostVolumeString  + &quot;:/WORKING_FOLDER/&quot; + filenameWithoutDir
  75. var hostConfig = &amp;container.HostConfig{
  76. Binds: []string{hostConfigBindString},
  77. }
  78. resp, err := cli.ContainerCreate(ctx, &amp;container.Config{
  79. Image: &quot;tusty53/ubuntu_c_runner:twelfth&quot;,
  80. Env: []string{&quot;F00=&quot; + filenameWithoutDir},
  81. Volumes: map[string]struct{}{
  82. hostVolumeString: struct{}{},
  83. },
  84. }, hostConfig, nil, &quot;&quot;)
  85. if err != nil {
  86. panic(err)
  87. }
  88. if err := cli.ContainerStart(ctx, resp.ID, dockertypes.ContainerStartOptions{}); err != nil {
  89. panic(err)
  90. }
  91. fmt.Println(resp.ID)
  92. var exited = false
  93. for !exited {
  94. json, err := cli.ContainerInspect(ctx, resp.ID)
  95. if err != nil {
  96. panic(err)
  97. }
  98. exited = json.State.Running
  99. fmt.Println(json.State.Status)
  100. }
  101. normalOut, err := cli.ContainerLogs(ctx, resp.ID, dockertypes.ContainerLogsOptions{ShowStdout: true, ShowStderr: false})
  102. if err != nil {
  103. panic(err)
  104. }
  105. errorOut, err := cli.ContainerLogs(ctx, resp.ID, dockertypes.ContainerLogsOptions{ShowStdout: false, ShowStderr: true})
  106. if err != nil {
  107. panic(err)
  108. }
  109. buf := new(bytes.Buffer)
  110. buf.ReadFrom(normalOut)
  111. sOut := buf.String()
  112. buf2 := new(bytes.Buffer)
  113. buf2.ReadFrom(errorOut)
  114. sErr := buf2.String()
  115. log.Printf(&quot;start\n&quot;)
  116. log.Printf(sOut)
  117. log.Printf(&quot;end\n&quot;)
  118. log.Printf(&quot;start error\n&quot;)
  119. log.Printf(sErr)
  120. log.Printf(&quot;end error\n&quot;)
  121. var testsPositive=0
  122. var testsTotal=0
  123. if(sErr!=&quot;&quot;){
  124. return 0,0,0,0
  125. }
  126. if(sOut!=&quot;&quot;){
  127. fmt.Sscanf(sOut, &quot;%d %d&quot;, &amp;testsPositive, &amp;testsTotal)
  128. return 1,1,testsPositive,testsTotal
  129. }
  130. return 1,0,0,0
  131. }
  132. // Creates examine directory if it doesn&#39;t exist.
  133. // If examine directory already exists, then comes an error.
  134. func prepareDir() {
  135. cmdMkdir := exec.Command(&quot;mkdir&quot;, os.Args[1])
  136. errMkdir := cmdMkdir.Run()
  137. if errMkdir != nil {
  138. log.Println(errMkdir)
  139. }
  140. }
  141. func main() {
  142. prepareDir()
  143. go http.HandleFunc(&quot;/submission&quot;, upload)
  144. http.ListenAndServe(&quot;:8123&quot;, nil)
  145. }

Dockerfile

  1. FROM ubuntu
  2. ENV DEBIAN_FRONTEND noninteractive
  3. RUN apt-get update &amp;&amp; \
  4. apt-get -y install gcc
  5. COPY . /WORKING_FOLDER
  6. WORKDIR /WORKING_FOLDER
  7. CMD [&quot;./chain&quot;]

Chain file

  1. #!/bin/bash
  2. gcc -Wall $F00  -o hello
  3. ./hello
  4. &gt;&amp;2 echo &quot;&quot;

答案1

得分: 2

我相信以下方法可以用来获取正在运行的容器的stdout和stderr。

  1. import "github.com/docker/docker/pkg/stdcopy"

从Docker SDK中导入这个包。

  1. data, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
  2. if err != nil {
  3.     panic(err)
  4. }

获取正在运行的容器的日志并将其存储到data中。现在创建两个缓冲区来存储流。

  1. // Demultiplex stdout and stderror
  2. // from the container logs
  3. stdoutput := new(bytes.Buffer)
  4. stderror := new(bytes.Buffer)

现在使用导入的stdcopy将这两个流保存到缓冲区中。

  1. stdcopy.StdCopy(stdoutput, stderror, data)
  2. if err != nil {
  3.     panic(err)
  4. }

这样就可以获取到运行容器的stdout和stderr了。

英文:

I believe the following method can be used to obtain the stdout and stderr of running containers.

  1. import &quot;github.com/docker/docker/pkg/stdcopy&quot;

import this package from the docker SDK.

  1.     data, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
  2. if err != nil {
  3. panic(err)
  4. }

get the logs from the running container and store it to data.Now create two buffers to store the streams.

  1. 	// Demultiplex stdout and stderror
  2. // from the container logs
  3. stdoutput := new(bytes.Buffer)
  4. stderror := new(bytes.Buffer)

now use the imported stdcopy to save the two streams to the buffers.

  1. 	stdcopy.StdCopy(stdoutput, stderror, data)
  2. if err != nil {
  3. panic(err)
  4. }

答案2

得分: 1

你应该考虑将Docker CLI作为一个独立的进程运行，并只读取其stdout和stderr，例如：

  1. cmd := exec.Command("docker", "run", "-v=<volumes-to-mount>", "<image>")
  2. out, err := cmd.CombinedOutput()
  3. if err != nil {
  4.     return err
  5. }
  6. // `out` 现在包含了来自Docker的stdout和stderr的合并内容
英文:

You should consider running the Docker CLI as a separate process and just reading its stdout and stderr, for example:

  1. cmd := exec.Command(&quot;docker&quot;, &quot;run&quot;, &quot;-v=&lt;volumes-to-mount&gt;&quot;, &quot;&lt;image&gt;&quot;)
  2. out, err := cmd.CombinedOutput()
  3. if err != nil {
  4. return err
  5. }
  6. // `out` now contains the combined stdout / stderr from Docker

答案3

得分: 0

我对Go语言不太熟悉，但是这段代码似乎是从Docker日志流中读取数据。可能是因为你的Docker守护进程配置了不同的日志驱动程序。除了json-file和journald之外，docker logs命令不适用于其他驱动程序。你可以使用以下命令进行检查：

  1. $ docker info |grep 'Logging Driver'

你可以像下面这样在每个容器上更改日志驱动程序：

  1. $ docker run -it --log-driver json-file <image> <command>

在创建容器时，你也可以通过API传递这个参数。

英文:

I'm not familiar with the Go but the code seems to be reading from the docker log stream. It could be that your Docker daemon is configured to use a different log driver. The docker logs command is not available for drivers other than json-file and journald. You can check with the following command:

  1. $ docker info |grep &#39;Logging Driver&#39;

You can change the logging driver on a per container basis like so:

  1. $ docker run -it --log-driver json-file &lt;image&gt; &lt;command&gt;

You should be able to pass this parameter via the api when creating the container as well.

huangapple
  • 本文由 发表于 2017年9月19日 01:37:47
  • 转载请务必保留本文链接：https://go.coder-hub.com/46285216.html
  • c
  • docker
  • go
  • logging
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定