GO: Confluence API无法获取所有附件。

huangapple
go评论113阅读模式
英文:

GO: Confluence API not getting all the attachments

问题

我正在使用golang开发我的应用程序，在这个应用程序中，我尝试从Confluence获取附件。以下是详细信息：

  1. req := "https://domain.atlassian.net/wiki/rest/api/content?expand=body.view,version&type=page&start=0&limit="
  2. res, err := w.sendRequest(req)
  3. if err != nil {
  4.     return nil, err
  5. }
  6. if strings.EqualFold(contentID, "") == false {
  7.     if len(res.Results) != 0 {
  8.         for i, _ := range res.Results {
  9.             Log.Info("files processed is:", i)
  10.             extension := filepath.Ext(res.Results[i].Title)
  11.             isExtenstionExclude := isExcludedExtenstion(sbConfig, extension)
  12.             ispathExclude := isExcludedFolder(sbConfig, res.Results[i].Links.Webui)
  13.             if sbgoclient.ExtesionMap[extension] == 0 || isExtenstionExclude == true || ispathExclude == true {
  14.                 binarycount++
  15.                 Log.Info("Excluded by extension" + extension + " for file" + res.Results[i].Title)
  16.             } else {
  17.                 md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
  18.                 md5HashInString := hex.EncodeToString(md5HashInBytes[:])
  19.                 file_path := parameter[0] + "/" + md5HashInString + strings.Replace(res.Results[i].Title, " ", "", -1)
  20.                 file, err := os.Create(file_path)
  22.                 if err != nil {
  23.                     fmt.Println(err)
  24.                     panic(err)
  25.                 }
  26.                 url_1 := sbConfig.ConfluenceUrl + res.Results[i].Links.Download
  27.                 req, err := http.NewRequest("GET", url_1, nil)
  28.                 resp, _ := w.client.Do(req) // add a filter to check redirect
  30.                 if err != nil {
  31.                     fmt.Println(err)
  32.                     panic(err)
  33.                 }
  34.                 // Close body on function exit
  35.                 defer resp.Body.Close()
  36.                 fmt.Println(resp.Status)
  38.                 size, err = io.Copy(file, resp.Body)
  40.                 if err != nil {
  41.                     panic(err)
  42.                 }
  44.                 defer file.Close()
  45.                 fmt.Printf("%s with %v bytes downloaded", res.Results[i].Title, size)
  46.                 meta := map[string]string{
  47.                     "size": strconv.FormatInt(size, 10),
  48.                 }
  49.             }
  50.         }
  51.     }
  52. } else {
  53.     if len(res.Results) != 0 {
  54.         for i, _ := range res.Results {
  55.             Log.Info("page indexing is", res.Results[i].Title, "and i value is:", i)
  56.             fmt.Println("hmtl content is", res.Results[i].Body.View.Value)
  57.             fmt.Println("page name is:", res.Results[i].Title)
  58.             md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
  59.             md5HashInString := hex.EncodeToString(md5HashInBytes[:])
  60.             file_path := parameter[0] + "/" + md5HashInString + strings.Replace(res.Results[i].Title, " ", "", -1) + ".html"
  61.             file, err := os.Create(file_path)
  63.             if err != nil {
  64.                 fmt.Println(err)
  65.                 panic(err)
  66.             }
  67.             defer file.Close()
  68.             html_content := "<html><body>" + res.Results[i].Body.View.Value + "</body></html>"
  69.             err = ioutil.WriteFile(file.Name(), []byte(html_content), 0777)
  70.             if err != nil {
  71.                 fmt.Println("error writing into file", err)
  72.                 panic(err)
  73.             }
  74.             file.Close()
  75.         }
  76.     }
  77. }
  1. func (w *Wiki) sendRequest(req *http.Request) (*vijay_content, error) {
  2.     var testjson vijay_content
  3.     req.Header.Add("Accept", "application/json, */*")
  4.     w.authMethod.auth(req)
  6.     resp, err := w.client.Do(req)
  7.     if err != nil {
  8.         return nil, err
  9.     }
  10.     bodyBytes, _ := ioutil.ReadAll(resp.Body)
  11.     body := string(bodyBytes)
  13.     fmt.Printf("response is %s\n", body)
  14.     err = json.Unmarshal(bodyBytes, &testjson)
  15.     if err != nil {
  16.         fmt.Println("error here is", err)
  17.         return nil, err
  18.     }
  20.     switch resp.StatusCode {
  21.     case http.StatusOK, http.StatusCreated, http.StatusPartialContent:
  22.         return &testjson, nil
  23.     case http.StatusNoContent, http.StatusResetContent:
  24.         return nil, nil
  25.     case http.StatusUnauthorized:
  26.         return nil, fmt.Errorf("Authentication failed.")
  27.     case http.StatusServiceUnavailable:
  28.         return nil, fmt.Errorf("Service is not available (%s).", resp.Status)
  29.     case http.StatusInternalServerError:
  30.         return nil, fmt.Errorf("Internal server error: %s", resp.Status)
  31.     }
  33.     return nil, fmt.Errorf("Unknown response status %s", resp.Status)
  34. }

在这个Confluence域中，我有1000多个文档，但我只能下载大约80到90个，我不知道发生了什么，请建议是否需要进行任何更改。

以下是用于从响应JSON中获取值的结构体：

  1. type Links struct {
  2.     Download string `json:"download,omitempty"`
  3.     Self     string `json:"self,omitempty"`
  4.     Webui    string `json:"webui,omitempty"`
  5. }
  6. type View_struct struct {
  7.     Value string `json:",innerxml"`
  8. }
  9. type Body_struct struct {
  10.     View View_struct `json:"view,omitempty"`
  11. }
  12. type Vijay_Results struct {
  13.     ID    string      `json:"id,omitempty"`
  14.     Links Links       `json:"_links,omitempty"`
  15.     Title string      `json:"title,omitempty"`
  16.     Body  Body_struct `json:"body,omitempty"`
  17. }
  18. type vijay_content struct {
  19.     Results []Vijay_Results `json:"results,omitempty"`
  20.     Start   int             `json:"start,omitempty"`
  21.     Limit   int             `json:"limit,omitempty"`
  22.     Size    int             `json:"size,omitempty"`
  23. }

请注意，我只提供了翻译的代码部分，不包括其他内容。

英文:

i am using golang for my application and in this application i tried getting attachments from Confluence, following are detail

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-golang-->

  1. req:=&quot;https://domain.atlassian.net/wiki/rest/api/content?expand=body.view,version&amp;type=page&amp;start=0&amp;limit=&quot;
  2. res, err := w.sendRequest(req)
  3. if err != nil {
  4. return nil, err
  5. }
  6. if strings.EqualFold(contentID, &quot;&quot;) == false {
  7. if len(res.Results) != 0 {
  8. for i, _ := range res.Results {
  9. Log.Info(&quot;files processed is:&quot;, i)
  10. extension := filepath.Ext(res.Results[i].Title)
  11. isExtenstionExclude := isExcludedExtenstion(sbConfig, extension)
  12. ispathExclude := isExcludedFolder(sbConfig, res.Results[i].Links.Webui)
  13. if sbgoclient.ExtesionMap[extension] == 0 || isExtenstionExclude == true || ispathExclude == true {
  14. binarycount++
  15. Log.Info(&quot;Excluded by extension&quot; + extension + &quot; for file&quot; + res.Results[i].Title)
  16. } else {
  17. md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
  18. md5HashInString := hex.EncodeToString(md5HashInBytes[:])
  19. file_path := parameter[0] + &quot;/&quot; + md5HashInString + strings.Replace(res.Results[i].Title, &quot; &quot;, &quot;&quot;, -1)
  20. file, err := os.Create(file_path)
  21. if err != nil {
  22. fmt.Println(err)
  23. panic(err)
  24. }
  25. url_1 := sbConfig.ConfluenceUrl + res.Results[i].Links.Download
  26. req, err := http.NewRequest(&quot;GET&quot;, url_1, nil)
  27. resp, _ := w.client.Do(req) // add a filter to check redirect
  28. if err != nil {
  29. fmt.Println(err)
  30. panic(err)
  31. }
  32. // Close body on function exit
  33. defer resp.Body.Close()
  34. fmt.Println(resp.Status)
  35. size, err = io.Copy(file, resp.Body)
  36. if err != nil {
  37. panic(err)
  38. }
  39. defer file.Close()
  40. fmt.Printf(&quot;%s with %v bytes downloaded&quot;, res.Results[i].Title, size)
  41. meta := map[string]string{
  42. &quot;size&quot;: strconv.FormatInt(size, 10),
  43. }
  44. }
  45. }
  46. }
  47. } else {
  48. if len(res.Results) != 0 {
  49. for i, _ := range res.Results {
  50. Log.Info(&quot;page indexing is&quot;, res.Results[i].Title, &quot;and i value is:&quot;, i)
  51. fmt.Println(&quot;hmtl content is&quot;, res.Results[i].Body.View.Value)
  52. fmt.Println(&quot;page name is:&quot;, res.Results[i].Title)
  53. md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
  54. md5HashInString := hex.EncodeToString(md5HashInBytes[:])
  55. file_path := parameter[0] + &quot;/&quot; + md5HashInString + strings.Replace(res.Results[i].Title, &quot; &quot;, &quot;&quot;, -1) + &quot;.html&quot;
  56. file, err := os.Create(file_path)
  57. if err != nil {
  58. fmt.Println(err)
  59. panic(err)
  60. }
  61. defer file.Close()
  62. html_content := &quot;&lt;html&gt;&lt;body&gt;&quot; + res.Results[i].Body.View.Value + &quot;&lt;/body&gt;&lt;/html&gt;&quot;
  63. err = ioutil.WriteFile(file.Name(), []byte(html_content), 0777)
  64. if err != nil {
  65. fmt.Println(&quot;error writing into file&quot;, err)
  66. panic(err)
  67. }
  68. file.Close()    				
  69. }

<!-- end snippet -->

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-golang -->

  1. func (w *Wiki) sendRequest(req *http.Request) (*vijay_content, error) {
  2. var testjson vijay_content
  3. req.Header.Add(&quot;Accept&quot;, &quot;application/json, */*&quot;)
  4. w.authMethod.auth(req)
  5. resp, err := w.client.Do(req)
  6. if err != nil {
  7. return nil, err
  8. }
  9. bodyBytes, _ := ioutil.ReadAll(resp.Body)
  10. body := string(bodyBytes)
  11. fmt.Printf(&quot;response is %s\n&quot;, body)
  12. err = json.Unmarshal(bodyBytes, &amp;testjson)
  13. if err != nil {
  14. fmt.Println(&quot;error here is&quot;, err)
  15. return nil, err
  16. }
  17. switch resp.StatusCode {
  18. case http.StatusOK, http.StatusCreated, http.StatusPartialContent:
  19. return &amp;testjson, nil
  20. case http.StatusNoContent, http.StatusResetContent:
  21. return nil, nil
  22. case http.StatusUnauthorized:
  23. return nil, fmt.Errorf(&quot;Authentication failed.&quot;)
  24. case http.StatusServiceUnavailable:
  25. return nil, fmt.Errorf(&quot;Service is not available (%s).&quot;, resp.Status)
  26. case http.StatusInternalServerError:
  27. return nil, fmt.Errorf(&quot;Internal server error: %s&quot;, resp.Status)
  28. }
  29. return nil, fmt.Errorf(&quot;Unknown response status %s&quot;, resp.Status)
  30. }

<!-- end snippet -->

and here in this confluence domain actually i have more than 1000 documents but i am able to download only around 80 to 90, i don't know whats happening here please suggest any changes to be done
and following is the struct used to get values from response json

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-golang -->

  1. type Links struct {
  2. Download string `json:&quot;download,omitempty&quot;`
  3. Self     string `json:&quot;self,omitempty&quot;`
  4. Webui    string `json:&quot;webui,omitempty&quot;`
  5. }
  6. type View_struct struct {
  7. Value string `json:&quot;,innerxml&quot;`
  8. }
  9. type Body_struct struct {
  10. View View_struct `json:&quot;view,omitempty&quot;`
  11. }
  12. type Vijay_Results struct {
  13. ID    string      `json:&quot;id,omitempty&quot;`
  14. Links Links       `json:&quot;_links,omitempty&quot;`
  15. Title string      `json:&quot;title,omitempty&quot;`
  16. Body  Body_struct `json:&quot;body,omitempty&quot;`
  17. }
  18. type vijay_content struct {
  19. Results []Vijay_Results `json:&quot;results,omitempty&quot;`
  20. Start   int             `json:&quot;start,omitempty&quot;`
  21. Limit   int             `json:&quot;limit,omitempty&quot;`
  22. Size    int             `json:&quot;size,omitempty&quot;`
  23. }

<!-- end snippet -->

答案1

得分: 1

API对结果进行分页。您应该通过指定start和limit来进行多次请求以获取整个列表。

例如，使用start=0&amp;limit=30请求前30个文档的列表，然后使用start=30&amp;limit=30请求接下来的30个文档，以此类推，直到收到一个空列表的响应。

您可以在分页文档中阅读更多详细信息。

英文:

The API paginates the results. You should fetch the whole list in multiple requests by specifying start and limit.

E.g. request the list of first 30 documents with start=0&amp;limit=30, then the next 30 with start=30&amp;limit=30, and so on, until you get a response with empty list.

You can read more details in the docs on pagination.

huangapple
  • 本文由 发表于 2017年6月30日 16:35:55
  • 转载请务必保留本文链接：https://go.coder-hub.com/44842132.html
  • api
  • confluence
  • confluence-rest-api
  • go
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定