How to import data to elasticsearch using golang

huangapple
go评论140阅读模式
英文:

How to import data to elasticsearch using golang

问题

我正在使用gopkg.in/olivere/elastic.v5，并尝试使用golang将数据从json文件导入到elasticsearch数据库。这是我的代码：

  1. package main
  3. import (
  4.     "gopkg.in/olivere/elastic.v5"
  5.     "golang.org/x/net/context"
  6.     "log"
  7.     "os"
  8.     "encoding/json"
  9. )
  11. type people struct {
  12.     Firstname    string `json:"firstname"`
  13.     Lastname     string `json:"lastname"`
  14.     Institution  string `json:"institution"`
  15.     Email        string `json:"email"`
  16. }
  18. type item struct {
  19.     Id        string   `json:"id"`
  20.     Title     string   `json:"title"`
  21.     Journal   string   `json:"journal"`
  22.     Volume    int      `json:"volume"`
  23.     Number    int      `json:"number"`
  24.     Pages     string   `json:"pages"`
  25.     Year      int      `json:"year"`
  26.     Authors   []people `json:"authors"`
  27.     Abstract  string   `json:"abstract"`
  28.     Link      string   `json:"link"`
  29.     Keywords  []string `json:"keywords"`
  30.     Body      string   `json:"body"`
  31. }
  33. var client *elastic.Client
  34. var err error
  36. func init() {
  37.     client, err = elastic.NewClient()
  38.     if err != nil {
  39.         log.Fatal(err)
  40.     }
  41. }
  43. func main() {
  44.     var data []item
  46.     file, err := os.Open("data.json")
  47.     if err != nil {
  48.         log.Fatal(err)
  49.     }
  50.     defer file.Close()
  52.     jsonDecoder := json.NewDecoder(file)
  53.     if err := jsonDecoder.Decode(&data); err != nil {
  54.         log.Fatal("Decode: ", err)
  55.     }
  57.     bulkIndex("library", "article", data)
  58. }
  60. func bulkIndex(index string, typ string, data []item) {
  61.     ctx := context.Background()
  62.     for _, item := range data {
  63.         _, err := client.Index().Index(index).Type(typ).BodyJson(item).Do(ctx)
  64.         if err != nil {
  65.             log.Fatal(err)
  66.         }
  67.     }
  68. }

这个代码可以编译通过，但是在运行后，当我使用GET /library/article/575084573a2404eec25acdcd?pretty（575084573a2404eec25acdcd是来自我的json文件的正确id）在kibana上检查我的elasticsearch数据库时，我得到以下响应：

  1. {
  2.   "_index": "library",
  3.   "_type": "article",
  4.   "_id": "575084573a2404eec25acdcd",
  5.   "found": false
  6. }

我该如何导入我的数据？

编辑：在kibana上执行GET /library?pretty后，我得到以下结果：

  1. {
  2.   "library": {
  3.     "aliases": {},
  4.     "mappings": {
  5.       "article": {
  6.         "properties": {
  7.           "abstract": {
  8.             "type": "text",
  9.             "fields": {
  10.               "keyword": {
  11.                 "type": "keyword",
  12.                 "ignore_above": 256
  13.               }
  14.             }
  15.           },
  16.           "authors": {
  17.             "properties": {
  18.               "email": {
  19.                 "type": "text",
  20.                 "fields": {
  21.                   "keyword": {
  22.                     "type": "keyword",
  23.                     "ignore_above": 256
  24.                   }
  25.                 }
  26.               },
  27.               "firstname": {
  28.                 "type": "text",
  29.                 "fields": {
  30.                   "keyword": {
  31.                     "type": "keyword",
  32.                     "ignore_above": 256
  33.                   }
  34.                 }
  35.               },
  36.               "institution": {
  37.                 "type": "text",
  38.                 "fields": {
  39.                   "keyword": {
  40.                     "type": "keyword",
  41.                     "ignore_above": 256
  42.                   }
  43.                 }
  44.               },
  45.               "lastname": {
  46.                 "type": "text",
  47.                 "fields": {
  48.                   "keyword": {
  49.                     "type": "keyword",
  50.                     "ignore_above": 256
  51.                   }
  52.                 }
  53.               }
  54.             }
  55.           },
  56.           "body": {
  57.             "type": "text",
  58.             "fields": {
  59.               "keyword": {
  60.                 "type": "keyword",
  61.                 "ignore_above": 256
  62.               }
  63.             }
  64.           },
  65.           "id": {
  66.             "type": "text",
  67.             "fields": {
  68.               "keyword": {
  69.                 "type": "keyword",
  70.                 "ignore_above": 256
  71.               }
  72.             }
  73.           },
  74.           "journal": {
  75.             "type": "text",
  76.             "fields": {
  77.               "keyword": {
  78.                 "type": "keyword",
  79.                 "ignore_above": 256
  80.               }
  81.             }
  82.           },
  83.           "keywords": {
  84.             "type": "text",
  85.             "fields": {
  86.               "keyword": {
  87.                 "type": "keyword",
  88.                 "ignore_above": 256
  89.               }
  90.             }
  91.           },
  92.           "link": {
  93.             "type": "text",
  94.             "fields": {
  95.               "keyword": {
  96.                 "type": "keyword",
  97.                 "ignore_above": 256
  98.               }
  99.             }
  100.           },
  101.           "number": {
  102.             "type": "long"
  103.           },
  104.           "pages": {
  105.             "type": "text",
  106.             "fields": {
  107.               "keyword": {
  108.                 "type": "keyword",
  109.                 "ignore_above": 256
  110.               }
  111.             }
  112.           },
  113.           "title": {
  114.             "type": "text",
  115.             "fields": {
  116.               "keyword": {
  117.                 "type": "keyword",
  118.                 "ignore_above": 256
  119.               }
  120.             }
  121.           },
  122.           "volume": {
  123.             "type": "long"
  124.           },
  125.           "year": {
  126.             "type": "long"
  127.           }
  128.         }
  129.       }
  130.     },
  131.     "settings": {
  132.       "index": {
  133.         "creation_date": "1486063182258",
  134.         "number_of_shards": "5",
  135.         "number_of_replicas": "1",
  136.         "uuid": "_SLeDWb4QPinFcSwOCUtCw",
  137.         "version": {
  138.           "created": "5020099"
  139.         },
  140.         "provided_name": "library"
  141.       }
  142.     }
  143.   }
  144. }
英文:

I am using gopkg.in/olivere/elastic.v5 and I am trying to import data from a json file to elasticsearch DB using golang. This is my code

  1. package main
  2. import(
  3. "gopkg.in/olivere/elastic.v5"
  4. "golang.org/x/net/context"
  5. "log"
  6. "os"
  7. "encoding/json"
  8. )
  9. type people struct{
  10. Firstname string `json:"firstname"`
  11. Lastname string `json:"lastname"`
  12. Institution string `json:"institution"`
  13. Email string `json:"email"`
  14. }
  15. type item struct{
  16. Id string `json:"id"`
  17. Title string `json:"title"`
  18. Journal  string `json:"journal"`
  19. Volume int `json:"volume"`
  20. Number int `json:"number"`
  21. Pages string `json:"pages"`
  22. Year int `json:"year"`
  23. Authors []people `json:"authors"`
  24. Abstract string `json:"abstract"`
  25. Link string `json:"link"`
  26. Keywords []string `json:"keywords"`
  27. Body string `json:"body"`
  28. }
  29. var client *elastic.Client
  30. var err error
  31. func init(){
  32. client,err = elastic.NewClient()
  33. if err!=nil{
  34. log.Fatal(err)
  35. }
  36. }
  37. func main() {
  38. var data []item
  39. file,err := os.Open("data.json")
  40. if err!=nil{
  41. log.Fatal(err)
  42. }
  43. defer file.Close()
  44. jsonDeocder := 	json.NewDecoder(file)
  45. if err := jsonDeocder.Decode(&data); err!=nil{
  46. log.Fatal("Decode: ",err)
  47. }
  48. bulkIndex("library","article",data)
  49. }
  50. func bulkIndex(index string,typ string ,data []item){
  51. ctx := context.Background()
  52. for _,item := range data{
  53. _,err := client.Index().Index(index).Type(typ).BodyJson(item).Do(ctx)	
  54. if err !=nil{
  55. log.Fatal(err)
  56. }
  57. }	
  58. }

The package documentation is huge and I am not sure if I have gone the right way. This compiles fine but after running this, when I check my elasticsearch DB on kibana using GET /library/article/575084573a2404eec25acdcd?pretty (575084573a2404eec25acdcd is the correct id from my json file), I am getting the following response

  1. {
  2. "_index": "library",
  3. "_type": "article",
  4. "_id": "575084573a2404eec25acdcd",
  5. "found": false
  6. }

How do I import my data?

EDIT: This is what I get on doing GET /library?pretty on kibana

  1. {
  2. "library": {
  3. "aliases": {},
  4. "mappings": {
  5. "article": {
  6. "properties": {
  7. "abstract": {
  8. "type": "text",
  9. "fields": {
  10. "keyword": {
  11. "type": "keyword",
  12. "ignore_above": 256
  13. }
  14. }
  15. },
  16. "authors": {
  17. "properties": {
  18. "email": {
  19. "type": "text",
  20. "fields": {
  21. "keyword": {
  22. "type": "keyword",
  23. "ignore_above": 256
  24. }
  25. }
  26. },
  27. "firstname": {
  28. "type": "text",
  29. "fields": {
  30. "keyword": {
  31. "type": "keyword",
  32. "ignore_above": 256
  33. }
  34. }
  35. },
  36. "institution": {
  37. "type": "text",
  38. "fields": {
  39. "keyword": {
  40. "type": "keyword",
  41. "ignore_above": 256
  42. }
  43. }
  44. },
  45. "lastname": {
  46. "type": "text",
  47. "fields": {
  48. "keyword": {
  49. "type": "keyword",
  50. "ignore_above": 256
  51. }
  52. }
  53. }
  54. }
  55. },
  56. "body": {
  57. "type": "text",
  58. "fields": {
  59. "keyword": {
  60. "type": "keyword",
  61. "ignore_above": 256
  62. }
  63. }
  64. },
  65. "id": {
  66. "type": "text",
  67. "fields": {
  68. "keyword": {
  69. "type": "keyword",
  70. "ignore_above": 256
  71. }
  72. }
  73. },
  74. "journal": {
  75. "type": "text",
  76. "fields": {
  77. "keyword": {
  78. "type": "keyword",
  79. "ignore_above": 256
  80. }
  81. }
  82. },
  83. "keywords": {
  84. "type": "text",
  85. "fields": {
  86. "keyword": {
  87. "type": "keyword",
  88. "ignore_above": 256
  89. }
  90. }
  91. },
  92. "link": {
  93. "type": "text",
  94. "fields": {
  95. "keyword": {
  96. "type": "keyword",
  97. "ignore_above": 256
  98. }
  99. }
  100. },
  101. "number": {
  102. "type": "long"
  103. },
  104. "pages": {
  105. "type": "text",
  106. "fields": {
  107. "keyword": {
  108. "type": "keyword",
  109. "ignore_above": 256
  110. }
  111. }
  112. },
  113. "title": {
  114. "type": "text",
  115. "fields": {
  116. "keyword": {
  117. "type": "keyword",
  118. "ignore_above": 256
  119. }
  120. }
  121. },
  122. "volume": {
  123. "type": "long"
  124. },
  125. "year": {
  126. "type": "long"
  127. }
  128. }
  129. }
  130. },
  131. "settings": {
  132. "index": {
  133. "creation_date": "1486063182258",
  134. "number_of_shards": "5",
  135. "number_of_replicas": "1",
  136. "uuid": "_SLeDWb4QPinFcSwOCUtCw",
  137. "version": {
  138. "created": "5020099"
  139. },
  140. "provided_name": "library"
  141. }
  142. }
  143. }
  144. }

答案1

得分: 2

好的，以下是翻译好的内容：

好的，我明白了。我应该指定我的项目的Id，而不仅仅指定索引和类型。

正确的语句应该是：

  1. _, err := client.Index().Index(index).Type(typ).Id(item.Id).BodyJson(item).Do(ctx)
英文:

Ok, I got it. I should have specified the Id for my item as well instead of just specifying the index and type.

The correct statement should be

  1. _,err := client.Index().Index(index).Type(typ).Id(item.Id).BodyJson(item).Do(ctx)

huangapple
  • 本文由 发表于 2017年2月3日 03:40:23
  • 转载请务必保留本文链接：https://go.coder-hub.com/42010937.html
  • elasticsearch
  • go
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定