英文:
I cannot print data side by side in JSON array in Golang Colly
问题
我正在使用golang colly从亚马逊上爬取图片,并希望将这些图片以JSON格式放入一个单独的数组中(每个产品图片只有一个数组)。我已经成功爬取到所需的图片,只是在JSON文件方面遇到了问题。非常感谢您的帮助。
以下是您的代码的翻译结果:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
AllInfos := make([]Info, 0)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// 获取图片
Counter := 0
var info Info
var theArray [10]string
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
theArray[Counter] = imgsrc
fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
Counter = Counter + 1
co.Visit(e.Request.AbsoluteURL(imgsrc))
info = Info{
Images: []string{
theArray[0],
theArray[1],
theArray[2],
theArray[3],
theArray[4],
theArray[5],
theArray[6],
},
}
AllInfos = append(AllInfos, info)
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(AllInfos)
writeJson(AllInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
您的代码的JSON结果如下:
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
""
]
}
]
您需要的JSON结果如下:
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg"
]
}
]
英文:
I'm taking pictures with golang colly from Amazon and I want to throw these pictures in JSON into a single array (only 1 array for each product images). I scraped the pictures I need, I'm just having a problem with the JSON file. Thank you very much in advance.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
AllInfos := make([]Info, 0)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
Counter := 0
var info Info
var theArray [10]string
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
theArray[Counter] = imgsrc
fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
Counter = Counter + 1
co.Visit(e.Request.AbsoluteURL(imgsrc))
info = Info{
Images: []string{
theArray[0],
theArray[1],
theArray[2],
theArray[3],
theArray[4],
theArray[5],
theArray[6],
},
}
AllInfos = append(AllInfos, info)
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(AllInfos)
writeJson(AllInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
My code's JSON result;
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
""
]
}
]
I need JSON result like this;
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
]
}
]
答案1
得分: 2
我很难理解你想要做什么,但是这是我想出来的代码:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := new(Info)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// 获取图片
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v\n", len(allInfos.Images), imgsrc)
allInfos.Images = append(allInfos.Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data *Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
根据@Flimzy的建议,我只将新的imagesrc
附加到了一个Info结构的实例中。看起来你期望的输出是一个长度为1的切片,其中包含所有的图片。
我还使用了strings.HasPrefix
来简化你的切片检查。这是等效的,但我认为它更清晰,所以这是一个可选的更改。
如果你有更多问题,请告诉我。
编辑,如果你需要继续使用切片,下面的代码可能会起作用。根据你的问题,很难确定何时需要清除一个新的Info:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := make([]Info, 1)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// 获取图片
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v\n", len(allInfos[0].Images), imgsrc)
allInfos[0].Images = append(allInfos[0].Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
希望能对你有所帮助!
英文:
I had a hard time understanding what you wanted to do but here's what I came up with:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := new(Info)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v", len(allInfos.Images), imgsrc+"\n")
allInfos.Images = append(allInfos.Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data *Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
As @Flimzy suggested, I appended only the new imagesrc
to a since instance of an info struct. Seeing as it seemed like your expected output was a slice of length 1 with all the images.
I also used a strings.HasPrefix
to clean up your slice check. This is equivalent but I think it's cleaner, so that's an optional change.
Let me know if you have any more questions.
EDIT, if you need to keep using a slice something like the following might work. From your question it's hard to tell when you need to clear a new Info:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := make([]Info, 1)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
allInfos[0].Images = append(allInfos[0].Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论