英文:
Switch between gzip and csv reader dynamically
问题
我正在使用以下代码片段来解压缩一个压缩的CSV文件并将其分割成均等的部分。
func split(feed MerchantFeed, db *gorm.DB) {
var merchant Merchant
db.First(&merchant, feed.MerchantID)
merchant.Title = strings.Replace(merchant.Title, ".", "_", -1)
fileID := merchant.Title + "__" + fmt.Sprint(feed.MerchantID) + "__" + uniuri.NewLen(20)
out, _ := os.Create("/tmp/feeds/" + fileID)
resp, err := http.Get(feed.Url)
if err != nil {
fmt.Println("Feed download error!")
}
defer resp.Body.Close()
// Calling Copy method with its parameters
bytes, err := io.Copy(out, resp.Body)
if err != nil {
fmt.Println("Could not create feed csv")
}
defer out.Close()
in, _ := os.Open("/tmp/feeds/" + fileID)
var reader io.Reader
if slices.Contains(zipMerchants, merchant.Network_ID) {
zipReader, err := zip.NewReader(in, bytes)
if err != nil {
fmt.Println("Could not unzip feed " + fmt.Sprint(feed.MerchantID))
return
}
reader = zipReader
} else {
reader = in
}
defer in.Close()
// Empty byte slice.
csvString := make([]byte, 100)
// Read in data.
_, errRead := reader.Read(csvString)
if errRead != nil {
fmt.Println(err)
}
path := "/tmp/split/feed__" + fileID + "__.csv"
csvfile, err := os.Create(path)
io.Copy(csvfile, reader)
if fileID != "" {
_, errx := splitter.Split(path, splitDir)
if errx != nil {
fmt.Println(errx)
}
defer csvfile.Close()
//fmt.Println(result)
result := os.Remove(path)
if result != nil {
fmt.Println(result)
}
}
fmt.Printf("Size in MB (Feed %d): %d\n", feed.MerchantID, bytes)
if bytes == 64 {
fmt.Println("could not download feed from " + fmt.Sprint(feed.MerchantID))
e := os.Remove("/tmp/feeds/" + fileID + ".gz")
if e != nil {
log.Fatal(e)
}
return
}
}
到目前为止一切都很好，它都正常工作。但是有些商家将他们的CSV文件打包成zip格式，而其他商家则没有。所以我的想法是在gzip.NewReader()和csv.NewReader()之间动态切换，但我不知道正确的方法。我希望尽可能少地重复代码。所以我尝试了以下代码：
var zipNetworks []int
zipNetworks = append(zipNetworks, 2)
var reader io.Reader
if slices.Contains(zipMerchants, merchant.Network_ID) {
zipReader, err := zip.NewReader(in, bytes)
if err != nil {
fmt.Println("Could not unzip feed " + fmt.Sprint(feed.MerchantID))
return
}
reader = zipReader
} else {
reader = in
}
但是我遇到了作用域的问题。有没有我不知道的聪明解决方案？
英文:
i am using the following code snippet to unpack a zipped csv file and split it into even parts.
func split(feed MerchantFeed, db *gorm.DB) {
var merchant Merchant
db.First(&merchant, feed.MerchantID)
merchant.Title = strings.Replace(merchant.Title, ".", "_", -1)
fileID := merchant.Title + "__" + fmt.Sprint(feed.MerchantID) + "__" + uniuri.NewLen(20)
out, _ := os.Create("/tmp/feeds/" + fileID)
resp, err := http.Get(feed.Url)
if err != nil {
fmt.Println("Feed download error!")
}
defer resp.Body.Close()
// Calling Copy method with its parameters
bytes, err := io.Copy(out, resp.Body)
if err != nil {
fmt.Println("Could not create feed csv")
}
defer out.Close()
in, _ := os.Open("/tmp/feeds/" + fileID)
reader, err := gzip.NewReader(in)
if err != nil {
fmt.Println("Could not unzip feed " + fmt.Sprint(feed.MerchantID))
err.Error()
return
}
defer in.Close()
// Empty byte slice.
csvString := make([]byte, 100)
// Read in data.
_, errRead := reader.Read(csvString)
if errRead != nil {
fmt.Println(err)
}
path := "/tmp/split/feed__" + fileID + "__.csv"
csvfile, err := os.Create(path)
io.Copy(csvfile, reader)
if fileID != "" {
_, errx := splitter.Split(path, splitDir)
if errx != nil {
fmt.Println(errx)
}
defer csvfile.Close()
//fmt.Println(result)
result := os.Remove(path)
if result != nil {
fmt.Println(result)
}
}
fmt.Printf("Size in MB (Feed %d): %d\n", feed.MerchantID, bytes)
if bytes == 64 {
fmt.Println("could not download feed from " + fmt.Sprint(feed.MerchantID))
e := os.Remove("/tmp/feeds/" + fileID + ".gz")
if e != nil {
log.Fatal(e)
}
return
}
}
So far so good. It's all working.
But some Merchants pack their csv as zip, and others don't. So my idea is to dynamically switch between gzip.NewReader() and csv.NewReader() but i don't know the proper way to do that. I want to repeat as few code as possible. So i tried to say
var zipNetworks []int
zipNetworks = append(zipNetworks, 2)
if slices.Contains(zipMerchants,merchant.Network_ID) {
reader := zip.NewReader(in)
} else {
reader := csv.NewReader(in)
}
but then I have problems with the scope.
Is there any smart solution around that I don't know?
答案1
得分: 1
你可以使用常见的io.Reader接口。
var reader io.Reader
if slices.Contains(zipMerchants, merchant.Network_ID) {
reader = zip.NewReader(in)
} else {
reader = csv.NewReader(in)
}
英文:
You can use the common io.Reader interface.
var reader io.Reader
if slices.Contains(zipMerchants, merchant.Network_ID) {
reader = zip.NewReader(in)
} else {
reader = csv.NewReader(in)
}
答案2
得分: 0
将读取器设置为文件。如果使用gzip压缩，则将读取器替换为围绕文件的gzip读取器：
var reader io.Reader = in
if shouldUseGzip {
reader, err = gzip.NewReader(reader)
if err != nil {
fmt.Println("无法解压缩 feed " + fmt.Sprint(feed.MerchantID), err)
return
}
}
英文:
Set the reader to the file. If gzipping, then replace reader with gzip reader around the file:
var reader io.Reader = in
if shouldUseGzip {
reader, err = gzip.NewReader(reader)
if err != nil {
fmt.Println("Could not unzip feed " + fmt.Sprint(feed.MerchantID), err)
return
}
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库，让每个人都能够通过互相帮助和分享经验来进步。
评论