Objects in go getting replaced

I'm learning go by writing a web spider. I'm trying to get a list of all the business categories from allpages.com.

Below is my entire program. Unfortunately I can't isolate the issue so I've pasted it all.

If you run this program, you'll see that first of all it correctly downloads the first page, and adds all the extracted categories to the list of categories.

However, when it then downloads subsequent pages, it seems to mess up the reference to the parent category. E.g. it incorrectly calculates the URL http://www.allpages.com/travel-tourism/political-ideological-organizations/, when in fact political-ideological-organizations/ is not a subcategory of travel-tourism/. Digging through the logs it seems to overwrite the data in the parent object. The error is more pronounced the more workers there are.

This was working a bit better before I started passing data by reference to the goroutine, but I had essentially the same issue.

I've got several questions:

1. How can I debug this without resorting to picking through log lines?

2. What's wrong/why isn't it working and how can it be fixed?

    package main
   import (
   "fmt"
   "github.com/PuerkitoBio/goquery"
   "log"
   "strconv"
   "strings"
   "regexp"
   )
   const domain = "http://www.allpages.com/"
   const categoryPage = "category.html"
   type Category struct {
   url string
   level uint
   name string
   entries int
   parent *Category
   }
   type DownloadResult struct {
   doc *goquery.Document
   category *Category
   }
   const WORKERS = 2
   const SEPARATOR = "§§§"
   func main() {
   allCategories := make([]Category, 0)
   downloadChannel := make(chan *Category)
   resultsChannel := make(chan *DownloadResult, 100)
   for w := 1; w <= WORKERS; w++ {
   go worker(downloadChannel, resultsChannel)
   }
   numRequests := 1
   downloadChannel <- &Category{ domain + categoryPage, 0, "root", 0, nil }
   for result := range resultsChannel {
   var extractor func(doc *goquery.Document) []string
   if result.category.level == 0 {
   extractor = topLevelExtractor
   } else if result.category.level == 1 {
   extractor = secondLevelExtractor
   } else {
   extractor = thirdLevelExtractor
   }
   categories := extractCategories(result.doc, result.category, extractor)
   allCategories = append(allCategories, *categories...)
   //fmt.Printf("Appending categories: %v", *categories)
   fmt.Printf("total categories = %d, total requests = %d\n", len(allCategories), numRequests)
   for _, category := range *categories {
   numRequests += 1
   downloadChannel <- &category
   }
   // close the channels when there are no more jobs
   if len(allCategories) > numRequests {
   close(downloadChannel)
   close(resultsChannel)
   }
   }
   fmt.Println("Done")
   }
   func worker(downloadChannel <-chan *Category, results chan<- *DownloadResult) {
   for target := range downloadChannel {
   fmt.Printf("Downloading %v (addr %p) ...", target, &target)
   doc, err := goquery.NewDocument(target.url)
   if err != nil {
   log.Fatal(err)
   panic(err)
   }
   fmt.Print("done \n")
   results <- &DownloadResult{doc, target}
   }
   }
   func extractCategories(doc *goquery.Document, parent *Category, extractor func(doc *goquery.Document) []string) *[]Category {
   numberRegex, _ := regexp.Compile("[0-9,]+")
   log.Printf("Extracting subcategories for page %s\n", parent)
   subCategories := extractor(doc)
   categories := make([]Category, 0)
   for _, subCategory := range subCategories {
   log.Printf("Got subcategory=%s from parent=%s", subCategory, parent)
   extracted := strings.Split(subCategory, SEPARATOR)
   numberWithComma := numberRegex.FindString(extracted[2])
   number := strings.Replace(numberWithComma, ",", "", -1)
   numRecords, err := strconv.Atoi(number)
   if err != nil {
   log.Fatal(err)
   panic(err)
   }
   var category Category
   level := parent.level + 1
   if parent.level == 0 {
   category = Category{ domain + extracted[1], level, extracted[0], numRecords, parent }
   } else {
   log.Printf("category URL=%s, parent=%s, parent=%v", extracted[1], parent.url, parent)
   category = Category{ parent.url + extracted[1], level, extracted[0], numRecords, parent }
   }
   log.Printf("Appending category=%v (pointer=%p)", category, &category)
   categories = append(categories, category)
   }
   return &categories
   }
   func topLevelExtractor(doc *goquery.Document) []string {
   return doc.Find(".cat-listings-td .c-1s-2m-1-td1").Map(func(i int, s *goquery.Selection) string {
   title := s.Find("a").Text()
   url := s.Find("a").Map(func(x int, a *goquery.Selection) string {
   v, _ := a.Attr("href")
   return v
   })
   records := s.Clone().Children().Remove().End().Text()
   //log.Printf("Item %d: %s, %s - %s\n", i, title, records, url)
   res := []string{title, url[0], records}
   return strings.Join(res, SEPARATOR)
   })
   }
   func secondLevelExtractor(doc *goquery.Document) []string {
   return doc.Find(".c-2m-3c-1-table .c-2m-3c-1-td1").Map(func(i int, s *goquery.Selection) string {
   title := s.Find("a").Text()
   url := s.Find("a").Map(func(x int, a *goquery.Selection) string {
   v, _ := a.Attr("href")
   return v
   })
   records := s.Clone().Children().Remove().End().Text()
   //log.Printf("Item %d: %s, %s - %s\n", i, title, records, url)
   res := []string{title, url[0], records}
   return strings.Join(res, SEPARATOR)
   })
   }
   func thirdLevelExtractor(doc *goquery.Document) []string {
   return doc.Find(".c-2m-3c-1-table .c-2m-3c-1-td1").Map(func(i int, s *goquery.Selection) string {
   title := s.Find("a").Text()
   url := s.Find("a").Map(func(x int, a *goquery.Selection) string {
   v, _ := a.Attr("href")
   return v
   })
   records := s.Clone().Children().Remove().End().Text()
   //log.Printf("Item %d: %s, %s - %s\n", i, title, records, url)
   res := []string{title, url[0], records}
   return strings.Join(res, SEPARATOR)
   })
   }
   

Update
Fixed - see comment below.

Looping over:

            for _, category := range *categories {
numRequests += 1
downloadChannel <- &category
}

meant I was sending a reference to the temporary variable category to the channel, instead of the actual memory address of that value.

I've fixed this by using a different loop:

	for i := 0; i < len(*categories); i++ {
fmt.Printf("Queuing category: %v (%p)", categoriesValues[i], categoriesValues[i])
downloadChannel <- &categoriesValues[i]
}