When and where to check if channel won't get any more data?

I'm trying to solve Exercise: Web Crawler

> In this exercise you'll use Go's concurrency features to parallelize a
> web crawler.
>
> Modify the Crawl function to fetch URLs in parallel without fetching
> the same URL twice.

When should I check if all urls already been crawled? (or how could I know if there will be no more data queued?)

package main

import (
	"fmt"
)

type Result struct {
	Url string
	Depth int
}

type Stor struct {
	Queue  chan Result
	Visited map[string]int
}    

func NewStor() *Stor {
	return &Stor{
		Queue:  make(chan Result,1000),
		Visited: map[string]int{},
	}
}

type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page.
	Fetch(url string) (body string, urls []string, err error)
}

// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(res Result, fetcher Fetcher, stor *Stor) {
	defer func() {			
		/*
		if len(stor.Queue) == 0 {
			close(stor.Queue)
		}	
		*/	// this is wrong, it makes the channel closes too early
	}()
	if res.Depth <= 0 {
		return
	}
	// TODO: Don't fetch the same URL twice.
	url := res.Url
	stor.Visited
++
	if stor.Visited
 > 1 {
		fmt.Println("skip:",stor.Visited
,url)
		return
	}
	body, urls, err := fetcher.Fetch(url)
	if err != nil {
		fmt.Println(err)
		return
	}	
	fmt.Printf("found: %s %q\n", url, body)
	for _, u := range urls {
		stor.Queue <- Result{u,res.Depth-1}
	}
	return
}

func main() {
	stor := NewStor()	
	Crawl(Result{"http://golang.org/", 4}, fetcher, stor)
	for res := range stor.Queue {
		// TODO: Fetch URLs in parallel.
		go Crawl(res,fetcher,stor)
	}
}

// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult

type fakeResult struct {
	body string
	urls []string
}

func (f fakeFetcher) Fetch(url string) (string, []string, error) {
	if res, ok := f
; ok {
		return res.body, res.urls, nil
	}
	return "", nil, fmt.Errorf("not found: %s", url)
}

// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
	"http://golang.org/": &fakeResult{
		"The Go Programming Language",
		[]string{
			"http://golang.org/pkg/",
			"http://golang.org/cmd/",
		},
	},
	"http://golang.org/pkg/": &fakeResult{
		"Packages",
		[]string{
			"http://golang.org/",
			"http://golang.org/cmd/",
			"http://golang.org/pkg/fmt/",
			"http://golang.org/pkg/os/",
		},
	},
	"http://golang.org/pkg/fmt/": &fakeResult{
		"Package fmt",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
	"http://golang.org/pkg/os/": &fakeResult{
		"Package os",
		[]string{
			"http://golang.org/",
			"http://golang.org/pkg/",
		},
	},
}

The output was a deadlock since the stor.Queue channel never closed.

Simplest way to wait until all goroutings are done is sync.WaitGroup in sync package

package main
import "sync"
var wg sync.WaitGroup
//then you do
func Crawl(res Result, fetcher Fetcher) { //what for you pass stor *Stor as arg? It just visible for all goroutings
defer wg.Done()
...
//why not to spawn new routing just inside Crowl?
for res := range urls {
wg.Add(1)
go Crawl(res,fetcher)
}
...
}
...
//And in main.main()
func main() {
wg.Add(1) 
Crawl(Result{"http://golang.org/", 4}, fetcher)
...
wg.Wait() //Will block until all routings Done
}

Complete solution will be:

package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
var visited map[string]int = map[string]int{}
type Result struct {
Url string
Depth int
}
type Fetcher interface {
// Fetch returns the body of URL and
// a slice of URLs found on that page.
Fetch(url string) (body string, urls []string, err error)
}
// Crawl uses fetcher to recursively crawl
// pages starting with url, to a maximum of depth.
func Crawl(res Result, fetcher Fetcher) {
defer wg.Done()
if res.Depth <= 0 {
return
}
// TODO: Don't fetch the same URL twice.
url := res.Url
visited
++
if visited
 > 1 {
fmt.Println("skip:",visited
,url)
return
}
body, urls, err := fetcher.Fetch(url)
if err != nil {
fmt.Println(err)
return
}   
fmt.Printf("found: %s %q\n", url, body)
for _, u := range urls {
wg.Add(1)
go Crawl( Result{u,res.Depth-1},fetcher)
//stor.Queue <- Result{u,res.Depth-1}
}
return
}
func main() {
wg.Add(1) 
Crawl(Result{"http://golang.org/", 4}, fetcher)
wg.Wait()
}
// fakeFetcher is Fetcher that returns canned results.
type fakeFetcher map[string]*fakeResult
type fakeResult struct {
body string
urls []string
}
func (f fakeFetcher) Fetch(url string) (string, []string, error) {
if res, ok := f
; ok {
return res.body, res.urls, nil
}
return "", nil, fmt.Errorf("not found: %s", url)
}
// fetcher is a populated fakeFetcher.
var fetcher = fakeFetcher{
"http://golang.org/": &fakeResult{
"The Go Programming Language",
[]string{
"http://golang.org/pkg/",
"http://golang.org/cmd/",
},
},
"http://golang.org/pkg/": &fakeResult{
"Packages",
[]string{
"http://golang.org/",
"http://golang.org/cmd/",
"http://golang.org/pkg/fmt/",
"http://golang.org/pkg/os/",
},
},
"http://golang.org/pkg/fmt/": &fakeResult{
"Package fmt",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
"http://golang.org/pkg/os/": &fakeResult{
"Package os",
[]string{
"http://golang.org/",
"http://golang.org/pkg/",
},
},
}

Checking the len of a channel is always a race, you can't use that for any sort of synchronization.

The producer is always the side that closes a channel, because it's a fatal error to try and send on a closed channel. Don't use a defer here, just close the channel when you're done sending.