Golang download multiple files in parallel using goroutines

Is it possible to download and save files in parallel using goroutines?

Below is my code which downloads files from my dropbox:

package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
)
const app_key string = "<app_key>"
const app_secret string = "<app_secret>"
var code string
type TokenResponse struct {
AccessToken string `json:"access_token"`
}
type File struct {
Path string
}
type FileListResponse struct {
FileList []File `json:"contents"`
}
func download_file(file File, token TokenResponse) {
download_file := fmt.Sprintf("https://api-content.dropbox.com/1/files/dropbox/%s?access_token=%s", file.Path, token.AccessToken)
resp, _ := http.Get(download_file)
defer resp.Body.Close()
filename := filepath.Base(file.Path)
out, err := os.Create(filename)
if err != nil {
panic(err)
}
defer out.Close()
io.Copy(out, resp.Body)
}
func main() {
authorize_url := fmt.Sprintf("https://www.dropbox.com/1/oauth2/authorize?response_type=code&client_id=%s", app_key)
// Get code
fmt.Printf("1. Go to: %s\n", authorize_url)
fmt.Println("2. Click 'Allow' (you might have to log in first)")
fmt.Println("3. Copy the authorization code.")
fmt.Printf("Enter the authorization code here: ")
fmt.Scanf("%s", &code)
// End get code
// Get access token
data := url.Values{}
data.Add("code", code)
data.Add("grant_type", "authorization_code")
data.Add("client_id", app_key)
data.Add("client_secret", app_secret)
resp, _ := http.PostForm("https://api.dropbox.com/1/oauth2/token", data)
defer resp.Body.Close()
contents, _ := ioutil.ReadAll(resp.Body)
var tr TokenResponse
json.Unmarshal(contents, &tr)
// End get access token
// Get file list
file_list_url := fmt.Sprintf("https://api.dropbox.com/1/metadata/dropbox/Camera Uploads?access_token=%s", tr.AccessToken)
resp2, _ := http.Get(file_list_url)
defer resp2.Body.Close()
contents2, _ := ioutil.ReadAll(resp2.Body)
var flr FileListResponse
json.Unmarshal(contents2, &flr)
// End get file list
for i, file := range flr.FileList {
download_file(file, tr)
if i >= 2 {
break
}
}
}

It doesn't work when I prefix the download_file function with the go command.

go download_file(file, tr)

That's because your main goroutine is exiting. You need to add a WaitGroup to wait until all the goroutines exit. For example,

var wg sync.WaitGroup
for i, file := range flr.FileList {
wg.Add(1)
go download_file(file, tr, wg)
if i >= 2 {
break
}
}
wg.Wait()
...
func download_file(file File, token TokenResponse, wg sync.WaitGroup) {
...
wg.Done()
}

This project might help you and others that are looking into achieving concurrency in Go:

Idea is to create a cluster in Go to execute parallel jobs. https://github.com/waqar-alamgir/mini-go-cluster

Source can be tweaked to download files concurrently.