How can I efficiently download a large file using Go?

Is there a way to download a large file using Go that will store the content directly into a file instead of storing it all in memory before writing it to a file? Because the file is so big, storing it all in memory before writing it to a file is going to use up all the memory.

I'll assume you mean download via http (error checks omitted for brevity):

import ("net/http"; "io"; "os")
...
out, err := os.Create("output.txt")
defer out.Close()
...
resp, err := http.Get("http://example.com/")
defer resp.Body.Close()
...
n, err := io.Copy(out, resp.Body)

The http.Response's Body is a Reader, so you can use any functions that take a Reader, to, e.g. read a chunk at a time rather than all at once. In this specific case, io.Copy() does the gruntwork for you.

A more descriptive version of Steve M's answer.

import (
    "os"
    "net/http"
    "io"
)

func downloadFile(filepath string, url string) (err error) {

  // Create the file
  out, err := os.Create(filepath)
  if err != nil  {
    return err
  }
  defer out.Close()

  // Get the data
  resp, err := http.Get(url)
  if err != nil {
    return err
  }
  defer resp.Body.Close()

  // Check server response
  if resp.StatusCode != http.StatusOK {
	return fmt.Errorf("bad status: %s", resp.Status)
  }

  // Writer the body to file
  _, err = io.Copy(out, resp.Body)
  if err != nil  {
    return err
  }

  return nil
}

The answer selected above using io.Copy is exactly what you need, but if you are interested in additional features like resuming broken downloads, auto-naming files, checksum validation or monitoring progress of multiple downloads, checkout the grab package.

I also think it's nice to have a progress indicator, especially for larger files. So I want to throw in my two cents for a solution to this problem while implementing a simple progress indicator.
(Most error handling also omitted for brevety).

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
)

func main() {
	temp_path := ".tmp"
	req, _ := http.NewRequest("GET", "http://212.183.159.230/200MB.zip", nil)
	resp, _ := http.DefaultClient.Do(req)
	defer resp.Body.Close()

	f, _ := os.OpenFile(temp_path, os.O_CREATE|os.O_WRONLY, 0644)
	defer f.Close()

	buf := make([]byte, 32*1024)
	var downloaded int64
	for {
		n, err := resp.Body.Read(buf)
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Fatalf("Error while downloading: %v", err)
		}
		if n > 0 {
			f.Write(buf[:n])
			downloaded += int64(n)
			fmt.Printf("\rDownloading... %.2f%%", float64(downloaded)/float64(resp.ContentLength)*100)
		}
	}
	os.Rename(temp_path, "wordpress.zip")
}

To use io.Copy we can implement an io.Reader . Which probably will be the preferred approach in a real world scenario to make it reusable and easier to test. So here is the second version:

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"
)

type ProgressReader struct {
	Reader io.Reader
	Size   int64
	Pos    int64
}

func (pr *ProgressReader) Read(p []byte) (int, error) {
	n, err := pr.Reader.Read(p)
	if err == nil {
		pr.Pos += int64(n)
		fmt.Printf("\rDownloading... %.2f%%", float64(pr.Pos)/float64(pr.Size)*100)
	}
	return n, err
}

func main() {
	start := time.Now().UnixMilli()
	tempPath := ".tmp"
	outPath := "200MB.zip"
	req, _ := http.NewRequest("GET", "http://212.183.159.230/200MB.zip", nil)
	resp, _ := http.DefaultClient.Do(req)
	if resp.StatusCode != 200 {
		log.Fatalf("Error while downloading: %v", resp.StatusCode)
	}
	defer resp.Body.Close()

	f, _ := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY, 0644)
	defer f.Close()

	progressReader := &ProgressReader{
		Reader: resp.Body,
		Size:   resp.ContentLength,
	}

	if _, err := io.Copy(f, progressReader); err != nil {
		log.Fatalf("Error while downloading: %v", err)
	}

	os.Rename(tempPath, outPath)
	fmt.Println(" - Download completed!")

	fmt.Printf("Took: %.2fs\n", float64(time.Now().UnixMilli()-start)/1000)
}

1. Here is a sample. https://github.com/thbar/golang-playground/blob/master/download-files.go

2. Also I give u some codes might help you.

code:

func HTTPDownload(uri string) ([]byte, error) {
fmt.Printf("HTTPDownload From: %s.\n", uri)
res, err := http.Get(uri)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
d, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("ReadFile: Size of download: %d\n", len(d))
return d, err
}
func WriteFile(dst string, d []byte) error {
fmt.Printf("WriteFile: Size of download: %d\n", len(d))
err := ioutil.WriteFile(dst, d, 0444)
if err != nil {
log.Fatal(err)
}
return err
}
func DownloadToFile(uri string, dst string) {
fmt.Printf("DownloadToFile From: %s.\n", uri)
if d, err := HTTPDownload(uri); err == nil {
fmt.Printf("downloaded %s.\n", uri)
if WriteFile(dst, d) == nil {
fmt.Printf("saved %s as %s\n", uri, dst)
}
}
}