Consuming Paginated REST API

I am trying to consume an API that on each successful request has a json key like the following:

{
    "hasNextPage": true,
    "endCursor": "some_guid_present_here"
}

The way the API works (I have used it many times in Python but am trying with Go for separate use case) is to basically check if there is a next page, then use the appropriate cursor to continue to paginate.

However, every time I use this structure, it will sometimes keep looping even though the response.Paginations.HasNextPage will be false.

I am trying to understand if it is my structure of consuming a paginated API like this that is causing this or else.

Also, I have say 4-5 requests to start off with, which I sent separately via goroutines. I am not sure if this is causing an issue but I've attached that as well in etl.go.

The main request making structs are in api.go.

I've confirmed that I do receive responses and they are unmarshaling properly, but am trying to wrangle this uncertain behavior.

api.go

package models


import (
	"encoding/json"
	"io/ioutil"
	"net/http"
    "fmt"
)

type Request struct {
    Url string
    ApiKey string
}

type Response struct {
    ...some fields...
	Paginations Pagination `json:"pagination"`
}

type Pagination struct {
	EndCursor   string `json:"endCursor"`
	HasNextPage bool   `json:"hasNextPage"`
}


func (request *Request) Get() ([]Response, error) {
    
    var responses []Response
    var response Response

	// Set up new request
	req, err := http.NewRequest("GET", request.Url, nil)
	if err != nil {
		fmt.Println("Error creating request...")
		return responses, err
	}

	// Add request headers
	req.Header = http.Header{
		"accept":        {"application/json"},
		"authorization": {"Bearer " + request.ApiKey},
	}

	// Get our initial response from the API and capture status code
	resp, _ := http.DefaultClient.Do(req)
	response.Status = resp.StatusCode

	// Read the response body and Unmarshal into struct
	respBody, err := ioutil.ReadAll(resp.Body)
	json.Unmarshal(respBody, &response)

	// If there was a parsing error, log it
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
    
    // This field will be in the response payload
    // It is verified to be of type bool (not string)
	fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)

	// Append response to our slice of responses
	responses = append(responses, response)

	// If paginations are available, continue to loop through until all paginations are exhausted
	for response.Paginations.HasNextPage == true {
		req, err := http.NewRequest("GET", request.Url, nil)
		if err != nil {
			return responses, err
		}

		// Append "after" cursor to query in order to redirect to paginated response
		qry := req.URL.Query()
		qry.Set("after", response.Paginations.EndCursor)
		req.URL.RawQuery = qry.Encode()
		fmt.Println("Paginated request query: ", req.URL.String())

		// Make request
		resp, err := http.DefaultClient.Do(req)
		response.Status = resp.StatusCode
		fmt.Printf("Status Code: %d\n", response.Status)

		// Read response and deserialize it
		respBody, err := ioutil.ReadAll(resp.Body)
		json.Unmarshal(respBody, &response)
		fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
		fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)

		// If there was a parsing error, log it
		if err != nil {
			fmt.Println(err)
		}
		defer resp.Body.Close()

		// Append response to our slice of responses
		responses = append(responses, response)
	}
	return responses, nil


}

etl.go

package models

import (
    "fmt"
	"sync"
)

type Etl struct {
	Requests []Request
}

func (etl *Etl) Extract() [][]Response {

	var wg sync.WaitGroup
	ch := make(chan []Response)
	defer close(ch)
	for _, req := range etl.Requests {
		wg.Add(1) // Increment
		fmt.Println("Incremented wait group")
		go func(i Request) {
			defer wg.Done() // Decrement
			resp, err := req.Get()
			if err != nil {
				fmt.Println(err)
			}
			ch <- resp
			fmt.Println("Decremented wait group")
		}(req)
	}

	results := make([][]Response, len(etl.Requests))
	for i, _ := range results {
		results[i] = <-ch
		//fmt.Println(results[i])
	}
	wg.Wait()
	return nil
}

I believe I found the issue. In my pagination loop for response.Paginations.HasNextPage == true I was creating a new request object (http.NewRequest) on each iteration which did not have headers added from the prior (initial request).

This caused a 401 unauthorized error to return and continuous querying of the API since it was not receiving a new response.Paginations.HasNextPage value.

My solution was to simply alter the for loop a bit like so:

package models
import (
"encoding/json"
"io/ioutil"
"net/http"
"fmt"
)
type Request struct {
Url string
ApiKey string
}
type Response struct {
...some fields...
Paginations Pagination `json:"pagination"`
}
type Pagination struct {
EndCursor   string `json:"endCursor"`
HasNextPage bool   `json:"hasNextPage"`
}
func (request *Request) Get() ([]Response, error) {
var responses []Response
var response Response
// Set up new request
req, err := http.NewRequest("GET", request.Url, nil)
if err != nil {
fmt.Println("Error creating request...")
return responses, err
}
// Add request headers
req.Header = http.Header{
"accept":        {"application/json"},
"authorization": {"Bearer " + request.ApiKey},
}
// Get our initial response from the API and capture status code
resp, _ := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
// Read the response body and Unmarshal into struct
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// This field will be in the response payload
// It is verified to be of type bool (not string)
fmt.Printf("Has Next Page? %t\n", resp.Paginations.HasNextPage)
// Append response to our slice of responses
responses = append(responses, response)
// If paginations are available, continue to loop through until all paginations are exhausted
for response.Paginations.HasNextPage == true {
// Append "after" cursor to query in order to redirect to paginated response
qry := req.URL.Query()
qry.Set("after", response.Paginations.EndCursor)
req.URL.RawQuery = qry.Encode()
fmt.Println("Paginated request query: ", req.URL.String())
// Make request
resp, err := http.DefaultClient.Do(req)
response.Status = resp.StatusCode
fmt.Printf("Status Code: %d\n", response.Status)
// Read response and deserialize it
respBody, err := ioutil.ReadAll(resp.Body)
json.Unmarshal(respBody, &response)
fmt.Println("Pagination Cursor: ", response.Paginations.EndCursor)
fmt.Printf("Has Next Page? %t\n", response.Paginations.HasNextPage)
// If there was a parsing error, log it
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
// Append response to our slice of responses
responses = append(responses, response)
}
return responses, nil
}