diff --git a/datasources/googlephotos/googlephotos.go b/datasources/googlephotos/googlephotos.go index 50f9a84..92d9d08 100644 --- a/datasources/googlephotos/googlephotos.go +++ b/datasources/googlephotos/googlephotos.go @@ -207,7 +207,7 @@ func (c *Client) getAlbumsAndTheirItemsNextPage(itemChan chan<- *timeliner.ItemG } var respBody listAlbums - err := c.apiRequest("GET", "/albums?"+vals.Encode(), nil, &respBody) + err := c.apiRequestWithRetry("GET", "/albums?"+vals.Encode(), nil, &respBody) if err != nil { return pageToken, err } @@ -283,48 +283,87 @@ func (c *Client) getAlbumItems(itemChan chan<- *timeliner.ItemGraph, album gpAlb func (c *Client) pageOfMediaItems(reqBody listMediaItemsRequest) (listMediaItems, error) { var respBody listMediaItems - err := c.apiRequest("POST", "/mediaItems:search", reqBody, &respBody) + err := c.apiRequestWithRetry("POST", "/mediaItems:search", reqBody, &respBody) return respBody, err } -func (c *Client) apiRequest(method, endpoint string, reqBodyData, respInto interface{}) error { +func (c *Client) apiRequestWithRetry(method, endpoint string, reqBodyData, respInto interface{}) error { + // do the request in a loop for controlled retries on error + var err error + const maxTries = 10 + for i := 0; i < maxTries; i++ { + var resp *http.Response + resp, err = c.apiRequest(method, endpoint, reqBodyData) + if err != nil { + log.Printf("[ERROR][%s/%s] Doing API request: >>> %v <<< - retrying... (attempt %d/%d)", + DataSourceID, c.userID, err, i+1, maxTries) + time.Sleep(2 * time.Second) + continue + } + + if resp.StatusCode != http.StatusOK { + bodyText, err2 := ioutil.ReadAll(io.LimitReader(resp.Body, 1024*256)) + resp.Body.Close() + + if err2 == nil { + err = fmt.Errorf("HTTP %d: %s: >>> %s <<<", resp.StatusCode, resp.Status, bodyText) + } else { + err = fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) + } + + // extra-long pause for rate limiting errors + if resp.StatusCode == http.StatusTooManyRequests { + log.Printf("[ERROR][%s/%s] Rate limited: HTTP %d: %s: %s - retrying in 35 seconds... (attempt %d/%d)", + DataSourceID, c.userID, resp.StatusCode, resp.Status, bodyText, i+1, maxTries) + time.Sleep(35 * time.Second) + continue + } + + // for any other error, wait a couple seconds and retry + log.Printf("[ERROR][%s/%s] Bad API response: %v - retrying... (attempt %d/%d)", + DataSourceID, c.userID, err, i+1, maxTries) + time.Sleep(2 * time.Second) + continue + } + + // successful request; read the response body + err = json.NewDecoder(resp.Body).Decode(&respInto) + if err != nil { + resp.Body.Close() + err = fmt.Errorf("decoding JSON: %v", err) + log.Printf("[ERROR][%s/%s] Reading API response: %v - retrying... (attempt %d/%d)", + DataSourceID, c.userID, err, i+1, maxTries) + time.Sleep(2 * time.Second) + continue + } + + // successful read; we're done here + resp.Body.Close() + break + } + + return err +} + +func (c *Client) apiRequest(method, endpoint string, reqBodyData interface{}) (*http.Response, error) { var reqBody io.Reader if reqBodyData != nil { reqBodyBytes, err := json.Marshal(reqBodyData) if err != nil { - return err + return nil, err } reqBody = bytes.NewReader(reqBodyBytes) } req, err := http.NewRequest(method, apiBase+endpoint, reqBody) if err != nil { - return err + return nil, err } if reqBody != nil { req.Header.Set("Content-Type", "application/json") } - resp, err := c.HTTPClient.Do(req) - if err != nil { - return fmt.Errorf("performing search request: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - bodyText, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024*256)) - if err == nil { - return fmt.Errorf("HTTP %d: %s: %s", resp.StatusCode, resp.Status, bodyText) - } - return fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) - } - - err = json.NewDecoder(resp.Body).Decode(&respInto) - if err != nil { - return fmt.Errorf("decoding JSON: %v", err) - } - - return nil + return c.HTTPClient.Do(req) } func dateRange(timeframe timeliner.Timeframe) listMediaItemsFilterRange { diff --git a/datasources/googlephotos/media.go b/datasources/googlephotos/media.go index e9110ee..cdb64ce 100644 --- a/datasources/googlephotos/media.go +++ b/datasources/googlephotos/media.go @@ -3,6 +3,7 @@ package googlephotos import ( "fmt" "io" + "io/ioutil" "log" "net/http" "strconv" @@ -62,16 +63,36 @@ func (m mediaItem) DataFileReader() (io.ReadCloser, error) { u += "=dv" } - resp, err := http.Get(u) - if err != nil { - return nil, fmt.Errorf("getting media contents: %v", err) - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) + const maxTries = 5 + var err error + var resp *http.Response + for i := 0; i < maxTries; i++ { + resp, err = http.Get(u) + if err != nil { + err = fmt.Errorf("getting media contents: %v", err) + log.Printf("[ERROR][%s] %v - retrying... (attempt %d/%d)", DataSourceID, err, i+1, maxTries) + time.Sleep(1 * time.Second) + continue + } + if resp.StatusCode != http.StatusOK { + bodyText, err2 := ioutil.ReadAll(io.LimitReader(resp.Body, 1024*256)) + resp.Body.Close() + + if err2 == nil { + err = fmt.Errorf("HTTP %d: %s: >>> %s <<<", resp.StatusCode, resp.Status, bodyText) + } else { + err = fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) + } + + log.Printf("[ERROR][%s] Bad response: %v - retrying... (attempt %d/%d)", + DataSourceID, err, i+1, maxTries) + time.Sleep(1 * time.Second) + continue + } + break } - return resp.Body, nil + return resp.Body, err } func (m mediaItem) DataFileHash() []byte {