Skip to content

Hardcode pg_rewind retrying (up-to 5m) #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dyson merged 1 commit into from
Aug 27, 2019
Merged

Hardcode pg_rewind retrying (up-to 5m) #16

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 28 additions & 9 deletions cmd/keeper/cmd/keeper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
Expand Down Expand Up @@ -842,15 +843,33 @@ func (p *PostgresKeeper) resync(db, followedDB *cluster.DB, tryPgrewind bool) er
// doesn't exists pgm.SyncFromFollowedPGRewind will return an error and
// fallback to pg_basebackup
if tryPgrewind && p.usePgrewind(db) {
connParams := p.getSUConnParams(db, followedDB)
log.Infow("syncing using pg_rewind", "followedDB", followedDB.UID, "keeper", followedDB.Spec.KeeperUID)
// TODO: Make the forceCheckpoint parameter use cluster specification
if err := pgm.SyncFromFollowedPGRewind(connParams, p.pgSUPassword, true); err != nil {
// log pg_rewind error and fallback to pg_basebackup
log.Errorw("error syncing with pg_rewind", zap.Error(err))
} else {
pgm.SetRecoveryParameters(p.createRecoveryParameters(true, standbySettings, nil, nil))
return nil
startedPgrewind := time.Now()
pgrewindRetries:
for {
connParams := p.getSUConnParams(db, followedDB)
log.Infow("syncing using pg_rewind", "followedDB", followedDB.UID, "keeper", followedDB.Spec.KeeperUID)
// TODO: Remove the final true once this is merged:
// https://github.com/sorintlab/stolon/pull/644 is
if err := pgm.SyncFromFollowedPGRewind(connParams, p.pgSUPassword, true); err != nil {
// log pg_rewind error and fallback to pg_basebackup
log.Errorw("error syncing with pg_rewind", zap.Error(err))

// TODO: This is a GoCardless modification that enables retrying pg_rewind for
// up-to 5m, to allow the follower to boot before we attempt to rewind. This
// avoids falling back on pg_basebackup unnecessarily, and will eventually require
// upstreaming in a more mature form.
if time.Since(startedPgrewind) > 5*time.Minute {
break pgrewindRetries
}

// Retry the pg_rewind 5-10s after our last attempt
log.Infow("sleeping before retrying pg_rewind")
time.Sleep((5 * time.Second) + time.Duration(rand.Int63n(int64(5*time.Second))))
continue pgrewindRetries
} else {
pgm.SetRecoveryParameters(p.createRecoveryParameters(true, standbySettings, nil, nil))
return nil
}
}
}

Expand Down