Skip to content

Hardcode pg_rewind retrying (up-to 5m) #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
VSpike merged 1 commit into from
Mar 22, 2022
Merged

Hardcode pg_rewind retrying (up-to 5m) #60

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 33 additions & 13 deletions cmd/keeper/cmd/keeper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
Expand Down Expand Up @@ -946,19 +947,38 @@ func (p *PostgresKeeper) resync(db, masterDB, followedDB *cluster.DB, tryPgrewin
// doesn't exists pgm.SyncFromFollowedPGRewind will return an error and
// fallback to pg_basebackup
if tryPgrewind && p.usePgrewind(db) {
// pg_rewind doesn't support running against a database that is in recovery, as it
// builds temporary tables and this is not supported on a hot-standby. Stolon doesn't
// currently support cascading replication, but we should be clear when issuing a
// rewind that it targets the current primary, rather than whatever database we
// follow.
connParams := p.getSUConnParams(db, masterDB)
log.Infow("syncing using pg_rewind", "masterDB", masterDB.UID, "keeper", followedDB.Spec.KeeperUID)
if err := pgm.SyncFromFollowedPGRewind(connParams, p.pgSUPassword, true); err != nil {
// log pg_rewind error and fallback to pg_basebackup
log.Errorw("error syncing with pg_rewind", zap.Error(err))
} else {
pgm.SetRecoveryOptions(p.createRecoveryOptions(pg.RecoveryModeStandby, standbySettings, nil, nil))
return nil
startedPgrewind := time.Now()
pgrewindRetries:
for {
// pg_rewind doesn't support running against a database that is in recovery, as it
// builds temporary tables and this is not supported on a hot-standby. Stolon doesn't
// currently support cascading replication, but we should be clear when issuing a
// rewind that it targets the current primary, rather than whatever database we
// follow.
connParams := p.getSUConnParams(db, masterDB)
log.Infow("syncing using pg_rewind", "masterDB", masterDB.UID, "keeper", followedDB.Spec.KeeperUID)
// TODO: Remove the final true once this is merged:
// https://github.com/sorintlab/stolon/pull/644 is
if err := pgm.SyncFromFollowedPGRewind(connParams, p.pgSUPassword, true); err != nil {
// log pg_rewind error and fallback to pg_basebackup
log.Errorw("error syncing with pg_rewind", zap.Error(err))

// TODO: This is a GoCardless modification that enables retrying pg_rewind for
// up-to 5m, to allow the follower to boot before we attempt to rewind. This
// avoids falling back on pg_basebackup unnecessarily, and will eventually require
// upstreaming in a more mature form.
if time.Since(startedPgrewind) > 5*time.Minute {
break pgrewindRetries
}

// Retry the pg_rewind 5-10s after our last attempt
log.Infow("sleeping before retrying pg_rewind")
time.Sleep((5 * time.Second) + time.Duration(rand.Int63n(int64(5*time.Second))))
continue pgrewindRetries
} else {
pgm.SetRecoveryOptions(p.createRecoveryOptions(pg.RecoveryModeStandby, standbySettings, nil, nil))
return nil
}
}
}

Expand Down