Skip to content

Commit 402ad8a

Browse files
rahearnrahearn
authored
Make s3 upload optional (#20)
1 parent 5061d67 commit 402ad8a

File tree

17 files changed

+192
-143
lines changed

17 files changed

+192
-143
lines changed

‎10/backup.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99

1010
BACKUP_DIR = os.environ["BACKUP_DIR"]
1111

12-
S3_PATH = os.environ["S3_PATH"]
13-
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS") or "STANDARD_IA"
14-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
12+
S3_PATH = os.environ.get("S3_PATH", "")
13+
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
14+
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
15+
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS", "STANDARD_IA")
16+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1517

16-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
18+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1719
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1820

1921
if not DB_NAME:
@@ -23,17 +25,17 @@
2325
DB_HOST = os.environ["DB_HOST"]
2426
DB_PASS = os.environ["DB_PASS"]
2527
DB_USER = os.environ["DB_USER"]
26-
DB_PORT = os.environ.get("DB_PORT") or "5432"
28+
DB_PORT = os.environ.get("DB_PORT", "5432")
2729

2830
MAIL_TO = os.environ.get("MAIL_TO")
2931
MAIL_FROM = os.environ.get("MAIL_FROM")
3032
WEBHOOK = os.environ.get("WEBHOOK")
3133
WEBHOOK_METHOD = os.environ.get("WEBHOOK_METHOD")
3234
WEBHOOK_DATA = os.environ.get("WEBHOOK_DATA")
33-
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
35+
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS", "")
3436
KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
3537
FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
36-
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
38+
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS", "")
3739

3840
file_name = dt.strftime(FILENAME)
3941
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -112,8 +114,12 @@ def main():
112114
take_backup()
113115
backup_size=os.path.getsize(backup_file)
114116

115-
log("Uploading to S3")
116-
upload_backup()
117+
if AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
118+
log("Uploading to S3")
119+
upload_backup()
120+
else:
121+
log("Skipping S3 upload, no AWS credentials provided")
122+
117123
log("Pruning local backup copies")
118124
prune_local_backup_files()
119125
end_time = datetime.now()

‎10/restore.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88
BACKUP_DIR = os.environ["BACKUP_DIR"]
99

10-
S3_PATH = os.environ["S3_PATH"]
11-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
10+
S3_PATH = os.environ.get("S3_PATH", "")
11+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1212

13-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
13+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1414
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1515

1616
if not DB_NAME:
@@ -20,7 +20,7 @@
2020
DB_HOST = os.environ["DB_HOST"]
2121
DB_PASS = os.environ["DB_PASS"]
2222
DB_USER = os.environ["DB_USER"]
23-
DB_PORT = os.environ.get("DB_PORT") or "5432"
23+
DB_PORT = os.environ.get("DB_PORT", "5432")
2424

2525
file_name = sys.argv[1]
2626
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -48,7 +48,7 @@ def restore_backup():
4848
if not backup_exists():
4949
sys.stderr.write("Backup file doesn't exists!\n")
5050
sys.exit(1)
51-
51+
5252
# restore postgres-backup
5353
env = os.environ.copy()
5454
if DB_USE_ENV:
@@ -71,10 +71,10 @@ def main():
7171
else:
7272
log("Downloading database dump")
7373
download_backup()
74-
74+
7575
log("Restoring database")
7676
restore_backup()
77-
77+
7878
log("Restore complete, took %.2f seconds" % (datetime.now() - start_time).total_seconds())
7979

8080
if __name__ == "__main__":

‎11/backup.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99

1010
BACKUP_DIR = os.environ["BACKUP_DIR"]
1111

12-
S3_PATH = os.environ["S3_PATH"]
13-
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS") or "STANDARD_IA"
14-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
12+
S3_PATH = os.environ.get("S3_PATH", "")
13+
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
14+
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
15+
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS", "STANDARD_IA")
16+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1517

16-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
18+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1719
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1820

1921
if not DB_NAME:
@@ -23,17 +25,17 @@
2325
DB_HOST = os.environ["DB_HOST"]
2426
DB_PASS = os.environ["DB_PASS"]
2527
DB_USER = os.environ["DB_USER"]
26-
DB_PORT = os.environ.get("DB_PORT") or "5432"
28+
DB_PORT = os.environ.get("DB_PORT", "5432")
2729

2830
MAIL_TO = os.environ.get("MAIL_TO")
2931
MAIL_FROM = os.environ.get("MAIL_FROM")
3032
WEBHOOK = os.environ.get("WEBHOOK")
3133
WEBHOOK_METHOD = os.environ.get("WEBHOOK_METHOD")
3234
WEBHOOK_DATA = os.environ.get("WEBHOOK_DATA")
33-
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
35+
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS", "")
3436
KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
3537
FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
36-
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
38+
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS", "")
3739

3840
file_name = dt.strftime(FILENAME)
3941
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -112,8 +114,12 @@ def main():
112114
take_backup()
113115
backup_size=os.path.getsize(backup_file)
114116

115-
log("Uploading to S3")
116-
upload_backup()
117+
if AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
118+
log("Uploading to S3")
119+
upload_backup()
120+
else:
121+
log("Skipping S3 upload, no AWS credentials provided")
122+
117123
log("Pruning local backup copies")
118124
prune_local_backup_files()
119125
end_time = datetime.now()

‎11/restore.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88
BACKUP_DIR = os.environ["BACKUP_DIR"]
99

10-
S3_PATH = os.environ["S3_PATH"]
11-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
10+
S3_PATH = os.environ.get("S3_PATH", "")
11+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1212

13-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
13+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1414
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1515

1616
if not DB_NAME:
@@ -20,7 +20,7 @@
2020
DB_HOST = os.environ["DB_HOST"]
2121
DB_PASS = os.environ["DB_PASS"]
2222
DB_USER = os.environ["DB_USER"]
23-
DB_PORT = os.environ.get("DB_PORT") or "5432"
23+
DB_PORT = os.environ.get("DB_PORT", "5432")
2424

2525
file_name = sys.argv[1]
2626
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -48,7 +48,7 @@ def restore_backup():
4848
if not backup_exists():
4949
sys.stderr.write("Backup file doesn't exists!\n")
5050
sys.exit(1)
51-
51+
5252
# restore postgres-backup
5353
env = os.environ.copy()
5454
if DB_USE_ENV:
@@ -71,10 +71,10 @@ def main():
7171
else:
7272
log("Downloading database dump")
7373
download_backup()
74-
74+
7575
log("Restoring database")
7676
restore_backup()
77-
77+
7878
log("Restore complete, took %.2f seconds" % (datetime.now() - start_time).total_seconds())
7979

8080
if __name__ == "__main__":

‎12/backup.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99

1010
BACKUP_DIR = os.environ["BACKUP_DIR"]
1111

12-
S3_PATH = os.environ["S3_PATH"]
13-
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS") or "STANDARD_IA"
14-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
12+
S3_PATH = os.environ.get("S3_PATH", "")
13+
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
14+
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
15+
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS", "STANDARD_IA")
16+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1517

16-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
18+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1719
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1820

1921
if not DB_NAME:
@@ -23,17 +25,17 @@
2325
DB_HOST = os.environ["DB_HOST"]
2426
DB_PASS = os.environ["DB_PASS"]
2527
DB_USER = os.environ["DB_USER"]
26-
DB_PORT = os.environ.get("DB_PORT") or "5432"
28+
DB_PORT = os.environ.get("DB_PORT", "5432")
2729

2830
MAIL_TO = os.environ.get("MAIL_TO")
2931
MAIL_FROM = os.environ.get("MAIL_FROM")
3032
WEBHOOK = os.environ.get("WEBHOOK")
3133
WEBHOOK_METHOD = os.environ.get("WEBHOOK_METHOD")
3234
WEBHOOK_DATA = os.environ.get("WEBHOOK_DATA")
33-
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
35+
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS", "")
3436
KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
3537
FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
36-
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
38+
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS", "")
3739

3840
file_name = dt.strftime(FILENAME)
3941
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -112,8 +114,12 @@ def main():
112114
take_backup()
113115
backup_size=os.path.getsize(backup_file)
114116

115-
log("Uploading to S3")
116-
upload_backup()
117+
if AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
118+
log("Uploading to S3")
119+
upload_backup()
120+
else:
121+
log("Skipping S3 upload, no AWS credentials provided")
122+
117123
log("Pruning local backup copies")
118124
prune_local_backup_files()
119125
end_time = datetime.now()

‎12/restore.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88
BACKUP_DIR = os.environ["BACKUP_DIR"]
99

10-
S3_PATH = os.environ["S3_PATH"]
11-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
10+
S3_PATH = os.environ.get("S3_PATH", "")
11+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1212

13-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
13+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1414
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1515

1616
if not DB_NAME:
@@ -20,7 +20,7 @@
2020
DB_HOST = os.environ["DB_HOST"]
2121
DB_PASS = os.environ["DB_PASS"]
2222
DB_USER = os.environ["DB_USER"]
23-
DB_PORT = os.environ.get("DB_PORT") or "5432"
23+
DB_PORT = os.environ.get("DB_PORT", "5432")
2424

2525
file_name = sys.argv[1]
2626
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -48,7 +48,7 @@ def restore_backup():
4848
if not backup_exists():
4949
sys.stderr.write("Backup file doesn't exists!\n")
5050
sys.exit(1)
51-
51+
5252
# restore postgres-backup
5353
env = os.environ.copy()
5454
if DB_USE_ENV:
@@ -71,10 +71,10 @@ def main():
7171
else:
7272
log("Downloading database dump")
7373
download_backup()
74-
74+
7575
log("Restoring database")
7676
restore_backup()
77-
77+
7878
log("Restore complete, took %.2f seconds" % (datetime.now() - start_time).total_seconds())
7979

8080
if __name__ == "__main__":

‎13/backup.py‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99

1010
BACKUP_DIR = os.environ["BACKUP_DIR"]
1111

12-
S3_PATH = os.environ["S3_PATH"]
13-
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS") or "STANDARD_IA"
14-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
12+
S3_PATH = os.environ.get("S3_PATH", "")
13+
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID")
14+
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY")
15+
S3_STORAGE_CLASS = os.environ.get("S3_STORAGE_CLASS", "STANDARD_IA")
16+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1517

16-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
18+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1719
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1820

1921
if not DB_NAME:
@@ -23,17 +25,17 @@
2325
DB_HOST = os.environ["DB_HOST"]
2426
DB_PASS = os.environ["DB_PASS"]
2527
DB_USER = os.environ["DB_USER"]
26-
DB_PORT = os.environ.get("DB_PORT") or "5432"
28+
DB_PORT = os.environ.get("DB_PORT", "5432")
2729

2830
MAIL_TO = os.environ.get("MAIL_TO")
2931
MAIL_FROM = os.environ.get("MAIL_FROM")
3032
WEBHOOK = os.environ.get("WEBHOOK")
3133
WEBHOOK_METHOD = os.environ.get("WEBHOOK_METHOD")
3234
WEBHOOK_DATA = os.environ.get("WEBHOOK_DATA")
33-
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
35+
WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS", "")
3436
KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
3537
FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
36-
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
38+
PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS", "")
3739

3840
file_name = dt.strftime(FILENAME)
3941
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -112,8 +114,12 @@ def main():
112114
take_backup()
113115
backup_size=os.path.getsize(backup_file)
114116

115-
log("Uploading to S3")
116-
upload_backup()
117+
if AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY:
118+
log("Uploading to S3")
119+
upload_backup()
120+
else:
121+
log("Skipping S3 upload, no AWS credentials provided")
122+
117123
log("Pruning local backup copies")
118124
prune_local_backup_files()
119125
end_time = datetime.now()

‎13/restore.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88
BACKUP_DIR = os.environ["BACKUP_DIR"]
99

10-
S3_PATH = os.environ["S3_PATH"]
11-
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS") or ""
10+
S3_PATH = os.environ.get("S3_PATH", "")
11+
S3_EXTRA_OPTIONS = os.environ.get("S3_EXTRA_OPTIONS", "")
1212

13-
DB_USE_ENV = os.environ.get("DB_USE_ENV") or False
13+
DB_USE_ENV = os.environ.get("DB_USE_ENV", False)
1414
DB_NAME = os.environ["DB_NAME"] if "DB_NAME" in os.environ else os.environ.get("PGDATABASE")
1515

1616
if not DB_NAME:
@@ -20,7 +20,7 @@
2020
DB_HOST = os.environ["DB_HOST"]
2121
DB_PASS = os.environ["DB_PASS"]
2222
DB_USER = os.environ["DB_USER"]
23-
DB_PORT = os.environ.get("DB_PORT") or "5432"
23+
DB_PORT = os.environ.get("DB_PORT", "5432")
2424

2525
file_name = sys.argv[1]
2626
backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -48,7 +48,7 @@ def restore_backup():
4848
if not backup_exists():
4949
sys.stderr.write("Backup file doesn't exists!\n")
5050
sys.exit(1)
51-
51+
5252
# restore postgres-backup
5353
env = os.environ.copy()
5454
if DB_USE_ENV:
@@ -71,10 +71,10 @@ def main():
7171
else:
7272
log("Downloading database dump")
7373
download_backup()
74-
74+
7575
log("Restoring database")
7676
restore_backup()
77-
77+
7878
log("Restore complete, took %.2f seconds" % (datetime.now() - start_time).total_seconds())
7979

8080
if __name__ == "__main__":

0 commit comments

Comments
 (0)