Merge remote-tracking branch 'origin/main' into codex/launchpad-service-deploy

Author: chrisbbreuer

ops/production/README.md

@@ -16,6 +16,12 @@ The inventory places future database dump staging on the mounted deployments vol
 
 Checks: `python3 test-preserve-backups.py` on Linux as root, shell syntax checks, ts-cloud driver tests and its isolated Linux mount tests. Verify database readiness, app HTTPS, workers, persistent mount identities, fstab, backups, mail protocols and failed units after applying.
 
+The hourly backup retry also selects newly discovered projects, missing provider
+receipts, unverified restores and backups older than the health check's 36-hour
+window. It skips only fresh, recovery-verified groups on every provider. Verify
+this selection with `python3 test-backup-retry.py`; the root integration exercises
+the real retry coordinator and confirms preservation mode never prunes snapshots.
+
 ReviewOS uses a separate private, versioned `reviewos-production` bucket in `hel1`, declared in its `cloud/object-storage.ts`. The shared host's existing operator credential can be installed into the app's private credential file using `reviewos-credentials.py --source <protected-S3-env-file>`. It never emits credentials or overwrites a different key pair. Prefer a dedicated project credential when rotating keys; the current trusted application processes already run as root on this host. Keep the backup bucket and its snapshots untouched.
 
 `install-preserving.py --apply` checks all operator baselines first, saves prior versions, installs the companion systemd declarations and reloads systemd. The declarations preserve app/mail backup history, put staging behind the required volume and make automated capacity maintenance read-only. The PostgreSQL declaration requires the logging volume before startup.

ops/production/host-backup.py

@@ -35,16 +35,20 @@ def save(path, data):
     temporary.replace(path)
 
 
+def project_backup_due(status, repositories, now):
+    """Failed, newly discovered and stale groups all need a verified backup."""
+    if status.get("error"):
+        return True
+    for name in repositories:
+        recovered = status.get(name, {})
+        if not recovered.get("restore_verified") or not 0 <= now - recovered.get("timestamp", 0) <= 36 * 3600:
+            return True
+    return False
+
+
 def all_projects_recovered(config, state, now):
-    for project in config["projects"]:
-        status = state.get("projects", {}).get(project, {})
-        if status.get("error"):
-            return False
-        for name in config["repositories"]:
-            recovered = status.get(name, {})
-            if not recovered.get("restore_verified") or not 0 <= now - recovered.get("timestamp", 0) <= 36 * 3600:
-                return False
-    return True
+    return all(not project_backup_due(state.get("projects", {}).get(project, {}), config["repositories"], now)
+               for project in config["projects"])
 
 
 def sqlite_snapshot(source, target):
@@ -319,7 +323,7 @@ def main():
     parser.add_argument("--config", type=Path, default=CONFIG)
     parser.add_argument("--check", action="store_true", help="rotate through a seventh of all repository data")
     parser.add_argument("--drill", action="store_true", help="load recovered PostgreSQL/Vitess dumps in disposable instances")
-    parser.add_argument("--retry-failed", action="store_true", help="retry failed app groups without repeating successful backups")
+    parser.add_argument("--retry-failed", action="store_true", help="retry failed, missing or stale app groups without repeating fresh verified backups")
     parser.add_argument("--preserve-backups", action="store_true", help="verify fresh backups without expiring any existing snapshots")
     args = parser.parse_args()
     os.umask(0o077)
@@ -362,8 +366,9 @@ def main():
                 save(STATE, state)
                 print(f"{name}: repository check {partition}/7 passed", flush=True)
             return
-        if args.retry_failed and not any(item.get("error") for item in state.get("projects", {}).values()):
-            print("No failed app backups to retry", flush=True)
+        retry_at = time.time()
+        if args.retry_failed and all_projects_recovered(config, state, retry_at):
+            print("All app backups are fresh and recovery-verified", flush=True)
             return
         stage = Path(config.get("staging", "/var/cache/host-backup"))
         stage.mkdir(mode=0o700, parents=True, exist_ok=True)
@@ -373,7 +378,7 @@ def main():
         failures = []
         for project, specification in config["projects"].items():
             previous = state.setdefault("projects", {}).get(project, {})
-            if args.retry_failed and not previous.get("error"):
+            if args.retry_failed and not project_backup_due(previous, config["repositories"], retry_at):
                 continue
             try:
                 results, error = project_backup(project, specification, archives, stage)

ops/production/operator-baseline.json

@@ -1,7 +1,7 @@
 {
   "host-backup.py": {
     "target": "/usr/local/sbin/host-backup.py",
-    "baseline": "17a7e96756aaae920386270c63f1e8c44d67fb4b0a97c70a5ed0a96fce6653ea"
+    "baseline": "bb46505bf0c110c63d61322ceac394aa296a8cc61dbff6fcf8c978ca47274766"
   },
   "mail-backup.sh": {
     "target": "/opt/mail/backup.sh",

ops/production/test-backup-retry.py

@@ -0,0 +1,80 @@
+"""Retry selection must cover discovered and stale groups without pruning history."""
+import importlib.machinery
+import importlib.util
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import time
+import types
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).parent
+
+
+def load_backup():
+    dependency = types.ModuleType('app_backup_retention')
+    dependency.ResticArchive = lambda _: None
+    dependency.digest = lambda _: 'fixture'
+    dependency.STAGING_MARKER = '.host-backup-managed'
+    spec = importlib.util.spec_from_file_location('retry_fixture', ROOT / 'host-backup.py')
+    module = importlib.util.module_from_spec(spec)
+    with patch.dict(sys.modules, {'app_backup_retention': dependency}), patch.object(importlib.machinery.SourceFileLoader, 'load_module', return_value=types.SimpleNamespace()):
+        spec.loader.exec_module(module)
+    return module
+
+
+class RetryTests(unittest.TestCase):
+    def test_only_fresh_verified_backups_can_be_skipped(self):
+        backup = load_backup()
+        now = 2_000_000
+        fresh = {'timestamp': now - 60, 'restore_verified': True}
+        for label, state, due in [
+            ('new discovery', {}, True),
+            ('one provider missing', {'aws': fresh}, True),
+            ('both verified', {'aws': fresh, 'hetzner': fresh}, False),
+            ('explicit failure', {'aws': fresh, 'hetzner': fresh, 'error': 'failed'}, True),
+            ('stale provider', {'aws': fresh, 'hetzner': {**fresh, 'timestamp': now - 36 * 3600 - 1}}, True),
+            ('unverified restore', {'aws': fresh, 'hetzner': {**fresh, 'restore_verified': False}}, True),
+            ('future timestamp', {'aws': fresh, 'hetzner': {**fresh, 'timestamp': now + 1}}, True),
+        ]:
+            with self.subTest(label=label):
+                self.assertEqual(backup.project_backup_due(state, ['aws', 'hetzner'], now), due)
+                self.assertEqual(backup.all_projects_recovered({'projects': {'app': {}}, 'repositories': ['aws', 'hetzner']}, {'projects': {'app': state}}, now), not due)
+        self.assertFalse(backup.all_projects_recovered({'projects': {'new-app': {}}, 'repositories': ['aws', 'hetzner']}, {'projects': {}}, now))
+
+    @unittest.skipUnless(os.geteuid() == 0, 'real root-owned staging integration')
+    def test_retry_creates_missing_and_stale_backups_and_preserves_every_snapshot(self):
+        backup = load_backup()
+        now = int(time.time())
+        fresh = {name: {'timestamp': now, 'restore_verified': True} for name in ['aws', 'hetzner']}
+        stale = {name: {**value, 'timestamp': now - 37 * 3600} for name, value in fresh.items()}
+        calls, archived = [], []
+        backup.ResticArchive = lambda _: types.SimpleNamespace(run=lambda *args: archived.append(args))
+        backup.command = lambda args: None
+        def recover(project, *_):
+            calls.append(project)
+            return fresh, None
+        backup.project_backup = recover
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            config = root / 'config.json'
+            config.write_text(json.dumps({'repositories': {'aws': '/unused', 'hetzner': '/unused'}, 'projects': {name: {} for name in ['fresh', 'new', 'stale', 'failed']}, 'staging': str(root / 'stage')}))
+            backup.STATE = root / 'state.json'
+            backup.STATE.write_text(json.dumps({'projects': {'fresh': fresh, 'stale': stale, 'failed': {**fresh, 'error': 'previous failure'}}}))
+            real_open = open
+            def locked_open(file, *args, **kwargs):
+                return real_open(root / 'lock' if file == '/run/lock/host-backup.lock' else file, *args, **kwargs)
+            with patch('sys.argv', ['host-backup.py', '--config', str(config), '--retry-failed', '--preserve-backups']), patch('builtins.open', side_effect=locked_open):
+                backup.main()
+                self.assertEqual(calls, ['new', 'stale', 'failed'])
+                self.assertEqual(archived, [], 'retry must never forget or prune an existing snapshot')
+                self.assertNotIn('error', json.loads(backup.STATE.read_text())['projects']['failed'])
+                backup.main()
+                self.assertEqual(calls, ['new', 'stale', 'failed'], 'fresh recovered groups must not be repeated')
+
+
+if __name__ == '__main__':
+    unittest.main()