feat(v4): consolidate audited data recovery economic replay and optional archival

2026-10-03 10:40:18 +00:00
parent 6c91d6bcc6
commit 8eb9688942
26 changed files with 1747 additions and 38 deletions
@@ -0,0 +1,106 @@
+"""Bounded online tables with verified lossless archives; never prune unarchived data.
+
+Runs at most hourly. Raw input history: 120 days (longer than 90-day training and
+comparison limits). Ordinary plans: 7 days, excluding active/acknowledged plans.
+Archives are retained; external backup/long-term archive lifecycle is operational.
+"""
+from pathlib import Path
+import gzip
+import hashlib
+import json
+import os
+import sqlite3
+import tempfile
+
+
+def schema(con):
+    con.executescript('''
+    CREATE TABLE IF NOT EXISTS planner_maintenance(
+      name TEXT PRIMARY KEY, checked_at INTEGER NOT NULL, status TEXT NOT NULL, detail TEXT NOT NULL);
+    CREATE TABLE IF NOT EXISTS planner_archives(
+      id TEXT PRIMARY KEY, created_at INTEGER NOT NULL, filename TEXT NOT NULL,
+      sha256 TEXT NOT NULL, records INTEGER NOT NULL, kind TEXT NOT NULL);
+    CREATE INDEX IF NOT EXISTS planner_plan_created ON planner_plans(created_at);
+    ''')
+
+
+def archive_batch(con,directory,now,kind,limit=100):
+    if kind not in ('observations','plans') or type(limit) is not int or not 1<=limit<=1000:
+        raise ValueError('Invalid archival scope')
+    directory=Path(directory)
+    if directory.is_symlink():raise ValueError('Archive symlink refused')
+    directory.mkdir(mode=0o700,parents=True,exist_ok=True)
+    if directory.stat().st_mode & 0o007:raise ValueError('Archive directory must not be public')
+    from datetime import datetime,timezone
+    with_context=False
+    con.execute('BEGIN IMMEDIATE')
+    try:
+        if kind=='observations':
+            rows=con.execute('SELECT * FROM planner_observations WHERE captured_at<? AND received_at<? ORDER BY captured_at LIMIT ?',
+                             (now-120*86400,now-7*86400,limit)).fetchall()
+        else:
+            before=datetime.fromtimestamp(now-7*86400,timezone.utc).isoformat()
+            rows=con.execute('SELECT * FROM planner_plans p WHERE created_at<? AND NOT EXISTS(SELECT 1 FROM planner_current c WHERE c.plan_id=p.plan_id) AND NOT EXISTS(SELECT 1 FROM planner_ack a WHERE a.plan_id=p.plan_id) ORDER BY created_at LIMIT ?',
+                             (before,limit)).fetchall()
+        if not rows:con.commit();return {'archived':0,'kind':kind}
+        records=[]
+        for r in rows:
+            item={'table':kind,'row':dict(r)}
+            if kind=='observations':
+                item['origins']=[dict(x) for x in con.execute('SELECT * FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',(r['plant'],r['dataset'],r['captured_at']))]
+            records.append(json.dumps(item,sort_keys=True,separators=(',',':'),allow_nan=False).encode()+b'\n')
+        raw=b''.join(records)
+        if len(raw)>67108864:raise ValueError('Archive batch exceeds memory budget')
+        ident=hashlib.sha256(raw).hexdigest();name=kind+'-'+ident+'.jsonl.gz';path=directory/name
+        if path.is_symlink():raise ValueError('Archive target symlink refused')
+        if not path.exists():
+            fd,tmp=tempfile.mkstemp(prefix='.archive-',dir=directory)
+            try:
+                os.fchmod(fd,0o600)
+                with os.fdopen(fd,'wb') as out:
+                    with gzip.GzipFile(fileobj=out,mode='wb',mtime=0) as zipped:zipped.write(raw)
+                    out.flush();os.fsync(out.fileno())
+                os.replace(tmp,path)
+                dfd=os.open(directory,os.O_RDONLY)
+                try:os.fsync(dfd)
+                finally:os.close(dfd)
+            finally:
+                if os.path.exists(tmp):os.unlink(tmp)
+        # Read back exact bytes before removing any database row.
+        with gzip.open(path,'rb') as f:verified=f.read(len(raw)+1)
+        if verified!=raw:raise ValueError('Archive verification failed; original rows retained')
+        for r in rows:
+            if kind=='observations':
+                key=(r['plant'],r['dataset'],r['captured_at'])
+                con.execute('DELETE FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',key)
+                con.execute('DELETE FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=? AND fingerprint=?',(*key,r['fingerprint']))
+            else:con.execute('DELETE FROM planner_plans WHERE plan_id=?',(r['plan_id'],))
+        con.execute('INSERT OR IGNORE INTO planner_archives VALUES(?,?,?,?,?,?)',(ident,now,name,ident,len(rows),kind))
+        con.commit()
+        return {'archived':len(rows),'kind':kind,'file':name,'sha256':ident,'sourceRecoverable':True}
+    except Exception:
+        con.rollback();raise
+
+
+def maintain(store,now):
+    if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return
+    con=store.con;stamp=int(now.timestamp());old=con.execute("SELECT checked_at FROM planner_maintenance WHERE name='archive'").fetchone()
+    if old and stamp-old[0]<3600:return
+    db=con.execute('PRAGMA database_list').fetchone()[2]
+    if not db:return  # In-memory test/ephemeral databases have no archival location.
+    try:
+        directory=Path(db).resolve().parent/'archives'
+        results=[archive_batch(con,directory,stamp,kind) for kind in ('plans','observations')]
+        if any(r['archived']>=100 for r in results):stamp-=3300
+        state='ok';detail={'results':results,'archivesRetained':True,'rawRetentionDays':120,'ordinaryPlanRetentionDays':7}
+    except (OSError,ValueError,sqlite3.Error) as exc:
+        state='archive_error';detail={'errorType':type(exc).__name__,'unverifiedDataNotDeleted':True}
+    with con:con.execute('INSERT INTO planner_maintenance VALUES(?,?,?,?) ON CONFLICT(name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,detail=excluded.detail',
+                         ('archive',stamp,state,json.dumps(detail,separators=(',',':'))))
+
+
+def status(con):
+    if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return {'status':'disabled_requires_operator_opt_in'}
+    r=con.execute("SELECT checked_at,status,detail FROM planner_maintenance WHERE name='archive'").fetchone()
+    # Public per-plant state must not expose archival filenames/rows of other tenants.
+    return {'checkedAtEpoch':r[0],'status':r[1],'rawRetentionDays':120,'ordinaryPlanRetentionDays':7,'archivesRetained':True} if r else {'status':'not_yet_run'}