feat(v4): consolidate audited data recovery economic replay and optional archival

services/netplan-v4/netplan_v4/economic_replay.py

@@ -0,0 +1,207 @@
+"""Causal daily economic family comparison using frozen plans and later physical data.
+
+This is a labelled *simulation*, not savings on an invoice. A cohort is captured
+near local midnight, uses only prices/forecasts then available, and all families
+share initial storage, physical outcomes, tariff and constraints. No device calls.
+Supported v1: one grid-charge-enabled battery; other topologies remain explicit.
+"""
+from dataclasses import asdict, replace
+from datetime import datetime, timedelta
+from hashlib import sha256
+from math import sqrt
+import json
+from .domain import ZURICH, utc, month_key, quarter_start
+from .selection import ReplayScore
+from . import measurement_pipeline as m
+
+
+def schema(con):
+    con.executescript('''
+    CREATE TABLE IF NOT EXISTS planner_economic_cohorts(
+      plant TEXT NOT NULL, local_day TEXT NOT NULL, issued_at INTEGER NOT NULL,
+      ends_at INTEGER NOT NULL, policy_id TEXT NOT NULL, value TEXT NOT NULL,
+      PRIMARY KEY(plant,local_day));
+    CREATE TABLE IF NOT EXISTS planner_economic_results(
+      plant TEXT NOT NULL, local_day TEXT NOT NULL, evaluated_at INTEGER NOT NULL,
+      policy_id TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(plant,local_day));
+    CREATE TABLE IF NOT EXISTS planner_economic_attempts(
+      plant TEXT NOT NULL, local_day TEXT NOT NULL, checked_at INTEGER NOT NULL,
+      reason TEXT NOT NULL, PRIMARY KEY(plant,local_day));
+    ''')
+
+
+def _policy(data, cfg):
+    batteries=[]
+    for b in data['batteries']:
+        v=asdict(b)
+        for k in ('soc_percent','measured_at','discharge_blocked'):v.pop(k)
+        batteries.append(v)
+    return sha256(m.canonical({'battery':batteries,'limits':asdict(data['limits']),
+        'mapping':cfg['mappingSha256'],'formula':cfg['formula'],'dataset':cfg['datasetId'],
+        'method':'frozen_day_ahead_grid_tracking_v1','external':'sdl_request_estimate','peakTariffs':data['peak_prices']}).encode()).hexdigest()
+
+
+def capture(store, plant, now, assemble, optimizer):
+    settings=store.settings(plant)
+    if settings.get('forecastSource')!='corrected_profile': return
+    local=utc(now).astimezone(ZURICH); day=local.date().isoformat()
+    # Freeze only at the beginning of a local day. Never reconstruct an old forecast from hindsight.
+    if local.hour!=0 or local.minute>=15:return
+    if store.con.execute('SELECT 1 FROM planner_economic_cohorts WHERE plant=? AND local_day=?',(plant,day)).fetchone():return
+    attempted=store.con.execute('SELECT checked_at FROM planner_economic_attempts WHERE plant=? AND local_day=?',(plant,day)).fetchone()
+    if attempted and now.timestamp()-attempted[0]<300:return
+    reason='awaiting_comparable_snapshot'
+    try:
+        cfg=m.configuration(store.con,plant,settings['measurementDataset']);plans={}; common=None;policy=None
+        end=datetime.combine(local.date()+timedelta(days=1),datetime.min.time(),tzinfo=ZURICH).astimezone(utc(now).tzinfo)
+        for family in store.registry.entries():
+            data,_,quality=assemble(store,plant,family.key,now)
+            data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']]
+            if len(data['batteries'])!=1 or not data['batteries'][0].grid_charging:
+                raise ValueError('unsupported_replay_topology')
+            if data['steps'][-1].end<end:raise ValueError('published_prices_do_not_cover_day')
+            p=optimizer(**data,config_revision=settings['revision'],family=family.key,timeout_seconds=1.0)
+            if not p.get('executable'):raise ValueError('candidate_not_feasible')
+            points=[x for x in p['points'] if utc(x['time'])<end]
+            p['points']=points
+            if utc(points[-1]['validUntil'])!=end:raise ValueError('day_boundary_not_covered')
+            this=_policy(data,cfg)
+            if policy is not None and this!=policy:raise ValueError('inconsistent_candidate_context')
+            policy=this
+            initial=asdict(data['batteries'][0]);initial['measured_at']=utc(initial['measured_at']).isoformat()
+            context={'battery':initial,'limits':asdict(data['limits']),'peaks':data['observed_peaks'],
+                     'peakTariffs':data['peak_prices'],'quarterPast':{utc(k).isoformat():asdict(v) for k,v in data['quarter_history'].items()}}
+            if common is not None and m.canonical(common)!=m.canonical(context):raise ValueError('different_initial_conditions')
+            common=context;plans[family.key]=p
+        billing=[[(p['time'],p['validUntil'],p['importPriceChfKwh'],p['exportPriceChfKwh']) for p in plan['points']] for plan in plans.values()]
+        if any(v!=billing[0] for v in billing):raise ValueError('Different priced intervals between families')
+        value={'datasetId':cfg['datasetId'],'start':next(iter(plans.values()))['points'][0]['time'],
+               'end':end.isoformat(),'context':common,'plans':plans,'policyId':policy,
+               'method':'frozen_day_ahead_grid_tracking_v1','actuation':False}
+        with store.con:store.con.execute('INSERT INTO planner_economic_cohorts VALUES(?,?,?,?,?,?)',
+                 (plant,day,int(now.timestamp()),int(end.timestamp()),policy,m.canonical(value)))
+        reason='captured'
+    except (ValueError,KeyError,TypeError):
+        # No exception text copied from arbitrary data. Retry bounded to one attempt per five minutes.
+        reason='awaiting_comparable_snapshot'
+    with store.con:store.con.execute('INSERT INTO planner_economic_attempts VALUES(?,?,?,?) ON CONFLICT(plant,local_day) DO UPDATE SET checked_at=excluded.checked_at,reason=excluded.reason',
+                                      (plant,day,int(now.timestamp()),reason))
+
+
+def actuals(records,cfg):
+    required=[s['key'] for s in cfg['sources'] if s['role'] in ('grid','physical_storage','sdl_request')]
+    if sum(s['role']=='sdl_request' for s in cfg['sources'])!=1:raise ValueError('Missing SDL outcome source')
+    def residual(values,c):
+        grid=storage=external=0.0
+        for s in c['sources']:
+            role=s['role']
+            if role not in ('grid','physical_storage','sdl_request'):continue
+            x=values[s['key']]*s['factorToW']
+            if role=='grid':grid+=x
+            elif role=='physical_storage':storage+=x
+            else:external+=x
+        # External SDL is an explicitly estimated historical contribution, never called a meter.
+        return grid-storage+external
+    return m.reconstruct(records,cfg,projection={'keys':required,'calculate':residual})
+
+
+def simulate(cohort,family,outcomes):
+    plan=cohort['plans'][family];ctx=cohort['context'];b=ctx['battery'];limits=ctx['limits']
+    capacity=b['capacity_kwh'];initial=energy=capacity*b['soc_percent']/100
+    low=capacity*b['min_soc_percent']/100; high=capacity*b['max_soc_percent']/100
+    eta=sqrt(b['roundtrip_efficiency']);peaks=dict(ctx['peaks']);quarter_max={};quarters={};money=wear=0.; coverage=[];breaches=0
+    blocked=b['discharge_blocked'];rearm=capacity*(b['rearm_soc_percent'] or b['min_soc_percent'])/100
+    for stamp,q in ctx['quarterPast'].items():quarters[stamp]=[q['import_kwh'],q['measured_seconds']]
+    for p in plan['points']:
+        start,end=utc(p['time']),utc(p['validUntil']);t=int(start.timestamp());seconds=int((end-start).total_seconds());dt=seconds/3600
+        w=outcomes.get(t//300*300)
+        if not w or not w['profileUsable'] or not m.numeric(w['loadW']):raise ValueError('Incomplete actual outcome')
+        residual=w['loadW'];coverage.append(w['coverage'])
+        charge=min(b['max_charge_w'],max(0.,(high-energy)/eta/dt*1000))
+        if blocked and energy>=rearm-1e-9:blocked=False
+        discharge=0. if blocked else min(b['max_discharge_w'],max(0.,(energy-low)*eta/dt*1000))
+        target=p['gridTargetW'];cap=limits['import_w'];monthly=limits['manager_month_limits_w'].get(str(start.astimezone(ZURICH).month),limits['manager_month_limits_w'].get(start.astimezone(ZURICH).month))
+        if monthly is not None:cap=monthly if cap is None else min(cap,monthly)
+        if cap is not None:target=min(target,cap)
+        if limits['export_w'] is not None:target=max(target,-limits['export_w'])
+        battery=min(charge,max(-discharge,target-residual));grid=residual+battery
+        energy+=battery/1000*dt*(eta if battery>=0 else 1/eta)
+        if energy<low-1e-6 or energy>high+1e-6:raise ValueError('Replay SOC invariant')
+        if energy<=low+1e-9:blocked=True
+        if cap is not None and grid>cap+1.:breaches+=1
+        if limits['export_w'] is not None and grid < -limits['export_w']-1.:breaches+=1
+        money+=(max(grid,0)*p['importPriceChfKwh']-max(-grid,0)*p['exportPriceChfKwh'])/1000*dt
+        wear+=abs(battery)/1000*dt*b['throughput_chf_kwh']
+        q=quarter_start(start).isoformat();v=quarters.setdefault(q,[0.,0]);v[0]+=max(grid,0)/1000*dt;v[1]+=seconds
+        if end==quarter_start(start)+timedelta(minutes=15):
+            if v[1]!=900:raise ValueError('Incomplete simulated billing quarter')
+            month=month_key(start);peaks[month]=max(peaks[month],v[0]/.25);quarter_max[month]=max(quarter_max.get(month,0.),v[0]/.25)
+    additional=sum(max(0,v-ctx['peaks'][month])*ctx['peakTariffs'][month] for month,v in peaks.items())
+    # Identical terminal valuation for all families, known at cohort creation; separate from cash.
+    last=plan['points'][-1];buy=max(0.,last['importPriceChfKwh']);sell=max(0.,min(buy,last['exportPriceChfKwh']))
+    terminal=max(initial-energy,0)/eta*buy-max(energy-initial,0)*eta*sell
+    return {'cashCostChf':money+additional,'energyCostChf':money,'peakCostChf':additional,'throughputCostChf':wear,
+            'terminalAdjustmentChf':terminal,'costChf':money+additional+wear+terminal,
+            'initialEnergyKwh':initial,'finalEnergyKwh':energy,'coverage':min(coverage),
+            'quarterMaximaKw':quarter_max,'initialPeaksKw':ctx['peaks'],'peakTariffs':ctx['peakTariffs'],
+            'constraintBreaches':breaches,'terminalNormalized':True,
+            'terminalMethod':'common_known_end_price_inventory_valuation_not_physical_restoration'}
+
+
+def advance(store,plant,now):
+    timestamp=int(now.timestamp())
+    rows=store.con.execute('SELECT c.* FROM planner_economic_cohorts c LEFT JOIN planner_economic_results r USING(plant,local_day) WHERE c.plant=? AND c.ends_at<=? AND c.ends_at>=? AND r.local_day IS NULL AND NOT EXISTS(SELECT 1 FROM planner_economic_attempts a WHERE a.plant=c.plant AND a.local_day=c.local_day AND a.checked_at>?) ORDER BY c.issued_at LIMIT 1',(plant,timestamp-120,timestamp-90*86400,timestamp-300)).fetchall()
+    for row in rows:
+        with store.con:store.con.execute('INSERT INTO planner_economic_attempts VALUES(?,?,?,?) ON CONFLICT(plant,local_day) DO UPDATE SET checked_at=excluded.checked_at,reason=excluded.reason',(plant,row['local_day'],timestamp,'evaluating_actuals'))
+        try:
+            cohort=json.loads(row['value']);cfg=m.configuration(store.con,plant,cohort['datasetId']);source=cfg.get('sourceDatasetId',cfg['datasetId']);start=m.epoch(cohort['start']);end=m.epoch(cohort['end'])
+            observed=store.con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',(plant,source,start-600,end+600,timestamp))
+            windows=actuals([json.loads(r[0]) for r in observed],cfg);outcomes={w['start']:w for w in windows}
+            results={f:simulate(cohort,f,outcomes) for f in cohort['plans']}
+            value={'status':'evaluated','results':results,'start':cohort['start'],'end':cohort['end'],
+                   'evaluationBasis':'physical_balance_with_sdl_request_estimate','method':cohort['method'],
+                   'actualCashSavings':False,'controlEnabled':False}
+            with store.con:store.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(plant,row['local_day'],timestamp,row['policy_id'],m.canonical(value)))
+        except (ValueError,KeyError,TypeError):
+            # Retain unscored cohort. Lack of actual data must never become a zero cost.
+            pass
+
+
+def scores(store,plant,now):
+    settings=store.settings(plant)
+    if settings.get('forecastSource')!='corrected_profile':return []
+    since=int((utc(now)-timedelta(days=settings['autoLookbackDays'])).timestamp())
+    recent=store.con.execute('SELECT policy_id,value FROM planner_economic_cohorts WHERE plant=? ORDER BY issued_at DESC LIMIT 1',(plant,)).fetchone()
+    if not recent or json.loads(recent[1])['datasetId']!=settings.get('measurementDataset'):return []
+    rows=store.con.execute('SELECT c.issued_at,c.ends_at,r.evaluated_at,r.value FROM planner_economic_results r JOIN planner_economic_cohorts c USING(plant,local_day) WHERE r.plant=? AND r.policy_id=? AND c.issued_at>=? AND r.evaluated_at<=? ORDER BY c.issued_at',(plant,recent[0],since,int(now.timestamp()))).fetchall()
+    if not rows:return []
+    keys=[f.key for f in store.registry.entries()];valid=[]
+    for r in rows:
+        v=json.loads(r['value']);out=v['results']
+        if set(out)!=set(keys) or any(out[k]['constraintBreaches'] for k in keys):continue
+        valid.append((r,v))
+    if not valid:return []
+    result=[]
+    for k in keys:
+        # Monthly demand cost is paid ONCE for the maximum, not once per replay day.
+        total=sum(v['results'][k]['energyCostChf']+v['results'][k]['throughputCostChf']+v['results'][k]['terminalAdjustmentChf'] for _,v in valid)
+        bases={}; maxima={}; rates={}
+        for _,v in valid:
+            d=v['results'][k]
+            for month,peak in d['quarterMaximaKw'].items():
+                bases.setdefault(month,d['initialPeaksKw'][month])
+                maxima[month]=max(maxima.get(month,0.),peak);rates[month]=d['peakTariffs'][month]
+        total+=sum(max(0.,maxima[month]-bases[month])*rates[month] for month in maxima)
+        cover=min(v['results'][k]['coverage'] for _,v in valid)
+        first=utc(valid[0][1]['start']);end=utc(valid[-1][1]['end']);available=datetime.fromtimestamp(max(r['evaluated_at'] for r,_ in valid),utc(now).tzinfo)
+        result.append(ReplayScore(k,recent[0],first,end,first,available,total,cover,len(valid)))
+    return result
+
+
+def status(store,plant):
+    rows=store.con.execute('SELECT local_day,value FROM planner_economic_results WHERE plant=? ORDER BY local_day DESC LIMIT 14',(plant,))
+    evaluated=[{'day':r[0],**json.loads(r[1])} for r in rows]
+    count=store.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts WHERE plant=?',(plant,)).fetchone()[0]
+    return {'connected':True,'method':'frozen_day_ahead_grid_tracking_v1','cohorts':count,'completedComparisons':evaluated,
+            'evaluationBasis':'physical_balance_with_sdl_request_estimate','isBillingEvidence':False,
+            'limitations':['one_grid_charging_battery','frozen_daily_plan_not_receding_horizon_field_replay']}

services/netplan-v4/netplan_v4/mapping_identity.py

@@ -0,0 +1,100 @@
+"""Audited numeric-representation compatibility, not relaxed device identity checks.
+
+Only the internal operator API can register equivalence. Raw journals are never
+rewritten, and the original received fingerprint is retained with each receipt.
+"""
+from hashlib import sha256
+import json
+import math
+import re
+
+
+def schema(con):
+    con.executescript('''
+    CREATE TABLE IF NOT EXISTS planner_mapping_compatibility(
+      plant TEXT NOT NULL, dataset TEXT NOT NULL, alias TEXT NOT NULL,
+      canonical TEXT NOT NULL, inventory TEXT NOT NULL, evidence TEXT NOT NULL,
+      created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset,alias));
+    CREATE TABLE IF NOT EXISTS planner_observation_origins(
+      plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL,
+      received_mapping TEXT NOT NULL, inventory TEXT NOT NULL,
+      evidence_id TEXT, received_at INTEGER NOT NULL,
+      PRIMARY KEY(plant,dataset,captured_at,received_mapping));
+    ''')
+
+
+def _decode(text):
+    if not isinstance(text, str) or len(text.encode()) > 131072:
+        raise ValueError('Bounded configuration evidence required')
+    def pairs(items):
+        d = {}
+        for k, v in items:
+            if k in d: raise ValueError('Duplicate configuration key')
+            d[k] = v
+        return d
+    def constant(_): raise ValueError('Nonfinite configuration')
+    return json.loads(text, object_pairs_hook=pairs, parse_constant=constant)
+
+
+def _same(a, b):
+    if type(a) is not type(b): return False
+    if isinstance(a, dict):
+        return list(a) == list(b) and all(_same(a[k], b[k]) for k in a)
+    if isinstance(a, list): return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b))
+    return a == b
+
+
+def validate_evidence(plant, config, payload):
+    if not isinstance(payload, dict) or set(payload) != {'version', 'canonicalJson', 'legacyJson'} or type(payload['version']) is not int or payload['version'] != 1:
+        raise ValueError('Explicit versioned representation evidence required')
+    ca, le = payload['canonicalJson'], payload['legacyJson']
+    a, b = _decode(ca), _decode(le)
+    canonical_hash, alias = sha256(ca.encode()).hexdigest(), sha256(le.encode()).hexdigest()
+    if canonical_hash != config['mappingSha256'] or alias == canonical_hash:
+        raise ValueError('Evidence does not match configured mapping')
+    for c in (a, b):
+        if not isinstance(c, dict) or c.get('installationId') != plant or c.get('reportedInventorySha256') != config['inventorySha256']:
+            raise ValueError('Evidence belongs to another installation or inventory')
+    x = a.get('accounting', {}).get('splitToleranceW')
+    y = b.get('accounting', {}).get('splitToleranceW')
+    if type(x) is not float or type(y) is not int or not math.isfinite(x) or x != y or not 0 <= x <= 500:
+        raise ValueError('Only demonstrated float/integer tolerance representation is compatible')
+    b['accounting']['splitToleranceW'] = float(y)
+    if not _same(a, b):
+        raise ValueError('Other configuration differences are not representation compatibility')
+    # Proof includes the exact hashed JSON strings; no arbitrary labels as evidence.
+    evidence = json.dumps(payload, sort_keys=True, separators=(',', ':'), allow_nan=False)
+    return {'alias': alias, 'canonical': canonical_hash, 'inventory': config['inventorySha256'],
+            'evidenceId': sha256(evidence.encode()).hexdigest(), 'evidence': evidence}
+
+
+def register(con, plant, config, payload, now):
+    if config.get('sourceDatasetId'): raise ValueError('Register compatibility on original dataset only')
+    v = validate_evidence(plant, config, payload)
+    con.execute('BEGIN IMMEDIATE')
+    try:
+        old = con.execute('SELECT canonical,inventory,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND alias=?',
+                          (plant, config['datasetId'], v['alias'])).fetchone()
+        expected = (v['canonical'], v['inventory'], v['evidence'])
+        if old and tuple(old) != expected: raise ValueError('Immutable mapping compatibility conflict')
+        con.execute('INSERT OR IGNORE INTO planner_mapping_compatibility VALUES(?,?,?,?,?,?,?)',
+                    (plant, config['datasetId'], v['alias'], *expected, now))
+        con.commit()
+    except Exception:
+        con.rollback(); raise
+    return {'status': 'registered', 'datasetId': config['datasetId'], 'evidenceId': v['evidenceId'],
+            'compatibleMapping': v['alias'], 'canonicalMapping': v['canonical'], 'controlEnabled': False}
+
+
+def approved(con, plant, config):
+    rows = con.execute('SELECT alias,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND canonical=? AND inventory=?',
+                       (plant, config['datasetId'], config['mappingSha256'], config['inventorySha256']))
+    return {r['alias']: sha256(r['evidence'].encode()).hexdigest() for r in rows}
+
+
+def save_origin(con, plant, config, record, captured_at, received_at, aliases):
+    source = record['mappingSha256']
+    if source != config['mappingSha256'] and source not in aliases:
+        raise ValueError('Unknown mapping; no receipt written')
+    con.execute('INSERT OR IGNORE INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)',
+                (plant, config['datasetId'], captured_at, source, config['inventorySha256'], aliases.get(source), received_at))

services/netplan-v4/netplan_v4/measurement_pipeline.py

@@ -14,6 +14,7 @@ from statistics import median
 from zoneinfo import ZoneInfo
 import json
 from .history_timing import validate_policy, endpoint_bridges
+from . import mapping_identity
 
 UTC = timezone.utc
 LOCAL = ZoneInfo('Europe/Zurich')
@@ -42,6 +43,7 @@ def numeric(value, bound=1e12):
 
 
 def schema(con):
+    mapping_identity.schema(con)
     con.executescript('''
     CREATE TABLE IF NOT EXISTS planner_data_sets(
       plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL,
@@ -162,10 +164,11 @@ def configuration(con, plant, dataset):
     return json.loads(row[0])
 
 
-def project(record, c, plant, now):
+def project(record, c, plant, now, approved_mappings=()):
     if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant:
         raise ValueError('Wrong capture identity')
-    if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']:
+    incoming = record.get('mappingSha256')
+    if not isinstance(incoming, str) or (incoming != c['mappingSha256'] and incoming not in approved_mappings) or record.get('reportedInventorySha256') != c['inventorySha256']:
         raise ValueError('Wrong capture mapping or inventory')
     t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt'))
     if start > t or t > now+30 or t < now-90*86400:
@@ -196,13 +199,14 @@ def ingest_batch(con, plant, payload, now):
     records = payload['records']
     if not isinstance(records,list) or not 1 <= len(records) <= 120:
         raise ValueError('Batch requires 1..120 captures')
-    rows = [project(r,c,plant,now) for r in records]
+    aliases = mapping_identity.approved(con, plant, c)
+    rows = [project(r,c,plant,now,aliases) for r in records]
     if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])):
         raise ValueError('Batch must be in increasing capture order')
     stored = duplicate = 0
     con.execute('BEGIN IMMEDIATE')
     try:
-        for r in rows:
+        for original, r in zip(records, rows):
             value = canonical(r); digest = sha256(value.encode()).hexdigest()
             old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone()
             if old:
@@ -211,6 +215,7 @@ def ingest_batch(con, plant, payload, now):
                 duplicate += 1
             else:
                 con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1
+            mapping_identity.save_origin(con, plant, c, original, r['capturedAt'], now, aliases)
         con.commit()
     except Exception:
         con.rollback(); raise
@@ -243,7 +248,7 @@ def physical_value(values, c):
     return load
 
 
-def reconstruct(records, c):
+def reconstruct(records, c, *, projection=None):
     """Bounded retrospective estimation, never a real-time feedback signal.
 
 Missing observations split support. Source timestamps are not refreshed. Small
@@ -260,6 +265,11 @@ uncovered portions remain quantified and are never filled with zero.
         for s in c['sources']:
             if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s
     timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources'])
+    if projection is not None:
+        configured={s['key']:s for s in c['sources']}
+        if set(projection)!= {'keys','calculate'} or not callable(projection['calculate']) or not projection['keys'] or not set(projection['keys']) <= set(configured):
+            raise ValueError('Invalid internal projection')
+        primary={k:configured[k] for k in projection['keys']}
     first,last = records[0]['capturedAt'],records[-1]['capturedAt']
     series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = []
     first_observed = {k:{} for k in primary}
@@ -313,7 +323,9 @@ uncovered portions remain quantified and are never filled with zero.
                     extended.append(k); known_at = max(known_at, bridge['availableAt'])
         load = None
         if usable:
-            try: load = physical_value(vals,c)
+            try:
+                load = physical_value(vals,c) if projection is None else projection['calculate'](vals,c)
+                if not numeric(load,1e9):raise ValueError('Invalid historical projection')
             except ValueError: usable = False
         if usable:
             item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0
@@ -490,9 +502,10 @@ def pipeline_status(con,plant):
     out=[]
     for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)):
         ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone()
-        count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone()
+        count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at),MAX(received_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone()
         out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0],
                     'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None,
+                    'lastReceived':iso(count[3]) if count[3] else None,
                     'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {},
                     'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'],
                     'observationDatasetId':c.get('sourceDatasetId',ds),

services/netplan-v4/netplan_v4/retention.py

@@ -0,0 +1,106 @@
+"""Bounded online tables with verified lossless archives; never prune unarchived data.
+
+Runs at most hourly. Raw input history: 120 days (longer than 90-day training and
+comparison limits). Ordinary plans: 7 days, excluding active/acknowledged plans.
+Archives are retained; external backup/long-term archive lifecycle is operational.
+"""
+from pathlib import Path
+import gzip
+import hashlib
+import json
+import os
+import sqlite3
+import tempfile
+
+
+def schema(con):
+    con.executescript('''
+    CREATE TABLE IF NOT EXISTS planner_maintenance(
+      name TEXT PRIMARY KEY, checked_at INTEGER NOT NULL, status TEXT NOT NULL, detail TEXT NOT NULL);
+    CREATE TABLE IF NOT EXISTS planner_archives(
+      id TEXT PRIMARY KEY, created_at INTEGER NOT NULL, filename TEXT NOT NULL,
+      sha256 TEXT NOT NULL, records INTEGER NOT NULL, kind TEXT NOT NULL);
+    CREATE INDEX IF NOT EXISTS planner_plan_created ON planner_plans(created_at);
+    ''')
+
+
+def archive_batch(con,directory,now,kind,limit=100):
+    if kind not in ('observations','plans') or type(limit) is not int or not 1<=limit<=1000:
+        raise ValueError('Invalid archival scope')
+    directory=Path(directory)
+    if directory.is_symlink():raise ValueError('Archive symlink refused')
+    directory.mkdir(mode=0o700,parents=True,exist_ok=True)
+    if directory.stat().st_mode & 0o007:raise ValueError('Archive directory must not be public')
+    from datetime import datetime,timezone
+    with_context=False
+    con.execute('BEGIN IMMEDIATE')
+    try:
+        if kind=='observations':
+            rows=con.execute('SELECT * FROM planner_observations WHERE captured_at<? AND received_at<? ORDER BY captured_at LIMIT ?',
+                             (now-120*86400,now-7*86400,limit)).fetchall()
+        else:
+            before=datetime.fromtimestamp(now-7*86400,timezone.utc).isoformat()
+            rows=con.execute('SELECT * FROM planner_plans p WHERE created_at<? AND NOT EXISTS(SELECT 1 FROM planner_current c WHERE c.plan_id=p.plan_id) AND NOT EXISTS(SELECT 1 FROM planner_ack a WHERE a.plan_id=p.plan_id) ORDER BY created_at LIMIT ?',
+                             (before,limit)).fetchall()
+        if not rows:con.commit();return {'archived':0,'kind':kind}
+        records=[]
+        for r in rows:
+            item={'table':kind,'row':dict(r)}
+            if kind=='observations':
+                item['origins']=[dict(x) for x in con.execute('SELECT * FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',(r['plant'],r['dataset'],r['captured_at']))]
+            records.append(json.dumps(item,sort_keys=True,separators=(',',':'),allow_nan=False).encode()+b'\n')
+        raw=b''.join(records)
+        if len(raw)>67108864:raise ValueError('Archive batch exceeds memory budget')
+        ident=hashlib.sha256(raw).hexdigest();name=kind+'-'+ident+'.jsonl.gz';path=directory/name
+        if path.is_symlink():raise ValueError('Archive target symlink refused')
+        if not path.exists():
+            fd,tmp=tempfile.mkstemp(prefix='.archive-',dir=directory)
+            try:
+                os.fchmod(fd,0o600)
+                with os.fdopen(fd,'wb') as out:
+                    with gzip.GzipFile(fileobj=out,mode='wb',mtime=0) as zipped:zipped.write(raw)
+                    out.flush();os.fsync(out.fileno())
+                os.replace(tmp,path)
+                dfd=os.open(directory,os.O_RDONLY)
+                try:os.fsync(dfd)
+                finally:os.close(dfd)
+            finally:
+                if os.path.exists(tmp):os.unlink(tmp)
+        # Read back exact bytes before removing any database row.
+        with gzip.open(path,'rb') as f:verified=f.read(len(raw)+1)
+        if verified!=raw:raise ValueError('Archive verification failed; original rows retained')
+        for r in rows:
+            if kind=='observations':
+                key=(r['plant'],r['dataset'],r['captured_at'])
+                con.execute('DELETE FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',key)
+                con.execute('DELETE FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=? AND fingerprint=?',(*key,r['fingerprint']))
+            else:con.execute('DELETE FROM planner_plans WHERE plan_id=?',(r['plan_id'],))
+        con.execute('INSERT OR IGNORE INTO planner_archives VALUES(?,?,?,?,?,?)',(ident,now,name,ident,len(rows),kind))
+        con.commit()
+        return {'archived':len(rows),'kind':kind,'file':name,'sha256':ident,'sourceRecoverable':True}
+    except Exception:
+        con.rollback();raise
+
+
+def maintain(store,now):
+    if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return
+    con=store.con;stamp=int(now.timestamp());old=con.execute("SELECT checked_at FROM planner_maintenance WHERE name='archive'").fetchone()
+    if old and stamp-old[0]<3600:return
+    db=con.execute('PRAGMA database_list').fetchone()[2]
+    if not db:return  # In-memory test/ephemeral databases have no archival location.
+    try:
+        directory=Path(db).resolve().parent/'archives'
+        results=[archive_batch(con,directory,stamp,kind) for kind in ('plans','observations')]
+        if any(r['archived']>=100 for r in results):stamp-=3300
+        state='ok';detail={'results':results,'archivesRetained':True,'rawRetentionDays':120,'ordinaryPlanRetentionDays':7}
+    except (OSError,ValueError,sqlite3.Error) as exc:
+        state='archive_error';detail={'errorType':type(exc).__name__,'unverifiedDataNotDeleted':True}
+    with con:con.execute('INSERT INTO planner_maintenance VALUES(?,?,?,?) ON CONFLICT(name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,detail=excluded.detail',
+                         ('archive',stamp,state,json.dumps(detail,separators=(',',':'))))
+
+
+def status(con):
+    if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return {'status':'disabled_requires_operator_opt_in'}
+    r=con.execute("SELECT checked_at,status,detail FROM planner_maintenance WHERE name='archive'").fetchone()
+    # Public per-plant state must not expose archival filenames/rows of other tenants.
+    return {'checkedAtEpoch':r[0],'status':r[1],'rawRetentionDays':120,'ordinaryPlanRetentionDays':7,'archivesRetained':True} if r else {'status':'not_yet_run'}

services/netplan-v4/netplan_v4/service.py

@@ -16,7 +16,7 @@ from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_star
 from .store import PlannerStore,canonical
 from .selection import choose_family
 from .optimizer import optimize
-from . import meter_runtime, controlled_trial, measurement_pipeline
+from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline
 from .forecast_quality import assess_family
 from .receiver_contract import provenance
 from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month
@@ -267,10 +267,12 @@ def assemble(store,plant,family,now):
     return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)}
 
 def run_once(store,now):
+    retention.maintain(store,now)
     stamp=int(now.timestamp())//300
     plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')]
     for plant in plants:
         config=store.settings(plant)
+        economic_replay.advance(store,plant,now)
         datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))]
         for dataset in datasets:
             try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp()))
@@ -286,14 +288,16 @@ def run_once(store,now):
     try:
         settings=store.settings(plant);previous=store.current(plant)
         current=previous['sourceFamily'] if previous else store.registry.entries()[0].key
-        # Productive replay ingestion is intentionally not fabricated from R2 metrics.
-        selection=choose_family(settings['family'],current,(),registry=store.registry,now=now)
+        selection=choose_family(settings['family'],current,economic_replay.scores(store,plant,now),registry=store.registry,now=now,
+            lookback_days=settings['autoLookbackDays'],minimum_days=settings['autoMinimumDays'],
+            minimum_coverage=settings['autoMinimumCoverage'],margin_chf=settings['autoSwitchMarginChf'])
         data,full_end,quality=assemble(store,plant,selection['family'],now)
         data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']]
         result=optimize(**data,config_revision=settings['revision'],family=selection['family'])
         if result['executable']:
             result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])})
             store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token'])
+            economic_replay.capture(store,plant,now,assemble,optimize)
         return result
     except (ValueError,TypeError,KeyError) as exc:
         result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]}
@@ -309,7 +313,7 @@ def status(store,plant,now):
     pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone()
     ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone()
     fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit'))
-    return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)}
+    return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)}
 
 def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()):
     allowed={str(UUID(p)) for p in plants}
@@ -343,7 +347,7 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial
         if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial')
         return factory()
     @app.get('/health')
-    def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1}
+    def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1,'applicationRelease':'unified-rc1','mappingCompatibilityVersion':1,'economicReplayVersion':1,'archiveVersion':1}
     @app.get('/internal/v2/prognosis/{plant}/planner')
     def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
         s=authorize(plant,token)
@@ -369,6 +373,17 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial
             return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc))
         except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300])
         finally:s.close()
+    @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}/mapping-compatibility')
+    def mapping_compatibility(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
+        s=authorize(plant,token)
+        try:
+            from .mapping_identity import register
+            cfg=measurement_pipeline.configuration(s.con,plant,dataset)
+            return register(s.con,plant,cfg,payload,int(datetime.now(timezone.utc).timestamp()))
+        except (ValueError,KeyError,TypeError,AttributeError):
+            raise HTTPException(400, 'Mapping compatibility proof rejected')
+        finally:s.close()
+
     @app.put('/internal/v2/prognosis/{plant}/planner/settings')
     def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
         s=authorize(plant,token)

services/netplan-v4/netplan_v4/store.py

@@ -4,7 +4,7 @@ import sqlite3
 from datetime import datetime,timedelta
 from uuid import uuid4
 from .domain import default_registry,month_key,number,quarter_start,utc
-from . import meter_runtime, controlled_trial, measurement_pipeline
+from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline
 
 def canonical(value):
     return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False)
@@ -33,6 +33,8 @@ class PlannerStore:
         ''')
         meter_runtime.schema(self.con)
         controlled_trial.schema(self.con)
+        economic_replay.schema(self.con)
+        retention.schema(self.con)
         measurement_pipeline.schema(self.con)
     def close(self):self.con.close()
     def settings(self,plant):