feat(v4): consolidate audited data recovery economic replay and optional archival
This commit is contained in:
ENELIX Agent
@@ -0,0 +1,144 @@
|
||||
import copy
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
from netplan_v4 import economic_replay as e, measurement_pipeline as m
|
||||
from netplan_v4.domain import Battery,Limits,Step,Price
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.selection import choose_family
|
||||
from test_measurement_pipeline import config,record
|
||||
from test_v4 import AID
|
||||
|
||||
AT=datetime(2026,10,2,22,tzinfo=timezone.utc)
|
||||
END=AT+timedelta(days=1)
|
||||
TS=int(AT.timestamp())
|
||||
|
||||
|
||||
def fixture():
|
||||
battery=Battery('b',20.,50.,10.,90.,5000.,5000.,AT,grid_charging=True)
|
||||
steps=[Step(AT+timedelta(minutes=5*i),1000.,0.,Price(.1 if i<144 else .4),Price(.03)) for i in range(288)]
|
||||
data={'steps':steps,'batteries':[battery],'limits':Limits(import_w=10000.),'observed_peaks':{'2026-10':5.},'peak_prices':{'2026-10':5.},'quarter_history':{},'at':AT}
|
||||
def optimizer(**kw):
|
||||
points=[]
|
||||
family=kw['family']
|
||||
for i,s in enumerate(kw['steps']):
|
||||
target=1000. if family in ('3','13') else (1600. if i<144 else 500.)
|
||||
points.append({'time':s.start.isoformat(),'validUntil':s.end.isoformat(),'gridTargetW':target,
|
||||
'importPriceChfKwh':s.import_price.chf_kwh,'exportPriceChfKwh':s.export_price.chf_kwh})
|
||||
return {'executable':True,'points':points}
|
||||
def assemble(*args):return copy.deepcopy(data),END,{}
|
||||
return data,assemble,optimizer
|
||||
|
||||
|
||||
class EconomicReplayTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'));self.c=config()
|
||||
m.register_dataset(self.s.con,AID,self.c,TS)
|
||||
self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':self.c['datasetId'],'autoMinimumDays':1},0,AT)
|
||||
self.data,self.assemble,self.optimizer=fixture()
|
||||
def tearDown(self):self.s.close();self.tmp.cleanup()
|
||||
def capture(self):e.capture(self.s,AID,AT,self.assemble,self.optimizer)
|
||||
def cohort(self):return json.loads(self.s.con.execute('SELECT value FROM planner_economic_cohorts').fetchone()[0])
|
||||
def populate(self):
|
||||
now=int(END.timestamp())+180
|
||||
rows=[record(t,self.c) for t in range(TS-60,now+1,30)]
|
||||
for i in range(0,len(rows),120):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows[i:i+120]},now)
|
||||
def outcomes(self,value=1000.):return {TS+300*i:{'loadW':value,'coverage':1.,'profileUsable':True} for i in range(288)}
|
||||
def test_snapshot_frozen_once(self):
|
||||
self.capture();first=self.cohort();self.capture();self.assertEqual(first,self.cohort())
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1)
|
||||
def test_no_retroactive_midday_snapshot(self):
|
||||
e.capture(self.s,AID,AT+timedelta(hours=12),self.assemble,self.optimizer)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0)
|
||||
def test_no_silent_legacy_replay(self):
|
||||
self.s.save_settings(AID,{'forecastSource':'legacy'},1,AT);self.capture()
|
||||
self.assertEqual(e.scores(self.s,AID,END),[])
|
||||
def test_all_families_same_initial_energy(self):
|
||||
self.capture();c=self.cohort();values=[e.simulate(c,f,self.outcomes()) for f in c['plans']]
|
||||
self.assertEqual(len({v['initialEnergyKwh'] for v in values}),1)
|
||||
for v in values:self.assertGreaterEqual(v['finalEnergyKwh'],2.-1e-6);self.assertLessEqual(v['finalEnergyKwh'],18.00001)
|
||||
def test_profitable_time_shift_has_lower_normalized_cost(self):
|
||||
self.capture();c=self.cohort();a=e.simulate(c,'3',self.outcomes());b=e.simulate(c,'23',self.outcomes())
|
||||
self.assertLess(b['costChf'],a['costChf']);self.assertTrue(b['terminalNormalized'])
|
||||
def test_end_energy_not_free_savings(self):
|
||||
self.capture();c=self.cohort()
|
||||
for p in c['plans']['3']['points']:p['gridTargetW']=0.;p['importPriceChfKwh']=.4
|
||||
d=e.simulate(c,'3',self.outcomes());self.assertGreater(d['terminalAdjustmentChf'],0)
|
||||
def test_missing_actual_not_zero(self):
|
||||
self.capture();out=self.outcomes();out.pop(TS+300)
|
||||
with self.assertRaises(ValueError):e.simulate(self.cohort(),'3',out)
|
||||
def test_export_and_import_limits_not_ignored(self):
|
||||
self.capture();c=self.cohort();c['context']['limits']['import_w']=500.
|
||||
result=e.simulate(c,'3',self.outcomes(20000.));self.assertGreater(result['constraintBreaches'],0)
|
||||
def test_peak_increment_not_total_billed_twice(self):
|
||||
self.capture();c=self.cohort()
|
||||
for p in c['plans']['3']['points']:p['gridTargetW']=6000.
|
||||
result=e.simulate(c,'3',self.outcomes(6000.));self.assertAlmostEqual(result['peakCostChf'],5.)
|
||||
def test_no_actual_results_before_day_end(self):
|
||||
self.capture();self.populate();e.advance(self.s,AID,AT+timedelta(hours=12))
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0)
|
||||
def test_real_pipeline_from_frozen_plans_to_actuals_to_selection(self):
|
||||
self.capture();self.populate();now=END+timedelta(minutes=4)
|
||||
e.advance(self.s,AID,now);scores=e.scores(self.s,AID,now)
|
||||
self.assertEqual(len(scores),3);self.assertEqual(scores[0].days,1)
|
||||
chosen=choose_family('auto','3',scores,minimum_days=1,margin_chf=0,now=now)
|
||||
self.assertEqual(chosen['family'],'23');self.assertEqual(chosen['mode'],'economic_replay')
|
||||
self.assertTrue(e.status(self.s,AID)['connected']);self.assertFalse(e.status(self.s,AID)['isBillingEvidence'])
|
||||
def test_evaluation_idempotent(self):
|
||||
self.capture();self.populate();now=END+timedelta(minutes=4);e.advance(self.s,AID,now);e.advance(self.s,AID,now)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],1)
|
||||
def test_missing_outcomes_preserve_cohort(self):
|
||||
self.capture();e.advance(self.s,AID,END+timedelta(minutes=4))
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1)
|
||||
def test_changed_tariffs_between_families_reject_cohort(self):
|
||||
def changed(**kwargs):
|
||||
v=self.optimizer(**kwargs)
|
||||
if kwargs['family']=='23':v['points'][0]['importPriceChfKwh']+=.5
|
||||
return v
|
||||
e.capture(self.s,AID,AT,self.assemble,changed)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0)
|
||||
def test_no_grid_charging_unsupported_is_explicit(self):
|
||||
self.data['batteries'][0]=__import__('dataclasses').replace(self.data['batteries'][0],grid_charging=False)
|
||||
e.capture(self.s,AID,AT,lambda *args:(self.data,END,{}),self.optimizer)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0)
|
||||
def test_short_price_horizon_no_fake_extension(self):
|
||||
data=copy.deepcopy(self.data);data['steps']=data['steps'][:12]
|
||||
e.capture(self.s,AID,AT,lambda *args:(data,END,{}),self.optimizer)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0)
|
||||
def test_other_plant_no_results(self):
|
||||
self.capture();self.populate();e.advance(self.s,AID,END+timedelta(minutes=4))
|
||||
self.assertEqual(e.scores(self.s,'30509683-4569-49e4-848f-4905e4cc813a',END+timedelta(minutes=4)),[])
|
||||
def test_residual_projection_does_not_change_load_profile(self):
|
||||
rows=[m.project(record(t,self.c),self.c,AID,TS) for t in range(TS-300,TS+1,30)]
|
||||
self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1000.)
|
||||
self.assertEqual(m.reconstruct(rows,self.c)[0]['loadW'],6000.)
|
||||
def test_monthly_peak_not_charged_again_every_comparison_day(self):
|
||||
self.capture();c=self.cohort()
|
||||
for family in c['plans']:
|
||||
for point in c['plans'][family]['points']:point['gridTargetW']=6000.
|
||||
result={f:e.simulate(c,f,self.outcomes(6000.)) for f in c['plans']}
|
||||
self.assertEqual(result['3']['peakCostChf'],5.)
|
||||
policy=self.s.con.execute('SELECT policy_id FROM planner_economic_cohorts').fetchone()[0]
|
||||
with self.s.con:
|
||||
for n in (0,1):
|
||||
start=AT+timedelta(days=n);end=END+timedelta(days=n);day=start.astimezone(__import__('zoneinfo').ZoneInfo('Europe/Zurich')).date().isoformat()
|
||||
if n:
|
||||
self.s.con.execute('INSERT INTO planner_economic_cohorts VALUES(?,?,?,?,?,?)',(AID,day,int(start.timestamp()),int(end.timestamp()),policy,json.dumps(c)))
|
||||
v={'results':result,'start':start.isoformat(),'end':end.isoformat()}
|
||||
self.s.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(AID,day,int(end.timestamp())+120,policy,json.dumps(v)))
|
||||
scores=e.scores(self.s,AID,END+timedelta(days=1,minutes=4))
|
||||
value=next(x.cost_chf for x in scores if x.family=='3')
|
||||
expected=2*(result['3']['energyCostChf']+result['3']['throughputCostChf']+result['3']['terminalAdjustmentChf'])+5.
|
||||
self.assertAlmostEqual(value,expected)
|
||||
|
||||
def test_external_sdl_not_assumed_zero(self):
|
||||
rows=[m.project(record(t,self.c,sdl=250.),self.c,AID,TS) for t in range(TS-300,TS+1,30)]
|
||||
self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1250.)
|
||||
for r in rows:r['raw']['sdl']['valid']=False
|
||||
self.assertFalse(e.actuals(rows,self.c)[0]['profileUsable'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,102 @@
|
||||
import copy
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from hashlib import sha256
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4 import mapping_identity as mi, measurement_pipeline as m
|
||||
from netplan_v4.service import create_app
|
||||
from fastapi.testclient import TestClient
|
||||
from test_measurement_pipeline import config, record, NOW
|
||||
from test_v4 import AID, TOKEN
|
||||
|
||||
|
||||
def proof(c):
|
||||
a = {'version':1, 'installationId':AID, 'reportedInventorySha256':c['inventorySha256'],
|
||||
'accounting':{'splitToleranceW':100.0,'sources':copy.deepcopy(c['sources'])}}
|
||||
b=copy.deepcopy(a); b['accounting']['splitToleranceW']=100
|
||||
p={'version':1, 'canonicalJson':json.dumps(a,separators=(',',':')), 'legacyJson':json.dumps(b,separators=(',',':'))}
|
||||
c['mappingSha256']=sha256(p['canonicalJson'].encode()).hexdigest()
|
||||
return p
|
||||
|
||||
|
||||
class MappingRecoveryTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory(); self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'))
|
||||
self.c=config(); self.proof=proof(self.c); self.alias=sha256(self.proof['legacyJson'].encode()).hexdigest()
|
||||
m.register_dataset(self.s.con,AID,self.c,NOW)
|
||||
def tearDown(self): self.s.close(); self.tmp.cleanup()
|
||||
def register(self): return mi.register(self.s.con,AID,self.c,self.proof,NOW)
|
||||
def batch(self,rows): return m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows},NOW)
|
||||
def r(self,t,legacy=False):
|
||||
r=record(t,self.c)
|
||||
if legacy:r['mappingSha256']=self.alias
|
||||
return r
|
||||
def test_original_mixed_batch_defect_and_atomic_recovery(self):
|
||||
rows=[self.r(NOW-3600+i*30,i>=45) for i in range(120)]; before=copy.deepcopy(rows)
|
||||
with self.assertRaises(ValueError):self.batch(rows)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],0)
|
||||
self.register(); result=self.batch(rows)
|
||||
self.assertEqual(result['stored'],120); self.assertEqual(rows,before)
|
||||
self.assertEqual(result['acceptedThrough'],rows[-1]['capturedAt'])
|
||||
origins=self.s.con.execute('SELECT received_mapping,COUNT(*) FROM planner_observation_origins GROUP BY received_mapping').fetchall()
|
||||
self.assertEqual(dict(origins)[self.alias],75)
|
||||
def test_retry_does_not_duplicate(self):
|
||||
self.register(); self.batch([self.r(NOW,True)])
|
||||
self.assertEqual(self.batch([self.r(NOW,True)])['duplicates'],1)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1)
|
||||
def test_canonical_retry_preserves_both_origins(self):
|
||||
self.register(); self.batch([self.r(NOW,True)]); self.batch([self.r(NOW)])
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],2)
|
||||
def test_unknown_alias_rolls_back_entire_block(self):
|
||||
self.register(); bad=self.r(NOW,True);bad['mappingSha256']='f'*64
|
||||
with self.assertRaises(ValueError): self.batch([self.r(NOW-30),bad])
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],0)
|
||||
def test_conflicting_measurement_keeps_existing_and_does_not_ack_prefix(self):
|
||||
self.register(); self.batch([self.r(NOW)])
|
||||
bad=self.r(NOW,True);bad['raw']['grid']['value']+=500
|
||||
with self.assertRaises(ValueError):self.batch([self.r(NOW-30),bad])
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1)
|
||||
def test_proof_wrong_canonical_hash_rejected(self):
|
||||
p=copy.deepcopy(self.proof);p['canonicalJson']+=' '
|
||||
with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW)
|
||||
def test_numeric_change_rejected(self):
|
||||
p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":101,')
|
||||
with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW)
|
||||
def test_boolean_rejected(self):
|
||||
p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":true,')
|
||||
with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW)
|
||||
def test_other_field_change_rejected(self):
|
||||
p=copy.deepcopy(self.proof);b=json.loads(p['legacyJson']);b['accounting']['sources'][0]['factorToW']=2;p['legacyJson']=json.dumps(b)
|
||||
with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW)
|
||||
def test_inventory_remains_strict(self):
|
||||
self.register();r=self.r(NOW,True);r['reportedInventorySha256']='d'*64
|
||||
with self.assertRaises(ValueError):self.batch([r])
|
||||
def test_other_plant_does_not_inherit_proof(self):
|
||||
self.register();other='30509683-4569-49e4-848f-4905e4cc813a';m.register_dataset(self.s.con,other,self.c,NOW)
|
||||
r=self.r(NOW,True);r['installationId']=other
|
||||
with self.assertRaises(ValueError):m.ingest_batch(self.s.con,other,{'version':1,'datasetId':self.c['datasetId'],'records':[r]},NOW)
|
||||
def test_other_dataset_does_not_inherit_proof(self):
|
||||
self.register();c=copy.deepcopy(self.c);c['datasetId']='unrelated';m.register_dataset(self.s.con,AID,c,NOW)
|
||||
with self.assertRaises(ValueError):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':'unrelated','records':[self.r(NOW,True)]},NOW)
|
||||
def test_direct_project_still_strict(self):
|
||||
self.register()
|
||||
with self.assertRaises(ValueError):m.project(self.r(NOW,True),self.c,AID,NOW)
|
||||
def test_registration_idempotent_and_persistent(self):
|
||||
a=self.register();b=self.register();self.assertEqual(a,b)
|
||||
self.s.close();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'))
|
||||
self.assertEqual(self.batch([self.r(NOW,True)])['stored'],1)
|
||||
def test_malformed_proof_no_registration(self):
|
||||
for x in ({}, {'version':True,'canonicalJson':'{}','legacyJson':'{}'}):
|
||||
with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,x,NOW)
|
||||
def test_internal_api_authentication(self):
|
||||
app=create_app(str(Path(self.tmp.name)/'db.sqlite'),TOKEN,[AID],start_worker=False)
|
||||
with TestClient(app) as client:
|
||||
url=f'/internal/v2/prognosis/{AID}/planner/datasets/{self.c["datasetId"]}/mapping-compatibility'
|
||||
self.assertEqual(client.put(url,json=self.proof).status_code,401)
|
||||
response=client.put(url,json=self.proof,headers={'X-Enelix-Service-Token':TOKEN})
|
||||
self.assertEqual(response.status_code,200,response.text);self.assertFalse(response.json()['controlEnabled'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -42,7 +42,8 @@ class ReleasePreflightTest(unittest.TestCase):
|
||||
text=(ROOT/'gui/netplan-v4.js').read_text()
|
||||
self.assertIn('SCHATTENBETRIEB',text)
|
||||
self.assertIn('measurementPolicy',text)
|
||||
self.assertIn('wartet weiterhin auf vergleichbare Kosten-Replays',text)
|
||||
self.assertIn('erstellt vergleichbare Kosten-Replays',text)
|
||||
self.assertIn('keine vollstaendige Nachbildung laufender Neuoptimierungen',text)
|
||||
self.assertIn('forecastSource',text)
|
||||
self.assertIn('trainingCadence',text)
|
||||
self.assertIn('Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe',text)
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import gzip,hashlib,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4 import retention
|
||||
from test_v4 import AID
|
||||
|
||||
NOW=1791028800
|
||||
|
||||
class RetentionTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name);self.s=PlannerStore(str(self.root/'db.sqlite'));self.archive=self.root/'archive'
|
||||
def tearDown(self):self.s.close();self.tmp.cleanup()
|
||||
def old(self,ago=121,received_ago=10):
|
||||
v='{"numeric":123}';fp=hashlib.sha256(v.encode()).hexdigest()
|
||||
with self.s.con:
|
||||
self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,NOW-received_ago*86400,fp,v))
|
||||
self.s.con.execute('INSERT INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,'m','i','e',NOW-received_ago*86400))
|
||||
def count(self):return self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0]
|
||||
def test_lossless_archive_before_removal(self):
|
||||
self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations');self.assertEqual(r['archived'],1);self.assertEqual(self.count(),0)
|
||||
raw=gzip.decompress((self.archive/r['file']).read_bytes());row=json.loads(raw)
|
||||
self.assertEqual(row['row']['value'],'{"numeric":123}');self.assertEqual(row['origins'][0]['received_mapping'],'m')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),r['sha256'])
|
||||
def test_recent_and_late_received_rows_not_pruned(self):
|
||||
self.old(ago=100);self.old(ago=121,received_ago=1)
|
||||
self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0);self.assertEqual(self.count(),2)
|
||||
def test_failed_durability_keeps_original(self):
|
||||
self.old()
|
||||
with patch('netplan_v4.retention.os.fsync',side_effect=OSError('synthetic')):
|
||||
with self.assertRaises(OSError):retention.archive_batch(self.s.con,self.archive,NOW,'observations')
|
||||
self.assertEqual(self.count(),1);self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1)
|
||||
def test_symlink_archive_rejected(self):
|
||||
self.old();other=self.root/'other';other.mkdir();self.archive.symlink_to(other)
|
||||
with self.assertRaises(ValueError):retention.archive_batch(self.s.con,self.archive,NOW,'observations')
|
||||
self.assertEqual(self.count(),1)
|
||||
def test_current_and_acknowledged_plans_retained(self):
|
||||
with self.s.con:
|
||||
for key in ('old','current','ack'):
|
||||
self.s.con.execute('INSERT INTO planner_plans VALUES(?,?,?,?,?,?)',(key,AID,0,'shadow','{}','2020-01-01T00:00:00+00:00'))
|
||||
self.s.con.execute('INSERT INTO planner_current VALUES(?,?,?)',(AID,'shadow','current'))
|
||||
self.s.con.execute('INSERT INTO planner_ack VALUES(?,?,?,?,?,?)',(AID,'ack',0,'2020-01-01T00:00:00+00:00',None,'shadow_seen'))
|
||||
r=retention.archive_batch(self.s.con,self.archive,NOW,'plans');self.assertEqual(r['archived'],1)
|
||||
self.assertEqual({r[0] for r in self.s.con.execute('SELECT plan_id FROM planner_plans')},{'current','ack'})
|
||||
def test_archived_rows_can_be_restored_exactly(self):
|
||||
self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations')
|
||||
obj=json.loads(gzip.decompress((self.archive/r['file']).read_bytes()));row=obj['row']
|
||||
with self.s.con:self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',tuple(row[k] for k in ('plant','dataset','captured_at','received_at','fingerprint','value')))
|
||||
self.assertEqual(self.count(),1)
|
||||
self.assertEqual(self.s.con.execute('SELECT fingerprint FROM planner_observations').fetchone()[0],hashlib.sha256(row['value'].encode()).hexdigest())
|
||||
def test_no_duplicate_archive_after_repeat(self):
|
||||
self.old();retention.archive_batch(self.s.con,self.archive,NOW,'observations')
|
||||
self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0)
|
||||
self.assertEqual(len(list(self.archive.glob('*.gz'))),1)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,81 @@
|
||||
import importlib.util,json,hashlib,tempfile,unittest,sqlite3,subprocess,sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from test_v4 import AID
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'commissioning'))
|
||||
import deploy_unified_release as dep
|
||||
|
||||
|
||||
class UnifiedReleaseTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name)/'service';self.root.mkdir()
|
||||
self.package=self.root/'commissioning/unified-release';(self.package/'context/netplan_v4').mkdir(parents=True)
|
||||
(self.package/'context/gui').mkdir();(self.root/'netplan_v4').mkdir();(self.root/'gui').mkdir();(self.root/'data').mkdir()
|
||||
self.gui=Path(self.tmp.name)/'portal.js';self.gui.write_text('old ui')
|
||||
self.proof=self.package/'mapping-proof.json';self.proof.write_text('{}')
|
||||
helper=self.root/'commissioning/deploy_application.py';helper.write_bytes((ROOT/'commissioning/deploy_application.py').read_bytes())
|
||||
self.files={}
|
||||
for name,old,new in [('netplan_v4/code.py','old','new'),('gui/netplan-v4.js','old ui','new ui')]:
|
||||
(self.root/name).write_text(old);(self.package/'context'/name).write_text(new)
|
||||
self.files[name]={'before':hashlib.sha256(old.encode()).hexdigest(),'after':hashlib.sha256(new.encode()).hexdigest()}
|
||||
with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:c.execute('CREATE TABLE original(value TEXT)');c.execute("INSERT INTO original VALUES('kept')")
|
||||
manifest={'scope':'unified_v4_candidate','releaseId':'a'*64,'installationId':AID,'files':self.files,
|
||||
'installerSha256':dep.digest(Path(dep.__file__).resolve()),'guiBefore':dep.digest(self.gui),'proofSha256':dep.digest(self.proof),'backupHelperSha256':dep.digest(helper)}
|
||||
(self.package/'RELEASE.json').write_text(json.dumps(manifest));self.calls=[];self.fail=None
|
||||
self.patches=[patch.object(dep,'ROOT',self.root),patch.object(dep,'PACKAGE',self.package),patch.object(dep,'GUI',self.gui),patch.object(dep.os,'geteuid',return_value=0),patch.object(dep.os,'chown'),patch.object(dep.subprocess,'run',side_effect=self.command)]
|
||||
for p in self.patches:p.start()
|
||||
def tearDown(self):
|
||||
for p in reversed(self.patches):p.stop()
|
||||
self.tmp.cleanup()
|
||||
def command(self,args,**kwargs):
|
||||
self.calls.append(args)
|
||||
if self.fail=='build' and args[:2]==['docker','build']:raise subprocess.CalledProcessError(1,args)
|
||||
if args[0]=='node':return SimpleNamespace(stdout='')
|
||||
if args[:2]==['docker','exec']:return SimpleNamespace(stdout=AID+'\n')
|
||||
if args[:2]==['docker','inspect']:return SimpleNamespace(stdout='sha256:'+'1'*64+'\n')
|
||||
if args[:3]==['docker','image','inspect']:return SimpleNamespace(stdout='sha256:'+'2'*64+'\n')
|
||||
if 'ps' in args:return SimpleNamespace(stdout='container\n')
|
||||
if 'exec' in args:
|
||||
if self.fail=='bootstrap':raise subprocess.CalledProcessError(1,args)
|
||||
return SimpleNamespace(stdout=json.dumps({'settingsUnchanged':True,'health':{'liveEnabled':False}}))
|
||||
return SimpleNamespace(stdout='')
|
||||
def test_source_only_never_deploys(self):
|
||||
dep.run(AID);self.assertEqual(self.calls,[]);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old')
|
||||
def test_drift_stops_before_docker(self):
|
||||
(self.root/'netplan_v4/code.py').write_text('other work')
|
||||
with self.assertRaises(ValueError):dep.run(AID,True)
|
||||
self.assertEqual(self.calls,[])
|
||||
def test_gui_parallel_change_preserved(self):
|
||||
self.gui.write_text('parallel gui')
|
||||
with self.assertRaises(ValueError):dep.run(AID,True)
|
||||
self.assertEqual(self.gui.read_text(),'parallel gui')
|
||||
def test_tests_before_replacement_and_no_other_service(self):
|
||||
dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new')
|
||||
tests=next(i for i,c in enumerate(self.calls) if c[:2]==['docker','run'])
|
||||
start=next(i for i,c in enumerate(self.calls) if 'up' in c)
|
||||
self.assertLess(tests,start);self.assertIn('none',self.calls[tests]);self.assertIn('--read-only',self.calls[tests])
|
||||
for c in self.calls:
|
||||
if 'up' in c:self.assertEqual(c[-1],'netplan-v4')
|
||||
self.assertEqual(self.gui.read_text(),'new ui')
|
||||
with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:self.assertEqual(c.execute('SELECT value FROM original').fetchone()[0],'kept')
|
||||
def test_failed_build_changes_no_running_sources(self):
|
||||
self.fail='build'
|
||||
with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True)
|
||||
self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertFalse(any('up' in c for c in self.calls))
|
||||
def test_bootstrap_failure_restores_previous_runtime_sources(self):
|
||||
self.fail='bootstrap'
|
||||
with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True)
|
||||
self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertEqual(self.gui.read_text(),'old ui')
|
||||
self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls))
|
||||
def test_repeat_is_safe(self):
|
||||
dep.run(AID,True);dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new')
|
||||
def test_no_test_or_production_dispatch_grant(self):
|
||||
text=Path(dep.__file__).read_text()
|
||||
self.assertNotIn('/trial/arm',text);self.assertNotIn("'runMode':'live'",text);self.assertNotIn('SET NetzfahrplanV4DatenCursor',text)
|
||||
def test_dockerfile_copies_installers_needed_by_test_suite(self):
|
||||
self.assertIn('COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py',(ROOT/'Dockerfile').read_text())
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user