ENELIX/Enelix-EMS
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(application): integrate measured-load ingestion training and planner source

Browse Source
This commit is contained in:
ENELIX Agent
2026-10-02 21:00:05 +00:00
parent 44362e4bf5
commit fc024fcfba
96 changed files with 8933 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/netplan-v4/tests/test_measurement_pipeline.py
+161
View File
@@ -0,0 +1,161 @@
import copy
import json
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from fastapi.testclient import TestClient
from netplan_v4 import measurement_pipeline as m
from netplan_v4.store import PlannerStore
from netplan_v4.service import create_app, ingest, run_once, status
from test_v4 import inputs, AID, TOKEN
NOW=int(datetime(2026,10,2,12,tzinfo=timezone.utc).timestamp())
def config(**changes):
c={'datasetId':'physical-v1','mappingSha256':'a'*64,'inventorySha256':'b'*64,
'formula':'physical_sum_v1','solarReference':None,'minimumCoverage':.95,
'maximumGapSeconds':10,'minimumTrainingHours':1,'historyDays':14,'sources':[]}
for i,(k,role) in enumerate((('grid','grid'),('pv','pv'),('battery','physical_storage'),('sdl','sdl_request'))):
c['sources'].append({'key':k,'role':role,'variableId':101+i,'factorToW':1,'maxAgeSeconds':60})
c.update(changes);return c
def record(t, c=None, **values):
c=c or config(); v={'grid':3000.,'pv':5000.,'battery':2000.,'sdl':0.,**values}
return {'schemaVersion':1,'kind':'raw_accounting_capture','installationId':AID,
'capturedAt':m.iso(t),'captureStartedAt':m.iso(t),'mappingSha256':c['mappingSha256'],
'reportedInventorySha256':c['inventorySha256'],'raw':{s['key']:{'variableId':s['variableId'],'value':v.get(s['key'],0),
'sourceUpdatedAt':t,'issues':[]} for s in c['sources']}}
class MeasurementPipelineTest(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.path=str(Path(self.tmp.name)/'pipeline.sqlite')
self.s=PlannerStore(self.path);self.c=config();m.register_dataset(self.s.con,AID,self.c,NOW)
def tearDown(self):self.s.close();self.tmp.cleanup()
def batch(self,rows,now=NOW,plant=AID):return m.ingest_batch(self.s.con,plant,{'version':1,'datasetId':'physical-v1','records':rows},now)
def history(self,hours=2):
rows=[record(t) for t in range(NOW-hours*3600,NOW+1,30)]
for i in range(0,len(rows),120):self.batch(rows[i:i+120])
def test_append_ack_idempotence(self):
r=record(NOW);self.assertEqual(self.batch([r])['stored'],1)
a=self.batch([r]);self.assertEqual(a['duplicates'],1);self.assertEqual(a['acceptedThrough'],m.iso(NOW))
def test_conflict_rolls_back_whole_batch(self):
self.batch([record(NOW)])
with self.assertRaises(ValueError):self.batch([record(NOW-30),record(NOW,grid=9000)])
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1)
def test_registration_immutable(self):
self.assertEqual(m.register_dataset(self.s.con,AID,self.c,NOW)['status'],'configured')
with self.assertRaises(ValueError):m.register_dataset(self.s.con,AID,{**self.c,'minimumCoverage':1.},NOW)
def test_plants_isolated(self):
with self.assertRaises(ValueError):self.batch([record(NOW)],plant='30509683-4569-49e4-848f-4905e4cc813a')
def test_mapping_is_not_device_supplied(self):
r=record(NOW);r['mappingSha256']='c'*64
with self.assertRaises(ValueError):self.batch([r])
def test_no_unknown_text_persisted(self):
r=record(NOW);r['unknown']='do-not-store';r['raw']['grid']['value']='do-not-store';r['assessment']={'status':'verified'}
self.batch([r]);data=self.s.con.execute('SELECT value FROM planner_observations').fetchone()[0]
self.assertNotIn('do-not-store',data);self.assertIn('null',data);self.assertNotIn('verified',data)
def test_missing_zero_not_invented(self):
r=record(NOW);r['raw'].pop('grid');p=m.project(r,self.c,AID,NOW)
self.assertIsNone(p['raw']['grid']['value']);self.assertFalse(p['raw']['grid']['valid'])
def test_boolean_not_numeric(self):
p=m.project(record(NOW,grid=True),self.c,AID,NOW);self.assertFalse(p['raw']['grid']['valid'])
def test_batch_order_required(self):
with self.assertRaises(ValueError):self.batch([record(NOW),record(NOW-30)])
def test_batch_size_limit(self):
with self.assertRaises(ValueError):self.batch([record(NOW)]*121)
def test_no_naive_time(self):
r=record(NOW);r['capturedAt']='2026-10-02T12:00:00'
with self.assertRaises(ValueError):self.batch([r])
def test_future_source_not_rejuvenated(self):
r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1
self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid'])
def test_profile_uses_physical_not_virtual(self):
r=record(NOW);r['raw']['virtual_ev']={'value':-90000}
self.assertEqual(m.physical_value({'grid':3000,'pv':5000,'battery':2000},self.c),6000)
def test_zero_numeric_valid(self):
p=m.project(record(NOW,grid=0,pv=0,battery=0),self.c,AID,NOW);self.assertTrue(p['raw']['grid']['valid'])
def projected(self,rows):return [m.project(r,self.c,AID,NOW) for r in rows]
def test_rollup_time_weighted(self):
rows=[record(t,grid=3000 if t<NOW-150 else 6000) for t in range(NOW-300,NOW+1,30)]
w=m.reconstruct(self.projected(rows),self.c)[0]
self.assertEqual(w['coveredSeconds'],300);self.assertAlmostEqual(w['loadW'],7500)
def test_gap_remains_explicit(self):
rows=[record(t) for t in range(NOW-300,NOW+1,30) if t!=NOW-120]
w=m.reconstruct(self.projected(rows),self.c)[0]
self.assertLess(w['coverage'],1);self.assertFalse(w['profileUsable'])
def test_stale_source_expires(self):
rows=[record(t) for t in range(NOW-300,NOW+1,30)]
for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW-300
w=m.reconstruct(self.projected(rows),self.c)[0]
self.assertEqual(w['coveredSeconds'],60)
def test_virtual_missing_not_veto_physical(self):
rows=[record(t,sdl=None) for t in range(NOW-300,NOW+1,30)]
w=m.reconstruct(self.projected(rows),self.c)[0];self.assertTrue(w['profileUsable'])
def test_last_bin_no_extrapolation(self):
rows=[record(t) for t in range(NOW-300,NOW,30)]
w=m.reconstruct(self.projected(rows),self.c)[0];self.assertEqual(w['coveredSeconds'],270);self.assertFalse(w['profileUsable'])
def test_short_history_collects(self):
self.batch([record(NOW-30),record(NOW)])
m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
self.assertIsNone(m.current_model(self.s.con,AID,'physical-v1',NOW))
def test_trained_model_persistent_and_versioned(self):
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
model=m.current_model(self.s.con,AID,'physical-v1',NOW)
self.assertIsNotNone(model);self.assertEqual(m.predict(model,'3',NOW),6000)
self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout')
self.s.close();self.s=PlannerStore(self.path)
self.assertEqual(m.current_model(self.s.con,AID,'physical-v1',NOW)['modelId'],model['modelId'])
def test_daily_training_not_every_sample(self):
self.history();settings=self.s.settings(AID);m.advance(self.s.con,AID,'physical-v1',settings,NOW)
m.advance(self.s.con,AID,'physical-v1',settings,NOW+300)
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_load_models').fetchone()[0],1)
def test_corrected_source_never_falls_back(self):
op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc))
with self.assertRaises(ValueError):m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW)
def test_external_is_labelled_persistence_not_published(self):
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc));out=m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW)
self.assertEqual(out['families']['3']['points'][0]['loadW'],6000)
self.assertFalse(out['families']['3']['dataPipeline']['futureSdlPublished'])
self.assertNotIn('accountingEvidenceId',out['families']['3'])
self.assertEqual(fc['families']['3']['points'][0]['loadW'],2000)
def test_unknown_external_no_fake_zero(self):
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
r=record(NOW+30,sdl=None);self.batch([r],NOW+30)
op,fc,tar=inputs(datetime.fromtimestamp(NOW+30,timezone.utc))
with self.assertRaises(ValueError):m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW+30)
def test_native_data_to_model_to_existing_optimizer(self):
self.history();now=datetime.fromtimestamp(NOW,timezone.utc)
self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v1'},0,now)
for kind,v in zip(('operation','forecast','tariffs'),inputs(now)):ingest(self.s,AID,kind,v,now)
p=run_once(self.s,now)
self.assertTrue(p.get('executable'),p);self.assertFalse(p['liveEnabled'])
self.assertEqual(p['inputQuality']['dataPipeline']['datasetId'],'physical-v1')
self.assertTrue(status(self.s,AID,now)['dataPipeline']['datasets'][0]['records'])
def test_model_not_available_before_training_time(self):
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
self.assertIsNone(m.current_model(self.s.con,AID,'physical-v1',NOW-1))
def test_dataset_setting_explicit(self):
with self.assertRaises(ValueError):self.s.save_settings(AID,{'forecastSource':'corrected_profile'},0,datetime.fromtimestamp(NOW,timezone.utc))
def test_http_scoped_ingest_and_configuration(self):
app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False)
path=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN}
with TestClient(app) as client:
self.assertEqual(client.put(path+'/datasets/physical-v1',json=self.c).status_code,401)
self.assertEqual(client.put(path+'/datasets/physical-v1',json=self.c,headers=h).status_code,200)
now=int(datetime.now(timezone.utc).timestamp());payload={'version':1,'datasetId':'physical-v1','records':[record(now)]}
self.assertEqual(client.post(path+'/measurements',json=payload,headers=h).status_code,200)
s=client.get(path,headers=h).json();self.assertEqual(s['dataPipeline']['datasets'][0]['records'],1)
self.assertFalse(s['liveEnabled'])
def test_solar_terminal_not_battery_twice(self):
c=copy.deepcopy(self.c);c['formula']='solar_terminal_v1'
c['solarReference']={'pvKey':'pv','batteryKey':'battery','rawKey':'ac','scaleKey':'sf'}
c['sources'] += [{'key':k,'role':role,'variableId':i,'factorToW':1,'maxAgeSeconds':60} for k,role,i in [('ac','solar_raw',105),('sf','solar_scale',106)]]
m.validate_config(c)
self.assertEqual(m.physical_value({'grid':3000,'ac':-4000,'sf':-2},c),2960)
def test_solar_invalid_sentinel(self):
c=copy.deepcopy(self.c);c['formula']='solar_terminal_v1';c['solarReference']={'pvKey':'pv','batteryKey':'battery','rawKey':'ac','scaleKey':'sf'}
with self.assertRaises(ValueError):m.physical_value({'grid':3000,'ac':-32768,'sf':0},c)
if __name__=='__main__':unittest.main()