feat(workflow): publish verified forecast controls and consolidated docs
Tests / test (push) Successful in 1m1s

Approved by Daniel Haefliger for develop and beta. Author dh_Agent, authenticated account dh. Preserve published battery, charging and overall Energy Pie changes. No deployment or plant control authorization.
This commit is contained in:
dh
2026-10-08 10:02:58 +00:00
parent af69151165
commit 74a2d6c685
82 changed files with 5054 additions and 617 deletions
@@ -144,7 +144,7 @@ def validate_config(c):
return c
def register_dataset(con, plant, c, now):
def register_dataset(con, plant, c, now, *, own_transaction=True):
"""Operator endpoint only; device append endpoint cannot change units or limits."""
validate_config(c)
if c.get('sourceDatasetId'):
@@ -155,15 +155,16 @@ def register_dataset(con, plant, c, now):
if canonical(comparable(origin)) != canonical(comparable(c)):
raise ValueError('Referenced observations must retain identical measurement meaning')
value = canonical(c)
con.execute('BEGIN IMMEDIATE')
if own_transaction: con.execute('BEGIN IMMEDIATE')
try:
old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone()
if old and old[0] != value:
raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning')
con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now))
con.commit()
if own_transaction: con.commit()
except Exception:
con.rollback(); raise
if own_transaction: con.rollback()
raise
return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False}
@@ -483,11 +484,14 @@ def apply_load_forecast(con,plant,dataset,forecast,decision):
last=json.loads(last[0])
if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds')
sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request']
if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario')
s=sdl_sources[0];r=last['raw'][s['key']]
if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']:
raise ValueError('No current external SDL request for the persistence scenario')
sdl=r['value']*s['factorToW']
if len(sdl_sources)>1: raise ValueError('At most one explicit SDL request channel is supported')
sdl=0.; external_observed=None; external_policy='no_external_sdl_channel'
if sdl_sources:
s=sdl_sources[0];r=last['raw'][s['key']]
if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']:
raise ValueError('No current external SDL request for the persistence scenario')
sdl=r['value']*s['factorToW'];external_observed=iso(r['sourceUpdatedAt'])
external_policy='last_sdl_request_persistence_estimate'
result=json.loads(canonical(forecast)); result['families']={}
result['observedAt']=iso(max(availability_epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt']))
for family,old in forecast['families'].items():
@@ -501,7 +505,7 @@ def apply_load_forecast(con,plant,dataset,forecast,decision):
'loadMethodVersion':model['methodVersion'],'loadVariant':family,
'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'),
'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False,
'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']),
'externalPolicy':external_policy,'externalObservedAt':external_observed,
'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation'],
'historyTimingMethod':model.get('historyTimingMethod','strict_expiry'),
'observationDatasetId':model.get('observationDatasetId',dataset)}}
+42 -9
View File
@@ -16,7 +16,7 @@ from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_star
from .store import PlannerStore,canonical
from .selection import choose_family
from .optimizer import optimize
from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline
from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline, workflow
from .forecast_quality import assess_family
from .receiver_contract import provenance
from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month
@@ -198,7 +198,9 @@ def assemble(store,plant,family,now,*,include_forecast_points=False):
input_quality['forecastAssessment']=assessment
if source.get('dataPipeline'):
input_quality['dataPipeline']=source['dataPipeline']
input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule')
input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping')
if source['dataPipeline'].get('externalPolicy')=='last_sdl_request_persistence_estimate':
input_quality['warnings'].append('External SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule')
for p in source['points']:
start=utc(p['time']);end=start+timedelta(minutes=5)
if end<=decision:continue
@@ -276,6 +278,7 @@ def run_once(store,now):
stamp=int(now.timestamp())//300
plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')]
for plant in plants:
if not workflow.learning_enabled(store.con,plant):continue
config=store.settings(plant)
economic_replay.advance(store,plant,now)
datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))]
@@ -291,6 +294,9 @@ def run_once(store,now):
if not claim:return {'status':'idle'}
plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]}
try:
if not workflow.learning_enabled(store.con,plant):
result={'status':'learning_disabled','executable':False,'points':[]}
return result
settings=store.settings(plant);previous=store.current(plant)
current=previous['sourceFamily'] if previous else store.registry.entries()[0].key
selection=choose_family(settings['family'],current,economic_replay.scores(store,plant,now),registry=store.registry,now=now,
@@ -316,16 +322,18 @@ def run_once(store,now):
def status(store,plant,now):
plan=store.current(plant);settings=store.settings(plant)
row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone()
pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone()
pending=store.con.execute('SELECT revision,reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone()
ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone()
fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit'))
return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)}
state={'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)}
state['workflow']=workflow.view(store,plant,state,now)
return state
def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()):
allowed={str(UUID(p)) for p in plants}
trial_allowed={str(UUID(p)) for p in controlled_trial_plants}
if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist')
if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required')
if not service_token or len(service_token)<24:raise ValueError('Private service token required')
path=Path(db_path).resolve()
if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required')
path.parent.mkdir(parents=True,exist_ok=True);stop=Event()
@@ -346,14 +354,38 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial
if start_worker:thread.join(35)
app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan)
app.state.store_factory=factory
def authorize(plant,token):
def authorize(plant,token,*,onboarding=False):
if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized')
try:plant=str(UUID(plant))
try:
if str(UUID(plant))!=plant:raise ValueError('Canonical UUID required')
except ValueError:raise HTTPException(400,'Invalid installation ID')
if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial')
return factory()
s=factory()
if not onboarding and plant not in allowed and not workflow.enrolled(s.con,plant):
s.close();raise HTTPException(403,'Installation not enabled for planning')
return s
@app.post('/internal/v2/prognosis/{plant}/planner/setup')
def setup(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
s=authorize(plant,token,onboarding=True)
try:return workflow.setup(s,plant,payload,datetime.now(timezone.utc))
except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(409 if 'immutable' in str(exc) else 400,str(exc)[:200])
finally:s.close()
@app.post('/internal/v2/prognosis/{plant}/planner/workflow')
def workflow_report(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
s=authorize(plant,token)
try:return workflow.report(s,plant,payload,datetime.now(timezone.utc))
except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:200])
finally:s.close()
@app.get('/health')
def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1,'applicationRelease':'unified-rc1','mappingCompatibilityVersion':1,'economicReplayVersion':1,'archiveVersion':1}
@app.get('/internal/v2/planner/installations')
def planning_installations(token:str=Header(default='',alias='X-Enelix-Service-Token')):
if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized')
s=factory()
try:
registered={r[0] for r in s.con.execute('SELECT plant FROM planner_enrollment WHERE learning_enabled=1')}
paused={r[0] for r in s.con.execute('SELECT plant FROM planner_enrollment WHERE learning_enabled=0')}
return {'installationIds':sorted((allowed|registered)-paused),'controlEnabled':False}
finally:s.close()
@app.get('/internal/v2/prognosis/{plant}/planner')
def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
s=authorize(plant,token)
@@ -419,6 +451,7 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial
def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
s=authorize(plant,token)
try:
if not workflow.learning_enabled(s.con,plant):raise ValueError('Learning is disabled for this installation')
now=datetime.now(timezone.utc)
result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp()))
# Existing five-minute worker handles rollup/training; no per-record optimizer flood.
+2 -1
View File
@@ -4,7 +4,7 @@ import sqlite3
from datetime import datetime,timedelta
from uuid import uuid4
from .domain import default_registry,month_key,number,quarter_start,utc
from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline
from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline, workflow
def canonical(value):
return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False)
@@ -36,6 +36,7 @@ class PlannerStore:
economic_replay.schema(self.con)
retention.schema(self.con)
measurement_pipeline.schema(self.con)
workflow.schema(self.con)
def close(self):self.con.close()
def settings(self,plant):
row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone()
+214
View File
@@ -0,0 +1,214 @@
"""Authenticated data onboarding and observed workflow, never actuator authority."""
from datetime import timedelta
from uuid import UUID
import json
from . import measurement_pipeline
from .domain import utc
from .receiver_contract import control_context
REPORT_MAX_AGE_SECONDS = 120
def canonical(value):
return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
def schema(con):
con.executescript('''
CREATE TABLE IF NOT EXISTS planner_enrollment(
plant TEXT PRIMARY KEY, dataset TEXT NOT NULL, learning_enabled INTEGER NOT NULL,
created_at TEXT NOT NULL, updated_at TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS planner_workflow_report(
plant TEXT PRIMARY KEY, observed_at TEXT NOT NULL, received_at TEXT NOT NULL,
value TEXT NOT NULL);
''')
def identity(plant, value):
if not isinstance(plant, str) or str(UUID(plant)) != plant:
raise ValueError('Canonical installation UUID required')
if type(value.get('version')) is not int or value['version'] != 1 or value.get('installationId') != plant:
raise ValueError('Installation path/payload mismatch or unsupported version')
def enrolled(con, plant):
return con.execute('SELECT * FROM planner_enrollment WHERE plant=?', (plant,)).fetchone()
def learning_enabled(con, plant):
row = enrolled(con, plant)
if row is not None:
return bool(row['learning_enabled'])
report_row = con.execute('SELECT value FROM planner_workflow_report WHERE plant=?', (plant,)).fetchone()
return report_row is None or json.loads(report_row['value'])['learningEnabled']
def setup(store, plant, value, now):
identity(plant, value)
if set(value) != {'version', 'installationId', 'dataset', 'learningEnabled'} or type(value['learningEnabled']) is not bool:
raise ValueError('Explicit dataset and boolean learningEnabled required')
config = measurement_pipeline.validate_config(value['dataset'])
# Compatibility proofs and relaxed history-timing policies remain operator-only.
existing = store.con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant, config['datasetId'])).fetchone()
if not existing and (config.get('sourceDatasetId') or config.get('historyTimingPolicy')):
raise ValueError('Device setup cannot grant history compatibility or timing exceptions')
if not existing and (config['minimumCoverage'] < .95 or config['maximumGapSeconds'] > 5 or config['minimumTrainingHours'] < 24):
raise ValueError('New datasets require coverage >= .95, gap <= 5s and training >= 24h')
con = store.con
con.execute('BEGIN IMMEDIATE')
try:
measurement_pipeline.register_dataset(con, plant, config, int(now.timestamp()), own_transaction=False)
current = store.settings(plant)
initialized = con.execute('SELECT 1 FROM planner_settings WHERE plant=?', (plant,)).fetchone() is None
if initialized:
# Native interval estimates retain explicit provenance and gap checks;
# this is a planning policy, never billing or actuator evidence.
current.update(forecastSource='corrected_profile', measurementDataset=config['datasetId'], trainingCadence='daily', measurementPolicy='allow_estimates')
current.pop('revision')
store._validate_settings(current)
con.execute('INSERT INTO planner_settings VALUES(?,?,?)', (plant, 1, canonical(current)))
store._request(plant, 1, 'onboarding', now)
con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',
(plant, utc(now).isoformat(), 'onboarding', canonical({'datasetId': config['datasetId'], 'controlEnabled': False})))
stamp = utc(now).isoformat()
con.execute('''INSERT INTO planner_enrollment VALUES(?,?,?,?,?)
ON CONFLICT(plant) DO UPDATE SET dataset=excluded.dataset,
learning_enabled=excluded.learning_enabled,updated_at=excluded.updated_at''',
(plant, config['datasetId'], int(value['learningEnabled']), stamp, stamp))
con.commit()
except Exception:
con.rollback()
raise
settings = store.settings(plant)
return {'status': 'configured', 'installationId': plant, 'datasetId': config['datasetId'],
'selectedDatasetId': settings['measurementDataset'], 'settingsInitialized': initialized,
'settingsRevision': settings['revision'], 'learningEnabled': value['learningEnabled'],
'controlEnabled': False, 'controlledTrialAuthorized': False}
def report(store, plant, value, now):
identity(plant, value)
if set(value) != {'version', 'installationId', 'observedAt', 'learningEnabled', 'controlRequested', 'controlActive', 'reason'}:
raise ValueError('Explicit workflow report required')
for key in ('learningEnabled', 'controlRequested', 'controlActive'):
if type(value[key]) is not bool:
raise ValueError('Workflow flags must be boolean')
reason = value['reason']
if not isinstance(reason, str) or len(reason) > 300 or any(ord(c) < 32 for c in reason):
raise ValueError('Workflow reason must be plain text up to 300 characters')
observed = measurement_pipeline.epoch(value['observedAt'])
if not -30 <= now.timestamp() - observed <= REPORT_MAX_AGE_SECONDS:
raise ValueError('Fresh whole-second UTC workflow report required')
if value['controlActive'] and not (value['learningEnabled'] and value['controlRequested']):
raise ValueError('Active control requires learning and requested control')
normalized = {**value, 'observedAt': measurement_pipeline.iso(observed)}
data = canonical(normalized)
con = store.con
con.execute('BEGIN IMMEDIATE')
try:
old = con.execute('SELECT observed_at,value FROM planner_workflow_report WHERE plant=?', (plant,)).fetchone()
if old and utc(old['observed_at']).timestamp() > observed:
raise ValueError('Workflow report moved backwards')
if old and utc(old['observed_at']).timestamp() == observed:
if old['value'] != data:
raise ValueError('Conflicting workflow report at the same time')
con.commit()
return {'status': 'duplicate', 'controlEnabled': False}
con.execute('''INSERT INTO planner_workflow_report VALUES(?,?,?,?)
ON CONFLICT(plant) DO UPDATE SET observed_at=excluded.observed_at,
received_at=excluded.received_at,value=excluded.value''',
(plant, normalized['observedAt'], utc(now).isoformat(), data))
# The authenticated first switch also pauses autonomous training. A stop
# remains effective after report expiry; stale telemetry cannot re-enable it.
registration = enrolled(con, plant)
if registration and observed >= int(utc(registration['updated_at']).timestamp()):
con.execute('UPDATE planner_enrollment SET learning_enabled=?,updated_at=? WHERE plant=?',
(int(value['learningEnabled']), utc(now).isoformat(), plant))
con.commit()
except Exception:
con.rollback()
raise
return {'status': 'recorded', 'controlEnabled': False}
def planning_continuity(store, plant, state, now):
"""Keep readiness during routine telemetry replans, never renew a receipt.
This does not change receiver ``fresh``. The Manager must still validate its
originally accepted envelope and stop when that local envelope expires.
"""
if state['fresh']:
return True
try:
pending = state['pending']
plan = state['plan']
settings = state['settings']
run = state['lastRun']
if not pending or not plan or not run:
return False
reasons = json.loads(pending['reasons'])
if not isinstance(reasons, list) or not reasons or not set(reasons) <= {'operation_changed', 'telemetry_changed', 'five_minute_tick'}:
return False
revision = settings['revision']
if pending['revision'] != revision or plan['configRevision'] != revision:
return False
if run['status'] not in ('optimal', 'feasible_time_limit') or run['detail'].get('planId') != plan['planId']:
return False
if run['detail'].get('configRevision') != revision:
return False
if not utc(plan['validFrom']) <= now < utc(plan['validUntil']) or not 0 <= (now-utc(plan['generatedAt'])).total_seconds() <= 900:
return False
rows = store.con.execute('''SELECT i.kind,i.value FROM planner_inputs i
JOIN planner_input_current c ON i.plant=c.plant AND i.kind=c.kind AND i.event_id=c.event_id
WHERE i.plant=? AND i.kind IN ('operation','forecast','tariffs')''', (plant,))
inputs = {row['kind']: json.loads(row['value']) for row in rows}
if any(plan['inputRefs'][kind] != inputs[kind]['eventId'] for kind in ('forecast', 'tariffs')):
return False
operation = inputs['operation']
if not 0 <= (now-utc(operation['observedAt'])).total_seconds() <= 120:
return False
return control_context(operation) == plan['controlContext']
except (KeyError, TypeError, ValueError, AttributeError):
return False
def view(store, plant, state, now):
settings = state['settings']
registration = enrolled(store.con, plant)
row = store.con.execute('SELECT value FROM planner_workflow_report WHERE plant=?', (plant,)).fetchone()
report_value = json.loads(row['value']) if row else None
source = utc(report_value['observedAt']) if report_value else None
report_fresh = bool(source and 0 <= (now-source).total_seconds() <= REPORT_MAX_AGE_SECONDS)
learning = bool(registration['learning_enabled']) if registration else bool(report_fresh and report_value['learningEnabled'])
requested = bool(report_value and report_value['controlRequested'])
dataset = settings['measurementDataset']
model = measurement_pipeline.current_model(store.con, plant, dataset, int(now.timestamp())) if dataset else None
pipeline = next((d for d in state['dataPipeline']['datasets'] if d['datasetId'] == dataset), {})
plan = state.get('plan') or {}
selected = plan.get('inputQuality', {}).get('dataPipeline', {})
ready = bool(settings['forecastSource'] == 'corrected_profile' and model and planning_continuity(store, plant, state, now)
and plan.get('executable') is True and selected.get('datasetId') == dataset
and selected.get('modelId') == model['modelId'])
active = bool(learning and requested and report_fresh and report_value['controlActive'] and ready)
if active:
name, reason = 'active', report_value['reason'] or 'manager_control_active'
elif requested:
name = 'interrupted'
reason = 'manager_report_stale' if not report_fresh else report_value['reason'] or ('planning_not_ready' if not ready else 'manager_control_not_active')
elif not learning:
name, reason = 'disabled', 'learning_disabled'
elif ready:
name, reason = 'ready', 'plan_and_model_ready'
elif model:
name = 'learning'
reason = (state.get('lastRun') or {}).get('detail', {}).get('reason') or 'awaiting_fresh_plan'
elif pipeline.get('status') in ('training', 'candidate_pending'):
name, reason = 'learning', 'model_training'
else:
name, reason = 'collecting', 'collecting_measurements' if registration else 'setup_required'
return {'state': name, 'reason': reason, 'learningEnabled': learning, 'controlRequested': requested,
'controlActive': active, 'planningReady': ready, 'checkedAt': utc(now).isoformat(),
'sourceReportAt': source.isoformat() if source else None,
'sourceFreshUntil': (source+timedelta(seconds=REPORT_MAX_AGE_SECONDS)).isoformat() if source else None,
'reportFresh': report_fresh, 'usingPreviousPlan': bool(ready and not state['fresh'])}