feat(application): integrate measured-load ingestion training and planner source
This commit is contained in:
ENELIX Agent
@@ -0,0 +1 @@
|
||||
"""ENELIX V4 shadow planner. No live actuator interface."""
|
||||
@@ -0,0 +1,44 @@
|
||||
"""SOC-aware battery model, including reserve recovery and hysteresis rearming."""
|
||||
from math import sqrt
|
||||
|
||||
|
||||
class BatteryModel:
|
||||
@staticmethod
|
||||
def build(model,battery,steps,direction):
|
||||
cap=battery.capacity_kwh;eta=sqrt(battery.roundtrip_efficiency)
|
||||
reserve=cap*battery.min_soc_percent/100
|
||||
initial=cap*battery.soc_percent/100
|
||||
lower=min(initial,reserve) if battery.recovery_allowed else reserve
|
||||
upper=cap*battery.max_soc_percent/100
|
||||
energies=[model.variable(lower,upper) for _ in range(len(steps)+1)]
|
||||
model.constraint({energies[0]:1},initial,initial)
|
||||
terminal=battery.terminal_soc_min_percent
|
||||
if terminal is None:terminal=max(battery.soc_percent,battery.min_soc_percent)
|
||||
model.constraint({energies[-1]:1},cap*terminal/100)
|
||||
model.cost[energies[-1]]=-battery.terminal_value_chf_kwh
|
||||
rearm=cap*(battery.rearm_soc_percent if battery.rearm_soc_percent is not None else battery.min_soc_percent)/100
|
||||
enabled=[model.variable(0,1,integer=True) for _ in steps]
|
||||
# Only a numerical threshold, not an undisclosed extra operating reserve.
|
||||
epsilon=min(1e-5,max(0.,upper-reserve)/1000)
|
||||
initially_enabled=not battery.discharge_blocked and initial>=reserve+max(epsilon,1e-8)
|
||||
model.constraint({enabled[0]:1},int(initially_enabled),int(initially_enabled))
|
||||
big=max(upper-lower+epsilon,1.)
|
||||
charge,discharge=[],[]
|
||||
for i,step in enumerate(steps):
|
||||
dt=step.seconds/3600;cmax=battery.max_charge_w/1000;dmax=battery.max_discharge_w/1000
|
||||
c=model.variable(0,cmax,battery.throughput_chf_kwh*dt)
|
||||
d=model.variable(0,dmax,battery.throughput_chf_kwh*dt)
|
||||
charge.append(c);discharge.append(d)
|
||||
model.constraint({c:1,direction[i]:-cmax},upper=0)
|
||||
model.constraint({d:1,direction[i]:dmax},upper=dmax)
|
||||
model.constraint({d:1,enabled[i]:-dmax},upper=0)
|
||||
model.constraint({energies[i]:1,enabled[i]:-big},lower=reserve+epsilon-big)
|
||||
# Once enabled, discharge may consume only energy above reserve.
|
||||
model.constraint({energies[i+1]:1,enabled[i]:-big},lower=reserve-big)
|
||||
if i:
|
||||
# A disabled battery can rearm only AFTER prior charging has
|
||||
# actually reached the configured hysteresis threshold.
|
||||
model.constraint({energies[i]:1,enabled[i]:-big,enabled[i-1]:big},lower=rearm-big)
|
||||
model.constraint({energies[i+1]:1,energies[i]:-1,c:-eta*dt,d:dt/eta},0,0)
|
||||
return {'charge':charge,'discharge':discharge,'energy':energies,
|
||||
'terminal_min_percent':terminal,'discharge_enabled':enabled}
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Explicit, time-limited commissioning authority; NEVER enables shadow execution.
|
||||
|
||||
Only a reviewed internal operator call can arm a trial, and only for a separate
|
||||
allowlist (empty by default). The manager and battery must additionally consent
|
||||
locally. Existing shadow plans/acknowledgements keep their original meaning.
|
||||
"""
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
import json
|
||||
|
||||
MAX_SECONDS = 1800
|
||||
MAX_POWER_W = 5000
|
||||
AUTHORITY_TTL_SECONDS = 90
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.execute('''CREATE TABLE IF NOT EXISTS planner_controlled_trials(
|
||||
plant TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE,
|
||||
value TEXT NOT NULL, revoked_at TEXT)''')
|
||||
|
||||
|
||||
def timestamp(v):
|
||||
if not isinstance(v, str):
|
||||
raise ValueError('Explicit timestamp required')
|
||||
t = datetime.fromisoformat(v.replace('Z', '+00:00'))
|
||||
if t.tzinfo is None:
|
||||
raise ValueError('Timezone required')
|
||||
return t.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def uuid(v):
|
||||
if not isinstance(v, str) or str(UUID(v)) != v:
|
||||
raise ValueError('Canonical UUID required')
|
||||
return v
|
||||
|
||||
|
||||
def power(v):
|
||||
if type(v) not in (int, float) or not 0 < v <= MAX_POWER_W:
|
||||
raise ValueError('Pilot limit must be explicit and at most 5000 W')
|
||||
return float(v)
|
||||
|
||||
|
||||
def accounting(plan):
|
||||
# A human checkbox alone must not relabel aggregate house consumption.
|
||||
quality = plan.get('inputQuality', {})
|
||||
if quality.get('loadBasis') != 'base_load':
|
||||
raise ValueError('Verified base-load/SDL adapter required before a control trial')
|
||||
evidence = quality.get('accountingEvidenceId')
|
||||
if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160:
|
||||
raise ValueError('Plan lacks traceable base-load accounting evidence')
|
||||
return evidence
|
||||
|
||||
|
||||
def eligibility(view, plant):
|
||||
if view.get('installationId') != plant or view.get('liveEnabled') is not False:
|
||||
raise ValueError('Wrong plant or service mode')
|
||||
p = view.get('plan')
|
||||
if view.get('fresh') is not True or not isinstance(p, dict):
|
||||
raise ValueError('Fresh independently validated planning input required')
|
||||
if (p.get('installationId') != plant or p.get('runMode') != 'shadow'
|
||||
or p.get('liveEnabled') is not False or p.get('executable') is not True):
|
||||
raise ValueError('Wrong source plan identity or mode')
|
||||
settings = view.get('settings', {})
|
||||
if settings.get('family') == 'auto' or p.get('sourceFamily') != settings.get('family'):
|
||||
raise ValueError('First controlled trial requires a fixed model family')
|
||||
if p.get('configRevision') != settings.get('revision'):
|
||||
raise ValueError('Configuration revision changed')
|
||||
if len(p.get('controlContext', {}).get('batteries', {})) != 1:
|
||||
raise ValueError('First controlled trial supports exactly one battery')
|
||||
accounting(p)
|
||||
return p
|
||||
|
||||
|
||||
def arm(store, plant, request, view, now, allowed_plants):
|
||||
"""Caller is authenticated internal operator; not exposed via device proxy."""
|
||||
if plant not in allowed_plants:
|
||||
raise ValueError('Controlled trial disabled by server allowlist')
|
||||
fields = {'sessionId', 'expectedPlanId', 'expectedRevision', 'durationSeconds',
|
||||
'maxChargeW', 'maxDischargeW', 'assetId', 'managerId', 'batteryInstanceId',
|
||||
'acceptEstimatedPeak', 'actuatorWatchdogEvidenceId', 'confirmation'}
|
||||
if set(request) != fields or request['confirmation'] != 'ARM_BOUNDED_CONTROL_TRIAL':
|
||||
raise ValueError('Explicit reviewed commissioning request required')
|
||||
session = uuid(request['sessionId']); p = eligibility(view, plant)
|
||||
if type(request['expectedRevision']) is not int or request['expectedRevision'] < 0:
|
||||
raise ValueError('Expected revision must be an integer')
|
||||
if request['expectedPlanId'] != p['planId'] or request['expectedRevision'] != p['configRevision']:
|
||||
raise ValueError('Source plan/revision changed; review again')
|
||||
seconds = request['durationSeconds']
|
||||
if type(seconds) is not int or not 30 <= seconds <= MAX_SECONDS:
|
||||
raise ValueError('Trial duration must be 30..1800 seconds')
|
||||
for k in ('managerId', 'batteryInstanceId'):
|
||||
if type(request[k]) is not int or not 1 <= request[k] <= 99999:
|
||||
raise ValueError('Explicit local instance binding required')
|
||||
if request['managerId'] == request['batteryInstanceId']:
|
||||
raise ValueError('Manager and battery instance must differ')
|
||||
if request['assetId'] not in p['controlContext']['batteries']:
|
||||
raise ValueError('Wrong trial battery')
|
||||
if type(request['acceptEstimatedPeak']) is not bool:
|
||||
raise ValueError('Explicit estimated-peak policy required')
|
||||
if p.get('peakCostIsEstimate') and not request['acceptEstimatedPeak']:
|
||||
raise ValueError('Estimated peak has not been accepted for this trial')
|
||||
evidence = request['actuatorWatchdogEvidenceId']
|
||||
if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160:
|
||||
raise ValueError('Device-side command-loss watchdog proof required')
|
||||
value = {'kind': 'controlled_trial_grant', 'version': 1, 'sessionId': session,
|
||||
'installationId': plant, 'issuedAt': now.isoformat(),
|
||||
'expiresAt': (now + timedelta(seconds=seconds)).isoformat(),
|
||||
'revision': p['configRevision'], 'family': p['sourceFamily'],
|
||||
'assetId': request['assetId'], 'managerId': request['managerId'],
|
||||
'batteryInstanceId': request['batteryInstanceId'],
|
||||
'maxChargeW': power(request['maxChargeW']),
|
||||
'maxDischargeW': power(request['maxDischargeW']),
|
||||
'acceptEstimatedPeak': request['acceptEstimatedPeak'],
|
||||
'accountingEvidenceId': accounting(p),
|
||||
'actuatorWatchdogEvidenceId': evidence,
|
||||
'controlContext': deepcopy(p['controlContext'])}
|
||||
with store.con:
|
||||
existing = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone()
|
||||
if existing and existing[1] is None and timestamp(json.loads(existing[0])['expiresAt']) > now:
|
||||
raise ValueError('Existing trial must first end; implicit extension forbidden')
|
||||
# Reusing an expired/revoked session ID must never resurrect it.
|
||||
used = store.con.execute("SELECT 1 FROM planner_audit WHERE plant=? AND kind='trial_arm' AND detail=?", (plant, session)).fetchone()
|
||||
if used:
|
||||
raise ValueError('Session ID already used')
|
||||
store.con.execute('INSERT INTO planner_controlled_trials VALUES(?,?,?,NULL) ON CONFLICT(plant) DO UPDATE SET session_id=excluded.session_id,value=excluded.value,revoked_at=NULL',
|
||||
(plant, session, json.dumps(value, sort_keys=True, allow_nan=False)))
|
||||
store.con.execute("INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,'trial_arm',?)", (plant, now.isoformat(), session))
|
||||
return value
|
||||
|
||||
|
||||
def revoke(store, plant, session, now):
|
||||
uuid(session)
|
||||
with store.con:
|
||||
store.con.execute('UPDATE planner_controlled_trials SET revoked_at=? WHERE plant=? AND session_id=?', (now.isoformat(), plant, session))
|
||||
return {'status': 'revoked', 'sessionId': session, 'remoteRevocationMaxSeconds': AUTHORITY_TTL_SECONDS}
|
||||
|
||||
|
||||
def authority(store, plant, view, now, allowed_plants):
|
||||
"""Separate authorization envelope, not a mutation of the shadow plan."""
|
||||
if plant not in allowed_plants:
|
||||
return None
|
||||
row = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone()
|
||||
if not row or row[1] is not None:
|
||||
return None
|
||||
grant = json.loads(row[0])
|
||||
try:
|
||||
p = eligibility(view, plant)
|
||||
if not timestamp(grant['issuedAt']) <= now < timestamp(grant['expiresAt']):
|
||||
return None
|
||||
if (p['configRevision'] != grant['revision'] or p['sourceFamily'] != grant['family']
|
||||
or p['controlContext'] != grant['controlContext']
|
||||
or accounting(p) != grant['accountingEvidenceId']
|
||||
or p.get('peakCostIsEstimate') and not grant['acceptEstimatedPeak']):
|
||||
return None
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
until = min(timestamp(grant['expiresAt']), timestamp(p['validUntil']),
|
||||
now + timedelta(seconds=AUTHORITY_TTL_SECONDS))
|
||||
return {**grant, 'kind': 'controlled_trial_authority',
|
||||
'sourceShadowPlanId': p['planId'], 'checkedAt': now.isoformat(),
|
||||
'validUntil': until.isoformat(), 'sourcePlanRemainsShadow': True}
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from math import isfinite
|
||||
from typing import Mapping
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
UTC = timezone.utc
|
||||
ZURICH = ZoneInfo('Europe/Zurich')
|
||||
|
||||
def utc(value):
|
||||
if isinstance(value, str):
|
||||
value = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError('Timezone required')
|
||||
return value.astimezone(UTC)
|
||||
|
||||
def number(value, name, minimum=None, maximum=None):
|
||||
if isinstance(value, bool) or not isinstance(value, (float, int)):
|
||||
raise ValueError(f'{name}: finite number required')
|
||||
value = float(value)
|
||||
if not isfinite(value) or minimum is not None and value < minimum or maximum is not None and value > maximum:
|
||||
raise ValueError(f'{name}: outside permitted range')
|
||||
return value
|
||||
|
||||
def quarter_start(value):
|
||||
value = utc(value)
|
||||
return value.replace(minute=value.minute // 15 * 15, second=0, microsecond=0)
|
||||
|
||||
def month_key(value):
|
||||
return utc(value).astimezone(ZURICH).strftime('%Y-%m')
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Family:
|
||||
key: str
|
||||
pv: str
|
||||
load: str
|
||||
schedule: str
|
||||
label: str
|
||||
|
||||
class FamilyRegistry:
|
||||
def __init__(self, families=()):
|
||||
self._families = {}
|
||||
for family in families:
|
||||
self.register(family)
|
||||
def register(self, family):
|
||||
if family.key in self._families or any(f.schedule == family.schedule for f in self._families.values()):
|
||||
raise ValueError('Duplicate family or schedule identifier')
|
||||
self._families[family.key] = family
|
||||
def get(self, key):
|
||||
if key not in self._families:
|
||||
raise ValueError(f'Unknown model family: {key}')
|
||||
return self._families[key]
|
||||
def entries(self):
|
||||
return tuple(self._families.values())
|
||||
|
||||
def default_registry():
|
||||
return FamilyRegistry([
|
||||
Family('3','prog_var_1','prog_var_2','prog_var_3','Variante 1 (1 / 2 / 3)'),
|
||||
Family('13','prog_var_10','prog_var_11','prog_var_13','Variante 2 (10 / 11 / 13)'),
|
||||
Family('23','prog_var_21','prog_var_22','prog_var_23','Variante 3 (21 / 22 / 23)'),
|
||||
])
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Price:
|
||||
chf_kwh: float
|
||||
published_at: datetime | None = None
|
||||
mode: str = 'static'
|
||||
def known_at(self, at):
|
||||
number(self.chf_kwh, 'energy price')
|
||||
if self.mode not in ('static','dynamic'):
|
||||
raise ValueError('Explicit static/dynamic price mode required')
|
||||
return self.mode == 'static' or self.published_at is not None and utc(self.published_at) <= utc(at)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
start: datetime
|
||||
base_load_w: float
|
||||
pv_w: float
|
||||
import_price: Price | None
|
||||
export_price: Price | None
|
||||
external_w: float = 0.0
|
||||
seconds: int = 300
|
||||
@property
|
||||
def end(self):
|
||||
return utc(self.start) + timedelta(seconds=self.seconds)
|
||||
@property
|
||||
def residual_w(self):
|
||||
return self.base_load_w + self.external_w - self.pv_w
|
||||
|
||||
def split_base_load(measured_house_w, flexible_w, external_w=0.0, external_already_removed=False):
|
||||
measured = number(measured_house_w, 'measured_house_w')
|
||||
flex = sum(number(v, 'flexible measurement') for v in flexible_w)
|
||||
external = number(external_w, 'external measurement')
|
||||
base = measured - flex - (0.0 if external_already_removed else external)
|
||||
if base < -1.0:
|
||||
raise ValueError('Negative base load: inconsistent measurement boundary or double subtraction')
|
||||
return max(0.0, base)
|
||||
|
||||
def priced_prefix(steps, at):
|
||||
result = []
|
||||
for step in steps:
|
||||
if not step.import_price or not step.export_price:
|
||||
break
|
||||
if not step.import_price.known_at(at) or not step.export_price.known_at(at):
|
||||
break
|
||||
result.append(step)
|
||||
while result and result[-1].end != quarter_start(result[-1].end):
|
||||
result.pop()
|
||||
return result
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Battery:
|
||||
asset_id: str
|
||||
capacity_kwh: float
|
||||
soc_percent: float
|
||||
min_soc_percent: float
|
||||
max_soc_percent: float
|
||||
max_charge_w: float
|
||||
max_discharge_w: float
|
||||
measured_at: datetime
|
||||
roundtrip_efficiency: float = 0.90
|
||||
grid_charging: bool = False
|
||||
throughput_chf_kwh: float = 0.0
|
||||
terminal_soc_min_percent: float | None = None
|
||||
terminal_value_chf_kwh: float = 0.0
|
||||
recovery_allowed: bool = False
|
||||
physical_min_soc_percent: float = 0.0
|
||||
discharge_blocked: bool = False
|
||||
rearm_soc_percent: float | None = None
|
||||
def validate(self, at):
|
||||
if not self.asset_id:
|
||||
raise ValueError('Battery asset_id required')
|
||||
number(self.capacity_kwh, 'capacity_kwh', 0.001)
|
||||
lo = number(self.min_soc_percent, 'min_soc_percent', 0, 100)
|
||||
hi = number(self.max_soc_percent, 'max_soc_percent', lo, 100)
|
||||
physical=number(self.physical_min_soc_percent,'physical minimum SOC',0,lo)
|
||||
if type(self.recovery_allowed) is not bool or type(self.discharge_blocked) is not bool:
|
||||
raise ValueError('Explicit battery recovery and hysteresis flags required')
|
||||
number(self.soc_percent, 'SOC', physical if self.recovery_allowed else lo, hi)
|
||||
if self.rearm_soc_percent is not None:number(self.rearm_soc_percent,'hysteresis rearm SOC',lo,hi)
|
||||
number(self.max_charge_w, 'max_charge_w', 0)
|
||||
number(self.max_discharge_w, 'max_discharge_w', 0)
|
||||
number(self.roundtrip_efficiency, 'roundtrip_efficiency', 0.01, 1)
|
||||
number(self.throughput_chf_kwh, 'throughput_chf_kwh', 0)
|
||||
number(self.terminal_value_chf_kwh, 'terminal_value_chf_kwh', 0)
|
||||
if self.terminal_soc_min_percent is not None:
|
||||
number(self.terminal_soc_min_percent, 'terminal_soc_min_percent', lo, hi)
|
||||
age = (utc(at) - utc(self.measured_at)).total_seconds()
|
||||
if age < -30 or age > 1800:
|
||||
raise ValueError('SOC is stale or from the future')
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuarterPast:
|
||||
import_kwh: float
|
||||
measured_seconds: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Limits:
|
||||
export_w: float | None = None
|
||||
import_w: float | None = None
|
||||
manager_month_limits_w: Mapping[int,float] = field(default_factory=dict)
|
||||
def import_limit(self, timestamp):
|
||||
values = []
|
||||
if self.import_w is not None:
|
||||
values.append(number(self.import_w, 'import limit', 0))
|
||||
m = utc(timestamp).astimezone(ZURICH).month
|
||||
if m in self.manager_month_limits_w:
|
||||
values.append(number(self.manager_month_limits_w[m], 'monthly manager limit', 0))
|
||||
return min(values) if values else None
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Minimal validity gates, not a claim of forecast accuracy or model ranking."""
|
||||
from math import isfinite
|
||||
|
||||
|
||||
def assess_family(family, *, minimum_steps=3):
|
||||
points=family.get('points',[])
|
||||
if len(points)<minimum_steps:
|
||||
return {'valid':False,'reason':'Forecast has too few intervals'}
|
||||
loads=[]
|
||||
for point in points:
|
||||
for key in ('loadW','pvW'):
|
||||
value=point.get(key)
|
||||
if isinstance(value,bool) or not isinstance(value,(int,float)) or not isfinite(value) or value<0:
|
||||
return {'valid':False,'reason':'Forecast contains missing or invalid power'}
|
||||
loads.append(point['loadW'])
|
||||
# A PV-only plant or confirmed shutdown may legitimately forecast zero load.
|
||||
# It must be explicit; missing observations filled with zero are not savings.
|
||||
if max(loads)==0 and family.get('zeroLoadConfirmed') is not True:
|
||||
return {'valid':False,'reason':'Unconfirmed all-zero load forecast; likely missing data, not zero electricity costs'}
|
||||
return {'valid':True,'reason':None,'loadMaximumW':max(loads),
|
||||
'loadZeroFraction':sum(v==0 for v in loads)/len(loads)}
|
||||
@@ -0,0 +1,449 @@
|
||||
"""Application data path: versioned numeric observations -> physical load -> trained profiles.
|
||||
|
||||
Lives in the existing planner service/database; no separate diagnostic service.
|
||||
The device may append only to an operator-configured dataset. Original observations,
|
||||
model revisions and prediction vintages are preserved. Output is never an actuator grant.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from bisect import bisect_right
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
from math import isfinite
|
||||
from statistics import median
|
||||
from zoneinfo import ZoneInfo
|
||||
import json
|
||||
|
||||
UTC = timezone.utc
|
||||
LOCAL = ZoneInfo('Europe/Zurich')
|
||||
FAMILIES = ('3', '13', '23')
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
|
||||
|
||||
|
||||
def epoch(value):
|
||||
if not isinstance(value, str):
|
||||
raise ValueError('UTC timestamp required')
|
||||
t = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond:
|
||||
raise ValueError('Explicit whole-second UTC timestamp required')
|
||||
return int(t.timestamp())
|
||||
|
||||
|
||||
def iso(t):
|
||||
return datetime.fromtimestamp(t, UTC).isoformat()
|
||||
|
||||
|
||||
def numeric(value, bound=1e12):
|
||||
return type(value) in (int, float) and isfinite(value) and abs(value) <= bound
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_data_sets(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_observations(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL,
|
||||
received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,captured_at));
|
||||
CREATE TABLE IF NOT EXISTS planner_load_windows(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL,
|
||||
available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,start));
|
||||
CREATE TABLE IF NOT EXISTS planner_load_models(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,model_id));
|
||||
CREATE TABLE IF NOT EXISTS planner_model_current(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_pipeline_state(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL,
|
||||
status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_prediction_vintages(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL,
|
||||
target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
load_w REAL NOT NULL, pv_w REAL NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,issued_at,target,family));
|
||||
CREATE INDEX IF NOT EXISTS planner_observation_window
|
||||
ON planner_observations(plant,dataset,captured_at);
|
||||
CREATE INDEX IF NOT EXISTS planner_prediction_target
|
||||
ON planner_prediction_vintages(plant,dataset,target);
|
||||
''')
|
||||
|
||||
|
||||
def validate_config(c):
|
||||
fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources',
|
||||
'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds',
|
||||
'minimumTrainingHours', 'historyDays'}
|
||||
if not isinstance(c, dict) or set(c) != fields:
|
||||
raise ValueError('Explicit dataset configuration required')
|
||||
name = c['datasetId']
|
||||
if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name):
|
||||
raise ValueError('Invalid dataset ID')
|
||||
for field in ('mappingSha256', 'inventorySha256'):
|
||||
h = c[field]
|
||||
if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h):
|
||||
raise ValueError('Explicit mapping/inventory fingerprint required')
|
||||
if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'):
|
||||
raise ValueError('Unknown physical formula')
|
||||
if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1:
|
||||
raise ValueError('Coverage must be .90..1; recorded gaps remain visible')
|
||||
for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)):
|
||||
if type(c[field]) is not int or not lo <= c[field] <= hi:
|
||||
raise ValueError('Invalid '+field)
|
||||
sources = c['sources']
|
||||
if not isinstance(sources, list) or not 3 <= len(sources) <= 80:
|
||||
raise ValueError('Source list required')
|
||||
seen, ids = set(), set()
|
||||
roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'}
|
||||
for s in sources:
|
||||
if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}:
|
||||
raise ValueError('Explicit source definition required')
|
||||
k = s['key']
|
||||
if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids:
|
||||
raise ValueError('Duplicate/invalid source')
|
||||
if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0:
|
||||
raise ValueError('Source role/factor invalid')
|
||||
if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300:
|
||||
raise ValueError('Source lifetime invalid')
|
||||
seen.add(k); ids.add(s['variableId'])
|
||||
if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources):
|
||||
raise ValueError('Grid and PV measurement sources required')
|
||||
sr = c['solarReference']
|
||||
if c['formula'] == 'solar_terminal_v1':
|
||||
if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}:
|
||||
raise ValueError('Solar terminal sources required')
|
||||
bykey = {s['key']: s['role'] for s in sources}
|
||||
if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))):
|
||||
raise ValueError('Solar origin roles mismatch')
|
||||
elif sr is not None:
|
||||
raise ValueError('No unused solar mapping allowed')
|
||||
canonical(c)
|
||||
return c
|
||||
|
||||
|
||||
def register_dataset(con, plant, c, now):
|
||||
"""Operator endpoint only; device append endpoint cannot change units or limits."""
|
||||
validate_config(c)
|
||||
value = canonical(c)
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone()
|
||||
if old and old[0] != value:
|
||||
raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning')
|
||||
con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now))
|
||||
con.commit()
|
||||
except Exception:
|
||||
con.rollback(); raise
|
||||
return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False}
|
||||
|
||||
|
||||
def configuration(con, plant, dataset):
|
||||
row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone()
|
||||
if row is None:
|
||||
raise ValueError('Dataset not configured for this installation')
|
||||
return json.loads(row[0])
|
||||
|
||||
|
||||
def project(record, c, plant, now):
|
||||
if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant:
|
||||
raise ValueError('Wrong capture identity')
|
||||
if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']:
|
||||
raise ValueError('Wrong capture mapping or inventory')
|
||||
t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt'))
|
||||
if start > t or t > now+30 or t < now-90*86400:
|
||||
raise ValueError('Capture timestamp outside permitted range')
|
||||
if not isinstance(record.get('raw'), dict):
|
||||
raise ValueError('Numeric raw observations required')
|
||||
out = {}; issues = []
|
||||
for s in c['sources']:
|
||||
r = record['raw'].get(s['key'], {})
|
||||
if not isinstance(r, dict):
|
||||
r = {}
|
||||
v, at = r.get('value'), r.get('sourceUpdatedAt')
|
||||
good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == []
|
||||
if not good:
|
||||
v = at = None
|
||||
issues.append(s['key'])
|
||||
# Unknown/free-text fields, credentials, client quality claims never persisted.
|
||||
out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)}
|
||||
return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues}
|
||||
|
||||
|
||||
def ingest_batch(con, plant, payload, now):
|
||||
if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1:
|
||||
raise ValueError('Measurement batch version/fields invalid')
|
||||
c = configuration(con,plant,payload['datasetId'])
|
||||
records = payload['records']
|
||||
if not isinstance(records,list) or not 1 <= len(records) <= 120:
|
||||
raise ValueError('Batch requires 1..120 captures')
|
||||
rows = [project(r,c,plant,now) for r in records]
|
||||
if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])):
|
||||
raise ValueError('Batch must be in increasing capture order')
|
||||
stored = duplicate = 0
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
for r in rows:
|
||||
value = canonical(r); digest = sha256(value.encode()).hexdigest()
|
||||
old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone()
|
||||
if old:
|
||||
if old[0] != digest:
|
||||
raise ValueError('Conflicting immutable observation')
|
||||
duplicate += 1
|
||||
else:
|
||||
con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1
|
||||
con.commit()
|
||||
except Exception:
|
||||
con.rollback(); raise
|
||||
return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate,
|
||||
'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False}
|
||||
|
||||
|
||||
def physical_value(values, c):
|
||||
"""Same explicit sign convention as configured acquisition. No virtual power in load."""
|
||||
total = defaultdict(float)
|
||||
sr = c['solarReference']
|
||||
for s in c['sources']:
|
||||
if s['role'] not in ('grid','pv','physical_storage','flexible_load'):
|
||||
continue
|
||||
if sr and s['key'] in (sr['pvKey'],sr['batteryKey']):
|
||||
continue
|
||||
val = values[s['key']]*s['factorToW']
|
||||
if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0):
|
||||
raise ValueError('Invalid physical power')
|
||||
total[s['role']] += val
|
||||
solar = 0.
|
||||
if sr:
|
||||
raw, sf = values[sr['rawKey']], values[sr['scaleKey']]
|
||||
if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6:
|
||||
raise ValueError('Invalid solar power/scaling sentinel')
|
||||
solar = raw*10**int(sf)
|
||||
load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar
|
||||
if not numeric(load,1e9) or load < 0:
|
||||
raise ValueError('Negative/nonfinite physical load')
|
||||
return load
|
||||
|
||||
|
||||
def reconstruct(records, c):
|
||||
"""Bounded retrospective estimation, never a real-time feedback signal.
|
||||
|
||||
Missing observations split support. Source timestamps are not refreshed. Small
|
||||
uncovered portions remain quantified and are never filled with zero.
|
||||
"""
|
||||
if len(records) < 2:
|
||||
return []
|
||||
if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])):
|
||||
raise ValueError('Capture sequence not ordered')
|
||||
sr = c['solarReference']
|
||||
primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')}
|
||||
if sr:
|
||||
primary.pop(sr['pvKey']); primary.pop(sr['batteryKey'])
|
||||
for s in c['sources']:
|
||||
if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s
|
||||
first,last = records[0]['capturedAt'],records[-1]['capturedAt']
|
||||
series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = []
|
||||
pending = {k:None for k in primary}; high = {k:0 for k in primary}
|
||||
edges = {first,last}
|
||||
for a,b in zip(records,records[1:]):
|
||||
if b['capturedAt']-a['capturedAt'] > 45:
|
||||
gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1])
|
||||
for r in records:
|
||||
at = r['capturedAt']
|
||||
for k in primary:
|
||||
v = r['raw'].get(k,{})
|
||||
t = v.get('sourceUpdatedAt')
|
||||
if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5:
|
||||
if pending[k] is None: pending[k] = at
|
||||
continue
|
||||
high[k] = max(high[k],t)
|
||||
if pending[k] is not None:
|
||||
blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None
|
||||
if t in series[k] and series[k][t] != v['value']:
|
||||
series[k][t] = None
|
||||
else:
|
||||
series[k].setdefault(t,v['value'])
|
||||
for k,s in primary.items():
|
||||
if pending[k] is not None:
|
||||
blocks[k].append((pending[k],last));edges.update(blocks[k][-1])
|
||||
for t in series[k]: edges.update((t,t+s['maxAgeSeconds']))
|
||||
edges.update(range(first//300*300+300,last,300))
|
||||
edges = sorted(x for x in edges if first <= x <= last)
|
||||
knots = {k:sorted(v) for k,v in series.items()}
|
||||
bins = {}
|
||||
for a,b in zip(edges,edges[1:]):
|
||||
start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0})
|
||||
vals = {}; usable = not any(x <= a < y for x,y in gaps)
|
||||
for k,s in primary.items():
|
||||
pos = bisect_right(knots[k],a)-1
|
||||
t = knots[k][pos] if pos >= 0 else None
|
||||
if t is None or a >= t+s['maxAgeSeconds'] or series[k][t] is None or any(x <= a < y for x,y in blocks[k]):
|
||||
usable = False
|
||||
else: vals[k] = series[k][t]
|
||||
load = None
|
||||
if usable:
|
||||
try: load = physical_value(vals,c)
|
||||
except ValueError: usable = False
|
||||
if usable:
|
||||
item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0
|
||||
else:
|
||||
item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap'])
|
||||
out=[]
|
||||
for t,item in sorted(bins.items()):
|
||||
# Partial beginning/end bins remain diagnostic and cannot train.
|
||||
complete_extent = first <= t and last >= t+300
|
||||
coverage = item['seconds']/300
|
||||
eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds']
|
||||
out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'],
|
||||
'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None,
|
||||
'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False,
|
||||
'meterBoundaryVerified':False,'method':c['formula']})
|
||||
return out
|
||||
|
||||
|
||||
def slot(t):
|
||||
local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL)
|
||||
return local.hour*12+local.minute//5
|
||||
|
||||
|
||||
def build_profiles(rows):
|
||||
samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)}
|
||||
anchor=max(r['start'] for r in rows)
|
||||
for r in rows:
|
||||
i=slot(r['start']); v=r['loadW']; samples[i].append(v)
|
||||
if anchor-r['start'] < 86400: recent[i].append(v)
|
||||
weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v)
|
||||
overall = median([r['loadW'] for r in rows])
|
||||
def profile(values):
|
||||
# Missing calendar slots are a model estimate, not invented historical measurements.
|
||||
result=[]
|
||||
for i in range(288):
|
||||
local=values.get(i,[])
|
||||
if not local:
|
||||
local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])]
|
||||
result.append(float(median(local)) if local else float(overall))
|
||||
return result
|
||||
return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)},
|
||||
'slotCoverage':len(samples)/288}
|
||||
|
||||
|
||||
def predict(model, family, t):
|
||||
p=model['profiles'][family]
|
||||
if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday']
|
||||
return p[slot(t)]
|
||||
|
||||
|
||||
def advance(con, plant, dataset, settings, now):
|
||||
"""Called by the existing worker; bounded data/model update once per five-minute tick."""
|
||||
c=configuration(con,plant,dataset); tick=now//300
|
||||
old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()
|
||||
if old and old[0]==tick: return
|
||||
fetched=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',
|
||||
(plant,dataset,now-172800-300,now,now)).fetchall()
|
||||
records=[json.loads(r[0]) for r in fetched]
|
||||
windows=reconstruct(records,c)
|
||||
with con:
|
||||
for w in windows:
|
||||
if w['start']+300 > now-30: continue
|
||||
con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value',
|
||||
(plant,dataset,w['start'],now,w['coverage'],canonical(w)))
|
||||
rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))]
|
||||
good=[r for r in rows if r['profileUsable']]
|
||||
active=current_model(con,plant,dataset,now)
|
||||
cadence=86400 if settings['trainingCadence']=='daily' else 604800
|
||||
detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'],
|
||||
'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'],
|
||||
'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True}
|
||||
state='collecting'
|
||||
if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12:
|
||||
state='model_ready' if active else 'training'
|
||||
attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0]
|
||||
if attempted is None or now-attempted>=cadence:
|
||||
profiles=build_profiles(good)
|
||||
candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good),
|
||||
'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'],
|
||||
'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'},
|
||||
'measurementBoundaryVerified':False}
|
||||
# Causal held-out validation: build validation profiles without the final day.
|
||||
split=good[-1]['start']-86400
|
||||
train=[r for r in good if r['start']+300<=split]; test=[r for r in good if r['start']>=split]
|
||||
if len(train)>=288 and len(test)>=240:
|
||||
val={'profiles':build_profiles(train)}
|
||||
errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES}
|
||||
candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors}
|
||||
# Initial model is labelled bootstrap, never a production measurement proof.
|
||||
# Existing model can be replaced only with held-out evidence and no aggregate regression.
|
||||
promote=active is None
|
||||
if active and candidate['validation']['status']=='causal_holdout':
|
||||
past_model_eligible=active['trainedThrough']<=split
|
||||
if past_model_eligible:
|
||||
incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test)
|
||||
challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test)
|
||||
promote=challenger<=incumbent
|
||||
candidate['validation']['incumbentCompared']=True
|
||||
else:
|
||||
candidate['validation']['status']='holdout_overlaps_active_training'
|
||||
ident=sha256(canonical(candidate).encode()).hexdigest()
|
||||
with con:
|
||||
con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate)))
|
||||
if promote:
|
||||
con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident))
|
||||
detail['candidateModelId']=ident;detail['candidatePromoted']=promote
|
||||
state='model_ready' if promote or active else 'candidate_pending'
|
||||
active=current_model(con,plant,dataset,now)
|
||||
if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']})
|
||||
with con:
|
||||
con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail',
|
||||
(plant,dataset,tick,state,canonical(detail)))
|
||||
|
||||
|
||||
def current_model(con,plant,dataset,at):
|
||||
row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone()
|
||||
return {**json.loads(row[1]),'modelId':row[0]} if row else None
|
||||
|
||||
|
||||
def apply_load_forecast(con,plant,dataset,forecast,decision):
|
||||
model=current_model(con,plant,dataset,decision)
|
||||
if not model:
|
||||
raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused')
|
||||
last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,dataset,decision,decision)).fetchone()
|
||||
if not last: raise ValueError('No recent corrected observation')
|
||||
last=json.loads(last[0]); c=configuration(con,plant,dataset)
|
||||
if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds')
|
||||
sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request']
|
||||
if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario')
|
||||
s=sdl_sources[0];r=last['raw'][s['key']]
|
||||
if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']:
|
||||
raise ValueError('No current external SDL request for the persistence scenario')
|
||||
sdl=r['value']*s['factorToW']
|
||||
result=json.loads(canonical(forecast)); result['families']={}
|
||||
result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt']))
|
||||
for family,old in forecast['families'].items():
|
||||
if family not in FAMILIES: continue
|
||||
points=[]
|
||||
for p in old['points']:
|
||||
t=epoch(p['time'])
|
||||
points.append({**p,'loadW':predict(model,family,t),'externalW':sdl})
|
||||
result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']),
|
||||
'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'],
|
||||
'loadMethodVersion':model['methodVersion'],'loadVariant':family,
|
||||
'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'),
|
||||
'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False,
|
||||
'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']),
|
||||
'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation']}}
|
||||
return result
|
||||
|
||||
|
||||
def pipeline_status(con,plant):
|
||||
out=[]
|
||||
for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)):
|
||||
ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone()
|
||||
count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,ds)).fetchone()
|
||||
out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0],
|
||||
'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None,
|
||||
'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {},
|
||||
'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds']})
|
||||
return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False}
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Persistent, explicitly estimated operational demand tracking.
|
||||
|
||||
No inference from a configured cap. Samples are device-reception observations;
|
||||
last-value integration is a labelled control estimate, not settlement metering.
|
||||
No reset, stale sample or long communication gap is bridged silently.
|
||||
"""
|
||||
from datetime import datetime,timedelta,timezone
|
||||
import json
|
||||
from .domain import number,utc,month_key,quarter_start,ZURICH
|
||||
from .peak_policy import basis_record
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_peak_assumptions(
|
||||
plant TEXT NOT NULL,month TEXT NOT NULL,kw REAL NOT NULL,value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,month));
|
||||
CREATE TABLE IF NOT EXISTS planner_runtime_samples(
|
||||
plant TEXT NOT NULL,meter_id TEXT NOT NULL,at INTEGER NOT NULL,
|
||||
power_w REAL NOT NULL,total_kwh REAL,policy_id TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,meter_id,at));
|
||||
CREATE TABLE IF NOT EXISTS planner_runtime_quarters(
|
||||
plant TEXT NOT NULL,meter_id TEXT NOT NULL,start INTEGER NOT NULL,
|
||||
import_kwh REAL NOT NULL,policy_id TEXT NOT NULL,value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,meter_id,start));
|
||||
CREATE INDEX IF NOT EXISTS idx_runtime_samples_time ON planner_runtime_samples(plant,at);
|
||||
''')
|
||||
|
||||
|
||||
def validate_observation(obs, observed_at):
|
||||
if not isinstance(obs,dict) or set(obs)!={'meterId','sampleAt','powerW','totalImportKwh','controlPolicyId'}:
|
||||
raise ValueError('Explicit meter observation schema required')
|
||||
import re
|
||||
if not isinstance(obs['meterId'],str) or not re.fullmatch(r'symcon-active-import:[0-9a-f]{64}',obs['meterId']):
|
||||
raise ValueError('Active import identity required')
|
||||
if not isinstance(obs['controlPolicyId'],str) or not 1<=len(obs['controlPolicyId'])<=160:
|
||||
raise ValueError('Control policy identity required')
|
||||
t=utc(obs['sampleAt'])
|
||||
if t.microsecond or not 0<=(utc(observed_at)-t).total_seconds()<=60:
|
||||
raise ValueError('Fresh whole-second acquisition timestamp required')
|
||||
number(obs['powerW'],'meter power',-1e9,1e9)
|
||||
number(obs['totalImportKwh'],'active import total',0,1e12)
|
||||
|
||||
|
||||
def integrate_power(samples,start,end,max_gap=120):
|
||||
"""samples sorted tuples (epoch,power,total,policy). Return None on gaps/reset.
|
||||
|
||||
A source can keep the same value while receiving fresh telemetry. The caller
|
||||
records all acquisitions, not only changes. A quarter is never labelled exact.
|
||||
"""
|
||||
if end<=start:return None
|
||||
rows=sorted(samples,key=lambda r:r[0])
|
||||
if len(rows)<2:return None
|
||||
covered=energy=0.;policies=set();largest_gap=0
|
||||
previous=None
|
||||
for row in rows:
|
||||
if previous is not None:
|
||||
ta,pa,ea,pola=previous;tb,pb,eb,polb=row
|
||||
if tb<=ta:return None
|
||||
left=max(start,ta);right=min(end,tb)
|
||||
if right>left:
|
||||
if tb-ta>max_gap or pola!=polb:return None
|
||||
if ea is not None and eb is not None and eb<ea-1e-8:return None
|
||||
seconds=right-left;covered+=seconds;energy+=max(0.,pa)*seconds/3600000
|
||||
policies.add(pola);largest_gap=max(largest_gap,tb-ta)
|
||||
previous=row
|
||||
if covered!=end-start or len(policies)!=1:return None
|
||||
return {'importKwh':energy,'measuredSeconds':end-start,'quality':'estimated',
|
||||
'source':'sampled_power_estimate','method':'positive_power_left_hold',
|
||||
'maxSampleGapSeconds':largest_gap,'controlPolicyId':policies.pop(),
|
||||
'billingEvidence':False}
|
||||
|
||||
|
||||
def save_assumption(con,plant,month,record,at):
|
||||
validated=basis_record(record,month,at,allow_estimates=True)
|
||||
if validated['quality']!='estimated':raise ValueError('Only estimates in assumption storage')
|
||||
current=con.execute('SELECT value FROM planner_peak_assumptions WHERE plant=? AND month=?',(plant,month)).fetchone()
|
||||
# Approximate historical maximum is monotonic within this quality channel.
|
||||
# An authoritative corrected source is stored separately and has precedence.
|
||||
if current:
|
||||
old=json.loads(current[0])
|
||||
if old.get('meterId') and validated.get('meterId') and old['meterId']!=validated['meterId']:
|
||||
raise ValueError('Peak assumption belongs to a different physical meter')
|
||||
if old['kw']>=validated['kw']:return old
|
||||
con.execute('INSERT INTO planner_peak_assumptions VALUES(?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET kw=excluded.kw,value=excluded.value',
|
||||
(plant,month,validated['kw'],json.dumps(validated,sort_keys=True,allow_nan=False)))
|
||||
return validated
|
||||
|
||||
|
||||
def assumptions(con,plant):
|
||||
return {r[0]:json.loads(r[1]) for r in con.execute('SELECT month,value FROM planner_peak_assumptions WHERE plant=?',(plant,))}
|
||||
|
||||
|
||||
def observe(con,plant,obs,at):
|
||||
validate_observation(obs,at)
|
||||
mid=obs['meterId'];stamp=int(utc(obs['sampleAt']).timestamp())
|
||||
existing=con.execute('SELECT power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at=?',(plant,mid,stamp)).fetchone()
|
||||
values=(float(obs['powerW']),float(obs['totalImportKwh']),obs['controlPolicyId'])
|
||||
if existing and tuple(existing)!=values:raise ValueError('Conflicting meter acquisition at same time')
|
||||
con.execute('INSERT OR IGNORE INTO planner_runtime_samples VALUES(?,?,?,?,?,?)',(plant,mid,stamp,*values))
|
||||
# Only a just-completed quarter is finalised; late acquisition never promotes
|
||||
# a historical gap to complete data without all supporting samples.
|
||||
current=stamp//900*900
|
||||
rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at',
|
||||
(plant,mid,current-900-120,stamp))]
|
||||
report=integrate_power(rows,current-900,current)
|
||||
if report:
|
||||
quarter=current-900
|
||||
previous=con.execute('SELECT import_kwh FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start=?',(plant,mid,quarter)).fetchone()
|
||||
if previous is None:
|
||||
con.execute('INSERT INTO planner_runtime_quarters VALUES(?,?,?,?,?,?)',(plant,mid,quarter,report['importKwh'],report['controlPolicyId'],json.dumps(report)))
|
||||
m=month_key(datetime.fromtimestamp(quarter,timezone.utc))
|
||||
save_assumption(con,plant,m,{'kw':report['importKwh']/.25,'quality':'estimated','source':'sampled_power_estimate',
|
||||
'observedAt':utc(at).isoformat(),'meterId':mid,'notes':'Maximum of available sampled quarters; earlier month may be incomplete'},at)
|
||||
# Samples are retained for reproducibility; production retention job required.
|
||||
return report
|
||||
|
||||
|
||||
def current_quarter(con,plant,obs,decision):
|
||||
q=quarter_start(decision);start=int(q.timestamp());end=int(utc(decision).timestamp())
|
||||
if start==end:return None
|
||||
rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at',
|
||||
(plant,obs['meterId'],start-120,end))]
|
||||
report=integrate_power(rows,start,end)
|
||||
if not report:return None
|
||||
return {'start':q.isoformat(),'measuredSeconds':end-start,'importKwh':report['importKwh'],
|
||||
'quality':'estimated','source':'sampled_power_estimate','coverage':1.0,'notes':'Acquisition power estimate, not an exact billing counter boundary'}
|
||||
|
||||
|
||||
def daily_peaks(con,plant,meter_id,now):
|
||||
rows=con.execute('SELECT start,import_kwh,policy_id FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start>=? ORDER BY start',
|
||||
(plant,meter_id,int((utc(now)-timedelta(days=91)).timestamp())))
|
||||
grouped={}
|
||||
for start,energy,policy in rows:
|
||||
t=datetime.fromtimestamp(start,timezone.utc).astimezone(ZURICH)
|
||||
grouped.setdefault((t.strftime('%Y-%m-%d'),policy),[]).append((start,energy))
|
||||
result=[]
|
||||
for (day,policy),items in grouped.items():
|
||||
begin=datetime.strptime(day,'%Y-%m-%d').replace(tzinfo=ZURICH);end=begin+timedelta(days=1)
|
||||
if utc(end)>utc(now):continue
|
||||
expected=set(range(int(begin.timestamp()),int(end.timestamp()),900))
|
||||
complete={t for t,e in items}==expected
|
||||
result.append({'day':day,'peakKw':max(e/.25 for t,e in items),'controlPolicyId':policy,'complete':complete,
|
||||
'observedAt':utc(end).isoformat(),'quality':'estimated'})
|
||||
return result
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Auditable import-energy evidence. Pure calculations; no device/database writes.
|
||||
|
||||
Raw change archives are NOT proof of uninterrupted meter communication. Boundaries
|
||||
without an exact counter record yield intervals, never silently interpolated
|
||||
billing facts. Only a separately verified source and chronology can be promoted
|
||||
into the existing strict planner evidence contract.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from bisect import bisect_left, bisect_right
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
import hashlib
|
||||
import json
|
||||
from math import isfinite
|
||||
from .domain import utc, month_key, quarter_start, ZURICH
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Reading:
|
||||
at: datetime
|
||||
kwh: float
|
||||
|
||||
|
||||
class CounterSeries:
|
||||
def __init__(self, source_id: str, rows, *, max_bracket_seconds=180,
|
||||
quantity='active_import', unit='kWh', timestamp_verified=False,
|
||||
identity_verified=False, chronology_verified=False):
|
||||
if quantity != 'active_import' or unit != 'kWh':
|
||||
raise ValueError('Active import energy in kWh required')
|
||||
if not isinstance(source_id, str) or not source_id:
|
||||
raise ValueError('Nonempty stable source identity required')
|
||||
if type(max_bracket_seconds) is not int or max_bracket_seconds < 1:
|
||||
raise ValueError('Positive bracket age required')
|
||||
self.source_id = source_id
|
||||
self.max_gap = max_bracket_seconds
|
||||
self.verified = all(x is True for x in
|
||||
(timestamp_verified, identity_verified, chronology_verified))
|
||||
values = {}
|
||||
for row in rows:
|
||||
at = utc(row.at)
|
||||
value = row.kwh
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value < 0:
|
||||
raise ValueError('Invalid cumulative active import value')
|
||||
if at in values and abs(values[at] - value) > 1e-9:
|
||||
raise ValueError('Conflicting duplicate timestamp')
|
||||
values[at] = float(value)
|
||||
self.times = sorted(values)
|
||||
self.values = [values[t] for t in self.times]
|
||||
self.resets = [self.times[i] for i in range(1,len(self.times))
|
||||
if self.values[i] < self.values[i-1] - 1e-9]
|
||||
|
||||
def boundary(self, at):
|
||||
at = utc(at)
|
||||
pos = bisect_left(self.times, at)
|
||||
if pos < len(self.times) and self.times[pos] == at:
|
||||
value = self.values[pos]
|
||||
return {'lowerKwh':value, 'upperKwh':value, 'exactRecord':True,
|
||||
'before':at.isoformat(), 'after':at.isoformat()}
|
||||
if pos == 0 or pos == len(self.times):
|
||||
return None
|
||||
before, after = self.times[pos-1], self.times[pos]
|
||||
if (after-before).total_seconds() > self.max_gap:
|
||||
return None
|
||||
if self.values[pos] < self.values[pos-1] - 1e-9:
|
||||
return None
|
||||
return {'lowerKwh':self.values[pos-1], 'upperKwh':self.values[pos],
|
||||
'exactRecord':False, 'before':before.isoformat(), 'after':after.isoformat()}
|
||||
|
||||
def energy(self, start, end):
|
||||
start, end = utc(start), utc(end)
|
||||
if end <= start:
|
||||
raise ValueError('Positive interval required')
|
||||
left, right = self.boundary(start), self.boundary(end)
|
||||
if not left or not right:
|
||||
return None
|
||||
support_start = utc(left['before'])
|
||||
support_end = utc(right['after'])
|
||||
if any(support_start < reset <= support_end for reset in self.resets):
|
||||
return None
|
||||
lower = max(0., right['lowerKwh'] - left['upperKwh'])
|
||||
upper = right['upperKwh'] - left['lowerKwh']
|
||||
if upper < lower - 1e-9:
|
||||
return None
|
||||
exact = bool(left['exactRecord'] and right['exactRecord'])
|
||||
return {'lowerKwh':lower, 'upperKwh':max(lower, upper),
|
||||
'exactRecords':exact, 'billingEvidence':exact and self.verified}
|
||||
|
||||
|
||||
def native_meter_id(sources):
|
||||
"""Same ordered JSON and SHA256 as NetzfahrplanV4Bezugszaehler::identitaet."""
|
||||
import re
|
||||
if not isinstance(sources,list) or not 1<=len(sources)<=8:
|
||||
raise ValueError('Explicit native import sources required')
|
||||
normalized=[];ids=set();idents=set();parents=set()
|
||||
for source in sources:
|
||||
if set(source)!={'VariableID','ElternID','Ident','FaktorZuKWh','Messgroesse'}:
|
||||
raise ValueError('Unknown native source fields')
|
||||
sid,pid,ident,factor=(source[k] for k in ('VariableID','ElternID','Ident','FaktorZuKWh'))
|
||||
if (type(sid) is not int or sid<=0 or type(pid) is not int or pid<=0
|
||||
or sid in ids or ident in idents or not isinstance(ident,str)
|
||||
or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}',ident)
|
||||
or isinstance(factor,bool) or not isinstance(factor,(float,int))
|
||||
or not isfinite(factor) or not 0<factor<=1e6
|
||||
or source['Messgroesse']!='WirkenergieBezug'):
|
||||
raise ValueError('Invalid or duplicate native source')
|
||||
ids.add(sid);idents.add(ident);parents.add(pid)
|
||||
normalized.append({'VariableID':sid,'ElternID':pid,'Ident':ident,
|
||||
'FaktorZuKWh':float(factor),'Messgroesse':'WirkenergieBezug'})
|
||||
if len(parents)!=1:raise ValueError('T1 and T2 must share physical meter')
|
||||
normalized.sort(key=lambda s:s['VariableID'])
|
||||
text=json.dumps(normalized,separators=(',',':'),ensure_ascii=True,allow_nan=False)
|
||||
# PHP preserves .0 on scientific floats and omits zero-padding in exponents.
|
||||
def exponent(match):
|
||||
mantissa=match.group(1)
|
||||
if '.' not in mantissa:mantissa+='.0'
|
||||
power=int(match.group(2))
|
||||
return '"FaktorZuKWh":'+mantissa+'e'+('+' if power>=0 else '')+str(power)
|
||||
text=re.sub(r'"FaktorZuKWh":([0-9]+(?:\.[0-9]+)?)e([+-]?[0-9]+)',exponent,text)
|
||||
return 'symcon-active-import:'+hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
class MeterEvidence:
|
||||
def __init__(self, series, *, native_sources=None):
|
||||
self.series = list(series)
|
||||
if not self.series or len({s.source_id for s in self.series}) != len(self.series):
|
||||
raise ValueError('Unique, nonempty set of counter sources required')
|
||||
self.native_bound = native_sources is not None
|
||||
if self.native_bound:
|
||||
self.meter_id=native_meter_id(native_sources)
|
||||
if {s.source_id for s in self.series}!={'symcon:'+str(s['VariableID']) for s in native_sources}:
|
||||
raise ValueError('Evidence stream does not match native source set')
|
||||
else:
|
||||
signature = json.dumps(sorted(s.source_id for s in self.series),separators=(',',':'))
|
||||
self.meter_id = 'audit-only:' + hashlib.sha256(signature.encode()).hexdigest()
|
||||
|
||||
def interval(self, start, end):
|
||||
start, end = utc(start), utc(end)
|
||||
parts = [s.energy(start,end) for s in self.series]
|
||||
missing = [s.source_id for s,p in zip(self.series,parts) if p is None]
|
||||
if missing:
|
||||
return {'status':'missing', 'start':start.isoformat(),'end':end.isoformat(),
|
||||
'missingSources':missing, 'billingEvidence':False}
|
||||
lo = sum(p['lowerKwh'] for p in parts)
|
||||
hi = sum(p['upperKwh'] for p in parts)
|
||||
seconds = (end-start).total_seconds()
|
||||
exact = all(p['exactRecords'] for p in parts)
|
||||
return {'status':'exact_records' if exact else 'bounded_records',
|
||||
'start':start.isoformat(),'end':end.isoformat(),'lowerKwh':lo,'upperKwh':hi,
|
||||
'lowerAverageKw':lo*3600/seconds,'upperAverageKw':hi*3600/seconds,
|
||||
'billingEvidence':all(p['billingEvidence'] for p in parts)}
|
||||
|
||||
def month(self, at):
|
||||
at = utc(at)
|
||||
local = at.astimezone(ZURICH)
|
||||
begin = utc(local.replace(day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
stop = quarter_start(at)
|
||||
count = int((stop-begin).total_seconds())//900
|
||||
covered = exact = verified = 0
|
||||
lo = hi = 0.
|
||||
gaps = []
|
||||
quarters = []
|
||||
for n in range(count):
|
||||
start = begin + timedelta(minutes=15*n)
|
||||
result = self.interval(start,start+timedelta(minutes=15))
|
||||
quarters.append(result)
|
||||
if result['status'] == 'missing':
|
||||
if len(gaps)<8:gaps.append(start.isoformat())
|
||||
continue
|
||||
covered += 1
|
||||
exact += result['status']=='exact_records'
|
||||
verified += result['billingEvidence']
|
||||
lo = max(lo,result['lowerAverageKw'])
|
||||
hi = max(hi,result['upperAverageKw'])
|
||||
complete = count>0 and covered==count
|
||||
certified = count>0 and verified==count
|
||||
return {'meterId':self.meter_id,'month':month_key(at),'completedQuarters':count,
|
||||
'coveredQuarters':covered,'exactRecordQuarters':exact,'verifiedQuarters':verified,
|
||||
'historyComplete':complete,'billingEvidence':certified,
|
||||
'observedPeakLowerKw':lo if covered else None,
|
||||
'observedPeakUpperKw':hi if covered else None,
|
||||
'monthPeakUpperKw':hi if complete else None,
|
||||
'verifiedMonthPeakKw':hi if certified else None,
|
||||
'firstMissingQuarters':gaps,'quarters':quarters}
|
||||
|
||||
def strict_contract(self, at):
|
||||
"""Fail closed; do not promote archive gaps, estimated boundaries or a cap."""
|
||||
at = utc(at)
|
||||
if not self.native_bound:
|
||||
raise ValueError('Explicit native counter identity required')
|
||||
result = self.month(at)
|
||||
if not result['billingEvidence']:
|
||||
raise ValueError('Full verified month history missing')
|
||||
q = quarter_start(at)
|
||||
past = None
|
||||
if at != q:
|
||||
partial = self.interval(q,at)
|
||||
if not partial['billingEvidence']:
|
||||
raise ValueError('Verified elapsed-quarter energy missing')
|
||||
past = {'start':q.isoformat(),'measuredSeconds':int((at-q).total_seconds()),
|
||||
'importKwh':partial['upperKwh']}
|
||||
return {'version':1,'meterId':self.meter_id,'measuredAt':at.isoformat(),
|
||||
'measuredPeaks':{month_key(at):{'kw':result['verifiedMonthPeakKw'],
|
||||
'source':'verified_month_history'}},
|
||||
'quarterPast':past}
|
||||
|
||||
|
||||
def audit_capture(payload):
|
||||
"""Read-only report from the dedicated Symcon export, never planner ingestion."""
|
||||
if payload.get('schemaVersion') != 1 or payload.get('kind') != 'v4_meter_capture':
|
||||
raise ValueError('Unknown capture contract')
|
||||
now = utc(payload['capturedAt'])
|
||||
channels = payload['channels']
|
||||
series = []
|
||||
summaries = []
|
||||
for variable in ('59607','26620'):
|
||||
channel = channels[variable]
|
||||
rows = channel.get('history',[])
|
||||
readings = [Reading(utc(datetime.fromtimestamp(r['TimeStamp'],now.tzinfo)),r['Value']) for r in rows]
|
||||
# An explicitly captured current value is usable from its capture, not
|
||||
# from an earlier last-change timestamp, and never before logging began.
|
||||
current = channel.get('snapshot')
|
||||
if current and channel.get('snapshotConsistent'):
|
||||
readings.append(Reading(utc(current['capturedAt']),current['value']))
|
||||
s = CounterSeries('symcon:'+variable, readings, timestamp_verified=False,
|
||||
identity_verified=False, chronology_verified=False)
|
||||
series.append(s)
|
||||
summaries.append({'variableId':int(variable),'ident':channel['ident'],
|
||||
'logging':channel.get('logging'), 'rows':len(rows),
|
||||
'queryComplete':channel.get('queryComplete',False),
|
||||
'firstRecord':s.times[0].isoformat() if s.times else None,
|
||||
'lastRecord':s.times[-1].isoformat() if s.times else None,
|
||||
'counterDecreases':[t.isoformat() for t in s.resets]})
|
||||
meter = MeterEvidence(series)
|
||||
month = meter.month(now)
|
||||
current = None
|
||||
q = quarter_start(now)
|
||||
if now>q:current=meter.interval(q,now)
|
||||
blockers=['Meter register identity/unit and acquisition timestamps still require verification.',
|
||||
'Change-only archive does not prove uninterrupted meter acquisition.']
|
||||
if any(not s['queryComplete'] for s in summaries):blockers.append('Archive export incomplete; do not infer complete history.')
|
||||
if not month['historyComplete']:blockers.append('At least one completed billing quarter lacks bounded readings for every tariff.')
|
||||
return {'capturedAt':now.isoformat(),'status':'audit_only','sourceSummary':summaries,
|
||||
'month':{k:v for k,v in month.items() if k!='quarters'},
|
||||
'recentQuarters':month['quarters'][-8:], 'currentQuarter':current,
|
||||
'billingEvidence':False, 'liveEnabled':False,'blockers':blockers}
|
||||
@@ -0,0 +1,199 @@
|
||||
from __future__ import annotations
|
||||
from math import sqrt
|
||||
from uuid import uuid4
|
||||
import numpy as np
|
||||
from scipy.optimize import Bounds, LinearConstraint, milp
|
||||
from scipy.sparse import coo_matrix
|
||||
from .domain import Battery, Limits, Step, month_key, number, quarter_start, utc
|
||||
from .peak_policy import basis_record, RestMonthOutlook
|
||||
|
||||
class Model:
|
||||
def __init__(self):
|
||||
self.lower,self.upper,self.cost,self.integer=[],[],[],[]
|
||||
self.rows,self.row_lo,self.row_hi=[],[],[]
|
||||
def variable(self, lower=0., upper=np.inf, cost=0., integer=False):
|
||||
index=len(self.lower)
|
||||
self.lower.append(lower);self.upper.append(upper);self.cost.append(cost);self.integer.append(int(integer))
|
||||
return index
|
||||
def constraint(self, coefficients, lower=-np.inf, upper=np.inf):
|
||||
self.rows.append(coefficients);self.row_lo.append(lower);self.row_hi.append(upper)
|
||||
def matrices(self):
|
||||
rr,cc,vv=[],[],[]
|
||||
for row,coefficients in enumerate(self.rows):
|
||||
for col,value in coefficients.items():
|
||||
rr.append(row);cc.append(col);vv.append(value)
|
||||
matrix=coo_matrix((vv,(rr,cc)),shape=(len(self.rows),len(self.lower))).tocsr()
|
||||
return matrix,np.array(self.row_lo),np.array(self.row_hi)
|
||||
|
||||
from .battery_model import BatteryModel
|
||||
|
||||
class AssetRegistry:
|
||||
"""Reviewed asset classes only; future EV/thermal models add their own constraints."""
|
||||
def __init__(self):self.models={Battery:BatteryModel}
|
||||
def register(self,asset_type,implementation):
|
||||
if asset_type in self.models:raise ValueError('Asset model already registered')
|
||||
self.models[asset_type]=implementation
|
||||
def build(self,model,asset,steps,direction):
|
||||
if type(asset) not in self.models:raise ValueError('Unsupported asset model')
|
||||
return self.models[type(asset)].build(model,asset,steps,direction)
|
||||
|
||||
def _error(reason,status='invalid_inputs'):
|
||||
return {'schemaVersion':2,'status':status,'executable':False,'points':[],'reason':str(reason)}
|
||||
|
||||
def _validate(steps,assets,at,past,observed_peaks,peak_prices,limits):
|
||||
if not steps or len(steps)>576:raise ValueError('Need 1..576 steps')
|
||||
for i,step in enumerate(steps):
|
||||
start=utc(step.start)
|
||||
if type(step.seconds) is not int or not 1<=step.seconds<=300:raise ValueError('Interval duration must be 1..300 seconds')
|
||||
if start.microsecond or step.end.second or step.end.microsecond or step.end.minute%5:raise ValueError('Intervals must end on a 5-minute boundary')
|
||||
if i and (step.seconds!=300 or start.second or start.minute%5):raise ValueError('Only first interval may be shortened')
|
||||
if i and start!=steps[i-1].end:raise ValueError('Missing/duplicated/overlapping interval')
|
||||
number(step.base_load_w,'base load',0);number(step.pv_w,'PV',0);number(step.external_w,'external flow')
|
||||
if not step.import_price or not step.export_price or not step.import_price.known_at(at) or not step.export_price.known_at(at):raise ValueError('Unknown or unpublished prices')
|
||||
limits.import_limit(start)
|
||||
if utc(at)!=utc(steps[0].start):raise ValueError('First interval must start at decision time')
|
||||
if steps[-1].end!=quarter_start(steps[-1].end):raise ValueError('End horizon on complete billing quarter')
|
||||
if limits.export_w is not None:number(limits.export_w,'export limit',0)
|
||||
ids=set()
|
||||
for asset in assets:
|
||||
asset.validate(at)
|
||||
if asset.asset_id in ids:raise ValueError('Duplicate asset_id')
|
||||
ids.add(asset.asset_id)
|
||||
q=quarter_start(steps[0].start);elapsed=int((utc(steps[0].start)-q).total_seconds())
|
||||
if any(key!=q for key in past):raise ValueError('Only elapsed energy of first quarter permitted')
|
||||
if not elapsed and q in past and (past[q].import_kwh!=0 or past[q].measured_seconds!=0):raise ValueError('No elapsed energy at quarter boundary')
|
||||
if elapsed:
|
||||
if q not in past or past[q].measured_seconds!=elapsed:raise ValueError('Actual elapsed quarter import energy missing')
|
||||
number(past[q].import_kwh,'quarter import energy',0)
|
||||
for month in {month_key(s.start) for s in steps}:
|
||||
if month not in observed_peaks or month not in peak_prices:raise ValueError(f'Measured peak state or tariff missing for {month}')
|
||||
number(observed_peaks[month],'measured peak',0);number(peak_prices[month],'peak tariff',0)
|
||||
|
||||
def optimize(steps,batteries,*,at,limits=None,observed_peaks=None,peak_prices=None,quarter_history=None,config_revision=1,family='3',timeout_seconds=30.,asset_registry=None,peak_context=None,peak_outlooks=None):
|
||||
"""Pure MILP. Peak state is measured, never a configured cap. No device calls."""
|
||||
limits=limits or Limits();observed_peaks=observed_peaks or {};peak_prices=peak_prices or {}
|
||||
past={utc(k):v for k,v in (quarter_history or {}).items()}
|
||||
try:
|
||||
_validate(steps,batteries,at,past,observed_peaks,peak_prices,limits)
|
||||
number(timeout_seconds,'solver timeout',.01,600)
|
||||
contexts={}
|
||||
for month in {month_key(s.start) for s in steps}:
|
||||
if peak_context is None:
|
||||
contexts[month]={'kw':observed_peaks[month],'quality':'verified','source':'legacy_verified_contract','observedAt':utc(at).isoformat()}
|
||||
else:
|
||||
contexts[month]=basis_record(peak_context[month],month,at,allow_estimates=True)
|
||||
if abs(contexts[month]['kw']-observed_peaks[month])>1e-8:raise ValueError('Peak context differs from numerical basis')
|
||||
outlooks=peak_outlooks or {}
|
||||
if set(outlooks)-set(contexts):raise ValueError('Outlook for month outside horizon')
|
||||
for month,outlook in outlooks.items():
|
||||
if not isinstance(outlook,RestMonthOutlook) or outlook.month!=month:raise ValueError('Invalid peak outlook')
|
||||
outlook.validate(at,steps[-1].end)
|
||||
except (ValueError,TypeError,KeyError,AttributeError) as exc:return _error(exc)
|
||||
model=Model();direction=[model.variable(0,1,integer=True) for _ in steps];registry=asset_registry or AssetRegistry()
|
||||
try:handles={b.asset_id:registry.build(model,b,steps,direction) for b in batteries}
|
||||
except ValueError as exc:return _error(exc)
|
||||
total_charge=sum(b.max_charge_w for b in batteries)/1000
|
||||
total_discharge=sum(b.max_discharge_w for b in batteries)/1000
|
||||
imp,exp,curtail=[],[],[];quarters={}
|
||||
for i,step in enumerate(steps):
|
||||
residual=step.residual_w/1000;dt=step.seconds/3600
|
||||
imax=max(0.,(step.base_load_w+step.external_w)/1000)+total_charge
|
||||
emax=max(0.,-residual)+total_discharge;limit=limits.import_limit(step.start)
|
||||
if limit is not None:imax=min(imax,limit/1000)
|
||||
if limits.export_w is not None:emax=min(emax,limits.export_w/1000)
|
||||
pi=model.variable(0,imax,step.import_price.chf_kwh*dt)
|
||||
pe=model.variable(0,emax,-step.export_price.chf_kwh*dt)
|
||||
pc=model.variable(0,step.pv_w/1000)
|
||||
imp.append(pi);exp.append(pe);curtail.append(pc)
|
||||
gm=model.variable(0,1,integer=True)
|
||||
model.constraint({pi:1,gm:-imax},upper=0);model.constraint({pe:1,gm:emax},upper=emax)
|
||||
balance={pi:1,pe:-1,pc:-1};pv_only={}
|
||||
for b in batteries:
|
||||
h=handles[b.asset_id];balance[h['charge'][i]]=-1;balance[h['discharge'][i]]=1
|
||||
if not b.grid_charging:pv_only[h['charge'][i]]=1
|
||||
model.constraint(balance,residual,residual)
|
||||
if pv_only:
|
||||
model.constraint(pv_only,upper=max(0.,-residual))
|
||||
no_grid_max=sum(b.max_charge_w for b in batteries if not b.grid_charging)/1000
|
||||
model.constraint({**pv_only,gm:no_grid_max},upper=no_grid_max)
|
||||
quarters.setdefault(quarter_start(step.start),[]).append(i)
|
||||
months=sorted({month_key(s.start) for s in steps})
|
||||
peaks={m:model.variable(observed_peaks[m]) for m in months}
|
||||
for m in months:
|
||||
if m in outlooks:outlooks[m].add_to_model(model,peaks[m],observed_peaks[m],peak_prices[m])
|
||||
else:model.cost[peaks[m]]=peak_prices[m]
|
||||
for quarter,positions in quarters.items():
|
||||
coefficients={imp[i]:steps[i].seconds/900 for i in positions};coefficients[peaks[month_key(quarter)]]=-1
|
||||
used=past[quarter].import_kwh if quarter in past else 0.
|
||||
model.constraint(coefficients,upper=-used/.25)
|
||||
matrix,row_lo,row_hi=model.matrices()
|
||||
try:
|
||||
result=milp(np.array(model.cost),integrality=np.array(model.integer),bounds=Bounds(model.lower,model.upper),constraints=LinearConstraint(matrix,row_lo,row_hi),options={'time_limit':float(timeout_seconds),'mip_rel_gap':1e-4})
|
||||
except Exception as exc:return _error(f'Solver exception: {type(exc).__name__}','solver_error')
|
||||
x=result.x
|
||||
if result.status not in (0,1) or x is None:return _error(result.message,'no_feasible_plan')
|
||||
if not np.isfinite(x).all():return _error('Non-finite solver result','validation_failed')
|
||||
ax=matrix@x;tol=1e-6
|
||||
if (np.any(x<np.array(model.lower)-tol) or np.any(x>np.array(model.upper)+tol) or np.any(ax<row_lo-tol) or np.any(ax>row_hi+tol) or any(abs(x[i]-round(x[i]))>tol for i,flag in enumerate(model.integer) if flag)):
|
||||
return _error('Solver incumbent violates constraints','validation_failed')
|
||||
running_peaks=dict(observed_peaks);baseline_peaks=dict(observed_peaks);peak_deltas={};baseline_peak_deltas={}
|
||||
for quarter,positions in quarters.items():
|
||||
used=past[quarter].import_kwh if quarter in past else 0.;m=month_key(quarter)
|
||||
demand=(used+sum(x[imp[i]]*steps[i].seconds/3600 for i in positions))/.25
|
||||
baseline=(used+sum(max(0.,steps[i].residual_w)*steps[i].seconds/3600000 for i in positions))/.25
|
||||
before=running_peaks[m];running_peaks[m]=max(before,demand);peak_deltas[positions[-1]]=(running_peaks[m]-before)*peak_prices[m]
|
||||
before=baseline_peaks[m];baseline_peaks[m]=max(before,baseline);baseline_peak_deltas[positions[-1]]=(baseline_peaks[m]-before)*peak_prices[m]
|
||||
points=[];cumulative_energy=cumulative_cash=cumulative_baseline_energy=cumulative_baseline_cash=cumulative_throughput=0.
|
||||
for i,step in enumerate(steps):
|
||||
dt=step.seconds/3600;p_import=max(0.,float(x[imp[i]]));p_export=max(0.,float(x[exp[i]]))
|
||||
energy_cost=(p_import*step.import_price.chf_kwh-p_export*step.export_price.chf_kwh)*dt
|
||||
baseline_import=max(0.,step.residual_w)/1000;baseline_export=max(0.,-step.residual_w)/1000
|
||||
if limits.export_w is not None:baseline_export=min(baseline_export,limits.export_w/1000)
|
||||
baseline_cost=(baseline_import*step.import_price.chf_kwh-baseline_export*step.export_price.chf_kwh)*dt
|
||||
targets,soc_end={},{};throughput_cost=0.
|
||||
for b in batteries:
|
||||
h=handles[b.asset_id];targets[b.asset_id]=float((x[h['charge'][i]]-x[h['discharge'][i]])*1000)
|
||||
soc_end[b.asset_id]=float(100*x[h['energy'][i+1]]/b.capacity_kwh)
|
||||
throughput_cost+=float((x[h['charge'][i]]+x[h['discharge'][i]])*dt*b.throughput_chf_kwh)
|
||||
cumulative_energy+=energy_cost;cumulative_cash+=energy_cost+peak_deltas.get(i,0.)
|
||||
cumulative_baseline_energy+=baseline_cost;cumulative_baseline_cash+=baseline_cost+baseline_peak_deltas.get(i,0.)
|
||||
cumulative_throughput+=throughput_cost;battery_w=sum(targets.values())
|
||||
if battery_w>1:intent='gridCharge' if p_import>.001 else 'pvCharge'
|
||||
elif battery_w< -1:intent='export' if p_export>.001 else 'discharge'
|
||||
else:intent='hold'
|
||||
points.append({'time':utc(step.start).isoformat(),'validUntil':step.end.isoformat(),
|
||||
'baselineGridW':float(step.residual_w),'gridTargetW':(p_import-p_export)*1000,
|
||||
'batteryTargetW':battery_w,'assetTargetsW':targets,'socEndPercent':soc_end,
|
||||
'pvCurtailmentW':float(x[curtail[i]]*1000),'intent':intent,
|
||||
'importLimitW':limits.import_limit(step.start),'exportLimitW':limits.export_w,
|
||||
'importPriceChfKwh':step.import_price.chf_kwh,'exportPriceChfKwh':step.export_price.chf_kwh,
|
||||
'energyCostChf':energy_cost,'additionalPeakCostChf':peak_deltas.get(i,0.),
|
||||
'throughputCostChf':throughput_cost,'cumulativeEnergyCostChf':cumulative_energy,
|
||||
'cumulativeCashCostChf':cumulative_cash,'cumulativeBaselineEnergyCostChf':cumulative_baseline_energy,
|
||||
'cumulativeBaselineCashCostChf':cumulative_baseline_cash,'baselineAdditionalPeakCostChf':baseline_peak_deltas.get(i,0.)})
|
||||
end_value=sum(float(x[handles[b.asset_id]['energy'][-1]])*b.terminal_value_chf_kwh for b in batteries)
|
||||
def planning_cost(chosen):
|
||||
return sum(outlooks[m].incremental_cost(observed_peaks[m],chosen[m],peak_prices[m]) if m in outlooks
|
||||
else max(0.,chosen[m]-observed_peaks[m])*peak_prices[m] for m in months)
|
||||
planning_peak=planning_cost(running_peaks)
|
||||
full_peak=cumulative_cash-cumulative_energy
|
||||
estimated=any(c['quality']=='estimated' for c in contexts.values())
|
||||
return {'schemaVersion':2,'planId':str(uuid4()),'configRevision':config_revision,'sourceFamily':family,
|
||||
'status':'optimal' if result.status==0 else 'feasible_time_limit','executable':True,
|
||||
'generatedAt':utc(at).isoformat(),'validFrom':points[0]['time'],'validUntil':points[-1]['validUntil'],
|
||||
'intervalMinutes':5,'points':points,'solverGap':float(result.mip_gap) if getattr(result,'mip_gap',None) is not None else None,
|
||||
'measuredPeaksKw':{m:observed_peaks[m] for m in months if contexts[m]['quality']=='verified'},
|
||||
'peakBasisKw':{m:observed_peaks[m] for m in months},'peakBasis':contexts,
|
||||
'peakCostIsEstimate':estimated,'planningPeakCostChf':planning_peak,
|
||||
'restMonthAdjustmentChf':planning_peak-full_peak,
|
||||
'peakScenarioOutlook':{m:o.as_dict() for m,o in outlooks.items()},
|
||||
'plannedPeaksKw':{m:running_peaks[m] for m in months},
|
||||
'additionalPeakCostChf':cumulative_cash-cumulative_energy,'energyCostChf':cumulative_energy,'cashCostChf':cumulative_cash,
|
||||
'baselineEnergyCostChf':cumulative_baseline_energy,'baselineCashCostChf':cumulative_baseline_cash,
|
||||
'baselineAdditionalPeakCostChf':sum(baseline_peak_deltas.values()),'baselinePeaksKw':baseline_peaks,
|
||||
'throughputCostChf':cumulative_throughput,'terminalValueChf':end_value,
|
||||
'objectiveChf':cumulative_energy+planning_peak+cumulative_throughput-end_value,
|
||||
'baselinePlanningPeakCostChf':planning_cost(baseline_peaks),
|
||||
'cashCostMeaning':'Projected horizon energy plus full incremental monthly tariff, relative to the labelled peak basis; not an invoice',
|
||||
'terminalMinSocPercent':{b.asset_id:handles[b.asset_id]['terminal_min_percent'] for b in batteries},
|
||||
'peakOutlook':'rest_month_scenarios' if outlooks else 'full_incremental_tariff'}
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Economic peak policy. Planning assumptions NEVER become metering facts.
|
||||
|
||||
Monthly marginal cost is convex. A rest-month scenario is a peak expected after
|
||||
this horizon under a comparable control policy, not a configured limit, free
|
||||
allowance, or a promise of savings. With insufficient evidence use full tariff.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from math import isclose
|
||||
from .domain import number, utc, month_key, ZURICH
|
||||
|
||||
VERIFIED_SOURCES = frozenset({'meter_month_register','verified_month_history','verified_new_month'})
|
||||
ESTIMATE_SOURCES = frozenset({'power_history_estimate','sampled_power_estimate','counter_interval_estimate','operator_estimate','new_month'})
|
||||
|
||||
|
||||
def valid_month(value):
|
||||
if not isinstance(value,str) or len(value)!=7:
|
||||
raise ValueError('Calendar month YYYY-MM required')
|
||||
if datetime.strptime(value,'%Y-%m').strftime('%Y-%m') != value:
|
||||
raise ValueError('Invalid calendar month')
|
||||
return value
|
||||
|
||||
|
||||
def basis_record(value, month, at, allow_estimates=False):
|
||||
"""Strict provenance; missing stays missing. Caller decides explicit opt-in."""
|
||||
valid_month(month)
|
||||
if not isinstance(value,dict) or set(value)-{'kw','quality','source','observedAt','notes','coverage','meterId'}:
|
||||
raise ValueError('Invalid peak-basis schema')
|
||||
kw=number(value.get('kw'),'peak basis kW',0,1e6)
|
||||
quality=value.get('quality')
|
||||
source=value.get('source')
|
||||
if quality=='verified':
|
||||
if source not in VERIFIED_SOURCES:raise ValueError('Not a verified peak source')
|
||||
elif quality=='estimated':
|
||||
if not allow_estimates or source not in ESTIMATE_SOURCES:
|
||||
raise ValueError('Estimated planning basis not permitted or source invalid')
|
||||
elif quality=='new_month':
|
||||
if source!='new_month' or kw!=0 or month<=month_key(at):
|
||||
raise ValueError('Zero future-month state is not a past measured peak')
|
||||
else:raise ValueError('Peak quality must be explicit')
|
||||
stamp=utc(value.get('observedAt'))
|
||||
if stamp>utc(at):raise ValueError('Peak evidence from the future')
|
||||
if quality!='new_month' and month>month_key(at):
|
||||
raise ValueError('Future peak is an outlook, not historical evidence')
|
||||
notes=value.get('notes','')
|
||||
if not isinstance(notes,str) or len(notes)>500:raise ValueError('Invalid peak notes')
|
||||
coverage=value.get('coverage')
|
||||
if coverage is not None:number(coverage,'peak coverage',0,1)
|
||||
mid=value.get('meterId')
|
||||
if mid is not None and (not isinstance(mid,str) or not 1<=len(mid)<=160):raise ValueError('Invalid meter identity')
|
||||
return {**value,'kw':kw,'quality':quality,'source':source,'observedAt':stamp.isoformat()}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PeakScenario:
|
||||
future_peak_kw: float
|
||||
probability: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RestMonthOutlook:
|
||||
month: str
|
||||
scenarios: tuple[PeakScenario, ...]
|
||||
issued_at: datetime
|
||||
future_from: datetime
|
||||
history_until: datetime
|
||||
control_policy_id: str
|
||||
method: str
|
||||
evidence_id: str
|
||||
reliance: float = 0.5
|
||||
|
||||
def validate(self, at, horizon_end):
|
||||
valid_month(self.month)
|
||||
at=utc(at);end=utc(horizon_end)
|
||||
if utc(self.issued_at)>at or utc(self.history_until)>utc(self.issued_at):
|
||||
raise ValueError('Rest-month outlook contains future information')
|
||||
if at-utc(self.issued_at)>timedelta(days=2):raise ValueError('Stale rest-month outlook')
|
||||
if utc(self.future_from)<end:
|
||||
raise ValueError('Rest-month outlook overlaps optimized horizon')
|
||||
if month_key(self.future_from)!=self.month:
|
||||
raise ValueError('Rest-month outlook outside target calendar month')
|
||||
if not self.control_policy_id or not self.evidence_id or not self.method:
|
||||
raise ValueError('Comparable control policy and historical evidence required')
|
||||
if not 1<=len(self.scenarios)<=100:raise ValueError('Need 1..100 scenarios')
|
||||
for item in self.scenarios:
|
||||
number(item.future_peak_kw,'future scenario peak',0,1e6)
|
||||
number(item.probability,'scenario probability',0,1)
|
||||
if not isclose(sum(s.probability for s in self.scenarios),1.,abs_tol=1e-8):
|
||||
raise ValueError('Scenario probabilities must sum to one')
|
||||
number(self.reliance,'outlook reliance',0,1)
|
||||
|
||||
def incremental_cost(self, basis_kw, planned_kw, tariff):
|
||||
"""Separate planning value, never an already-paid or guaranteed saving."""
|
||||
full=max(0.,planned_kw-basis_kw)*tariff
|
||||
expected=tariff*sum(s.probability*(max(planned_kw,basis_kw,s.future_peak_kw)-max(basis_kw,s.future_peak_kw)) for s in self.scenarios)
|
||||
return (1-self.reliance)*full+self.reliance*expected
|
||||
|
||||
def add_to_model(self, model, peak_variable, basis_kw, tariff):
|
||||
model.cost[peak_variable]+=(1-self.reliance)*tariff
|
||||
for s in self.scenarios:
|
||||
if s.probability<=0:continue
|
||||
end_peak=model.variable(max(basis_kw,s.future_peak_kw),cost=self.reliance*tariff*s.probability)
|
||||
model.constraint({end_peak:1.,peak_variable:-1.},lower=0.)
|
||||
|
||||
def as_dict(self):
|
||||
return {'month':self.month,'issuedAt':utc(self.issued_at).isoformat(),'futureFrom':utc(self.future_from).isoformat(),
|
||||
'historyUntil':utc(self.history_until).isoformat(),'controlPolicyId':self.control_policy_id,'method':self.method,
|
||||
'evidenceId':self.evidence_id,'reliance':self.reliance,
|
||||
'scenarios':[{'peakKw':s.future_peak_kw,'probability':s.probability} for s in self.scenarios]}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value):
|
||||
if set(value)!={'month','issuedAt','futureFrom','historyUntil','controlPolicyId','method','evidenceId','reliance','scenarios'}:
|
||||
raise ValueError('Unexpected rest-month outlook fields')
|
||||
if any(set(s)!={'peakKw','probability'} for s in value['scenarios']):raise ValueError('Unexpected scenario fields')
|
||||
return cls(value['month'],tuple(PeakScenario(s['peakKw'],s['probability']) for s in value['scenarios']),
|
||||
utc(value['issuedAt']),utc(value['futureFrom']),utc(value['historyUntil']),
|
||||
value['controlPolicyId'],value['method'],value['evidenceId'],value['reliance'])
|
||||
|
||||
|
||||
def next_month_start(at):
|
||||
local=utc(at).astimezone(ZURICH)
|
||||
if local.month==12:return utc(local.replace(year=local.year+1,month=1,day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
return utc(local.replace(month=local.month+1,day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
|
||||
|
||||
def empirical_rest_month(daily_records, *, month, at, horizon_end, control_policy_id, minimum_days=14, reliance=.5):
|
||||
"""Deterministic circular block bootstrap over comparable completed daily peaks.
|
||||
|
||||
Returns None without sufficient historical evidence; never fills with a cap.
|
||||
Input requires valid whole-day coverage and recorded policy identity. Historical
|
||||
measured daily peaks are only a planning proxy for future comparable operation.
|
||||
No fabricated future energy prices are needed for this peak-only outlook.
|
||||
"""
|
||||
valid_month(month)
|
||||
start=max(utc(horizon_end),utc(datetime.strptime(month,'%Y-%m').replace(tzinfo=ZURICH)))
|
||||
end=next_month_start(start)
|
||||
if month_key(start)!=month or start>=end:return None
|
||||
good={}
|
||||
for r in daily_records:
|
||||
if r.get('controlPolicyId')!=control_policy_id or r.get('complete') is not True:continue
|
||||
if r.get('quality') not in ('verified','estimated'):continue
|
||||
day=datetime.strptime(r['day'],'%Y-%m-%d').replace(tzinfo=ZURICH)
|
||||
finished=utc(day+timedelta(days=1))
|
||||
observed=utc(r['observedAt'])
|
||||
if finished>utc(at) or observed>utc(at) or observed<finished:continue
|
||||
if utc(at)-finished>timedelta(days=90):continue
|
||||
kw=number(r['peakKw'],'historical daily peak',0,1e6)
|
||||
if r['day'] in good and good[r['day']]!=kw:raise ValueError('Conflicting daily peak evidence')
|
||||
good[r['day']]=kw
|
||||
if len(good)<minimum_days:return None
|
||||
days=sorted(good)
|
||||
# Use a continuous segment. Missing days cannot be disguised as complete coverage.
|
||||
longest=[];segment=[]
|
||||
for day in days:
|
||||
if segment and datetime.strptime(day,'%Y-%m-%d')-datetime.strptime(segment[-1],'%Y-%m-%d')!=timedelta(days=1):
|
||||
if len(segment)>len(longest):longest=segment
|
||||
segment=[]
|
||||
segment.append(day)
|
||||
if len(segment)>len(longest):longest=segment
|
||||
if len(longest)<minimum_days:return None
|
||||
future_days=max(1,(end.astimezone(ZURICH).date()-start.astimezone(ZURICH).date()).days)
|
||||
values=[good[d] for d in longest]
|
||||
maxima=[max(values[(offset+n)%len(values)] for n in range(future_days)) for offset in range(len(values))]
|
||||
weights={}
|
||||
for value in maxima:weights[value]=weights.get(value,0)+1
|
||||
weights={value:count/len(maxima) for value,count in weights.items()}
|
||||
import hashlib,json
|
||||
evidence=hashlib.sha256(json.dumps({'history':[(d,good[d]) for d in longest],'policy':control_policy_id},sort_keys=True).encode()).hexdigest()
|
||||
return RestMonthOutlook(month,tuple(PeakScenario(k,v) for k,v in sorted(weights.items())),utc(at),start,
|
||||
utc(datetime.strptime(longest[-1],'%Y-%m-%d').replace(tzinfo=ZURICH)+timedelta(days=1)),
|
||||
control_policy_id,'comparable_daily_peak_block_bootstrap',evidence,reliance)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Explicit identity/provenance for the OBSERVATION-ONLY Symcon receiver.
|
||||
This is not a dispatch permit and does not make a shadow plan executable locally.
|
||||
"""
|
||||
from copy import deepcopy
|
||||
|
||||
|
||||
def control_context(operation):
|
||||
assets = {}
|
||||
for b in operation['batteries']:
|
||||
rearm = b.get('rearmSocPercent')
|
||||
assets[b['id']] = {
|
||||
'capacityKwh': b['capacityKwh'],
|
||||
'minSocPercent': b['minSocPercent'],
|
||||
'maxSocPercent': b['maxSocPercent'],
|
||||
'physicalMinSocPercent': b.get('physicalMinSocPercent', 0.0),
|
||||
'rearmSocPercent': b['minSocPercent'] if rearm is None else rearm,
|
||||
'gridCharging': b['gridCharging'],
|
||||
}
|
||||
return {'batteries': assets, 'limits': deepcopy(operation['limits'])}
|
||||
|
||||
|
||||
def provenance(inputs):
|
||||
# Capture BEFORE optimization. Never attach a newer operation to an older plan.
|
||||
return {'inputRefs': {kind: inputs[kind]['eventId'] for kind in ('operation', 'forecast', 'tariffs')},
|
||||
'controlContext': control_context(inputs['operation'])}
|
||||
@@ -0,0 +1,53 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from .domain import default_registry,number,utc,ZURICH
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReplayScore:
|
||||
family:str
|
||||
comparison_key:str
|
||||
first_decision:object
|
||||
last_decision:object
|
||||
forecast_issued_at:object
|
||||
actual_available_at:object
|
||||
cost_chf:float
|
||||
coverage:float
|
||||
days:int
|
||||
constraint_breaches:int=0
|
||||
terminal_normalized:bool=True
|
||||
def valid(self):
|
||||
number(self.cost_chf,'replay cost');number(self.coverage,'coverage',0,1)
|
||||
return (utc(self.forecast_issued_at)<=utc(self.first_decision) and utc(self.actual_available_at)>=utc(self.last_decision) and self.terminal_normalized and self.constraint_breaches==0)
|
||||
|
||||
def choose_family(setting,current,scores,*,registry=None,lookback_days=14,minimum_days=7,minimum_coverage=.9,margin_chf=1.,now):
|
||||
registry=registry or default_registry()
|
||||
if setting!='auto':
|
||||
registry.get(setting)
|
||||
return {'family':setting,'mode':'configured','reason':'Explicit installation setting'}
|
||||
registry.get(current);number(margin_chf,'margin',0);valid=[]
|
||||
for score in scores:
|
||||
registry.get(score.family)
|
||||
if (score.valid() and score.coverage>=minimum_coverage and score.days>=minimum_days
|
||||
and utc(score.first_decision)>=utc(now)-timedelta(days=lookback_days)
|
||||
and utc(score.last_decision)<=utc(now) and utc(score.actual_available_at)<=utc(now)):
|
||||
valid.append(score)
|
||||
groups={}
|
||||
for score in valid:
|
||||
key=(score.comparison_key,utc(score.first_decision),utc(score.last_decision),score.days)
|
||||
groups.setdefault(key,{})[score.family]=score
|
||||
complete=[g for g in groups.values() if len(g)==len(registry.entries())]
|
||||
if not complete:return {'family':current,'mode':'collecting','reason':'Insufficient comparable out-of-sample replay evidence'}
|
||||
group=max(complete,key=lambda g:utc(next(iter(g.values())).last_decision))
|
||||
best=min(group,key=lambda f:(group[f].cost_chf,f));improvement=group[current].cost_chf-group[best].cost_chf
|
||||
chosen=best if improvement>margin_chf else current
|
||||
return {'family':chosen,'mode':'economic_replay','improvementChf':improvement,'reason':'Matched historical cost replay; switching margin applied','costByFamilyChf':{k:v.cost_chf for k,v in group.items()}}
|
||||
|
||||
def training_due(last_trained_at,now,cadence='daily'):
|
||||
if cadence not in ('daily','weekly'):raise ValueError('Unknown training cadence')
|
||||
if last_trained_at is None:return True
|
||||
now,last=utc(now).astimezone(ZURICH),utc(last_trained_at).astimezone(ZURICH)
|
||||
return (now.date()-last.date()).days >= (1 if cadence=='daily' else 7)
|
||||
|
||||
def promote_candidate(*,active_cost,candidate_cost,valid_coverage,no_data_leakage,constraints_passed):
|
||||
number(active_cost,'active cost');number(candidate_cost,'candidate cost')
|
||||
return bool(valid_coverage and no_data_leakage and constraints_passed and candidate_cost<active_cost)
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Isolated V4 service: immutable inputs, coalesced replan queue, SHADOW publication.
|
||||
No external actuator endpoint, no reads of users.db, no legacy schedule changes.
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import asdict,replace
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
from threading import Event,Thread
|
||||
from uuid import UUID
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import logging
|
||||
from fastapi import FastAPI,Header,HTTPException
|
||||
from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_start,utc,number,priced_prefix
|
||||
from .store import PlannerStore,canonical
|
||||
from .selection import choose_family
|
||||
from .optimizer import optimize
|
||||
from . import meter_runtime, controlled_trial, measurement_pipeline
|
||||
from .forecast_quality import assess_family
|
||||
from .receiver_contract import provenance
|
||||
from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month
|
||||
|
||||
KINDS={'forecast','operation','tariffs','prices','planning_basis','peak_outlook'}
|
||||
|
||||
def latest(store,plant,kind):
|
||||
row=store.con.execute('''SELECT value FROM planner_inputs i JOIN planner_input_current c
|
||||
ON i.plant=c.plant AND i.kind=c.kind AND i.event_id=c.event_id WHERE i.plant=? AND i.kind=?''',(plant,kind)).fetchone()
|
||||
return json.loads(row[0]) if row else None
|
||||
|
||||
def batteries(value):
|
||||
result=[]
|
||||
for b in value:
|
||||
result.append(Battery(asset_id=b['id'],capacity_kwh=b['capacityKwh'],soc_percent=b['socPercent'],
|
||||
min_soc_percent=b['minSocPercent'],max_soc_percent=b['maxSocPercent'],
|
||||
max_charge_w=b['maxChargeW'],max_discharge_w=b['maxDischargeW'],measured_at=utc(b['measuredAt']),
|
||||
grid_charging=b['gridCharging'],throughput_chf_kwh=b.get('throughputChfKwh',0.),
|
||||
terminal_soc_min_percent=b.get('terminalMinSocPercent'),terminal_value_chf_kwh=b.get('terminalValueChfKwh',0.),
|
||||
recovery_allowed=b.get('recoveryAllowed',False),physical_min_soc_percent=b.get('physicalMinSocPercent',0.),
|
||||
discharge_blocked=b.get('dischargeBlocked',False),rearm_soc_percent=b.get('rearmSocPercent')))
|
||||
if type(b['gridCharging']) is not bool:raise ValueError('Explicit boolean grid-charging permission required')
|
||||
if len(result)>20 or len({b.asset_id for b in result})!=len(result):raise ValueError('Duplicate/too many batteries')
|
||||
return result
|
||||
|
||||
def validate(kind,value,now,registry):
|
||||
if kind not in KINDS or type(value.get('version')) is not int or value['version']!=1:raise ValueError('Unsupported event version/kind')
|
||||
identifier=value['eventId']
|
||||
if not isinstance(identifier,str) or not 1<=len(identifier)<=160:raise ValueError('Event ID required')
|
||||
observed=utc(value['observedAt'])
|
||||
if observed>now+timedelta(seconds=30):raise ValueError('Observation from future')
|
||||
fields={'version','eventId','observedAt'}
|
||||
if kind=='forecast':
|
||||
fields|={'families','modelVersions'}
|
||||
if not value['families']:raise ValueError('No forecast families')
|
||||
for key,family in value['families'].items():
|
||||
registry.get(key)
|
||||
if family['loadBasis'] not in ('base_load','house_total'):raise ValueError('Explicit metering basis required')
|
||||
evidence=family.get('accountingEvidenceId')
|
||||
if evidence is not None and (family['loadBasis']!='base_load' or not isinstance(evidence,str) or not 8<=len(evidence)<=160):raise ValueError('Explicit base-load accounting evidence required')
|
||||
if family.get('trainedUntil') and utc(family['trainedUntil'])>observed:raise ValueError('Training leakage')
|
||||
if not 1<=len(family['points'])<=576:raise ValueError('Need 1..576 forecast intervals')
|
||||
previous=None
|
||||
for p in family['points']:
|
||||
t=utc(p['time'])
|
||||
if t.second or t.microsecond or t.minute%5:raise ValueError('Forecast interval alignment')
|
||||
if previous and t-previous!=timedelta(minutes=5):raise ValueError('Forecast gap/overlap')
|
||||
previous=t;number(p['pvW'],'PV',0,1e9);number(p['loadW'],'load',0,1e9);number(p.get('externalW',0.),'external',-1e9,1e9)
|
||||
elif kind=='operation':
|
||||
fields|={'gridW','meteringBoundary','batteries','limits','measuredPeaks','quarterPast','planningPeaks','quarterEstimate','meterObservation'}
|
||||
if value['meteringBoundary']!='common_pcc':raise ValueError('Common metering boundary required')
|
||||
number(value['gridW'],'grid W',-1e9,1e9)
|
||||
for b in batteries(value['batteries']):b.validate(observed)
|
||||
limits=value['limits']
|
||||
if set(limits)!={'importW','exportW','managerMonthLimitsW'}:raise ValueError('Explicit limits required; null unlimited, zero zero')
|
||||
for k in ('importW','exportW'):
|
||||
if limits[k] is not None:number(limits[k],k,0,1e9)
|
||||
for k,v in limits['managerMonthLimitsW'].items():
|
||||
if str(int(k))!=k or not 1<=int(k)<=12:raise ValueError('Invalid manager month')
|
||||
number(v,'manager limit',0,1e9)
|
||||
for m,p in value.get('measuredPeaks',{}).items():
|
||||
datetime.strptime(m,'%Y-%m');number(p['kw'],'peak kW',0)
|
||||
if m>month_key(observed) or p['source'] not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Measured peak source invalid; cap is not paid peak')
|
||||
past=value.get('quarterPast')
|
||||
if past:
|
||||
q=quarter_start(observed)
|
||||
if utc(past['start'])!=q or type(past['measuredSeconds']) is not int or past['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Quarter measurement timestamp mismatch')
|
||||
number(past['importKwh'],'quarter energy',0)
|
||||
for m,p in value.get('planningPeaks',{}).items():
|
||||
record=basis_record(p,m,observed,allow_estimates=True)
|
||||
if record['quality']!='estimated':raise ValueError('planningPeaks contains estimates only')
|
||||
estimate=value.get('quarterEstimate')
|
||||
if estimate:
|
||||
if set(estimate)-{'start','measuredSeconds','importKwh','quality','source','coverage','notes'}:raise ValueError('Unknown quarter estimate field')
|
||||
if estimate.get('quality')!='estimated' or estimate.get('source') not in ('power_history_estimate','sampled_power_estimate','counter_interval_estimate'):raise ValueError('Explicit quarter estimate provenance required')
|
||||
q=quarter_start(observed)
|
||||
if utc(estimate['start'])!=q or type(estimate['measuredSeconds']) is not int or estimate['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Estimated quarter timing mismatch')
|
||||
number(estimate['importKwh'],'estimated quarter energy',0)
|
||||
if number(estimate.get('coverage'),'quarter estimate coverage',0,1)<1.:raise ValueError('Missing current-quarter coverage')
|
||||
if value.get('meterObservation') is not None:meter_runtime.validate_observation(value['meterObservation'],observed)
|
||||
elif kind=='planning_basis':
|
||||
fields|={'peaks'}
|
||||
if not isinstance(value.get('peaks'),dict) or not value['peaks']:raise ValueError('Explicit peak estimates required')
|
||||
for m,p in value['peaks'].items():
|
||||
if basis_record(p,m,observed,allow_estimates=True)['quality']!='estimated':raise ValueError('Planning basis is not a metering import')
|
||||
elif kind=='peak_outlook':
|
||||
fields|={'outlooks'}
|
||||
for m,v in value['outlooks'].items():
|
||||
outlook=RestMonthOutlook.from_dict(v)
|
||||
if m!=outlook.month:raise ValueError('Outlook month mismatch')
|
||||
outlook.validate(observed,observed)
|
||||
elif kind=='tariffs':
|
||||
fields|={'import','export','peakChfKwMonth'}
|
||||
for side in ('import','export'):
|
||||
p=value[side]
|
||||
if p['mode'] not in ('static','dynamic') or not p['tariffId']:raise ValueError('Explicit price mode/id required')
|
||||
if p['mode']=='static':number(p['staticChfKwh'],'static price')
|
||||
peaks=value['peakChfKwMonth']
|
||||
if isinstance(peaks,dict):
|
||||
for m,v in peaks.items():datetime.strptime(m,'%Y-%m');number(v,'peak tariff',0)
|
||||
else:number(peaks,'peak tariff',0)
|
||||
else:
|
||||
fields|={'periods'}
|
||||
if len(value['periods'])>3000:raise ValueError('Too many price intervals')
|
||||
for p in value['periods']:
|
||||
if p['unit'] not in ('CHF/kWh','CHF_kWh','Rp/kWh','CHF/MWh') or p['side'] not in ('import','export'):raise ValueError('Explicit price unit/direction required')
|
||||
if p['sourceKind'] not in ('published_interval','estimate','carried_forward'):raise ValueError('Explicit price provenance required')
|
||||
number(p['value'],'price')
|
||||
if utc(p['end'])<=utc(p['start']) or utc(p['observedAt'])>observed:raise ValueError('Invalid price interval or observation')
|
||||
if p.get('publishedAt') and utc(p['publishedAt'])>utc(p['observedAt']):raise ValueError('Price not published when observed')
|
||||
if set(value)-fields:raise ValueError('Unknown fields: extra device data/credentials must not be submitted')
|
||||
canonical(value)
|
||||
|
||||
def ingest(store,plant,kind,value,now):
|
||||
validate(kind,value,now,store.registry);data=canonical(value);con=store.con;con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
old=con.execute('SELECT value FROM planner_inputs WHERE plant=? AND kind=? AND event_id=?',(plant,kind,value['eventId'])).fetchone()
|
||||
if old:
|
||||
if old[0]!=data:raise ValueError('Immutable event conflict')
|
||||
con.commit();return {'status':'duplicate','queued':False}
|
||||
current=latest(store,plant,kind)
|
||||
if current and utc(current['observedAt'])==utc(value['observedAt']):
|
||||
left,right=dict(current),dict(value);left.pop('eventId');right.pop('eventId')
|
||||
if canonical(left)!=canonical(right):raise ValueError('Conflicting simultaneous observations')
|
||||
con.execute('INSERT INTO planner_inputs VALUES(?,?,?,?,?)',(plant,kind,value['eventId'],utc(value['observedAt']).isoformat(timespec='microseconds'),data))
|
||||
newer=not current or utc(current['observedAt'])<=utc(value['observedAt'])
|
||||
if newer:
|
||||
if kind=='operation':
|
||||
known=store.peaks(plant)
|
||||
for m,p in value.get('measuredPeaks',{}).items():
|
||||
if m in known and p['kw']<known[m]-1e-9:raise ValueError('Measured peak decreased')
|
||||
con.execute('''INSERT INTO planner_month_peaks VALUES(?,?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET peak_kw=excluded.peak_kw,source=excluded.source,updated_at=excluded.updated_at''',(plant,m,p['kw'],p['source'],utc(now).isoformat()))
|
||||
if kind=='operation':
|
||||
for m,p in value.get('planningPeaks',{}).items():meter_runtime.save_assumption(con,plant,m,p,now)
|
||||
if value.get('meterObservation'):meter_runtime.observe(con,plant,value['meterObservation'],utc(value['observedAt']))
|
||||
elif kind=='planning_basis':
|
||||
for m,p in value['peaks'].items():meter_runtime.save_assumption(con,plant,m,p,now)
|
||||
con.execute('INSERT INTO planner_input_current VALUES(?,?,?) ON CONFLICT(plant,kind) DO UPDATE SET event_id=excluded.event_id',(plant,kind,value['eventId']))
|
||||
store._request(plant,store.settings(plant)['revision'],kind+'_changed',now)
|
||||
con.commit();return {'status':'stored' if newer else 'archived_older','queued':newer}
|
||||
except Exception:con.rollback();raise
|
||||
|
||||
def price_at(store,plant,side,config,start,end,now):
|
||||
if config['mode']=='static':return Price(config['staticChfKwh'])
|
||||
rows=store.con.execute('''SELECT value FROM planner_inputs WHERE plant=? AND kind='prices' AND observed_at<=? ORDER BY observed_at DESC''',(plant,utc(now).isoformat(timespec='microseconds')))
|
||||
candidates=[]
|
||||
for row in rows:
|
||||
for p in json.loads(row[0])['periods']:
|
||||
if (p['tariffId']==config['tariffId'] and p['side']==side and p['sourceKind']=='published_interval'
|
||||
and utc(p['start'])<=start and utc(p['end'])>=end and utc(p['observedAt'])<=now):
|
||||
factor={'CHF/kWh':1.,'CHF_kWh':1.,'Rp/kWh':.01,'CHF/MWh':.001}[p['unit']]
|
||||
candidates.append((utc(p['observedAt']),p['value']*factor))
|
||||
if not candidates:return None
|
||||
last=max(t for t,v in candidates);values={v for t,v in candidates if t==last}
|
||||
if len(values)!=1:raise ValueError('Conflicting published price intervals')
|
||||
return Price(values.pop(),last,'dynamic')
|
||||
|
||||
class AwaitingInput(ValueError):pass
|
||||
|
||||
def assemble(store,plant,family,now):
|
||||
values={k:latest(store,plant,k) for k in ('operation','forecast','tariffs')}
|
||||
for k,v in values.items():
|
||||
if not v:raise AwaitingInput('Missing '+k+' input')
|
||||
op,forecast,tariffs=(values[k] for k in ('operation','forecast','tariffs'))
|
||||
decision=utc(op['observedAt']).replace(microsecond=0)
|
||||
settings=store.settings(plant);allow_estimates=settings['measurementPolicy']=='allow_estimates'
|
||||
input_quality={'quarter':'verified','warnings':[]}
|
||||
if not 0<=(now-decision).total_seconds()<=120:raise AwaitingInput('Fresh manager observation required (120s maximum)')
|
||||
if (now-utc(forecast['observedAt'])).total_seconds()>5400:raise AwaitingInput('Forecast older than 90 minutes')
|
||||
if utc(forecast['observedAt'])>decision or utc(tariffs['observedAt'])>decision:raise AwaitingInput('New data awaiting fresh manager observation')
|
||||
if settings['forecastSource']=='corrected_profile':
|
||||
try:forecast=measurement_pipeline.apply_load_forecast(store.con,plant,settings['measurementDataset'],forecast,int(decision.timestamp()))
|
||||
except ValueError as exc:raise AwaitingInput(str(exc)) from exc
|
||||
if family not in forecast['families']:raise AwaitingInput('Chosen family is unavailable; no silent switch')
|
||||
source=forecast['families'][family];steps=[]
|
||||
future_source={**source,'points':[p for p in source['points'] if utc(p['time'])+timedelta(minutes=5)>decision]}
|
||||
assessment=assess_family(future_source)
|
||||
if not assessment['valid']:raise AwaitingInput(assessment['reason'])
|
||||
input_quality['forecastAssessment']=assessment
|
||||
if source.get('dataPipeline'):
|
||||
input_quality['dataPipeline']=source['dataPipeline']
|
||||
input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule')
|
||||
for p in source['points']:
|
||||
start=utc(p['time']);end=start+timedelta(minutes=5)
|
||||
if end<=decision:continue
|
||||
start=max(start,decision)
|
||||
external=p.get('externalW',0.) if source['loadBasis']=='base_load' else 0.
|
||||
steps.append(Step(start,p['loadW'],p['pvW'],price_at(store,plant,'import',tariffs['import'],start,end,decision),price_at(store,plant,'export',tariffs['export'],start,end,decision),external,int((end-start).total_seconds())))
|
||||
if not steps or steps[0].start!=decision:raise AwaitingInput('Forecast has no current interval')
|
||||
full_end=steps[-1].end;steps=priced_prefix(steps,decision)
|
||||
if not steps:raise AwaitingInput('No complete published-price billing quarter')
|
||||
q=quarter_start(decision);elapsed=int((decision-q).total_seconds());past={}
|
||||
if elapsed:
|
||||
p=op.get('quarterPast')
|
||||
if not p and allow_estimates:
|
||||
p=op.get('quarterEstimate')
|
||||
if not p and op.get('meterObservation'):p=meter_runtime.current_quarter(store.con,plant,op['meterObservation'],decision)
|
||||
if p:
|
||||
input_quality['quarter']='estimated'
|
||||
input_quality['warnings'].append('Current quarter uses an explicitly estimated energy value, not a billing measurement')
|
||||
if not p or utc(p['start'])!=q or p['measuredSeconds']!=elapsed:raise AwaitingInput('Current-quarter energy missing (measured or explicitly permitted estimate)')
|
||||
past[q]=QuarterPast(p['importKwh'],elapsed)
|
||||
peaks=store.peaks(plant);months={month_key(s.start) for s in steps};contexts={}
|
||||
estimates=meter_runtime.assumptions(store.con,plant)
|
||||
for m in months:
|
||||
if m>month_key(decision):
|
||||
peaks[m]=0.;contexts[m]={'kw':0.,'quality':'new_month','source':'new_month','observedAt':decision.isoformat()}
|
||||
elif m in peaks:
|
||||
r=store.con.execute('SELECT source,updated_at FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone()
|
||||
known_at=op['observedAt'] if m in op.get('measuredPeaks',{}) else r['updated_at']
|
||||
contexts[m]={'kw':peaks[m],'quality':'verified','source':r['source'],'observedAt':known_at}
|
||||
# A later acquired larger quarter may raise an older verified baseline,
|
||||
# but the resulting combined planning basis must then say estimated.
|
||||
if allow_estimates and m in estimates and estimates[m]['kw']>peaks[m] and utc(estimates[m]['observedAt'])>utc(known_at):
|
||||
contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True)
|
||||
peaks[m]=contexts[m]['kw']
|
||||
input_quality['warnings'].append('A newer sampled quarter increased the earlier verified peak baseline; current planning maximum is estimated')
|
||||
elif allow_estimates and m in estimates:
|
||||
contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True);peaks[m]=contexts[m]['kw']
|
||||
input_quality['warnings'].append('Monthly peak '+m+' is a planning estimate, not an authoritative billing maximum')
|
||||
else:raise AwaitingInput('Peak basis missing; supply a verified maximum or explicitly permit a labelled estimate')
|
||||
peak_prices=tariffs['peakChfKwMonth']
|
||||
if not isinstance(peak_prices,dict):peak_prices={m:peak_prices for m in months}
|
||||
lim=op['limits'];limits=Limits(lim['exportW'],lim['importW'],{int(k):v for k,v in lim['managerMonthLimitsW'].items()})
|
||||
outlooks={};horizon_end=steps[-1].end
|
||||
if settings['peakOutlookPolicy']=='empirical_if_available':
|
||||
explicit=latest(store,plant,'peak_outlook')
|
||||
for m in months:
|
||||
if explicit and m in explicit['outlooks']:
|
||||
candidate=RestMonthOutlook.from_dict(explicit['outlooks'][m])
|
||||
try:
|
||||
candidate.validate(decision,horizon_end)
|
||||
policy=op.get('meterObservation',{}).get('controlPolicyId')
|
||||
if not policy or candidate.control_policy_id!=policy:raise ValueError('Different or unknown control policy')
|
||||
except ValueError:input_quality['warnings'].append('Stale, overlapping or incomparable rest-month outlook ignored; full incremental tariff used')
|
||||
else:outlooks[m]=replace(candidate,reliance=min(candidate.reliance,settings['peakOutlookReliance']))
|
||||
elif op.get('meterObservation'):
|
||||
observation=op['meterObservation']
|
||||
candidate=empirical_rest_month(meter_runtime.daily_peaks(store.con,plant,observation['meterId'],decision),
|
||||
month=m,at=decision,horizon_end=horizon_end,control_policy_id=observation['controlPolicyId'],
|
||||
reliance=settings['peakOutlookReliance'])
|
||||
if candidate is not None:outlooks[m]=candidate
|
||||
if not outlooks:input_quality['warnings'].append('Insufficient comparable rest-month history; full incremental peak tariff used, no arbitrary free peak allowance')
|
||||
data={'steps':steps,'batteries':batteries(op['batteries']),'limits':limits,'observed_peaks':peaks,'peak_prices':peak_prices,
|
||||
'quarter_history':past,'at':decision,'peak_context':contexts,'peak_outlooks':outlooks}
|
||||
if source['loadBasis']=='base_load' and source.get('accountingEvidenceId'):
|
||||
input_quality['accountingEvidenceId']=source['accountingEvidenceId']
|
||||
return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)}
|
||||
|
||||
def run_once(store,now):
|
||||
stamp=int(now.timestamp())//300
|
||||
plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')]
|
||||
for plant in plants:
|
||||
config=store.settings(plant)
|
||||
datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))]
|
||||
for dataset in datasets:
|
||||
try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp()))
|
||||
except ValueError as exc:logging.getLogger(__name__).warning('Data pipeline unavailable for configured dataset: %s',type(exc).__name__)
|
||||
with store.con:
|
||||
old=store.con.execute('SELECT tick FROM planner_ticks WHERE plant=?',(plant,)).fetchone()
|
||||
if not old or old[0]!=stamp:
|
||||
store._request(plant,store.settings(plant)['revision'],'five_minute_tick',now)
|
||||
store.con.execute('INSERT INTO planner_ticks VALUES(?,?) ON CONFLICT(plant) DO UPDATE SET tick=excluded.tick',(plant,stamp))
|
||||
claim=store.claim(now)
|
||||
if not claim:return {'status':'idle'}
|
||||
plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]}
|
||||
try:
|
||||
settings=store.settings(plant);previous=store.current(plant)
|
||||
current=previous['sourceFamily'] if previous else store.registry.entries()[0].key
|
||||
# Productive replay ingestion is intentionally not fabricated from R2 metrics.
|
||||
selection=choose_family(settings['family'],current,(),registry=store.registry,now=now)
|
||||
data,full_end,quality=assemble(store,plant,selection['family'],now)
|
||||
data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']]
|
||||
result=optimize(**data,config_revision=settings['revision'],family=selection['family'])
|
||||
if result['executable']:
|
||||
result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])})
|
||||
store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token'])
|
||||
return result
|
||||
except (ValueError,TypeError,KeyError) as exc:
|
||||
result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]}
|
||||
return result
|
||||
finally:
|
||||
detail={k:v for k,v in result.items() if k in ('status','reason','planId','configRevision','sourceFamily')}
|
||||
with store.con:store.con.execute('INSERT INTO planner_run_status VALUES(?,?,?,?) ON CONFLICT(plant) DO UPDATE SET updated_at=excluded.updated_at,status=excluded.status,detail=excluded.detail',(plant,now.isoformat(),result['status'],canonical(detail)))
|
||||
store.finish(claim)
|
||||
|
||||
def status(store,plant,now):
|
||||
plan=store.current(plant);settings=store.settings(plant)
|
||||
row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone()
|
||||
pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone()
|
||||
fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit'))
|
||||
return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)}
|
||||
|
||||
def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()):
|
||||
allowed={str(UUID(p)) for p in plants}
|
||||
trial_allowed={str(UUID(p)) for p in controlled_trial_plants}
|
||||
if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist')
|
||||
if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required')
|
||||
path=Path(db_path).resolve()
|
||||
if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required')
|
||||
path.parent.mkdir(parents=True,exist_ok=True);stop=Event()
|
||||
def factory():return PlannerStore(str(path))
|
||||
def loop():
|
||||
while not stop.is_set():
|
||||
s=factory()
|
||||
try:run_once(s,datetime.now(timezone.utc).replace(microsecond=0))
|
||||
except Exception as exc:logging.getLogger(__name__).error('V4 worker error: %s',type(exc).__name__)
|
||||
finally:s.close()
|
||||
stop.wait(1.)
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
thread=Thread(target=loop,name='v4-shadow',daemon=True)
|
||||
if start_worker:thread.start()
|
||||
yield
|
||||
stop.set()
|
||||
if start_worker:thread.join(35)
|
||||
app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan)
|
||||
app.state.store_factory=factory
|
||||
def authorize(plant,token):
|
||||
if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized')
|
||||
try:plant=str(UUID(plant))
|
||||
except ValueError:raise HTTPException(400,'Invalid installation ID')
|
||||
if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial')
|
||||
return factory()
|
||||
@app.get('/health')
|
||||
def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1}
|
||||
@app.get('/internal/v2/prognosis/{plant}/planner')
|
||||
def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc);view=status(s,plant,now)
|
||||
view['controlledTrial']=controlled_trial.authority(s,plant,view,now,trial_allowed)
|
||||
view['controlledTrialAuthorized']=view['controlledTrial'] is not None
|
||||
return view
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/trial/arm')
|
||||
def arm_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc)
|
||||
return controlled_trial.arm(s,plant,payload,status(s,plant,now),now,trial_allowed)
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/trial/revoke')
|
||||
def revoke_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
if set(payload)!={'sessionId'}:raise ValueError('Session ID only')
|
||||
return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.put('/internal/v2/prognosis/{plant}/planner/settings')
|
||||
def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:return s.save_settings(plant,payload['changes'],payload['expectedRevision'],datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409 if 'Revision conflict' in str(exc) else 400,str(exc))
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/replan')
|
||||
def replan(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:s.request(plant,'manual',datetime.now(timezone.utc));return {'status':'queued','liveEnabled':False}
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/inputs/{kind}')
|
||||
def input_event(plant:str,kind:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:return ingest(s,plant,kind,payload,datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}')
|
||||
def configure_dataset(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
if payload.get('datasetId')!=dataset:raise ValueError('Dataset path/payload mismatch')
|
||||
return measurement_pipeline.register_dataset(s.con,plant,payload,int(datetime.now(timezone.utc).timestamp()))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/measurements')
|
||||
def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc)
|
||||
result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp()))
|
||||
# Existing five-minute worker handles rollup/training; no per-record optimizer flood.
|
||||
return result
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/ack')
|
||||
def ack(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:s.acknowledge(plant,payload['planId'],payload['revision'],datetime.now(timezone.utc),payload.get('step'),payload.get('status','shadow_seen'));return {'status':'recorded'}
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
from starlette.responses import JSONResponse
|
||||
class BodyLimit:
|
||||
def __init__(self,app):self.app=app
|
||||
async def __call__(self,scope,receive,send):
|
||||
if scope['type']!='http' or scope['method'] not in ('POST','PUT'):return await self.app(scope,receive,send)
|
||||
chunks=[];total=0
|
||||
while True:
|
||||
msg=await receive()
|
||||
if msg['type']=='http.disconnect':return
|
||||
total+=len(msg.get('body',b''))
|
||||
if total>2000000:return await JSONResponse({'detail':'Request too large'},status_code=413)(scope,receive,send)
|
||||
chunks.append(msg)
|
||||
if not msg.get('more_body',False):break
|
||||
async def replay():return chunks.pop(0) if chunks else await receive()
|
||||
return await self.app(scope,replay,send)
|
||||
app.add_middleware(BodyLimit)
|
||||
return app
|
||||
|
||||
def from_environment():
|
||||
return create_app(os.environ.get('NETPLAN_V4_DB','/data/netplan-v4.sqlite'),os.environ.get('PROGNOSIS_SERVICE_TOKEN',''),[p.strip() for p in os.environ.get('NETPLAN_V4_PLANTS','').split(',') if p.strip()], controlled_trial_plants=[p.strip() for p in os.environ.get('NETPLAN_V4_CONTROL_TRIAL_PLANTS','').split(',') if p.strip()])
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime,timedelta
|
||||
from uuid import uuid4
|
||||
from .domain import default_registry,month_key,number,quarter_start,utc
|
||||
from . import meter_runtime, controlled_trial, measurement_pipeline
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False)
|
||||
|
||||
DEFAULT_SETTINGS={'family':'3','autoLookbackDays':14,'autoMinimumDays':7,'autoMinimumCoverage':.9,'autoSwitchMarginChf':1.,'tariffPolicy':'published_only','trainingCadence':'daily','trainingPromotion':'validated_only','runMode':'shadow','roundtripEfficiency':.90,'measurementPolicy':'verified_only','peakOutlookPolicy':'empirical_if_available','peakOutlookReliance':.5,'forecastSource':'legacy','measurementDataset':''}
|
||||
|
||||
class PlannerStore:
|
||||
"""Own SQLite file, no mutation of legacy application databases."""
|
||||
def __init__(self,path,registry=None):
|
||||
self.registry=registry or default_registry();self.con=sqlite3.connect(path,timeout=10)
|
||||
self.con.row_factory=sqlite3.Row;self.con.execute('PRAGMA foreign_keys=ON')
|
||||
self.con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_settings(plant TEXT PRIMARY KEY,revision INTEGER NOT NULL,value TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_audit(id INTEGER PRIMARY KEY,plant TEXT NOT NULL,at TEXT NOT NULL,kind TEXT NOT NULL,detail TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_work(plant TEXT PRIMARY KEY,sequence INTEGER NOT NULL,revision INTEGER NOT NULL,reasons TEXT NOT NULL,requested_at TEXT NOT NULL,lease_until TEXT,lease_token TEXT);
|
||||
CREATE TABLE IF NOT EXISTS planner_measurements(plant TEXT NOT NULL,start TEXT NOT NULL,import_kwh REAL NOT NULL,PRIMARY KEY(plant,start));
|
||||
CREATE TABLE IF NOT EXISTS planner_month_peaks(plant TEXT NOT NULL,month TEXT NOT NULL,peak_kw REAL NOT NULL,source TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(plant,month));
|
||||
CREATE TABLE IF NOT EXISTS planner_snapshots(id TEXT PRIMARY KEY,plant TEXT NOT NULL,issued_at TEXT NOT NULL,value TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_plans(plan_id TEXT PRIMARY KEY,plant TEXT NOT NULL,revision INTEGER NOT NULL,mode TEXT NOT NULL,value TEXT NOT NULL,created_at TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_current(plant TEXT NOT NULL,mode TEXT NOT NULL,plan_id TEXT NOT NULL REFERENCES planner_plans(plan_id),PRIMARY KEY(plant,mode));
|
||||
CREATE TABLE IF NOT EXISTS planner_ack(plant TEXT PRIMARY KEY,plan_id TEXT NOT NULL,revision INTEGER NOT NULL,received_at TEXT NOT NULL,applied_step TEXT,status TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_inputs(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,observed_at TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(plant,kind,event_id));
|
||||
CREATE TABLE IF NOT EXISTS planner_input_current(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,PRIMARY KEY(plant,kind));
|
||||
CREATE TABLE IF NOT EXISTS planner_run_status(plant TEXT PRIMARY KEY,updated_at TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_ticks(plant TEXT PRIMARY KEY,tick INTEGER NOT NULL);
|
||||
''')
|
||||
meter_runtime.schema(self.con)
|
||||
controlled_trial.schema(self.con)
|
||||
measurement_pipeline.schema(self.con)
|
||||
def close(self):self.con.close()
|
||||
def settings(self,plant):
|
||||
row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone()
|
||||
return {'revision':row['revision'],**DEFAULT_SETTINGS,**json.loads(row['value'])} if row else {'revision':0,**DEFAULT_SETTINGS}
|
||||
def _validate_settings(self,value):
|
||||
if set(value)!=set(DEFAULT_SETTINGS):raise ValueError('Unknown or missing setting')
|
||||
if value['family']!='auto':self.registry.get(value['family'])
|
||||
for key in ('autoLookbackDays','autoMinimumDays'):
|
||||
if type(value[key]) is not int:raise ValueError('Days must be integers')
|
||||
number(value['autoLookbackDays'],'lookback',7,90);number(value['autoMinimumDays'],'minimum days',1,value['autoLookbackDays'])
|
||||
number(value['autoMinimumCoverage'],'coverage',.5,1);number(value['autoSwitchMarginChf'],'margin',0)
|
||||
if value['tariffPolicy']!='published_only':raise ValueError('Only published-price policy implemented')
|
||||
if value['trainingCadence'] not in ('daily','weekly') or value['trainingPromotion']!='validated_only':raise ValueError('Training must use validated promotion')
|
||||
if value['runMode']!='shadow':raise ValueError('Shadow-only: live release requires separate validation')
|
||||
number(value['roundtripEfficiency'],'roundtrip efficiency',.01,1)
|
||||
if value['measurementPolicy'] not in ('verified_only','allow_estimates'):raise ValueError('Invalid measurement policy')
|
||||
if value['peakOutlookPolicy'] not in ('full_incremental','empirical_if_available'):raise ValueError('Invalid peak outlook policy')
|
||||
number(value['peakOutlookReliance'],'peak outlook reliance',0,1)
|
||||
if value['forecastSource'] not in ('legacy','corrected_profile'):raise ValueError('Unknown forecast source')
|
||||
if not isinstance(value['measurementDataset'],str) or len(value['measurementDataset'])>80:raise ValueError('Invalid measurement dataset')
|
||||
if value['forecastSource']=='corrected_profile' and not value['measurementDataset']:raise ValueError('Corrected forecast requires an explicit dataset')
|
||||
def _request(self,plant,revision,reason,now):
|
||||
row=self.con.execute('SELECT * FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
reasons=set(json.loads(row['reasons'])) if row else set();reasons.add(reason)
|
||||
seq=row['sequence']+1 if row else 1
|
||||
self.con.execute('''INSERT INTO planner_work(plant,sequence,revision,reasons,requested_at) VALUES(?,?,?,?,?)
|
||||
ON CONFLICT(plant) DO UPDATE SET sequence=excluded.sequence,revision=excluded.revision,reasons=excluded.reasons,requested_at=excluded.requested_at''',(plant,seq,revision,canonical(sorted(reasons)),utc(now).isoformat()))
|
||||
def save_settings(self,plant,changes,expected_revision,now):
|
||||
if type(expected_revision) is not int or expected_revision<0:raise ValueError('Invalid expected revision')
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
current=self.settings(plant)
|
||||
if current.pop('revision')!=expected_revision:raise ValueError('Revision conflict; reload before saving')
|
||||
current.update(changes);self._validate_settings(current);revision=expected_revision+1
|
||||
self.con.execute('''INSERT INTO planner_settings VALUES(?,?,?) ON CONFLICT(plant) DO UPDATE SET revision=excluded.revision,value=excluded.value''',(plant,revision,canonical(current)))
|
||||
self._request(plant,revision,'configuration_changed',now)
|
||||
self.con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',(plant,utc(now).isoformat(),'settings',canonical({'revision':revision,'changes':changes})))
|
||||
self.con.commit();return {'revision':revision,**current}
|
||||
except Exception:self.con.rollback();raise
|
||||
def request(self,plant,reason,now):
|
||||
if reason not in ('prices_changed','telemetry_changed','five_minute_tick','manual','model_promoted','forecast_changed','operation_changed','tariffs_changed'):raise ValueError('Unknown trigger')
|
||||
with self.con:self._request(plant,self.settings(plant)['revision'],reason,now)
|
||||
def claim(self,now,lease_seconds=120):
|
||||
at=utc(now);self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
row=self.con.execute('SELECT * FROM planner_work WHERE lease_until IS NULL OR lease_until < ? ORDER BY requested_at LIMIT 1',(at.isoformat(),)).fetchone()
|
||||
if not row:self.con.commit();return None
|
||||
token=str(uuid4());self.con.execute('UPDATE planner_work SET lease_until=?,lease_token=? WHERE plant=?',((at+timedelta(seconds=lease_seconds)).isoformat(),token,row['plant']))
|
||||
self.con.commit();return {**dict(row),'lease_token':token}
|
||||
except Exception:self.con.rollback();raise
|
||||
def finish(self,claim):
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
row=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(claim['plant'],)).fetchone()
|
||||
if not row or row['lease_token']!=claim['lease_token']:self.con.commit();return False
|
||||
if row['sequence']==claim['sequence']:self.con.execute('DELETE FROM planner_work WHERE plant=?',(claim['plant'],))
|
||||
else:self.con.execute('UPDATE planner_work SET lease_until=NULL,lease_token=NULL WHERE plant=?',(claim['plant'],))
|
||||
self.con.commit();return True
|
||||
except Exception:self.con.rollback();raise
|
||||
def initialize_peak(self,plant,month,peak_kw,source,now):
|
||||
number(peak_kw,'authoritative measured peak',0)
|
||||
if source not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Configured cap is NOT measured peak')
|
||||
datetime.strptime(month,'%Y-%m')
|
||||
if month>month_key(now):raise ValueError('Future month cannot have a measured peak')
|
||||
with self.con:
|
||||
old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,month)).fetchone()
|
||||
if old and peak_kw<old[0]:raise ValueError('Cannot lower measured peak silently')
|
||||
self.con.execute('''INSERT INTO planner_month_peaks VALUES(?,?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET peak_kw=excluded.peak_kw,source=excluded.source,updated_at=excluded.updated_at''',(plant,month,peak_kw,source,utc(now).isoformat()))
|
||||
def record_import_interval(self,plant,start,import_kwh,received_at):
|
||||
start,received_at=utc(start),utc(received_at);number(import_kwh,'metered import energy',0)
|
||||
if start.minute%5 or start.second or start.microsecond or start+timedelta(minutes=5)>received_at:raise ValueError('Completed aligned intervals required')
|
||||
with self.con:
|
||||
old=self.con.execute('SELECT import_kwh FROM planner_measurements WHERE plant=? AND start=?',(plant,start.isoformat())).fetchone()
|
||||
if old and abs(old[0]-import_kwh)>1e-9:raise ValueError('Conflicting metering fact')
|
||||
self.con.execute('INSERT OR IGNORE INTO planner_measurements VALUES(?,?,?)',(plant,start.isoformat(),import_kwh))
|
||||
q=quarter_start(start);rows=self.con.execute('SELECT start,import_kwh FROM planner_measurements WHERE plant=? AND start>=? AND start<? ORDER BY start',(plant,q.isoformat(),(q+timedelta(minutes=15)).isoformat())).fetchall()
|
||||
if len(rows)!=3:return None
|
||||
peak=sum(r['import_kwh'] for r in rows)/.25;m=month_key(q)
|
||||
old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone()
|
||||
if not old:return {'quarterPeakKw':peak,'monthState':'needs_initialization'}
|
||||
self.con.execute('UPDATE planner_month_peaks SET peak_kw=MAX(peak_kw,?),updated_at=? WHERE plant=? AND month=?',(peak,received_at.isoformat(),plant,m))
|
||||
return {'quarterPeakKw':peak,'monthState':'measured'}
|
||||
def peaks(self,plant):return {r['month']:r['peak_kw'] for r in self.con.execute('SELECT month,peak_kw FROM planner_month_peaks WHERE plant=?',(plant,))}
|
||||
def snapshot(self,plant,issued_at,value,snapshot_id=None):
|
||||
identifier=snapshot_id or str(uuid4())
|
||||
with self.con:self.con.execute('INSERT INTO planner_snapshots VALUES(?,?,?,?)',(identifier,plant,utc(issued_at).isoformat(),canonical(value)))
|
||||
return identifier
|
||||
def publish_shadow(self,plant,plan,expected_revision,now,work_sequence=None,work_token=None):
|
||||
if not plan.get('executable') or plan.get('configRevision')!=expected_revision:raise ValueError('Only validated plans for exact revision')
|
||||
if plan.get('runMode','shadow')!='shadow':raise ValueError('Only shadow publication permitted')
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
if self.settings(plant)['revision']!=expected_revision:raise ValueError('Configuration changed while computing')
|
||||
if work_sequence is not None:
|
||||
work=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
if not work or work[0]!=work_sequence or work_token is not None and work[1]!=work_token:raise ValueError('Newer request arrived while computing')
|
||||
identifier=plan['planId']
|
||||
self.con.execute('INSERT INTO planner_plans VALUES(?,?,?,?,?,?)',(identifier,plant,expected_revision,'shadow',canonical(plan),utc(now).isoformat()))
|
||||
self.con.execute('INSERT INTO planner_current VALUES(?,?,?) ON CONFLICT(plant,mode) DO UPDATE SET plan_id=excluded.plan_id',(plant,'shadow',identifier))
|
||||
self.con.commit()
|
||||
except Exception:self.con.rollback();raise
|
||||
def current(self,plant,mode='shadow'):
|
||||
row=self.con.execute('SELECT value FROM planner_plans JOIN planner_current USING(plan_id) WHERE planner_current.plant=? AND planner_current.mode=?',(plant,mode)).fetchone()
|
||||
return json.loads(row[0]) if row else None
|
||||
def acknowledge(self,plant,plan_id,revision,now,step=None,status='received'):
|
||||
if status not in ('received','applied','rejected','shadow_seen'):raise ValueError('Unknown acknowledgement')
|
||||
row=self.con.execute('SELECT mode,revision,value FROM planner_plans WHERE plan_id=? AND plant=?',(plan_id,plant)).fetchone()
|
||||
if not row or row['revision']!=revision:raise ValueError('Unknown plan/revision')
|
||||
if status=='applied' and row['mode']!='live':raise ValueError('Shadow plan must never be applied')
|
||||
if step is not None and step not in {p['time'] for p in json.loads(row['value'])['points']}:raise ValueError('Step does not belong to plan')
|
||||
with self.con:self.con.execute('''INSERT INTO planner_ack VALUES(?,?,?,?,?,?) ON CONFLICT(plant) DO UPDATE SET plan_id=excluded.plan_id,revision=excluded.revision,received_at=excluded.received_at,applied_step=excluded.applied_step,status=excluded.status''',(plant,plan_id,revision,utc(now).isoformat(),step,status))
|
||||
Reference in New Issue
Block a user