From 640fe537209fcb8ea56fc788430431bb0e0da1cb Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Thu, 1 Oct 2026 16:20:36 +0000 Subject: [PATCH 01/22] feat(manager): add opt-in native V4 shadow operation telemetry --- CHANGELOG.md | 4 + Manager/module.php | 12 ++ docs/testing/Netzfahrplan-V4-Schatten.md | 83 ++++++++++ libs/ManagerNetzfahrplanV4Trait.php | 157 ++++++++++++++++++ libs/NetzfahrplanV4Betriebsdaten.php | 191 ++++++++++++++++++++++ tests/NetzfahrplanV4BetriebsdatenTest.php | 29 ++++ tests/fixtures/NativeV4Scenarios.php | 124 ++++++++++++++ 7 files changed, 600 insertions(+) create mode 100644 docs/testing/Netzfahrplan-V4-Schatten.md create mode 100644 libs/ManagerNetzfahrplanV4Trait.php create mode 100644 libs/NetzfahrplanV4Betriebsdaten.php create mode 100644 tests/NetzfahrplanV4BetriebsdatenTest.php create mode 100644 tests/fixtures/NativeV4Scenarios.php diff --git a/CHANGELOG.md b/CHANGELOG.md index a316bca..4f6db07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Alle wesentlichen Aenderungen an Enelix EMS werden in dieser Datei dokumentiert. +## Unreleased + +- Optionaler nativer V4-Betriebsdatensender fuer den Schattenversuch (standardmaessig aus), separate Diagnose und sichere Batterie-/Messquellenzuordnung. Kein neuer Fahrplan wird lokal ausgefuehrt; vollstaendige Monatspeak-/Viertelstunden-Messnachweise und deren Historienadapter bleiben Voraussetzung. + ## 0.1 Build 1 - Erste Beta - 2026-09-29 ### Enthalten diff --git a/Manager/module.php b/Manager/module.php index 1090d94..ac74605 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -11,6 +11,8 @@ require_once __DIR__ . '/../libs/Anlagentopologie.php'; require_once __DIR__ . '/../libs/StoerungsSnapshot.php'; require_once __DIR__ . '/../libs/EnergieMessung.php'; require_once __DIR__ . '/../libs/ManagerEnergieTrait.php'; +require_once __DIR__ . '/../libs/NetzfahrplanV4Betriebsdaten.php'; +require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4Trait.php'; use Belevo\EnelixEMS\Anlagentopologie; use Belevo\EnelixEMS\EinspeiseRegler; @@ -25,6 +27,7 @@ use Belevo\EnelixEMS\StoerungsSnapshot; class Manager extends IPSModule implements ManagerSchnittstelle { use ManagerEnergieTrait; + use \Belevo\EnelixEMS\ManagerNetzfahrplanV4Trait; private const STATUS_AKTIV = 102; private const STATUS_MESSWERT_UNGUELTIG = 201; @@ -112,6 +115,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle public function Create(): void { parent::Create(); + $this->registriereNetzfahrplanV4(); $this->RegisterPropertyInteger('Rolle', 0); $this->RegisterPropertyInteger('NetzleistungVariableID', 0); @@ -232,6 +236,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle public function ApplyChanges(): void { parent::ApplyChanges(); + $this->SetTimerInterval('NetzfahrplanV4Senden', 0); $this->initialisiereLizenzInstallationID(); $this->aktualisiereVariablen(); @@ -287,6 +292,9 @@ class Manager extends IPSModule implements ManagerSchnittstelle return; } $this->SetStatus(self::STATUS_AKTIV); + if ($this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->SetTimerInterval('NetzfahrplanV4Senden', 60000); + } $this->synchronisiereAnlagentopologie(); if ($this->prognoseTelemetrieKonfiguriert()) { $this->SetTimerInterval( @@ -449,6 +457,10 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->regeln((bool) $wert); return; + case 'NetzfahrplanV4Senden': + $this->sendeNetzfahrplanV4(); + return; + case 'PrognoseSenden': $this->sendePrognoseTelemetrie(); return; diff --git a/docs/testing/Netzfahrplan-V4-Schatten.md b/docs/testing/Netzfahrplan-V4-Schatten.md new file mode 100644 index 0000000..92e33c8 --- /dev/null +++ b/docs/testing/Netzfahrplan-V4-Schatten.md @@ -0,0 +1,83 @@ +# V4: native Betriebsdaten, ausschliesslich Schattenbetrieb + +Stand 2026-10-01: Der optionale Sender und die Diagnose sind implementiert. +Kein V4-Plan wird damit uebernommen, kein Aktor geschrieben, kein V1-Sollwert +geaendert. Der Sender ist nach Installation standardmaessig AUS. + +## Konfiguration im Manager + +- NetzfahrplanV4SchattenAktiv: bool, Standard false. Eigener 60-s-Timer, + unabhaengig vom bestehenden Prognose-Timer. Fehler betreffen nur den Sendestatus. +- NetzfahrplanV4NetzladenErlaubt: bool, Standard false. Betrifft ausschliesslich + die im Schattenplan angenommene Berechtigung, keine reale Netzladefreigabe. +- NetzfahrplanV4BatterieOptionen: JSON-Objekt je Topologie-ID. Optional + SOCKapazitaet_kWh und MaxSOC_Prozent (sonst 100). Bei unterschiedlicher Nenn- + und Nutzkapazitaet ist SOCKapazitaet_kWh zwingend explizit anzugeben. +- NetzfahrplanV4MessnachweisVariableID: JSON-Stringvariable, Standard 0. + Ein vollstaendiger historischer/registerbasierter Messnachweis-Produzent + ist NOCH NICHT integriert. Fehlende Nachweise bleiben fehlend. + +Die Quelle muss der gemeinsame Netzanschluss sein. Topologiebatterien werden +nur einer eindeutigen aktiven Batterieinstanz mit derselben SOC- UND +Leistungsmessquelle zugeordnet. Externe SDL-Batterien werden nicht automatisch +hinzugefuegt. SDL-/Grundlastbereinigung bleibt eine separate, noch offene Aufgabe. +BMS-Grenzen werden mit Topologieleistungen begrenzt. Normale Betriebsreserve +und technisches Minimum werden nicht verwechselt: die hoehere Grenze gilt. + +Bei aktueller Entladehysterese wird die Entladeleistung konservativ auf 0 +begrenzt. Eine spaetere Freigabe innerhalb der 48h wird noch nicht modelliert. +Das ist sichtbar diagnostiziert und kann den Schattenplan wirtschaftlich +beschraenken; eine zeitabhaengige Hysterese-Modellierung bleibt offen. +SOC unterhalb der Reserve wird nicht kuenstlich angehoben: Diagnosefehler, +kein angeblich gueltiger Plan. Recovery-/Nachladungsmodell bleibt offen. + +## Diagnose ohne Netzwerk oder Stellbefehl + +Im Symcon-Skript nach Installation des neuen Modulstands: + +```php +RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); + $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); + $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4Senden', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); + } + + public function GetNetzfahrplanV4Diagnose(): string + { + try { + $manager = []; + foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', + 'Lastspitzenmodus', 'NetzbezugEnergieVariableID'] as $key) { + $manager[$key] = $this->ReadPropertyInteger($key); + } + foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { + $manager[$key] = $this->ReadPropertyFloat($key); + } + $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); + $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); + $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); + $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); + $cache = $this->leseJsonAttribut('VerbraucherCache'); + $controllers = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + $c = ['InstanzID' => $id]; + foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + $c[$key] = IPS_GetProperty($id, $key); + } + $entry = $cache[(string) $id] ?? []; + $c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0; + $c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false; + foreach (($entry['Daten']['Zustand'] ?? []) as $state) { + if (($state['Kennung'] ?? '') === 'HystereseAktiv') { + $c['HystereseAktiv'] = $state['Wert']; + } + } + $controllers[] = $c; + } + $evidence = []; + $evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID'); + if ($evidenceID > 0) { + if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) { + throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.'); + } + $evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($evidence)) { + throw new RuntimeException('Ungueltiger Messnachweis.'); + } + } + $read = static function (int $id): array { + if (!IPS_VariableExists($id)) { + throw new RuntimeException('Messvariable ' . $id . ' fehlt.'); + } + $before = IPS_GetVariable($id); + $value = GetValue($id); + $after = IPS_GetVariable($id); + if ($before['VariableUpdated'] !== $after['VariableUpdated']) { + throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); + } + return ['value' => $value, 'updated' => (int) $after['VariableUpdated']]; + }; + return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, + $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } catch (Throwable $error) { + return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR); + } + } + + public function GetNetzfahrplanV4Sendestatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4Sendestatus'); + } + + private function sendeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + return; + } + try { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $snapshot = $this->GetNetzfahrplanV4Diagnose(); + $data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR); + if (!isset($data['operation'])) { + throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.'); + } + // Decoding as objects preserves empty JSON dictionaries in the API contract. + $payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR); + if (!isset($payload->operation)) { + throw new RuntimeException('Keine konsistenten Betriebsdaten.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation'; + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.'); + } + try { + curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, + CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR), + CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]); + $answer = curl_exec($handle); + $status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($answer === false || $status !== 200) { + throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.'); + } + if (strlen($answer) > 65536) { + throw new RuntimeException('V4-Antwort ist zu gross.'); + } + $ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR); + if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) { + throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.'); + } + } finally { + curl_close($handle); + } + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt, + 'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + // Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands. + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } + } +} diff --git a/libs/NetzfahrplanV4Betriebsdaten.php b/libs/NetzfahrplanV4Betriebsdaten.php new file mode 100644 index 0000000..b3bf27e --- /dev/null +++ b/libs/NetzfahrplanV4Betriebsdaten.php @@ -0,0 +1,191 @@ + $max) { + throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.'); + } + return (float) $value; + } + + private static function timestamp($value): int + { + if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) { + throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.'); + } + return (new DateTimeImmutable($value))->getTimestamp(); + } + + /** $read returns ['value' => int|float, 'updated' => int], without changing a device. */ + private static function measurement(callable $read, int $id, int $now, int $age, string $label): array + { + if ($id <= 0) { + throw new InvalidArgumentException($label . ': Messquelle fehlt.'); + } + $m = $read($id); + if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now + || $now - $m['updated'] > $age) { + throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.'); + } + return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']]; + } + + /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ + private static function evidence(array $input, int $now, int $meterID, array &$warnings): array + { + $peaks = []; + $past = null; + if ($input === []) { + $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; + return [(object) [], null]; + } + if ($meterID <= 0 || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== 'symcon:' . $meterID) { + throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); + } + $measured = self::timestamp($input['measuredAt'] ?? null); + if ($measured > $now || $now - $measured > 120) { + throw new InvalidArgumentException('Messnachweis ist nicht aktuell.'); + } + $month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m'); + foreach (($input['measuredPeaks'] ?? []) as $key => $p) { + if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month + || !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) { + throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.'); + } + $peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']]; + } + if (!array_key_exists($month, $peaks)) { + $warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.'; + } + $q = intdiv($now, 900) * 900; + if (isset($input['quarterPast'])) { + $p = $input['quarterPast']; + // Stale quarter evidence remains absent, not extrapolated to the decision time. + if ($measured === $now && self::timestamp($p['start'] ?? null) === $q + && ($p['measuredSeconds'] ?? null) === $now - $q) { + $past = ['start' => gmdate('c', $q), + 'measuredSeconds' => $now - $q, + 'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)]; + } else { + $warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.'; + } + } + if ($now !== $q && $past === null) { + $warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.'; + } + return [(object) $peaks, $past]; + } + + /** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */ + public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array + { + $warnings = []; + $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, + (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); + $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); + $mode = $manager['Lastspitzenmodus'] ?? 0; + $import = null; + $months = []; + if ($mode === 1) { + $import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0); + } elseif ($mode === 2) { + foreach (($manager['Monatsgrenzen'] ?? []) as $row) { + $m = $row['MonatIndex'] ?? null; + if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) { + throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.'); + } + $months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0); + } + if (count($months) !== 12) { + throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.'); + } + } elseif ($mode !== 0) { + throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.'); + } + $export = !empty($manager['EinspeisebegrenzungAktiv']) + ? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null; + $result = []; + $seen = []; + $used = []; + foreach ($assets as $asset) { + $id = $asset['ID'] ?? ''; + if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) { + throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.'); + } + $seen[$id] = true; + $socID = (int) ($asset['SOCVariableID'] ?? 0); + $matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool { + return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID; + })); + if (count($matches) !== 1) { + throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.'); + } + $c = $matches[0]; + $controllerID = $c['InstanzID']; + if (isset($used[$controllerID])) { + throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.'); + } + $used[$controllerID] = true; + if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) { + throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.'); + } + $opts = $manager['BatterieOptionen'][$id] ?? []; + $nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001); + $usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal); + if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) { + throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.'); + } + $capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal); + $age = (int) ($c['MesswertMaxAlter'] ?? 30); + $soc = self::measurement($read, $socID, $now, $age, $id . ' SOC'); + $charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden'); + $discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen'); + $min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100), + self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); + $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); + $socValue = self::number($soc['value'], 'SOC', 0, 100); + if ($socValue < $min || $socValue > $max) { + throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); + } + if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { + throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.'); + } + $maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0)); + $maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0)); + if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) { + $maxCharge = $maxDischarge = 0.0; + $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; + } + if (!empty($c['HystereseAktiv'])) { + $maxDischarge = 0.0; + $warnings[] = $id . ': aktuelle Entladesperre konservativ uebernommen; keine spaetere Freigabe erfunden.'; + } + $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, + 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, + 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true]; + } + if ($result === []) { + throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); + } + [$peaks, $past] = self::evidence($evidence, $now, (int) ($manager['NetzbezugEnergieVariableID'] ?? 0), $warnings); + $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, + 'meteringBoundary' => 'common_pcc', 'batteries' => $result, + 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], + 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'operation' => $payload]; + } +} diff --git a/tests/NetzfahrplanV4BetriebsdatenTest.php b/tests/NetzfahrplanV4BetriebsdatenTest.php new file mode 100644 index 0000000..64f90a0 --- /dev/null +++ b/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -0,0 +1,29 @@ +regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) { + self::assertStringNotContainsString($forbidden, $source); + } + self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source); + self::assertStringContainsString('/prognosis/planner-v4/operation', $source); + self::assertStringNotContainsString('/prognosis/schedule', $source); + } +} diff --git a/tests/fixtures/NativeV4Scenarios.php b/tests/fixtures/NativeV4Scenarios.php new file mode 100644 index 0000000..cc0ebff --- /dev/null +++ b/tests/fixtures/NativeV4Scenarios.php @@ -0,0 +1,124 @@ + 1, 'NetzbezugEnergieVariableID' => 9, + 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, + 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, + 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; + $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, + 'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5, + 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, + 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; + $read = static function (int $id) use ($now): array { + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0][$id], 'updated' => $now]; + }; + $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; + $fail = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid-input rejection'); + }; + $cases = []; + $build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array { + return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now); + }; + $cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void { + $x = $build()['operation']; $b = $x['batteries'][0]; + $check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0 + && $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0); + }; + $cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void { + $x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null); + }; + $cases['empty_maps_are_objects'] = static function () use ($build, $check): void { + $x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW)); + }; + $cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void { + $m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0); + }; + $cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void { + $m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === null && $x['limits']['exportW'] === null); + }; + $cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void { + $m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = []; + for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; } + $x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0); + array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m)); + }; + $cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void { + $a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a)); + $m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]]; + $check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44); + }; + $cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void { + $a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a)); + }; + $cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void { + $c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c)); + }; + $cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void { + $c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c)); + }; + $cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void { + $r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { + $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); + }; + $cases['soc_not_lifted_to_minimum'] = static function () use ($build, $read, $fail): void { + $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { + $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; + $check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0); + }; + $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { + $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); + $check($x['operation']['batteries'][0]['maxDischargeW'] === 0.0 && count($x['warnings']) > 1); + }; + $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { + $check($build()['operation']['batteries'][0]['gridCharging'] === false); + $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); + }; + $ev = ['version' => 1, 'meterId' => 'symcon:9', 'measuredAt' => gmdate('c', $now), + 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], + 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; + $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { + $x = $build(null, null, null, null, $ev); + $check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5); + }; + $cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void { + $ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev); + $check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0); + }; + $cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void { + $ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { + $check($build()['operation']['eventId'] === $build()['operation']['eventId']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} From 12174a59b4ccaf57dcd942d23b629349d097cf5b Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Thu, 1 Oct 2026 16:56:39 +0000 Subject: [PATCH 02/22] fix(manager): isolate V4 active import sources from legacy counter --- docs/testing/Netzfahrplan-V4-Bezugszaehler.md | 66 ++++++++++ docs/testing/Netzfahrplan-V4-Schatten.md | 7 +- libs/ManagerNetzfahrplanV4Trait.php | 10 +- libs/NetzfahrplanV4Betriebsdaten.php | 13 +- libs/NetzfahrplanV4Bezugszaehler.php | 108 +++++++++++++++++ tests/NetzfahrplanV4BetriebsdatenTest.php | 2 +- tests/NetzfahrplanV4BezugszaehlerTest.php | 29 +++++ tests/fixtures/NativeV4Scenarios.php | 19 ++- tests/fixtures/V4BezugszaehlerScenarios.php | 113 ++++++++++++++++++ 9 files changed, 356 insertions(+), 11 deletions(-) create mode 100644 docs/testing/Netzfahrplan-V4-Bezugszaehler.md create mode 100644 libs/NetzfahrplanV4Bezugszaehler.php create mode 100644 tests/NetzfahrplanV4BezugszaehlerTest.php create mode 100644 tests/fixtures/V4BezugszaehlerScenarios.php diff --git a/docs/testing/Netzfahrplan-V4-Bezugszaehler.md b/docs/testing/Netzfahrplan-V4-Bezugszaehler.md new file mode 100644 index 0000000..533aa3a --- /dev/null +++ b/docs/testing/Netzfahrplan-V4-Bezugszaehler.md @@ -0,0 +1,66 @@ +# V4: separate Wirkenergie-Bezugsquelle (Entwicklung, nicht live) + +## Lihrenmoos - Befund und geplante Zuordnung + +User hat die erfolgreiche Aktivierung der T2-Zaehlerarchivierung fuer 26620 +im Archiv 16207 bestaetigt. Der exakte Beginn einer lueckenlosen Messhistorie +ist daraus NICHT nachgewiesen; keine historische Null setzen. + +Die vorbereitete separate V4-Konfiguration fuer diese Anlage lautet: + +```json +[ + {"VariableID":59607,"ElternID":11490,"Ident":"Energy_0","FaktorZuKWh":1.0,"Messgroesse":"WirkenergieBezug"}, + {"VariableID":26620,"ElternID":11490,"Ident":"Energy_1","FaktorZuKWh":1.0,"Messgroesse":"WirkenergieBezug"} +] +``` + +Property: NetzfahrplanV4BezugszaehlerQuellen. Diese Datei setzt KEINE Property. +IDs duerfen nicht fuer andere Anlagen kopiert werden. Zuordnung ist durch +Variablenstruktur, Standardtelegramm und Leistungs-/Energiedifferenz stark +plausibilisiert; Abgleich mit Rohtelegramm/Zaehleranzeige steht noch aus. + +53476, dessen gespeicherte Historie, die alte Manager-Property +NetzbezugEnergieVariableID und alle vorhandenen Spiegelereignisse bleiben +unveraendert. V4 liest seine eigenen Quellen, erzeugt keine Spruenge in alten +Summenzaehlern und kann keine Blindenergie durch einen Faktor korrigieren. + +## Vertrag + +Jede Quelle hat eine ausdrueckliche Messgroesse und Umrechnung nach kWh. +Variable, Elterninstanz und Ident werden bei jedem Lesen geprueft. Ein +fehlender, veralteter, ungueltiger oder zeitlich unpassender Teilwert verwirft +die Summe. Ein aktueller echter Nullwert ist gueltig. Zwei aufeinanderfolgende +Lesevorgaenge reduzieren das Risiko einer gemischten Momentaufnahme; sie sind +kein atomarer Zugriff auf das physische Zaehlertelegramm. + +Die Summe wird unter bezugszaehler in der lokalen V4-Diagnose ausgegeben, mit +den Beobachtungszeitpunkten jedes Teilregisters. VariableUpdated ist ein +Symcon-Aktualisierungszeitpunkt, kein Beweis einer exakten Abrechnungsgrenze. +Die Diagnose hat deshalb billingEvidence=false und historyComplete=false. +Eine falsche Messgroessen-Deklaration kann ohne Rohtelegramm nicht automatisch +erkannt werden; Installationspruefung bleibt erforderlich. + +Ein aus Quell-IDs, Eltern-ID, Idents und Umrechnungsfaktoren gebildeter +stabiler SHA256-Schluessel bindet die spaeteren Messnachweise an genau diese +Quelle. Die Reihenfolge von T1/T2 aendert den Schluessel nicht. Quell-/Faktor- +wechsel dagegen invalidieren alte Nachweise. symcon:53476 wird nicht mehr +als V4-Messnachweis akzeptiert. Der Schluessel beweist Zuordnung, nicht den +Wahrheitsgehalt eines gelieferten Monatspeaks. + +## Noch offen + +Der Erzeuger fuer exakte Viertelstundenenergie und den vollstaendigen +Monatspeak ist weiterhin nicht implementiert. Keine Ableitung eines bereits +bezahlten Monatspeaks aus einer Managergrenze oder unvollstaendiger Historie. +Auch 15 Minuten neuer Aufzeichnung beweisen nicht den bisherigen Monatspeak. +Die vorbereitete native Summe ist noch nicht im laufenden Symcon installiert. +V4 bleibt ohne belastbare Nachweise in awaiting_inputs. + +## Nachweis dieses Schritts + +44 synthetische Offline-Szenarien mit PHP CLI 8.4.23 erfolgreich: +20 Quellen-/Summenpruefungen, 24 native Betriebsdatenkonvertierungen. +Die vom Server exportierten Betriebsdaten-, Trait- und Fixture-Dateien wurden +vor dem Test per SHA256 verglichen. Keine Symcon-Laufzeitpruefung und keine +neue Zielcontainer-/vollstaendige PHPUnit-Pruefung in diesem Schritt. diff --git a/docs/testing/Netzfahrplan-V4-Schatten.md b/docs/testing/Netzfahrplan-V4-Schatten.md index 92e33c8..2770226 100644 --- a/docs/testing/Netzfahrplan-V4-Schatten.md +++ b/docs/testing/Netzfahrplan-V4-Schatten.md @@ -13,6 +13,9 @@ geaendert. Der Sender ist nach Installation standardmaessig AUS. - NetzfahrplanV4BatterieOptionen: JSON-Objekt je Topologie-ID. Optional SOCKapazitaet_kWh und MaxSOC_Prozent (sonst 100). Bei unterschiedlicher Nenn- und Nutzkapazitaet ist SOCKapazitaet_kWh zwingend explizit anzugeben. +- NetzfahrplanV4BezugszaehlerQuellen: JSON-Liste, Standard []. Eigene, explizite + Wirkenergie-Bezugsquellen fuer V4; KEIN Rueckfall auf den Altzaehler. + Details und Lihrenmoos-Kandidat: Netzfahrplan-V4-Bezugszaehler.md. - NetzfahrplanV4MessnachweisVariableID: JSON-Stringvariable, Standard 0. Ein vollstaendiger historischer/registerbasierter Messnachweis-Produzent ist NOCH NICHT integriert. Fehlende Nachweise bleiben fehlend. @@ -51,7 +54,7 @@ Nicht ungeprueft deployen: Host-CLI- und Zielsystemtests getrennt nachweisen. ```json { "version": 1, - "meterId": "symcon:53476", + "meterId": "symcon-active-import:", "measuredAt": "2026-10-01T12:05:00Z", "measuredPeaks": { "2026-10": {"kw": 18.4, "source": "meter_month_register"} @@ -79,5 +82,5 @@ muessen separat aktiviert werden. Keine neue Modellwahl oder Aktorfreigabe hier. ## Tests -21 offline Konvertierungsszenarien plus PHPUnit-Strukturpruefungen. Werte sind +24 offline Konvertierungsszenarien und 20 Quellen-/Summen-Szenarien sowie PHPUnit-Strukturpruefungen. Werte sind synthetisch; keine reale HTTP-Anbindung oder Symcon-Laufzeit damit behauptet. diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php index b501330..bc187a3 100644 --- a/libs/ManagerNetzfahrplanV4Trait.php +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -16,6 +16,7 @@ trait ManagerNetzfahrplanV4Trait $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]'); $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); $this->RegisterTimer('NetzfahrplanV4Senden', 0, "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); @@ -26,7 +27,7 @@ trait ManagerNetzfahrplanV4Trait try { $manager = []; foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', - 'Lastspitzenmodus', 'NetzbezugEnergieVariableID'] as $key) { + 'Lastspitzenmodus'] as $key) { $manager[$key] = $this->ReadPropertyInteger($key); } foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { @@ -36,6 +37,9 @@ trait ManagerNetzfahrplanV4Trait $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $manager['V4BezugszaehlerQuellen'] = json_decode( + $this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR + ); $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); $cache = $this->leseJsonAttribut('VerbraucherCache'); $controllers = []; @@ -80,7 +84,9 @@ trait ManagerNetzfahrplanV4Trait if ($before['VariableUpdated'] !== $after['VariableUpdated']) { throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); } - return ['value' => $value, 'updated' => (int) $after['VariableUpdated']]; + $object = IPS_GetObject($id); + return ['value' => $value, 'updated' => (int) $after['VariableUpdated'], + 'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']]; }; return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); diff --git a/libs/NetzfahrplanV4Betriebsdaten.php b/libs/NetzfahrplanV4Betriebsdaten.php index b3bf27e..1502536 100644 --- a/libs/NetzfahrplanV4Betriebsdaten.php +++ b/libs/NetzfahrplanV4Betriebsdaten.php @@ -8,6 +8,8 @@ use DateTimeImmutable; use DateTimeZone; use InvalidArgumentException; +require_once __DIR__ . '/NetzfahrplanV4Bezugszaehler.php'; + /** Read-only conversion. Meter evidence is never derived from a configured limit. */ final class NetzfahrplanV4Betriebsdaten { @@ -43,7 +45,7 @@ final class NetzfahrplanV4Betriebsdaten } /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ - private static function evidence(array $input, int $now, int $meterID, array &$warnings): array + private static function evidence(array $input, int $now, string $meterID, array &$warnings): array { $peaks = []; $past = null; @@ -51,7 +53,7 @@ final class NetzfahrplanV4Betriebsdaten $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; return [(object) [], null]; } - if ($meterID <= 0 || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== 'symcon:' . $meterID) { + if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) { throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); } $measured = self::timestamp($input['measuredAt'] ?? null); @@ -92,6 +94,9 @@ final class NetzfahrplanV4Betriebsdaten public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array { $warnings = []; + $meter = NetzfahrplanV4Bezugszaehler::lesen( + $manager['V4BezugszaehlerQuellen'] ?? [], $read, $now + ); $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); @@ -180,12 +185,12 @@ final class NetzfahrplanV4Betriebsdaten if ($result === []) { throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); } - [$peaks, $past] = self::evidence($evidence, $now, (int) ($manager['NetzbezugEnergieVariableID'] ?? 0), $warnings); + [$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings); $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, 'meteringBoundary' => 'common_pcc', 'batteries' => $result, 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], 'measuredPeaks' => $peaks, 'quarterPast' => $past]; $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); - return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'operation' => $payload]; + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; } } diff --git a/libs/NetzfahrplanV4Bezugszaehler.php b/libs/NetzfahrplanV4Bezugszaehler.php new file mode 100644 index 0000000..dda9b72 --- /dev/null +++ b/libs/NetzfahrplanV4Bezugszaehler.php @@ -0,0 +1,108 @@ + 8 + || array_keys($sources) !== range(0, count($sources) - 1)) { + throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.'); + } + $result = []; + $parent = null; + foreach ($sources as $source) { + if (!is_array($source) || count($source) !== 5 + || ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') { + throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.'); + } + $id = $source['VariableID'] ?? null; + $pid = $source['ElternID'] ?? null; + $ident = $source['Ident'] ?? null; + $factor = $source['FaktorZuKWh'] ?? null; + if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0 + || !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) + || $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) { + throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.'); + } + if ($parent !== null && $parent !== $pid) { + throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.'); + } + $parent = $pid; + foreach ($result as $existing) { + if ($existing['Ident'] === $ident) { + throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.'); + } + } + $result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident, + 'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug']; + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + public static function identitaet(array $sources): string + { + // Source/factor changes invalidate old measurement evidence; names do not matter. + return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources), + JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + } + + private static function probe(array $source, callable $read, int $now, int $maxAge): array + { + $m = $read($source['VariableID']); + if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID'] + || ($m['ident'] ?? null) !== $source['Ident']) { + throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.'); + } + $value = $m['value'] ?? null; + $updated = $m['updated'] ?? null; + if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0 + || !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) { + throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.'); + } + $kwh = (float) $value * $source['FaktorZuKWh']; + if (!is_finite($kwh) || $kwh > 1.0e12) { + throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.'); + } + return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value, + 'totalKwh' => $kwh, 'updatedAt' => $updated]; + } + + /** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */ + public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array + { + if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) { + throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.'); + } + $sources = self::quellen($sources); + $samples = []; + foreach ($sources as $source) { + $samples[] = self::probe($source, $read, $now, $maxAge); + } + // Reread all channels to reject concurrent updates, including changes in the same second. + foreach ($sources as $i => $source) { + if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) { + throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.'); + } + } + $times = array_column($samples, 'updatedAt'); + if (max($times) - min($times) > $maxSkew) { + throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.'); + } + return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh', + 'totalKwh' => array_sum(array_column($samples, 'totalKwh')), + 'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)), + 'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples, + 'billingEvidence' => false, 'historyComplete' => false]; + } +} diff --git a/tests/NetzfahrplanV4BetriebsdatenTest.php b/tests/NetzfahrplanV4BetriebsdatenTest.php index 64f90a0..adb80c8 100644 --- a/tests/NetzfahrplanV4BetriebsdatenTest.php +++ b/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -13,7 +13,7 @@ final class NetzfahrplanV4BetriebsdatenTest extends TestCase { public function testOfflineInputScenarios(): void { - self::assertCount(21, NativeV4Scenarios::run()); + self::assertCount(24, NativeV4Scenarios::run()); } public function testSenderCannotChangeLivePlanOrActuators(): void diff --git a/tests/NetzfahrplanV4BezugszaehlerTest.php b/tests/NetzfahrplanV4BezugszaehlerTest.php new file mode 100644 index 0000000..41a2992 --- /dev/null +++ b/tests/NetzfahrplanV4BezugszaehlerTest.php @@ -0,0 +1,29 @@ + 1, 'NetzbezugEnergieVariableID' => 9, 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $m['V4BezugszaehlerQuellen'] = [ + ['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, @@ -24,7 +28,8 @@ final class NativeV4Scenarios 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; $read = static function (int $id) use ($now): array { - return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0][$id], 'updated' => $now]; + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id], + 'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other']; }; $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; $fail = static function (callable $fn): void { @@ -97,7 +102,7 @@ final class NativeV4Scenarios $check($build()['operation']['batteries'][0]['gridCharging'] === false); $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); }; - $ev = ['version' => 1, 'meterId' => 'symcon:9', 'measuredAt' => gmdate('c', $now), + $ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now), 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { @@ -117,6 +122,16 @@ final class NativeV4Scenarios $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { $check($build()['operation']['eventId'] === $build()['operation']['eventId']); }; + $cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void { + unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m)); + }; + $cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void { + $v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203 + && !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []); + }; $passed = []; foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } return $passed; diff --git a/tests/fixtures/V4BezugszaehlerScenarios.php b/tests/fixtures/V4BezugszaehlerScenarios.php new file mode 100644 index 0000000..7e08052 --- /dev/null +++ b/tests/fixtures/V4BezugszaehlerScenarios.php @@ -0,0 +1,113 @@ + 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $samples = [ + 59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'], + 26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'], + ]; + $check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } }; + $reject = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid source/sample rejection'); + }; + $snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array { + $mm = $mm ?? $samples; + return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now); + }; + $cases = []; + $cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void { + $v = $snapshot(); + $check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']); + $check(!isset($v['measuredPeaks']) && !isset($v['quarterPast'])); + }; + $cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void { + $samples[26620]['value'] = 27.25; + $check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9); + }; + $cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void { + $samples[59607]['value'] = 0; + $check($snapshot(null, $samples)['totalKwh'] === 0.0); + }; + $cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void { + $check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources))); + }; + $cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001; + $check($old !== Meter::identitaet($sources)); + }; + $cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345; + $check($old !== Meter::identitaet($sources)); + }; + $cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); }; + $cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources)); + }; + $cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources)); + }; + $cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources)); + }; + $cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void { + unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources)); + }; + $cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void { + unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void { + foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) { + $samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples)); + } + }; + $cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void { + $calls = 0; + $read = static function (int $id) use ($samples, &$calls): array { + $v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v; + }; + $reject(static fn() => Meter::lesen($sources, $read, $now)); + }; + $cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void { + $samples[59607]['updated'] -= 1; + $v = $snapshot(null, $samples); + $check($v['sourceObservationFrom'] === gmdate('c', $now - 1)); + $check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now)); + $check(!$v['billingEvidence']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} From fc4c4632cec0b119a3f260c765b1520dc9f2833f Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Thu, 1 Oct 2026 18:05:59 +0000 Subject: [PATCH 03/22] feat(manager): transmit estimated meter samples and SOC recovery state --- docs/testing/Netzfahrplan-V4-Schatten.md | 22 +++++++++++----- libs/ManagerNetzfahrplanV4Trait.php | 3 ++- libs/NetzfahrplanV4Betriebsdaten.php | 33 ++++++++++++++++++++---- tests/fixtures/NativeV4Scenarios.php | 7 ++--- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/testing/Netzfahrplan-V4-Schatten.md b/docs/testing/Netzfahrplan-V4-Schatten.md index 2770226..19a6a4c 100644 --- a/docs/testing/Netzfahrplan-V4-Schatten.md +++ b/docs/testing/Netzfahrplan-V4-Schatten.md @@ -27,12 +27,22 @@ hinzugefuegt. SDL-/Grundlastbereinigung bleibt eine separate, noch offene Aufgab BMS-Grenzen werden mit Topologieleistungen begrenzt. Normale Betriebsreserve und technisches Minimum werden nicht verwechselt: die hoehere Grenze gilt. -Bei aktueller Entladehysterese wird die Entladeleistung konservativ auf 0 -begrenzt. Eine spaetere Freigabe innerhalb der 48h wird noch nicht modelliert. -Das ist sichtbar diagnostiziert und kann den Schattenplan wirtschaftlich -beschraenken; eine zeitabhaengige Hysterese-Modellierung bleibt offen. -SOC unterhalb der Reserve wird nicht kuenstlich angehoben: Diagnosefehler, -kein angeblich gueltiger Plan. Recovery-/Nachladungsmodell bleibt offen. +Der erweiterte Sender uebermittelt Entladesperre und Wiederfreigabe-SOC separat +von der maximalen Entladeleistung. Der neue Python-Rechenkern darf die Sperre +im Zukunftsplan erst nach vorherigem Laden bis zur Wiederfreigabe aufheben. +SOC unterhalb der Betriebsreserve, aber oberhalb des technischen Minimums, +bleibt unveraendert erhalten: Es ist Wiederaufladung statt erfundener Energie +vorgesehen. Unterhalb des technischen Minimums wird kein normaler Plan erzeugt. + +Zusaetzlich wird meterObservation mit frischen Empfangswerten der Netzleistung +und T1/T2-Summe uebermittelt. Dessen laufende Viertelstundenintegration ist eine +explizite Planungsschaetzung, KEIN verifizierter Abrechnungsnachweis. Verwendung +von Schaetzungen muss im V4-Dienst ausdruecklich aktiviert werden. Historische +Peakannahmen werden getrennt von verifizierten Maxima gespeichert. + +Diese Protokollerweiterung benoetigt zuerst den passenden Backendstand. Der +bisherige laufende V4-Container und die Testanlage sind noch nicht aktualisiert. +Offline-Szenarien und Quelltextpruefungen ersetzen keinen Symcon-Laufzeittest. ## Diagnose ohne Netzwerk oder Stellbefehl diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php index bc187a3..c6e7412 100644 --- a/libs/ManagerNetzfahrplanV4Trait.php +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -33,6 +33,7 @@ trait ManagerNetzfahrplanV4Trait foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { $manager[$key] = $this->ReadPropertyFloat($key); } + $manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified'; $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); @@ -50,7 +51,7 @@ trait ManagerNetzfahrplanV4Trait } $c = ['InstanzID' => $id]; foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', - 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { $c[$key] = IPS_GetProperty($id, $key); } $entry = $cache[(string) $id] ?? []; diff --git a/libs/NetzfahrplanV4Betriebsdaten.php b/libs/NetzfahrplanV4Betriebsdaten.php index 1502536..35495da 100644 --- a/libs/NetzfahrplanV4Betriebsdaten.php +++ b/libs/NetzfahrplanV4Betriebsdaten.php @@ -161,7 +161,8 @@ final class NetzfahrplanV4Betriebsdaten self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); $socValue = self::number($soc['value'], 'SOC', 0, 100); - if ($socValue < $min || $socValue > $max) { + $physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min); + if ($socValue < $physicalMin || $socValue > $max) { throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); } if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { @@ -173,14 +174,20 @@ final class NetzfahrplanV4Betriebsdaten $maxCharge = $maxDischarge = 0.0; $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; } - if (!empty($c['HystereseAktiv'])) { - $maxDischarge = 0.0; - $warnings[] = $id . ': aktuelle Entladesperre konservativ uebernommen; keine spaetere Freigabe erfunden.'; + $rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100)); + $blocked = !empty($c['HystereseAktiv']) || $socValue <= $min; + if ($blocked) { + $warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.'; + } + if ($socValue < $min) { + $warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.'; } $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), - 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true]; + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true, + 'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true, + 'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm]; } if ($result === []) { throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); @@ -190,6 +197,22 @@ final class NetzfahrplanV4Betriebsdaten 'meteringBoundary' => 'common_pcc', 'batteries' => $result, 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + // A fresh acquisition sample is an operational estimate. It does NOT + // claim a calibrated billing-period boundary or full historical coverage. + $policy = [ + 'adapterVersion' => 'v4-native-estimated-meter-1', + 'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified', + 'mode' => $mode, 'importW' => $import, 'exportW' => $export, + 'months' => $months, + 'batteryPolicy' => array_map(static function (array $b): array { + return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent'])); + }, $result), + ]; + $payload['meterObservation'] = [ + 'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now), + 'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'], + 'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)), + ]; $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; } diff --git a/tests/fixtures/NativeV4Scenarios.php b/tests/fixtures/NativeV4Scenarios.php index e85d817..0fa58a0 100644 --- a/tests/fixtures/NativeV4Scenarios.php +++ b/tests/fixtures/NativeV4Scenarios.php @@ -86,9 +86,10 @@ final class NativeV4Scenarios $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); }; - $cases['soc_not_lifted_to_minimum'] = static function () use ($build, $read, $fail): void { + $cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void { $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; - $fail(static fn() => $build(null, null, null, $r)); + $b = $build(null, null, null, $r)['operation']['batteries'][0]; + $check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true); }; $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; @@ -96,7 +97,7 @@ final class NativeV4Scenarios }; $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); - $check($x['operation']['batteries'][0]['maxDischargeW'] === 0.0 && count($x['warnings']) > 1); + $check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1); }; $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { $check($build()['operation']['batteries'][0]['gridCharging'] === false); From b5c1fa6be2a5dd3dad66b35f6ddaddbcfcb6055f Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 13:01:04 +0000 Subject: [PATCH 04/22] feat(manager): receive and preview V4 shadow plans without dispatch --- Manager/module.php | 11 + docs/netplan-v4-receiver.md | 48 ++++ libs/ManagerNetzfahrplanV4EmpfangTrait.php | 223 +++++++++++++++ libs/ManagerNetzfahrplanV4Trait.php | 6 + libs/NetzfahrplanV4Planpruefung.php | 310 +++++++++++++++++++++ tests/NetzfahrplanV4EmpfangTest.php | 29 ++ tests/V4Receiver/fixture.php | 30 ++ tests/V4Receiver/protocol_checks.php | 73 +++++ tests/V4Receiver/receiver_checks.php | 117 ++++++++ 9 files changed, 847 insertions(+) create mode 100644 docs/netplan-v4-receiver.md create mode 100644 libs/ManagerNetzfahrplanV4EmpfangTrait.php create mode 100644 libs/NetzfahrplanV4Planpruefung.php create mode 100644 tests/NetzfahrplanV4EmpfangTest.php create mode 100644 tests/V4Receiver/fixture.php create mode 100644 tests/V4Receiver/protocol_checks.php create mode 100644 tests/V4Receiver/receiver_checks.php diff --git a/Manager/module.php b/Manager/module.php index ac74605..f4b2158 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -237,6 +237,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle { parent::ApplyChanges(); $this->SetTimerInterval('NetzfahrplanV4Senden', 0); + $this->stoppeNetzfahrplanV4Vorschau(); $this->initialisiereLizenzInstallationID(); $this->aktualisiereVariablen(); @@ -295,6 +296,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle if ($this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { $this->SetTimerInterval('NetzfahrplanV4Senden', 60000); } + $this->konfiguriereNetzfahrplanV4Vorschau(); $this->synchronisiereAnlagentopologie(); if ($this->prognoseTelemetrieKonfiguriert()) { $this->SetTimerInterval( @@ -424,6 +426,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->monatsgrenzenFuerFormular() ); + $formular['elements'][] = $this->netzfahrplanV4EmpfangFormular(); return json_encode($formular, JSON_THROW_ON_ERROR); } @@ -457,6 +460,14 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->regeln((bool) $wert); return; + case 'NetzfahrplanV4Empfangen': + $this->empfangeNetzfahrplanV4(); + return; + + case 'NetzfahrplanV4VorschauAktualisieren': + $this->aktualisiereNetzfahrplanV4Vorschau(); + return; + case 'NetzfahrplanV4Senden': $this->sendeNetzfahrplanV4(); return; diff --git a/docs/netplan-v4-receiver.md b/docs/netplan-v4-receiver.md new file mode 100644 index 0000000..387e383 --- /dev/null +++ b/docs/netplan-v4-receiver.md @@ -0,0 +1,48 @@ +# V4 plan reception and local preview (observation only) + +This module does not execute a battery command. The existing regulator and its +setpoints remain unchanged. `NetzfahrplanAktiv` must remain false. + +The optional `NetzfahrplanV4EmpfangAktiv` property defaults to false. When enabled +with the existing shadow sender, one bounded GET is made before the next native +operation POST, avoiding reading the plan immediately after queuing its replan. +A successful inspection acknowledges `shadow_seen` once per plan/step. This is +not `applied`. An HTTP failure clears the local preview; it does not change the +old control mode or its connection credentials. + +The V4 service must first support `receiverProtocolVersion=1`, installation IDs +in envelope and plan, per-input event references, and the stable controlContext +used at calculation time. Old or cross-plant responses are deliberately refused. +The client checks fresh/lastRun/pending status, exact configuration revision, +model-family registry, known price coverage, contiguous half-open five-minute +intervals, asset mapping and power balance. Numerical feasibility checks are +not evidence of economic performance or permission to control devices. + +A cache watchdog refreshes the diagnostic display every 10 seconds, without any +network requests. Cached response older than 90 seconds, plan older than 900 +seconds, an expired step, invalid local measurement or policy change invalidates +the preview. Missing targets are JSON null and '-' in the display, never zero. +Restart/ApplyChanges clears the cache. Existing control functions are not edited. + +The preview is currently limited to ONE battery. Multiple batteries are still +accepted by the server planner; local allocation is explicitly unsupported in +this stage. Preview feedback uses actual grid and battery power, local capacity, +available power, minimum/maximum SOC, discharge hysteresis, network-charging +permission and import/export/month limits. It exposes any remaining grid-limit +violation instead of pretending the battery can resolve it. It is a momentary +calculation only, NOT the commissioned real-time regulator. PV curtailment and +external SDL/base-load separation must still be validated before any live test. + +Two read-only variables below the manager expose a HTML summary and detailed +JSON. No EnableAction, consumer RequestAction, SendData or actor property writes +are used. The existing form.json and web portal assets are not modified. + +Offline checks: + + php tests/V4Receiver/protocol_checks.php + php tests/V4Receiver/receiver_checks.php + +The PHPUnit wrapper runs these in separate subprocesses, so fake IPS and curl +functions cannot contaminate other module tests. The mocked transport uses no +real credentials or connections. Full Symcon kernel testing and installation +are separate commissioning steps; these unit tests are not field acceptance. diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php new file mode 100644 index 0000000..569a7ad --- /dev/null +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -0,0 +1,223 @@ +RegisterPropertyBoolean('NetzfahrplanV4EmpfangAktiv', false); + $this->RegisterAttributeString('NetzfahrplanV4Empfang', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4EmpfangStatus', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4LetzteBestaetigung', '{}'); + $this->RegisterAttributeInteger('NetzfahrplanV4NaechsterAbruf', 0); + $this->RegisterVariableString('NetzfahrplanV4Vorschau', 'V4 Sollwert-Vorschau (KEINE Stellfreigabe)', '~HTMLBox', 950); + $this->RegisterVariableString('NetzfahrplanV4VorschauJSON', 'V4 Vorschau Diagnose (nur lesen)', '', 951); + $this->RegisterTimer('NetzfahrplanV4VorschauAktualisieren', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4VorschauAktualisieren', true);"); + } + + private function stoppeNetzfahrplanV4Vorschau(): void + { + $this->SetTimerInterval('NetzfahrplanV4VorschauAktualisieren', 0); + $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); + $this->WriteAttributeString('NetzfahrplanV4LetzteBestaetigung', '{}'); + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', '{"status":"not_received"}'); + $this->zeigeNetzfahrplanV4Vorschau(['status' => 'not_received', 'reason' => 'Neu initialisiert; keinen alten Sollwert weiterverwenden.']); + } + + private function konfiguriereNetzfahrplanV4Vorschau(): void + { + $enabled = $this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') + && $this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') + && !$this->ReadPropertyBoolean('NetzfahrplanAktiv'); + $this->SetTimerInterval('NetzfahrplanV4VorschauAktualisieren', $enabled ? 10000 : 0); + $this->aktualisiereNetzfahrplanV4Vorschau(); + } + + /** Appended dynamically; existing form.json and parallel portal GUI stay untouched. */ + private function netzfahrplanV4EmpfangFormular(): array + { + return ['type' => 'ExpansionPanel', 'caption' => 'V4 Planempfang / Sollwert-Vorschau', 'items' => [ + ['type' => 'Label', 'caption' => 'NUR BEOBACHTEN. Kein Lade-/Entladebefehl. Der alte intelligente Netzfahrplan muss AUS bleiben.'], + ['type' => 'CheckBox', 'name' => 'NetzfahrplanV4EmpfangAktiv', 'caption' => 'V4 Schattenplan empfangen und Sollwert anzeigen (keine Steuerung)'], + ['type' => 'Label', 'caption' => 'Diagnose und Sollwerte stehen in den beiden V4-Vorschauvariablen unterhalb des Managers.'], + ['type' => 'Button', 'caption' => 'Plan jetzt pruefen (Abruf begrenzt auf 1/min)', + 'onClick' => "IPS_RequestAction(\$id, 'NetzfahrplanV4Empfangen', true);"], + ]]; + } + + /** Called before the next operation upload, so the preceding completed plan can be seen. */ + private function empfangeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv')) return; + $lock = 'ENELIX.V4.Receive.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { + $now = time(); + if ($now < $this->ReadAttributeInteger('NetzfahrplanV4NaechsterAbruf')) return; + $this->WriteAttributeInteger('NetzfahrplanV4NaechsterAbruf', $now + 60); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') || $this->ReadPropertyBoolean('NetzfahrplanAktiv')) { + throw new RuntimeException('Schatten-Sender fehlt oder alter Fahrplan ist eingeschaltet.'); + } + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $response = $this->netzfahrplanV4EmpfangHttp('GET', ''); + $receipt = time(); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') || $this->ReadPropertyBoolean('NetzfahrplanAktiv')) { + throw new RuntimeException('Freigabe waehrend des Abrufs geaendert.'); + } + // Clear the old envelope before validation: invalid replies never extend its lifetime. + $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); + [$operation, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $checked = NetzfahrplanV4Planpruefung::pruefen($response, + $this->ReadAttributeString('LizenzInstallationID'), $operation, time(), $receipt); + $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); + // Only the known, non-secret protocol envelope is cached. + $keys = ['receiverProtocolVersion', 'installationId', 'checkedAt', 'liveEnabled', 'fresh', + 'pending', 'settings', 'plan', 'families', 'lastRun']; + $safe = array_intersect_key($response, array_flip($keys)); + $this->WriteAttributeString('NetzfahrplanV4Empfang', json_encode( + ['receivedAt' => $receipt, 'response' => $safe], JSON_THROW_ON_ERROR)); + $state = ['status' => 'received_shadow', 'at' => gmdate('c', $receipt), 'acknowledged' => false]; + $ack = ['planId' => $preview['planId'], 'revision' => $preview['revision'], 'step' => $preview['step'], 'status' => 'shadow_seen']; + $last = json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteBestaetigung'), true); + if ($last !== $ack) { + try { + $answer = $this->netzfahrplanV4EmpfangHttp('POST', '/ack', $ack); + if (($answer['status'] ?? null) !== 'recorded') throw new RuntimeException('Empfangsbestaetigung abgelehnt.'); + $this->WriteAttributeString('NetzfahrplanV4LetzteBestaetigung', json_encode($ack, JSON_THROW_ON_ERROR)); + $state['acknowledged'] = true; + } catch (Throwable $error) { + $state['acknowledgementError'] = substr($error->getMessage(), 0, 200); + } + } else { + $state['acknowledged'] = true; + } + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', json_encode($state, JSON_THROW_ON_ERROR)); + $this->aktualisiereNetzfahrplanV4Vorschau(); + } catch (Throwable $error) { + $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); + $state = ['status' => 'rejected_or_unavailable', 'at' => gmdate('c'), 'reason' => substr($error->getMessage(), 0, 300)]; + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', json_encode($state, JSON_THROW_ON_ERROR)); + $this->zeigeNetzfahrplanV4Vorschau($state); + } finally { + IPS_SemaphoreLeave($lock); + } + } + + /** Fixed authenticated read/ack endpoints only; never operation, settings, replan or dispatch. */ + private function netzfahrplanV4EmpfangHttp(string $method, string $suffix, ?array $body = null): array + { + if (!(($method === 'GET' && $suffix === '' && $body === null) + || ($method === 'POST' && $suffix === '/ack' && ($body['status'] ?? null) === 'shadow_seen'))) { + throw new RuntimeException('Unzulaessiger V4-Empfangsaufruf.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $id) + || !preg_match('/^[A-Za-z0-9_-]{43,128}$/D', $token)) { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $handle = curl_init('https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4' . $suffix); + if ($handle === false) throw new RuntimeException('V4-Empfang konnte nicht vorbereitet werden.'); + $response = ''; + try { + $options = [CURLOPT_CUSTOMREQUEST => $method, CURLOPT_FOLLOWLOCATION => false, + CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, CURLOPT_HTTPHEADER => ['Authorization: Bearer ' . $token, + 'Accept: application/json', 'Content-Type: application/json', 'Cache-Control: no-cache'], + CURLOPT_WRITEFUNCTION => static function ($curl, string $chunk) use (&$response): int { + if (strlen($response) + strlen($chunk) > NetzfahrplanV4Planpruefung::MAX_RESPONSE_BYTES) return 0; + $response .= $chunk;return strlen($chunk); + }]; + if ($body !== null) $options[CURLOPT_POSTFIELDS] = json_encode($body, JSON_THROW_ON_ERROR); + curl_setopt_array($handle, $options); + $ok = curl_exec($handle);$code = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($ok === false || $code !== 200) throw new RuntimeException('V4-Planempfang HTTP ' . $code . '; kein gueltiger Vorschauwert.'); + return NetzfahrplanV4Planpruefung::dekodieren($response); + } finally { + curl_close($handle); + } + } + + /** Reads independently of the old control loop. Never manufactures fresh timestamps. */ + private function netzfahrplanV4LokaleVorschauwerte(): array + { + $d = json_decode($this->GetNetzfahrplanV4Diagnose(), true, 64, JSON_THROW_ON_ERROR); + if (!is_array($d['operation'] ?? null)) throw new RuntimeException('Lokale Betriebsdaten ungueltig; Vorschau gesperrt.'); + $topology = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 64, JSON_THROW_ON_ERROR); + $actual = [];$now = time(); + foreach ($topology as $asset) { + $id = (int) ($asset['LeistungVariableID'] ?? 0); + if (!IPS_VariableExists($id)) throw new RuntimeException('Batterie-Istleistung fehlt.'); + $before = IPS_GetVariable($id);$v = GetValue($id);$after = IPS_GetVariable($id); + $factor = $asset['Leistungsfaktor'] ?? null; + if ((!is_int($v) && !is_float($v)) || !is_finite((float) $v) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) || $factor == 0 + || $before['VariableUpdated'] !== $after['VariableUpdated'] + || $after['VariableUpdated'] > $now || $now - $after['VariableUpdated'] > $this->ReadPropertyInteger('MesswertMaxAlter')) { + throw new RuntimeException('Batterie-Istleistung fehlt, ist veraltet oder inkonsistent.'); + } + $actual[$asset['ID']] = (float) $v * $factor; + } + return [$d['operation'], $actual]; + } + + public function GetNetzfahrplanV4Vorschau(): string + { + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') || !$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + throw new RuntimeException('V4-Empfang ausgeschaltet.'); + } + if ($this->ReadPropertyBoolean('NetzfahrplanAktiv')) throw new RuntimeException('Alter Fahrplan ist aktiv; V4-Vorschau gesperrt.'); + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { + throw new RuntimeException('Noch kein gueltiger V4-Plan empfangen.'); + } + [$operation, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $checked = NetzfahrplanV4Planpruefung::pruefen($envelope['response'], + $this->ReadAttributeString('LizenzInstallationID'), $operation, time(), $envelope['receivedAt']); + $out = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); + $out['reception'] = json_decode($this->ReadAttributeString('NetzfahrplanV4EmpfangStatus'), true); + } catch (Throwable $error) { + $out = ['status' => 'unavailable', 'reason' => substr($error->getMessage(), 0, 300)]; + } + return json_encode(array_merge(['gridTargetW' => null, 'plannedBatteryW' => null, 'previewBatteryW' => null], + $out, ['canDispatch' => false, 'liveEnabled' => false]), JSON_THROW_ON_ERROR); + } + + private function aktualisiereNetzfahrplanV4Vorschau(): void + { + $this->zeigeNetzfahrplanV4Vorschau(json_decode($this->GetNetzfahrplanV4Vorschau(), true, 64, JSON_THROW_ON_ERROR)); + } + + private function zeigeNetzfahrplanV4Vorschau(array $value): void + { + $value = array_merge(['gridTargetW' => null, 'plannedBatteryW' => null, 'previewBatteryW' => null], $value, + ['canDispatch' => false, 'liveEnabled' => false]); + $escape = static fn($v): string => htmlspecialchars((string) $v, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8'); + $html = '

V4: NUR VORSCHAU - KEINE STELLFREIGABE

'; + foreach (['status' => 'Status', 'planId' => 'Plan-ID', 'family' => 'Modellfamilie', 'step' => 'Zeitschritt (UTC)', + 'stepValidUntil' => 'Gueltig bis', 'gridTargetW' => 'Geplantes Netzziel W', 'plannedBatteryW' => 'Geplante Batterie W', + 'previewBatteryW' => 'Lokal berechnete Batterie W (nicht ausgegeben)', 'reason' => 'Ablehnungsgrund'] as $key => $label) { + $item = $value[$key] ?? '-';if (is_float($item)) $item = number_format($item, 1, '.', "'"); + $html .= ''; + } + $html .= '
' . $escape($label) . '' . $escape($item) . '

Batterie + = Laden, - = Entladen. Netz + = Bezug, - = Einspeisung.

'; + foreach ($value['reasons'] ?? [] as $reason) $html .= '

' . $escape($reason) . '

'; + $html .= '

Empfangsbestaetigung ist keine Ausfuehrungsbestaetigung.

'; + $this->SetValue('NetzfahrplanV4VorschauJSON', json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + $this->SetValue('NetzfahrplanV4Vorschau', $html); + } +} diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php index c6e7412..1d32ca6 100644 --- a/libs/ManagerNetzfahrplanV4Trait.php +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -7,11 +7,15 @@ namespace Belevo\EnelixEMS; use RuntimeException; use Throwable; +require_once __DIR__ . '/ManagerNetzfahrplanV4EmpfangTrait.php'; + /** Shadow telemetry only. Does not read/apply a V4 schedule or write actuators. */ trait ManagerNetzfahrplanV4Trait { + use ManagerNetzfahrplanV4EmpfangTrait; private function registriereNetzfahrplanV4(): void { + $this->registriereNetzfahrplanV4Empfang(); $this->RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); @@ -107,6 +111,8 @@ trait ManagerNetzfahrplanV4Trait $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); return; } + // Read the preceding completed plan BEFORE sending a new operation/replan trigger. + $this->empfangeNetzfahrplanV4(); try { if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); diff --git a/libs/NetzfahrplanV4Planpruefung.php b/libs/NetzfahrplanV4Planpruefung.php new file mode 100644 index 0000000..3138201 --- /dev/null +++ b/libs/NetzfahrplanV4Planpruefung.php @@ -0,0 +1,310 @@ +getTimestamp(); + } + + private static function zahl($value, string $name, float $low = -1.0e9, float $high = 1.0e9): float + { + if ((!is_float($value) && !is_int($value)) || !is_finite((float) $value) || $value < $low || $value > $high) { + throw new InvalidArgumentException($name . ': endlicher Zahlenwert erforderlich.'); + } + return (float) $value; + } + + private static function uuid($value): string + { + if (!is_string($value) || !preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $value)) { + throw new InvalidArgumentException('Ungueltige UUID.'); + } + return $value; + } + + public static function dekodieren(string $raw): array + { + if (strlen($raw) > self::MAX_RESPONSE_BYTES) { + throw new InvalidArgumentException('V4-Antwort zu gross.'); + } + $result = json_decode($raw, true, 64, JSON_THROW_ON_ERROR); + if (!is_array($result) || self::liste($result)) { + throw new InvalidArgumentException('V4-Antwort ist kein Objekt.'); + } + return $result; + } + + /** Stable local planning policy. Volatile SOC/availability are checked separately. */ + public static function kontext(array $operation): array + { + $assets = []; + foreach ($operation['batteries'] ?? [] as $b) { + if (!is_array($b) || !is_string($b['id'] ?? null) || isset($assets[$b['id']])) { + throw new InvalidArgumentException('Batteriezuordnung ungueltig.'); + } + $assets[$b['id']] = [ + 'capacityKwh' => self::zahl($b['capacityKwh'] ?? null, 'Kapazitaet', .001), + 'minSocPercent' => self::zahl($b['minSocPercent'] ?? null, 'Reserve', 0, 100), + 'maxSocPercent' => self::zahl($b['maxSocPercent'] ?? null, 'Maximal-SOC', 0, 100), + 'physicalMinSocPercent' => self::zahl($b['physicalMinSocPercent'] ?? 0.0, 'Minimum', 0, 100), + 'rearmSocPercent' => self::zahl($b['rearmSocPercent'] ?? $b['minSocPercent'], 'Hysterese', 0, 100), + 'gridCharging' => $b['gridCharging'] ?? null, + ]; + if (!is_bool($assets[$b['id']]['gridCharging'])) { + throw new InvalidArgumentException('Netzladefreigabe fehlt.'); + } + } + if (!$assets || count($assets) > 20 || !is_array($operation['limits'] ?? null)) { + throw new InvalidArgumentException('Batterien oder lokale Grenzen fehlen.'); + } + ksort($assets); + $limits = $operation['limits']; + $expected = ['importW', 'exportW', 'managerMonthLimitsW']; + if (count($limits) !== 3 || array_diff($expected, array_keys($limits)) || !is_array($limits['managerMonthLimitsW'])) { + throw new InvalidArgumentException('Unvollstaendige lokale Grenzen.'); + } + foreach (['importW', 'exportW'] as $key) { + if ($limits[$key] !== null) $limits[$key] = self::zahl($limits[$key], 'Grenze', 0); + } + foreach ($limits['managerMonthLimitsW'] as $month => $limit) { + if ((string) (int) $month !== (string) $month || (int) $month < 1 || (int) $month > 12) { + throw new InvalidArgumentException('Ungueltige Monatsgrenze.'); + } + $limits['managerMonthLimitsW'][$month] = self::zahl($limit, 'Monatsgrenze', 0); + } + ksort($limits['managerMonthLimitsW']); + return ['batteries' => $assets, 'limits' => $limits]; + } + + /** Authenticated endpoint must echo installation identity; old protocol is refused. */ + public static function pruefen(array $response, string $installation, array $operation, int $now, int $receivedAt): array + { + self::uuid($installation); + if (($response['receiverProtocolVersion'] ?? null) !== 1 + || ($response['installationId'] ?? null) !== $installation + || ($response['liveEnabled'] ?? null) !== false) { + throw new InvalidArgumentException('V4-Protokoll/Anlagenzuordnung oder Schattenmodus ungueltig.'); + } + if ($receivedAt > $now || $now - $receivedAt > self::MAX_RECEIPT_AGE) { + throw new InvalidArgumentException('Empfang veraltet; Vorschau verworfen.'); + } + $serverAt = self::zeit($response['checkedAt'] ?? null); + if ($serverAt > $now + 5 || $now - $serverAt > self::MAX_RECEIPT_AGE) { + throw new InvalidArgumentException('Serverantwort nicht aktuell.'); + } + if (($response['fresh'] ?? null) !== true || ($response['pending'] ?? null) !== null) { + throw new InvalidArgumentException('Server hat noch keinen aktuellen freigegebenen Schattenplan.'); + } + $settings = $response['settings'] ?? []; + $p = $response['plan'] ?? []; + if (!is_array($p) || !is_int($settings['revision'] ?? null) || $settings['revision'] < 0 + || ($p['schemaVersion'] ?? null) !== 2 || ($p['installationId'] ?? null) !== $installation + || ($p['configRevision'] ?? null) !== $settings['revision'] + || ($settings['runMode'] ?? null) !== 'shadow' + || ($p['runMode'] ?? null) !== 'shadow' || ($p['liveEnabled'] ?? null) !== false + || ($p['executable'] ?? null) !== true + || !in_array($p['status'] ?? '', ['optimal', 'feasible_time_limit'], true)) { + throw new InvalidArgumentException('Planrevision, Status oder Schattenmodus ungueltig.'); + } + self::uuid($p['planId'] ?? null); + if (($response['lastRun']['detail']['planId'] ?? null) !== $p['planId'] + || !in_array($response['lastRun']['status'] ?? '', ['optimal', 'feasible_time_limit'], true)) { + throw new InvalidArgumentException('Letzte Rechnung passt nicht zum empfangenen Plan.'); + } + $families = array_column($response['families'] ?? [], 'key'); + if (!is_string($p['sourceFamily'] ?? null) || !in_array($p['sourceFamily'], $families, true) + || (($settings['family'] ?? null) !== 'auto' && $settings['family'] !== $p['sourceFamily'])) { + throw new InvalidArgumentException('Modellfamilie unbekannt oder nicht ausgewaehlt.'); + } + $generated = self::zeit($p['generatedAt'] ?? null); + $start = self::zeit($p['validFrom'] ?? null); + $until = self::zeit($p['validUntil'] ?? null); + if ($generated > $now || $now - $generated > self::MAX_PLAN_AGE || $start > $now || $until <= $now + || $start !== $generated || $until - $start > 48 * 3600 + || self::zeit($p['pricesKnownUntil'] ?? null) < $until) { + throw new InvalidArgumentException('Plan abgelaufen, zukuenftig oder ohne bekannten Preishorizont.'); + } + $opAt = self::zeit($operation['observedAt'] ?? null); + if ($opAt > $now || $now - $opAt > 60) { + throw new InvalidArgumentException('Lokale Betriebsdaten veraltet.'); + } + $context = self::kontext($operation); + $pc = $p['controlContext'] ?? null; + if (!is_array($pc) || count($pc) !== 2 || !is_array($pc['batteries'] ?? null) || !is_array($pc['limits'] ?? null)) { + throw new InvalidArgumentException('Planungskontext fehlt.'); + } + $policyAssets = []; + foreach ($pc['batteries'] as $id => $policy) { + if (!is_array($policy) || count($policy) !== 6) { + throw new InvalidArgumentException('Unbekannter Batteriekontext.'); + } + $policyAssets[] = ['id' => $id] + $policy; + } + // Normalize explicitly typed numeric values; strings/bools are not numbers. + $plannedContext = self::kontext(['batteries' => $policyAssets, 'limits' => $pc['limits']]); + if ($plannedContext != $context) { + throw new InvalidArgumentException('Lokale Batterie-/Grenzkonfiguration passt nicht zum Plan.'); + } + if (!is_array($p['inputRefs'] ?? null) || !is_string($p['inputRefs']['operation'] ?? null) + || $p['inputRefs']['operation'] === '') { + throw new InvalidArgumentException('Plan ohne nachvollziehbare Eingangsdaten.'); + } + self::zahl($settings['roundtripEfficiency'] ?? null, 'Wirkungsgrad', .01, 1); + $points = $p['points'] ?? null; + if (!is_array($points) || !self::liste($points) || count($points) < 1 || count($points) > 576) { + throw new InvalidArgumentException('Ungueltige Zeitschrittliste.'); + } + $current = null; + $last = $start; + $ids = array_keys($context['batteries']); + foreach ($points as $point) { + $a = self::zeit($point['time'] ?? null); + $b = self::zeit($point['validUntil'] ?? null); + if ($a !== $last || $b <= $a || $b - $a > 300 || $b % 300 !== 0) { + throw new InvalidArgumentException('Luecke/Ueberlappung oder falsches Fuenfminutenraster.'); + } + $last = $b; + $grid = self::zahl($point['gridTargetW'] ?? null, 'Netzziel'); + $bat = self::zahl($point['batteryTargetW'] ?? null, 'Batterieziel'); + $base = self::zahl($point['baselineGridW'] ?? null, 'Basisnetz'); + $curtail = self::zahl($point['pvCurtailmentW'] ?? null, 'PV-Abregelung', 0); + if (abs($base + $bat + $curtail - $grid) > 1.0) { + throw new InvalidArgumentException('Leistungsbilanz des Plans verletzt.'); + } + if (!in_array($point['intent'] ?? '', ['hold', 'gridCharge', 'pvCharge', 'discharge', 'export'], true)) { + throw new InvalidArgumentException('Unbekannte Planabsicht.'); + } + $targets = $point['assetTargetsW'] ?? null; + $socs = $point['socEndPercent'] ?? null; + if (!is_array($targets) || !is_array($socs)) { + throw new InvalidArgumentException('Batterieziele fehlen.'); + } + $keys = array_keys($targets);sort($keys);$socKeys = array_keys($socs);sort($socKeys); + if ($keys !== $ids || $socKeys !== $ids) { + throw new InvalidArgumentException('Plan enthaelt falsche Batteriezuordnung.'); + } + $sum = 0.0; + foreach ($targets as $id => $w) { + $sum += self::zahl($w, 'Einzelziel'); + self::zahl($socs[$id], 'Plan-SOC', 0, 100); + } + if (abs($sum - $bat) > 1.0) { + throw new InvalidArgumentException('Summe der Batterieziele passt nicht.'); + } + foreach (['importLimitW', 'exportLimitW'] as $limit) { + if (!array_key_exists($limit, $point)) { + throw new InvalidArgumentException('Plangrenze fehlt.'); + } + if ($point[$limit] !== null) { + $v = self::zahl($point[$limit], 'Plangrenze', 0); + if (($limit === 'importLimitW' ? $grid : -$grid) > $v + 1) { + throw new InvalidArgumentException('Netzgrenze im Plan verletzt.'); + } + } + } + if ($a <= $now && $now < $b) { + $current = $point; + } + } + if ($last !== $until || $current === null) { + throw new InvalidArgumentException('Kein derzeit gueltiger Planabschnitt.'); + } + return ['plan' => $p, 'point' => $current, 'efficiency' => (float) $settings['roundtripEfficiency']]; + } + + /** Proposed single-battery correction only. Never use this result as a dispatch permit. */ + public static function vorschau(array $checked, array $operation, array $actualBatteryW, int $now): array + { + $point = $checked['point'];$p = $checked['plan']; + if (count($operation['batteries']) !== 1) { + throw new InvalidArgumentException('Lokale Vorschau fuer mehrere Batterien noch nicht freigegeben.'); + } + $b = $operation['batteries'][0];$id = $b['id']; + $age = $now - self::zeit($b['measuredAt'] ?? null); + if ($age < 0 || $age > 60) { + throw new InvalidArgumentException('Batteriemessung veraltet.'); + } + $grid = self::zahl($operation['gridW'], 'Netzleistung'); + $battery = self::zahl($actualBatteryW[$id] ?? null, 'Batterie-Istleistung'); + $soc = self::zahl($b['socPercent'], 'Aktueller SOC', $b['physicalMinSocPercent'] ?? 0, $b['maxSocPercent']); + $remainingHours = (self::zeit($point['validUntil']) - $now) / 3600; + if ($remainingHours <= 0) { + throw new InvalidArgumentException('Planabschnitt abgelaufen.'); + } + $eta = sqrt($checked['efficiency']);$capacity = self::zahl($b['capacityKwh'], 'Kapazitaet', .001); + $charge = min(self::zahl($b['maxChargeW'], 'Ladeleistung', 0), + max(0., $b['maxSocPercent'] - $soc) * $capacity / 100 / $eta / $remainingHours * 1000); + $discharge = min(self::zahl($b['maxDischargeW'], 'Entladeleistung', 0), + max(0., $soc - $b['minSocPercent']) * $capacity / 100 * $eta / $remainingHours * 1000); + $reasons = []; + if (($b['dischargeBlocked'] ?? false) === true || $soc <= $b['minSocPercent']) { + $discharge = 0.0;$reasons[] = 'Lokale Entladesperre/Reserve'; + } + $residual = $grid - $battery; + if ($b['gridCharging'] !== true) { + $charge = min($charge, max(0., -$residual));$reasons[] = 'Netzladen lokal gesperrt'; + } + $target = (float) $point['gridTargetW'];$limits = $operation['limits']; + $import = $limits['importW'];$export = $limits['exportW']; + $month = (int) (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('n'); + if (isset($limits['managerMonthLimitsW'][(string) $month])) { + $monthly = self::zahl($limits['managerMonthLimitsW'][(string) $month], 'Monatsgrenze', 0); + $import = $import === null ? $monthly : min($import, $monthly); + } + if ($import !== null) $target = min($target, self::zahl($import, 'Bezugsgrenze', 0)); + if ($export !== null) $target = max($target, -self::zahl($export, 'Einspeisegrenze', 0)); + $raw = $battery + $target - $grid; + $wanted = $raw; + if (abs($target) > 1.0) { + $wanted = match ($point['intent']) { + 'gridCharge' => max(0., $wanted), + 'pvCharge' => max(0., min(max(0., -$residual), $wanted)), + 'discharge', 'export' => min(0., $wanted), + 'hold' => 0., + }; + } + // Hard connection limits override economic intent, never BMS/availability. + if ($import !== null) $wanted = min($wanted, $import - $residual); + if ($export !== null) $wanted = max($wanted, -$export - $residual); + $clipped = min($charge, max(-$discharge, $wanted)); + if (abs($clipped - $raw) > 1) $reasons[] = 'Leistung, SOC, Planabsicht oder Netzgrenze begrenzt'; + $expectedGrid = $residual + $clipped; + $violation = max(0., $import === null ? 0. : $expectedGrid - $import, $export === null ? 0. : -$export - $expectedGrid); + if ($violation > 1) $reasons[] = 'Netzgrenze durch Batterie allein nicht erreichbar; lokale Schutzregelung erforderlich'; + return ['status' => 'preview_only', 'liveEnabled' => false, 'canDispatch' => false, + 'installationId' => $p['installationId'], 'planId' => $p['planId'], 'revision' => $p['configRevision'], + 'family' => $p['sourceFamily'], 'checkedAt' => gmdate('c', $now), 'step' => $point['time'], + 'stepValidUntil' => $point['validUntil'], 'gridTargetW' => (float) $point['gridTargetW'], + 'plannedBatteryW' => (float) $point['batteryTargetW'], 'previewBatteryW' => $clipped, + 'actualGridW' => $grid, 'actualBatteryW' => $battery, 'expectedGridW' => $expectedGrid, + 'unmetGridLimitW' => $violation, 'reasons' => $reasons, 'inputWarnings' => $p['warnings'] ?? [], + 'validityNote' => 'Nur Momentvorschau; keine Stellfreigabe und keine nachgewiesene Netzreaktion.']; + } +} diff --git a/tests/NetzfahrplanV4EmpfangTest.php b/tests/NetzfahrplanV4EmpfangTest.php new file mode 100644 index 0000000..fd22d29 --- /dev/null +++ b/tests/NetzfahrplanV4EmpfangTest.php @@ -0,0 +1,29 @@ + ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + self::assertIsResource($process); + fclose($pipes[0]); + $out = stream_get_contents($pipes[1]);$err = stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0, proc_close($process), $out . $err); + self::assertStringContainsString('Total:', $out); + } + + public function scripts(): array + { + return [['protocol_checks.php'], ['receiver_checks.php']]; + } +} diff --git a/tests/V4Receiver/fixture.php b/tests/V4Receiver/fixture.php new file mode 100644 index 0000000..b8afa19 --- /dev/null +++ b/tests/V4Receiver/fixture.php @@ -0,0 +1,30 @@ + 'op-original', 'observedAt' => gmdate('c', $now), 'gridW' => -3000.0, + 'batteries' => [['id' => 'b', 'capacityKwh' => 20.0, 'socPercent' => 50.0, 'minSocPercent' => 15.0, + 'maxSocPercent' => 90.0, 'physicalMinSocPercent' => 3.0, 'rearmSocPercent' => 17.0, + 'gridCharging' => true, 'dischargeBlocked' => false, 'maxChargeW' => 5000.0, 'maxDischargeW' => 4000.0, + 'measuredAt' => gmdate('c', $now)]], + 'limits' => ['importW' => 25000.0, 'exportW' => 25000.0, 'managerMonthLimitsW' => []]]; + $points = []; + for ($i = 0; $i < 6; $i++) $points[] = ['time' => gmdate('c', $now + $i*300), 'validUntil' => gmdate('c', $now + ($i+1)*300), + 'baselineGridW' => -3000.0, 'gridTargetW' => 0.0, 'batteryTargetW' => 3000.0, 'pvCurtailmentW' => 0.0, + 'assetTargetsW' => ['b' => 3000.0], 'socEndPercent' => ['b' => 50.0 + $i], 'intent' => 'pvCharge', + 'importLimitW' => 25000.0, 'exportLimitW' => 25000.0]; + $plan = ['schemaVersion' => 2, 'planId' => '10000000-0000-4000-8000-000000000001', 'installationId' => $id, + 'configRevision' => 1, 'sourceFamily' => '3', 'runMode' => 'shadow', 'liveEnabled' => false, 'executable' => true, + 'status' => 'optimal', 'generatedAt' => gmdate('c', $now), 'validFrom' => gmdate('c', $now), + 'validUntil' => gmdate('c', $now + 1800), 'pricesKnownUntil' => gmdate('c', $now + 1800), + 'controlContext' => \Belevo\EnelixEMS\NetzfahrplanV4Planpruefung::kontext($operation), + 'inputRefs' => ['operation' => 'op-original', 'forecast' => 'fc-original', 'tariffs' => 'tar-original'], + 'points' => $points, 'warnings' => ['Estimated peak; shadow only']]; + $response = ['receiverProtocolVersion' => 1, 'installationId' => $id, 'checkedAt' => gmdate('c', $now), + 'liveEnabled' => false, 'fresh' => true, 'pending' => null, + 'settings' => ['revision' => 1, 'family' => '3', 'runMode' => 'shadow', 'roundtripEfficiency' => .9], + 'families' => [['key' => '3'], ['key' => '13'], ['key' => '23']], 'plan' => $plan, + 'lastRun' => ['status' => 'optimal', 'detail' => ['planId' => $plan['planId']]]]; + return [$response, $id, $operation, $now]; +} diff --git a/tests/V4Receiver/protocol_checks.php b/tests/V4Receiver/protocol_checks.php new file mode 100644 index 0000000..de7cd5b --- /dev/null +++ b/tests/V4Receiver/protocol_checks.php @@ -0,0 +1,73 @@ +0.0],$now); +check($v['previewBatteryW']===3000.0 && $v['canDispatch']===false && $v['liveEnabled']===false,'zero grid uses actual surplus, never permission'); +invalid('wrong installation',fn(&$r)=>$r['installationId']='00000000-0000-4000-8000-000000000001'); +invalid('foreign plan',fn(&$r)=>$r['plan']['installationId']='00000000-0000-4000-8000-000000000001'); +invalid('old protocol',function(&$r){unset($r['receiverProtocolVersion']);}); +invalid('revision mismatch',fn(&$r)=>$r['plan']['configRevision']=2); +invalid('boolean revision',fn(&$r)=>$r['plan']['configRevision']=true); +invalid('live response',fn(&$r)=>$r['liveEnabled']=true); +invalid('live plan',fn(&$r)=>$r['plan']['runMode']='live'); +invalid('fake executable',fn(&$r)=>$r['plan']['executable']='true'); +invalid('stale server flag',fn(&$r)=>$r['fresh']=false); +invalid('pending computation',fn(&$r)=>$r['pending']=['reason'=>'new price']); +invalid('last run different plan',fn(&$r)=>$r['lastRun']['detail']['planId']='wrong'); +invalid('last run invalid',fn(&$r)=>$r['lastRun']['status']='invalid_inputs'); +invalid('family mismatch',fn(&$r)=>$r['settings']['family']='23'); +invalid('unknown family',fn(&$r)=>$r['plan']['sourceFamily']='unregistered'); +invalid('receipt stale',function(&$r,&$op,&$now,&$receipt){$receipt-=91;}); +invalid('future receipt',function(&$r,&$op,&$now,&$receipt){$receipt++;}); +invalid('stale server response',function(&$r)use($now){$r['checkedAt']=gmdate('c',$now-91);}); +invalid('future plan',function(&$r)use($now){$r['plan']['generatedAt']=gmdate('c',$now+10);}); +invalid('stale plan',function(&$r)use($now){$r['plan']['generatedAt']=gmdate('c',$now-901);}); +invalid('price horizon too short',function(&$r)use($now){$r['plan']['pricesKnownUntil']=gmdate('c',$now+300);}); +invalid('stale local operation',function(&$r,&$op)use($now){$op['observedAt']=gmdate('c',$now-61);}); +invalid('changed local reserve',fn(&$r,&$op)=>$op['batteries'][0]['minSocPercent']=30.0); +invalid('changed local grid limit',fn(&$r,&$op)=>$op['limits']['importW']=2000.0); +invalid('typed context required',fn(&$r)=>$r['plan']['controlContext']['batteries']['b']['capacityKwh']='20'); +invalid('input provenance missing',function(&$r){unset($r['plan']['inputRefs']);}); +invalid('unknown assets',function(&$r){$r['plan']['points'][0]['assetTargetsW']=['c'=>3000];}); +invalid('overlap',function(&$r){$r['plan']['points'][1]['time']=$r['plan']['points'][0]['time'];}); +invalid('gap',function(&$r)use($now){$r['plan']['points'][1]['time']=gmdate('c',$now+310);}); +invalid('bool target',fn(&$r)=>$r['plan']['points'][0]['gridTargetW']=false); +invalid('string target',fn(&$r)=>$r['plan']['points'][0]['gridTargetW']='0'); +invalid('NaN target',fn(&$r)=>$r['plan']['points'][0]['batteryTargetW']=NAN); +invalid('broken balance',fn(&$r)=>$r['plan']['points'][0]['baselineGridW']=10000.0); +invalid('unknown intent',fn(&$r)=>$r['plan']['points'][0]['intent']='discharge_all'); +invalid('no timezone',fn(&$r)=>$r['plan']['points'][0]['time']='2026-10-02T12:00:00'); +invalid('bad calendar date',fn(&$r)=>$r['plan']['points'][0]['time']='2026-02-30T12:00:00Z'); +invalid('missing step',fn(&$r)=>$r['plan']['points']=[]); +invalid('negative curtailment',fn(&$r)=>$r['plan']['points'][0]['pvCurtailmentW']=-100.0); +[$r,$id,$op,$now]=receiverFixture();$r['checkedAt']=$op['observedAt']=gmdate('c',$now+300);$op['batteries'][0]['measuredAt']=$op['observedAt']; +$x=P::pruefen($r,$id,$op,$now+300,$now+300); +check($x['point']['time']===gmdate('c',$now+300),'half-open boundary picks current, not interpolated'); +[$r,$id,$op,$now]=receiverFixture();$r['plan']['sourceFamily']=$r['settings']['family']='future-family';$r['families'][]=['key'=>'future-family']; +check(P::pruefen($r,$id,$op,$now,$now)['plan']['sourceFamily']==='future-family','registry is extensible'); +[$r,$id,$op,$now]=receiverFixture();$op['gridW']=3000.0;$op['batteries'][0]['dischargeBlocked']=true; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now); +check($v['previewBatteryW']===0.0,'local hysteresis prevents preview discharge'); +[$r,$id,$op,$now]=receiverFixture();$op['gridW']=3000.0;$op['batteries'][0]['socPercent']=15.001; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now); +check($v['previewBatteryW']>-3 && $v['previewBatteryW']<0,'remaining-step energy cannot cross reserve'); +[$r,$id,$op,$now]=receiverFixture();$op['batteries'][0]['socPercent']=90; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.0,'full battery cannot charge'); +[$r,$id,$op,$now]=receiverFixture();$op['batteries'][0]['maxChargeW']=0;$op['limits']['exportW']=0.;$r['plan']['controlContext']=P::kontext($op); +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now);check($v['unmetGridLimitW']===3000.0 && !$v['canDispatch'],'zero export limit reports unmet constraint'); +[$r,$id,$op,$now]=receiverFixture();$x=P::pruefen($r,$id,$op,$now,$now);$op['gridW']=2000;$op['batteries'][0]['gridCharging']=false; +$x['point']['gridTargetW']=8000.;$x['point']['intent']='gridCharge';$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.,'grid charging opt-in enforced'); +$op['batteries'][0]['gridCharging']=true;$x['point']['intent']='hold';$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.,'nonzero hold does not accidentally charge'); +$x['point']['gridTargetW']=0.;$x['point']['intent']='discharge';$op['gridW']=0.;$v=P::vorschau($x,$op,['b'=>-2000],$now);check($v['previewBatteryW']===-2000.,'uses actual battery feedback, not incremental zero'); +try { P::dekodieren(str_repeat('x',P::MAX_RESPONSE_BYTES+1));check(false,'body limit'); } catch (InvalidArgumentException $e) {check(true,'body limit');} +echo "Total: $count checks. Synthetic data only.\n"; diff --git a/tests/V4Receiver/receiver_checks.php b/tests/V4Receiver/receiver_checks.php new file mode 100644 index 0000000..31cd174 --- /dev/null +++ b/tests/V4Receiver/receiver_checks.php @@ -0,0 +1,117 @@ +$name) { + if (!defined($name)) define($name,50000+$i); + } + function time() { return $GLOBALS['clock']; } + function IPS_SemaphoreEnter($name,$wait) { if ($GLOBALS['lock']) return false;$GLOBALS['lock']=true;return true; } + function IPS_SemaphoreLeave($name) { $GLOBALS['lock']=false; } + function IPS_VariableExists($id) {return $id===100;} + function IPS_GetVariable($id) {return ['VariableUpdated'=>$GLOBALS['clock']];} + function GetValue($id) {return $GLOBALS['actual'];} + function curl_init($url) { return (object)['url'=>$url,'options'=>[],'code'=>0]; } + function curl_setopt_array($h,$options) { $h->options=$options;return true; } + function curl_getinfo($h,$key) { return $h->code; } + function curl_close($h) {} + function curl_exec($h) { + $GLOBALS['requests'][]=['url'=>$h->url,'options'=>$h->options]; + $ack=str_ends_with($h->url,'/ack'); + $h->code=$ack?$GLOBALS['ackCode']:$GLOBALS['getCode']; + $body=$ack?['status'=>'recorded']:$GLOBALS['response']; + $json=json_encode($body,JSON_THROW_ON_ERROR); + $callback=$h->options[CURLOPT_WRITEFUNCTION]; + return $callback($h,$json)===strlen($json); + } + final class ReceiverHarness { + use ManagerNetzfahrplanV4EmpfangTrait; + public int $InstanceID=17004; + public array $props=[],$attrs=[],$vars=[],$timers=[]; + public bool $licensed=true; + public array $op=[]; + public function __construct($id,$op) { + $this->op=$op; + $this->props=['NetzfahrplanV4EmpfangAktiv'=>false,'NetzfahrplanV4SchattenAktiv'=>true, + 'NetzfahrplanAktiv'=>false,'MesswertMaxAlter'=>60, + 'AnlagenBatterien'=>json_encode([['ID'=>'b','LeistungVariableID'=>100,'Leistungsfaktor'=>1]])]; + $this->attrs=['LizenzInstallationID'=>$id,'PrognoseInstallationsToken'=>str_repeat('x',43)]; + $this->registriereNetzfahrplanV4Empfang(); + } + public function receive() {$this->empfangeNetzfahrplanV4();} + public function update() {$this->aktualisiereNetzfahrplanV4Vorschau();} + public function stop() {$this->stoppeNetzfahrplanV4Vorschau();} + public function configure() {$this->konfiguriereNetzfahrplanV4Vorschau();} + public function form() {return $this->netzfahrplanV4EmpfangFormular();} + public function GetNetzfahrplanV4Diagnose() { return json_encode(['operation'=>$this->op]); } + private function berechtigungLizenziert($key) {return $this->licensed;} + private function ReadPropertyBoolean($key) {return (bool)$this->props[$key];} + private function ReadPropertyString($key) {return $this->props[$key];} + private function ReadPropertyInteger($key) {return (int)$this->props[$key];} + private function ReadAttributeString($key) {return $this->attrs[$key]??'';} + private function ReadAttributeInteger($key) {return (int)($this->attrs[$key]??0);} + private function WriteAttributeString($key,$val) { + if (!str_starts_with($key,'NetzfahrplanV4')) throw new \RuntimeException('Unexpected external attribute'); + $this->attrs[$key]=$val; + } + private function WriteAttributeInteger($key,$val) { $this->attrs[$key]=$val; } + private function RegisterPropertyBoolean($key,$default) {$this->props[$key]??=$default;} + private function RegisterAttributeString($key,$default) {$this->attrs[$key]??=$default;} + private function RegisterAttributeInteger($key,$default) {$this->attrs[$key]??=$default;} + private function RegisterVariableString($key,$name,$profile,$pos) {$this->vars[$key]='';} + private function RegisterTimer($key,$interval,$script) {$this->timers[$key]=$interval;} + private function SetTimerInterval($key,$interval) {$this->timers[$key]=$interval;} + private function SetValue($key,$value) { + if (!in_array($key,['NetzfahrplanV4Vorschau','NetzfahrplanV4VorschauJSON'],true)) throw new \RuntimeException('Actuator/foreign variable write'); + $this->vars[$key]=$value; + } + } +} +namespace { + require_once __DIR__.'/fixture.php'; + use Belevo\EnelixEMS\ReceiverHarness; + $count=0; + function check($ok,$name) {global $count;if (!$ok) throw new RuntimeException('FAIL '.$name);$count++;echo 'PASS '.$name."\n";} + function fixture() { + [$r,$id,$op,$now]=receiverFixture(); + $GLOBALS['response']=$r;$GLOBALS['clock']=$now;$GLOBALS['actual']=0.;$GLOBALS['requests']=[]; + $GLOBALS['lock']=false;$GLOBALS['getCode']=200;$GLOBALS['ackCode']=200; + return new ReceiverHarness($id,$op); + } + function advance($h,$seconds) { + $GLOBALS['clock']+=$seconds;$GLOBALS['response']['checkedAt']=gmdate('c',$GLOBALS['clock']); + $h->op['observedAt']=gmdate('c',$GLOBALS['clock']);$h->op['batteries'][0]['measuredAt']=$h->op['observedAt']; + } + $h=fixture();$h->receive();check(count($requests)===0,'default disabled no network'); + $h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->configure();check($h->timers['NetzfahrplanV4VorschauAktualisieren']===10000,'watchdog without network'); + $h->receive();$v=json_decode($h->GetNetzfahrplanV4Vorschau(),true); + check($v['status']==='preview_only' && $v['previewBatteryW']===3000 && $v['canDispatch']===false,'read and show only'); + check(count($requests)===2 && str_ends_with($requests[1]['url'],'/ack'),'one read and one acknowledgement'); + $body=json_decode($requests[1]['options'][CURLOPT_POSTFIELDS],true);check($body['status']==='shadow_seen' && $body['step']===$response['plan']['points'][0]['time'],'ack is shadow_seen, never applied'); + $h->receive();check(count($requests)===2,'repeated click throttled'); + advance($h,60);$h->receive();check(count($requests)===3,'same plan step not acknowledged twice'); + advance($h,91);$h->update();$v=json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true); + check($v['status']==='unavailable' && $v['previewBatteryW']===null,'expired cache clears numeric targets, not fake zero'); + check(!str_contains($h->vars['NetzfahrplanV4Vorschau'],'3000'),'expired HTML clears previous setpoint'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();advance($h,61);$GLOBALS['getCode']=429;$h->receive(); + check($h->attrs['NetzfahrplanV4Empfang']==='{}' && json_decode($h->GetNetzfahrplanV4Vorschau(),true)['previewBatteryW']===null,'429 invalidates cache'); + check(!str_contains(json_encode($h->vars),str_repeat('x',43)),'no credentials in diagnosis'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->props['NetzfahrplanAktiv']=true;$h->receive();check(count($requests)===0,'legacy execution on blocks preview acquisition'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['response']['installationId']='00000000-0000-4000-8000-000000000001';$h->receive(); + check(count($requests)===1 && $h->attrs['NetzfahrplanV4Empfang']==='{}','foreign response not acknowledged'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['ackCode']=503;$h->receive();$v=json_decode($h->GetNetzfahrplanV4Vorschau(),true); + check($v['status']==='preview_only' && !$v['reception']['acknowledged'] && isset($v['reception']['acknowledgementError']),'ack failure is visible, no false execution confirmation'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->op['batteries'][0]['minSocPercent']=30.;$h->update(); + check(json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true)['previewBatteryW']===null,'local policy change invalidates cache'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->stop();check($h->attrs['NetzfahrplanV4Empfang']==='{}' && $h->timers['NetzfahrplanV4VorschauAktualisieren']===0,'apply/restart invalidates cached plan'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->licensed=false;$h->receive();check(count($requests)===0,'no license no read'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['lock']=true;$h->receive();check(count($requests)===0,'concurrent receiver does not send'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->props['NetzfahrplanV4EmpfangAktiv']=false;$h->update();check(json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true)['previewBatteryW']===null,'disable clears displayed target'); + $form=$h->form();check($form['items'][1]['name']==='NetzfahrplanV4EmpfangAktiv','separate dynamic form field'); + $source=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4EmpfangTrait.php'); + check(!preg_match('/\b(?:SendDataToChildren|SendDataToParent|sendeManagerdaten|regeln|SetStatus|EnableAction|IPS_SetProperty|IPS_ApplyChanges)\s*\(/',$source),'no control or external configuration call in receiver'); + check(!str_contains($source,"'status' => 'applied'"),'never acknowledges applied'); + echo "Total: $count checks. Mocked IPS and HTTP only.\n"; +} From 9394fe3eb1435ec2ef444700a647fe562eb1a07e Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 14:01:41 +0000 Subject: [PATCH 05/22] feat(manager): add gated short-lease V4 control trial and local fallback --- Batterie/module.php | 48 +++++- Manager/module.php | 30 ++++ docs/netplan-v4-controlled-trial.md | 96 +++++++++++ libs/BatterieNetzfahrplanV4TestTrait.php | 186 +++++++++++++++++++++ libs/ManagerNetzfahrplanV4EmpfangTrait.php | 2 +- libs/ManagerNetzfahrplanV4TestTrait.php | 177 ++++++++++++++++++++ libs/NetzfahrplanV4Regeltest.php | 168 +++++++++++++++++++ tests/NetzfahrplanV4RegeltestTest.php | 24 +++ tests/V4ControlTrial/additional_checks.php | 30 ++++ tests/V4ControlTrial/checks.php | 161 ++++++++++++++++++ 10 files changed, 918 insertions(+), 4 deletions(-) create mode 100644 docs/netplan-v4-controlled-trial.md create mode 100644 libs/BatterieNetzfahrplanV4TestTrait.php create mode 100644 libs/ManagerNetzfahrplanV4TestTrait.php create mode 100644 libs/NetzfahrplanV4Regeltest.php create mode 100644 tests/NetzfahrplanV4RegeltestTest.php create mode 100644 tests/V4ControlTrial/additional_checks.php create mode 100644 tests/V4ControlTrial/checks.php diff --git a/Batterie/module.php b/Batterie/module.php index 97c7a90..8510535 100644 --- a/Batterie/module.php +++ b/Batterie/module.php @@ -6,6 +6,7 @@ require_once __DIR__ . '/../libs/VerbraucherSchnittstelle.php'; require_once __DIR__ . '/../libs/VerbraucherBasisTrait.php'; require_once __DIR__ . '/../libs/Nachrichtenvertrag.php'; require_once __DIR__ . '/../libs/BatterieRegler.php'; +require_once __DIR__ . '/../libs/BatterieNetzfahrplanV4TestTrait.php'; use Belevo\EnelixEMS\BatterieRegler; use Belevo\EnelixEMS\Nachrichtenvertrag; @@ -15,6 +16,7 @@ use Belevo\EnelixEMS\VerbraucherSchnittstelle; class Batterie extends IPSModule implements VerbraucherSchnittstelle { use VerbraucherBasisTrait; + use \Belevo\EnelixEMS\BatterieNetzfahrplanV4TestTrait; private const MANAGER_MODULE_ID = '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}'; private const STATUS_AKTIV = 102; @@ -68,6 +70,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle parent::Create(); $this->registriereVerbraucherBasis(); + $this->registriereV4BatterieTest(); $this->RegisterPropertyInteger('Batterietyp', BatterieRegler::TYP_UNKONFIGURIERT); $this->RegisterPropertyInteger( 'Batteriemanagement', @@ -152,6 +155,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle { parent::ApplyChanges(); + $this->V4BatterieTestStoppen(); $this->aktualisiereVariablen(); $this->registriereMesswertmeldungen(); $this->WriteAttributeBoolean('RegisterInitialisiert', false); @@ -248,7 +252,22 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->aktualisiere(true); return; + case 'NetzfahrplanV4TestStart': + if (!is_string($wert)) throw new InvalidArgumentException('JSON-Testanmeldung erforderlich.'); + $this->V4BatterieTestStarten($wert); + return; + case 'NetzfahrplanV4TestCommand': + if (!is_string($wert)) throw new InvalidArgumentException('JSON-Testbefehl erforderlich.'); + $this->V4BatterieTestBefehl($wert); + return; + case 'NetzfahrplanV4TestStop': + $this->V4BatterieTestStoppen(); + return; + case 'NetzfahrplanV4TestWatchdog': + $this->aktualisiere(true); + return; case 'SichererZustand': + $this->V4BatterieTestStoppen(); $this->setzeZustand('SollwertGueltig', false); $this->schreibeRegister(0, true); $this->aktualisiere(true); @@ -287,6 +306,14 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle /** @param array $daten */ public function ManagerdatenEmpfangen(array $daten): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterieannahme beschaeftigt.'); + try { $this->ManagerdatenEmpfangenIntern($daten); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function ManagerdatenEmpfangenIntern(array $daten): void { Nachrichtenvertrag::pruefeManagerdaten($daten); if ($daten['Kopf']['EmpfaengerID'] !== $this->InstanceID) { @@ -302,8 +329,13 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ); } + if ($this->v4BatterieTestAktiv()) { + // Ordinary manager messages cannot overwrite or renew the independent short test lease. + $this->planeMeldung(); + return; + } $this->WriteAttributeString('Betriebsart', $daten['Betriebsart']); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); $sollleistung = $daten['Sollleistung_W']; if ($sollleistung === null) { @@ -314,7 +346,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ) ) { $this->setzeZustand('SollwertGueltig', false); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); } $this->planeMeldung(); return; @@ -341,11 +373,19 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->setzeZustand('Sollleistung', $sollleistung); $this->setzeZustand('SollwertGueltig', true); $this->planeVorgabeTimeout(time()); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); $this->planeMeldung(); } private function aktualisiere(bool $meldungPlanen): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { $this->aktualisiereIntern($meldungPlanen); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function aktualisiereIntern(bool $meldungPlanen): void { try { $jetzt = time(); @@ -426,6 +466,8 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $ziel = $verfuegbar && (bool) $this->leseZustand('SollwertGueltig') ? (int) $this->leseZustand('Sollleistung') : 0; + $v4Ziel = $this->v4BatterieTestZiel($messwerte, $verfuegbar, $jetzt); + if ($v4Ziel !== null) $ziel = $v4Ziel; $this->schreibeRegister($ziel, false); $this->setzeZustand('Sollleistung', $ziel); $this->setzeZustand('Verfuegbar', $verfuegbar); diff --git a/Manager/module.php b/Manager/module.php index f4b2158..b2b75ac 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -13,6 +13,7 @@ require_once __DIR__ . '/../libs/EnergieMessung.php'; require_once __DIR__ . '/../libs/ManagerEnergieTrait.php'; require_once __DIR__ . '/../libs/NetzfahrplanV4Betriebsdaten.php'; require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4Trait.php'; +require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4TestTrait.php'; use Belevo\EnelixEMS\Anlagentopologie; use Belevo\EnelixEMS\EinspeiseRegler; @@ -28,6 +29,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle { use ManagerEnergieTrait; use \Belevo\EnelixEMS\ManagerNetzfahrplanV4Trait; + use \Belevo\EnelixEMS\ManagerNetzfahrplanV4TestTrait; private const STATUS_AKTIV = 102; private const STATUS_MESSWERT_UNGUELTIG = 201; @@ -116,6 +118,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle { parent::Create(); $this->registriereNetzfahrplanV4(); + $this->registriereV4ManagerTest(); $this->RegisterPropertyInteger('Rolle', 0); $this->RegisterPropertyInteger('NetzleistungVariableID', 0); @@ -236,6 +239,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle public function ApplyChanges(): void { parent::ApplyChanges(); + $this->v4ManagerTestAbbrechen('Manager neu initialisiert'); $this->SetTimerInterval('NetzfahrplanV4Senden', 0); $this->stoppeNetzfahrplanV4Vorschau(); @@ -468,6 +472,14 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->aktualisiereNetzfahrplanV4Vorschau(); return; + case 'NetzfahrplanV4TestStart': + if (!is_string($wert)) throw new InvalidArgumentException('Explizite JSON-Testfreigabe erforderlich.'); + $this->V4ManagerTestStarten($wert); + return; + case 'NetzfahrplanV4TestStop': + $this->V4ManagerTestStoppen(); + return; + case 'NetzfahrplanV4Senden': $this->sendeNetzfahrplanV4(); return; @@ -703,6 +715,22 @@ class Manager extends IPSModule implements ManagerSchnittstelle } private function regeln(bool $erneuern): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { + $this->regelnIntern($erneuern); + if ($this->v4ManagerTestSession() + && (!$this->GetValue('Aktiv') || IPS_GetInstance($this->InstanceID)['InstanceStatus'] !== self::STATUS_AKTIV)) { + $this->v4ManagerTestAbbrechen('Manager nicht mehr betriebsbereit'); + } + } catch (Throwable $e) { + $this->v4ManagerTestAbbrechen('Regelungsfehler'); + throw $e; + } finally { IPS_SemaphoreLeave($lock); } + } + + private function regelnIntern(bool $erneuern): void { if (!$this->aktualisiereLizenzfreigabe($this->ReadPropertyString('Lizenzcode'), false)) { $this->SetValue('Betriebsart', ManagerRegler::BETRIEBSART_INAKTIV); @@ -807,6 +835,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle : $this->nichtGeregelteGeraeteText($nichtGeregelt) ); + $this->v4ManagerTestUebergabe($ergebnis, $verbraucher, $synchronisiert); $letzteSollwerte = $this->leseJsonAttribut('LetzteSollwerte'); foreach ($ergebnis['Sollwerte'] as $instanzID => $sollleistung) { if (!$erneuern && ($letzteSollwerte[(string) $instanzID] ?? null) === $sollleistung) { @@ -816,6 +845,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->sendeManagerdaten((int) $instanzID, $betriebsart, $sollleistung); $letzteSollwerte[(string) $instanzID] = $sollleistung; } catch (Throwable $fehler) { + $this->v4ManagerTestAbbrechen('Andere Verbraucher-Stellwertuebergabe fehlgeschlagen'); $stoerungen[] = 'Sollwert an Instanz ' . $instanzID . ': ' . $fehler->getMessage(); } } diff --git a/docs/netplan-v4-controlled-trial.md b/docs/netplan-v4-controlled-trial.md new file mode 100644 index 0000000..b9b0936 --- /dev/null +++ b/docs/netplan-v4-controlled-trial.md @@ -0,0 +1,96 @@ +# V4: begrenzter Regeltest (Entwicklungskandidat, NICHT in Betrieb) + +Stand 2026-10-02. Diese Erweiterung ist weder Produktionsfreigabe noch ein +Installations-/Startauftrag. Bestehende Schattenplaene und ihre `shadow_seen` +Bestaetigungen bleiben unveraendert. Ein Vorschauwert allein darf niemals +als Stellfreigabe interpretiert werden. + +## Freigabekette + +1. Serverseitige separate Allowlist `NETPLAN_V4_CONTROL_TRIAL_PLANTS` (standardmaessig + leer). Interner authentifizierter `planner/trial/arm`-Aufruf mit expliziter + Bestaetigung, neuer Sitzung, gepruefter Plan-ID/Revision, Batteriezuordnung, + Zeitfenster, Leistungsgrenzen und dokumentierter Geraete-Watchdog-Abnahme. + Die bestehenden oeffentlichen Portal-/Geraeterouten proxien diesen Aufruf NICHT. +2. `NetzfahrplanV4RegeltestErlaubt` lokal im Manager UND im Batteriemodul (jeweils + Standard false). Alte `NetzfahrplanAktiv`-Freigabe muss false bleiben. +3. Zusaetzlicher bewusster lokaler Start einer bestimmten Sitzung. Ein Reload, + Netzwiederkehr oder gespeicherter Haken startet keine Sitzung automatisch. +4. Der Manager prueft den empfangenen Schattenplan und die getrennte + `controlled_trial_authority` gemeinsam. Sie sind gebunden an Anlage, Batterie, + beide Instanz-IDs, Revision, manuelle Modellfamilie und stabilen Planungskontext. +5. Die Batterie nimmt nur einen separaten `controlled_trial_command` mit eigener + Sequenz und kurzer Gueltigkeit an. Normale Managerbefehle erneuern diese + Gueltigkeit nicht. `shadow_seen` bleibt eine Empfangs-, keine Ausfuehrungsmeldung. + +Die Serverantwort bleibt eine Schattenplan-Antwort (`liveEnabled:false`). Eine +getrennte Testfreigabe ist unter `controlledTrial` und +`controlledTrialAuthorized` sichtbar. Der Schattenmodus allein ist bei kuenftig +bewusst gestartetem Test deshalb kein Nachweis fuer 'keine Batteriebewegung'. +Die Controller-Diagnosen benennen eine angenommene Sollvorgabe, nicht bereits +physisch gemessene Umsetzung oder Ersparnis. + +## Bewusst begrenzter Pilotumfang + +- Genau eine eindeutig zugeordnete Batterie je Test; Familie manuell festgelegt. +- 30 bis 1800 Sekunden Sitzung, maximal 5000 W Laden und 5000 W Entladen. +- Server-, Manager- und Batteriegrenze koennen strenger sein; nie grosszuegig + ersetzen oder still auf 39 kW hochsetzen. +- Gesamtsollleistung, NICHT Zusatzleistung zur bereits fliessenden Batterieleistung. +- Andere Verbraucher bleiben im alten Verteiler. Ihre erwartete Leistungsaenderung + wird einmal bei der Batteriekorrektur beruecksichtigt. +- Batterie wird nur einem Stellwertpfad zugeteilt, nicht gleichzeitig dem alten + und neuen Verteiler. Ist der Test ungueltig, stoppt er und es wird eine NEUE + normale Verteilung angefordert; kein alter gespeicherter Sollwert wird restauriert. +- Physische Grenzverletzung, die innerhalb der Pilotleistung nicht behebbar ist, + beendet den Test zugunsten der bestehenden lokalen Regelung. + +## Ausfall- und Zeitverhalten + +- Serverfreigabe maximal 90 Sekunden, lokal regelmaessig frisch abgerufen. + Serverseitiger Widerruf wirkt nicht magisch sofort: Cachefrist beachten. +- Einzelner Batteriebefehl maximal 10 Sekunden; Batterie kontrolliert im laufenden + Kernel mit 1-Sekunden-Timer zusaetzlich zu Messwert-Callbacks. +- Manager liefert kurze Befehle im Test mit zusaetzlichem 2-Sekunden-Takt. +- Monotone Laufzeit und Kalenderzeit werden getrennt geprueft. Eine rueckwaerts + springende Systemuhr darf eine Sitzung nicht verlaengern. +- Abbruch widerruft die Sitzung VOR dem Nullstellversuch. Schreibfehler werden als + `stop_failed`/nicht bestaetigter Stopp sichtbar, nicht als sicherer Zustand. +- Reserve, Hysterese, aktuelle Lade-/Entladeleistung, SOC und Verfuegbarkeit werden + auch im Batteriemodul erneut geprueft. Sicherheitsreduktionen werden nicht durch + die Umschaltsperre verzoegert; Richtungswechsel laufen ueber null. +- Laufzeitsitzungen liegen nur im Buffer, nicht als automatische Wiederanlauf-Freigabe. + Ein persistenter Marker sorgt bei Wiederinitialisierung fuer einen erneuten + Nullstellversuch, falls ein Testwert zuvor ausgegeben worden sein koennte. + +WICHTIGE GRENZE: PHP-Timer und Software-Nullstellversuche funktionieren nur bei +laufendem Kernel und erreichbarem Wechselrichter. Sie sind KEIN physischer +Not-Aus und kein Beweis fuer Abschaltung bei Serverabsturz, Stromausfall oder +unterbrochener Stellwertverbindung. Dafuer ist ein separat nachgewiesener +Befehlsausfall-/Watchdogmechanismus am Geraet bzw. Gateway erforderlich. +Das Feld `NetzfahrplanV4GeraeteWatchdogNachweis` ist eine Referenz auf die gepruefte +Dokumentation/Abnahme, keine automatische Hardwarepruefung und kein Defaultwert. + +## Noch offene Freigabepunkte (nicht mit Platzhaltern umgehen) + +- Der laufende Lihrenmoos-Prognosestrom ist noch `house_total`. Fuer Testfreigabe + muss `base_load` mit nachgewiesener Quelle/SDL-Abgrenzung und + `accountingEvidenceId` aus dem geprueften Adapter vorliegen. Ein Labelwechsel + oder ein erfundener Nachweis reicht nicht; Quelle und Messbilanz pruefen. +- Geraete-/Gateway-Watchdog bei ausbleibenden Stellbefehlen nachweisen. +- Vollstaendiges Manager-/Batterie-Callbackverhalten im IP-Symcon-Kernel pruefen. + Die isolierten Tests verwenden echte neue Traits, aber simulierte IPS-Aufrufe, + Messwerte, Zeit und Register. Reale Modbus-/VGT-Stellreaktion steht aus. +- Zielcontainerpruefung des neuesten Serverstands und Feldabnahme. +- Keine durchgehende unbeaufsichtigte Produktion, kein automatischer Mehranlagenstart. + Modellvergleich, Trainingsanbindung, Mehranlagenabnahme und Betriebs-Release + bleiben weitere Aufgaben. + +## Quelltests + +`php tests/V4ControlTrial/checks.php` nur im separaten CLI-Testprozess ausfuehren, +niemals im laufenden Symcon-Skripteditor. Die Datei verweigert einen realen +Symcon-Kernel. Eine PHPUnit-Huelle bindet sie an die normale Testsuite an. + +Der Server-Test verwendet ausschliesslich synthetische Daten/Temporaerdatenbanken. +Keine Testfunktion liest Zugangsdaten oder schreibt eine reale Stellvariable. diff --git a/libs/BatterieNetzfahrplanV4TestTrait.php b/libs/BatterieNetzfahrplanV4TestTrait.php new file mode 100644 index 0000000..56a20d2 --- /dev/null +++ b/libs/BatterieNetzfahrplanV4TestTrait.php @@ -0,0 +1,186 @@ +RegisterPropertyBoolean('NetzfahrplanV4RegeltestErlaubt', false); + $this->RegisterAttributeString('NetzfahrplanV4LetzteTestsitzung', ''); + $this->RegisterPropertyString('NetzfahrplanV4GeraeteWatchdogNachweis', ''); + $this->RegisterAttributeString('NetzfahrplanV4TestStatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4TestWatchdog', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4TestWatchdog', 0);"); + } + + private function v4BatterieTestSitzung(): array + { + $s = json_decode($this->GetBuffer('V4BatteryTrialSession') ?: '{}', true); + return is_array($s) ? $s : []; + } + + private function v4BatterieTestAktiv(): bool + { + return ($this->v4BatterieTestSitzung()['active'] ?? false) === true; + } + + /** Local test arming by the bound manager; never called by ordinary forecast traffic. */ + public function V4BatterieTestStarten(string $json): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->v4BatterieStartIntern($json); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function v4BatterieStartIntern(string $json): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt')) throw new RuntimeException('Batterietest lokal nicht erlaubt.'); + $s = json_decode($json, true, 32, JSON_THROW_ON_ERROR);$now = time(); + if (!is_array($s) || ($s['kind'] ?? null) !== 'local_control_trial_consent' + || ($s['batteryInstanceId'] ?? null) !== $this->InstanceID || $this->v4BatterieTestAktiv()) { + throw new RuntimeException('Batterie-Testanmeldung ungueltig oder bereits aktiv.'); + } + NetzfahrplanV4Regeltest::uuid($s['sessionId'] ?? null); + if ($this->ReadAttributeString('NetzfahrplanV4LetzteTestsitzung') === $s['sessionId']) throw new RuntimeException('Testsitzung bereits beendet/verwendet.'); + NetzfahrplanV4Regeltest::uuid($s['installationId'] ?? null); + $manager = NetzfahrplanV4Regeltest::integer($s['managerId'] ?? null, 'Manager', 1, 99999); + if (!in_array($manager, $this->zugeordneteManagerIDs(), true) + || IPS_GetProperty($manager, 'NetzfahrplanV4RegeltestErlaubt') !== true) { + throw new RuntimeException('Manager nicht fuer Batterietest freigegeben/zugeordnet.'); + } + $evidence = $this->ReadPropertyString('NetzfahrplanV4GeraeteWatchdogNachweis'); + if (strlen($evidence) < 8 || $evidence !== ($s['actuatorWatchdogEvidenceId'] ?? null)) { + throw new RuntimeException('Externer Geraete-Watchdog muss separat nachgewiesen sein.'); + } + $end = NetzfahrplanV4Regeltest::integer($s['expiresAt'] ?? null, 'Testende', $now + 1, $now + 1800); + $s['maxChargeW'] = NetzfahrplanV4Regeltest::number($s['maxChargeW'] ?? null, 'Testladen', 1, 5000); + $s['maxDischargeW'] = NetzfahrplanV4Regeltest::number($s['maxDischargeW'] ?? null, 'Testentladen', 1, 5000); + if (!is_string($s['assetId'] ?? null) || $s['assetId'] === '') throw new RuntimeException('Batterie-ID fehlt.'); + if (!$this->GetValue('Aktiv') || $this->ReadPropertyInteger('Batteriemanagement') !== 2) { + throw new RuntimeException('Batterie nicht unter ENELIX-Regelung.'); + } + $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $s['sessionId']); + $s['active'] = true;$s['lastSequence'] = 0;$s['lastWallClockAt'] = $now; + $s['monotonicDeadline'] = hrtime(true) / 1e9 + $end - $now; + $s['commandMonotonicDeadline'] = hrtime(true) / 1e9 + 10; + $this->SetBuffer('V4BatteryTrialCommand', '{}'); + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + // Old ordinary values may not resume after the test without a NEW manager message. + $this->setzeZustand('SollwertGueltig', false); + $this->SetTimerInterval('NetzfahrplanV4TestWatchdog', 1000); + $this->WriteAttributeString('NetzfahrplanV4TestStatus', '{"status":"armed_waiting_command","physicalResponseVerified":false}'); + } + + public function V4BatterieTestBefehl(string $json): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->v4BatterieBefehlIntern($json); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function v4BatterieBefehlIntern(string $json): void + { + $s = $this->v4BatterieTestSitzung();$now = time();$mono = hrtime(true) / 1e9; + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt')) throw new RuntimeException('Batterietest gesperrt.'); + $c = json_decode($json, true, 32, JSON_THROW_ON_ERROR); + NetzfahrplanV4Regeltest::pruefeBatteriebefehl($c, $s, $now, $mono); + $s['lastSequence'] = $c['sequence'];$s['lastWallClockAt'] = $now; + $s['commandMonotonicDeadline'] = $mono + min(10, $c['expiresAt'] - $now); + $this->SetBuffer('V4BatteryTrialCommand', json_encode($c, JSON_THROW_ON_ERROR)); + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + $this->aktualisiereIntern(true); + if (!$this->v4BatterieTestAktiv() || $this->ReadAttributeString('Registerfehler') !== '') { + throw new RuntimeException('Batterie konnte Testbefehl nicht sicher uebernehmen.'); + } + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode([ + 'status' => 'controller_command_accepted', 'sessionId' => $s['sessionId'], + 'sourceShadowPlanId' => $c['sourceShadowPlanId'], 'sequence' => $c['sequence'], + 'commandExpiresAt' => $c['expiresAt'], 'actualRequestedW' => $this->ReadAttributeInteger('LetzterSollwert'), + 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->beendeV4BatterieTest('Befehl abgelehnt', true); + throw $e; + } + } + + /** Explicit local abort or ApplyChanges: revoke before any register action. */ + private function beendeV4BatterieTest(string $reason, bool $writeNow): void + { + $oldStatus = json_decode($this->ReadAttributeString('NetzfahrplanV4TestStatus'), true); + $active = $this->v4BatterieTestAktiv() || in_array($oldStatus['status'] ?? '', ['controller_command_accepted', 'armed_waiting_command', 'stop_pending', 'stop_failed'], true); + $this->SetBuffer('V4BatteryTrialSession', '{}');$this->SetBuffer('V4BatteryTrialCommand', '{}'); + $this->SetTimerInterval('NetzfahrplanV4TestWatchdog', 0); + if (!$active) return; + $this->setzeZustand('SollwertGueltig', false);$this->setzeZustand('Sollleistung', 0); + $report = ['status' => 'stop_pending', 'reason' => $reason, 'physicalResponseVerified' => false]; + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + if ($writeNow) { + try { $this->schreibeRegister(0, true);$report['status'] = 'stopped'; } + catch (Throwable $e) { + $report['status'] = 'stop_failed'; + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + throw $e; + } + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + } + } + + public function V4BatterieTestStoppen(): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->beendeV4BatterieTest('Lokaler Testabbruch', true); } + finally { IPS_SemaphoreLeave($lock); } + } + + public function GetV4BatterieTestStatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4TestStatus'); + } + + /** Called inside the existing battery update, immediately before writing registers. */ + private function v4BatterieTestZiel(array $measurements, bool $available, int $now): ?int + { + if (!$this->v4BatterieTestAktiv()) return null; + $s = $this->v4BatterieTestSitzung();$mono = hrtime(true) / 1e9; + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt')) throw new RuntimeException('Lokale Testfreigabe entzogen.'); + $c = json_decode($this->GetBuffer('V4BatteryTrialCommand') ?: '{}', true); + if ($now >= $s['expiresAt'] || $mono >= $s['monotonicDeadline'] || $now < $s['lastWallClockAt']) throw new RuntimeException('Testsitzung abgelaufen/Zeitsprung.'); + if (!$c) { + if ($mono >= $s['commandMonotonicDeadline']) throw new RuntimeException('Kein Startbefehl innerhalb Watchdog.'); + return 0; + } + $w = NetzfahrplanV4Regeltest::batterieZiel($c, $s, $measurements, [ + 'available' => $available, 'reserve' => $this->ReadPropertyFloat('ReserveLadezustand'), + 'minimum' => $this->ReadPropertyFloat('MindestLadezustand'), + 'blocked' => (bool) $this->leseZustand('HystereseAktiv'), + 'rechargeLimitW' => $this->ReadPropertyBoolean('NachladenMitMaximalleistung') ? null : $this->ReadPropertyInteger('MaximaleNachladeleistung'), + ], $now, $mono); + // Respect switching lock for increases/reversals; safety reductions and zero are immediate. + $last = $this->ReadAttributeInteger('LetzterSollwert'); + if ($now < $this->ReadAttributeInteger('AenderungFreigabeAb') && $w !== $last) { + if ($last * $w < 0) $w = 0; + elseif (abs($w) > abs($last)) $w = $last; + } + $s['lastWallClockAt'] = $now; + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + return $w; + } catch (Throwable $e) { + $this->beendeV4BatterieTest($e->getMessage(), false); + return 0; // revoke stale value even if ordinary cached setpoint existed + } + } +} diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php index 569a7ad..0761567 100644 --- a/libs/ManagerNetzfahrplanV4EmpfangTrait.php +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -84,7 +84,7 @@ trait ManagerNetzfahrplanV4EmpfangTrait $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); // Only the known, non-secret protocol envelope is cached. $keys = ['receiverProtocolVersion', 'installationId', 'checkedAt', 'liveEnabled', 'fresh', - 'pending', 'settings', 'plan', 'families', 'lastRun']; + 'pending', 'settings', 'plan', 'families', 'lastRun', 'controlledTrial']; $safe = array_intersect_key($response, array_flip($keys)); $this->WriteAttributeString('NetzfahrplanV4Empfang', json_encode( ['receivedAt' => $receipt, 'response' => $safe], JSON_THROW_ON_ERROR)); diff --git a/libs/ManagerNetzfahrplanV4TestTrait.php b/libs/ManagerNetzfahrplanV4TestTrait.php new file mode 100644 index 0000000..7a052e4 --- /dev/null +++ b/libs/ManagerNetzfahrplanV4TestTrait.php @@ -0,0 +1,177 @@ +RegisterPropertyBoolean('NetzfahrplanV4RegeltestErlaubt', false); + $this->RegisterAttributeString('NetzfahrplanV4LetzteTestsitzung', ''); + $this->RegisterAttributeString('NetzfahrplanV4RegeltestStatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4TestTick', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'Regeln', true);"); + } + + private function v4ManagerTestSession(): array + { + $value = json_decode($this->GetBuffer('V4ManagerTrialSession') ?: '{}', true); + return is_array($value) ? $value : []; + } + + private function v4ManagerTestBedingungen(): void + { + if (IPS_GetInstance($this->InstanceID)['InstanceStatus'] !== 102 + || !$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt') + || !$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') + || !$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') + || $this->ReadPropertyBoolean('NetzfahrplanAktiv') || !$this->GetValue('Aktiv') + || !$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Keine lokale Testberechtigung oder konkurrierender Fahrplan.'); + } + } + + /** Explicit kernel-local user action only. No network request can call this via the portal. */ + public function V4ManagerTestStarten(string $json): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Regelung beschaeftigt.'); + $armedBattery = null; + try { + $this->v4ManagerTestBedingungen(); + if ($this->v4ManagerTestSession()) throw new RuntimeException('Testsitzung bereits aktiv; nicht verlaengern.'); + $request = json_decode($json, true, 16, JSON_THROW_ON_ERROR); + if (!is_array($request) || ($request['confirmation'] ?? null) !== 'START_BOUNDED_V4_CONTROL_TRIAL') { + throw new RuntimeException('Ausdrueckliche lokale Testbestaetigung erforderlich.'); + } + $session = NetzfahrplanV4Regeltest::uuid($request['sessionId'] ?? null); + if ($this->ReadAttributeString('NetzfahrplanV4LetzteTestsitzung') === $session) throw new RuntimeException('Testsitzung bereits benutzt; neue Freigabe erforderlich.'); + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + $a = $envelope['response']['controlledTrial'] ?? null; + if (!is_array($a) || ($a['sessionId'] ?? null) !== $session) throw new RuntimeException('Keine passende separate Serverfreigabe empfangen.'); + [$op, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $batteryID = (int) ($a['batteryInstanceId'] ?? 0); + if (!in_array($batteryID, $this->aktiveVerbraucherIDs(), true) + || !IPS_InstanceExists($batteryID) + || IPS_GetInstance($batteryID)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + throw new RuntimeException('Testbatterie ist keine aktive zugeordnete Batterieinstanz.'); + } + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 32, JSON_THROW_ON_ERROR); + if (count($assets) !== 1 || $assets[0]['ID'] !== ($a['assetId'] ?? null) + || $assets[0]['SOCVariableID'] !== IPS_GetProperty($batteryID, 'LadezustandVariableID') + || $assets[0]['LeistungVariableID'] !== IPS_GetProperty($batteryID, 'IstleistungVariableID')) { + throw new RuntimeException('Physische Batteriezuordnung ist nicht eindeutig.'); + } + $now = time();$until = min(NetzfahrplanV4Planpruefung::zeit($a['expiresAt']), + $now + NetzfahrplanV4Regeltest::integer($request['durationSeconds'] ?? null, 'Lokale Testdauer', 30, 1800)); + $s = ['kind' => 'local_control_trial_consent', 'enabled' => true, 'legacyEnabled' => false, + 'sessionId' => $session, 'installationId' => $this->ReadAttributeString('LizenzInstallationID'), + 'managerId' => $this->InstanceID, 'batteryInstanceId' => $batteryID, 'assetId' => $a['assetId'], + 'expiresAt' => $until, 'maxChargeW' => NetzfahrplanV4Regeltest::number($request['maxChargeW'] ?? null, 'Testladegrenze', 1, 5000), + 'maxDischargeW' => NetzfahrplanV4Regeltest::number($request['maxDischargeW'] ?? null, 'Testentladegrenze', 1, 5000), + 'actuatorWatchdogEvidenceId' => IPS_GetProperty($batteryID, 'NetzfahrplanV4GeraeteWatchdogNachweis'), + 'sequence' => 1, 'lastWallClockAt' => $now, + 'monotonicDeadline' => hrtime(true) / 1e9 + $until - $now]; + // Validate everything BEFORE arming either module. This rejects all normal shadow replies. + NetzfahrplanV4Regeltest::befehl($envelope['response'], $op, $actual, $s, $now, $envelope['receivedAt']); + $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $session); + IPS_RequestAction($batteryID, 'NetzfahrplanV4TestStart', json_encode($s, JSON_THROW_ON_ERROR)); + $armedBattery = $batteryID; + $this->SetBuffer('V4ManagerTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + $this->SetTimerInterval('NetzfahrplanV4TestTick', 2000); + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode([ + 'status' => 'armed_local_trial', 'sessionId' => $session, 'expiresAt' => $until, + 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->SetBuffer('V4ManagerTrialSession', '{}');$this->SetTimerInterval('NetzfahrplanV4TestTick', 0); + if ($armedBattery !== null) IPS_RequestAction($armedBattery, 'NetzfahrplanV4TestStop', 0); + throw $e; + } finally { IPS_SemaphoreLeave($lock); } + } + + /** Revocation precedes I/O; if the device call fails its own ten-second lease still expires. */ + private function v4ManagerTestAbbrechen(string $reason): void + { + $s = $this->v4ManagerTestSession(); + $this->SetBuffer('V4ManagerTrialSession', '{}');$this->SetTimerInterval('NetzfahrplanV4TestTick', 0); + if (!$s) return; + $result = ['status' => 'stopped_fallback_pending', 'reason' => $reason, + 'sessionId' => $s['sessionId'], 'physicalResponseVerified' => false]; + // Reissue the next ordinary target even if numerically equal to a previously cached value. + $this->WriteAttributeString('LetzteSollwerte', '{}'); + try { + IPS_RequestAction($s['batteryInstanceId'], 'NetzfahrplanV4TestStop', 0); + $result['controllerStopAccepted'] = true; + } catch (Throwable $e) { + $result['controllerStopAccepted'] = false;$result['reason'] .= '; Geraete-Abbruch fehlgeschlagen, Watchdog pruefen.'; + } + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode($result, JSON_THROW_ON_ERROR)); + } + + public function V4ManagerTestStoppen(): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Regelung beschaeftigt; Stop erneut anfordern.'); + try { $this->v4ManagerTestAbbrechen('Lokaler Testabbruch'); } + finally { IPS_SemaphoreLeave($lock); } + $this->regeln(true); // fresh ordinary command; never restore a cached pre-trial target + } + + public function GetV4ManagerTestStatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4RegeltestStatus'); + } + + /** Called under the existing regulator's serialization, after allocation but before writes. */ + private function v4ManagerTestUebergabe(array &$allocation, array $consumers, bool $synchronized): void + { + $s = $this->v4ManagerTestSession(); + if (!$s) return; + try { + $this->v4ManagerTestBedingungen();$now = time(); + if (!$synchronized || $now >= $s['expiresAt'] || hrtime(true) / 1e9 >= $s['monotonicDeadline']) { + throw new RuntimeException('Testende oder Verbraucher nicht synchronisiert.'); + } + [$op, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + // Other consumers stay under the existing controller. Compensate their predicted + // change once; no double-counting of the battery's already flowing actual power. + $otherDelta = 0.0;$found = false; + foreach ($consumers as $consumer) { + $id = $consumer['InstanzID']; + if ($id === $s['batteryInstanceId']) { $found = true;continue; } + if (!array_key_exists($id, $allocation['Sollwerte'])) continue; + $d = $consumer['Daten']; + if (($d['Leistungsquelle'] ?? null) !== Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) throw new RuntimeException('Andere Verbraucherleistung nicht gemessen.'); + $otherDelta += NetzfahrplanV4Regeltest::number($allocation['Sollwerte'][$id], 'Anderer Sollwert', -1e9, 1e9) + - NetzfahrplanV4Regeltest::number($d['Istleistung_W'] ?? null, 'Andere Istleistung', -1e9, 1e9); + } + if (!$found) throw new RuntimeException('Testbatterie nicht mehr in aktiver Zuordnung.'); + $op['gridW'] += $otherDelta; + if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { + throw new RuntimeException('Kein gueltiger empfangener Testkontext.'); + } + $command = NetzfahrplanV4Regeltest::befehl($envelope['response'], $op, $actual, $s, $now, $envelope['receivedAt']); + // A controller-side refusal stops the session; never downgrade it to an ordinary target. + IPS_RequestAction($s['batteryInstanceId'], 'NetzfahrplanV4TestCommand', json_encode($command, JSON_THROW_ON_ERROR)); + $s['sequence']++;$s['lastWallClockAt'] = $now; + $this->SetBuffer('V4ManagerTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + unset($allocation['Sollwerte'][$s['batteryInstanceId']]); // one owner of this actuator + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode([ + 'status' => 'controller_command_sent', 'sessionId' => $s['sessionId'], + 'sourceShadowPlanId' => $command['sourceShadowPlanId'], 'command' => $command, + 'otherConsumerPredictedDeltaW' => $otherDelta, 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->v4ManagerTestAbbrechen($e->getMessage()); + // Stop writes zero and next NEW ordinary allocation runs on this same pass. + } + } +} diff --git a/libs/NetzfahrplanV4Regeltest.php b/libs/NetzfahrplanV4Regeltest.php new file mode 100644 index 0000000..a377053 --- /dev/null +++ b/libs/NetzfahrplanV4Regeltest.php @@ -0,0 +1,168 @@ + $max) { + throw new InvalidArgumentException($name . ': ungueltiger Wert.'); + } + return (float) $value; + } + + public static function uuid($value): string + { + if (!is_string($value) || !preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $value)) { + throw new InvalidArgumentException('Eindeutige Testkennung erforderlich.'); + } + return $value; + } + + public static function integer($value, string $name, int $min, int $max): int + { + if (!is_int($value) || $value < $min || $value > $max) { + throw new InvalidArgumentException($name . ': Ganzzahl erforderlich.'); + } + return $value; + } + + /** Validated server authorization plus independent LOCAL consent and physical mapping. */ + public static function befehl(array $response, array $operation, array $actual, array $local, int $now, int $receivedAt): array + { + if (($local['enabled'] ?? null) !== true || ($local['legacyEnabled'] ?? null) !== false) { + throw new InvalidArgumentException('Keine lokale Testfreigabe oder konkurrierender Fahrplan.'); + } + $installation = self::uuid($local['installationId'] ?? null); + $session = self::uuid($local['sessionId'] ?? null); + $manager = self::integer($local['managerId'] ?? null, 'Manager', 1, 99999); + $controller = self::integer($local['batteryInstanceId'] ?? null, 'Batterieinstanz', 1, 99999); + $end = self::integer($local['expiresAt'] ?? null, 'Lokales Testende', $now + 1, $now + self::MAX_TEST_SECONDS); + if (($local['lastWallClockAt'] ?? $now) > $now) throw new InvalidArgumentException('Systemzeit rueckwaerts; Test abbrechen.'); + $checked = NetzfahrplanV4Planpruefung::pruefen($response, $installation, $operation, $now, $receivedAt); + $plan = $checked['plan'];$a = $response['controlledTrial'] ?? null; + if (!is_array($a) || ($a['version'] ?? null) !== 1 || ($a['kind'] ?? null) !== 'controlled_trial_authority' + || ($a['sourcePlanRemainsShadow'] ?? null) !== true || ($a['installationId'] ?? null) !== $installation + || ($a['sessionId'] ?? null) !== $session || ($a['managerId'] ?? null) !== $manager + || ($a['batteryInstanceId'] ?? null) !== $controller || ($a['sourceShadowPlanId'] ?? null) !== $plan['planId'] + || ($a['revision'] ?? null) !== $plan['configRevision'] || ($a['family'] ?? null) !== $plan['sourceFamily']) { + throw new InvalidArgumentException('Keine passende separate Server-Testfreigabe.'); + } + $issued = NetzfahrplanV4Planpruefung::zeit($a['issuedAt'] ?? null); + $expires = NetzfahrplanV4Planpruefung::zeit($a['expiresAt'] ?? null); + $authorityAt = NetzfahrplanV4Planpruefung::zeit($a['checkedAt'] ?? null); + $authorityUntil = NetzfahrplanV4Planpruefung::zeit($a['validUntil'] ?? null); + if ($issued > $now || $expires <= $now || $expires - $issued > self::MAX_TEST_SECONDS + || $end > $expires || $authorityAt > $now + 2 || $authorityUntil <= $now + || $authorityUntil - $authorityAt > 90 || $authorityUntil > $expires) { + throw new InvalidArgumentException('Testfreigabe abgelaufen oder ungueltig.'); + } + $quality = $plan['inputQuality'] ?? []; + $evidence = $quality['accountingEvidenceId'] ?? null; + if (($quality['loadBasis'] ?? null) !== 'base_load' || !is_string($evidence) || strlen($evidence) < 8 + || $evidence !== ($a['accountingEvidenceId'] ?? null) + || !is_string($a['actuatorWatchdogEvidenceId'] ?? null) || strlen($a['actuatorWatchdogEvidenceId']) < 8 + || $a['actuatorWatchdogEvidenceId'] !== ($local['actuatorWatchdogEvidenceId'] ?? null)) { + throw new InvalidArgumentException('Anlagenbilanz oder Geraete-Watchdog nicht nachgewiesen.'); + } + if (($plan['peakCostIsEstimate'] ?? true) && ($a['acceptEstimatedPeak'] ?? null) !== true) { + throw new InvalidArgumentException('Geschaetzte Peakbasis nicht fuer Test akzeptiert.'); + } + if (count($operation['batteries']) !== 1 || $operation['batteries'][0]['id'] !== ($a['assetId'] ?? null) + || ($local['assetId'] ?? null) !== $a['assetId']) { + throw new InvalidArgumentException('Test braucht eindeutige Batteriezuordnung.'); + } + if (($a['controlContext'] ?? null) != $plan['controlContext']) { + throw new InvalidArgumentException('Testkonfiguration stimmt nicht mehr.'); + } + $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, $now); + $charge = min(self::number($a['maxChargeW'] ?? null, 'Server-Ladegrenze', 1, self::MAX_TEST_POWER_W), + self::number($local['maxChargeW'] ?? null, 'Lokale Ladegrenze', 1, self::MAX_TEST_POWER_W)); + $discharge = min(self::number($a['maxDischargeW'] ?? null, 'Server-Entladegrenze', 1, self::MAX_TEST_POWER_W), + self::number($local['maxDischargeW'] ?? null, 'Lokale Entladegrenze', 1, self::MAX_TEST_POWER_W)); + $watts = (int) (max(-$discharge, min($charge, $preview['previewBatteryW']))); // truncate towards zero + $residual = $preview['actualGridW'] - $preview['actualBatteryW']; + $grid = $residual + $watts; + $p = $checked['point']; + if (($p['importLimitW'] !== null && $grid > $p['importLimitW'] + 1) + || ($p['exportLimitW'] !== null && -$grid > $p['exportLimitW'] + 1)) { + throw new InvalidArgumentException('Testleistungsgrenze reicht fuer Netzgrenze nicht; lokale Regelung hat Vorrang.'); + } + return ['kind' => 'controlled_trial_command', 'version' => 1, + 'sessionId' => $session, 'installationId' => $installation, 'assetId' => $a['assetId'], + 'managerId' => $manager, 'batteryInstanceId' => $controller, + 'sequence' => self::integer($local['sequence'] ?? null, 'Befehlsfolge', 1, PHP_INT_MAX), + 'issuedAt' => $now, 'expiresAt' => min($now + self::COMMAND_TTL_SECONDS, $authorityUntil, $end, + NetzfahrplanV4Planpruefung::zeit($p['validUntil'])), + 'watts' => $watts, 'maxChargeW' => (int) $charge, 'maxDischargeW' => (int) $discharge, + 'sourceShadowPlanId' => $plan['planId'], 'sourceStep' => $p['time'], + 'revision' => $plan['configRevision'], 'expectedGridW' => $grid, + 'executionMeaning' => 'separate_local_control_trial_not_shadow_plan_execution']; + } + + /** Independently checked by the battery before any physical register call. */ + public static function pruefeBatteriebefehl(array $command, array $session, int $now, float $monotonicNow): void + { + if (($command['kind'] ?? null) !== 'controlled_trial_command' || ($command['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Vorschau ist kein Stellbefehl.'); + } + foreach (['sessionId', 'installationId', 'assetId', 'managerId', 'batteryInstanceId'] as $key) { + if (!array_key_exists($key, $session) || ($command[$key] ?? null) !== $session[$key]) { + throw new InvalidArgumentException('Befehl gehoert nicht zur lokal angemeldeten Testsitzung.'); + } + } + if (($session['active'] ?? null) !== true || $now >= ($session['expiresAt'] ?? 0) + || $monotonicNow >= ($session['monotonicDeadline'] ?? 0) + || $now < ($session['lastWallClockAt'] ?? $now)) { + throw new InvalidArgumentException('Lokale Testsitzung abgelaufen/abgebrochen.'); + } + $sent = self::integer($command['issuedAt'] ?? null, 'Befehlszeit', $now - self::COMMAND_TTL_SECONDS, $now); + $end = self::integer($command['expiresAt'] ?? null, 'Befehlsende', $now + 1, $sent + self::COMMAND_TTL_SECONDS); + if ($end > $session['expiresAt']) throw new InvalidArgumentException('Befehl verlaengert Test.'); + $seq = self::integer($command['sequence'] ?? null, 'Befehlsfolge', 1, PHP_INT_MAX); + if ($seq <= ($session['lastSequence'] ?? 0)) throw new InvalidArgumentException('Alter/doppelter Befehl verworfen.'); + $w = self::integer($command['watts'] ?? null, 'Sollwert W', -self::MAX_TEST_POWER_W, self::MAX_TEST_POWER_W); + $c = self::number($command['maxChargeW'] ?? null, 'Befehl-Ladegrenze', 1, self::MAX_TEST_POWER_W); + $d = self::number($command['maxDischargeW'] ?? null, 'Befehl-Entladegrenze', 1, self::MAX_TEST_POWER_W); + if ($c > $session['maxChargeW'] || $d > $session['maxDischargeW'] || $w > $c || $w < -$d) { + throw new InvalidArgumentException('Befehl ueberschreitet lokale Testgrenze.'); + } + self::uuid($command['sourceShadowPlanId'] ?? null); + } + + /** Re-evaluated on every battery update; no reliance on the manager's earlier clipping. */ + public static function batterieZiel(array $command, array $session, array $m, array $policy, int $now, float $monotonicNow): int + { + if ($now >= $command['expiresAt'] || $now < $session['lastWallClockAt'] + || $monotonicNow >= $session['commandMonotonicDeadline'] || $monotonicNow >= $session['monotonicDeadline']) { + throw new InvalidArgumentException('Befehls-Watchdog abgelaufen.'); + } + if (($m['Gueltig'] ?? null) !== true || ($policy['available'] ?? null) !== true) { + throw new InvalidArgumentException('Batterie nicht verfuegbar oder Messwerte ungueltig.'); + } + $soc = self::number($m['Ladezustand'] ?? null, 'SOC', 0, 100); + $reserve = self::number($policy['reserve'] ?? null, 'Reserve', 0, 100); + $minimum = self::number($policy['minimum'] ?? null, 'Minimum', 0, $reserve); + if ($soc < $minimum) throw new InvalidArgumentException('SOC unter technischer Untergrenze.'); + $charge = min($session['maxChargeW'], $command['maxChargeW'], self::number($m['MaxLaden'] ?? null, 'Max Laden', 0, 1e9)); + $discharge = min($session['maxDischargeW'], $command['maxDischargeW'], self::number($m['MaxEntladen'] ?? null, 'Max Entladen', 0, 1e9)); + if ($soc >= 99) $charge = 0; // existing battery hardware policy is more conservative than planner max 100 + if ($soc <= $reserve || ($policy['blocked'] ?? true)) $discharge = 0; + if (($soc <= $reserve || ($policy['blocked'] ?? true)) && $policy['rechargeLimitW'] !== null) { + $charge = min($charge, self::number($policy['rechargeLimitW'], 'Nachladegrenze', 0, 1e9)); + } + return (int) max(-$discharge, min($charge, $command['watts'])); + } +} diff --git a/tests/NetzfahrplanV4RegeltestTest.php b/tests/NetzfahrplanV4RegeltestTest.php new file mode 100644 index 0000000..a49725f --- /dev/null +++ b/tests/NetzfahrplanV4RegeltestTest.php @@ -0,0 +1,24 @@ + ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + self::assertIsResource($process); + fclose($pipes[0]); + $out = stream_get_contents($pipes[1]);$err = stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0, proc_close($process), $out . $err); + self::assertSame('', $err); + self::assertStringContainsString('TOTAL 59 checks passed.', $out); + } +} diff --git a/tests/V4ControlTrial/additional_checks.php b/tests/V4ControlTrial/additional_checks.php new file mode 100644 index 0000000..3669c8b --- /dev/null +++ b/tests/V4ControlTrial/additional_checks.php @@ -0,0 +1,30 @@ +op['gridW']=3000.;tick($m); +check(end($b->writes)===-3000,'discharge sign and current net load compensation'); +$b->measurements['Ladezustand']=15.;$b->watchdog();check(end($b->writes)===0,'reserve stops an active discharge independently'); +[$m,$b,$s]=fixture();start($m,$s);$m->op['gridW']=3000.;tick($m);$b->state['HystereseAktiv']=true;$b->watchdog(); +check(end($b->writes)===0,'local discharge hysteresis overrides previous command'); +[$m,$b,$s]=fixture();start($m,$s);$m->op['gridW']=3000.;tick($m);$b->measurements['MaxEntladen']=500.;$b->watchdog(); +check(end($b->writes)===-500,'discharge availability decrease is immediate'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=2.;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'technical minimum violation aborts rather than inventing SOC'); +[$m,$b,$s]=fixture();$s['maxChargeW']=1000.;start($m,$s);tick($m); +check(end($b->writes)===1000,'independent local cap can be stricter than server cap'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=500;$b->attrs['AenderungFreigabeAb']=$clock+4;tick($m); +check(end($b->writes)===500,'switch lock holds power increase'); +advance(5);tick($m);check(end($b->writes)===3000,'power increase only after switching lock'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=500;$b->attrs['AenderungFreigabeAb']=$clock+4;$m->op['gridW']=3000.;tick($m); +check(end($b->writes)===0,'sign reversal first passes zero during switching lock'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=3000;$b->attrs['AenderungFreigabeAb']=$clock+4;$b->measurements['MaxLaden']=100.;tick($m); +check(end($b->writes)===100,'safety reduction not delayed by switching lock'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=15.;$b->props['NachladenMitMaximalleistung']=false;$b->props['MaximaleNachladeleistung']=200;$b->watchdog(); +check(end($b->writes)===200,'configured reserve-recovery charging limit remains enforced'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$m->licensed=false;tick($m); +check(!$m->active()&&!$b->active()&&end($b->writes)===0,'license withdrawal revokes both software sessions'); +[$m,$b,$s]=fixture();$s['durationSeconds']=30;start($m,$s);tick($m);advance(30);tick($m); +check(!$m->active()&&!$b->active()&&end($b->writes)===0,'local shorter test expiration respected'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$m->vars['Aktiv']=false;tick($m); +check(!$m->active()&&!$b->active(),'manual manager disable ends trial'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->vars['Aktiv']=false;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'manual battery disable independently ends trial'); diff --git a/tests/V4ControlTrial/checks.php b/tests/V4ControlTrial/checks.php new file mode 100644 index 0000000..eb0d903 --- /dev/null +++ b/tests/V4ControlTrial/checks.php @@ -0,0 +1,161 @@ +$GLOBALS['objects'][$id]->ready?102:201, + 'ModuleInfo'=>['ModuleID'=>$id===44234?'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}':'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']]; } + function IPS_GetProperty($id,$key) { return $GLOBALS['objects'][$id]->props[$key]; } + function IPS_RequestAction($id,$key,$value) { + $GLOBALS['calls'][]=[$id,$key];$b=$GLOBALS['objects'][$id]; + if ($key==='NetzfahrplanV4TestStart') $b->V4BatterieTestStarten($value); + elseif ($key==='NetzfahrplanV4TestCommand') $b->V4BatterieTestBefehl($value); + elseif ($key==='NetzfahrplanV4TestStop') $b->V4BatterieTestStoppen(); + else throw new \RuntimeException('Unexpected external action in offline test'); + } + trait StoreHarness { + public array $props=[],$attrs=[],$buf=[],$timers=[],$vars=['Aktiv'=>true]; + public bool $ready=true; + private function RegisterPropertyBoolean($k,$v) { $this->props[$k]??=$v; } + private function RegisterPropertyString($k,$v) { $this->props[$k]??=$v; } + private function RegisterAttributeString($k,$v) { $this->attrs[$k]??=$v; } + private function RegisterTimer($k,$ms,$script) { $this->timers[$k]=$ms; } + private function SetTimerInterval($k,$ms) { $this->timers[$k]=$ms; } + private function ReadPropertyBoolean($k) { return $this->props[$k]; } + private function ReadPropertyString($k) { return $this->props[$k]; } + private function ReadPropertyInteger($k) { return $this->props[$k]; } + private function ReadPropertyFloat($k) { return (float)$this->props[$k]; } + private function ReadAttributeString($k) { return $this->attrs[$k]??''; } + private function ReadAttributeInteger($k) { return $this->attrs[$k]??0; } + private function WriteAttributeString($k,$v) { $this->attrs[$k]=$v; } + private function GetBuffer($k) { return $this->buf[$k]??''; } + private function SetBuffer($k,$v) { $this->buf[$k]=$v; } + private function GetValue($k) { return $this->vars[$k]; } + } + class BatteryHarness { + use StoreHarness, BatterieNetzfahrplanV4TestTrait; + public int $InstanceID=44234; + public array $state=['HystereseAktiv'=>false],$writes=[],$measurements=[]; + public bool $failRegisters=false; + public function __construct() { + $this->props=['Batteriemanagement'=>2,'ReserveLadezustand'=>15.,'MindestLadezustand'=>3., + 'NachladenMitMaximalleistung'=>true,'MaximaleNachladeleistung'=>1000, + 'LadezustandVariableID'=>1001,'IstleistungVariableID'=>1002]; + $this->measurements=['Gueltig'=>true,'Ladezustand'=>50.,'MaxLaden'=>5000.,'MaxEntladen'=>4000.]; + $this->attrs['LetzterSollwert']=0;$this->attrs['AenderungFreigabeAb']=0; + $this->registriereV4BatterieTest(); + } + private function zugeordneteManagerIDs() { return [17004]; } + private function setzeZustand($k,$v) { $this->state[$k]=$v; } + private function leseZustand($k) { return $this->state[$k]??false; } + private function schreibeRegister($w,$force) { if ($this->failRegisters) throw new \RuntimeException('synthetic register failure');$this->writes[]=$w;$this->attrs['LetzterSollwert']=$w; } + private function aktualisiereIntern($notify) { $w=$this->v4BatterieTestZiel($this->measurements,$this->vars['Aktiv'],time());if ($w!==null) $this->schreibeRegister($w,false); } + public function watchdog() { $this->aktualisiereIntern(true); } + public function active() { return $this->v4BatterieTestAktiv(); } + } + class ManagerHarness { + use StoreHarness, ManagerNetzfahrplanV4TestTrait; + public int $InstanceID=17004; + public array $op=[],$actual=['b'=>0.];public int $fallback=0; + public bool $licensed=true; + public function __construct($response,$op) { + $this->props=['NetzfahrplanV4SchattenAktiv'=>true,'NetzfahrplanV4EmpfangAktiv'=>true,'NetzfahrplanAktiv'=>false, + 'AnlagenBatterien'=>json_encode([['ID'=>'b','SOCVariableID'=>1001,'LeistungVariableID'=>1002]])]; + $this->op=$op;$this->attrs=['LizenzInstallationID'=>$response['installationId'], + 'NetzfahrplanV4Empfang'=>json_encode(['response'=>$response,'receivedAt'=>time()])]; + $this->registriereV4ManagerTest(); + } + private function berechtigungLizenziert($k) { return $this->licensed; } + private function actieveDummy() {} + private function aktiveVerbraucherIDs() { return [44234]; } + private function netzfahrplanV4LokaleVorschauwerte() { return [$this->op,$this->actual]; } + private function regeln($renew) { $this->fallback++; } + public function tick(&$allocation,$consumers=null,$sync=true) { + $this->v4ManagerTestUebergabe($allocation,$consumers??[['InstanzID'=>44234,'Daten'=>[]]],$sync); + } + public function active() { return (bool)$this->v4ManagerTestSession(); } + public function command() { return json_decode($this->attrs['NetzfahrplanV4RegeltestStatus'],true)['command']??null; } + } +} +namespace { + require_once __DIR__.'/../V4Receiver/fixture.php'; + use Belevo\EnelixEMS\NetzfahrplanV4Regeltest as Gate; + use Belevo\EnelixEMS\BatteryHarness; + use Belevo\EnelixEMS\ManagerHarness; + $count=0; + function check($ok,$name) { global $count;if (!$ok) throw new RuntimeException('FAIL '.$name);$count++;echo "PASS $name\n"; } + function rejects(callable $call,$name) { try {$call();}catch(Throwable $e){check(true,$name);return;}throw new RuntimeException('FAIL did not reject '.$name); } + function fixture($permissions=true,$serverAuthority=true): array { + [$r,$id,$op,$now]=receiverFixture();$GLOBALS['clock']=$now;$GLOBALS['mono']=1000.;$GLOBALS['locks']=[];$GLOBALS['calls']=[]; + $r['plan']['inputQuality']=['loadBasis'=>'base_load','accountingEvidenceId'=>'synthetic-meter-boundary-v1']; + $r['plan']['peakCostIsEstimate']=true; + $a=['kind'=>'controlled_trial_authority','version'=>1,'sourcePlanRemainsShadow'=>true, + 'sessionId'=>'20000000-0000-4000-8000-000000000001','installationId'=>$id, + 'managerId'=>17004,'batteryInstanceId'=>44234,'assetId'=>'b', + 'sourceShadowPlanId'=>$r['plan']['planId'],'revision'=>1,'family'=>'3', + 'issuedAt'=>gmdate('c',$now),'expiresAt'=>gmdate('c',$now+300),'checkedAt'=>gmdate('c',$now),'validUntil'=>gmdate('c',$now+90), + 'maxChargeW'=>5000.,'maxDischargeW'=>5000.,'actuatorWatchdogEvidenceId'=>'synthetic-hardware-watchdog', + 'accountingEvidenceId'=>'synthetic-meter-boundary-v1','acceptEstimatedPeak'=>true,'controlContext'=>$r['plan']['controlContext']]; + $r['controlledTrial']=$serverAuthority?$a:null; + $m=new ManagerHarness($r,$op);$b=new BatteryHarness();$GLOBALS['objects']=[17004=>$m,44234=>$b]; + $m->props['NetzfahrplanV4RegeltestErlaubt']=$permissions;$b->props['NetzfahrplanV4RegeltestErlaubt']=$permissions; + $b->props['NetzfahrplanV4GeraeteWatchdogNachweis']='synthetic-hardware-watchdog'; + $start=['confirmation'=>'START_BOUNDED_V4_CONTROL_TRIAL','sessionId'=>$a['sessionId'],'durationSeconds'=>300,'maxChargeW'=>5000.,'maxDischargeW'=>5000.]; + return [$m,$b,$start,$r,$op,$now]; + } + function start($m,$s) {$m->V4ManagerTestStarten(json_encode($s));} + function tick($m) {$a=['Sollwerte'=>[44234=>999]];$m->tick($a);return $a;} + function advance($s) {$GLOBALS['clock']+=$s;$GLOBALS['mono']+=$s;} + + [$m,$b,$s]=fixture(false);rejects(fn()=>start($m,$s),'default local permission blocks');check($b->writes===[],'no register writes without consent'); + [$m,$b,$s]=fixture(true,false);rejects(fn()=>start($m,$s),'ordinary shadow response never authorizes');check($calls===[],'no driver call for preview alone'); + [$m,$b,$s]=fixture();$b->props['NetzfahrplanV4RegeltestErlaubt']=false;rejects(fn()=>start($m,$s),'second battery gate independent'); + [$m,$b,$s]=fixture();$m->props['NetzfahrplanAktiv']=true;rejects(fn()=>start($m,$s),'old schedule excludes new controller'); + [$m,$b,$s,$r]=fixture();$r['plan']['inputQuality']['loadBasis']='house_total';$m->attrs['NetzfahrplanV4Empfang']=json_encode(['response'=>$r,'receivedAt'=>$clock]);rejects(fn()=>start($m,$s),'unverified aggregate accounting blocks test'); + [$m,$b,$s]=fixture();$b->props['NetzfahrplanV4GeraeteWatchdogNachweis']='';rejects(fn()=>start($m,$s),'unknown device watchdog blocks test'); + [$m,$b,$s]=fixture();$m->ready=false;rejects(fn()=>start($m,$s),'unready manager blocks test'); + [$m,$b,$s]=fixture();$s['maxChargeW']=5001;rejects(fn()=>start($m,$s),'test limit above 5kW refused'); + [$m,$b,$s]=fixture();start($m,$s);check($m->active()&&$b->active(),'both sessions armed');check($b->writes===[],'arm itself sends no register command'); + $allocation=tick($m);check(!isset($allocation['Sollwerte'][44234]),'one actuator owner removes ordinary target'); + check(end($b->writes)===3000,'actual trait chain delivers bounded 3kW command'); + $c=$m->command();check($c['kind']==='controlled_trial_command'&&$c['expiresAt']===$clock+10,'independent command type and ten-second lease'); + check(json_decode($b->GetV4BatterieTestStatus(),true)['physicalResponseVerified']===false,'controller acceptance is not physical proof'); + advance(2);tick($m);check($m->command()['sequence']===2,'monotonic command sequence'); + rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'replayed command refused');check(!$b->active()&&end($b->writes)===0,'replay aborts and releases output'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);advance(11);$b->watchdog();check(!$b->active()&&end($b->writes)===0,'watchdog expiry drops to zero'); + [$m,$b,$s]=fixture();start($m,$s);advance(11);$b->watchdog();check(!$b->active()&&end($b->writes)===0,'missing first command expires'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$mono+=11;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'monotonic watchdog works without wall clock advance'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$clock-=1;$b->watchdog();check(!$b->active(),'backwards wall clock aborts'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->V4ManagerTestStoppen();check(!$m->active()&&!$b->active()&&end($b->writes)===0&&$m->fallback===1,'manual stop releases before ordinary fallback'); + rejects(fn()=>start($m,$s),'stopped local session cannot resurrect'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->buf=[];$b->V4BatterieTestStoppen();check(end($b->writes)===0&&!$b->active(),'runtime restart clears persisted command marker'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Gueltig']=false;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'invalid device measurements release command'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['MaxLaden']=1000.;$b->watchdog();check(end($b->writes)===1000,'device availability reduction rechecked'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=99.;$b->watchdog();check(end($b->writes)===0,'full battery cannot keep charging'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->props['NetzfahrplanV4RegeltestErlaubt']=false;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'battery consent withdrawn aborts'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->props['NetzfahrplanV4RegeltestErlaubt']=false;$a=tick($m);check(!$m->active()&&isset($a['Sollwerte'][44234])&&end($b->writes)===0,'manager consent withdrawn resumes ordinary allocation'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->attrs['NetzfahrplanV4Empfang']='{}';$a=tick($m);check(!$m->active()&&end($b->writes)===0,'invalidated receiver cache ends trial'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$a=['Sollwerte'=>[44234=>123]];$m->tick($a,null,false);check(!$m->active()&&end($b->writes)===0,'unsynchronized consumers abort'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->failRegisters=true;advance(2);tick($m);check(!$m->active()&&!$b->active(),'register failure revokes both sessions');check(json_decode($m->GetV4ManagerTestStatus(),true)['controllerStopAccepted']===false,'stop failure not disguised as safe'); + [$m,$b,$s]=fixture();start($m,$s);$a=['Sollwerte'=>[44234=>999,33333=>1000]];$m->tick($a,[['InstanzID'=>44234,'Daten'=>[]],['InstanzID'=>33333,'Daten'=>['Leistungsquelle'=>'measured','Istleistung_W'=>0]]]); + check(end($b->writes)===2000&&$a['Sollwerte'][33333]===1000,'other consumer delta counted once without takeover'); + [$m,$b,$s]=fixture();start($m,$s);$m->actual['b']=2000.;$m->op['gridW']=-1000.;tick($m);check(end($b->writes)===3000,'command is total battery power not additive'); + [$m,$b,$s]=fixture();start($m,$s);$m->actual['b']=0.;$m->op['gridW']=-40000.;tick($m);check(!$m->active(),'test cap cannot override hard export limit'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$clock+=91;$mono+=91;tick($m);check(!$m->active()&&end($b->writes)===0,'expired server observation cancels even with future prices'); + [$m,$b,$s,$r,$op,$now]=fixture();$local=['enabled'=>true,'legacyEnabled'=>false,'installationId'=>$r['installationId'],'sessionId'=>$s['sessionId'],'managerId'=>17004,'batteryInstanceId'=>44234,'assetId'=>'b','expiresAt'=>$now+300,'maxChargeW'=>5000,'maxDischargeW'=>5000,'sequence'=>1,'actuatorWatchdogEvidenceId'=>'synthetic-hardware-watchdog']; + foreach (['sessionId','installationId','sourceShadowPlanId'] as $key) { + $bad=$r;$bad['controlledTrial'][$key]='ffffffff-ffff-4fff-8fff-ffffffffffff';rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'wrong authority binding '.$key); + } + $bad=$r;$bad['controlledTrial']['validUntil']=gmdate('c',$now+120);rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'oversized server authorization lease'); + $bad=$r;$bad['controlledTrial']['acceptEstimatedPeak']=false;rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'peak uncertainty needs explicit acceptance'); + require __DIR__ . '/additional_checks.php'; + echo "TOTAL $count checks passed. Mocked IPS, clocks and register writes only.\n"; +} From e0e52d2454c24d462d84414a8cd4db47b26eab1d Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 14:23:19 +0000 Subject: [PATCH 06/22] feat(accounting): separate physical storage and SDL with source-age validation --- docs/netplan-v4-accounting.md | 99 +++++++++++++++++++ libs/NetzfahrplanV4Bilanzierung.php | 120 +++++++++++++++++++++++ tests/NetzfahrplanV4BilanzierungTest.php | 16 +++ tests/V4Accounting/checks.php | 56 +++++++++++ 4 files changed, 291 insertions(+) create mode 100644 docs/netplan-v4-accounting.md create mode 100644 libs/NetzfahrplanV4Bilanzierung.php create mode 100644 tests/NetzfahrplanV4BilanzierungTest.php create mode 100644 tests/V4Accounting/checks.php diff --git a/docs/netplan-v4-accounting.md b/docs/netplan-v4-accounting.md new file mode 100644 index 0000000..f921543 --- /dev/null +++ b/docs/netplan-v4-accounting.md @@ -0,0 +1,99 @@ +# V4: getrennte Last-/Batterie-/SDL-Bilanz (Kandidat, keine Stellfreigabe) + +## Status 2026-10-02 + +`NetzfahrplanV4Bilanzierung` ist ein reiner Rechen-/Validierungsbaustein. Noch keine +Einbindung in die laufende Telemetrie, den Trainer oder den Kontrolltest. Er ruft +keine IPS-, Netzwerk-, Archiv- oder Stellfunktionen auf. Ergebnis ist immer +`controlEligible=false`, auch bei geschlossener mathematischer Bilanz. + +34 isolierte PHP-Pruefungen wurden mit der gleichen SHA256-Datei auf iot-symcon01 +bestanden. Das ist kein Nachweis fuer die reale Hardware-/Messgrenzenzuordnung +und kein vollstaendiger PHPUnit- oder Symcon-Kernel-Test. + +## Gefundener Unterschied in Lihrenmoos + +Installierter Manager `Manager/module.php::leseMessleistungen()` verwendet: + + Haus = PV + Netz - Summe der konfigurierten Batterie-Istleistungen + +Konfiguriert ist hier die virtuelle EV-Leistung 52020, nicht die gesamte physische +Batterieleistung. Daher bleibt der externe SDL-Anteil und eine allfaellige +Abweichung der virtuellen Aufteilung im an den Prognosedienst gesendeten Hauswert. +`sendePrognoseTelemetrie()` sendet genau diesen errechneten Wert. Die parallel +existierenden Hausvariablen 46714 und 40713 sind NICHT automatisch diese Quelle. + +Der bisherige Gateway `Bat_EV_SDL_V4` liest die physische Batterieleistung aus +47725, 35724 und 36380 und invertiert diese Vorzeichen. EV und SDL sind daraus +abgeleitete, separat gefilterte virtuelle Konten, keine zwei unabhaengigen Zaehler. +Die Filter koennen alte Teilwerte halten. `UpdateActualPowerSplit` aktualisiert die +Ausgabevariablen trotzdem. Ein neuer Ausgabezeitstempel beweist deshalb nicht die +Frische aller darunter liegenden Originalmessungen. + +## Kandidat fuer die korrekte Bilanz + +Mit einheitlicher Messgrenze und Vorzeichen Batterie+ = Laden, Netz+ = Bezug: + + Verbraucherlast = Netz + PV - gesamte physische Batterieleistung + Grundlast = Verbraucherlast - separat geplante flexible Verbraucher + Zuordnungsrest = physische Batterie - virtuell steuerbare Batterie - externe virtuelle Konten + Externer Netzeffekt = externe virtuelle Konten + Zuordnungsrest + Netz = Grundlast + flexible Last + steuerbare Batterie + externer Netzeffekt - PV + +Der Zuordnungsrest wird nicht erneut als Hausverbrauch trainiert, sondern separat +angezeigt. Eine grosse Abweichung sperrt die Datenqualitaet. Gegenlaeufige virtuelle +Konten (z.B. EV +42 kW, SDL -40 kW, physisch +2 kW) bleiben korrekt getrennt. +Spaetere Ladestationen/Boiler sind ueber `flexible_load` erweiterbar. Solange sie +nicht separat geplant werden, verbleiben sie in der nicht separat steuerbaren Last. + +Keine Messgroesse wird doppelt subtrahiert. Ein negatives Grundlastergebnis wird +nicht heimlich auf null begrenzt. Numerische Nullen sind echte Werte; fehlende, +nicht endliche oder umtypisierte Werte sind keine Nullen. + +## Originalzeit und Datenqualitaet + +Jede Quelle hat explizite Variable/Parent/Ident, Faktor, maximales Alter und optionale +Ursprungsabhaengigkeiten. Alle Quellen werden zweimal gelesen. Wertewechsel waehrend +der Aufnahme werden abgewiesen; Zeitstempel werden nie kuenstlich verjuengt. +Abhaengigkeiten sind zyklenfrei, doppelte Quellen werden verhindert. +Eine virtuelle Summenvariable erbt das aelteste Datum ihrer physischen Eingangsquellen. +Konfigurierbare Alters-/Synchronitaetspruefung ist erforderlich. Diagnoseberechnungen +mit ungeeigneten Daten bleiben `quality_hold`, nicht `base_load`-Trainingsfreigabe. +Die momentanen Beispielgrenzen (60 s Alter, 30 s Versatz) sind konservative +Diagnoseparameter, keine garantierten Geraeteeigenschaften. + +Die konkrete Solar-PV-Herkunft sowie AC/DC-/Wirkungsgradgrenzen aller Hybridgeraete +sind noch zu verifizieren. Eine algebraisch geschlossene Gleichung beweist diese +physische Messgrenze NICHT. Die vorgeschlagene Quellkonfiguration ist nicht freigegeben. + +## Migration in die Prognose (noch offen) + +1. Originalmessquellen und Messgrenze dokumentieren und Qualitaet ueberwachen. +2. Korrigierte Grundlast und externe Kanaele als NEUE versionierte Zeitreihen erfassen. +3. Bestehende Historien nicht ueberschreiben oder als bereinigte Historie umbenennen. + Rekonstruktion nur bei ausreichenden zeitlich passenden Originaldaten. +4. Prognosemodell auf der korrigierten Basis trainieren/validieren. +5. Zukunfts-SDL separat liefern. Unbekannt ist nicht null; eine geschaetzte Annahme + muss explizit bezeichnet werden und darf nicht als veroeffentlichter SDL-Fahrplan gelten. +6. Erst nach diesen Pruefungen eine echte Bilanzierungsreferenz im Trial-Gate verwenden. + `mappingId` aus dieser Klasse ist ein Konfigurationsfingerabdruck, KEIN solcher Nachweis. + +## Gateway-Ausfallverhalten (nicht getestet/freigegeben) + +Quelltextpruefung des installierten Alt-Gateways ergab keinen Frischetest fuer +`Nennleistung_Soll_EV` im Weg `Update -> ApplySetpoints`. Der zwischengespeicherte +EV-Sollwert wird weiter verwendet, solange die Gateway-Schleife laeuft. Der Eingang +wird von `scripts/31800.ips.php` geschrieben; dieser enthaelt ebenfalls keine Lease. +Der vorhandene 30-s-Timeout des EMS-Batterieadapters ist ein Schutz innerhalb von +Symcon, nicht ein unabhaengiger Wechselrichter-Timeout bei Rechnerausfall. + +`State=false` im Alt-Gateway beendet `Update` ohne dort Nullwerte zu schreiben. +Diesen Schalter nicht als nachgewiesenen Hardware-Notstopp verwenden. +Keine Kommunikationsunterbrechung, kein Abschalttest und keine neue Stellwertausgabe +wurden an der realen Anlage durchgefuehrt. Auch aus dem fehlenden Code kann NICHT +gefolgert werden, dass das Geraet selbst keinen Timeout hat. Exakte Typen/Firmware +und das wirksame geraeteseitige Verhalten muessen noch festgestellt werden. + +Wichtig fuer spaetere Anpassung: Ein EV-Befehlsablauf darf nicht pauschal den +externen SDL-Auftrag loeschen. Ein Kernel-interner Timer ersetzt wiederum keinen +Schutz bei komplettem Kernel-/Rechner-/Kommunikationsausfall. diff --git a/libs/NetzfahrplanV4Bilanzierung.php b/libs/NetzfahrplanV4Bilanzierung.php new file mode 100644 index 0000000..38f61a2 --- /dev/null +++ b/libs/NetzfahrplanV4Bilanzierung.php @@ -0,0 +1,120 @@ + $max) { + throw new InvalidArgumentException($name . ': ungueltiger Zahlenwert.'); + } + return (float) $v; + } + + private static function listArray(array $a): bool + { + return $a === [] || array_keys($a) === range(0, count($a)-1); + } + + public static function konfigurieren(array $c): array + { + $fields = ['version','sources','maxSkewSeconds','splitToleranceW']; + if (count($c) !== count($fields) || array_diff($fields,array_keys($c)) || $c['version'] !== 1 + || !is_array($c['sources']) || !self::listArray($c['sources']) || count($c['sources']) < 3 || count($c['sources']) > 80 + || !is_int($c['maxSkewSeconds']) || $c['maxSkewSeconds'] < 0 || $c['maxSkewSeconds'] > 60) { + throw new InvalidArgumentException('Bilanzkonfiguration unvollstaendig.'); + } + self::number($c['splitToleranceW'], 'Zuordnungstoleranz W', 0, 500); + $seen=[];$keys=[];$roles=[]; + foreach ($c['sources'] as &$s) { + if (!is_array($s) || count($s)!==8 || array_diff(['key','role','variableId','parentId','ident','factorToW','maxAgeSeconds','dependsOn'],array_keys($s))) { + throw new InvalidArgumentException('Explizite Quellenzuordnung erforderlich.'); + } + if (!is_string($s['key']) || !preg_match('/^[A-Za-z][A-Za-z0-9_-]{0,63}$/D',$s['key']) || isset($keys[$s['key']]) + || !in_array($s['role'], ['grid','pv','physical_storage','controlled_virtual','external_virtual','flexible_load','reference'],true) + || !is_int($s['variableId']) || $s['variableId']<=0 || isset($seen[$s['variableId']]) + || !is_int($s['parentId']) || $s['parentId']<=0 || !is_string($s['ident']) || strlen($s['ident'])>100 + || !is_int($s['maxAgeSeconds']) || $s['maxAgeSeconds']<1 || $s['maxAgeSeconds']>300) { + throw new InvalidArgumentException('Doppelte oder ungueltige Bilanzquelle.'); + } + $s['factorToW']=self::number($s['factorToW'],'Einheiten-/Vorzeichenfaktor',-1e6,1e6); + if ($s['factorToW']==0 || !is_array($s['dependsOn']) || !self::listArray($s['dependsOn']) || count($s['dependsOn'])>40) { + throw new InvalidArgumentException('Abhaengigkeiten oder Faktor fehlen.'); + } + foreach ($s['dependsOn'] as $dep) if (!is_string($dep)) throw new InvalidArgumentException('Quellenschluessel erforderlich.'); + if (count(array_unique($s['dependsOn']))!==count($s['dependsOn'])) throw new InvalidArgumentException('Doppelte Abhaengigkeit.'); + sort($s['dependsOn']);$keys[$s['key']]=true;$seen[$s['variableId']]=true;$roles[$s['role']]=($roles[$s['role']]??0)+1; + } + unset($s); + if (($roles['grid']??0)!==1 || !isset($roles['pv'],$roles['physical_storage'],$roles['controlled_virtual'])) { + throw new InvalidArgumentException('Netz, PV, physische und steuerbare virtuelle Batterie muessen explizit zugeordnet sein.'); + } + $graph=[];foreach ($c['sources'] as $s) $graph[$s['key']]=$s['dependsOn']; + $visit=function($key,$stack=[]) use (&$visit,$graph): void { + if (!array_key_exists($key,$graph) || isset($stack[$key])) throw new InvalidArgumentException('Fehlende oder zyklische Ursprungsquelle.'); + $stack[$key]=true;foreach ($graph[$key] as $dep) $visit($dep,$stack); + }; + foreach (array_keys($graph) as $key) $visit($key); + usort($c['sources'],static fn($a,$b)=>strcmp($a['key'],$b['key']));return $c; + } + + /** Reader: {value:number, updated:int, parentID:int, ident:string}. */ + public static function aufnehmen(array $configuration, callable $read, int $now): array + { + if ($now<=0) throw new InvalidArgumentException('Ungueltige Aufnahmezeit.'); + $c=self::konfigurieren($configuration);$samples=[];$sources=[];$issues=[]; + $probe=static function(array $s) use ($read): array { + $v=$read($s['variableId']); + if (!is_array($v) || ($v['parentID']??null)!==$s['parentId'] || ($v['ident']??null)!==$s['ident'] + || !is_int($v['updated']??null) || $v['updated']<=0) throw new InvalidArgumentException('Quelle/Originalzeitpunkt ungueltig: '.$s['key']); + return ['raw'=>self::number($v['value']??null,$s['key']), 'updated'=>$v['updated']]; + }; + foreach ($c['sources'] as $s) {$sources[$s['key']]=$s;$samples[$s['key']]=$probe($s);} + foreach ($c['sources'] as $s) if ($samples[$s['key']]!==$probe($s)) throw new InvalidArgumentException('Aufnahme waehrend des Lesens geaendert: '.$s['key']); + $origin=function($key) use (&$origin,$sources,$samples): int { + $stamp=$samples[$key]['updated'];foreach ($sources[$key]['dependsOn'] as $dep) $stamp=min($stamp,$origin($dep));return $stamp; + }; + $totals=array_fill_keys(['grid','pv','physical_storage','controlled_virtual','external_virtual','flexible_load','reference'],0.0); + $out=[];$times=[]; + foreach ($sources as $key=>$s) { + $t=$origin($key);$w=$samples[$key]['raw']*$s['factorToW']; + if (!is_finite($w) || abs($w)>1e9) throw new InvalidArgumentException('Leistung ausserhalb des Bereichs.'); + if ($samples[$key]['updated']>$now || $t>$now) $issues[]='future_source:'.$key; + if ($now-$t>$s['maxAgeSeconds']) $issues[]='stale_origin:'.$key; + if (in_array($s['role'],['pv','flexible_load'],true) && $w<0) $issues[]='negative_unsigned_source:'.$key; + $totals[$s['role']]+=$w; + if ($s['role']!=='reference') $times[]=$t; + $out[$key]=['variableId'=>$s['variableId'],'role'=>$s['role'],'powerW'=>$w,'sourceUpdatedAt'=>$samples[$key]['updated'],'oldestOriginAt'=>$t]; + } + $skew=max($times)-min($times);if ($skew>$c['maxSkewSeconds']) $issues[]='source_time_skew'; + $load=$totals['grid']+$totals['pv']-$totals['physical_storage']; + $base=$load-$totals['flexible_load']; + // This channel preserves unallocated dynamics instead of hiding them in house load. + $residual=$totals['physical_storage']-$totals['controlled_virtual']-$totals['external_virtual']; + $external=$totals['external_virtual']+$residual; + if ($base<0) $issues[]='negative_base_load'; + if (abs($residual)>$c['splitToleranceW']) $issues[]='virtual_physical_allocation_mismatch'; + $reconstructed=$base+$totals['flexible_load']+$totals['controlled_virtual']+$external-$totals['pv']; + $identity=hash('sha256',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + return ['version'=>1,'status'=>$issues===[]?'consistent_candidate':'quality_hold', + 'capturedAt'=>gmdate('c',$now),'sourceObservedFrom'=>gmdate('c',min($times)),'sourceObservedUntil'=>gmdate('c',max($times)), + 'sourceSkewSeconds'=>$skew,'mappingId'=>'accounting-map:'.$identity, + 'baseLoadCandidateW'=>$base,'consumerLoadCandidateW'=>$load,'pvW'=>$totals['pv'],'gridW'=>$totals['grid'], + 'physicalStorageW'=>$totals['physical_storage'],'controlledVirtualW'=>$totals['controlled_virtual'], + 'flexibleLoadW'=>$totals['flexible_load'],'reportedExternalW'=>$totals['external_virtual'], + 'allocationResidualW'=>$residual,'externalEffectW'=>$external,'reconstructedGridW'=>$reconstructed, + 'arithmeticBalanceErrorW'=>$reconstructed-$totals['grid'],'issues'=>$issues,'sources'=>$out, + 'controlEligible'=>false,'meteringBoundaryVerified'=>false,'forecastHistoryMigrated'=>false, + 'note'=>'Algebraic closure is not independent meter validation. No relabelling of old forecasts, no future SDL assumption.']; + } +} diff --git a/tests/NetzfahrplanV4BilanzierungTest.php b/tests/NetzfahrplanV4BilanzierungTest.php new file mode 100644 index 0000000..6e1145a --- /dev/null +++ b/tests/NetzfahrplanV4BilanzierungTest.php @@ -0,0 +1,16 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]);self::assertSame(0,proc_close($p),$out.$err);self::assertSame('',$err); + self::assertStringContainsString('TOTAL 34 checks passed',$out); + } +} diff --git a/tests/V4Accounting/checks.php b/tests/V4Accounting/checks.php new file mode 100644 index 0000000..2d9c806 --- /dev/null +++ b/tests/V4Accounting/checks.php @@ -0,0 +1,56 @@ +1000.,'pv'=>5000.,'physical_storage'=>3000.,'controlled_virtual'=>2000.,'external_virtual'=>1000.] as $role=>$value){ + $id=101+count($s);$s[]=['key'=>$role,'role'=>$role,'variableId'=>$id,'parentId'=>10,'ident'=>$role,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>[]]; + $m[$id]=['value'=>$value,'updated'=>$now,'parentID'=>10,'ident'=>$role]; + } + $s[3]['dependsOn']=['physical_storage'];$s[4]['dependsOn']=['physical_storage']; + return [['version'=>1,'sources'=>$s,'maxSkewSeconds'=>30,'splitToleranceW'=>100.],$m,$now]; +} +function take(array $c,array $m,int $now):array{return B::aufnehmen($c,static fn($id)=>$m[$id],$now);} +[$c,$m,$n]=fixture();$r=take($c,$m,$n); +check($r['baseLoadCandidateW']===3000.,'subtract physical battery once, not virtual twice'); +check($r['externalEffectW']===1000. && $r['allocationResidualW']===0.,'SDL kept outside household'); +check($r['reconstructedGridW']===1000. && $r['arithmeticBalanceErrorW']===0.,'signed PCC balance closes'); +check($r['controlEligible']===false && !$r['meteringBoundaryVerified'],'algebra never grants control'); +check(!$r['forecastHistoryMigrated'],'old forecasts never relabelled'); +$m[101]['value']=-5000.;$m[103]['value']=-3000.;$m[104]['value']=-2000.;$m[105]['value']=-1000.;$r=take($c,$m,$n); +check($r['baseLoadCandidateW']===3000. && $r['externalEffectW']===-1000.,'discharge and export signs correct'); +[$c,$m,$n]=fixture();$m[103]['value']=2000.;$m[104]['value']=42000.;$m[105]['value']=-40000.;$r=take($c,$m,$n); +check($r['allocationResidualW']===0. && $r['baseLoadCandidateW']===4000.,'opposed virtual accounts are not two physical loads'); +[$c,$m,$n]=fixture();$m[104]['value']=0.;$m[105]['value']=0.;$r=take($c,$m,$n); +check($r['externalEffectW']===3000. && $r['baseLoadCandidateW']===3000.,'held virtual values cannot contaminate base load'); +check(in_array('virtual_physical_allocation_mismatch',$r['issues'],true),'split discrepancy explicit'); +[$c,$m,$n]=fixture();$m[103]['updated']=$n-120;$r=take($c,$m,$n); +check(in_array('stale_origin:physical_storage',$r['issues'],true),'old physical reading rejected despite new acquisition'); +check(in_array('stale_origin:controlled_virtual',$r['issues'],true),'fresh virtual timestamp cannot mask old physical input'); +check($r['sources']['controlled_virtual']['oldestOriginAt']===$n-120,'original dependency timestamp preserved'); +check($r['status']==='quality_hold','bad quality remains a diagnostic, not a training sample'); +[$c,$m,$n]=fixture();$m[102]['updated']=$n-31;$r=take($c,$m,$n);check(in_array('source_time_skew',$r['issues'],true),'skew rejected separately from age'); +$m[102]['updated']=$n+1;$r=take($c,$m,$n);check(in_array('future_source:pv',$r['issues'],true),'future timestamp rejected'); +[$c,$m,$n]=fixture();foreach($m as &$v)$v['value']=0.;unset($v);$r=take($c,$m,$n);check($r['status']==='consistent_candidate' && $r['baseLoadCandidateW']===0.,'true all-zero observation accepted without fake history'); +[$c,$m,$n]=fixture();$m[101]['value']=-6000.;$r=take($c,$m,$n);check($r['baseLoadCandidateW']===-4000. && in_array('negative_base_load',$r['issues'],true),'negative residual not silently clipped to zero'); +[$c,$m,$n]=fixture();$m[102]['value']=-10.;check(in_array('negative_unsigned_source:pv',take($c,$m,$n)['issues'],true),'PV sign validity'); +[$c,$m,$n]=fixture();$m[103]['value']=-3000.;$c['sources'][2]['factorToW']=-1.;check(take($c,$m,$n)['baseLoadCandidateW']===3000.,'explicit vendor sign conversion'); +[$c,$m,$n]=fixture();$m[101]['value']=1.;$c['sources'][0]['factorToW']=1000.;check(take($c,$m,$n)['baseLoadCandidateW']===3000.,'kW to W factor applied once'); +[$c,$m,$n]=fixture();$c['sources'][]=['key'=>'boiler','role'=>'flexible_load','variableId'=>200,'parentId'=>10,'ident'=>'boiler','factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>[]];$m[200]=['value'=>500.,'updated'=>$n,'parentID'=>10,'ident'=>'boiler'];$r=take($c,$m,$n);check($r['baseLoadCandidateW']===2500. && $r['reconstructedGridW']===1000.,'future flexible load excluded exactly once'); +foreach([true,'1000',NAN,INF] as $bad){[$c,$m,$n]=fixture();$m[101]['value']=$bad;fails(fn()=>take($c,$m,$n),'invalid numeric input '.gettype($bad));} +[$c,$m,$n]=fixture();$m[101]['parentID']=99;fails(fn()=>take($c,$m,$n),'different source parent rejected'); +[$c,$m,$n]=fixture();$m[101]['ident']='other';fails(fn()=>take($c,$m,$n),'different source ident rejected'); +[$c,$m,$n]=fixture();$c['sources'][]=$c['sources'][0];fails(fn()=>take($c,$m,$n),'duplicate physical source rejected'); +[$c,$m,$n]=fixture();$c['sources'][3]['dependsOn']=['absent'];fails(fn()=>take($c,$m,$n),'missing origin rejected'); +[$c,$m,$n]=fixture();$c['sources'][2]['dependsOn']=['controlled_virtual'];fails(fn()=>take($c,$m,$n),'dependency loop rejected'); +[$c,$m,$n]=fixture();$a=take($c,$m,$n)['mappingId'];$c['sources']=array_reverse($c['sources']);check(take($c,$m,$n)['mappingId']===$a,'source order does not change mapping identity'); +$c['sources'][0]['factorToW']=2.;check(take($c,$m,$n)['mappingId']!==$a,'factor changes invalidate old mapping identity'); +[$c,$m,$n]=fixture();$calls=[]; +fails(function()use($c,$m,$n,&$calls){B::aufnehmen($c,function($id)use($m,&$calls){$calls[$id]=($calls[$id]??0)+1;$v=$m[$id];if($calls[$id]>1)$v['value']++;return $v;},$n);},'same-second concurrent value change rejected'); +$source=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Bilanzierung.php');check(!preg_match('/\b(?:SetValue|IPS_RequestAction|RequestAction|curl_exec|IPS_SetProperty|AC_SetLoggingStatus)\s*\(/',$source),'adapter has no runtime or external write calls'); +echo "TOTAL $count checks passed; synthetic snapshots only.\n"; From 76f8b067cc72d7c7e2574d157b70b318c9634de9 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 14:55:43 +0000 Subject: [PATCH 07/22] feat(data): add passive versioned raw capture independent of control --- docs/netplan-v4-data-capture.md | 67 +++++++++ examples/V4DataCapture/install.php | 89 ++++++++++++ examples/V4DataCapture/runtime.php | 143 ++++++++++++++++++++ libs/NetzfahrplanV4Messaufnahme.php | 165 +++++++++++++++++++++++ tests/NetzfahrplanV4MessaufnahmeTest.php | 16 +++ tests/V4DataCapture/checks.php | 62 +++++++++ tests/V4DataCapture/native_checks.php | 61 +++++++++ 7 files changed, 603 insertions(+) create mode 100644 docs/netplan-v4-data-capture.md create mode 100644 examples/V4DataCapture/install.php create mode 100644 examples/V4DataCapture/runtime.php create mode 100644 libs/NetzfahrplanV4Messaufnahme.php create mode 100644 tests/NetzfahrplanV4MessaufnahmeTest.php create mode 100644 tests/V4DataCapture/checks.php create mode 100644 tests/V4DataCapture/native_checks.php diff --git a/docs/netplan-v4-data-capture.md b/docs/netplan-v4-data-capture.md new file mode 100644 index 0000000..a2237be --- /dev/null +++ b/docs/netplan-v4-data-capture.md @@ -0,0 +1,67 @@ +# Passive V4 data capture - user inventory and raw acquisition + +Status 2026-10-02: source implementation, CLI/mocked tests and staged user installation. +Not part of the live manager, not a control or training release. + +## Lihrenmoos user-reported inventory (not applied to running configuration) + +- GoodWe 1/2: each 50 kW AC, each 156 kWh reported physical battery capacity. +- SolarEdge: 10 kW AC, 10 kWh reported physical battery capacity. +- EV allocation: 30 kW / 160 kWh total, understood as 10 kW EV per inverter. +- PV roof: approximately 20 kWp east and 20 kWp west. Tilt/individual strings unknown. +- Nominal battery sum: 322 kWh. 162 kWh nominal difference to EV allocation is NOT a + verified usable SDL capacity. AC ratings are not verified battery charge/discharge ratings. +- Exact types, firmware, common AC meter boundary and device watchdog remain unverified. +- Current saved manager topology still reports 161.44 kWh / 39 kW. Gateway currently + lists 130/130/8 kWh and 57/57/5 kW. These may use a different usable/nominal basis. + Do not rebase virtual SOC or change current controls based on these notes alone. + +## Why collect before boundary approval + +Record the actual original numerical channels now, with source metadata and timestamps. +The algebraic base-load candidate is useful for diagnosis, but NEVER a certified base-load +training sample until the physical AC/DC boundary and historical recipe are verified. +No dependence on hardware-watchdog approval for passive recording. + +21 channels: PCC power, legacy displayed PCC power, PV for three inverters, three physical +battery powers, EV/SDL virtual powers and SOCs, three physical SOCs, dynamic EV limits, +T1/T2 import counters, EV/SDL requested setpoints (READ ONLY). +A cyclic script reads values already present in Symcon every 30 seconds. It does not issue +additional Modbus polls. Original VariableUpdated/VariableChanged are retained; these are +Symcon timestamps, not independent device-measurement timestamps. Two-pass reads detect +concurrent changes; missing/non-numeric/re-used channels remain explicit quality failures. + +## Storage and interpretation + +Daily append-only raw-YYYYMMDD.jsonl in the separate capture stage data directory, UTC +capture times plus original source timestamps, immutable mapping/inventory snapshots by +SHA256. Raw errors and quality_hold are recorded too; nothing is invented as zero. +The raw observations are a source for later documented 5-minute aggregation, not themselves +aligned 5-minute means. No interpolation, training publication or future SDL assumption. + +A 512 MiB journal quota, 64 MiB daily limit, 256 KiB record cap and 100 MiB free-space +reserve bound recording. Limits fail visibly without deleting old data. File permissions +0640; no upload credentials, request bodies, complete instance configurations or secrets. +A separate status variable and STATUS.json show latest capture and failures. Process restart +retains files; an incomplete final JSONL row is preserved for manual review, not extended. + +## Install / stop + +Staged user entry point: /srv/agent/netplan-v4-data-capture-stage/install.php +Execute in the Symcon PHP editor (not a Linux shell): + + require '/srv/agent/netplan-v4-data-capture-stage/install.php'; + +Installer checks exact source hashes and numeric source identities before creating a NEW +root category ENELIX_V4_PASSIVE_CAPTURE, a status string, collector script and stop script. +IPS_SetScriptTimer enables only this new script at 30 seconds. Repeating installation is +idempotent. Unrelated/edited scripts or object identity collisions are not overwritten. + +Stop using the newly created script 'Nur diese Messaufnahme stoppen'; it disables only +this collection timer, retains collected files, and does not stop the manager or battery. +No module reload, existing archive change, existing timer/poller change, service restart, +forecast selection, battery reservation or actuator command is included. + +Current tests: 36 core/file tests + 19 mocked native lifecycle tests with PHP CLI on the +test host. First real capture and future timer execution require the user activation. +These tests do not certify hardware mappings or a Symcon-kernel run. diff --git a/examples/V4DataCapture/install.php b/examples/V4DataCapture/install.php new file mode 100644 index 0000000..fe2d8b3 --- /dev/null +++ b/examples/V4DataCapture/install.php @@ -0,0 +1,89 @@ + IPS_CreateCategory(), 2 => IPS_CreateVariable(3), 3 => IPS_CreateScript(0) }; + if (!is_int($id) || $id <= 0) throw new RuntimeException('Cannot create capture object'); + $created[] = $id; + if (!IPS_SetParent($id, $parent) || !IPS_SetIdent($id, $ident) || !IPS_SetName($id, $name) || !IPS_SetInfo($id, $marker)) { + throw new RuntimeException('Cannot label capture object'); + } + return $id; +} + +function v4captureScript(int $id, string $content, array $created): void +{ + $current = IPS_GetScriptContent($id); + if ($current === $content) return; + if (!in_array($id, $created, true)) throw new RuntimeException('Existing capture script changed; not overwritten'); + if (!IPS_SetScriptContent($id, $content) || IPS_GetScriptContent($id) !== $content) throw new RuntimeException('Cannot store capture script'); +} + +function v4captureInstall(string $stage): array +{ + if (!function_exists('IPS_GetKernelVersion') || version_compare(IPS_GetKernelVersion(),'8.0','<')) { + throw new RuntimeException('Nur im IP-Symcon-Skripteditor der Testanlage ausfuehren.'); + } + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + if (!IPS_InstanceExists($c['managerId']) + || IPS_GetInstance($c['managerId'])['ModuleInfo']['ModuleID'] !== '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}') { + throw new RuntimeException('Expected manager not found'); + } + // Verify layout only; stale/missing sensor updates remain diagnostic raw data, not fake zeros. + foreach (array_merge($c['accounting']['sources'], $c['extraSources']) as $s) { + if (!IPS_VariableExists($s['variableId'])) throw new RuntimeException('Expected source missing: ' . $s['key']); + $o = IPS_GetObject($s['variableId']); $v = IPS_GetVariable($s['variableId']); + if ($o['ParentID'] !== $s['parentId'] || $o['ObjectIdent'] !== $s['ident'] || !in_array($v['VariableType'],[1,2],true)) { + throw new RuntimeException('Source identity changed: ' . $s['key']); + } + } + $data = $stage . '/data'; + if (is_link($data) || !is_dir($data) || !is_writable($data)) throw new RuntimeException('Dedicated capture directory not writable'); + $key = 'ENELIX.V4.CaptureInstall.' . $c['installationId']; + if (!IPS_SemaphoreEnter($key, 3000)) throw new RuntimeException('Capture installation busy'); + $created = []; $script = null; $timerSet = false; + $report = ['startedAt' => gmdate('c'), 'scope' => 'passive_raw_capture_only', 'controlEnabled' => false, + 'existingModulesChanged' => false, 'archivesChanged' => false, 'existingHistoriesChanged' => false]; + try { + $category = v4captureEnsure(0,'ENELIX_V4_PASSIVE_CAPTURE',0,'V4 - separate Messaufnahme (keine Steuerung)',$marker,$created); + v4captureEnsure($category,'CaptureStatus',2,'Aufzeichnungsstatus (nur lesen)',$marker,$created); + $script = v4captureEnsure($category,'RawCapture',3,'Messwerte alle 30 Sekunden erfassen',$marker,$created); + $stop = v4captureEnsure($category,'StopCapture',3,'Nur diese Messaufnahme stoppen',$marker,$created); + $include = ' 'capture_installed', 'categoryId' => $category, 'scriptId' => $script, + 'stopScriptId' => $stop, 'intervalSeconds' => 30, 'firstCapture' => $first, + 'futureTimerExecutionVerified' => false]; + echo "OK: Separate Rohdatenaufnahme eingerichtet, alle 30 Sekunden.\n"; + echo 'Erste Aufnahme: ' . $first['capturedAt'] . '; Quellen: ' . $first['sourceCount'] . ".\n"; + echo 'Bilanzqualitaet: ' . $first['accountingQuality'] . " (noch keine Trainings-/Stellfreigabe).\n"; + echo "Bestehende Manager-, Batterie-, Archiv- und Pollingkonfiguration unveraendert.\n"; + } catch (Throwable $e) { + $report['status'] = 'needs_review'; $report['reason'] = substr($e->getMessage(),0,300); + if ($timerSet && $script !== null) IPS_SetScriptTimer($script,0); + // Never delete unrelated objects or stored data during recovery. + echo 'FEHLER: ' . $report['reason'] . "\n"; + } finally { + $report['createdObjects'] = $created; $report['finishedAt'] = gmdate('c'); + try { v4captureAtomicJson($stage . '/INSTALL_RESULT.json', $report); } + finally { IPS_SemaphoreLeave($key); } + echo 'CAPTURE REPORT: ' . $stage . "/INSTALL_RESULT.json\n"; + } + return $report; +} diff --git a/examples/V4DataCapture/runtime.php b/examples/V4DataCapture/runtime.php new file mode 100644 index 0000000..7b0ba2a --- /dev/null +++ b/examples/V4DataCapture/runtime.php @@ -0,0 +1,143 @@ + GetValue($id), 'updated' => (int)$v['VariableUpdated'], 'changed' => (int)$v['VariableChanged'], + 'parentID' => $o['ParentID'], 'ident' => $o['ObjectIdent']]; +} + +function v4captureRun(string $stage, int $category): array +{ + if (!function_exists('IPS_GetKernelVersion')) throw new RuntimeException('Run inside IP-Symcon only'); + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + v4captureAssertObject($category, 0, 'ENELIX_V4_PASSIVE_CAPTURE', 0, $marker); + $statusId = v4captureChild($category, 'CaptureStatus'); + if ($statusId === null) throw new RuntimeException('Capture status missing'); + v4captureAssertObject($statusId, $category, 'CaptureStatus', 2, $marker); + $lock = 'ENELIX.V4.RawCapture.' . $c['installationId']; + if (!IPS_SemaphoreEnter($lock, 0)) return ['status' => 'busy', 'controlEnabled' => false]; + try { + try { + $record = NetzfahrplanV4Messaufnahme::capture($c, 'v4captureRead', static fn(): int => time()); + v4captureKeepSnapshot($stage . '/data/mapping-' . $record['mappingSha256'] . '.json', json_encode($c, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + v4captureKeepSnapshot($stage . '/data/inventory-' . $record['reportedInventorySha256'] . '.json', (string)file_get_contents($stage . '/USER_REPORTED_INVENTORY.json')); + $path = NetzfahrplanV4Messaufnahme::append($stage . '/data', $record); + $status = ['status' => 'recorded', 'capturedAt' => $record['capturedAt'], 'journal' => $path, + 'sourceCount' => count($record['raw']), 'rawIssueCount' => count($record['issues']), + 'accountingQuality' => $record['assessment']['status'] ?? 'unavailable', + 'qualityIssues' => $record['assessment']['issues'] ?? $record['issues'], + 'baseLoadCandidateW' => $record['assessment']['baseLoadCandidateW'] ?? null, + 'sourceSkewSeconds' => $record['assessment']['sourceSkewSeconds'] ?? null, + 'trainingEligible' => false, 'controlEnabled' => false, 'measurementBoundaryVerified' => false]; + v4captureAtomicJson($stage . '/data/STATUS.json', $status); + } catch (Throwable $e) { + $status = ['status' => 'capture_error', 'at' => gmdate('c'), 'reason' => substr($e->getMessage(),0,250), + 'trainingEligible' => false, 'controlEnabled' => false]; + try { v4captureAtomicJson($stage . '/data/STATUS.json', $status); } catch (Throwable $ignored) { } + } + // Only this newly created diagnosis variable, never a meter or controller variable. + SetValue($statusId, json_encode($status, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + return $status; + } finally { IPS_SemaphoreLeave($lock); } +} + +function v4captureStop(string $stage, int $category): void +{ + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + v4captureAssertObject($category, 0, 'ENELIX_V4_PASSIVE_CAPTURE', 0, $marker); + $script = v4captureChild($category, 'RawCapture'); $status = v4captureChild($category, 'CaptureStatus'); + if ($script === null || $status === null) throw new RuntimeException('Capture objects missing'); + v4captureAssertObject($script, $category, 'RawCapture', 3, $marker); + v4captureAssertObject($status, $category, 'CaptureStatus', 2, $marker); + if (!IPS_SetScriptTimer($script, 0)) throw new RuntimeException('Cannot stop capture timer'); + $r = ['status' => 'stopped', 'at' => gmdate('c'), 'oldDataRetained' => true, 'controlEnabled' => false]; + SetValue($status, json_encode($r, JSON_THROW_ON_ERROR)); + v4captureAtomicJson($stage . '/data/STATUS.json', $r); +} diff --git a/libs/NetzfahrplanV4Messaufnahme.php b/libs/NetzfahrplanV4Messaufnahme.php new file mode 100644 index 0000000..e818ba2 --- /dev/null +++ b/libs/NetzfahrplanV4Messaufnahme.php @@ -0,0 +1,165 @@ + 60 + || !is_string($c['reportedInventorySha256'] ?? null) + || !preg_match('/^[0-9a-f]{64}$/D', $c['reportedInventorySha256'])) { + throw new InvalidArgumentException('Invalid capture configuration'); + } + $c['accounting'] = NetzfahrplanV4Bilanzierung::konfigurieren($c['accounting'] ?? []); + $keys = []; $ids = []; + foreach ($c['accounting']['sources'] as $s) { $keys[$s['key']] = true; $ids[$s['variableId']] = true; } + foreach ($c['extraSources'] as $s) { + if (count($s) !== 5 || !is_string($s['key'] ?? null) + || !preg_match('/^[A-Za-z][A-Za-z0-9_-]{0,63}$/D', $s['key']) || isset($keys[$s['key']]) + || !is_int($s['variableId'] ?? null) || $s['variableId'] < 1 || isset($ids[$s['variableId']]) + || !is_int($s['parentId'] ?? null) || $s['parentId'] < 1 + || !is_string($s['ident'] ?? null) || strlen($s['ident']) > 100 + || !in_array($s['unit'] ?? null, ['W','kWh','percent'], true)) { + throw new InvalidArgumentException('Invalid or duplicate raw source'); + } + $keys[$s['key']] = true; $ids[$s['variableId']] = true; + } + usort($c['extraSources'], static fn(array $a, array $b): int => strcmp($a['key'], $b['key'])); + return $c; + } + + private static function readSource(array $s, callable $read): array + { + try { + $r = $read($s['variableId']); + if (!is_array($r)) throw new RuntimeException('Missing reading'); + } catch (Throwable $e) { + return ['value' => null, 'sourceUpdatedAt' => null, 'sourceChangedAt' => null, 'issues' => ['read_failed']]; + } + $issues = []; + $v = $r['value'] ?? null; + if ((!is_int($v) && !is_float($v)) || !is_finite((float) $v) || abs((float)$v) > 1.0e12) { + $v = null; $issues[] = 'invalid_numeric_value'; + } + $at = $r['updated'] ?? null; + if (!is_int($at) || $at < 1) { $at = null; $issues[] = 'missing_original_timestamp'; } + $changed = $r['changed'] ?? null; + if (!is_int($changed) || $changed < 0) $changed = null; + if (($r['parentID'] ?? null) !== $s['parentId'] || ($r['ident'] ?? null) !== $s['ident']) { + // Do not record a numeric value from an object re-used for a different purpose. + $v = null; $issues[] = 'source_identity_mismatch'; + } + return ['value' => $v, 'sourceUpdatedAt' => $at, 'sourceChangedAt' => $changed, 'issues' => $issues]; + } + + public static function capture(array $config, callable $read, callable $clock): array + { + $c = self::configuration($config); + $start = $clock(); + if (!is_int($start) || $start <= 0) throw new InvalidArgumentException('Integer UTC epoch required'); + $sources = array_merge($c['accounting']['sources'], $c['extraSources']); + $first = []; $second = []; $raw = []; $issues = []; + foreach ($sources as $s) $first[$s['key']] = self::readSource($s, $read); + foreach ($sources as $s) $second[$s['key']] = self::readSource($s, $read); + $end = $clock(); + if (!is_int($end) || $end < $start) throw new RuntimeException('Acquisition clock moved backwards'); + foreach ($sources as $s) { + $key = $s['key']; $r = $first[$key]; + if ($r !== $second[$key]) { + $r['issues'][] = 'changed_during_capture'; + $r['secondValue'] = $second[$key]['value']; + $r['secondUpdatedAt'] = $second[$key]['sourceUpdatedAt']; + } + if ($r['sourceUpdatedAt'] !== null && $r['sourceUpdatedAt'] > $end) $r['issues'][] = 'future_source'; + $r['sourceAgeSeconds'] = $r['sourceUpdatedAt'] === null ? null : $end - $r['sourceUpdatedAt']; + $raw[$key] = ['variableId' => $s['variableId']] + $r; + foreach ($r['issues'] as $issue) $issues[] = $key . ':' . $issue; + } + $assessment = null; $accountingUsable = true; $samples = []; + foreach ($c['accounting']['sources'] as $s) { + $r = $raw[$s['key']]; + if ($r['issues'] !== []) $accountingUsable = false; + $samples[$s['variableId']] = ['value' => $r['value'], 'updated' => $r['sourceUpdatedAt'], 'parentID' => $s['parentId'], 'ident' => $s['ident']]; + } + if ($accountingUsable) { + try { $assessment = NetzfahrplanV4Bilanzierung::aufnehmen($c['accounting'], static fn(int $id): array => $samples[$id], $end); } + catch (Throwable $e) { $issues[] = 'accounting_input_invalid'; } + } else { $issues[] = 'accounting_snapshot_unusable'; } + if ($end - $start > 5) $issues[] = 'long_acquisition'; + return ['schemaVersion' => 1, 'kind' => 'raw_accounting_capture', 'installationId' => $c['installationId'], + 'captureStartedAt' => gmdate('c', $start), 'capturedAt' => gmdate('c', $end), + 'mappingSha256' => hash('sha256', json_encode($c, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)), + 'reportedInventorySha256' => $c['reportedInventorySha256'], + 'timestampMeaning' => 'Symcon VariableUpdated; not proof of an independent device timestamp', + 'raw' => $raw, 'issues' => $issues, 'assessment' => $assessment, + 'trainingEligible' => false, 'controlEligible' => false, 'measurementBoundaryVerified' => false, + 'futureSdl' => 'unknown_no_assumption', 'historicalDataRewritten' => false]; + } + + /** Append-only daily JSONL with exclusive lock, bounds and crash-tail detection. + * No retention deletion; quota stops capture visibly, never deletes old histories. + */ + public static function append(string $directory, array $record, int $quota = 536870912): string + { + if (!is_dir($directory) || is_link($directory) || realpath($directory) !== $directory || $quota < 1) { + throw new RuntimeException('Invalid dedicated data directory'); + } + $json = json_encode($record, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION) . "\n"; + if (strlen($json) > 262144) throw new RuntimeException('Capture exceeds record limit'); + $epoch = strtotime($record['capturedAt'] ?? ''); + if ($epoch === false) throw new RuntimeException('Capture timestamp missing'); + $file = $directory . '/raw-' . gmdate('Ymd', $epoch) . '.jsonl'; + $lockPath = $directory . '/.writer.lock'; + if (is_link($lockPath) || is_link($file)) throw new RuntimeException('Symbolic link refused'); + $lock = fopen($lockPath, 'c+b'); + if ($lock === false) throw new RuntimeException('Capture lock unavailable'); + try { + if (!flock($lock, LOCK_EX | LOCK_NB)) throw new RuntimeException('Capture writer busy'); + clearstatcache(); $used = 0; $files = glob($directory . '/raw-*.jsonl'); + if ($files === false || count($files) > 400) throw new RuntimeException('Data directory bounds exceeded'); + foreach ($files as $p) { + if (is_link($p)) throw new RuntimeException('Symbolic journal refused'); + $size = filesize($p); if ($size === false) throw new RuntimeException('Cannot size journal'); + $used += $size; + } + if ($used + strlen($json) > $quota) throw new RuntimeException('Capture quota reached; no data deleted'); + $free = disk_free_space($directory); + if ($free === false || $free < 104857600 + strlen($json)) throw new RuntimeException('Insufficient free disk; capture paused'); + $h = fopen($file, 'c+b'); + if ($h === false) throw new RuntimeException('Cannot open capture journal'); + try { + fseek($h, 0, SEEK_END); $old = ftell($h); + if ($old === false || $old + strlen($json) > 67108864) throw new RuntimeException('Daily capture bounds exceeded'); + if ($old > 0) { + fseek($h, -1, SEEK_END); + if (fread($h, 1) !== "\n") throw new RuntimeException('Incomplete journal tail; preserve for review'); + } + fseek($h, 0, SEEK_END); $n = 0; + try { + while ($n < strlen($json)) { + $written = fwrite($h, substr($json, $n)); + if ($written === false || $written === 0) throw new RuntimeException('Capture write incomplete'); + $n += $written; + } + if (!fflush($h)) throw new RuntimeException('Capture flush failed'); + } catch (Throwable $e) { ftruncate($h, $old); fflush($h); throw $e; } + if (!chmod($file, 0640)) throw new RuntimeException('Cannot restrict journal permissions'); + } finally { fclose($h); } + return $file; + } finally { flock($lock, LOCK_UN); fclose($lock); } + } +} diff --git a/tests/NetzfahrplanV4MessaufnahmeTest.php b/tests/NetzfahrplanV4MessaufnahmeTest.php new file mode 100644 index 0000000..421b42c --- /dev/null +++ b/tests/NetzfahrplanV4MessaufnahmeTest.php @@ -0,0 +1,16 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]);fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err);self::assertStringContainsString('TOTAL',$out); + } + public function scripts(): array { return [['checks.php'],['native_checks.php']]; } +} diff --git a/tests/V4DataCapture/checks.php b/tests/V4DataCapture/checks.php new file mode 100644 index 0000000..4630aff --- /dev/null +++ b/tests/V4DataCapture/checks.php @@ -0,0 +1,62 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004, + 'reportedInventorySha256'=>str_repeat('a',64),'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]], + 'extraSources'=>[['key'=>'soc','variableId'=>106,'parentId'=>200,'ident'=>'soc','unit'=>'percent']]]; +$values=[3000,5000,2000,1500,500]; $readings=[]; +foreach($roles as $i=>$role){ + $key='s'.$i;$id=101+$i; + $c['accounting']['sources'][]=['key'=>$key,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>in_array($i,[3,4],true)?['s2']:[]]; + $readings[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key]; +} +$readings[106]=['value'=>0.,'updated'=>$now,'changed'=>0,'parentID'=>200,'ident'=>'soc']; +function take(array $c,array $r,int $now): array { return Capture::capture($c,static fn(int $id):array=>$r[$id],static fn():int=>$now); } +$r=take($c,$readings,$now); +ck($r['assessment']['baseLoadCandidateW']===6000.,'physical battery subtracted exactly once'); +ck($r['raw']['soc']['value']===0. && $r['raw']['soc']['sourceChangedAt']===0,'real zero and unchanged timestamp retained'); +ck(!$r['trainingEligible'] && !$r['controlEligible'] && !$r['measurementBoundaryVerified'],'algebra never enables training or control'); +ck($r['futureSdl']==='unknown_no_assumption','no fabricated future SDL'); +ck(count($r['raw'])===6,'raw extra sources retained separately'); +$a=$readings;$a[103]['updated']=$now-90;$a[104]['updated']=$now; +$x=take($c,$a,$now);ck($x['assessment']['status']==='quality_hold','old physical measurement quarantines candidate'); +ck($x['raw']['s2']['sourceUpdatedAt']===$now-90,'original old timestamp preserved'); +ck(in_array('stale_origin:s3',$x['assessment']['issues'],true),'fresh virtual write cannot mask old dependency'); +$a=$readings;$a[101]['value']=null;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && $x['assessment']===null,'missing source not zero'); +$a=$readings;$a[101]['value']='sensitive-test-text';$x=take($c,$a,$now);ck(!str_contains(json_encode($x),'sensitive-test-text'),'unexpected strings never logged'); +foreach([true,NAN,INF,1.0e20] as $v){$a=$readings;$a[101]['value']=$v;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && is_string(json_encode($x,JSON_THROW_ON_ERROR)),'invalid numeric represented safely');} +$a=$readings;$a[101]['parentID']=999;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && $x['assessment']===null,'reused ID not recorded as correct measurement'); +$a=$readings;$a[102]['updated']=$now+1;$x=take($c,$a,$now);ck($x['assessment']===null && in_array('s1:future_source',$x['issues'],true),'future timestamp marked'); +$a=$readings;$a[101]['updated']=0;$x=take($c,$a,$now);ck($x['assessment']===null,'absent timestamp not invented'); +$counts=[];$x=Capture::capture($c,static function($id)use(&$counts,$readings){$r=$readings[$id];$counts[$id]=($counts[$id]??0)+1;if($id===101 && $counts[$id]===2)$r['value']++;return $r;},static fn()=>$now); +ck($x['assessment']===null && $x['raw']['s0']['secondValue']===3001,'same-second concurrent change visible'); +$x=Capture::capture($c,static function($id)use($readings){if($id===101)throw new RuntimeException('do-not-emit-secret');return $readings[$id];},static fn()=>$now); +ck(!str_contains(json_encode($x),'do-not-emit-secret') && $x['assessment']===null,'source errors bounded without secret messages'); +$a=$readings;$a[106]['value']=null;$x=take($c,$a,$now);ck($x['assessment']!==null && !$x['trainingEligible'],'optional SOC error does not destroy available raw powers'); +$a=$readings;$a[101]['value']=-10000;$x=take($c,$a,$now);ck($x['assessment']['baseLoadCandidateW']<0 && $x['assessment']['status']==='quality_hold','negative candidate not clipped'); +$a=$c;$a['extraSources'][]=$a['extraSources'][0];refuses(fn()=>take($a,$readings,$now),'duplicate source rejected'); +$a=$c;$a['accounting']['sources']=array_reverse($a['accounting']['sources']);ck(take($a,$readings,$now)['mappingSha256']===$r['mappingSha256'],'mapping identity stable'); +$t=0;refuses(fn()=>Capture::capture($c,static fn($id)=>$readings[$id],static function()use(&$t,$now){return $now-$t++;}),'backwards clock rejected'); +$base=sys_get_temp_dir().'/v4-raw-test-'.bin2hex(random_bytes(5));mkdir($base,0700);$dir=$base.'/data';mkdir($dir,0700); +try{ + $file=Capture::append($dir,$r);$first=file_get_contents($file);Capture::append($dir,$r); + ck(file_get_contents($file)===$first.$first,'append retains existing observations'); + ck(count(file($file))===2,'one record per line'); + ck((fileperms($file)&0777)===0640,'data permissions restricted'); + $old=file_get_contents($file);refuses(fn()=>Capture::append($dir,$r,1),'quota refuses without deleting data');ck(file_get_contents($file)===$old,'quota leaves journal unchanged'); + file_put_contents($file,'partial',FILE_APPEND);refuses(fn()=>Capture::append($dir,$r),'crash-truncated tail not silently extended');ck(str_ends_with(file_get_contents($file),'partial'),'incomplete record retained for review'); + unlink($file);file_put_contents($base.'/foreign','untouched');symlink($base.'/foreign',$file);refuses(fn()=>Capture::append($dir,$r),'journal symlink refused');ck(file_get_contents($base.'/foreign')==='untouched','unrelated file preserved');unlink($file); + $lock=fopen($dir.'/.writer.lock','c+b');flock($lock,LOCK_EX);refuses(fn()=>Capture::append($dir,$r),'parallel writer refused');flock($lock,LOCK_UN);fclose($lock); + $a=$r;$a['blob']=str_repeat('x',300000);refuses(fn()=>Capture::append($dir,$a),'oversized record refused'); +}finally{foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($base,FilesystemIterator::SKIP_DOTS),RecursiveIteratorIterator::CHILD_FIRST)as$p){$p->isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base);} +$source=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Messaufnahme.php'); +ck(!preg_match('/\b(?:IPS_|GetValue|SetValue|curl_|RequestAction|shell_exec|exec\s*\()/',$source),'capture core has no device or network calls'); +echo "TOTAL $n checks passed; synthetic values and temporary files only.\n"; diff --git a/tests/V4DataCapture/native_checks.php b/tests/V4DataCapture/native_checks.php new file mode 100644 index 0000000..1b2f062 --- /dev/null +++ b/tests/V4DataCapture/native_checks.php @@ -0,0 +1,61 @@ +$o['ParentID']===$id));} +function IPS_InstanceExists($id){return $id===17004;} +function IPS_GetInstance($id){return ['ModuleInfo'=>['ModuleID'=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']];} +function IPS_VariableExists($id){return isset($GLOBALS['values'][$id]);} +function IPS_GetVariable($id){$v=$GLOBALS['values'][$id];return ['VariableType'=>is_string($v)?3:2,'VariableUpdated'=>time(),'VariableChanged'=>time()-10];} +function GetValue($id){return $GLOBALS['values'][$id];} +function SetValue($id,$v){if($id<60000||IPS_GetObject($id)['ObjectIdent']!=='CaptureStatus')throw new RuntimeException('Foreign variable write');$GLOBALS['values'][$id]=$v;return true;} +function IPS_SemaphoreEnter($key,$wait){if($GLOBALS['locks'][$key]??false)return false;$GLOBALS['locks'][$key]=true;return true;} +function IPS_SemaphoreLeave($key){$GLOBALS['locks'][$key]=false;} +function makeObj($type){$id=$GLOBALS['nextId']++;$GLOBALS['objects'][$id]=['ObjectType'=>$type,'ObjectIdent'=>'','ParentID'=>0,'ObjectInfo'=>''];return $id;} +function IPS_CreateCategory(){return makeObj(0);} +function IPS_CreateScript($type){$id=makeObj(3);$GLOBALS['scripts'][$id]=''synthetic_test','physicalKwh'=>322,'evKwh'=>160];file_put_contents($base.'/USER_REPORTED_INVENTORY.json',json_encode($info)); +$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004,'reportedInventorySha256'=>hash_file('sha256',$base.'/USER_REPORTED_INVENTORY.json'), + 'extraSources'=>[],'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]]]; +foreach(['grid','pv','physical_storage','controlled_virtual','external_virtual']as$i=>$role){$id=101+$i;$c['accounting']['sources'][]=['key'=>'x'.$i,'variableId'=>$id,'parentId'=>200,'ident'=>'v'.$i,'role'=>$role,'factorToW'=>1,'maxAgeSeconds'=>60,'dependsOn'=>[]];$objects[$id]=['ParentID'=>200,'ObjectIdent'=>'v'.$i,'ObjectType'=>2,'ObjectInfo'=>'original'];$values[$id]=[1000,2000,500,500,0][$i];} +file_put_contents($base.'/config.json',json_encode($c));$hashes['config.json']=hash_file('sha256',$base.'/config.json');$hashes['USER_REPORTED_INVENTORY.json']=hash_file('sha256',$base.'/USER_REPORTED_INVENTORY.json'); +file_put_contents($base.'/MANIFEST.json',json_encode(['scope'=>'passive_raw_capture_only','hashes'=>$hashes])); +$before=$objects;$beforeValues=$values; +try{ + ob_start();$r=v4captureInstall($base);ob_end_clean(); + yes($r['status']==='capture_installed','passive installation completes'); + yes(count($r['createdObjects'])===4,'only category, two scripts and one status variable'); + yes($timers[$r['scriptId']]===30,'only new capture timer runs at 30 seconds'); + yes($r['firstCapture']['status']==='recorded'&&count(file($r['firstCapture']['journal']))===1,'first capture actually written before activation'); + yes($r['firstCapture']['accountingQuality']==='consistent_candidate','candidate is not a training permit'); + foreach($before as$id=>$o){yes($objects[$id]===$o && $values[$id]===$beforeValues[$id],'existing source preserved '.$id);} + $size=count($objects);ob_start();$again=v4captureInstall($base);ob_end_clean();yes(count($objects)===$size&&$again['createdObjects']===[],'installation idempotent, no duplicate timers'); + $bad=$scripts[$r['scriptId']];$scripts[$r['scriptId']]="isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base);} +echo "TOTAL $count native-mock checks passed; no Symcon kernel calls.\n"; From 44362e4bf5fb2bc599046567961279fcfdda17a2 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 17:00:32 +0000 Subject: [PATCH 08/22] feat(observation): separate physical load and unfiltered EV-SDL model without dispatch --- docs/netplan-v4-separated-observation.md | 88 ++++++++++ examples/V4SeparatedObservation/install.php | 59 +++++++ examples/V4SeparatedObservation/runtime.php | 89 ++++++++++ libs/NetzfahrplanV4Messaufnahme.php | 2 +- libs/NetzfahrplanV4Messsicht.php | 162 ++++++++++++++++++ tests/NetzfahrplanV4MesssichtTest.php | 16 ++ tests/V4SeparatedObservation/checks.php | 59 +++++++ tests/V4SeparatedObservation/fixture.php | 14 ++ .../V4SeparatedObservation/native_checks.php | 61 +++++++ 9 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 docs/netplan-v4-separated-observation.md create mode 100644 examples/V4SeparatedObservation/install.php create mode 100644 examples/V4SeparatedObservation/runtime.php create mode 100644 libs/NetzfahrplanV4Messsicht.php create mode 100644 tests/NetzfahrplanV4MesssichtTest.php create mode 100644 tests/V4SeparatedObservation/checks.php create mode 100644 tests/V4SeparatedObservation/fixture.php create mode 100644 tests/V4SeparatedObservation/native_checks.php diff --git a/docs/netplan-v4-separated-observation.md b/docs/netplan-v4-separated-observation.md new file mode 100644 index 0000000..7647d28 --- /dev/null +++ b/docs/netplan-v4-separated-observation.md @@ -0,0 +1,88 @@ +# Separate physical load / EV-SDL observation (NO dispatch or training permission) + +## Implemented 2026-10-02 + +- `NetzfahrplanV4Messsicht` recalculates independent physical-load, physical-storage, + unfiltered modelled allocation and SolarEdge-origin views from explicitly identified + raw snapshots. Quality failures in held virtual accounts do not veto otherwise + consistent physical inputs. Missing/stale physical inputs still invalidate their + respective view. No automatic relaxation of age/skew limits. +- EV/SDL model: delta = physical - requestedEV - requestedSDL; distribute delta in + proportion to the absolute requested powers. The SDL component is obtained from + physical minus modelled EV to preserve numerical closure. This is an accounting + convention, NOT independently measured EV and SDL and NOT a verified response. +- Command changes more recent than the oldest physical feedback are flagged. Unknown + change timestamps, tracking mismatch, aged/missing origins and idle unallocated + power remain explicit. There is no held-value filter and no forced convergence of + displayed power to the requested value. All proposals keep `canDispatch=false`. +- Actual legacy filter outputs are displayed solely for comparison. They are not + reused as inputs of physical-load or unfiltered partition calculations. No changes + to the legacy filter, physical driver, EV/SDL setpoints, SOC integration or accounts. + +## New read-only origin finding + +Selected event 38921 writes legacy PV variable 20335 using: + + a = 10 ** GetValue(41853) * GetValue(37975) + GetValue(21447) + if (a < 0) a = 0 + SetValue(20335, a) + +Arithmetic statements were inspected without exposing arbitrary scripts/credentials. +Verified saved configuration: variable 37975 parent 35514, register 40083; scale +41853 parent 48996, register 40084; battery 21447 parent 30789. An independently +updated 20335 is therefore not an independent PV reading and cannot certify all +input timestamps. A second constrained arithmetic read confirmed the condition +`if ($a < 0)`; this clamp applies to the legacy calculated PV value, not to the +new signed terminal-power observation. Neither proves the full AC/DC boundary. + +The staged observer additionally records the ORIGINAL mantissa and exponent, not +only their periodically rewritten derived PV result. The alternative candidate +uses the scaled terminal power ONCE for this inverter (removing its derived-PV +and battery terms). GoodWe AC/DC boundaries and full-site physical boundary are +still unverified. This is not a definitive corrected house meter or live forecast. + +## Standalone installation / operational scope + +Stage on test host: `/srv/agent/netplan-v4-separated-observer-stage`. +User invocation inside Symcon only: + + require '/srv/agent/netplan-v4-separated-observer-stage/install.php'; + +Creates only a NEW root category, three textual diagnostic variables, observe and +stop scripts and a NEW 30-second timer. Reads 23 existing numeric variables twice; +no Modbus polling requests, network calls, archive settings, existing module or +control timer updates. Existing 21-channel raw collector continues untouched. + +New daily journals contain the selected numeric raw snapshot and derived views. +A new immutable mapping/inventory reference binds the corrected user EV allocation +161.44 kWh / 39 kW. Original collector metadata and its correction addendum remain +unchanged. No rescaling or resetting of virtual SOC/energy accounts. + +Derived journal and LATEST/STATUS reports are 0644 WITHIN mode-0700 private stage +and data directories, allowing the authorized agent to inspect these selected +new outputs without repeated manual exports. No permissions on original journals, +Symcon directories or unrelated data are changed. Quotas reuse 64 MiB/day, 512 MiB +in total and 100 MiB disk reserve; exceeding them stops recording visibly and does +not delete old data. No network transfer or cloud trainer is activated. + +## Validation and limitations + +40 pure view checks + 32 native-mock checks passed on PHP 8.3 CLI at test host. +Regression: 36 raw-capture + 19 native-capture + 34 accounting checks passed with +new additive `raw_power`/`scale` units. Total 161 isolated checks. Full relevant PHP +source lint passed. Not a full PHPUnit run, Symcon kernel trial or hardware test. + +Tests cover opposite EV/SDL requests, held output, non-unique partition labelling, +negative terminal power, scale/sentinel validity, old dependency timestamps, changed +commands, missing source, no duplicate native objects, unchanged existing signals, +append preservation, source drift, private output restrictions and stop behavior. + +The observer is PREPARED, not started by the assistant. It does not replace the +installed receiver or its existing preview calculation. The preview/driver must +only be switched after measurement-boundary and feedback policy acceptance. Model +training and five-minute reconstruction must consume the new versioned snapshots +separately; there is no training eligibility or zero future-SDL assumption here. + +Device-side command-loss protection and exact model/firmware identification remain +open. Official SolarEdge PDF fetch was unavailable (403/error); no device-specific +register suitability or watchdog behavior was inferred from a third-party source. diff --git a/examples/V4SeparatedObservation/install.php b/examples/V4SeparatedObservation/install.php new file mode 100644 index 0000000..902d259 --- /dev/null +++ b/examples/V4SeparatedObservation/install.php @@ -0,0 +1,59 @@ +IPS_CreateCategory(),2=>IPS_CreateVariable(3),3=>IPS_CreateScript(0)}; + if(!is_int($id)||$id<=0)throw new RuntimeException('Cannot create observer object');$created[]=$id; + if(!IPS_SetParent($id,$parent)||!IPS_SetIdent($id,$ident)||!IPS_SetName($id,$name)||!IPS_SetInfo($id,$marker))throw new RuntimeException('Cannot label observer object'); + return $id; +} +function v4viewsScript(int $id,string $source,array $created): void +{ + if(IPS_GetScriptContent($id)===$source)return; + if(!in_array($id,$created,true))throw new RuntimeException('Existing observer script differs; not overwritten'); + if(!IPS_SetScriptContent($id,$source)||IPS_GetScriptContent($id)!==$source)throw new RuntimeException('Cannot write observer script'); +} +function v4viewsInstall(string $stage): array +{ + if(!function_exists('IPS_GetKernelVersion')||version_compare(IPS_GetKernelVersion(),'8.0','<'))throw new RuntimeException('Nur im IP-Symcon-Skripteditor ausfuehren.'); + [$c,$p]=v4viewsConfig($stage);$marker=v4viewsMarker($c); + if(!IPS_InstanceExists($c['managerId'])||IPS_GetInstance($c['managerId'])['ModuleInfo']['ModuleID']!=='{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}')throw new RuntimeException('Expected manager missing'); + foreach(array_merge($c['accounting']['sources'],$c['extraSources']) as $s){ + if(!IPS_VariableExists($s['variableId']))throw new RuntimeException('Expected observation source missing'); + $o=IPS_GetObject($s['variableId']);$v=IPS_GetVariable($s['variableId']); + if($o['ParentID']!==$s['parentId']||$o['ObjectIdent']!==$s['ident']||!in_array($v['VariableType'],[1,2],true))throw new RuntimeException('Source identity differs'); + } + if(is_link($stage.'/data')||!is_dir($stage.'/data')||!is_writable($stage.'/data')||(fileperms($stage.'/data')&0077)!==0)throw new RuntimeException('Private data directory required'); + $lock='ENELIX.V4.ViewsInstall.'.$c['installationId'];if(!IPS_SemaphoreEnter($lock,3000))throw new RuntimeException('Observer installation busy'); + $created=[];$script=null;$newTimer=false;$report=['scope'=>'passive_separated_observation','startedAt'=>gmdate('c'), + 'oldCollectorChanged'=>false,'modulesChanged'=>false,'pollingChanged'=>false,'socAccountsChanged'=>false,'trainingEnabled'=>false,'controlEnabled'=>false]; + try{ + $cat=v4viewsEnsure(0,'ENELIX_V4_SEPARATED_OBSERVATION',0,'V4 - getrennte Last und EV-SDL (nur Beobachtung)',$marker,$created); + foreach(['PhysicalLoad'=>'Physische Last - Kandidat, keine Trainingsfreigabe','AllocationModel'=>'EV-SDL-Modell - KEINE separate Messung','ObservationStatus'=>'Beobachtungsstatus'] as $key=>$name)v4viewsEnsure($cat,$key,2,$name,$marker,$created); + $script=v4viewsEnsure($cat,'Observe',3,'Getrennte Messsicht alle 30 Sekunden',$marker,$created); + $stop=v4viewsEnsure($cat,'Stop',3,'Nur diese zusaetzliche Beobachtung stoppen',$marker,$created); + $include=''observer_installed','categoryId'=>$cat,'scriptId'=>$script,'stopId'=>$stop,'firstObservation'=>$first,'futureTimerExecutionVerified'=>false]; + echo "OK: Getrennte Last-/EV-SDL-Beobachtung eingerichtet, alle 30 Sekunden.\n"; + echo 'Erste Aufnahme: '.$first['capturedAt'].'; Quellen: '.$first['sourceCount'].".\n"; + echo 'Physische Last: '.$first['physicalLoadStatus'].'; EV-/SDL-Modell: '.$first['allocationModelStatus'].".\n"; + echo "Keine Trainings-/Stellfreigabe. Bisheriger Sammler, Manager, Gateway, SDL und Polling unveraendert.\n"; + }catch(Throwable $e){ + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,180); + if($newTimer&&$script!==null)IPS_SetScriptTimer($script,0); + echo 'FEHLER: '.$report['reason']."\n"; + }finally{ + $report['createdObjects']=$created;$report['finishedAt']=gmdate('c'); + try{v4viewsJson($stage.'/INSTALL_RESULT.json',$report);}finally{IPS_SemaphoreLeave($lock);} + echo 'OBSERVATION REPORT: '.$stage."/INSTALL_RESULT.json\n"; + } + return $report; +} diff --git a/examples/V4SeparatedObservation/runtime.php b/examples/V4SeparatedObservation/runtime.php new file mode 100644 index 0000000..62171ce --- /dev/null +++ b/examples/V4SeparatedObservation/runtime.php @@ -0,0 +1,89 @@ +GetValue($id),'updated'=>(int)$a['VariableUpdated'],'changed'=>(int)$a['VariableChanged'],'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; +} +function v4viewsJson(string $path,array $data): void +{ + if(is_link($path)||realpath(dirname($path))!==dirname($path)||(fileperms(dirname($path))&0077)!==0)throw new RuntimeException('Private observer output required'); + $tmp=tempnam(dirname($path),'.views-');if($tmp===false)throw new RuntimeException('Cannot create observer report'); + try{$json=json_encode($data,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT|JSON_PRESERVE_ZERO_FRACTION)."\n"; + if(file_put_contents($tmp,$json,LOCK_EX)!==strlen($json)||!chmod($tmp,0644)||!rename($tmp,$path))throw new RuntimeException('Cannot store observer report'); + }finally{if(is_file($tmp))unlink($tmp);} +} +function v4viewsRun(string $stage,int $category): array +{ + if(!function_exists('IPS_GetKernelVersion'))throw new RuntimeException('Run inside Symcon'); + [$c,$policy]=v4viewsConfig($stage);$marker=v4viewsMarker($c); + v4viewsAssert($category,0,'ENELIX_V4_SEPARATED_OBSERVATION',0,$marker); + $ids=[];foreach(['PhysicalLoad','AllocationModel','ObservationStatus'] as $key){ + $id=v4viewsChild($category,$key);if($id===null)throw new RuntimeException('Observer object missing');v4viewsAssert($id,$category,$key,2,$marker);$ids[$key]=$id;} + $lock='ENELIX.V4.SeparatedView.'.$c['installationId'];if(!IPS_SemaphoreEnter($lock,0))return ['status'=>'busy','controlEnabled'=>false]; + try{ + try{ + $raw=\Belevo\EnelixEMS\NetzfahrplanV4Messaufnahme::capture($c,'v4viewsRead',static fn():int=>time()); + $view=\Belevo\EnelixEMS\NetzfahrplanV4Messsicht::calculate($raw,$c,$policy); + $record=['schemaVersion'=>1,'kind'=>'physical_and_allocation_observation','installationId'=>$c['installationId'],'capturedAt'=>$raw['capturedAt'], + 'capture'=>$raw,'views'=>$view,'trainingEligible'=>false,'controlEnabled'=>false]; + $dir=$stage.'/data';if(is_link($dir)||realpath($dir)!==$dir||(fileperms($dir)&0077)!==0)throw new RuntimeException('Private data directory required'); + $path=\Belevo\EnelixEMS\NetzfahrplanV4Messaufnahme::append($dir,$record); + // Derived-only files readable to authorized agent through a mode-0700 stage/data directory. + // Original collector files and permissions are never touched. + if(!chmod($path,0644))throw new RuntimeException('Cannot expose selected derived observation'); + $status=['status'=>'recorded','capturedAt'=>$raw['capturedAt'],'sourceCount'=>count($raw['raw']),'journal'=>$path, + 'physicalLoadStatus'=>$view['physicalLoad']['status'],'allocationModelStatus'=>$view['allocationModel']['status'], + 'solarOriginStatus'=>$view['solarOrigin']['status'],'trainingEligible'=>false,'controlEnabled'=>false]; + v4viewsJson($dir.'/LATEST.json',$record); + SetValue($ids['PhysicalLoad'],json_encode(['captureAt'=>$raw['capturedAt'],'physical'=>$view['physicalLoad'],'solarAcAlternative'=>$view['solarAcAlternative'],'trainingEligible'=>false],JSON_THROW_ON_ERROR)); + SetValue($ids['AllocationModel'],json_encode(['captureAt'=>$raw['capturedAt'],'model'=>$view['allocationModel'],'storage'=>$view['storageFeedback'],'canDispatch'=>false],JSON_THROW_ON_ERROR)); + }catch(Throwable $e){ + $status=['status'=>'observation_error','at'=>gmdate('c'),'errorType'=>get_class($e),'sourceLine'=>$e->getLine(),'trainingEligible'=>false,'controlEnabled'=>false]; + // Never leave a stale numeric proposal in the new diagnosis variables. + SetValue($ids['PhysicalLoad'],json_encode($status,JSON_THROW_ON_ERROR));SetValue($ids['AllocationModel'],json_encode($status,JSON_THROW_ON_ERROR)); + try{v4viewsJson($stage.'/data/LATEST.json',$status);}catch(Throwable $ignored){} + } + SetValue($ids['ObservationStatus'],json_encode($status,JSON_THROW_ON_ERROR));v4viewsJson($stage.'/data/STATUS.json',$status);return $status; + }finally{IPS_SemaphoreLeave($lock);} +} +function v4viewsStop(string $stage,int $category): void +{ + [$c,$p]=v4viewsConfig($stage);$marker=v4viewsMarker($c);v4viewsAssert($category,0,'ENELIX_V4_SEPARATED_OBSERVATION',0,$marker); + $id=v4viewsChild($category,'Observe');if($id===null)throw new RuntimeException('Observer script missing');v4viewsAssert($id,$category,'Observe',3,$marker); + if(!IPS_SetScriptTimer($id,0))throw new RuntimeException('Cannot stop observer'); + $s=['status'=>'stopped','at'=>gmdate('c'),'controlEnabled'=>false,'existingCollectorChanged'=>false]; + foreach(['PhysicalLoad','AllocationModel','ObservationStatus'] as $key){$var=v4viewsChild($category,$key);if($var===null)throw new RuntimeException('Observer status missing');v4viewsAssert($var,$category,$key,2,$marker);SetValue($var,json_encode($s,JSON_THROW_ON_ERROR));} + v4viewsJson($stage.'/data/STATUS.json',$s);v4viewsJson($stage.'/data/LATEST.json',$s); +} diff --git a/libs/NetzfahrplanV4Messaufnahme.php b/libs/NetzfahrplanV4Messaufnahme.php index e818ba2..19af299 100644 --- a/libs/NetzfahrplanV4Messaufnahme.php +++ b/libs/NetzfahrplanV4Messaufnahme.php @@ -33,7 +33,7 @@ final class NetzfahrplanV4Messaufnahme || !is_int($s['variableId'] ?? null) || $s['variableId'] < 1 || isset($ids[$s['variableId']]) || !is_int($s['parentId'] ?? null) || $s['parentId'] < 1 || !is_string($s['ident'] ?? null) || strlen($s['ident']) > 100 - || !in_array($s['unit'] ?? null, ['W','kWh','percent'], true)) { + || !in_array($s['unit'] ?? null, ['W','kWh','percent','raw_power','scale'], true)) { throw new InvalidArgumentException('Invalid or duplicate raw source'); } $keys[$s['key']] = true; $ids[$s['variableId']] = true; diff --git a/libs/NetzfahrplanV4Messsicht.php b/libs/NetzfahrplanV4Messsicht.php new file mode 100644 index 0000000..b390b19 --- /dev/null +++ b/libs/NetzfahrplanV4Messsicht.php @@ -0,0 +1,162 @@ + 300 + || !self::numeric($p['trackingToleranceW']) || $p['trackingToleranceW'] < 0 || $p['trackingToleranceW'] > 1000) { + throw new InvalidArgumentException('Invalid measurement-view policy'); + } + $extra = array_column($c['extraSources'], null, 'key'); + foreach (['evRequestKey','sdlRequestKey'] as $k) { + if (!is_string($p[$k]) || !isset($extra[$p[$k]]) || $extra[$p[$k]]['unit'] !== 'W') { + throw new InvalidArgumentException('Explicit separate request channels required'); + } + } + if ($p['evRequestKey'] === $p['sdlRequestKey']) throw new InvalidArgumentException('Duplicate request channel'); + if ($p['solarReference'] !== null) { + $s=$p['solarReference']; $keys=['pvKey','batteryKey','rawKey','scaleKey']; + if (!is_array($s) || count($s)!==4 || array_diff($keys,array_keys($s))) throw new InvalidArgumentException('Invalid solar-origin policy'); + foreach ($keys as $k) if (!is_string($s[$k])) throw new InvalidArgumentException('Invalid solar-origin key'); + $primary=array_column($c['accounting']['sources'],null,'key'); + if (($primary[$s['pvKey']]['role']??null)!=='pv' || ($primary[$s['batteryKey']]['role']??null)!=='physical_storage' + || ($extra[$s['rawKey']]['unit']??null)!=='raw_power' || ($extra[$s['scaleKey']]['unit']??null)!=='scale') { + throw new InvalidArgumentException('Solar references must be explicit original sources'); + } + } + return $p; + } + + public static function calculate(array $r, array $config, array $policy): array + { + $c=NetzfahrplanV4Messaufnahme::configuration($config);$p=self::policy($policy,$c); + $mapping=hash('sha256',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + if (($r['kind']??null)!=='raw_accounting_capture' || ($r['schemaVersion']??null)!==1 + || ($r['installationId']??null)!==$c['installationId'] || ($r['mappingSha256']??null)!==$mapping + || ($r['reportedInventorySha256']??null)!==$c['reportedInventorySha256']) throw new InvalidArgumentException('Wrong capture identity'); + foreach (['capturedAt','captureStartedAt'] as $k) { + if (!is_string($r[$k]??null) || !preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$/D',$r[$k])) throw new InvalidArgumentException('Explicit UTC capture time required'); + $date=new \DateTimeImmutable($r[$k]);$err=\DateTimeImmutable::getLastErrors(); + if ($err!==false && ($err['warning_count']||$err['error_count'])) throw new InvalidArgumentException('Invalid capture time'); + $time[$k]=$date->getTimestamp(); + } + $now=$time['capturedAt'];$duration=$now-$time['captureStartedAt']; + if ($duration<0) throw new InvalidArgumentException('Reversed acquisition clock'); + $sources=array_column(array_merge($c['accounting']['sources'],$c['extraSources']),null,'key'); + $v=[]; + foreach ($sources as $key=>$s) { + $raw=$r['raw'][$key]??[]; $issues=[]; $value=$raw['value']??null; $updated=$raw['sourceUpdatedAt']??null; + if (($raw['variableId']??null)!==$s['variableId']) {$issues[]='identity';$value=null;} + if (!self::numeric($value)) {$issues[]='not_numeric';$value=null;} + if (!is_int($updated) || $updated<=0) {$issues[]='timestamp';$updated=null;} + if (($raw['issues']??null)!==[]) $issues[]='capture_quality'; // never copy arbitrary source text + if ($updated!==null && ($updated>$now || $now-$updated>($s['maxAgeSeconds']??$p['extraMaxAgeSeconds']))) $issues[]='source_age'; + $scaled=$value===null?null:(float)$value*($s['factorToW']??1.0); + if ($scaled!==null && (!is_finite($scaled)||abs($scaled)>1e12)) {$scaled=null;$issues[]='converted_value';} + $changed=$raw['sourceChangedAt']??null; + if (!is_int($changed)||$changed<0||$updated===null||$changed>$updated) $changed=null; + $v[$key]=['value'=>$scaled,'updated'=>$updated,'changed'=>$changed,'issues'=>$issues]; + } + $closure=function(array $keys)use(&$closure,$sources):array { + $out=[];foreach($keys as $key){if(!isset($sources[$key]))throw new InvalidArgumentException('Missing origin');$out[]=$key; + foreach($closure($sources[$key]['dependsOn']??[]) as $dep)$out[]=$dep;} + return array_values(array_unique($out)); + }; + $quality=function(array $keys)use($closure,$v,$c,$duration):array { + $keys=$closure($keys);$issues=[];$ts=[]; + foreach($keys as $key){foreach($v[$key]['issues'] as $i)$issues[]=$key.':'.$i;if($v[$key]['updated']!==null)$ts[]=$v[$key]['updated'];} + $skew=$ts===[]?null:max($ts)-min($ts); + if($duration>5)$issues[]='long_acquisition'; + if($skew!==null && $skew>$c['accounting']['maxSkewSeconds'])$issues[]='source_time_skew'; + return ['status'=>$issues===[]?'consistent_candidate':'quality_hold','issues'=>$issues,'oldestSourceUpdatedAt'=>$ts===[]?null:min($ts),'sourceSkewSeconds'=>$skew]; + }; + $byRole=[];foreach($c['accounting']['sources'] as $s)$byRole[$s['role']][]=$s['key']; + $physical=$byRole['physical_storage'];$pv=$byRole['pv'];$grid=$byRole['grid'];$flex=$byRole['flexible_load']??[]; + $sum=function(array $keys)use($v):?float { $total=0.0;foreach($keys as $key){if($v[$key]['value']===null)return null;$total+=$v[$key]['value'];}return $total;}; + $storage=$quality($physical);$storage['physicalStorageCandidateW']=$sum($physical); + $solar=['status'=>'not_configured','acTerminalCandidateW'=>null,'reconstructedPvCandidateW'=>null,'derivedPvDifferenceW'=>null,'issues'=>['origin_not_configured']]; + $loadKeys=array_merge($grid,$pv,$physical,$flex);$sr=$p['solarReference']; + if($sr!==null){ + $origin=[$sr['rawKey'],$sr['scaleKey'],$sr['batteryKey']];$loadKeys=array_merge($loadKeys,$origin); + $solar=$quality($origin);$raw=$v[$sr['rawKey']]['value'];$sf=$v[$sr['scaleKey']]['value']; + $solar+=['acTerminalCandidateW'=>null,'reconstructedPvCandidateW'=>null,'derivedPvDifferenceW'=>null]; + // int16 invalid sentinel excluded, including scale sentinel; no unsigned guessing. + if($raw===null||$sf===null||floor($raw)!==$raw||$raw<=-32768||$raw>32767||floor($sf)!==$sf||$sf < -6||$sf>6){ + $solar['status']='quality_hold';$solar['issues'][]='invalid_raw_or_scale'; + }else{ + $ac=$raw*(10**(int)$sf); + if(abs($ac)>1e9){$solar['status']='quality_hold';$solar['issues'][]='scaled_power_range';} + else{$solar['acTerminalCandidateW']=$ac;$b=$v[$sr['batteryKey']]['value']; + if($b!==null)$solar['reconstructedPvCandidateW']=$ac+$b; + $reported=$v[$sr['pvKey']]['value']; + if($reported!==null&&$solar['reconstructedPvCandidateW']!==null)$solar['derivedPvDifferenceW']=$reported-max(0.0,$solar['reconstructedPvCandidateW']);} + } + } + $load=$quality($loadKeys);$net=$sum($grid);$pvSum=$sum($pv);$phys=$sum($physical);$fl=$sum($flex); + $candidate=($net===null||$pvSum===null||$phys===null||$fl===null)?null:$net+$pvSum-$phys-$fl; + if($candidate!==null&&$candidate<0){$load['status']='quality_hold';$load['issues'][]='negative_load';} + foreach(array_merge($pv,$flex) as $key)if($v[$key]['value']!==null&&$v[$key]['value']<0){$load['status']='quality_hold';$load['issues'][]='negative_unsigned:'.$key;} + if($sr!==null&&$solar['status']!=='consistent_candidate'){$load['status']='quality_hold';$load['issues'][]='solar_origin_unverified_value';} + $load+=['loadCandidateW'=>$candidate,'gridW'=>$net,'pvCandidateW'=>$pvSum,'physicalStorageCandidateW'=>$phys,'flexibleLoadW'=>$fl, + 'virtualAllocationUsed'=>false,'measurementBoundaryVerified'=>false]; + // AC terminal alternative eliminates SolarEdge battery/PV cancellation and zero-clamp artefact. + $alternative=null;$altQuality=null; + if($sr!==null){ + $altKeys=array_values(array_diff(array_merge($grid,$pv,$physical,$flex),[$sr['pvKey'],$sr['batteryKey']])); + $altKeys=array_merge($altKeys,[$sr['rawKey'],$sr['scaleKey']]);$altQuality=$quality($altKeys); + $pvRest=$sum(array_values(array_diff($pv,[$sr['pvKey']])));$bRest=$sum(array_values(array_diff($physical,[$sr['batteryKey']]))); + if($net!==null&&$pvRest!==null&&$bRest!==null&&$fl!==null&&$solar['acTerminalCandidateW']!==null){ + $alternative=$net+$pvRest-$bRest+$solar['acTerminalCandidateW']-$fl; + if($alternative<0){$altQuality['status']='quality_hold';$altQuality['issues'][]='negative_load';} + }else{$altQuality['status']='quality_hold';$altQuality['issues'][]='solar_ac_unavailable';} + foreach(array_merge(array_values(array_diff($pv,[$sr['pvKey']])),$flex) as $key)if($v[$key]['value']!==null&&$v[$key]['value']<0){$altQuality['status']='quality_hold';$altQuality['issues'][]='negative_unsigned:'.$key;} + $altQuality+=['loadCandidateW'=>$alternative,'measurementBoundaryVerified'=>false,'solarBatterySubtractedAgain'=>false]; + } + $requestKeys=[$p['evRequestKey'],$p['sdlRequestKey']];$alloc=$quality(array_merge($physical,$requestKeys)); + $ev=$v[$p['evRequestKey']]['value'];$sdl=$v[$p['sdlRequestKey']]['value'];$evModel=null;$sdlModel=null;$unallocated=$phys;$delta=null; + foreach($requestKeys as $key){ + if($v[$key]['changed']===null){$alloc['issues'][]='request_change_time_unknown:'.$key;$alloc['status']='quality_hold';} + elseif($storage['oldestSourceUpdatedAt']!==null && $v[$key]['changed']>$storage['oldestSourceUpdatedAt']){$alloc['issues'][]='request_newer_than_feedback:'.$key;$alloc['status']='quality_hold';} + } + if($phys!==null&&$ev!==null&&$sdl!==null){ + $abs=abs($ev)+abs($sdl);$delta=$phys-$ev-$sdl; + if($abs>=1.0){$evModel=$ev+$delta*abs($ev)/$abs;$sdlModel=$phys-$evModel;$unallocated=0.0; + if(abs($delta)>$p['trackingToleranceW']){$alloc['issues'][]='physical_request_tracking_error';$alloc['status']='quality_hold';} + }else{$alloc['issues'][]='idle_power_unallocated';$alloc['status']='quality_hold';} + } + $reportedEv=$sum($byRole['controlled_virtual']);$reportedSdl=$sum($byRole['external_virtual']??[]); + $reportedResidual=($phys===null||$reportedEv===null||$reportedSdl===null)?null:$phys-$reportedEv-$reportedSdl; + $alloc+=['physicalStorageCandidateW'=>$phys,'requestedEvW'=>$ev,'requestedSdlW'=>$sdl,'unfilteredModelEvW'=>$evModel, + 'unfilteredModelSdlW'=>$sdlModel,'unallocatedPhysicalW'=>$unallocated,'physicalRequestErrorW'=>$delta, + 'reportedEvW'=>$reportedEv,'reportedSdlW'=>$reportedSdl,'reportedAllocationResidualW'=>$reportedResidual, + 'reportedVirtualQuality'=>$quality(array_merge($byRole['controlled_virtual'],$byRole['external_virtual']??[])), + 'underlyingTimesMeaning'=>'Symcon VariableUpdated; not independently measured EV/SDL channel times', + 'reportedAllocationWithinTolerance'=>$reportedResidual!==null&&abs($reportedResidual)<=$c['accounting']['splitToleranceW'], + 'separatelyMeasured'=>false,'method'=>'request_weighted_residual_unfiltered_not_unique_measurement']; + return ['schemaVersion'=>1,'kind'=>'separated_measurement_views','installationId'=>$c['installationId'],'capturedAt'=>$r['capturedAt'], + 'mappingSha256'=>$mapping,'policySha256'=>hash('sha256',json_encode($p,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)), + 'physicalLoad'=>$load,'storageFeedback'=>$storage,'allocationModel'=>$alloc,'solarOrigin'=>$solar,'solarAcAlternative'=>$altQuality, + 'trainingEligible'=>false,'canDispatch'=>false,'controlEnabled'=>false,'socAccountsChanged'=>false,'futureSdl'=>'unknown_no_assumption', + 'note'=>'Observation only. Source times are Symcon update times. Algebraic partition is not separate EV/SDL measurement. GoodWe and combined AC/DC boundaries remain unverified.']; + } +} diff --git a/tests/NetzfahrplanV4MesssichtTest.php b/tests/NetzfahrplanV4MesssichtTest.php new file mode 100644 index 0000000..cc21418 --- /dev/null +++ b/tests/NetzfahrplanV4MesssichtTest.php @@ -0,0 +1,16 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]);fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err);self::assertStringContainsString('TOTAL ',$out);self::assertSame('',$err); + } + public function scripts(): array {return [['checks.php'],['native_checks.php']];} +} diff --git a/tests/V4SeparatedObservation/checks.php b/tests/V4SeparatedObservation/checks.php new file mode 100644 index 0000000..45c13c1 --- /dev/null +++ b/tests/V4SeparatedObservation/checks.php @@ -0,0 +1,59 @@ +$r[$id],static fn()=>$now),$c,$p);} +[$c,$p,$r,$now]=separatedFixture();$x=take($c,$p,$r,$now); +ck($x['physicalLoad']['loadCandidateW']===6000.,'physical sources subtracted once'); +ck($x['solarAcAlternative']['loadCandidateW']===6000.,'solar AC terminal used without second battery subtraction'); +ck($x['solarOrigin']['reconstructedPvCandidateW']===1000.,'original mantissa scale and battery retained'); +ck($x['allocationModel']['unfilteredModelEvW']===2500.&&$x['allocationModel']['unfilteredModelSdlW']===500.,'unfiltered partition uses physical feedback'); +ck(!$x['canDispatch']&&!$x['trainingEligible']&&!$x['allocationModel']['separatelyMeasured'],'no measurement or control permission invented'); +$a=$r;$a[106]['value']=-9000;$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='consistent_candidate'&&$x['physicalLoad']['loadCandidateW']===6000.,'held virtual output cannot contaminate physical load'); +ck($x['allocationModel']['unfilteredModelEvW']===2500.&&!$x['allocationModel']['reportedAllocationWithinTolerance'],'held EV value only compared not reused'); +$a=$r;$a[107]['value']='never-export-this';$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='consistent_candidate','invalid SDL output does not veto physical load'); +ck(!str_contains(json_encode($x),'never-export-this'),'unknown string is not propagated'); +$a=$r;$a[108]['value']=42000;$a[109]['value']=-40000;$a[104]['value']=2000;$a[105]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===42000.&&$x['allocationModel']['unfilteredModelSdlW']===-40000.,'opposed commands do not divide by near-zero net'); +$a=$r;$a[108]['value']=0;$a[109]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===null&&$x['allocationModel']['unallocatedPhysicalW']===3000.,'idle physical power kept unallocated not invented as EV'); +$a=$r;$a[104]['value']=-3125;$a[105]['value']=0;$a[108]['value']=-10500;$a[109]['value']=0;$a[106]['value']=-8334;$a[107]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===-3125.,'real export-example candidate is physical -3125 not held -8334'); +ck($x['allocationModel']['status']==='quality_hold'&&in_array('physical_request_tracking_error',$x['allocationModel']['issues'],true),'tracking disagreement remains visible'); +$a=$r;$a[108]['changed']=$now;$a[104]['updated']=$now-10;$x=take($c,$p,$a,$now); +ck(in_array('request_newer_than_feedback:req_ev',$x['allocationModel']['issues'],true),'post-command feedback not pretended'); +$a=$r;unset($a[108]['changed']);$x=take($c,$p,$a,$now); +ck(in_array('request_change_time_unknown:req_ev',$x['allocationModel']['issues'],true),'unknown command change time explicit'); +$a=$r;$a[105]['updated']=$now-70;$x=take($c,$p,$a,$now); +ck($x['storageFeedback']['status']==='quality_hold'&&$x['physicalLoad']['status']==='quality_hold','old physical source blocks physical feedback'); +ck($x['solarAcAlternative']['status']==='consistent_candidate','AC alternative does not inherit unrelated solar battery age'); +$a=$r;$a[111]['updated']=$now-70;$x=take($c,$p,$a,$now); +ck($x['solarAcAlternative']['status']==='quality_hold'&&$x['physicalLoad']['status']==='quality_hold','fresh solar derived output cannot hide old scale origin'); +$a=$r;$a[110]['value']=500;$a[111]['value']=1;$a[103]['value']=6000;$x=take($c,$p,$a,$now); +ck($x['solarOrigin']['acTerminalCandidateW']===5000.&&$x['solarAcAlternative']['loadCandidateW']===11000.,'signed scale applied exactly once'); +$a=$r;$a[110]['value']=-500;$a[111]['value']=0;$a[105]['value']=0;$a[103]['value']=0;$x=take($c,$p,$a,$now); +ck($x['solarOrigin']['reconstructedPvCandidateW']===-500.&&$x['solarAcAlternative']['loadCandidateW']===5500.,'negative AC not clamped or disguised as PV'); +foreach([true,'0',INF,NAN,-32768]as$invalid){$a=$r;$a[111]['value']=$invalid;$x=take($c,$p,$a,$now);ck($x['solarOrigin']['acTerminalCandidateW']===null,'invalid scale has no numeric fallback');} +$a=$r;$a[110]['value']=-32768;$x=take($c,$p,$a,$now);ck($x['solarOrigin']['acTerminalCandidateW']===null,'SunSpec invalid power sentinel rejected'); +$a=$r;$a[102]['updated']=$now-40;$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='quality_hold'&&$x['storageFeedback']['status']==='consistent_candidate','slow PV does not veto independent storage feedback'); +$a=$r;$a[108]['updated']=$now-90;$x=take($c,$p,$a,$now);ck($x['allocationModel']['status']==='quality_hold','old request timestamp visible'); +$a=$r;$a[104]['updated']=$now+1;$x=take($c,$p,$a,$now);ck($x['storageFeedback']['status']==='quality_hold','future physical value refused'); +$a=$r;$a[104]['parentID']=999;$x=take($c,$p,$a,$now);ck($x['storageFeedback']['physicalStorageCandidateW']===null,'reused measurement identity not accepted'); +$a=$r;$a[101]['value']=-20000;$x=take($c,$p,$a,$now);ck($x['physicalLoad']['loadCandidateW']<0&&$x['physicalLoad']['status']==='quality_hold','negative load candidate retained as invalid not zero'); +$a=$r;foreach($a as &$v)$v['value']=0;unset($v);$x=take($c,$p,$a,$now);ck($x['physicalLoad']['loadCandidateW']===0.&&$x['physicalLoad']['status']==='consistent_candidate','real zero observation accepted'); +$raw=Capture::capture($c,static fn($id)=>$r[$id],static fn()=>$now);$raw['assessment']['baseLoadCandidateW']=999999.;$x=View::calculate($raw,$c,$p);ck($x['physicalLoad']['loadCandidateW']===6000.,'assessment independently recalculated'); +foreach(['installationId','mappingSha256','reportedInventorySha256']as$key){$bad=$raw;$bad[$key]='wrong';refuses(fn()=>View::calculate($bad,$c,$p),'wrong identity '.$key);} +$bad=$p;$bad['sdlRequestKey']='req_ev';refuses(fn()=>View::calculate($raw,$c,$bad),'duplicate request channels refused'); +$bad=$p;$bad['solarReference']['scaleKey']='soc';refuses(fn()=>View::calculate($raw,$c,$bad),'scale cannot be an arbitrary percent channel'); +$before=json_encode([$raw,$c,$p]);View::calculate($raw,$c,$p);ck(json_encode([$raw,$c,$p])===$before,'inputs not mutated'); +$src=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Messsicht.php');ck(!preg_match('/\b(?:IPS_|GetValue\(|SetValue\(|RequestAction\(|curl_|exec\s*\()/',$src),'pure model has no control or native calls'); +echo "TOTAL $n checks passed. Synthetic observations only.\n"; diff --git a/tests/V4SeparatedObservation/fixture.php b/tests/V4SeparatedObservation/fixture.php new file mode 100644 index 0000000..2f9ee5f --- /dev/null +++ b/tests/V4SeparatedObservation/fixture.php @@ -0,0 +1,14 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004,'reportedInventorySha256'=>str_repeat('a',64), + 'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]],'extraSources'=>[]]; + $defs=[['grid','grid',3000],['pv_g','pv',5000],['pv_se','pv',1000],['bat_g','physical_storage',2000],['bat_se','physical_storage',1000],['ev','controlled_virtual',2500],['sdl','external_virtual',500]]; + $r=[]; + foreach($defs as $i=>[$key,$role,$value]){$id=101+$i;$c['accounting']['sources'][]=['key'=>$key,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>in_array($role,['controlled_virtual','external_virtual'],true)?['bat_g','bat_se']:[]];$r[$id]=['value'=>$value,'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key];} + foreach([['req_ev','W',2500],['req_sdl','W',500],['se_ac','raw_power',0],['se_sf','scale',0],['soc','percent',50]] as $i=>[$key,$unit,$value]){ + $id=108+$i;$c['extraSources'][]=['key'=>$key,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'unit'=>$unit];$r[$id]=['value'=>$value,'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key];} + $p=['version'=>1,'evRequestKey'=>'req_ev','sdlRequestKey'=>'req_sdl','extraMaxAgeSeconds'=>60,'trackingToleranceW'=>100.,'solarReference'=>['pvKey'=>'pv_se','batteryKey'=>'bat_se','rawKey'=>'se_ac','scaleKey'=>'se_sf']]; + return [$c,$p,$r,$now]; +} diff --git a/tests/V4SeparatedObservation/native_checks.php b/tests/V4SeparatedObservation/native_checks.php new file mode 100644 index 0000000..db578a6 --- /dev/null +++ b/tests/V4SeparatedObservation/native_checks.php @@ -0,0 +1,61 @@ +['ModuleID'=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']];} +function IPS_GetChildrenIDs($parent){return array_keys(array_filter($GLOBALS['objects'],fn($o)=>$o['ParentID']===$parent));} +function IPS_GetObject($id){return $GLOBALS['objects'][$id];} +function IPS_GetVariable($id){return $GLOBALS['variables'][$id];} +function IPS_VariableExists($id){return isset($GLOBALS['variables'][$id]);} +function GetValue($id){return $GLOBALS['values'][$id];} +function SetValue($id,$v){if($id<50000||!is_string($v))throw new RuntimeException('FORBIDDEN source/actuator write');$GLOBALS['values'][$id]=$v;return true;} +function IPS_SemaphoreEnter($name,$wait){if($GLOBALS['locks'][$name]??false)return false;$GLOBALS['locks'][$name]=true;return true;} +function IPS_SemaphoreLeave($name){$GLOBALS['locks'][$name]=false;} +function makeObject($type){$id=++$GLOBALS['next'];$GLOBALS['objects'][$id]=['ParentID'=>0,'ObjectIdent'=>'','ObjectType'=>$type,'ObjectInfo'=>''];return $id;} +function IPS_CreateCategory(){return makeObject(0);} +function IPS_CreateVariable($type){ck($type===3,'only new textual observation variables');$id=makeObject(2);$GLOBALS['variables'][$id]=['VariableType'=>3];$GLOBALS['values'][$id]='';return $id;} +function IPS_CreateScript($type){if($type!==0)throw new RuntimeException('script type');$id=makeObject(3);$GLOBALS['scripts'][$id]='';return $id;} +function changeObject($id,$key,$v){if($id<50000)throw new RuntimeException('FORBIDDEN existing object change');$GLOBALS['objects'][$id][$key]=$v;return true;} +function IPS_SetParent($id,$v){return changeObject($id,'ParentID',$v);} +function IPS_SetIdent($id,$v){return changeObject($id,'ObjectIdent',$v);} +function IPS_SetName($id,$v){return changeObject($id,'ObjectName',$v);} +function IPS_SetInfo($id,$v){return changeObject($id,'ObjectInfo',$v);} +function IPS_GetScriptContent($id){return $GLOBALS['scripts'][$id];} +function IPS_SetScriptContent($id,$s){if($id<50000)throw new RuntimeException('existing script');$GLOBALS['scripts'][$id]=$s;return true;} +function IPS_SetScriptTimer($id,$seconds){if($id<50000||!isset($GLOBALS['scripts'][$id])||!in_array($seconds,[0,30],true))throw new RuntimeException('FORBIDDEN timer change');$GLOBALS['timers'][$id]=$seconds;return true;} +[$c,$p,$r,$ignored]=separatedFixture();$now=time();$objects=[];$variables=[];$values=[];$scripts=[];$timers=[];$locks=[];$next=50000; +foreach($r as$id=>$v){$objects[$id]=['ParentID'=>$v['parentID'],'ObjectIdent'=>$v['ident'],'ObjectType'=>2,'ObjectInfo'=>'original'];$variables[$id]=['VariableType'=>is_int($v['value'])?1:2,'VariableUpdated'=>$now,'VariableChanged'=>$now-30];$values[$id]=$v['value'];} +$sourceValues=$values;$base=sys_get_temp_dir().'/v4views-test-'.bin2hex(random_bytes(6));mkdir($base,0700);mkdir($base.'/data',0700); +$files=['source/libs/NetzfahrplanV4Bilanzierung.php','source/libs/NetzfahrplanV4Messaufnahme.php','source/libs/NetzfahrplanV4Messsicht.php','source/examples/V4SeparatedObservation/runtime.php','source/examples/V4SeparatedObservation/install.php']; +foreach($files as$name){$dest=$base.'/'.$name;if(!is_dir(dirname($dest)))mkdir(dirname($dest),0700,true);copy(__DIR__.'/../../'.substr($name,7),$dest);} +file_put_contents($base.'/INVENTORY.json','{"effectiveEvCapacityKwh":161.44,"evPowerKw":39}');$c['reportedInventorySha256']=hash_file('sha256',$base.'/INVENTORY.json'); +file_put_contents($base.'/config.json',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION));file_put_contents($base.'/policy.json',json_encode($p,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); +$manifest=['scope'=>'passive_separated_observation','hashes'=>[]];foreach(array_merge($files,['config.json','policy.json','INVENTORY.json'])as$name)$manifest['hashes'][$name]=hash_file('sha256',$base.'/'.$name);file_put_contents($base.'/MANIFEST.json',json_encode($manifest)); +require_once $base.'/source/examples/V4SeparatedObservation/install.php'; +try{ + ob_start();$report=v4viewsInstall($base);ob_end_clean();ck($report['status']==='observer_installed','native installation completes');$cat=$report['categoryId']; + ck(count($report['createdObjects'])===6,'one new category three read-only variables two scripts'); + ck($timers===[$report['scriptId']=>30],'only new passive observation timer'); + ck($report['firstObservation']['status']==='recorded','actual first record precedes timer'); + $journal=$report['firstObservation']['journal'];$original=file_get_contents($journal); + ck((fileperms($journal)&0777)===0644&&(fileperms(dirname($journal))&0077)===0,'selected new output readable inside private directory'); + foreach($sourceValues as$id=>$value)ck($values[$id]===$value,'source unchanged '.$id); + ob_start();$again=v4viewsInstall($base);ob_end_clean();ck($again['createdObjects']===[],'idempotent no duplicate scripts or variables'); + ck(str_starts_with(file_get_contents($journal),$original),'repeat preserves historical observations'); + $before=file_get_contents($journal);$locks['ENELIX.V4.SeparatedView.'.$c['installationId']]=true;$busy=v4viewsRun($base,$cat);ck($busy['status']==='busy'&&file_get_contents($journal)===$before,'parallel observer does not duplicate writes');$locks=[]; + unset($variables[101]);$failed=v4viewsRun($base,$cat);ck($failed['status']==='recorded','missing measurement explicitly recorded'); + $latest=json_decode(file_get_contents($base.'/data/LATEST.json'),true);ck($latest['views']['physicalLoad']['loadCandidateW']===null&&!$latest['controlEnabled'],'bad source never old target or fake zero');$variables[101]=['VariableType'=>1,'VariableUpdated'=>$now,'VariableChanged'=>$now-30]; + $before=file_get_contents($journal);v4viewsStop($base,$cat);ck($timers[$report['scriptId']]===0&&file_get_contents($journal)===$before,'stop only observer no deletion'); + ck(json_decode(file_get_contents($base.'/data/LATEST.json'),true)['status']==='stopped','stopped view clears latest numbers'); + $scripts[$report['scriptId']].='// parallel edit';ob_start();$bad=v4viewsInstall($base);ob_end_clean();ck($bad['status']==='needs_review'&&$timers[$report['scriptId']]===0,'parallel script edit not overwritten or activated'); + $old=file_get_contents($base.'/config.json');file_put_contents($base.'/config.json',$old.' ');try{v4viewsConfig($base);$refused=false;}catch(Throwable $e){$refused=true;}ck($refused,'changed config refused');file_put_contents($base.'/config.json',$old); + chmod($base,0755);clearstatcache();try{v4viewsConfig($base);$refused=false;}catch(Throwable $e){$refused=true;}ck($refused,'public stage refused before reading output');chmod($base,0700); + foreach(['runtime.php','install.php']as$file){$text=file_get_contents($base.'/source/examples/V4SeparatedObservation/'.$file);ck(!preg_match('/\b(?:RequestAction|IPS_ApplyChanges|IPS_SetProperty|IPS_SetConfiguration|MC_Reload|curl_exec|shell_exec)\s*\(/',$text),'no actuation or configuration calls in '.$file);} +}finally{ + foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($base,FilesystemIterator::SKIP_DOTS),RecursiveIteratorIterator::CHILD_FIRST)as$f){$f->isDir()&&!$f->isLink()?rmdir($f->getPathname()):unlink($f->getPathname());}rmdir($base); +} +echo "TOTAL $n native-mock checks passed. No actual Symcon calls.\n"; From fc024fcfba376f797c1bbe60932b87b885279c9c Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 21:00:05 +0000 Subject: [PATCH 09/22] feat(application): integrate measured-load ingestion training and planner source --- Manager/module.php | 5 + docs/netplan-v4-application.md | 97 ++ libs/ManagerNetzfahrplanV4DatenTrait.php | 159 +++ libs/ManagerNetzfahrplanV4Trait.php | 3 + services/netplan-v4/.dockerignore | 5 + services/netplan-v4/.gitignore | 7 + services/netplan-v4/Dockerfile | 23 + services/netplan-v4/SOURCE_MANIFEST.json | 93 ++ services/netplan-v4/acceptance/Dockerfile.php | 7 + .../netplan-v4/acceptance/check_forecast.py | 36 + .../forecast-src/SOURCE_MANIFEST.json | 25 + .../acceptance/forecast-src/main.py | 908 ++++++++++++++++++ .../forecast-src/methods/__init__.py | 0 .../forecast-src/methods/battery_optimizer.py | 240 +++++ .../acceptance/forecast-src/methods/common.py | 32 + .../acceptance/forecast-src/methods/var_1.py | 66 ++ .../acceptance/forecast-src/methods/var_10.py | 12 + .../acceptance/forecast-src/methods/var_11.py | 12 + .../acceptance/forecast-src/methods/var_13.py | 13 + .../acceptance/forecast-src/methods/var_2.py | 169 ++++ .../acceptance/forecast-src/methods/var_21.py | 62 ++ .../acceptance/forecast-src/methods/var_22.py | 121 +++ .../acceptance/forecast-src/methods/var_23.py | 13 + .../acceptance/forecast-src/methods/var_3.py | 13 + .../forecast-src/model_isolation.py | 45 + .../forecast-src/netplan_v4_publisher.py | 108 +++ .../acceptance/forecast-src/shared_utils.py | 93 ++ .../forecast-src/soc_diagnostics.py | 66 ++ .../forecast-src/telemetry_quality.py | 106 ++ .../tests/test_battery_optimizer.py | 133 +++ .../forecast-src/tests/test_load_forecast.py | 66 ++ .../tests/test_model_isolation.py | 71 ++ .../tests/test_telemetry_integrity.py | 213 ++++ .../acceptance/php-src/SOURCE_MANIFEST.json | 9 + .../netplan-v4/acceptance/php-src/check.php | 45 + .../libs/ManagerNetzfahrplanV4Trait.php | 164 ++++ .../libs/NetzfahrplanV4Betriebsdaten.php | 219 +++++ .../libs/NetzfahrplanV4Bezugszaehler.php | 108 +++ .../tests/NetzfahrplanV4BetriebsdatenTest.php | 29 + .../tests/NetzfahrplanV4BezugszaehlerTest.php | 29 + .../tests/fixtures/NativeV4Scenarios.php | 140 +++ .../fixtures/V4BezugszaehlerScenarios.php | 113 +++ .../acceptance/read_native_forecasts.py | 100 ++ .../netplan-v4/approved_previous_assets.json | 5 + .../commissioning/APPLICATION_STATUS.md | 80 ++ .../application-source/RELEASE.json | 99 ++ .../application-source/server-dataset.json | 1 + .../commissioning/deploy_application.py | 131 +++ .../netplan-v4/compose.portal-bridge.yaml | 6 + services/netplan-v4/compose.yaml | 37 + .../netplan-v4/deploy_integrated_shadow.py | 175 ++++ services/netplan-v4/forecast_acceptance.py | 38 + services/netplan-v4/gui/netplan-v4.css | 1 + services/netplan-v4/gui/netplan-v4.html | 1 + services/netplan-v4/gui/netplan-v4.js | 107 +++ services/netplan-v4/install_hooks.py | 131 +++ .../integrations/NetzfahrplanV4.php | 71 ++ .../integrations/netplan-v4-bridge.mjs | 39 + .../integrations/netplan_v4_publisher.py | 108 +++ services/netplan-v4/netplan_v4/__init__.py | 1 + .../netplan-v4/netplan_v4/battery_model.py | 44 + .../netplan-v4/netplan_v4/controlled_trial.py | 162 ++++ services/netplan-v4/netplan_v4/domain.py | 170 ++++ .../netplan-v4/netplan_v4/forecast_quality.py | 21 + .../netplan_v4/measurement_pipeline.py | 449 +++++++++ .../netplan-v4/netplan_v4/meter_runtime.py | 145 +++ services/netplan-v4/netplan_v4/metering.py | 245 +++++ services/netplan-v4/netplan_v4/optimizer.py | 199 ++++ services/netplan-v4/netplan_v4/peak_policy.py | 173 ++++ .../netplan_v4/receiver_contract.py | 25 + services/netplan-v4/netplan_v4/selection.py | 53 + services/netplan-v4/netplan_v4/service.py | 432 +++++++++ services/netplan-v4/netplan_v4/store.py | 147 +++ services/netplan-v4/release_preflight.py | 128 +++ services/netplan-v4/requirements.txt | 5 + services/netplan-v4/run_tests.py | 21 + services/netplan-v4/runtime_preflight.py | 46 + .../netplan-v4/tests/manager_protocol.php | 24 + services/netplan-v4/tests/portal.test.mjs | 32 + .../tests/test_application_deployment.py | 39 + .../netplan-v4/tests/test_battery_recovery.py | 43 + .../netplan-v4/tests/test_container_access.py | 60 ++ .../netplan-v4/tests/test_controlled_trial.py | 120 +++ .../tests/test_controlled_trial_pipeline.py | 44 + services/netplan-v4/tests/test_delivery.py | 82 ++ .../tests/test_forecast_acceptance.py | 80 ++ .../netplan-v4/tests/test_forecast_quality.py | 31 + .../tests/test_integrated_deployment.py | 71 ++ .../tests/test_measurement_pipeline.py | 161 ++++ services/netplan-v4/tests/test_metering.py | 134 +++ .../netplan-v4/tests/test_peak_release.py | 155 +++ .../tests/test_receiver_contract.py | 45 + .../tests/test_release_preflight.py | 50 + services/netplan-v4/tests/test_v4.py | 184 ++++ tests/NetzfahrplanV4DatenTest.php | 21 + tests/V4ApplicationData/checks.php | 83 ++ 96 files changed, 8933 insertions(+) create mode 100644 docs/netplan-v4-application.md create mode 100644 libs/ManagerNetzfahrplanV4DatenTrait.php create mode 100644 services/netplan-v4/.dockerignore create mode 100644 services/netplan-v4/.gitignore create mode 100644 services/netplan-v4/Dockerfile create mode 100644 services/netplan-v4/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/Dockerfile.php create mode 100644 services/netplan-v4/acceptance/check_forecast.py create mode 100644 services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/forecast-src/main.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/__init__.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/common.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_1.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_10.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_11.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_13.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_2.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_21.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_22.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_23.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_3.py create mode 100644 services/netplan-v4/acceptance/forecast-src/model_isolation.py create mode 100644 services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py create mode 100644 services/netplan-v4/acceptance/forecast-src/shared_utils.py create mode 100644 services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py create mode 100644 services/netplan-v4/acceptance/forecast-src/telemetry_quality.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py create mode 100644 services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/php-src/check.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/fixtures/NativeV4Scenarios.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php create mode 100644 services/netplan-v4/acceptance/read_native_forecasts.py create mode 100644 services/netplan-v4/approved_previous_assets.json create mode 100644 services/netplan-v4/commissioning/APPLICATION_STATUS.md create mode 100644 services/netplan-v4/commissioning/application-source/RELEASE.json create mode 100644 services/netplan-v4/commissioning/application-source/server-dataset.json create mode 100644 services/netplan-v4/commissioning/deploy_application.py create mode 100644 services/netplan-v4/compose.portal-bridge.yaml create mode 100644 services/netplan-v4/compose.yaml create mode 100644 services/netplan-v4/deploy_integrated_shadow.py create mode 100644 services/netplan-v4/forecast_acceptance.py create mode 100644 services/netplan-v4/gui/netplan-v4.css create mode 100644 services/netplan-v4/gui/netplan-v4.html create mode 100644 services/netplan-v4/gui/netplan-v4.js create mode 100644 services/netplan-v4/install_hooks.py create mode 100644 services/netplan-v4/integrations/NetzfahrplanV4.php create mode 100644 services/netplan-v4/integrations/netplan-v4-bridge.mjs create mode 100644 services/netplan-v4/integrations/netplan_v4_publisher.py create mode 100644 services/netplan-v4/netplan_v4/__init__.py create mode 100644 services/netplan-v4/netplan_v4/battery_model.py create mode 100644 services/netplan-v4/netplan_v4/controlled_trial.py create mode 100644 services/netplan-v4/netplan_v4/domain.py create mode 100644 services/netplan-v4/netplan_v4/forecast_quality.py create mode 100644 services/netplan-v4/netplan_v4/measurement_pipeline.py create mode 100644 services/netplan-v4/netplan_v4/meter_runtime.py create mode 100644 services/netplan-v4/netplan_v4/metering.py create mode 100644 services/netplan-v4/netplan_v4/optimizer.py create mode 100644 services/netplan-v4/netplan_v4/peak_policy.py create mode 100644 services/netplan-v4/netplan_v4/receiver_contract.py create mode 100644 services/netplan-v4/netplan_v4/selection.py create mode 100644 services/netplan-v4/netplan_v4/service.py create mode 100644 services/netplan-v4/netplan_v4/store.py create mode 100644 services/netplan-v4/release_preflight.py create mode 100644 services/netplan-v4/requirements.txt create mode 100644 services/netplan-v4/run_tests.py create mode 100644 services/netplan-v4/runtime_preflight.py create mode 100644 services/netplan-v4/tests/manager_protocol.php create mode 100644 services/netplan-v4/tests/portal.test.mjs create mode 100644 services/netplan-v4/tests/test_application_deployment.py create mode 100644 services/netplan-v4/tests/test_battery_recovery.py create mode 100644 services/netplan-v4/tests/test_container_access.py create mode 100644 services/netplan-v4/tests/test_controlled_trial.py create mode 100644 services/netplan-v4/tests/test_controlled_trial_pipeline.py create mode 100644 services/netplan-v4/tests/test_delivery.py create mode 100644 services/netplan-v4/tests/test_forecast_acceptance.py create mode 100644 services/netplan-v4/tests/test_forecast_quality.py create mode 100644 services/netplan-v4/tests/test_integrated_deployment.py create mode 100644 services/netplan-v4/tests/test_measurement_pipeline.py create mode 100644 services/netplan-v4/tests/test_metering.py create mode 100644 services/netplan-v4/tests/test_peak_release.py create mode 100644 services/netplan-v4/tests/test_receiver_contract.py create mode 100644 services/netplan-v4/tests/test_release_preflight.py create mode 100644 services/netplan-v4/tests/test_v4.py create mode 100644 tests/NetzfahrplanV4DatenTest.php create mode 100644 tests/V4ApplicationData/checks.php diff --git a/Manager/module.php b/Manager/module.php index b2b75ac..82ab561 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -244,6 +244,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->stoppeNetzfahrplanV4Vorschau(); $this->initialisiereLizenzInstallationID(); + $this->konfiguriereNetzfahrplanV4Daten(); $this->aktualisiereVariablen(); try { @@ -480,6 +481,10 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->V4ManagerTestStoppen(); return; + case 'NetzfahrplanV4DatenErfassen': + $this->ErfasseNetzfahrplanV4Daten(); + return; + case 'NetzfahrplanV4Senden': $this->sendeNetzfahrplanV4(); return; diff --git a/docs/netplan-v4-application.md b/docs/netplan-v4-application.md new file mode 100644 index 0000000..bf964b0 --- /dev/null +++ b/docs/netplan-v4-application.md @@ -0,0 +1,97 @@ +# V4 application data and forecast integration + +## Implemented application path + +`ManagerNetzfahrplanV4DatenTrait` records the configured raw power/SOC/counter sources +inside the existing Manager. This replaces the need for permanent standalone +observation categories once the native delivery path has been accepted. It never +issues device commands. New acquisition properties default disabled. + +The private outbox at `data/enelix-v4/` is append-only and delivery +is acknowledged by dataset and capture time. Unacknowledged data survives network +failures and 429 responses. The native payload budget is below the existing portal +1 MiB request-body limit. Existing authentication and V4 rate limiting are reused. + +Server application source is now versioned under `services/netplan-v4`, rather than +existing only as an untracked server working tree. No live measurements, databases, +credentials, dependency binaries or settings exports are included in this directory. + +The existing worker handles per-plant dataset ingestion, physical five-minute +rollup, model training, model storage and corrected-load substitution into the V4 +optimizer. It continues to consume the existing PV forecasts and price/operation +inputs. The public device route can append measurements but cannot alter source +meaning, register a dataset, change live permissions or arm a controlled trial. + +## Explicit settings and model meaning + +- `forecastSource=legacy` preserves the existing forecast input by default. +- `forecastSource=corrected_profile` plus `measurementDataset` uses the trained + corrected physical load. Missing models are reported; legacy house values are + not substituted silently. +- `trainingCadence=daily|weekly` is now connected to real profile fitting in the + application worker, independently from the optimizer refresh frequency. +- Corrected load profiles are tagged `physical-profile-v1`: robust daily profile, + recent-day profile and weekday/weekend profile paired with PV families 3/13/23. + They are not claimed to be the previous load models unchanged. Economic automatic + selection still requires the separate, not yet finished cost-replay integration. +- Initial models are explicitly bootstrap models. Later candidates record causal + holdout errors; an overlapping validation window cannot certify a promotion. +- Current SDL request persistence is only a labelled shadow scenario. Unknown or + stale SDL is not silently zero, and the scenario is not a guaranteed future + schedule or a robust SDL-aware production control policy. + +## Lihrenmoos package scope + +Effective EV capacity 161.44 kWh and power 39 kW remain unchanged; SDL reserve is +already excluded. The prepared dataset `lihrenmoos-physical-v1` uses a configured +physical estimate with SolarEdge signed terminal power counted once. It requires +at least 24 equivalent usable hours before the first bootstrap profile; the +history window is 28 days. Coverage policy 95% with maximum 10-second unsupported +portion is an explicit modelling assumption. Gaps and source ages remain visible. +It is not an independent electrical metering-boundary proof. + +Server entry point on the service host: + + python3 /home/agent/services/netplan-v4-shadow/commissioning/deploy_application.py \ + --plant e3a08f9e-af12-4695-99bd-8b51c0520021 --apply + +Default without --apply checks source only. The explicit apply command rebuilds +and tests V4 in Python 3.11, tests portal routing, backs up the V4 database and +recreates only V4 and the portal. It configures the dataset but DOES NOT select it +for the existing plan automatically. No forecasts/tariff importer or controllers +are restarted by the server command. A previous-image rollback is prepared. + +Then, inside Lihrenmoos Symcon: + + require '/srv/agent/netplan-v4-application-build/install.php'; + +This is a data-only patch of the currently installed passive manager, with file +backup and library reload. It does NOT install the development control-trial +changes in Manager/Batterie. The first runtime invocation imports at most 48h of +the existing 23-channel observer as a durable backlog, preserving the original +files, then activates acquisition and delivery. If Symcon module registration is +not immediately available, the installer reports that a single repeat is needed. + +The two temporary standalone samplers are not stopped automatically by this +initial package; disable them only after native batch acceptance and history +continuity are confirmed. This package creates no new root diagnostic category. + +## Validation performed before deployment + +219 Python tests passed in the isolated host QA runtime, including actual synthetic +measurement -> profile -> existing optimizer -> stored shadow plan. 16 Node proxy +checks passed. 21 PHP data-path checks and 5 installer scenarios passed with mocked +IPS/HTTP and temporary files on the test host. Full staged PHP syntax passed. +No new container or new manager data integration has yet been run in production. + +Evidence: service `APPLICATION_PIPELINE_TEST_RESULTS.txt`, +`APPLICATION_PORTAL_TEST_RESULTS.txt`, `commissioning/application-source/RELEASE.json`; +test host `netplan-v4-application-build/PREPARATION.json` and `TEST_RESULTS.txt`. + +## Remaining product scope (do not disguise as complete) + +This integrates application data and load forecasting, not completed production +commissioning. Economic replay-based automatic family selection, full corrected +feedback/control integration, long-term outbox/server retention and real live/ +failure acceptance remain open. Existing trial gates are unchanged. No sensor, +accounting or hardware evidence identifier is fabricated by data ingestion. diff --git a/libs/ManagerNetzfahrplanV4DatenTrait.php b/libs/ManagerNetzfahrplanV4DatenTrait.php new file mode 100644 index 0000000..16001c6 --- /dev/null +++ b/libs/ManagerNetzfahrplanV4DatenTrait.php @@ -0,0 +1,159 @@ +RegisterPropertyBoolean('NetzfahrplanV4MessdatenAktiv', false); + $this->RegisterPropertyString('NetzfahrplanV4Messkonfiguration', '{}'); + $this->RegisterPropertyString('NetzfahrplanV4Datensatz', ''); + $this->RegisterAttributeString('NetzfahrplanV4DatenCursor', '{}'); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->RegisterVariableString('NetzfahrplanV4Datenstatus', 'V4 Messdaten und Modellversorgung', '', 199); + $this->RegisterTimer('NetzfahrplanV4DatenErfassen', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4DatenErfassen', true);"); + } + + private function konfiguriereNetzfahrplanV4Daten(): void + { + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 0); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + try { + $this->v4DatenKonfiguration(); + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 30000); + } catch (\Throwable $e) { + $this->v4DatenStatus(['status'=>'configuration_error','errorType'=>get_class($e)]); + } + } + + private function v4DatenKonfiguration(): array + { + $id=$this->ReadAttributeString('LizenzInstallationID'); + $dataset=$this->ReadPropertyString('NetzfahrplanV4Datensatz'); + if (!preg_match('/^[a-zA-Z0-9_-]{1,80}$/D',$dataset) + || !preg_match('/^[0-9a-f-]{36}$/D',$id)) throw new \RuntimeException('Dataset/installation not configured'); + $c=json_decode($this->ReadPropertyString('NetzfahrplanV4Messkonfiguration'),true,64,JSON_THROW_ON_ERROR); + $c=NetzfahrplanV4Messaufnahme::configuration($c); + if ($c['installationId']!==$id || ($c['managerId']??null)!==$this->InstanceID) throw new \RuntimeException('Measurement mapping belongs to another manager'); + return $c; + } + + private function v4DatenVerzeichnis(): string + { + $base=rtrim(IPS_GetKernelDir(),'/').'/data/enelix-v4'; + $directory=$base.'/'.$this->ReadAttributeString('LizenzInstallationID'); + // Fixed application-owned location; no configurable path or arbitrary file access. + foreach ([$base,$directory] as $p) { + if (is_link($p)) throw new \RuntimeException('Measurement directory symlink refused'); + if (!is_dir($p) && !mkdir($p,0700,true) && !is_dir($p)) throw new \RuntimeException('Measurement directory unavailable'); + if ((fileperms($p)&0007)!==0) throw new \RuntimeException('Measurement directory must remain private'); + } + return $directory; + } + + private function v4DatenStatus(array $data): void + { + $this->SetValue('NetzfahrplanV4Datenstatus',json_encode($data+['checkedAt'=>gmdate('c'), + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'controlEnabled'=>false],JSON_THROW_ON_ERROR)); + } + + public function ErfasseNetzfahrplanV4Daten(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + $lock='ENELIX.V4.ApplicationData.'.$this->InstanceID; + if (!IPS_SemaphoreEnter($lock,0)) return; + try { + $c=$this->v4DatenKonfiguration(); + $reader=static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('Missing measurement'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id); + if (!in_array($v['VariableType'],[1,2],true)) throw new \RuntimeException('Numeric source required'); + return ['value'=>GetValue($id),'updated'=>(int)$v['VariableUpdated'],'changed'=>(int)$v['VariableChanged'], + 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + }; + $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); + $dir=$this->v4DatenVerzeichnis(); + NetzfahrplanV4Messaufnahme::append($dir,$r); + $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), + 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + $last=$this->ReadAttributeInteger('NetzfahrplanV4DatenVersuch'); + $failures=min(5,$this->ReadAttributeInteger('NetzfahrplanV4DatenFehlerzahl')); + $retry=min(900,60*(2**$failures)); + if (time()>=$last+$retry) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch',time()); + try { + $result['delivery']=$this->v4DatenUebertragen($dir); + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',0); + } catch (\Throwable $e) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',$failures+1); + $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e),'dataRetained'=>true]; + } + } else $result['delivery']=['status'=>'scheduled','nextAttemptAt'=>gmdate('c',$last+$retry)]; + $this->v4DatenStatus($result); + } catch (\Throwable $e) { + // Sensor/configuration exceptions never enter the actuator or module-status path. + $this->v4DatenStatus(['status'=>'capture_error','errorType'=>get_class($e)]); + } finally { IPS_SemaphoreLeave($lock); } + } + + /** Returns only complete records from the append-only prefix; cursor moves after matching ACK. */ + private function v4DatenUebertragen(string $dir): array + { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) throw new \RuntimeException('Forecast licence missing'); + $cursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $day=$cursor['day']??'';$offset=$cursor['offset']??0; + if (!is_string($day) || ($day!==''&&!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day)) || !is_int($offset)||$offset<0) throw new \RuntimeException('Invalid data cursor'); + $files=glob($dir.'/raw-*.jsonl'); + if ($files===false||count($files)>400) throw new \RuntimeException('Measurement journal bounds exceeded'); + sort($files,SORT_STRING); + $records=[];$candidate=$cursor;$bytes=0;$lastTime=null; + foreach ($files as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||($day!==''&&$name<$day)) continue; + if (is_link($file)||!is_file($file)||realpath($file)!==$file) throw new \RuntimeException('Invalid measurement journal'); + $h=fopen($file,'rb');if($h===false)throw new \RuntimeException('Cannot open measurement journal'); + try { + $size=fstat($h)['size'];$position=$name===$day?$offset:0; + if ($position>$size||fseek($h,$position)!==0) throw new \RuntimeException('Measurement journal shortened'); + while (ftell($h)<$size) { + $line=fgets($h,min(262146,$size-ftell($h)+1)); + if ($line===false||!str_ends_with($line,"\n")) break; + if (strlen($line)>262144) throw new \RuntimeException('Oversized measurement'); + if (count($records)>=120||$bytes+strlen($line)>900000) break 2; + $r=json_decode($line,false,64,JSON_THROW_ON_ERROR); + if (!is_object($r)||($r->installationId??null)!==$this->ReadAttributeString('LizenzInstallationID') + ||($r->kind??null)!=='raw_accounting_capture') throw new \RuntimeException('Journal identity invalid'); + $records[]=$r;$bytes+=strlen($line);$lastTime=$r->capturedAt; + $candidate=['day'=>$name,'offset'=>ftell($h)]; + } + } finally { fclose($h); } + } + if ($records===[]) return ['status'=>'up_to_date']; + $id=$this->ReadAttributeString('LizenzInstallationID');$token=$this->ReadAttributeString('PrognoseInstallationsToken'); + if ($token==='') throw new \RuntimeException('Existing device token unavailable'); + $payload=json_encode(['version'=>1,'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'records'=>$records],JSON_THROW_ON_ERROR); + $h=curl_init('https://license.enelix.ch/api/v1/installations/'.rawurlencode($id).'/prognosis/planner-v4/measurements'); + if ($h===false) throw new \RuntimeException('Cannot prepare data transfer'); + try { + curl_setopt_array($h,[CURLOPT_POST=>true,CURLOPT_RETURNTRANSFER=>true,CURLOPT_FOLLOWLOCATION=>false, + CURLOPT_CONNECTTIMEOUT=>2,CURLOPT_TIMEOUT=>8,CURLOPT_SSL_VERIFYPEER=>true,CURLOPT_SSL_VERIFYHOST=>2, + CURLOPT_HTTPHEADER=>['Content-Type: application/json','Accept: application/json','Authorization: Bearer '.$token], + CURLOPT_POSTFIELDS=>$payload]); + $answer=curl_exec($h);$code=(int)curl_getinfo($h,CURLINFO_HTTP_CODE); + if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged'); + $ack=json_decode($answer,true,32,JSON_THROW_ON_ERROR); + if (!in_array($ack['status']??null,['stored','duplicate'],true) + ||($ack['datasetId']??null)!==$this->ReadPropertyString('NetzfahrplanV4Datensatz') + ||($ack['acceptedThrough']??null)!==$lastTime) throw new \RuntimeException('Measurement acknowledgement does not match batch'); + $this->WriteAttributeString('NetzfahrplanV4DatenCursor',json_encode($candidate,JSON_THROW_ON_ERROR)); + return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime]; + } finally { curl_close($h); } + } +} diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php index 1d32ca6..31aecc6 100644 --- a/libs/ManagerNetzfahrplanV4Trait.php +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -8,14 +8,17 @@ use RuntimeException; use Throwable; require_once __DIR__ . '/ManagerNetzfahrplanV4EmpfangTrait.php'; +require_once __DIR__ . '/ManagerNetzfahrplanV4DatenTrait.php'; /** Shadow telemetry only. Does not read/apply a V4 schedule or write actuators. */ trait ManagerNetzfahrplanV4Trait { use ManagerNetzfahrplanV4EmpfangTrait; + use ManagerNetzfahrplanV4DatenTrait; private function registriereNetzfahrplanV4(): void { $this->registriereNetzfahrplanV4Empfang(); + $this->registriereNetzfahrplanV4Daten(); $this->RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); diff --git a/services/netplan-v4/.dockerignore b/services/netplan-v4/.dockerignore new file mode 100644 index 0000000..7264271 --- /dev/null +++ b/services/netplan-v4/.dockerignore @@ -0,0 +1,5 @@ +data/ +.git/ +__pycache__/ +**/__pycache__/ +*.log diff --git a/services/netplan-v4/.gitignore b/services/netplan-v4/.gitignore new file mode 100644 index 0000000..eb476e8 --- /dev/null +++ b/services/netplan-v4/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +data/ +*-reports/ +application-releases/ +.env +*.env diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile new file mode 100644 index 0000000..b98928f --- /dev/null +++ b/services/netplan-v4/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY netplan_v4 ./netplan_v4 +COPY tests ./tests +COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile . +COPY integrations ./integrations +COPY gui ./gui +COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ +COPY acceptance ./acceptance +COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +# Host sources can be 0600/0700. COPY makes them root-owned. +# Normalize only packaged application code; never change host secrets or sockets. +RUN find /app -type d -exec chmod 0755 {} + \ + && find /app -type f -exec chmod 0644 {} + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results +USER 1000:1000 +# Fail the build early if the actual unprivileged runtime cannot read the code. +RUN python /app/runtime_preflight.py +CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"] diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json new file mode 100644 index 0000000..878da98 --- /dev/null +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -0,0 +1,93 @@ +{ + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "packagedAt": "2026-10-02T20:57:00.387145+00:00", + "runtimeChanged": false +} diff --git a/services/netplan-v4/acceptance/Dockerfile.php b/services/netplan-v4/acceptance/Dockerfile.php new file mode 100644 index 0000000..d7aa137 --- /dev/null +++ b/services/netplan-v4/acceptance/Dockerfile.php @@ -0,0 +1,7 @@ +FROM php:8.3-cli +WORKDIR /check +COPY php-src/ ./ +RUN find /check -type d -exec chmod 0755 {} + \ + && find /check -type f -exec chmod 0644 {} + +USER 1000:1000 +CMD ["php", "/check/check.php"] diff --git a/services/netplan-v4/acceptance/check_forecast.py b/services/netplan-v4/acceptance/check_forecast.py new file mode 100644 index 0000000..f42cabb --- /dev/null +++ b/services/netplan-v4/acceptance/check_forecast.py @@ -0,0 +1,36 @@ +"""Offline candidate-forecast tests: no telemetry, model loading or publication. +Run in an unprivileged, networkless test container with no live data volumes. +""" +from pathlib import Path +import ast +import hashlib +import json +import sys +import unittest + + +def main(): + root = Path('/app/forecast') + sys.path.insert(0, str(root)) + manifest = json.loads((root / 'SOURCE_MANIFEST.json').read_text()) + for relative, expected in manifest.items(): + p = root / relative + if Path(relative).is_absolute() or '..' in Path(relative).parts or not p.resolve().is_relative_to(root): + raise ValueError('Unsafe manifest path') + raw = p.read_bytes() + if hashlib.sha256(raw).hexdigest() != expected: + raise ValueError('Test image source checksum mismatch') + if p.suffix == '.py': + ast.parse(raw, filename=str(p)) + import pandas, numpy, scipy, sklearn + versions = {'python': sys.version.split()[0], 'pandas': pandas.__version__, + 'numpy': numpy.__version__, 'scipy': scipy.__version__, 'sklearn': sklearn.__version__} + print('Candidate forecast runtime:', json.dumps(versions), flush=True) + suite = unittest.defaultTestLoader.discover(str(root / 'tests')) + result = unittest.TextTestRunner(verbosity=2).run(suite) + print('Forecast tests only; no live data, no publication, no training job.', flush=True) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..f35881c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json @@ -0,0 +1,25 @@ +{ + "main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" +} diff --git a/services/netplan-v4/acceptance/forecast-src/main.py b/services/netplan-v4/acceptance/forecast-src/main.py new file mode 100644 index 0000000..02748d7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/main.py @@ -0,0 +1,908 @@ +from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts +from model_isolation import collect_predictions +import os +import sqlite3 +import json +import time +import datetime +import traceback +import warnings +import subprocess +import sys +import threading + +import numpy as np +import pandas as pd +import pytz +from influxdb_client import InfluxDBClient, Point, WritePrecision +from influxdb_client.client.write_api import SYNCHRONOUS +from soc_diagnostics import battery_soc_points +from telemetry_quality import require_recent_telemetry, sanitize_measured_frame +try: + from influxdb_client.client.warnings import MissingPivotFunction + warnings.simplefilter("ignore", MissingPivotFunction) +except Exception: + pass + +import methods.var_1 as v1 +import methods.var_2 as v2 +import methods.var_3 as v3 +import methods.var_10 as v10 +import methods.var_11 as v11 +import methods.var_13 as v13 +import methods.var_21 as v21 +import methods.var_22 as v22 +import methods.var_23 as v23 + +INFLUX_URL = os.getenv("INFLUX_URL", "http://influxdb:8086") +INFLUX_TOKEN = os.environ["INFLUX_TOKEN"] +INFLUX_ORG = os.getenv("INFLUX_ORG", "belevo") +INFLUX_BUCKET = os.getenv("INFLUX_BUCKET", "energy_data") +SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "/app/data/users.db") +HISTORY_START = os.getenv("FORECAST_HISTORY_START", "1970-01-01T00:00:00Z") +QUALITY_LOOKBACK_DAYS = int(os.getenv("FORECAST_QUALITY_LOOKBACK_DAYS", "14")) +LOCAL_TZ = pytz.timezone(os.getenv("TZ", "Europe/Zurich")) +INFLUX_TIMEOUT_MS = int(os.getenv("FORECAST_INFLUX_TIMEOUT_MS", "120000")) +FORECAST_TIMEOUT_SECONDS = int(os.getenv("FORECAST_RUN_TIMEOUT_SECONDS", "900")) +FORECAST_HORIZON_HOURS = max(24, min(72, int(os.getenv("FORECAST_HORIZON_HOURS", "48")))) +TRAINING_TIMEOUT_SECONDS = int(os.getenv("FORECAST_TRAINING_TIMEOUT_SECONDS", "3600")) +FORECAST_STALE_SECONDS = int(os.getenv("FORECAST_STALE_SECONDS", "5400")) +WATCHDOG_INTERVAL_SECONDS = int(os.getenv("FORECAST_WATCHDOG_INTERVAL_SECONDS", "300")) +TRAINING_HOUR = int(os.getenv("FORECAST_TRAINING_HOUR", "2")) +LAST_SUCCESS_PATH = os.getenv("FORECAST_LAST_SUCCESS_PATH", "/tmp/forecast_engine_last_success.json") +LAST_TRAINING_PATH = os.getenv("FORECAST_LAST_TRAINING_PATH", "/app/data/forecast_training_status.json") +last_trained_day = None +_FORECAST_PROCESS_LOCK = threading.Lock() +_TRAINING_THREAD = None + +MODEL_MODULES = {1: v1, 2: v2, 3: v3, 10: v10, 11: v11, 13: v13, 21: v21, 22: v22, 23: v23} +QUALITY_TARGETS = {1: "PV", 10: "PV", 21: "PV", 2: "Hausverbrauch", 11: "Hausverbrauch", 22: "Hausverbrauch"} + + +def active(config, n): + return bool(int(config.get(f"prog_var_{n}", config.get(f"var_{n}", 0)) or 0)) + + +def get_configs(): + conn = sqlite3.connect(SQLITE_DB_PATH) + conn.row_factory = sqlite3.Row + columns = {row["name"] for row in conn.execute("PRAGMA table_info(anlagen_meta)")} + if "batt_grid_charging_enabled" not in columns: + conn.execute( + "ALTER TABLE anlagen_meta ADD COLUMN batt_grid_charging_enabled INTEGER NOT NULL DEFAULT 0" + ) + conn.commit() + rows = conn.execute("SELECT * FROM anlagen_meta").fetchall() + conn.close() + configs = [] + for r in rows: + d = dict(r) + try: + d["daecher"] = json.loads(d.get("daecher") or "[]") + except Exception: + d["daecher"] = [] + for key, default in [ + ("ac_leistung", 10.0), + ("batt_capacity_kwh", 0.0), + ("batt_power_kw", 0.0), + ("tarif_bezug_fest", 0.30), + ("tarif_einspeisung_fest", 0.10), + ("tarif_peak_fest", 5.0), + ]: + raw_value = d.get(key) + d[key] = float(default if raw_value is None or raw_value == "" else raw_value) + configs.append(d) + return configs + + +def _time_literal(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _query_df(query): + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + try: + df = client.query_api().query_data_frame(org=INFLUX_ORG, query=query) + finally: + client.close() + if isinstance(df, list): + df = pd.concat(df, ignore_index=True) if df else pd.DataFrame() + if df is None or df.empty or "_time" not in df.columns: + return pd.DataFrame() + df["_time"] = pd.to_datetime(df["_time"], utc=True).dt.tz_localize(None) + return df + + +def _pivot_frame(df, fields): + if df.empty: + return pd.DataFrame() + out = df.set_index("_time") + keep = [c for c in fields if c in out.columns] + out = out[keep] if keep else pd.DataFrame(index=out.index) + out = out.apply(pd.to_numeric, errors="coerce") + out = out[~out.index.duplicated(keep="last")] + return out.sort_index().resample("5min").mean(numeric_only=True) + + +def _tariff_frame(df, config=None): + if df is None or df.empty: + return pd.DataFrame() + df = df.copy() + if "_time" in df.columns: + df["_time"] = pd.to_datetime(df["_time"], errors="coerce") + df = df.dropna(subset=["_time"]).set_index("_time") + if df.empty: + return pd.DataFrame() + + price = df["price_chf_kwh"] if "price_chf_kwh" in df.columns else df.get("_value") + if price is None: + return pd.DataFrame() + price = pd.to_numeric(price, errors="coerce") + model = df.get("tariff_model", pd.Series("", index=df.index)).astype(str).str.lower() + typ = df.get("type", pd.Series("", index=df.index)).astype(str).str.lower() + tariff_name = df.get("tariff_name", pd.Series("", index=df.index)).astype(str).str.lower() + provider = df.get("provider", pd.Series("", index=df.index)).astype(str).str.lower() + key = (model + " " + tariff_name + " " + provider).str.lower() + + rows = pd.DataFrame({"price": price, "key": key, "type": typ}, index=df.index) + rows = rows[pd.notna(rows["price"])].sort_index() + if rows.empty: + return pd.DataFrame() + + cfg = config or {} + import_choice = str(cfg.get("tarif_bezug", "") or "").lower() + export_choice = str(cfg.get("tarif_einspeisung", "") or "").lower() + + out = pd.DataFrame(index=rows.index.unique().sort_values()) + + import_base = ( + rows["type"].str.contains("consumption|import|bezug", regex=True, na=False) + | rows["key"].str.contains("dynamic|dynamisch|home|business", regex=True, na=False) + ) + if "business" in import_choice: + import_mask = import_base & rows["key"].str.contains("business|gewerbe|commercial", regex=True, na=False) + elif "home" in import_choice or "privat" in import_choice: + import_mask = import_base & rows["key"].str.contains("home|privat|private", regex=True, na=False) + elif "dynam" in import_choice: + import_mask = import_base + else: + import_mask = pd.Series(False, index=rows.index) + if not import_mask.any() and "dynam" in import_choice: + import_mask = import_base + + export_base = ( + rows["type"].str.contains("feed|einspeis|export", regex=True, na=False) + | rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|feed", regex=True, na=False) + ) + if "referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice: + export_mask = export_base & rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|belevo", regex=True, na=False) + else: + export_mask = rows["key"].str.contains("standard_feedin|ckw statisch", regex=True, na=False) & export_base + if not export_mask.any() and ("referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice): + export_mask = export_base + + if import_mask.any(): + out["import_price"] = rows.loc[import_mask, "price"].groupby(level=0).last() + if export_mask.any(): + out["export_price"] = rows.loc[export_mask, "price"].groupby(level=0).last() + + if out.empty: + return out + return out.sort_index().resample("5min").mean().ffill().bfill() + +def fetch_influx_frames(config, training): + aid = config["anlagen_id"] + start = HISTORY_START if training else "-14d" + future_stop = _time_literal(datetime.datetime.utcnow() + datetime.timedelta(hours=FORECAST_HORIZON_HOURS)) + + q_tel = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "PV" or r["_field"] == "Hausverbrauch" or r["_field"] == "Netzleistung" or r["_field"] == "SOC") + |> filter(fn: (r) => not exists r["data_type"] or (r["data_type"] != "forecast" and r["data_type"] != "forecast_snapshot")) + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false, timeSrc: "_start") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_tel = _pivot_frame(_query_df(q_tel), ["PV", "Hausverbrauch", "Netzleistung", "SOC"]) + + q_wea = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "weather_forecast") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_wea = _pivot_frame(_query_df(q_wea), ["temp_c", "temperature", "cloud", "cloud_cover", "precip_mm", "wind_kph", "chance_of_snow"]) + if "temperature" in df_wea.columns and "temp_c" not in df_wea.columns: + df_wea["temp_c"] = df_wea["temperature"] + if "cloud_cover" in df_wea.columns and "cloud" not in df_wea.columns: + df_wea["cloud"] = df_wea["cloud_cover"] + if "cloud" in df_wea.columns and "cloud_cover" not in df_wea.columns: + df_wea["cloud_cover"] = df_wea["cloud"] + + q_tar = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -7d, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "tariffs") + |> filter(fn: (r) => r["_field"] == "price_chf_kwh") +''' + df_tar = _tariff_frame(_query_df(q_tar), config) + return df_tel, df_wea, df_tar + + +def _consistent_tail(df): + if df.empty or "PV" not in df.columns or "Hausverbrauch" not in df.columns: + return df + probe = df[["PV", "Hausverbrauch"]].copy() + filled = probe.interpolate(limit=3, limit_direction="both") + valid = filled.notna().all(axis=1) + if not valid.any(): + return df.iloc[0:0] + run = 0 + last_break = -1 + for i, ok in enumerate(valid.to_numpy()): + if ok: + run = 0 + else: + run += 1 + if run >= 4: + last_break = i + if last_break >= 0: + after = np.where(valid.iloc[last_break + 1:].to_numpy())[0] + if len(after): + return df.iloc[last_break + 1 + after[0]:] + return df.iloc[0:0] + return df.loc[valid[valid].index[0]:] + + + +def _longest_consistent_segment(df, column): + if df.empty or column not in df.columns: + return pd.DataFrame() + series = pd.to_numeric(df[column], errors="coerce") + valid = series.notna().to_numpy() + if not valid.any(): + return pd.DataFrame() + best_start = best_end = None + start = 0 + gap = 0 + for i, ok in enumerate(valid): + if ok: + gap = 0 + else: + gap += 1 + if gap >= 4: + end = i - gap + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + start = i + 1 + gap = 0 + end = len(valid) - 1 + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + if best_start is None: + return pd.DataFrame() + segment = df.iloc[best_start:best_end + 1].copy() + first = pd.to_numeric(segment[column], errors="coerce").first_valid_index() + last = pd.to_numeric(segment[column], errors="coerce").last_valid_index() + if first is None or last is None: + return pd.DataFrame() + return segment.loc[first:last] + + +def _add_time_features(df): + hour = df.index.hour + df.index.minute / 60.0 + doy = df.index.dayofyear + df["hour_float"] = hour + df["hour_sin"] = np.sin(2 * np.pi * hour / 24.0) + df["hour_cos"] = np.cos(2 * np.pi * hour / 24.0) + df["sin_year"] = np.sin(2 * np.pi * doy / 365.25) + df["cos_year"] = np.cos(2 * np.pi * doy / 365.25) + df["weekday"] = df.index.weekday + df["is_weekday"] = (df.index.weekday < 5).astype(int) + return df + + +def _fill_defaults(df, history): + defaults = { + "PV": np.nan, + "Hausverbrauch": np.nan, + "SOC": np.nan, + "Netzleistung": np.nan, + "temp_c": 15.0, + "cloud": 20.0, + "cloud_cover": 20.0, + "precip_mm": 0.0, + "wind_kph": 0.0, + "chance_of_snow": 0.0, + "import_price": np.nan, + "export_price": np.nan, + } + for col, default in defaults.items(): + if col not in df.columns: + df[col] = default + df[col] = pd.to_numeric(df[col], errors="coerce") + # Measurements are never interpolated/filled here. Outages are not zero load, + # zero PV, a fresh SOC, or a measured grid peak. Weather defaults are separate. + df = sanitize_measured_frame(df) + for col, default in defaults.items(): + if col not in ["PV", "Hausverbrauch", "SOC", "Netzleistung"]: + df[col] = df[col].ffill().bfill() + if np.isfinite(default): + df[col] = df[col].fillna(default) + if "cloud_cover" in df.columns: + df["cloud"] = df["cloud"].fillna(df["cloud_cover"]) + df["import_price"] = df["import_price"].fillna(0.30) + df["export_price"] = df["export_price"].fillna(0.10) + return _add_time_features(df) + + +def build_data_object(config, training=False): + now = datetime.datetime.utcnow().replace(second=0, microsecond=0) + now = now - datetime.timedelta(minutes=now.minute % 5) + df_tel, df_wea, df_tar = fetch_influx_frames(config, training) + + frames = [f for f in [df_tel, df_wea, df_tar] if not f.empty] + combined = frames[0] if frames else pd.DataFrame() + for f in frames[1:]: + combined = combined.join(f, how="outer") + combined = combined.sort_index() + + full_hist_raw = combined.loc[:now - datetime.timedelta(minutes=5)] if not combined.empty else pd.DataFrame() + # A trailing telemetry gap must not discard all earlier valid observations. + hist_raw = full_hist_raw.copy() + # Freshness is determined from measurements, not the outer-joined weather grid. + recent_raw = sanitize_measured_frame(df_tel.loc[:now - datetime.timedelta(minutes=5)].copy()) if not df_tel.empty else pd.DataFrame() + start_hist = hist_raw.index.min().floor("5min") if training and not hist_raw.empty else now - datetime.timedelta(days=14) + + idx_hist = pd.date_range(start=start_hist, end=now - datetime.timedelta(minutes=5), freq="5min") + df_hist = pd.DataFrame(index=idx_hist).join(hist_raw, how="left") + df_hist = _fill_defaults(df_hist, history=True) + + df_load_training = pd.DataFrame() + if training and "Hausverbrauch" in full_hist_raw.columns: + load_segment = _longest_consistent_segment(full_hist_raw, "Hausverbrauch") + if not load_segment.empty: + idx_load = pd.date_range( + start=load_segment.index.min().floor("5min"), + end=load_segment.index.max().floor("5min"), + freq="5min", + ) + df_load_training = pd.DataFrame(index=idx_load).join(load_segment, how="left") + df_load_training = _fill_defaults(df_load_training, history=True) + + df_pv_training = pd.DataFrame() + if training and "PV" in full_hist_raw.columns: + pv_first = full_hist_raw["PV"].first_valid_index() + if pv_first is not None: + idx_pv = pd.date_range( + start=pv_first.floor("5min"), + end=now - datetime.timedelta(minutes=5), + freq="5min", + ) + df_pv_training = pd.DataFrame(index=idx_pv).join(full_hist_raw, how="left") + df_pv_training = _fill_defaults(df_pv_training, history=True) + + idx_fut = pd.date_range(start=now, periods=FORECAST_HORIZON_HOURS * 12, freq="5min") + fut_raw = combined.reindex(combined.index.union(idx_fut)).sort_index() if not combined.empty else pd.DataFrame(index=idx_fut) + df_fut = pd.DataFrame(index=idx_fut).join(fut_raw, how="left") + df_fut = _fill_defaults(df_fut, history=False) + + month_hist = df_hist[(df_hist.index.year == now.year) & (df_hist.index.month == now.month)] + current_peak_kw = 0.0 + if not month_hist.empty and "Netzleistung" in month_hist.columns: + measured_grid = pd.to_numeric(month_hist["Netzleistung"], errors="coerce").dropna() + measured_peak = measured_grid.resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not measured_peak.empty: + current_peak_kw = max(0.0, float(measured_peak.max()) / 1000.0) + if current_peak_kw <= 0.0 and not month_hist.empty and {"Hausverbrauch", "PV"}.issubset(month_hist.columns): + fallback_residual = (month_hist["Hausverbrauch"] - month_hist["PV"]).resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not fallback_residual.empty: + current_peak_kw = max(0.0, float(fallback_residual.max()) / 1000.0) + + min_soc = float( + config.get("batt_min_soc", config.get("batt_min_soc_percent", 0.0)) or 0.0 + ) + current_soc = max(0.0, min(100.0, min_soc)) + current_soc_source = "safe_minimum" + current_soc_age_minutes = None + if "SOC" in full_hist_raw.columns: + soc_values = pd.to_numeric(full_hist_raw["SOC"], errors="coerce").dropna() + if not soc_values.empty: + latest_soc_time = pd.Timestamp(soc_values.index[-1]) + current_soc_age_minutes = max( + 0.0, + (pd.Timestamp(now) - latest_soc_time).total_seconds() / 60.0, + ) + try: + max_soc_age_minutes = max( + 5.0, + float(config.get("batt_soc_max_age_minutes", 30.0) or 30.0), + ) + except (TypeError, ValueError): + max_soc_age_minutes = 30.0 + if current_soc_age_minutes <= max_soc_age_minutes: + current_soc = float(soc_values.iloc[-1]) + current_soc_source = "telemetry" + current_soc = max(0.0, min(100.0, current_soc)) + + return { + "config": config, + "metadata": config, + "now": now, + "df_hist": df_hist, + "df_recent_raw": recent_raw, + "df_load_training": df_load_training, + "df_pv_training": df_pv_training, + "df_fut": df_fut, + "current_soc_perc": current_soc, + "current_soc_source": current_soc_source, + "current_soc_age_minutes": current_soc_age_minutes, + "current_month_peak_kw": current_peak_kw, + } + + +def run_training(): + for config in get_configs(): + if not any(active(config, n) for n in MODEL_MODULES): + continue + print(f"Training Anlage {config['anlagen_id']}...") + data_obj = build_data_object(config, training=True) + for n, module in MODEL_MODULES.items(): + if active(config, n) and hasattr(module, "train"): + try: + target = QUALITY_TARGETS.get(n) + if target: + require_recent_telemetry(data_obj, [target]) + print(f" var_{n}: {module.train(data_obj)}") + except Exception: + print(f" var_{n}: Training fehlgeschlagen") + traceback.print_exc() + + +def _forecast_point(aid, field, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast") + .field(field, float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _snapshot_point(aid, field, run_hour, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast_snapshot") + .tag("run_hour", run_hour) + .field(f"{field}_run_{run_hour}", float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _quality_query(aid, forecast_field, target_field): + stop = _time_literal(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) + return f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -{QUALITY_LOOKBACK_DAYS}d, stop: {stop}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "{target_field}" or r["_field"] == "{forecast_field}") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> group() + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + + +def _forecast_quality(aid, variant, target_field): + forecast_field = f"prog_var_{variant}" + df = _query_df(_quality_query(aid, forecast_field, target_field)) + if df.empty or forecast_field not in df.columns or target_field not in df.columns: + return None + pair = df[[target_field, forecast_field]].apply(pd.to_numeric, errors="coerce").dropna() + if len(pair) < 3: + return None + actual = pair[target_field].to_numpy(dtype=float) + pred = pair[forecast_field].to_numpy(dtype=float) + ss_res = float(np.sum((actual - pred) ** 2)) + ss_tot = float(np.sum((actual - np.mean(actual)) ** 2)) + r2 = None if ss_tot <= 0 else 1.0 - (ss_res / ss_tot) + mae = float(np.mean(np.abs(actual - pred))) + rmse = float(np.sqrt(np.mean((actual - pred) ** 2))) + return {"r2": r2, "mae": mae, "rmse": rmse, "samples": int(len(pair))} + + +def _quality_point(aid, variant, target, metrics): + p = ( + Point("forecast_metrics") + .tag("anlagen_id", aid) + .tag("forecast", f"prog_var_{variant}") + .tag("target", target) + .field("samples", int(metrics["samples"])) + .field("mae", float(metrics["mae"])) + .field("rmse", float(metrics["rmse"])) + .time(datetime.datetime.utcnow(), WritePrecision.S) + ) + if metrics["r2"] is not None: + p.field("r2", float(metrics["r2"])) + return p + + +def write_quality_metrics(write_api, aid, active_variants): + points = [] + + for variant, target in QUALITY_TARGETS.items(): + if variant not in active_variants: + continue + try: + metrics = _forecast_quality(aid, variant, target) + if metrics: + points.append(_quality_point(aid, variant, target, metrics)) + r2_text = "nan" if metrics["r2"] is None else f"{metrics['r2']:.3f}" + print(f"R2 Anlage {aid} prog_var_{variant}: {r2_text} / samples={metrics['samples']}") + except Exception: + print(f"R2 Anlage {aid} prog_var_{variant}: Berechnung fehlgeschlagen") + traceback.print_exc() + if points: + try: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + except Exception: + print(f"R2 Anlage {aid}: Schreiben der Qualitaetswerte fehlgeschlagen") + traceback.print_exc() + + +def run_forecast(only_anlagen_id=None, manual=False): + configs = get_configs() + if only_anlagen_id: + configs = [c for c in configs if c.get("anlagen_id") == only_anlagen_id] + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + write_api = client.write_api(write_options=SYNCHRONOUS) + completed = [] + failures = [] + try: + for config in configs: + aid = config["anlagen_id"] + try: + data_obj = build_data_object(config, training=False) + targets = [] + if any(active(config, n) for n in (1, 3, 10, 13, 21, 23)): + targets.append("PV") + if any(active(config, n) for n in (2, 3, 11, 13, 22, 23)): + targets.append("Hausverbrauch") + if float(config.get("batt_capacity_kwh", 0.0) or 0.0) > 0 and any(active(config, n) for n in (3, 13, 23)): + targets.append("SOC") + # Fail BEFORE model prediction, snapshot publication or V1/V4 plan writes. + data_obj["telemetry_quality"] = require_recent_telemetry(data_obj, targets) + soc_age = data_obj.get("current_soc_age_minutes") + soc_age_text = "keine Messung" if soc_age is None else f"{soc_age:.1f} min" + print( + f"Forecast Anlage {aid}: Batterie-SOC {data_obj['current_soc_perc']:.1f}% " + f"({data_obj['current_soc_source']}, Alter {soc_age_text})." + ) + forecasts, model_errors = collect_predictions( + data_obj, config, + {1: v1, 2: v2, 10: v10, 11: v11, 21: v21, 22: v22}, active, + ) + data_obj['forecast_model_status'] = model_errors + for variant, detail in model_errors.items(): + print(f"Forecast Anlage {aid} prog_var_{variant}: unavailable ({detail['errorType']}); keine Nullwerte eingesetzt.") + p_1, p_2, p_10, p_11, p_21, p_22 = ( + forecasts[n] for n in (1, 2, 10, 11, 21, 22) + ) + + # ENELIX_V4_SHADOW_BRIDGE + _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)]) + + p_3 = v3.predict(data_obj, p_1, p_2) if active(config, 3) and p_1 and p_2 else {} + p_13 = v13.predict(data_obj, p_10, p_11) if active(config, 13) and p_10 and p_11 else {} + p_23 = v23.predict(data_obj, p_21, p_22) if active(config, 23) and p_21 and p_22 else {} + + forecast_sets = [ + (1, p_1), (2, p_2), (3, p_3), + (10, p_10), (11, p_11), (13, p_13), + (21, p_21), (22, p_22), (23, p_23), + ] + run_hour = str(int(datetime.datetime.now(LOCAL_TZ).strftime("%H"))) + points = [] + for variant_id, pv_src, load_src, grid_src in [ + (3, p_1, p_2, p_3), + (13, p_10, p_11, p_13), + (23, p_21, p_22, p_23), + ]: + if grid_src and pv_src and load_src: + points.extend(battery_soc_points(data_obj, variant_id, pv_src, load_src, grid_src)) + active_variants = set() + battery_plans = data_obj.get("battery_plans", {}) + for t in data_obj["df_fut"].index: + for n, values in forecast_sets: + if t in values: + field = f"prog_var_{n}" + value = float(values[t]) + points.append(_forecast_point(aid, field, t, value)) + points.append(_snapshot_point(aid, field, run_hour, t, value)) + if n in battery_plans and t in battery_plans[n].get("battery", {}): + points.append(_forecast_point( + aid, + f"prog_var_{n}_battery", + t, + float(battery_plans[n]["battery"][t]), + )) + active_variants.add(n) + if points: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + print(f"Forecast Anlage {aid}: {len(points)} Punkte geschrieben inkl. run_{run_hour}.") + else: + print(f"Forecast Anlage {aid}: keine aktiven Prognosen oder keine Daten.") + write_quality_metrics(write_api, aid, active_variants) + completed.append(aid) + except Exception as exc: + failures.append({"anlagen_id": aid, "error": f"{type(exc).__name__}: {exc}"}) + print(f"Forecast Anlage {aid}: Lauf fehlgeschlagen.") + traceback.print_exc() + finally: + client.close() + if failures: + raise RuntimeError(f"Forecast-Fehler: {failures}") + return {"completed": completed, "manual": bool(manual)} + + +def _write_json_atomic(path, payload): + directory = os.path.dirname(path) or "." + os.makedirs(directory, exist_ok=True) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + with open(tmp_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +def _read_json(path): + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + return {} + + +def _last_success_age_seconds(): + stamp = _read_json(LAST_SUCCESS_PATH).get("timestamp") + if not stamp: + return None + try: + then = datetime.datetime.fromisoformat(str(stamp).replace("Z", "+00:00")) + now = datetime.datetime.now(datetime.timezone.utc) + return max(0.0, (now - then).total_seconds()) + except Exception: + return None + + +def _child_environment(training=False): + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + if training: + max_threads = str(max(1, int(env.get("FORECAST_TRAINING_MAX_THREADS", "1")))) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "LOKY_MAX_CPU_COUNT", + ): + env[name] = max_threads + return env + + +def _run_child(mode, timeout_seconds, only_anlagen_id=None): + command = [sys.executable, "-u", os.path.abspath(__file__), mode] + if only_anlagen_id: + command.append(str(only_anlagen_id)) + label = "Training" if mode == "--train-once" else "Forecast" + print(f"{label}-Kindprozess startet (Timeout {timeout_seconds}s).") + process = subprocess.Popen(command, env=_child_environment(training=mode == "--train-once")) + try: + return_code = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + print(f"{label}-Kindprozess hat das Zeitlimit erreicht und wird beendet.") + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=15) + return 124 + if return_code != 0: + print(f"{label}-Kindprozess beendet mit Status {return_code}.") + return return_code + + +def run_forecast_isolated(only_anlagen_id=None, source="scheduler"): + if not _FORECAST_PROCESS_LOCK.acquire(blocking=False): + print(f"Forecast-Aufruf ({source}) uebersprungen: bereits ein Lauf aktiv.") + return {"status": "busy", "source": source} + try: + return_code = _run_child("--forecast-once", FORECAST_TIMEOUT_SECONDS, only_anlagen_id) + return { + "status": "ok" if return_code == 0 else "error", + "source": source, + "return_code": return_code, + } + finally: + _FORECAST_PROCESS_LOCK.release() + + +def _training_worker(day_text): + global _TRAINING_THREAD + started = datetime.datetime.now(datetime.timezone.utc).isoformat() + _write_json_atomic(LAST_TRAINING_PATH, { + "day": day_text, + "status": "running", + "started_at": started, + }) + return_code = 1 + error = None + try: + return_code = _run_child("--train-once", TRAINING_TIMEOUT_SECONDS) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + finally: + status = "ok" if return_code == 0 else ("timeout" if return_code == 124 else "error") + payload = { + "day": day_text, + "status": status, + "started_at": started, + "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "return_code": return_code, + } + if error: + payload["error"] = error + _write_json_atomic(LAST_TRAINING_PATH, payload) + print(f"Nachttraining beendet: {status}.") + _TRAINING_THREAD = None + + +def start_training_async(day): + global _TRAINING_THREAD + if _TRAINING_THREAD is not None and _TRAINING_THREAD.is_alive(): + print("Nachttraining laeuft bereits.") + return False + day_text = day.isoformat() + _TRAINING_THREAD = threading.Thread(target=_training_worker, args=(day_text,), daemon=True) + _TRAINING_THREAD.start() + return True + + +def _watchdog_loop(): + while True: + time.sleep(max(60, WATCHDOG_INTERVAL_SECONDS)) + try: + age = _last_success_age_seconds() + if age is None or age > FORECAST_STALE_SECONDS: + age_text = "unbekannt" if age is None else f"{age / 60.0:.1f} Minuten" + print(f"Forecast-Watchdog: letzter erfolgreicher Lauf {age_text}; neuer Lauf wird gestartet.") + run_forecast_isolated(source="watchdog") + except Exception: + print("Forecast-Watchdog: Pruefung fehlgeschlagen.") + traceback.print_exc() + + +# BEGIN EMS RESIMULATE HTTP SERVER +_RESIMULATE_SERVER_STARTED = False + + +def _start_resimulate_server(): + global _RESIMULATE_SERVER_STARTED + if _RESIMULATE_SERVER_STARTED: + return + _RESIMULATE_SERVER_STARTED = True + + import json as _json + import os as _os + import threading as _threading + import traceback as _traceback + import urllib.parse as _urlparse + from http.server import BaseHTTPRequestHandler as _BaseHTTPRequestHandler, ThreadingHTTPServer as _ThreadingHTTPServer + + class _Handler(_BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def _send(self, code, body): + raw = _json.dumps(body).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def _handle(self): + try: + parsed = _urlparse.urlparse(self.path) + if parsed.path == "/health": + age = _last_success_age_seconds() + healthy = age is not None and age <= FORECAST_STALE_SECONDS + self._send(200 if healthy else 503, { + "status": "ok" if healthy else "stale", + "last_success_age_seconds": age, + "training": _read_json(LAST_TRAINING_PATH), + }) + return + if parsed.path not in ("/run_now", "/resimulate"): + self._send(404, {"detail": "not found"}) + return + params = _urlparse.parse_qs(parsed.query) + anlagen_id = (params.get("anlagen_id") or [None])[0] + result = run_forecast_isolated(only_anlagen_id=anlagen_id, source="http") + result["anlagen_id"] = anlagen_id + code = 200 if result["status"] == "ok" else (409 if result["status"] == "busy" else 500) + self._send(code, result) + except Exception as exc: + _traceback.print_exc() + self._send(500, {"detail": str(exc)}) + + port = int(_os.getenv("FORECAST_ENGINE_HTTP_PORT", "9000")) + server = _ThreadingHTTPServer(("0.0.0.0", port), _Handler) + thread = _threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + print(f"Forecast Resimulate HTTP Server startet auf Port {port}.") +# END EMS RESIMULATE HTTP SERVER + +def main(): + _start_resimulate_server() + global last_trained_day + print("Forecast Engine startet.") + threading.Thread(target=_watchdog_loop, daemon=True).start() + run_forecast_isolated(source="startup") + while True: + try: + now_local = datetime.datetime.now(LOCAL_TZ) + if now_local.hour == TRAINING_HOUR and last_trained_day != now_local.date(): + if start_training_async(now_local.date()): + last_trained_day = now_local.date() + next_run = (now_local + datetime.timedelta(hours=1)).replace(minute=0, second=0, microsecond=0) + time.sleep(max(60.0, (next_run - now_local).total_seconds())) + run_forecast_isolated(source="scheduler") + except Exception: + traceback.print_exc() + time.sleep(60) + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "--forecast-once": + selected_anlage = sys.argv[2] if len(sys.argv) >= 3 else None + run_forecast(only_anlagen_id=selected_anlage, manual=True) + _write_json_atomic(LAST_SUCCESS_PATH, { + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "anlagen_id": selected_anlage, + "pid": os.getpid(), + }) + elif len(sys.argv) >= 2 and sys.argv[1] == "--train-once": + run_training() + else: + main() diff --git a/services/netplan-v4/acceptance/forecast-src/methods/__init__.py b/services/netplan-v4/acceptance/forecast-src/methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py new file mode 100644 index 0000000..8615394 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py @@ -0,0 +1,240 @@ +import numpy as np +import pandas as pd +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import lil_matrix + +DT_H = 5.0 / 60.0 + + +def train_artifact(kind): + return {"trained": True, "type": "battery_48h_cost_milp_v3", "source": kind} + + +def _cfg_float(config, key, default): + try: + value = config.get(key, default) + return float(default if value is None or value == "" else value) + except Exception: + return float(default) + + +def _cfg_bool(config, key, default=False): + value = config.get(key, default) + if value is None or value == "": + return bool(default) + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "ja", "on"} + + +def _use_dynamic(config, key): + value = str(config.get(key, "") or "").strip().lower() + return any(token in value for token in ("dynam", "marktpreis", "referenzmarktpreis", "market")) + + +def _price(data_obj, timestamp, column, fallback, dynamic_enabled): + if not dynamic_enabled: + return fallback + frame = data_obj["df_fut"] + if column in frame.columns and timestamp in frame.index: + try: + value = float(frame.at[timestamp, column]) + if np.isfinite(value): + return value + except Exception: + pass + return fallback + + +def _battery_meta(config, data_obj): + cap_kwh = _cfg_float(config, "batt_capacity_kwh", 0.0) + max_power_w = _cfg_float(config, "batt_power_kw", 0.0) * 1000.0 + min_soc = _cfg_float(config, "batt_min_soc", _cfg_float(config, "batt_min_soc_percent", 0.0)) + max_soc = _cfg_float(config, "batt_max_soc", _cfg_float(config, "batt_max_soc_percent", 100.0)) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None: + start_soc_value = min_soc + try: + start_soc = float(start_soc_value) + except (TypeError, ValueError): + start_soc = min_soc + min_soc = max(0.0, min(100.0, min_soc)) + max_soc = max(min_soc, min(100.0, max_soc)) + start_soc = max(min_soc, min(max_soc, start_soc)) + charge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_discharge_efficiency", 0.95))) + return cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff + + +def _quarter_groups(index): + groups = {} + for position, timestamp in enumerate(index): + quarter = timestamp.floor("15min") if hasattr(timestamp, "floor") else position // 3 + groups.setdefault(quarter, []).append(position) + return list(groups.values()) + + +def _fallback_plan(index, residual_w): + grid = {timestamp: float(value) for timestamp, value in zip(index, residual_w)} + battery = {timestamp: 0.0 for timestamp in index} + return {"grid": grid, "battery": battery, "solver": "fallback"} + + +def optimize_battery_plan(data_obj, pv_dict, load_dict): + config = data_obj["config"] + cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff = _battery_meta(config, data_obj) + index = list(data_obj["df_fut"].index) + if not index: + return {"grid": {}, "battery": {}, "solver": "empty"} + + load_w = np.array([max(0.0, float(load_dict.get(t, 0.0))) for t in index]) + pv_w = np.array([max(0.0, float(pv_dict.get(t, 0.0))) for t in index]) + residual_w = load_w - pv_w + if cap_kwh <= 0.0 or max_power_w <= 0.0: + return _fallback_plan(index, residual_w) + + import_fixed = _cfg_float(config, "tarif_bezug_fest", 0.30) + export_fixed = _cfg_float(config, "tarif_einspeisung_fest", 0.10) + import_dynamic = _use_dynamic(config, "tarif_bezug") + export_dynamic = _use_dynamic(config, "tarif_einspeisung") + import_price = np.array([ + _price(data_obj, t, "import_price", import_fixed, import_dynamic) for t in index + ]) + export_price = np.array([ + _price(data_obj, t, "export_price", export_fixed, export_dynamic) for t in index + ]) + + n = len(index) + imp, exp, charge, discharge, curtail, soc = 0, n, 2 * n, 3 * n, 4 * n, 5 * n + peak = 6 * n + 1 + battery_mode = peak + 1 + grid_mode = battery_mode + n + variable_count = grid_mode + n + + max_import_w = max(float(load_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_import_limit = _cfg_float(config, "grid_import_limit_w", 0.0) + if configured_import_limit > 0.0: + max_import_w = min(max_import_w, configured_import_limit) + max_export_w = max(float(pv_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_export_limit = _cfg_float(config, "grid_export_limit_w", 0.0) + if configured_export_limit > 0.0: + max_export_w = min(max_export_w, configured_export_limit) + + lower = np.zeros(variable_count) + upper = np.full(variable_count, np.inf) + upper[imp:imp + n] = max_import_w + upper[exp:exp + n] = max_export_w + upper[charge:charge + n] = max_power_w + if not _cfg_bool(config, "batt_grid_charging_enabled", False): + upper[charge:charge + n] = np.minimum(max_power_w, np.maximum(0.0, pv_w - load_w)) + upper[discharge:discharge + n] = max_power_w + upper[curtail:curtail + n] = pv_w + reserve_soc = max( + min_soc, + min(100.0, _cfg_float(config, "batt_economic_reserve_soc_percent", 10.0)), + ) + economic_min_soc = max(min_soc, min(start_soc, reserve_soc)) + min_kwh = cap_kwh * economic_min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + lower[soc:soc + n + 1] = min_kwh + upper[soc:soc + n + 1] = max_kwh + current_peak_kw = max(0.0, float(data_obj.get("current_month_peak_kw", 0.0) or 0.0)) + lower[peak] = current_peak_kw + upper[peak] = max(current_peak_kw, max_import_w / 1000.0) + upper[battery_mode:battery_mode + n] = 1.0 + upper[grid_mode:grid_mode + n] = 1.0 + + objective = np.zeros(variable_count) + objective[imp:imp + n] = import_price * DT_H / 1000.0 + objective[exp:exp + n] = -export_price * DT_H / 1000.0 + degradation = max(0.0, _cfg_float(config, "batt_degradation_chf_kwh", 0.03)) + objective[charge:charge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[discharge:discharge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[curtail:curtail + n] = 1e-9 * DT_H / 1000.0 + objective[peak] = max(0.0, _cfg_float(config, "tarif_peak_fest", 0.0)) + terminal_value = _cfg_float(config, "batt_terminal_value_chf_kwh", np.median(import_price)) + objective[soc + n] = -max(0.0, terminal_value) * discharge_eff + + equality_rows = 2 * n + 1 + equality = lil_matrix((equality_rows, variable_count), dtype=float) + equality_rhs = np.zeros(equality_rows) + for i in range(n): + equality[i, imp + i] = 1.0 + equality[i, exp + i] = -1.0 + equality[i, charge + i] = -1.0 + equality[i, discharge + i] = 1.0 + equality[i, curtail + i] = -1.0 + equality_rhs[i] = residual_w[i] + + row = n + i + equality[row, soc + i] = -1.0 + equality[row, soc + i + 1] = 1.0 + equality[row, charge + i] = -charge_eff * DT_H / 1000.0 + equality[row, discharge + i] = DT_H / (1000.0 * discharge_eff) + equality[2 * n, soc] = 1.0 + equality_rhs[2 * n] = cap_kwh * start_soc / 100.0 + + quarter_groups = _quarter_groups(pd.Index(index)) + inequality_rows = 4 * n + len(quarter_groups) + inequality = lil_matrix((inequality_rows, variable_count), dtype=float) + inequality_upper = np.zeros(inequality_rows) + row = 0 + for i in range(n): + inequality[row, charge + i] = 1.0 + inequality[row, battery_mode + i] = -max_power_w + row += 1 + inequality[row, discharge + i] = 1.0 + inequality[row, battery_mode + i] = max_power_w + inequality_upper[row] = max_power_w + row += 1 + inequality[row, imp + i] = 1.0 + inequality[row, grid_mode + i] = -max_import_w + row += 1 + inequality[row, exp + i] = 1.0 + inequality[row, grid_mode + i] = max_export_w + inequality_upper[row] = max_export_w + row += 1 + for group in quarter_groups: + for i in group: + inequality[row, imp + i] = 1.0 / (len(group) * 1000.0) + inequality[row, peak] = -1.0 + row += 1 + + integrality = np.zeros(variable_count, dtype=int) + integrality[battery_mode:battery_mode + n] = 1 + integrality[grid_mode:grid_mode + n] = 1 + constraints = [ + LinearConstraint(equality.tocsr(), equality_rhs, equality_rhs), + LinearConstraint(inequality.tocsr(), -np.inf, inequality_upper), + ] + result = milp( + objective, + integrality=integrality, + bounds=Bounds(lower, upper), + constraints=constraints, + options={"time_limit": max(5.0, _cfg_float(config, "batt_optimizer_timeout_seconds", 30.0))}, + ) + if not result.success or result.x is None: + return _fallback_plan(index, residual_w) + + grid_values = result.x[imp:imp + n] - result.x[exp:exp + n] + battery_values = result.x[charge:charge + n] - result.x[discharge:discharge + n] + threshold_w = max(25.0, max_power_w * 0.005) + grid_values[np.abs(grid_values) < threshold_w] = 0.0 + battery_values[np.abs(battery_values) < threshold_w] = 0.0 + return { + "grid": {t: float(v) for t, v in zip(index, grid_values)}, + "battery": {t: float(v) for t, v in zip(index, battery_values)}, + "solver": "scipy-milp", + "objective_chf": float(result.fun), + "planned_peak_kw": float(result.x[peak]), + "start_soc_percent": float(start_soc), + "economic_min_soc_percent": float(economic_min_soc), + } + + +def optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=None): + plan = optimize_battery_plan(data_obj, pv_dict, load_dict) + if forecast_id is not None: + data_obj.setdefault("battery_plans", {})[int(forecast_id)] = plan + return plan["grid"] diff --git a/services/netplan-v4/acceptance/forecast-src/methods/common.py b/services/netplan-v4/acceptance/forecast-src/methods/common.py new file mode 100644 index 0000000..a0ab5e9 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/common.py @@ -0,0 +1,32 @@ +import os +import joblib + +MODEL_DIR = "/app/data/models" + + +def model_path(aid, forecast_id): + os.makedirs(MODEL_DIR, exist_ok=True) + return os.path.join(MODEL_DIR, f"forecast_var_{forecast_id}_{aid}.pkl") + + +def save_model(aid, forecast_id, artifact): + path = model_path(aid, forecast_id) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + joblib.dump(artifact, tmp_path) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + return path + + +def load_model(aid, forecast_id): + path = model_path(aid, forecast_id) + if not os.path.exists(path): + return None + try: + return joblib.load(path) + except Exception as exc: + print(f"Modell {path} konnte nicht geladen werden: {exc}") + return None diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_1.py b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py new file mode 100644 index 0000000..98cbe44 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py @@ -0,0 +1,66 @@ +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model +from shared_utils import calc_pure_math_pv, roof_features + +FORECAST_ID = 1 +FEATURES = [ + "math_pv", + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "sin_year", + "cos_year", + "pv_kwp_total", + "roof_azimuth_sin", + "roof_azimuth_cos", + "roof_tilt_avg", + "roof_south_factor", +] + + +def _features(frame, config): + out = frame.copy() + out["math_pv"] = [calc_pure_math_pv(config, t) for t in out.index] + rf = roof_features(config) + for k, v in rf.items(): + out[k] = v + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out[FEATURES].astype(float) + + +def train(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + df = data_obj.get("df_pv_training", data_obj["df_hist"]).copy() + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + X = _features(df, config) + y = pd.to_numeric(df["PV"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + model = RandomForestRegressor(n_estimators=400, max_depth=18, min_samples_leaf=2, random_state=42, n_jobs=-1) + model.fit(X, y) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES}) + return {"trained": True, "samples": int(len(X)), "path": path} + + +def predict(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + X = _features(data_obj["df_fut"], config) + if artifact and "model" in artifact: + values = artifact["model"].predict(X) + else: + values = X["math_pv"].to_numpy() + ac_limit = float(config.get("ac_leistung", 10.0) or 10.0) * 1000.0 + return {t: max(0.0, min(float(v), ac_limit)) for t, v in zip(data_obj["df_fut"].index, values)} diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_10.py b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py new file mode 100644 index 0000000..02ea37d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 10, {"type": "repeat_pv_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'PV') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_11.py b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py new file mode 100644 index 0000000..7e1dc3c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 11, {"type": "repeat_load_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'Hausverbrauch') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_13.py b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py new file mode 100644 index 0000000..9775eeb --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 13 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_10_11"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=13) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_2.py b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py new file mode 100644 index 0000000..fbb3d59 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py @@ -0,0 +1,169 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor + +from methods.common import load_model, save_model +from telemetry_quality import profile_source_value + +FORECAST_ID = 2 +FEATURES = [ + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "weekday", + "is_weekday", + "load_24h_ago", + "load_7d_ago", + "energy_24h_rolling", + "load_3d_same_time_mean", + "load_7d_same_time_mean", +] + + +def _history_features(df): + out = df.copy() + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_24h_rolling"] = (out["Hausverbrauch"] * 5 / 60 / 1000).shift(1).rolling(288).sum() + same_time_lags = [out["Hausverbrauch"].shift(288 * d) for d in range(1, 8)] + out["load_3d_same_time_mean"] = pd.concat(same_time_lags[:3], axis=1).mean(axis=1) + out["load_7d_same_time_mean"] = pd.concat(same_time_lags, axis=1).mean(axis=1) + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = data_obj.get("df_load_training", data_obj["df_hist"]).copy() + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = _history_features(df) + X = df[FEATURES].astype(float) + y = pd.to_numeric(df["Hausverbrauch"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + validation_day = X.index.max().normalize() + validation_mask = X.index >= validation_day + if int(validation_mask.sum()) < 144: + validation_day -= datetime.timedelta(days=1) + validation_mask = (X.index >= validation_day) & (X.index < validation_day + datetime.timedelta(days=1)) + score = None + if int((~validation_mask).sum()) >= 288 and int(validation_mask.sum()) >= 96: + eval_model = HistGradientBoostingRegressor( + max_iter=900, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + eval_model.fit(X.loc[~validation_mask], y.loc[~validation_mask]) + score = float(eval_model.score(X.loc[validation_mask], y.loc[validation_mask])) + + model = HistGradientBoostingRegressor( + max_iter=1200, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + model.fit(X, y) + path = save_model( + aid, + FORECAST_ID, + {"model": model, "features": FEATURES, "r2_last_day": score}, + ) + return { + "trained": True, + "samples": int(len(X)), + "r2_last_day": score, + "path": path, + } + + +def _history_value(history, ts): + if history is None or history.empty or ts not in history.index or "Hausverbrauch" not in history.columns: + return None + value = history.at[ts, "Hausverbrauch"] + return float(value) if pd.notna(value) and np.isfinite(value) and value >= 0 else None + + +def _same_time_values(history, t, days): + values = [] + for day in range(1, days + 1): + value = _history_value(history, t - datetime.timedelta(days=day)) + if value is not None and np.isfinite(value): + values.append(max(0.0, value)) + return values + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + + frames = [ + frame + for frame in ( + data_obj.get("df_load_training"), + data_obj.get("df_hist"), + data_obj.get("df_recent_raw"), + ) + if frame is not None and not frame.empty and "Hausverbrauch" in frame.columns + ] + history = pd.concat(frames).sort_index() if frames else pd.DataFrame(columns=["Hausverbrauch"]) + history = history[~history.index.duplicated(keep="last")] + fut = data_obj["df_fut"] + res = {} + + for t in fut.index: + same_values = _same_time_values(history, t, 7) + load_24 = _history_value(history, t - datetime.timedelta(days=1)) + load_7d = _history_value(history, t - datetime.timedelta(days=7)) + if load_24 is None: + load_24 = load_7d if load_7d is not None else profile_source_value(history, t, "Hausverbrauch") + if load_7d is None: + load_7d = load_24 + same_median = float(np.median(same_values)) if same_values else load_7d + profile = max(0.0, (0.50 * load_24) + (0.30 * load_7d) + (0.20 * same_median)) + + window = history.loc[ + t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), + "Hausverbrauch", + ] + energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + same_3 = same_values[:3] + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "cloud"]), + float(fut.at[t, "hour_sin"]), + float(fut.at[t, "hour_cos"]), + int(fut.at[t, "weekday"]), + int(fut.at[t, "is_weekday"]), + load_24, + load_7d, + energy, + float(np.mean(same_3)) if same_3 else profile, + float(np.mean(same_values)) if same_values else profile, + ]], columns=FEATURES) + + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + model_pred = min(max(model_pred, profile * 0.25), max(500.0, profile * 3.0)) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + + res[t] = pred + history.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_21.py b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py new file mode 100644 index 0000000..1c84981 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py @@ -0,0 +1,62 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model + +FORECAST_ID = 21 +FEATURES = ["temp_c", "hour_cos", "pv_24h_ago"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + if "PV" in out.columns: + out["pv_24h_ago"] = out["PV"].shift(288) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_pv_training", data_obj["df_hist"]).copy()) + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + df = df.dropna(subset=FEATURES + ["PV"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["PV"]) + score = float(eval_model.score(test_df[FEATURES], test_df["PV"])) + print(f"[var_21] R2 PV letzter kompletter Tag: {score:.3f}") + + model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + model.fit(df[FEATURES], df["PV"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + hist = data_obj["df_hist"] + fut = data_obj["df_fut"].copy() + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + pv_24 = float(hist.at[t_24, "PV"]) if t_24 in hist.index else 0.0 + row = pd.DataFrame([[float(fut.at[t, "temp_c"]), float(fut.at[t, "hour_cos"]), pv_24]], columns=FEATURES) + pred = float(model.predict(row)[0]) if model is not None else pv_24 + res[t] = max(0.0, pred) + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_22.py b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py new file mode 100644 index 0000000..3f1d6d8 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py @@ -0,0 +1,121 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.inspection import permutation_importance + +from methods.common import load_model, save_model + +FORECAST_ID = 22 +FEATURES = ["temp_c", "hour_cos", "load_24h_ago", "load_7d_ago", "energy_24h_rolling", "weekday"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + out["weekday"] = out.index.weekday + if "Hausverbrauch" in out.columns: + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_5m_kwh"] = out["Hausverbrauch"] * (5 / 60) / 1000 + out["energy_24h_rolling"] = out["energy_5m_kwh"].shift(1).rolling(window=288).sum() + out = out.drop(columns=["energy_5m_kwh"]) + return out + + +def _history_value(history, recent, reference, ts): + for frame in (history, recent, reference): + if frame is not None and not frame.empty and ts in frame.index and "Hausverbrauch" in frame.columns: + value = frame.at[ts, "Hausverbrauch"] + if pd.notna(value): + return float(value) + return None + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_load_training", data_obj["df_hist"]).copy()) + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = df.dropna(subset=FEATURES + ["Hausverbrauch"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + importance = {} + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["Hausverbrauch"]) + score = float(eval_model.score(test_df[FEATURES], test_df["Hausverbrauch"])) + print(f"[var_22] R2 Hausverbrauch letzter kompletter Tag: {score:.3f}") + try: + perm = permutation_importance(eval_model, test_df[FEATURES], test_df["Hausverbrauch"], n_repeats=10, random_state=42) + order = perm.importances_mean.argsort()[::-1] + importance = {FEATURES[i]: float(perm.importances_mean[i]) for i in order} + print("[var_22] Feature-Wichtigkeit Hausverbrauch:") + for name, val in importance.items(): + print(f" {name}: {val:.4f}") + except Exception as exc: + print(f"[var_22] permutation_importance nicht berechnet: {exc}") + + model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + model.fit(df[FEATURES], df["Hausverbrauch"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score, "importance": importance}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + hist = data_obj["df_hist"].copy() + recent = data_obj.get("df_recent_raw", pd.DataFrame()) + reference = data_obj.get("df_load_training", pd.DataFrame()) + fut = data_obj["df_fut"] + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + t_7d = t - datetime.timedelta(days=7) + load_7d = _history_value(hist, recent, reference, t_7d) + load_24 = _history_value(hist, recent, reference, t_24) + if load_24 is None: + load_24 = load_7d if load_7d is not None else 0.0 + if load_7d is None: + load_7d = load_24 + ref_end = t - datetime.timedelta(days=7) + ref_start = ref_end - datetime.timedelta(days=1) + if not reference.empty and "Hausverbrauch" in reference.columns: + window = reference.loc[ref_start:ref_end - datetime.timedelta(minutes=5), "Hausverbrauch"] + else: + window = pd.Series(dtype=float) + if window.empty and "Hausverbrauch" in recent.columns: + window = recent.loc[t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), "Hausverbrauch"] + roll_energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "hour_cos"]), + load_24, + load_7d, + roll_energy, + int(fut.at[t, "weekday"]), + ]], columns=FEATURES) + profile = max(0.0, (0.65 * load_24) + (0.35 * load_7d)) + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + res[t] = pred + hist.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_23.py b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py new file mode 100644 index 0000000..3cafe6d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 23 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_21_22"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=23) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_3.py b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py new file mode 100644 index 0000000..db46d6f --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 3 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_1_2"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=3) diff --git a/services/netplan-v4/acceptance/forecast-src/model_isolation.py b/services/netplan-v4/acceptance/forecast-src/model_isolation.py new file mode 100644 index 0000000..e64d3d1 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/model_isolation.py @@ -0,0 +1,45 @@ +"""Forecast family fault isolation; no IO, fabricated samples or model switching.""" +from collections.abc import Mapping +from math import isfinite +from numbers import Real + + +def collect_predictions(data, config, models, enabled): + """Return complete finite PV/load series independently for each requested model. + + A model with insufficient history must not prevent other valid families being + published. It is omitted, not filled with zeros or replaced by another model. + Global raw-telemetry checks still run BEFORE this helper in run_forecast. + """ + expected = tuple(data['df_fut'].index) + if not expected or len(set(expected)) != len(expected): + raise ValueError('Nonempty unique future interval index required') + expected_keys = set(expected) + predictions, errors = {}, {} + requested = 0 + for variant, model in models.items(): + predictions[variant] = {} + if not enabled(config, variant): + continue + requested += 1 + try: + values = model.predict(data) + if not isinstance(values, Mapping) or set(values) != expected_keys: + raise ValueError('Missing, extra or non-matching forecast timestamps') + cleaned = {} + for at in expected: + value = values[at] + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError('Non-numeric forecast power') + value = float(value) + if not isfinite(value) or value < 0: + raise ValueError('Invalid nonnegative forecast power') + cleaned[at] = value + predictions[variant] = cleaned + except Exception as error: + # Expected data failures and unexpected model failures are visible, + # but model exception text may contain filesystem paths. Do not leak it. + errors[variant] = {'status': 'unavailable', 'errorType': type(error).__name__} + if requested and all(not p for p in predictions.values()): + raise ValueError('All requested PV/load models failed; no forecasts published') + return predictions, errors diff --git a/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/acceptance/forecast-src/shared_utils.py b/services/netplan-v4/acceptance/forecast-src/shared_utils.py new file mode 100644 index 0000000..fb40dce --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/shared_utils.py @@ -0,0 +1,93 @@ +import math + + +def as_float(value, default=0.0): + try: + if value is None: + return default + return float(value) + except Exception: + return default + + +def roof_kwp(roof): + return as_float(roof.get("kwp", roof.get("leistung", roof.get("pv_kwp", 0.0))), 0.0) + + +def roof_azimuth_deg(roof): + return as_float(roof.get("azimut", roof.get("azimuth", roof.get("ausrichtung", 180.0))), 180.0) + + +def roof_tilt_deg(roof): + return as_float(roof.get("neigung", roof.get("tilt", 30.0)), 30.0) + + +def roof_features(config): + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "azimut": 180.0, "neigung": 30.0}] + total_kwp = sum(roof_kwp(r) for r in roofs) + if total_kwp <= 0: + total_kwp = as_float(config.get("ac_leistung", 0.0), 0.0) + weighted_az = 0.0 + weighted_tilt = 0.0 + south_factor = 0.0 + for roof in roofs: + w = roof_kwp(roof) / total_kwp if total_kwp > 0 else 0.0 + az = roof_azimuth_deg(roof) + tilt = roof_tilt_deg(roof) + weighted_az += w * az + weighted_tilt += w * tilt + south_factor += w * max(0.0, math.cos(math.radians(az - 180.0))) + return { + "pv_kwp_total": total_kwp, + "roof_azimuth_sin": math.sin(math.radians(weighted_az)), + "roof_azimuth_cos": math.cos(math.radians(weighted_az)), + "roof_tilt_avg": weighted_tilt, + "roof_south_factor": south_factor, + } + + +def calc_pure_math_pv(config, dt): + ac_limit = as_float(config.get("ac_leistung", 10.0), 10.0) * 1000.0 + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "neigung": 30.0, "azimut": 180.0}] + + day = dt.timetuple().tm_yday + hour = dt.hour + dt.minute / 60.0 + lat = math.radians(as_float(config.get("latitude", 47.0), 47.0)) + decl = math.radians(23.45 * math.sin(math.radians(360.0 * (day - 81) / 365.0))) + hour_angle = math.radians(15.0 * (hour - 12.0)) + + sin_alt = math.sin(lat) * math.sin(decl) + math.cos(lat) * math.cos(decl) * math.cos(hour_angle) + sun_alt = math.asin(max(-1.0, min(1.0, sin_alt))) + if sun_alt <= 0.0: + return 0.0 + + sun_az = math.atan2( + math.sin(hour_angle), + math.cos(hour_angle) * math.sin(lat) - math.tan(decl) * math.cos(lat), + ) + + total = 0.0 + for roof in roofs: + kwp = roof_kwp(roof) + if kwp <= 0: + continue + tilt = math.radians(roof_tilt_deg(roof)) + az = math.radians(roof_azimuth_deg(roof)) + cos_inc = math.sin(sun_alt) * math.cos(tilt) + math.cos(sun_alt) * math.sin(tilt) * math.cos(sun_az - az) + if cos_inc > 0: + total += kwp * 1000.0 * cos_inc + + return max(0.0, min(total, ac_limit)) + + +def calc_pure_math_load(dt): + hour = dt.hour + dt.minute / 60.0 + base = 650.0 + morning = 220.0 * math.exp(-((hour - 7.0) ** 2) / 5.0) + evening = 380.0 * math.exp(-((hour - 19.0) ** 2) / 8.0) + weekend = 1.12 if dt.weekday() >= 5 else 1.0 + return max(0.0, (base + morning + evening) * weekend) diff --git a/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py new file mode 100644 index 0000000..f2bf2e7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py @@ -0,0 +1,66 @@ +from influxdb_client import Point, WritePrecision + +DT_H = 5.0 / 60.0 + + +def _f(config, key, default): + try: + return float(config.get(key, default) or default) + except Exception: + return float(default) + + +def battery_soc_points(data_obj, forecast_var, pv_dict, load_dict, grid_dict): + config = data_obj["config"] + aid = str(config["anlagen_id"]) + cap_kwh = _f(config, "batt_capacity_kwh", 0.0) + if cap_kwh <= 0 or not grid_dict: + return [] + + min_soc = _f(config, "batt_min_soc", _f(config, "batt_min_soc_percent", 0.0)) + max_soc = _f(config, "batt_max_soc", _f(config, "batt_max_soc_percent", 100.0)) + charge_eff = max(0.01, min(1.0, _f(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _f(config, "batt_discharge_efficiency", 0.95))) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None or start_soc_value == "": + start_soc_value = config.get("batt_soc_percent", 50.0) + start_soc = float(start_soc_value) + start_soc = max(min_soc, min(max_soc, start_soc)) + + min_kwh = cap_kwh * min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + soc_kwh = max(min_kwh, min(max_kwh, cap_kwh * start_soc / 100.0)) + + planned_battery = ( + data_obj.get("battery_plans", {}) + .get(int(forecast_var), {}) + .get("battery", {}) + ) + points = [] + for t in data_obj["df_fut"].index: + if t not in grid_dict: + continue + if t in planned_battery: + battery_target_w = float(planned_battery[t]) # positiv = laden + else: + residual_w = float(load_dict.get(t, 0.0)) - float(pv_dict.get(t, 0.0)) + grid_w = float(grid_dict.get(t, 0.0)) + battery_target_w = grid_w - residual_w + + if battery_target_w > 0: + soc_kwh += battery_target_w * DT_H / 1000.0 * charge_eff + elif battery_target_w < 0: + soc_kwh += battery_target_w * DT_H / 1000.0 / discharge_eff + + soc_kwh = max(min_kwh, min(max_kwh, soc_kwh)) + soc_percent = max(0.0, min(100.0, soc_kwh / cap_kwh * 100.0)) + + points.append( + Point("forecast_diagnostics") + .tag("anlagen_id", aid) + .tag("data_type", "battery_soc_simulation") + .tag("forecast_var", f"prog_var_{forecast_var}") + .field("soc_percent", float(soc_percent)) + .time(t.to_pydatetime() if hasattr(t, "to_pydatetime") else t, WritePrecision.S) + ) + return points diff --git a/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py new file mode 100644 index 0000000..bf316d2 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py @@ -0,0 +1,106 @@ +"""Measured telemetry integrity and repeat-profile helpers. +No IO, no fabricated measurements and no fixed household-load fallback. +Forecast freshness is checked against raw telemetry, never against filled features. +""" +from __future__ import annotations +import datetime +import math +import numpy as np +import pandas as pd + +MEASURED_COLUMNS = ('PV', 'Hausverbrauch', 'Netzleistung', 'SOC') + +class TelemetryUnavailable(ValueError): + """No publishable forecast can be derived from the supplied observations.""" + + +def sanitize_measured_frame(frame): + out = frame.copy() + for column in MEASURED_COLUMNS: + if column not in out: + continue + raw = out[column] + numeric = pd.to_numeric(raw, errors='coerce').astype(float) + bad = ~np.isfinite(numeric) | raw.map(lambda x: isinstance(x, (bool, np.bool_))) + if column in ('PV', 'Hausverbrauch', 'SOC'): + bad |= numeric < 0 + if column == 'SOC': + bad |= numeric > 100 + out[column] = numeric.mask(bad) + return out + + +def _naive_utc(value): + stamp = pd.Timestamp(value) + if stamp.tzinfo is not None: + stamp = stamp.tz_convert('UTC').tz_localize(None) + return stamp + + +def require_recent_telemetry(data_obj, fields=('PV','Hausverbrauch'), max_age_minutes=30.0): + """Refuse missing/stale inputs without changing real zeros or raw samples.""" + if isinstance(max_age_minutes, bool) or not math.isfinite(max_age_minutes) or max_age_minutes <= 0: + raise ValueError('Positive telemetry age limit required') + now = _naive_utc(data_obj['now']) + raw = data_obj.get('df_recent_raw') + raw = sanitize_measured_frame(raw) if raw is not None else pd.DataFrame() + if not raw.empty: + raw.index = pd.DatetimeIndex([_naive_utc(t) for t in raw.index]) + raw = raw.loc[raw.index < now].sort_index() + report = {} + errors = [] + for field in fields: + if field not in MEASURED_COLUMNS: + raise ValueError('Unknown telemetry target') + values = raw[field].dropna() if field in raw else pd.Series(dtype=float) + if values.empty: + errors.append(field + ': keine gemessenen Werte') + continue + stamp = values.index[-1] + age = (now-stamp).total_seconds()/60.0 + report[field] = {'lastObservedInterval': stamp.isoformat()+'Z', 'ageMinutes': age, + 'observedIntervals': int(len(values)), 'lastValue': float(values.iloc[-1])} + if age > max_age_minutes: + errors.append(field + ': Messdaten veraltet (' + format(age,'.1f') + ' min)') + if errors: + raise TelemetryUnavailable('; '.join(errors) + '. Keine neuen Prognosen/Fahrplaene veroeffentlicht.') + return report + + +def _finite_nonnegative(value): + if isinstance(value, (bool, np.bool_)): + return None + try: + value = float(value) + except (ValueError, TypeError): + return None + return value if math.isfinite(value) and value >= 0.0 else None + + +def profile_source_value(history, at, column, predictions=None): + """Repeat yesterday; prefer same weekday if yesterday is missing, then older days. + Explicitly generated first-day values may be repeated on the second forecast day. + Missing historical values are not zeros. No future measurement is read. + """ + if column not in ('PV','Hausverbrauch'): + raise ValueError('Unsupported repeat-profile target') + predictions = {} if predictions is None else predictions + at = pd.Timestamp(at) + for day in (1, 7, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14): + source = at - datetime.timedelta(days=day) + if source in predictions: + value = _finite_nonnegative(predictions[source]) + if value is not None: + return value + if history is not None and column in history and source in history.index: + value = _finite_nonnegative(history.at[source,column]) + if value is not None: + return value + raise TelemetryUnavailable(column + ': kein gemessener Tagesprofilwert fuer ' + str(at)) + + +def repeat_daily_profile(history, future_index, column): + result = {} + for at in future_index: + result[at] = profile_source_value(history, at, column, result) + return result diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py new file mode 100644 index 0000000..e533d76 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py @@ -0,0 +1,133 @@ +import unittest + +import pandas as pd + +from methods.battery_optimizer import DT_H, optimize_battery_plan + + +class BatteryCostOptimizerTest(unittest.TestCase): + def plan(self, load, pv, import_prices=None, export_prices=None, **overrides): + index = pd.date_range("2026-09-29T00:00:00", periods=len(load), freq="5min") + config = { + "batt_capacity_kwh": 2.0, + "batt_power_kw": 1.0, + "batt_min_soc": 0.0, + "batt_max_soc": 100.0, + "batt_charge_efficiency": 1.0, + "batt_discharge_efficiency": 1.0, + "batt_soc_percent": 0.0, + "batt_grid_charging_enabled": False, + "batt_degradation_chf_kwh": 0.0, + "tarif_bezug": "dynamic", + "tarif_einspeisung": "dynamic", + "tarif_peak_fest": 0.0, + } + config.update(overrides) + data = { + "config": config, + "df_fut": pd.DataFrame({ + "import_price": import_prices or [0.30] * len(index), + "export_price": export_prices or [0.10] * len(index), + }, index=index), + "current_soc_perc": config["batt_soc_percent"], + "current_month_peak_kw": overrides.get("current_month_peak_kw", 0.0), + } + return index, optimize_battery_plan(data, dict(zip(index, pv)), dict(zip(index, load))) + + def test_grid_charging_is_opt_in(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + ) + self.assertTrue(all(plan["battery"][t] <= 1e-6 for t in index)) + + def test_cheap_grid_energy_is_shifted_to_expensive_period(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + batt_grid_charging_enabled=True, + ) + self.assertGreater(plan["battery"][index[0]], 0.0) + self.assertLess(plan["battery"][index[-1]], 0.0) + self.assertGreater(plan["grid"][index[0]], 500.0) + self.assertAlmostEqual(plan["grid"][index[-1]], 0.0, places=5) + + def test_high_feed_in_value_prefers_export(self): + index, plan = self.plan( + [0.0, 1000.0], + [1000.0, 0.0], + import_prices=[0.20, 0.20], + export_prices=[0.60, 0.60], + ) + self.assertAlmostEqual(plan["grid"][index[0]], -1000.0, places=5) + self.assertAlmostEqual(plan["grid"][index[1]], 1000.0, places=5) + + def test_peak_tariff_prevents_grid_charge_above_existing_peak(self): + index, plan = self.plan( + [1000.0] * 6, + [0.0] * 6, + import_prices=[0.05] * 3 + [0.50] * 3, + batt_grid_charging_enabled=True, + tarif_peak_fest=20.0, + current_month_peak_kw=1.0, + ) + self.assertLessEqual(plan["planned_peak_kw"], 1.0 + 1e-7) + self.assertTrue(all(plan["grid"][t] <= 1000.0 + 1e-5 for t in index)) + + def test_nearly_empty_battery_is_not_discharged_further_at_low_value(self): + index, plan = self.plan( + [1000.0] * 4, + [0.0] * 4, + import_prices=[0.01] * 4, + export_prices=[0.0] * 4, + batt_soc_percent=5.0, + batt_economic_reserve_soc_percent=10.0, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(all(plan["battery"][timestamp] >= -1e-6 for timestamp in index)) + self.assertTrue(all(abs(plan["grid"][timestamp] - 1000.0) <= 1e-5 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 5.0) + + def test_profitable_export_discharge_remains_allowed_above_reserve(self): + index, plan = self.plan( + [100.0, 100.0], + [1000.0, 1000.0], + import_prices=[0.20, 0.20], + export_prices=[0.80, 0.80], + batt_soc_percent=100.0, + batt_economic_reserve_soc_percent=10.0, + batt_degradation_chf_kwh=0.03, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(any(plan["battery"][timestamp] < -1e-6 for timestamp in index)) + self.assertTrue(all(plan["grid"][timestamp] <= -899.0 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 10.0) + + def test_power_and_soc_limits_hold(self): + load = [0.0] * 12 + [1000.0] * 12 + pv = [1000.0] * 12 + [0.0] * 12 + index, plan = self.plan( + load, + pv, + batt_capacity_kwh=1.0, + batt_power_kw=0.5, + batt_min_soc=20.0, + batt_max_soc=80.0, + batt_soc_percent=20.0, + ) + soc = 0.2 + for timestamp in index: + target = plan["battery"][timestamp] + self.assertLessEqual(abs(target), 500.0 + 1e-6) + if target >= 0.0: + soc += target * DT_H / 1000.0 + else: + soc += target * DT_H / 1000.0 + self.assertGreaterEqual(soc, 0.2 - 1e-8) + self.assertLessEqual(soc, 0.8 + 1e-8) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py new file mode 100644 index 0000000..42767ef --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py @@ -0,0 +1,66 @@ +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from methods.var_2 import predict +from methods.var_11 import predict as predict_repeat + + +class LoadProfileForecastTest(unittest.TestCase): + def test_profile_keeps_daily_shape_across_48_hours(self): + history_index = pd.date_range("2026-09-21", periods=8 * 288, freq="5min") + phase = 2 * np.pi * ( + history_index.hour.to_numpy() + history_index.minute.to_numpy() / 60.0 + ) / 24.0 + history = pd.DataFrame( + {"Hausverbrauch": 1800.0 + 900.0 * np.cos(phase - np.pi)}, + index=history_index, + ) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=2 * 288, freq="5min") + future_phase = 2 * np.pi * ( + future_index.hour.to_numpy() + future_index.minute.to_numpy() / 60.0 + ) / 24.0 + future = pd.DataFrame( + { + "temp_c": 15.0, + "cloud": 20.0, + "hour_sin": np.sin(future_phase), + "hour_cos": np.cos(future_phase), + "weekday": future_index.weekday, + "is_weekday": (future_index.weekday < 5).astype(int), + }, + index=future_index, + ) + data = { + "config": {"anlagen_id": "test"}, + "df_hist": history, + "df_load_training": history, + "df_recent_raw": history.iloc[-2 * 288 :], + "df_fut": future, + } + + with patch("methods.var_2.load_model", return_value=None): + values = np.array(list(predict(data).values())) + + self.assertGreater(float(values.max() - values.min()), 1200.0) + np.testing.assert_allclose(values[:288], values[288:], rtol=0.0, atol=1e-6) + + def test_repeat_profile_remains_available_on_second_day(self): + history_index = pd.date_range("2026-09-29", periods=288, freq="5min") + daily_values = np.arange(288, dtype=float) + 1000.0 + history = pd.DataFrame({"Hausverbrauch": daily_values}, index=history_index) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=576, freq="5min") + + values = np.array(list(predict_repeat({ + "df_hist": history, + "df_fut": pd.DataFrame(index=future_index), + }).values())) + + np.testing.assert_allclose(values[:288], daily_values) + np.testing.assert_allclose(values[288:], daily_values) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py new file mode 100644 index 0000000..dd6220c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py @@ -0,0 +1,71 @@ +import ast +import contextlib +import datetime +import io +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock +from model_isolation import collect_predictions + + +class ModelIsolationTest(unittest.TestCase): + def setUp(self): + self.index = [datetime.datetime(2026,10,2,6,0) + datetime.timedelta(minutes=5*i) for i in range(3)] + self.data = {'df_fut': SimpleNamespace(index=self.index)} + self.good = {t: 1000.0 for t in self.index} + def model(self, values=None): + return SimpleNamespace(predict=Mock(return_value=self.good if values is None else values)) + def call(self, models, enabled=lambda c,n:True): + return collect_predictions(self.data, {}, models, enabled) + def test_one_failed_model_does_not_remove_other_families(self): + bad=self.model();bad.predict.side_effect=ValueError('private path not logged') + forecasts,errors=self.call({1:self.model(),10:bad,21:self.model()}) + self.assertEqual(forecasts[1],self.good);self.assertEqual(forecasts[21],self.good) + self.assertEqual(forecasts[10],{});self.assertEqual(errors[10]['errorType'],'ValueError') + self.assertNotIn('private',str(errors)) + def test_missing_timestamp_does_not_get_filled(self): + forecasts,errors=self.call({1:self.model(),10:self.model({self.index[0]:20.0})}) + self.assertFalse(forecasts[10]);self.assertIn(10,errors) + def test_nonfinite_negative_and_boolean_rejected(self): + for value in (float('nan'),float('inf'),-1.0,True,'2'): + with self.subTest(value=value): + result,errors=self.call({1:self.model(),10:self.model({t:value for t in self.index})}) + self.assertFalse(result[10]);self.assertIn(10,errors) + def test_explicit_zero_forecast_is_not_imputed(self): + result,errors=self.call({10:self.model({t:0.0 for t in self.index})}) + self.assertEqual(errors,{});self.assertEqual(sum(result[10].values()),0.) + def test_all_requested_models_fail_closed(self): + with self.assertRaisesRegex(ValueError,'All requested'): + self.call({10:self.model({})}) + def test_disabled_models_not_called(self): + model=self.model();result,errors=self.call({10:model},lambda c,n:False) + model.predict.assert_not_called();self.assertEqual(result,{10:{}});self.assertEqual(errors,{}) + def test_input_time_order_preserved(self): + result,_=self.call({1:self.model(dict(reversed(list(self.good.items()))))}) + self.assertEqual(list(result[1]),self.index) + def test_run_forecast_publishes_valid_pairs_after_repeat_failure(self): + root=Path(__file__).resolve().parents[1] + tree=ast.parse((root/'main.py').read_text()) + fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run_forecast') + modules={n:self.model() for n in (1,2,3,10,11,13,21,22,23)} + modules[10].predict.side_effect=ValueError('profile gap') + client=Mock();published=Mock() + data={**self.data,'current_soc_perc':20.,'current_soc_source':'telemetry'} + ns={'datetime':datetime,'LOCAL_TZ':datetime.timezone.utc,'traceback':Mock(), + 'get_configs':lambda:[{'anlagen_id':'test','batt_capacity_kwh':10.}], + 'InfluxDBClient':Mock(return_value=client),'SYNCHRONOUS':object(), + 'INFLUX_URL':'offline','INFLUX_TOKEN':'synthetic','INFLUX_ORG':'offline', + 'INFLUX_BUCKET':'offline','INFLUX_TIMEOUT_MS':1, + 'build_data_object':lambda *a,**k:data,'active':lambda c,n:True, + 'require_recent_telemetry':lambda *a,**k:{},'collect_predictions':collect_predictions, + '_v4_publish_forecasts':published,'battery_soc_points':lambda *a:[], + '_forecast_point':lambda *a:a,'_snapshot_point':lambda *a:a, + 'write_quality_metrics':Mock(),**{'v'+str(n):m for n,m in modules.items()}} + exec(compile(ast.Module(body=[fn],type_ignores=[]),'source-run-forecast','exec'),ns) + with contextlib.redirect_stdout(io.StringIO()):r=ns['run_forecast']() + self.assertEqual(r['completed'],['test']) + families=published.call_args.args[1] + self.assertTrue(families[0][1] and families[0][2]);self.assertFalse(families[1][1]) + self.assertTrue(families[2][1] and families[2][2]);modules[13].predict.assert_not_called() + client.write_api.return_value.write.assert_called_once() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py new file mode 100644 index 0000000..693c530 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py @@ -0,0 +1,213 @@ +"""Offline regressions against the exact reviewed orchestration source. +Only selected function definitions are compiled; main is NOT imported, and there +is no database, network, model-file, prediction-publication or device access. +""" +import ast +import contextlib +import datetime +import io +from pathlib import Path +import unittest +from unittest.mock import Mock, patch + +import numpy as np +import pandas as pd +from telemetry_quality import (TelemetryUnavailable, require_recent_telemetry, + sanitize_measured_frame, repeat_daily_profile, profile_source_value) + +ROOT = Path(__file__).resolve().parents[1] +AT = pd.Timestamp('2026-10-01T20:00:00') + + +def functions(*names, **extra): + tree = ast.parse((ROOT/'main.py').read_text()) + nodes = [n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names] + if {n.name for n in nodes} != set(names): + raise AssertionError('Reviewed function missing') + ns = {'pd':pd,'np':np,'datetime':datetime, + 'sanitize_measured_frame':sanitize_measured_frame, + 'require_recent_telemetry':require_recent_telemetry} + ns.update(extra) + exec(compile(ast.Module(body=nodes,type_ignores=[]),str(ROOT/'main.py'),'exec'),ns) + return ns + + +def raw_frame(start=None, periods=288): + index=pd.date_range(start if start is not None else AT-pd.Timedelta(days=1),periods=periods,freq='5min') + return pd.DataFrame({'PV':200.,'Hausverbrauch':3200.,'SOC':25.,'Netzleistung':3000.},index=index) + + +class MeasuredTelemetryTest(unittest.TestCase): + def test_empty_history_is_not_zero_consumption(self): + ns=functions('_fill_defaults','_add_time_features') + frame=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=3,freq='5min')),True) + self.assertTrue(frame[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + self.assertEqual(frame['temp_c'].tolist(),[15.]*3) + + def test_recorded_zero_is_preserved(self): + frame=raw_frame(periods=3);frame[:]=0. + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(out['Hausverbrauch'].tolist(),[0.]*3) + self.assertEqual(out['PV'].tolist(),[0.]*3) + + def test_internal_and_trailing_measurement_gaps_remain_missing(self): + frame=raw_frame(periods=7) + frame.loc[frame.index[[0,2,3,6]],['PV','Hausverbrauch','SOC','Netzleistung']]=np.nan + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(int(out['Hausverbrauch'].isna().sum()),4) + self.assertEqual(int(out['SOC'].isna().sum()),4) + self.assertEqual(int(out['Netzleistung'].isna().sum()),4) + + def test_bad_values_not_real_measurements(self): + frame=pd.DataFrame({'Hausverbrauch':[np.inf,-1.,True,0.,250.], 'Netzleistung':[-100.,np.nan,0.,1.,2.], 'SOC':[101.,-1.,np.inf,0.,100.]}) + out=sanitize_measured_frame(frame) + self.assertTrue(out['Hausverbrauch'].iloc[:3].isna().all()) + self.assertEqual(out['Hausverbrauch'].iloc[3],0.) + self.assertEqual(out['Netzleistung'].iloc[0],-100.) + self.assertTrue(out['SOC'].iloc[:3].isna().all()) + + def test_recent_recorded_zero_passes(self): + frame=raw_frame();frame[['PV','Hausverbrauch']]=0. + report=require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_missing_raw_cannot_be_hidden_by_filled_feature_grid(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':pd.DataFrame(),'df_hist':raw_frame()}) + + def test_stale_values_fail(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':raw_frame(AT-pd.Timedelta(days=2))}) + + def test_freshness_is_checked_per_field(self): + frame=raw_frame();frame.loc[frame.index[-12:],'Hausverbrauch']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'Hausverbrauch'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_future_measurements_do_not_rescue_freshness(self): + frame=raw_frame(AT,periods=3) + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_aware_timestamps_normalized_to_utc(self): + frame=raw_frame();frame.index=frame.index.tz_localize('UTC').tz_convert('Europe/Zurich') + report=require_recent_telemetry({'now':AT.tz_localize('UTC'),'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_soc_staleness_cannot_be_hidden(self): + frame=raw_frame();frame.loc[frame.index[-24:],'SOC']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'SOC'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame},['SOC']) + + def test_only_exogenous_future_defaults_are_filled(self): + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=6,freq='5min')),False) + self.assertTrue(out[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + + def test_input_unchanged(self): + frame=raw_frame();before=frame.copy(deep=True) + sanitize_measured_frame(frame);require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + pd.testing.assert_frame_equal(frame,before) + + +class HistoryOrchestrationTest(unittest.TestCase): + def data(self,telemetry): + class FrozenDateTime(datetime.datetime): + @classmethod + def utcnow(cls):return AT.to_pydatetime() + import types + dates=types.SimpleNamespace(datetime=FrozenDateTime,timedelta=datetime.timedelta) + weather=pd.DataFrame({'temp_c':15.,'cloud':20.},index=pd.date_range(AT-pd.Timedelta(days=14),AT+pd.Timedelta(days=2),freq='5min')) + fetch=Mock(return_value=(telemetry,weather,pd.DataFrame())) + ns=functions('build_data_object','_fill_defaults','_add_time_features','_longest_consistent_segment',datetime=dates,fetch_influx_frames=fetch,FORECAST_HORIZON_HOURS=48) + return ns['build_data_object']({'anlagen_id':'offline'},False) + + def test_weather_tail_does_not_erase_historical_load(self): + telemetry=raw_frame(AT-pd.Timedelta(days=2),periods=288) + data=self.data(telemetry) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw'].index.max(),telemetry.index.max()) + self.assertTrue(data['df_hist']['Hausverbrauch'].iloc[-24:].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_no_measurements_preserves_all_missing(self): + data=self.data(pd.DataFrame()) + self.assertTrue(data['df_hist']['Hausverbrauch'].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_reconnected_short_tail_does_not_delete_past_profile(self): + old=raw_frame(AT-pd.Timedelta(days=2),periods=288) + new=raw_frame(AT-pd.Timedelta(minutes=10),periods=2) + data=self.data(pd.concat([old,new])) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),290) + self.assertEqual(require_recent_telemetry(data)['Hausverbrauch']['ageMinutes'],5.) + + def test_stale_forecast_never_predicts_or_publishes(self): + import types + models={n:types.SimpleNamespace(predict=Mock()) for n in (1,2,3,10,11,13,21,22,23)} + client=Mock();write=Mock();client.write_api.return_value=write + publish=Mock();trace=Mock() + ns=functions('run_forecast',get_configs=lambda:[{'anlagen_id':'offline','batt_capacity_kwh':10}], + InfluxDBClient=Mock(return_value=client),SYNCHRONOUS=object(), + INFLUX_URL='offline',INFLUX_TOKEN='synthetic',INFLUX_ORG='offline',INFLUX_BUCKET='offline',INFLUX_TIMEOUT_MS=1, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},active=lambda *args:True, + _v4_publish_forecasts=publish,traceback=trace,**{'v'+str(k):v for k,v in models.items()}) + with contextlib.redirect_stdout(io.StringIO()),self.assertRaises(RuntimeError):ns['run_forecast']() + for model in models.values():model.predict.assert_not_called() + publish.assert_not_called();write.write.assert_not_called();client.close.assert_called_once() + + def test_stale_training_does_not_overwrite_model(self): + import types + train=Mock();mod=types.SimpleNamespace(train=train) + ns=functions('run_training',get_configs=lambda:[{'anlagen_id':'offline'}], + MODEL_MODULES={2:mod},QUALITY_TARGETS={2:'Hausverbrauch'},active=lambda *a:True, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},traceback=Mock()) + with contextlib.redirect_stdout(io.StringIO()):ns['run_training']() + train.assert_not_called() + + def test_queries_exclude_forecasts_and_align_interval_starts(self): + source=(ROOT/'main.py').read_text();ns=functions('fetch_influx_frames', + HISTORY_START='1970-01-01T00:00:00Z',FORECAST_HORIZON_HOURS=48,INFLUX_BUCKET='offline', + _query_df=Mock(return_value=pd.DataFrame()),_pivot_frame=lambda df,fields:df, + _tariff_frame=lambda df,cfg:df,_time_literal=lambda t:t.isoformat()) + ns['fetch_influx_frames']({'anlagen_id':'offline'},False) + query=ns['_query_df'].call_args_list[0].args[0] + self.assertIn('timeSrc: "_start"',query) + self.assertIn('!= "forecast_snapshot"',query) + self.assertIn('!= "forecast"',query) + + +class RepeatProfileIntegrityTest(unittest.TestCase): + def test_pv_repeats_on_second_day(self): + history=raw_frame();history['PV']=np.maximum(0.,np.sin(np.arange(288)*2*np.pi/288))*12000. + idx=pd.date_range(AT,periods=576,freq='5min') + actual=np.array(list(repeat_daily_profile(history,idx,'PV').values())) + np.testing.assert_allclose(actual[:288],history['PV']) + np.testing.assert_allclose(actual[288:],history['PV']) + + def test_missing_yesterday_uses_older_finite_day(self): + t=AT + history=pd.DataFrame({'Hausverbrauch':[3500.,np.nan]},index=[t-pd.Timedelta(days=7),t-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,t,'Hausverbrauch'),3500.) + + def test_missing_profile_is_not_zero(self): + history=pd.DataFrame({'Hausverbrauch':[np.nan]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):repeat_daily_profile(history,[AT],'Hausverbrauch') + + def test_recorded_profile_zero_is_valid(self): + history=pd.DataFrame({'PV':[0.]},index=[AT-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,AT,'PV'),0.) + + def test_negative_profile_cannot_be_silently_clamped(self): + history=pd.DataFrame({'Hausverbrauch':[-100.]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + def test_future_value_is_never_used_as_history(self): + history=pd.DataFrame({'Hausverbrauch':[3300.]},index=[AT+pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..c667975 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json @@ -0,0 +1,9 @@ +{ + "libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" +} diff --git a/services/netplan-v4/acceptance/php-src/check.php b/services/netplan-v4/acceptance/php-src/check.php new file mode 100644 index 0000000..feb59a8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/check.php @@ -0,0 +1,45 @@ + $expected) { + $path = __DIR__ . '/' . $relative; + if (!is_file($path) || hash_file('sha256', $path) !== $expected) { + throw new RuntimeException('Source manifest mismatch: ' . $relative); + } + $command = escapeshellarg(PHP_BINARY) . ' -l ' . escapeshellarg($path); + $lines = []; + exec($command, $lines, $code); + if ($code !== 0) { + throw new RuntimeException('PHP syntax failure: ' . $relative); + } + } + require __DIR__ . '/libs/NetzfahrplanV4Bezugszaehler.php'; + require __DIR__ . '/libs/NetzfahrplanV4Betriebsdaten.php'; + require __DIR__ . '/libs/ManagerNetzfahrplanV4Trait.php'; + require __DIR__ . '/tests/fixtures/NativeV4Scenarios.php'; + require __DIR__ . '/tests/fixtures/V4BezugszaehlerScenarios.php'; + $native = \Belevo\EnelixEMS\Tests\NativeV4Scenarios::run(); + $meters = \Belevo\EnelixEMS\Tests\V4BezugszaehlerScenarios::run(); + if (count($native) !== 24 || count($meters) !== 20) { + throw new RuntimeException('Expected 24 native and 20 meter scenarios.'); + } + foreach (array_merge($native, $meters) as $name) { + echo 'PASS ', $name, "\n"; + } + echo 'PHP ', PHP_VERSION, ': 44 offline conversion scenarios passed.', "\n"; + echo "NOT a full EMS suite and NOT an IP-Symcon runtime/physical test.\n"; +} catch (Throwable $error) { + fwrite(STDERR, 'FAIL: ' . $error->getMessage() . "\n"); + exit(1); +} diff --git a/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php new file mode 100644 index 0000000..c6e7412 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php @@ -0,0 +1,164 @@ +RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); + $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); + $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]'); + $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4Senden', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); + } + + public function GetNetzfahrplanV4Diagnose(): string + { + try { + $manager = []; + foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', + 'Lastspitzenmodus'] as $key) { + $manager[$key] = $this->ReadPropertyInteger($key); + } + foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { + $manager[$key] = $this->ReadPropertyFloat($key); + } + $manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified'; + $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); + $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); + $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); + $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $manager['V4BezugszaehlerQuellen'] = json_decode( + $this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR + ); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); + $cache = $this->leseJsonAttribut('VerbraucherCache'); + $controllers = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + $c = ['InstanzID' => $id]; + foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + $c[$key] = IPS_GetProperty($id, $key); + } + $entry = $cache[(string) $id] ?? []; + $c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0; + $c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false; + foreach (($entry['Daten']['Zustand'] ?? []) as $state) { + if (($state['Kennung'] ?? '') === 'HystereseAktiv') { + $c['HystereseAktiv'] = $state['Wert']; + } + } + $controllers[] = $c; + } + $evidence = []; + $evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID'); + if ($evidenceID > 0) { + if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) { + throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.'); + } + $evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($evidence)) { + throw new RuntimeException('Ungueltiger Messnachweis.'); + } + } + $read = static function (int $id): array { + if (!IPS_VariableExists($id)) { + throw new RuntimeException('Messvariable ' . $id . ' fehlt.'); + } + $before = IPS_GetVariable($id); + $value = GetValue($id); + $after = IPS_GetVariable($id); + if ($before['VariableUpdated'] !== $after['VariableUpdated']) { + throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); + } + $object = IPS_GetObject($id); + return ['value' => $value, 'updated' => (int) $after['VariableUpdated'], + 'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']]; + }; + return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, + $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } catch (Throwable $error) { + return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR); + } + } + + public function GetNetzfahrplanV4Sendestatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4Sendestatus'); + } + + private function sendeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + return; + } + try { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $snapshot = $this->GetNetzfahrplanV4Diagnose(); + $data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR); + if (!isset($data['operation'])) { + throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.'); + } + // Decoding as objects preserves empty JSON dictionaries in the API contract. + $payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR); + if (!isset($payload->operation)) { + throw new RuntimeException('Keine konsistenten Betriebsdaten.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation'; + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.'); + } + try { + curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, + CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR), + CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]); + $answer = curl_exec($handle); + $status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($answer === false || $status !== 200) { + throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.'); + } + if (strlen($answer) > 65536) { + throw new RuntimeException('V4-Antwort ist zu gross.'); + } + $ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR); + if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) { + throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.'); + } + } finally { + curl_close($handle); + } + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt, + 'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + // Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands. + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php new file mode 100644 index 0000000..35495da --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php @@ -0,0 +1,219 @@ + $max) { + throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.'); + } + return (float) $value; + } + + private static function timestamp($value): int + { + if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) { + throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.'); + } + return (new DateTimeImmutable($value))->getTimestamp(); + } + + /** $read returns ['value' => int|float, 'updated' => int], without changing a device. */ + private static function measurement(callable $read, int $id, int $now, int $age, string $label): array + { + if ($id <= 0) { + throw new InvalidArgumentException($label . ': Messquelle fehlt.'); + } + $m = $read($id); + if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now + || $now - $m['updated'] > $age) { + throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.'); + } + return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']]; + } + + /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ + private static function evidence(array $input, int $now, string $meterID, array &$warnings): array + { + $peaks = []; + $past = null; + if ($input === []) { + $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; + return [(object) [], null]; + } + if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) { + throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); + } + $measured = self::timestamp($input['measuredAt'] ?? null); + if ($measured > $now || $now - $measured > 120) { + throw new InvalidArgumentException('Messnachweis ist nicht aktuell.'); + } + $month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m'); + foreach (($input['measuredPeaks'] ?? []) as $key => $p) { + if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month + || !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) { + throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.'); + } + $peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']]; + } + if (!array_key_exists($month, $peaks)) { + $warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.'; + } + $q = intdiv($now, 900) * 900; + if (isset($input['quarterPast'])) { + $p = $input['quarterPast']; + // Stale quarter evidence remains absent, not extrapolated to the decision time. + if ($measured === $now && self::timestamp($p['start'] ?? null) === $q + && ($p['measuredSeconds'] ?? null) === $now - $q) { + $past = ['start' => gmdate('c', $q), + 'measuredSeconds' => $now - $q, + 'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)]; + } else { + $warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.'; + } + } + if ($now !== $q && $past === null) { + $warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.'; + } + return [(object) $peaks, $past]; + } + + /** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */ + public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array + { + $warnings = []; + $meter = NetzfahrplanV4Bezugszaehler::lesen( + $manager['V4BezugszaehlerQuellen'] ?? [], $read, $now + ); + $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, + (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); + $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); + $mode = $manager['Lastspitzenmodus'] ?? 0; + $import = null; + $months = []; + if ($mode === 1) { + $import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0); + } elseif ($mode === 2) { + foreach (($manager['Monatsgrenzen'] ?? []) as $row) { + $m = $row['MonatIndex'] ?? null; + if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) { + throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.'); + } + $months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0); + } + if (count($months) !== 12) { + throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.'); + } + } elseif ($mode !== 0) { + throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.'); + } + $export = !empty($manager['EinspeisebegrenzungAktiv']) + ? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null; + $result = []; + $seen = []; + $used = []; + foreach ($assets as $asset) { + $id = $asset['ID'] ?? ''; + if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) { + throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.'); + } + $seen[$id] = true; + $socID = (int) ($asset['SOCVariableID'] ?? 0); + $matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool { + return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID; + })); + if (count($matches) !== 1) { + throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.'); + } + $c = $matches[0]; + $controllerID = $c['InstanzID']; + if (isset($used[$controllerID])) { + throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.'); + } + $used[$controllerID] = true; + if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) { + throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.'); + } + $opts = $manager['BatterieOptionen'][$id] ?? []; + $nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001); + $usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal); + if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) { + throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.'); + } + $capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal); + $age = (int) ($c['MesswertMaxAlter'] ?? 30); + $soc = self::measurement($read, $socID, $now, $age, $id . ' SOC'); + $charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden'); + $discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen'); + $min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100), + self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); + $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); + $socValue = self::number($soc['value'], 'SOC', 0, 100); + $physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min); + if ($socValue < $physicalMin || $socValue > $max) { + throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); + } + if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { + throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.'); + } + $maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0)); + $maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0)); + if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) { + $maxCharge = $maxDischarge = 0.0; + $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; + } + $rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100)); + $blocked = !empty($c['HystereseAktiv']) || $socValue <= $min; + if ($blocked) { + $warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.'; + } + if ($socValue < $min) { + $warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.'; + } + $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, + 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, + 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true, + 'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true, + 'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm]; + } + if ($result === []) { + throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); + } + [$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings); + $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, + 'meteringBoundary' => 'common_pcc', 'batteries' => $result, + 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], + 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + // A fresh acquisition sample is an operational estimate. It does NOT + // claim a calibrated billing-period boundary or full historical coverage. + $policy = [ + 'adapterVersion' => 'v4-native-estimated-meter-1', + 'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified', + 'mode' => $mode, 'importW' => $import, 'exportW' => $export, + 'months' => $months, + 'batteryPolicy' => array_map(static function (array $b): array { + return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent'])); + }, $result), + ]; + $payload['meterObservation'] = [ + 'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now), + 'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'], + 'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)), + ]; + $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php new file mode 100644 index 0000000..dda9b72 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php @@ -0,0 +1,108 @@ + 8 + || array_keys($sources) !== range(0, count($sources) - 1)) { + throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.'); + } + $result = []; + $parent = null; + foreach ($sources as $source) { + if (!is_array($source) || count($source) !== 5 + || ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') { + throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.'); + } + $id = $source['VariableID'] ?? null; + $pid = $source['ElternID'] ?? null; + $ident = $source['Ident'] ?? null; + $factor = $source['FaktorZuKWh'] ?? null; + if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0 + || !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) + || $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) { + throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.'); + } + if ($parent !== null && $parent !== $pid) { + throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.'); + } + $parent = $pid; + foreach ($result as $existing) { + if ($existing['Ident'] === $ident) { + throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.'); + } + } + $result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident, + 'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug']; + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + public static function identitaet(array $sources): string + { + // Source/factor changes invalidate old measurement evidence; names do not matter. + return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources), + JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + } + + private static function probe(array $source, callable $read, int $now, int $maxAge): array + { + $m = $read($source['VariableID']); + if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID'] + || ($m['ident'] ?? null) !== $source['Ident']) { + throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.'); + } + $value = $m['value'] ?? null; + $updated = $m['updated'] ?? null; + if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0 + || !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) { + throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.'); + } + $kwh = (float) $value * $source['FaktorZuKWh']; + if (!is_finite($kwh) || $kwh > 1.0e12) { + throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.'); + } + return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value, + 'totalKwh' => $kwh, 'updatedAt' => $updated]; + } + + /** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */ + public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array + { + if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) { + throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.'); + } + $sources = self::quellen($sources); + $samples = []; + foreach ($sources as $source) { + $samples[] = self::probe($source, $read, $now, $maxAge); + } + // Reread all channels to reject concurrent updates, including changes in the same second. + foreach ($sources as $i => $source) { + if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) { + throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.'); + } + } + $times = array_column($samples, 'updatedAt'); + if (max($times) - min($times) > $maxSkew) { + throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.'); + } + return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh', + 'totalKwh' => array_sum(array_column($samples, 'totalKwh')), + 'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)), + 'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples, + 'billingEvidence' => false, 'historyComplete' => false]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php new file mode 100644 index 0000000..adb80c8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -0,0 +1,29 @@ +regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) { + self::assertStringNotContainsString($forbidden, $source); + } + self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source); + self::assertStringContainsString('/prognosis/planner-v4/operation', $source); + self::assertStringNotContainsString('/prognosis/schedule', $source); + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php new file mode 100644 index 0000000..41a2992 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php @@ -0,0 +1,29 @@ + 1, 'NetzbezugEnergieVariableID' => 9, + 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, + 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $m['V4BezugszaehlerQuellen'] = [ + ['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, + 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; + $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, + 'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5, + 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, + 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; + $read = static function (int $id) use ($now): array { + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id], + 'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other']; + }; + $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; + $fail = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid-input rejection'); + }; + $cases = []; + $build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array { + return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now); + }; + $cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void { + $x = $build()['operation']; $b = $x['batteries'][0]; + $check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0 + && $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0); + }; + $cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void { + $x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null); + }; + $cases['empty_maps_are_objects'] = static function () use ($build, $check): void { + $x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW)); + }; + $cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void { + $m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0); + }; + $cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void { + $m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === null && $x['limits']['exportW'] === null); + }; + $cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void { + $m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = []; + for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; } + $x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0); + array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m)); + }; + $cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void { + $a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a)); + $m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]]; + $check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44); + }; + $cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void { + $a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a)); + }; + $cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void { + $c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c)); + }; + $cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void { + $c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c)); + }; + $cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void { + $r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { + $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); + }; + $cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void { + $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; + $b = $build(null, null, null, $r)['operation']['batteries'][0]; + $check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true); + }; + $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { + $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; + $check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0); + }; + $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { + $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); + $check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1); + }; + $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { + $check($build()['operation']['batteries'][0]['gridCharging'] === false); + $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); + }; + $ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now), + 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], + 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; + $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { + $x = $build(null, null, null, null, $ev); + $check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5); + }; + $cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void { + $ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev); + $check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0); + }; + $cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void { + $ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { + $check($build()['operation']['eventId'] === $build()['operation']['eventId']); + }; + $cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void { + unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m)); + }; + $cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void { + $v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203 + && !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php new file mode 100644 index 0000000..7e08052 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php @@ -0,0 +1,113 @@ + 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $samples = [ + 59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'], + 26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'], + ]; + $check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } }; + $reject = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid source/sample rejection'); + }; + $snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array { + $mm = $mm ?? $samples; + return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now); + }; + $cases = []; + $cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void { + $v = $snapshot(); + $check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']); + $check(!isset($v['measuredPeaks']) && !isset($v['quarterPast'])); + }; + $cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void { + $samples[26620]['value'] = 27.25; + $check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9); + }; + $cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void { + $samples[59607]['value'] = 0; + $check($snapshot(null, $samples)['totalKwh'] === 0.0); + }; + $cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void { + $check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources))); + }; + $cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001; + $check($old !== Meter::identitaet($sources)); + }; + $cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345; + $check($old !== Meter::identitaet($sources)); + }; + $cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); }; + $cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources)); + }; + $cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources)); + }; + $cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources)); + }; + $cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void { + unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources)); + }; + $cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void { + unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void { + foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) { + $samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples)); + } + }; + $cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void { + $calls = 0; + $read = static function (int $id) use ($samples, &$calls): array { + $v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v; + }; + $reject(static fn() => Meter::lesen($sources, $read, $now)); + }; + $cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void { + $samples[59607]['updated'] -= 1; + $v = $snapshot(null, $samples); + $check($v['sourceObservationFrom'] === gmdate('c', $now - 1)); + $check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now)); + $check(!$v['billingEvidence']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/read_native_forecasts.py b/services/netplan-v4/acceptance/read_native_forecasts.py new file mode 100644 index 0000000..abae9e8 --- /dev/null +++ b/services/netplan-v4/acceptance/read_native_forecasts.py @@ -0,0 +1,100 @@ +"""Read-only probe executed INSIDE the existing forecast-engine container. + +No get_configs() (it may migrate SQL), no training/prediction, no forecast writing, +no manual /run_now request, and no user credentials in output. Reads one plant's +stored raw 5-minute forecast values and the input frames used by the engine. +""" +import contextlib +from datetime import datetime, timezone +import hashlib +import io +import json +import math +import os +from pathlib import Path +import sqlite3 +from urllib.parse import quote +from uuid import UUID + +class Discard(io.TextIOBase): + def write(self,text):return len(text) + +def clean_time(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def run(): + aid=str(UUID(os.environ['ENELIX_ACCEPTANCE_PLANT'])) + os.environ['FORECAST_INFLUX_TIMEOUT_MS']='20000' + # These assignments affect only this diagnostic process, not the service. + with contextlib.redirect_stdout(Discard()),contextlib.redirect_stderr(Discard()): + import main + import numpy as np + import pandas as pd + path=Path(main.SQLITE_DB_PATH) + if not path.is_file():raise RuntimeError('Existing configuration database missing') + con=sqlite3.connect('file:'+quote(str(path))+'?mode=ro',uri=True) + con.row_factory=sqlite3.Row + try:row=con.execute('SELECT * FROM anlagen_meta WHERE anlagen_id=?',(aid,)).fetchone() + finally:con.close() + if row is None:raise RuntimeError('Installation not found') + cfg=dict(row) + cfg['daecher']=json.loads(cfg.get('daecher') or '[]') + for key,default in [('ac_leistung',10.),('batt_capacity_kwh',0.),('batt_power_kw',0.),('tarif_bezug_fest',.3),('tarif_einspeisung_fest',.1),('tarif_peak_fest',5.)]: + value=cfg.get(key);cfg[key]=float(default if value is None or value=='' else value) + data=main.build_data_object(cfg,training=False) + frames={} + for key in ('df_hist','df_recent_raw','df_load_training','df_fut'): + frame=data.get(key) + if frame is None:frames[key]={'available':False};continue + item={'rows':len(frame),'from':clean_time(frame.index.min()) if len(frame) else None,'until':clean_time(frame.index.max()) if len(frame) else None,'columns':{}} + for col in ('Hausverbrauch','PV','Netzleistung','SOC'): + if col not in frame.columns:continue + values=pd.to_numeric(frame[col],errors='coerce');valid=values[np.isfinite(values)] + item['columns'][col]={'finite':len(valid),'zeros':int((valid==0).sum()),'median':float(valid.median()) if len(valid) else None,'maximum':float(valid.max()) if len(valid) else None,'lastFiniteAt':clean_time(valid.index[-1]) if len(valid) else None,'recentValues':[{'time':clean_time(t),'value':float(v)} for t,v in valid.tail(12).items()]} + frames[key]=item + start=datetime.now(timezone.utc).replace(second=0,microsecond=0) + from datetime import timedelta + end=start+timedelta(hours=48) + fields=('prog_var_1','prog_var_2','prog_var_10','prog_var_11','prog_var_21','prog_var_22') + field_filter=' or '.join('r["_field"] == '+json.dumps(f) for f in fields) + query='''from(bucket: %s) + |> range(start: %s, stop: %s) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == %s) + |> filter(fn: (r) => r["data_type"] == "forecast") + |> filter(fn: (r) => %s) + |> keep(columns: ["_time", "_field", "_value"]) + ''' % (json.dumps(main.INFLUX_BUCKET),start.isoformat(),end.isoformat(),json.dumps(aid),field_filter) + client=main.InfluxDBClient(url=main.INFLUX_URL,token=main.INFLUX_TOKEN,org=main.INFLUX_ORG,timeout=20000) + series={field:[] for field in fields} + try: + tables=client.query_api().query(org=main.INFLUX_ORG,query=query) + for table in tables: + for record in table.records: + value=record.get_value();field=record.get_field() + if field in series and isinstance(value,(int,float)) and math.isfinite(value): + series[field].append({'time':clean_time(record.get_time()),'value':float(value)}) + finally:client.close() + for field in series:series[field].sort(key=lambda p:p['time']) + versions={} + for relative in ('main.py','methods/var_1.py','methods/var_2.py','methods/var_10.py','methods/var_11.py','methods/var_21.py','methods/var_22.py'): + source=Path('/app')/relative + if source.is_file():versions[relative]=hashlib.sha256(source.read_bytes()).hexdigest() + summary={} + for field,points in series.items(): + values=[p['value'] for p in points] + summary[field]={'points':len(values),'allZero':bool(values) and max(abs(v) for v in values)==0,'maximumW':max(values) if values else None,'from':points[0]['time'] if points else None,'until':points[-1]['time'] if points else None} + native=True + for points in series.values(): + stamps=[datetime.fromisoformat(p['time']).timestamp() for p in points] + if not stamps or any(t%300 for t in stamps) or any(b-a!=300 for a,b in zip(stamps,stamps[1:])):native=False + return {'status':'read_only_acquired','installationId':aid,'observedAt':datetime.now(timezone.utc).isoformat(),'queryResolution':'raw_no_chart_resampling','nativeFiveMinuteForecast':native,'generationTimeVerified':False,'inputFrames':frames,'forecastSummary':summary,'forecastSeries':series,'runningSourceHashes':versions,'modelWrite':False,'liveControlChanged':False} + +try: + result=run() +except Exception as error: + # Never echo exception details containing SQL payloads, credentials or URLs. + result={'status':'read_only_probe_failed','errorType':type(error).__name__,'liveControlChanged':False} +print(json.dumps(result,allow_nan=False)) diff --git a/services/netplan-v4/approved_previous_assets.json b/services/netplan-v4/approved_previous_assets.json new file mode 100644 index 0000000..3c59814 --- /dev/null +++ b/services/netplan-v4/approved_previous_assets.json @@ -0,0 +1,5 @@ +{ + "license/public/netplan-v4.js": [ + "e859e91a52dd1d6c96f411d16caed17375f6925bf867d09992ab32ce93bff5d7" + ] +} diff --git a/services/netplan-v4/commissioning/APPLICATION_STATUS.md b/services/netplan-v4/commissioning/APPLICATION_STATUS.md new file mode 100644 index 0000000..b716bd1 --- /dev/null +++ b/services/netplan-v4/commissioning/APPLICATION_STATUS.md @@ -0,0 +1,80 @@ +# Integrated application data/model path - 2026-10-02 + +User request: finish application implementation, stop asking for repeated hardware +confirmations and adding standalone diagnostic samplers. No actuator permission is +implied by that development request. Confirmed EV allocation remains 161.44 kWh / +39 kW each direction, with SDL reserve already excluded. + +## Implemented in the existing application + +- Native ManagerNetzfahrplanV4DatenTrait: 30-second acquisition of configured numeric + sources; durable private app-owned outbox; original timestamps, no new Modbus + polls; acknowledged per-plant batches with retries/backoff. Cursor advances only + after exact dataset/acceptedThrough acknowledgement. Batches bounded below the + existing portal's 1 MiB body limit. No credentials in data payload or reports. +- Existing device proxy now forwards /planner-v4/measurements through the existing + authentication, plant binding, licence check and separate V4 rate limiter. + Device route cannot modify the dataset mapping or issue a trial grant. +- Existing V4 database stores immutable versioned source mappings and observations. + Same-time conflicts reject the whole batch; repeated identical data is idempotent. + No writes to portal/users/legacy measurement databases. +- Existing V4 worker aggregates physical load using original source times. Virtual + EV/SDL filter outputs are not load inputs. Solar terminal variant counts signed + terminal power once. Small unobserved parts remain quantified, never zero-filled. + This configured formula remains an estimate, not independent AC/DC certification. +- Application profile training actually runs daily/weekly. Models are immutable, + bootstrap/holdout status explicit. Incumbent replacement requires a causal + holdout when available. Existing data and model versions are not relabelled. +- Three corrected load variants (daily robust profile, last-day profile, + weekday/weekend profile) are paired with the existing PV families 3/13/23. + They are tagged physical-profile-v1; they are NOT claimed to be the old trained + load algorithms under a new data name. Automatic future family registry work + and economic replay are still distinct from this load-profile implementation. +- Selecting corrected_profile in the existing GUI/setting now actually changes + the load entering the existing optimizer. Missing trained data cannot silently + fall back to the contaminated legacy household series. +- A current SDL request is an explicitly labelled PERSISTENCE SCENARIO for the + shadow economic calculation, not a published future schedule. Missing/stale SDL + is not zero. This is not a robust/full SDL production-dispatch policy. +- GUI now exposes dataset/source selection, records, usable equivalent hours, + model version, validation and real training cadence. Existing normal portal + app.js/index files are not changed. + +## Deployment package + +Server: commissioning/deploy_application.py --plant --apply +Rebuild/test V4 with Python 3.11, portal routing tests, consistent SQLite backup, +replace ONLY V4 and portal, configure the new per-plant dataset. No automatic +forecast-source switch and no control grant. Atomic file-bind replacement requires +portal recreation, not a simple restart. Additive DB rollback retains new data. +Default invocation without --apply validates source manifests only. + +Test host: /srv/agent/netplan-v4-application-build/install.php inside Symcon. +Installs a DATA-ONLY patch of the currently installed passive manager (not the +unreleased controller-trial candidate), reloads EMS, configures acquisition and +imports up to 48h of the existing 23-source observer as a delivery backlog. +Original observer and old raw journals remain intact. Existing separate observers +are NOT automatically stopped by this initial installer; consolidation should +follow confirmed native batch acceptance. No new root diagnostic category. +An asynchronous module registration may require one repeat, handled explicitly. + +Dataset lihrenmoos-physical-v1: minimum coverage 95%, longest unsupported portion +10s, at least 24 equivalent usable hours, 28-day profile history. The 95% rule is +an explicit modelling policy, not a statement that missing energy was measured. +Insufficient data remains collecting; no promised time to good forecasts. +All original acquisition thresholds remain unchanged. + +## Tests and honest limits + +Host tests use existing isolated QA dependencies; native tests mock IPS/HTTP and +use temporary files. Real container and kernel tests occur on the user-run deploy +commands, not during preparation. No services/modules/settings/devices were modified +by this development session. + +Still NOT a complete production commissioning: the corrected profile-to-local +feedback/dispatch integration, full automatic cost replay, independent long-term +outbox/server retention and staged live/failure acceptance remain open. The trial +code from previous commits remains gated and is not included in the data-only +native installation. No accountingEvidenceId/device watchdog proof is fabricated. +This release is an integrated application build, NOT a claim that the full original +multi-plant production scope is done. No main/beta release is authorized by tests alone. diff --git a/services/netplan-v4/commissioning/application-source/RELEASE.json b/services/netplan-v4/commissioning/application-source/RELEASE.json new file mode 100644 index 0000000..596cf5c --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/RELEASE.json @@ -0,0 +1,99 @@ +{ + "scope": "integrated_measurement_application", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "portalBefore": { + "netplan-v4-bridge.mjs": "f58c652b04cd2ac20f2bec11e1161c44b72c26267b4dfb42096a1fad36a3c9c2", + "public/netplan-v4.js": "6cd1d887bb08c73a815c92eb684a0992e8cb092f748770cc94debcfd9705a628" + }, + "createdAt": "2026-10-02T20:57:00.387145+00:00", + "controlEnabled": false +} diff --git a/services/netplan-v4/commissioning/application-source/server-dataset.json b/services/netplan-v4/commissioning/application-source/server-dataset.json new file mode 100644 index 0000000..7fd1190 --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/server-dataset.json @@ -0,0 +1 @@ +{"datasetId":"lihrenmoos-physical-v1","mappingSha256":"517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b","inventorySha256":"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc","sources":[{"key":"grid","variableId":40348,"factorToW":1000,"maxAgeSeconds":60,"role":"grid"},{"key":"pv_goodwe1","variableId":48459,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_goodwe2","variableId":53802,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_solaredge","variableId":20335,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"physical_goodwe1","variableId":47725,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_goodwe2","variableId":35724,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_solaredge","variableId":21447,"factorToW":1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"ev_account","variableId":52020,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"sdl_account","variableId":25085,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"grid_display","variableId":49301,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe1","variableId":27361,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe2","variableId":23109,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_solaredge","variableId":51938,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_ev","variableId":32871,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_sdl","variableId":23879,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_charge","variableId":50230,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_discharge","variableId":43899,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t1","variableId":59607,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t2","variableId":26620,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_requested","variableId":19651,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"sdl_requested","variableId":38943,"role":"sdl_request","factorToW":1,"maxAgeSeconds":120},{"key":"solar_ac_value","variableId":37975,"role":"solar_raw","factorToW":1,"maxAgeSeconds":60},{"key":"solar_ac_scale","variableId":41853,"role":"solar_scale","factorToW":1,"maxAgeSeconds":60}],"formula":"solar_terminal_v1","solarReference":{"pvKey":"pv_solaredge","batteryKey":"physical_solaredge","rawKey":"solar_ac_value","scaleKey":"solar_ac_scale"},"minimumCoverage":0.95,"maximumGapSeconds":10,"minimumTrainingHours":24,"historyDays":28} diff --git a/services/netplan-v4/commissioning/deploy_application.py b/services/netplan-v4/commissioning/deploy_application.py new file mode 100644 index 0000000..4e27762 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_application.py @@ -0,0 +1,131 @@ +"""Deploy the integrated measurement/training application, NOT actuator permission. + +Only V4 and the portal are recreated. Existing legacy forecasts, tariffs, Symcon and +battery dispatch are unchanged. Source-only checks are the default; --apply is explicit. +""" +from pathlib import Path +from datetime import datetime, timezone +from uuid import UUID +import argparse,hashlib,json,os,sqlite3,subprocess,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PORTAL=ROOT.parent/'license' +MANIFEST=Path(__file__).with_name('application-source')/'RELEASE.json' +DATASET=Path(__file__).with_name('application-source')/'server-dataset.json' +PORTAL_FILES={'integrations/netplan-v4-bridge.mjs':'netplan-v4-bridge.mjs','gui/netplan-v4.js':'public/netplan-v4.js'} + +def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest() + +def verify(): + m=json.loads(MANIFEST.read_text()) + if m.get('scope')!='integrated_measurement_application' or not m.get('sourceHashes'): + raise ValueError('Unexpected application release manifest') + for n,h in m['sourceHashes'].items(): + p=ROOT/n + if Path(n).is_absolute() or '..' in Path(n).parts or p.is_symlink() or not p.is_file() or digest(p)!=h: + raise ValueError('Source drift: '+n) + for n,old in m['portalBefore'].items(): + if n not in PORTAL_FILES.values():raise ValueError('Unexpected portal target') + p=PORTAL/n + if p.is_symlink() or not p.is_file():raise ValueError('Portal target changed') + source=next(s for s,t in PORTAL_FILES.items() if t==n) + if digest(p) not in (old,m['sourceHashes'][source]):raise ValueError('Concurrent portal change; not overwritten') + return m + +def atomic(path,data,mode=0o644): + if path.is_symlink():raise ValueError('Symlink refused') + fd,name=tempfile.mkstemp(prefix='.v4-app-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.chmod(name,mode);os.replace(name,path) + finally: + if os.path.exists(name):os.unlink(name) + +def backup_database(source,destination): + if not source.is_file() or source.is_symlink() or destination.exists():raise ValueError('Explicit existing database and new backup path required') + with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as a,sqlite3.connect(destination) as b: + a.backup(b) + if b.execute('PRAGMA integrity_check').fetchone()[0]!='ok':raise ValueError('Backup failed') + destination.chmod(0o600) + +BOOTSTRAP='''import json,os,urllib.request,sys +p=json.load(sys.stdin) +base='http://127.0.0.1:9100/internal/v2/prognosis/'+p['plant']+'/planner' +h={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +req=urllib.request.Request(base+'/datasets/'+p['dataset']['datasetId'],data=json.dumps(p['dataset']).encode(),headers=h,method='PUT') +receipt=json.load(urllib.request.urlopen(req,timeout=10)) +state=json.load(urllib.request.urlopen(urllib.request.Request(base,headers=h),timeout=10)) +assert state['liveEnabled'] is False +print(json.dumps({'dataset':receipt,'existingForecastSource':state['settings']['forecastSource'],'liveEnabled':False})) +''' + +def run(plant,apply=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Use the prepared installation-specific mapping') + if not apply: + print('APPLICATION SOURCE CHECK PASSED:',len(m['sourceHashes']),'files. No deployment.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker permissions') + folder=ROOT/'application-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ') + folder.mkdir(parents=True,mode=0o700) + env={**os.environ,'NETPLAN_V4_PLANTS':plant} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')] + portal=['docker','compose','-f',str(PORTAL/'compose.yaml'),'-f',str(ROOT/'compose.portal-bridge.yaml')] + result={'scope':'integrated_measurement_application','startedAt':datetime.now(timezone.utc).isoformat(), + 'liveEnabled':False,'productionCommissioned':False,'steps':[],'sourceHashes':m['sourceHashes']} + changed={};old_image=None;v4_replaced=False;portal_replaced=False + def cmd(args,timeout=180,capture=False,input=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=input, + stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + try: + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip() + result['previousV4Image']=old_image + cmd(compose+['build','netplan-v4'],timeout=900) + cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240) + cmd(['node','--test',str(ROOT/'tests/portal.test.mjs')]) + cmd(['node','--check',str(ROOT/'gui/netplan-v4.js')]) + result['steps'].append('target_python_and_portal_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite') + result['steps'].append('consistent_database_backup') + for source,target in PORTAL_FILES.items(): + p=PORTAL/target;old=p.read_bytes();new=(ROOT/source).read_bytes() + if old==new:continue + if hashlib.sha256(old).hexdigest()!=m['portalBefore'][target]:raise ValueError('Concurrent portal change') + dest=folder/'portal-before'/target;dest.parent.mkdir(parents=True,exist_ok=True);dest.write_bytes(old) + atomic(p,new);changed[target]=(old,new) + v4_replaced=True;cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4']) + # Recreate rather than restart: atomic replacement of a file bind mount needs a new mount. + portal_replaced=True;cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + payload=json.dumps({'plant':plant,'dataset':json.loads(DATASET.read_text())}) + receipt=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,input=payload) + result['application']=json.loads(receipt.stdout) + result['status']='application_deployed_no_actuator_permission' + result['steps'].append('configured_dataset_and_health_verified') + except Exception as e: + result['status']='needs_review';result['errorType']=type(e).__name__ + restored=[] + for target,(old,new) in changed.items(): + p=PORTAL/target + if p.read_bytes()==new:atomic(p,old);restored.append(target) + result['restoredPortalFiles']=restored + try: + if v4_replaced and old_image: + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + cmd(compose+['-f',str(rollback),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + if portal_replaced:cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + result['rollback']='previous_runtime_restored_additive_data_retained' + except Exception:result['rollback']='manual_review_required' + raise + finally: + result['finishedAt']=datetime.now(timezone.utc).isoformat() + report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n') + uid=ROOT.stat().st_uid;gid=ROOT.stat().st_gid + os.chown(folder,uid,gid);os.chown(folder.parent,uid,gid);os.chown(report,uid,gid);report.chmod(0o640) + print('APPLICATION RELEASE REPORT:',report) + print('V4 data/model application and portal updated. Install manager data integration separately. No V4 actuation enabled.') + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True,type=lambda v:str(UUID(v)));p.add_argument('--apply',action='store_true');a=p.parse_args() + try:run(a.plant,a.apply) + except Exception as e:raise SystemExit('Stopped: '+type(e).__name__+'. See the application release report.') diff --git a/services/netplan-v4/compose.portal-bridge.yaml b/services/netplan-v4/compose.portal-bridge.yaml new file mode 100644 index 0000000..0bf8355 --- /dev/null +++ b/services/netplan-v4/compose.portal-bridge.yaml @@ -0,0 +1,6 @@ +services: + license-portal: + environment: + NETPLAN_V4_URL: http://netplan-v4:9100 + volumes: + - /home/agent/services/license/netplan-v4-bridge.mjs:/app/netplan-v4-bridge.mjs:ro diff --git a/services/netplan-v4/compose.yaml b/services/netplan-v4/compose.yaml new file mode 100644 index 0000000..84c7075 --- /dev/null +++ b/services/netplan-v4/compose.yaml @@ -0,0 +1,37 @@ +name: enelix-netplan-v4-shadow +services: + netplan-v4: + build: . + restart: unless-stopped + init: true + user: "1000:1000" + read_only: true + env_file: + - /home/agent/services/prognosis-integration.env + environment: + NETPLAN_V4_DB: /data/netplan-v4.sqlite + NETPLAN_V4_PLANTS: ${NETPLAN_V4_PLANTS:?Explicit installation ID required} + NETPLAN_V4_TEST_REPORT_DIR: /tmp/test-results + volumes: + - ./data:/data + tmpfs: + - /tmp:size=64m,mode=1777,noexec,nosuid + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + networks: + prognosis_integration: + aliases: + - netplan-v4 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9100/health', timeout=3).read()"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + # No public port, no Docker socket, no access to legacy databases. +networks: + prognosis_integration: + external: true + name: enelix-platform-internal diff --git a/services/netplan-v4/deploy_integrated_shadow.py b/services/netplan-v4/deploy_integrated_shadow.py new file mode 100644 index 0000000..6e4d614 --- /dev/null +++ b/services/netplan-v4/deploy_integrated_shadow.py @@ -0,0 +1,175 @@ +"""Install the reviewed server-side SHADOW integration; no Symcon/device writes. + +Default: verify source and print scope only. --apply requires root. Build and test +all changed code before recreating services. Save old image/config references and +an integrity-checked online backup of the V4 DB. Does not enable native telemetry +or dispatch: operation ingress may still be awaiting installation on Symcon. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from uuid import UUID + +ROOT = Path(__file__).resolve().parent +SERVICES = ROOT.parent +PROJECT = SERVICES / 'prognosis-manager-enelix2' +PORTAL = SERVICES / 'license' +PREFLIGHT = ROOT / 'acceptance-reports/20261002T043021Z/REPORT.json' +MANIFEST = ROOT / 'INTEGRATED_SHADOW_SOURCE.json' + + +def verify_sources(): + report = json.loads(PREFLIGHT.read_text()) + if report.get('preflightChecksPassed') is not True: + raise ValueError('Expected successful earlier runtime preflight is missing') + expected = json.loads(MANIFEST.read_text())['files'] + for relative, digest in expected.items(): + path = SERVICES / relative + if not path.resolve().is_relative_to(SERVICES) or path.is_symlink() or not path.is_file(): + raise ValueError('Invalid release source path: ' + relative) + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError('Source changed since preparation: ' + relative) + from forecast_acceptance import verify_forecast_bundle + verify_forecast_bundle(ROOT/'acceptance/forecast-src', PROJECT/'forecast_engine') + from release_preflight import verify_native_bundle + verify_native_bundle() + return expected + + +def compose_groups(): + return { + 'v4': (ROOT, [ROOT/'compose.yaml'], ['netplan-v4']), + 'forecast': (PROJECT, [PROJECT/'compose.yaml', ROOT/'compose.forecast-bridge.yaml'], ['api','forecast-engine','tariff-importer']), + 'portal': (PORTAL, [PORTAL/'compose.yaml', ROOT/'compose.portal-bridge.yaml'], ['license-portal']), + } + + +def compose_args(files): + result=['docker','compose'] + for path in files:result.extend(['-f',str(path)]) + return result + + +def backup_database(source, destination): + if not source.exists():return {'exists':False} + src=sqlite3.connect(source.resolve().as_uri()+'?mode=ro',uri=True,timeout=15) + dst=sqlite3.connect(destination,timeout=15) + try: + src.backup(dst) + if dst.execute('PRAGMA integrity_check').fetchone()[0]!='ok': + raise ValueError('V4 database backup failed integrity check') + finally:dst.close();src.close() + os.chmod(destination,0o640) + if os.geteuid()==0:os.chown(destination,1000,1000) + return {'exists':True,'path':str(destination),'sha256':hashlib.sha256(destination.read_bytes()).hexdigest(),'integrity':'ok'} + + +def save_json(path, value): + path.write_text(json.dumps(value,indent=2,allow_nan=False)+'\n') + os.chmod(path,0o640) + if os.geteuid()==0:os.chown(path,1000,1000) + + +def run(plant): + sources=verify_sources() + env=dict(os.environ,NETPLAN_V4_PLANTS=plant) + stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + folder=ROOT/'deployment-reports'/stamp + folder.mkdir(parents=True,exist_ok=False) + if os.geteuid()==0: + os.chown(folder.parent,1000,1000);os.chown(folder,1000,1000) + report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant, + 'scope':'server_shadow_only','liveEnabled':False,'productionReady':False, + 'sourceHashes':sources,'steps':[],'before':{},'existingServiceRecreated':False} + def command(args,cwd=ROOT,capture=False,stdin=None,timeout=900): + result=subprocess.run(args,cwd=cwd,env=env,input=stdin,text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None,timeout=timeout,check=False) + if result.returncode: + raise RuntimeError('Step failed (exit '+str(result.returncode)+'): '+args[0]) + return result.stdout if capture else '' + def step(name,args,cwd=ROOT,timeout=900): + print('\n=== '+name+' ===',flush=True) + command(args,cwd,timeout=timeout) + report['steps'].append(name);save_json(folder/'DEPLOYMENT.json',report) + try: + # Capture only selected nonsecret Docker fields, never the environment. + for group,(cwd,files,services) in compose_groups().items(): + oldfiles=[files[0]] + items=[] + for service in services: + cid=command(compose_args(oldfiles)+['ps','-q',service],cwd,True).strip() + if not cid or '\n' in cid:raise ValueError('Expected exactly one existing container: '+service) + image=command(['docker','inspect','--format','{{.Image}}',cid],cwd,True).strip() + configs=command(['docker','inspect','--format','{{index .Config.Labels "com.docker.compose.project.config_files"}}',cid],cwd,True).strip() + items.append({'service':service,'containerId':cid,'imageId':image,'previousComposeFiles':configs}) + report['before'][group]=items + save_json(folder/'DEPLOYMENT.json',report) + step('review optional source installer',[sys.executable,str(ROOT/'install_hooks.py')]) + # All new forecasting tests run offline in a disposable Python 3.11 image. + step('build forecast acceptance image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')]) + step('forecast candidate offline tests',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local']) + step('V4 and bridge target tests',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only']) + verify_sources() + from install_hooks import plan,apply + receipt=apply(plan()) + report['sourceInstallReceipt']=str(receipt) if receipt else None + step('portal syntax',['node','--check',str(PORTAL/'server.mjs')]) + # Build every production-shaped candidate before changing a running container. + cwd,files,svcs=compose_groups()['forecast'] + step('build server-side images',compose_args(files)+['build']+svcs,cwd) + report['v4DatabaseBackup']=backup_database(ROOT/'data/netplan-v4.sqlite',folder/'netplan-v4-before.sqlite') + report['existingServiceRecreated']=True + save_json(folder/'DEPLOYMENT.json',report) + for group,(cwd,files,svcs) in compose_groups().items(): + step('start shadow integration '+group,compose_args(files)+['up','-d','--no-deps','--no-build','--wait','--wait-timeout','180']+svcs,cwd,timeout=300) + cwd,files,_=compose_groups()['v4'] + # Check shadow guard and report incoming types without disclosing payloads. + probe='''import json,os,sqlite3,urllib.request +from datetime import datetime,timezone +plant=os.environ['NETPLAN_V4_PLANTS'] +health=json.loads(urllib.request.urlopen('http://127.0.0.1:9100/health',timeout=5).read()) +assert health.get('mode')=='shadow' and health.get('liveEnabled') is False +con=sqlite3.connect('file:/data/netplan-v4.sqlite?mode=ro',uri=True) +rows=con.execute('SELECT kind,count(*) FROM planner_inputs WHERE plant=? GROUP BY kind',(plant,)).fetchall() +status=con.execute('SELECT status FROM planner_run_status WHERE plant=?',(plant,)).fetchone() +con.close() +print(json.dumps({'checkedAt':datetime.now(timezone.utc).isoformat(),'health':health,'inputEventsByKind':dict(rows),'lastRunStatus':status[0] if status else None,'nativeSenderInstalledByThisCommand':False})) +''' + report['readiness']=json.loads(command(compose_args(files)+['exec','-T','netplan-v4','python','-B','-'],cwd,True,probe,60)) + report['status']='server_shadow_installed' + report['remaining']='Native operation sender, plant-level initialization and real shadow-plan acceptance still required; no live dispatch installed' + except Exception as error: + report['status']='failed';report['errorType']=type(error).__name__;report['error']=str(error)[:500] + report['remaining']='Inspect completed steps and before-image references. No automatic database restore or device changes.' + raise + finally: + save_json(folder/'DEPLOYMENT.json',report) + print('\nDEPLOYMENT REPORT: '+str(folder/'DEPLOYMENT.json'),flush=True) + print('SERVER SHADOW ONLY. Symcon, device permissions and actuators were not modified.',flush=True) + print(json.dumps(report['readiness'],indent=2)) + print('Native operating-state input may still be missing; this is NOT production commissioning.') + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda p:str(UUID(p))) + parser.add_argument('--apply',action='store_true') + args=parser.parse_args() + if args.plant!='e3a08f9e-af12-4695-99bd-8b51c0520021': + raise SystemExit('This first integration rollout is allowlisted for Lihrenmoos only.') + if not args.apply: + files=verify_sources() + print('CHECK ONLY:',len(files),'reviewed source files. No running service changed.') + for group,(_,_,svcs) in compose_groups().items():print(group,', '.join(svcs)) + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not change Docker socket access.') + try:run(args.plant) + except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + raise SystemExit('Deployment stopped: '+str(error)) diff --git a/services/netplan-v4/forecast_acceptance.py b/services/netplan-v4/forecast_acceptance.py new file mode 100644 index 0000000..686765f --- /dev/null +++ b/services/netplan-v4/forecast_acceptance.py @@ -0,0 +1,38 @@ +"""Verify the separately staged forecast repair before target-runtime testing. +Only explicit Python source and requirements are copied. No models, credentials, +live databases or telemetry are needed by this offline test image. +""" +from pathlib import Path +import hashlib +import json + + +def source_paths(root): + root = Path(root) + paths = [root / 'requirements.txt'] + list(root.glob('*.py')) + for folder in ('methods', 'tests'): + paths.extend((root / folder).glob('*.py')) + if not (root / 'telemetry_quality.py').is_file(): + raise ValueError('Forecast telemetry repair missing') + for path in paths: + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError('External forecast source is not allowed') + return sorted(paths) + + +def verify_forecast_bundle(bundle, source): + bundle, source = Path(bundle), Path(source) + manifest_path = bundle / 'SOURCE_MANIFEST.json' + manifest = json.loads(manifest_path.read_text()) + expected_paths = {str(p.relative_to(source)) for p in source_paths(source)} + if not isinstance(manifest, dict) or not manifest or set(manifest) != expected_paths: + raise ValueError('Forecast source set changed; rebuild reviewed test bundle') + if {str(p.relative_to(bundle)) for p in source_paths(bundle)} != expected_paths: + raise ValueError('Staged forecast source set does not match development source') + for relative, expected in manifest.items(): + if Path(relative).is_absolute() or '..' in Path(relative).parts: + raise ValueError('Unsafe forecast manifest path') + for path in (source / relative, bundle / relative): + if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise ValueError('Forecast source changed after staging; review before test') + return dict(manifest) diff --git a/services/netplan-v4/gui/netplan-v4.css b/services/netplan-v4/gui/netplan-v4.css new file mode 100644 index 0000000..2460732 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.css @@ -0,0 +1 @@ +body{font:16px system-ui,sans-serif;background:#eef4f4;color:#243c42}main{max-width:1100px;margin:2rem auto;padding:1.5rem;background:white}a{color:#006b83}label{display:block;margin:.8rem 0}select,button{font:inherit;padding:.6rem;margin:.3rem;border:1px solid #bacaca;border-radius:.25rem}button{background:#006b83;color:white;cursor:pointer}.notice{background:#e9f7fa;padding:1rem;border-left:4px solid #006b83}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#eef4f4;padding:1rem}svg{width:100%;height:auto}.curve{fill:none;stroke-width:2}.baseline{stroke:#7d888c}.optimized{stroke:#006b83}.axis{stroke:#b7c8cc}button:disabled{opacity:.5} diff --git a/services/netplan-v4/gui/netplan-v4.html b/services/netplan-v4/gui/netplan-v4.html new file mode 100644 index 0000000..1a96991 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.html @@ -0,0 +1 @@ +ENELIX Netzfahrplan V4
Zurueck zum Portal

Netzfahrplan V4

Schattenbetrieb. Diese Ansicht steuert keine Verbraucher. Die bestehende Regelung bleibt unveraendert.

diff --git a/services/netplan-v4/gui/netplan-v4.js b/services/netplan-v4/gui/netplan-v4.js new file mode 100644 index 0000000..a67ac55 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.js @@ -0,0 +1,107 @@ +const message = document.getElementById('message'); +const panel = document.getElementById('panel'); +const plants = document.getElementById('plants'); +let session, generation = 0; +const element = (tag, text) => { const e = document.createElement(tag); if (text !== undefined) e.textContent = text; return e; }; +const time = value => value ? new Date(value).toLocaleString('de-CH', { timeZone: 'Europe/Zurich' }) : 'nicht vorhanden'; +const num = (v, digits = 2) => typeof v === 'number' && Number.isFinite(v) ? v.toLocaleString('de-CH', { maximumFractionDigits: digits }) : 'unbekannt'; + +async function request(url, method = 'GET', body) { + const headers = {}; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + if (method !== 'GET') headers['X-CSRF-Token'] = session.csrfToken; + const response = await fetch(url, { method, headers, credentials: 'same-origin', body: body === undefined ? undefined : JSON.stringify(body) }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || data.detail || 'Anfrage fehlgeschlagen'); + return data; +} +function select(label, choices, selected) { + const wrap = element('label', label + ' '), field = element('select'); + for (const [value, caption] of choices) { const option = element('option', caption); option.value = value; field.append(option); } + field.value = selected; wrap.append(field); return { wrap, field }; +} +function table(title, rows) { + panel.append(element('h3', title)); const grid = element('table'); + for (const [name, value] of rows) { const row = element('tr'); row.append(element('th', name), element('td', String(value))); grid.append(row); } + panel.append(grid); +} +function curve(title, points, series) { + panel.append(element('h3', title)); + if (!points.length) { panel.append(element('p', 'Keine Daten.')); return; } + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + svg.setAttribute('viewBox', '0 0 1000 210'); svg.setAttribute('role', 'img'); svg.setAttribute('aria-label', title); + const values = points.flatMap(p => series.map(s => s.value(p))).filter(v => typeof v === 'number' && Number.isFinite(v)); + if (!values.length) { panel.append(element('p', 'Keine gueltigen Werte.')); return; } + const low = Math.min(0, ...values), high = Math.max(1, ...values); + const start = Date.parse(points[0].time), end = Date.parse(points[points.length - 1].validUntil || points[points.length - 1].time); + for (const [i, source] of series.entries()) { + let section = []; + const flush = () => { if (!section.length) return; const line = document.createElementNS(svg.namespaceURI, 'polyline'); line.setAttribute('class', 'curve ' + (i ? 'optimized' : 'baseline')); line.setAttribute('points', section.join(' ')); svg.append(line); section = []; }; + for (const p of points) { + const value = source.value(p); + if (typeof value !== 'number' || !Number.isFinite(value)) { flush(); continue; } + const x = 10 + 980 * (Date.parse(p.time) - start) / Math.max(1, end - start); + const y = 190 - (value - low) / (high - low) * 180; section.push(`${x},${y}`); + } + flush(); + } + panel.append(svg, element('p', `${num(low)} bis ${num(high)}. ${series.map((s, i) => (i ? 'Blau: ' : 'Grau: ') + s.label).join('; ')}. ${time(points[0].time)} – ${time(points[points.length - 1].validUntil)}.`)); +} +async function render() { + const ticket = ++generation, id = plants.value; if (!id) return; + const base = `/api/plants/${encodeURIComponent(id)}/prognosis/planner-v4`; + message.textContent = 'Lade Planungsstand ...'; + try { + const state = await request(base); if (ticket !== generation) return; panel.replaceChildren(); + panel.append(element('p', 'SCHATTENBETRIEB: Die angezeigten Vorgaben steuern keine Geraete.')); + const form = element('form'); + const family = select('Modellfamilie', [['auto', 'Automatisch – historischer Kostenvergleich'], ...state.families.map(f => [f.key, f.label])], state.settings.family); + const measurement = select('Messdatenbasis', [['verified_only', 'Nur verifizierte Peak-/Viertelstundenwerte'], ['allow_estimates', 'Gekennzeichnete Schaetzungen fuer die Planung zulassen']], state.settings.measurementPolicy || 'verified_only'); + const outlook = select('Peakbewertung', [['full_incremental', 'Voller zusaetzlicher Monatstarif'], ['empirical_if_available', 'Restmonats-Szenarien bei ausreichender Historie']], state.settings.peakOutlookPolicy || 'full_incremental'); + const dataSource = select('Verbrauchsprognose', [['legacy', 'Bisherige Datenquelle'], ['corrected_profile', 'Bereinigte physische Lastprofile']], state.settings.forecastSource || 'legacy'); + const datasets = state.dataPipeline?.datasets || []; + const dataset = select('Messdatensatz', [['', 'Datensatz auswaehlen'], ...datasets.map(d => [d.datasetId, d.datasetId + ' (' + d.status + ')'])], state.settings.measurementDataset || ''); + const cadence = select('Nachtraining', [['daily', 'Taeglich'], ['weekly', 'Woechentlich']], state.settings.trainingCadence); + const relianceWrap = element('label', 'Gewicht der Restmonatsprognose (0 bis 100 %) '), reliance = element('input'); + reliance.type = 'number'; reliance.min = '0'; reliance.max = '100'; reliance.step = '1'; reliance.value = String(100 * (state.settings.peakOutlookReliance ?? .5)); relianceWrap.append(reliance); + const save = element('button', 'Speichern und neu berechnen'), replan = element('button', 'Neu berechnen'), refresh = element('button', 'Status aktualisieren'); + save.type = 'submit'; replan.type = refresh.type = 'button'; + form.append(family.wrap, dataSource.wrap, dataset.wrap, measurement.wrap, outlook.wrap, relianceWrap, cadence.wrap, save, replan, refresh); panel.append(form); + let revision = state.settings.revision; + form.addEventListener('submit', async event => { + event.preventDefault(); save.disabled = true; + try { + const r = await request(base + '/settings', 'PUT', { expectedRevision: revision, changes: { family: family.field.value, forecastSource: dataSource.field.value, measurementDataset: dataset.field.value, measurementPolicy: measurement.field.value, peakOutlookPolicy: outlook.field.value, peakOutlookReliance: Number(reliance.value) / 100, trainingCadence: cadence.field.value } }); + revision = r.revision; message.textContent = `Revision ${revision} gespeichert. Neuberechnung angefordert, noch nicht abgeschlossen.`; + } catch (error) { message.textContent = error.message; } finally { save.disabled = false; } + }); + replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } }); + refresh.addEventListener('click', render); + panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.')); + panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik wartet weiterhin auf vergleichbare Kosten-Replays, nicht bloss auf gute R2-Werte.')); + for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.')); + if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.')); + for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]); + const plan = state.plan; + if (plan) { + if (!state.fresh) panel.append(element('p', 'ACHTUNG: Plan veraltet oder Neuberechnung ausstehend.')); + table('Plan und Kosten', [['Plan-ID', plan.planId], ['Modellfamilie', plan.sourceFamily], ['Revision', plan.configRevision], ['Prognose bis', time(plan.forecastUntil)], ['Bekannte Preise bis', time(plan.pricesKnownUntil)], ['Energiekosten', num(plan.energyCostChf) + ' CHF'], ['Zusaetzliche Peakkosten zum vollen Tarif', num(plan.additionalPeakCostChf) + ' CHF'], ['Peakbewertung unter Restmonats-Szenarien', num(plan.planningPeakCostChf) + ' CHF'], ['Planung mit geschaetzter Peakbasis', plan.peakCostIsEstimate ? 'Ja – kein Abrechnungsnachweis' : 'Nein'], ['V4 steuert die Anlage', 'Nein']]); + for (const warning of plan.warnings || []) panel.append(element('p', warning)); + for (const [month, basis] of Object.entries(plan.peakBasis || {})) table(`Peak ${month}`, [['Ausgangsbasis', num(basis.kw) + ' kW (' + basis.quality + ')'], ['Geplant', num(plan.plannedPeaksKw?.[month]) + ' kW'], ['Bewertung', plan.peakOutlook]]); + curve('Netzleistung (W)', plan.points, [{ label: 'ohne Optimierung', value: p => p.baselineGridW }, { label: 'optimiert', value: p => p.gridTargetW }]); + curve('Batterieleistung (W; positiv = Laden)', plan.points, [{ label: 'Batterieplan', value: p => p.batteryTargetW }]); + for (const asset of Object.keys(plan.points[0]?.socEndPercent || {})) curve(`SOC ${asset} (%)`, plan.points, [{ label: 'geplanter SOC', value: p => p.socEndPercent?.[asset] }]); + curve('Kumulierte Zahlungen inklusive vollem Peakpreis (CHF)', plan.points, [{ label: 'ohne Optimierung', value: p => p.cumulativeBaselineCashCostChf }, { label: 'optimiert', value: p => p.cumulativeCashCostChf }]); + panel.append(element('p', 'Kostenverlaeufe sind Prognosen. Restwert der Batterie und Restmonatsbewertung sind keine bereits erzielten Erloese.')); + } + message.textContent = ''; + } catch (error) { if (ticket === generation) { panel.replaceChildren(); message.textContent = error.message; } } +} +try { + session = await request('/api/session'); if (!session.authenticated) throw new Error('Bitte zuerst im Portal anmelden.'); + const data = await request('/api/plants'); + for (const plant of data.plants || []) { if (!plant.installation_id) continue; const option = element('option', plant.name); option.value = plant.id; plants.append(option); } + if (!plants.options.length) throw new Error('Keine verknuepfte Anlage vorhanden.'); + plants.addEventListener('change', render); await render(); +} catch (error) { message.textContent = error.message; } diff --git a/services/netplan-v4/install_hooks.py b/services/netplan-v4/install_hooks.py new file mode 100644 index 0000000..b6e8c76 --- /dev/null +++ b/services/netplan-v4/install_hooks.py @@ -0,0 +1,131 @@ +"""Count-checked optional bridge installation. Default CHECK ONLY, no service action. +Only dedicated V4 GUI files are added. app.js/index.html/styles.css stay untouched. +Every changed legacy source is backed up with hashes. Concurrent changes abort. +""" +from pathlib import Path +from datetime import datetime,timezone +import argparse,ast,hashlib,json,os,tempfile +ROOT=Path(__file__).resolve().parent +SERVICES=Path('/home/agent/services') + +def once(text,old,new): + if text.count(old)!=1:raise ValueError('Source anchor changed/ambiguous; review before updating') + return text.replace(old,new,1) + +def patch_forecast(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts\n'+text + anchor=' p_3 = v3.predict(data_obj, p_1, p_2)' + text=once(text,anchor,' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)])\n\n'+anchor) + text=once(text,' d[key] = float(d.get(key) or default)',' raw_value = d.get(key)\n d[key] = float(default if raw_value is None or raw_value == "" else raw_value)') + ast.parse(text);return text + +def patch_api(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_tariffs as _v4_publish_tariffs\n'+text + anchor=' _portal_store_configuration(anlagen_id, configuration)\n' + text=once(text,anchor,anchor+' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_tariffs(anlagen_id, configuration)\n') + ast.parse(text);return text + +def patch_tariff(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_ckw as _v4_publish_ckw\n'+text + anchor=' return candidate_points\n' + text=once(text,anchor," # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_ckw(source['label'], rows, data.get('publication_timestamp') if isinstance(data, dict) else None, tariff_type)\n"+anchor) + for name in ('row','integrated'): + old=f'{name}.get("value") or {name}.get("price") or {name}.get("amount")' + if old in text:text=once(text,old,f'next(({name}[key] for key in ("value", "price", "amount") if {name}.get(key) is not None), None)') + ast.parse(text);return text + +def patch_portal(text): + if '// ENELIX_V4_SHADOW_BRIDGE' in text:return text + anchor='const server = createServer(async (req, res) => {' + extra="""// ENELIX_V4_SHADOW_BRIDGE +// No dependency or changed route while NETPLAN_V4_URL is absent. +const plannerV4Bridge = process.env.NETPLAN_V4_URL + ? (await import('./netplan-v4-bridge.mjs')).createPlannerV4Bridge({ + configuredPrognosisPlant, roleAllowed, bodyJson, json, deviceActivation, + checkDeviceRate, ownedLicenseState, serviceToken: prognosisServiceToken, + upstreamUrl: process.env.NETPLAN_V4_URL + }) + : null; + +""" + text=once(text,anchor,extra+anchor) + anchor=' if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {' + return once(text,anchor,' if (plannerV4Bridge && await plannerV4Bridge(req, res, url)) return;\n'+anchor) + +def plan(root=SERVICES, approved_assets=None): + root=Path(root).resolve();project=root/'prognosis-manager-enelix2';result={} + if approved_assets is None: + manifest=ROOT/'approved_previous_assets.json' + approved_assets=json.loads(manifest.read_text()) if manifest.is_file() else {} + for path,patch in [(project/'api/main.py',patch_api),(project/'forecast_engine/main.py',patch_forecast),(project/'tariff_importer/main.py',patch_tariff),(root/'license/server.mjs',patch_portal)]: + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes();new=patch(old.decode()).encode() + if old!=new:result[path]=(old,new) + files={project/service/'netplan_v4_publisher.py':ROOT/'integrations/netplan_v4_publisher.py' for service in ('api','forecast_engine','tariff_importer')} + files[root/'license/netplan-v4-bridge.mjs']=ROOT/'integrations/netplan-v4-bridge.mjs' + for name in ('netplan-v4.html','netplan-v4.js','netplan-v4.css'):files[root/'license/public'/name]=ROOT/'gui'/name + for path,source in files.items(): + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes() if path.exists() else None;new=source.read_bytes() + if old is not None and old!=new: + approved=approved_assets.get(str(path.relative_to(root)),[]) + if hashlib.sha256(old).hexdigest() not in approved: + raise ValueError('Existing V4 file differs from the reviewed previous release; explicit merge required: '+str(path)) + if old!=new:result[path]=(old,new) + return result + +def atomic(path,data,mode=0o644): + path.parent.mkdir(parents=True,exist_ok=True);fd,tmp=tempfile.mkstemp(prefix=path.name+'.v4-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as out:out.write(data);out.flush();os.fsync(out.fileno()) + os.chmod(tmp,mode);os.replace(tmp,path) + finally: + if os.path.exists(tmp):os.unlink(tmp) + +def apply(entries,root=SERVICES): + root=Path(root).resolve() + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent change: no files modified') + if not entries:return None + backup=root/'change-backups'/'netplan-v4'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + backup.mkdir(parents=True);receipt={'root':str(root),'files':[]};written=[] + try: + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent source change during install') + rel=path.relative_to(root);mode=path.stat().st_mode&0o777 if path.exists() else 0o644 + if old is not None: + original=backup/'originals'/rel;original.parent.mkdir(parents=True,exist_ok=True);original.write_bytes(old) + atomic(path,new,mode);written.append(path) + receipt['files'].append({'path':str(rel),'before':hashlib.sha256(old).hexdigest() if old is not None else None,'after':hashlib.sha256(new).hexdigest(),'mode':mode}) + atomic(backup/'receipt.json',json.dumps(receipt,indent=2).encode());return backup/'receipt.json' + except Exception: + for path in reversed(written): + old,new=entries[path] + if path.read_bytes()==new: + if old is None:path.unlink() + else:atomic(path,old) + raise + +def rollback(receipt_path): + path=Path(receipt_path).resolve();data=json.loads(path.read_text());root=Path(data['root']).resolve() + for item in data['files']: + target=(root/item['path']).resolve() + if not target.is_relative_to(root) or hashlib.sha256(target.read_bytes()).hexdigest()!=item['after']:raise ValueError('Later change detected; automatic rollback refused') + for item in reversed(data['files']): + target=root/item['path'] + if item['before'] is None:target.unlink() + else: + original=(path.parent/'originals'/item['path']).read_bytes() + if hashlib.sha256(original).hexdigest()!=item['before']:raise ValueError('Backup checksum mismatch') + atomic(target,original,item['mode']) + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--apply',action='store_true');parser.add_argument('--rollback');args=parser.parse_args() + if args.rollback:rollback(args.rollback);print('Source rollback completed. No service restart.') + else: + entries=plan() + for path in entries:print(path) + print('Receipt:',apply(entries)) if args.apply else print('CHECK ONLY:',len(entries),'files. No source/database/service changed.') diff --git a/services/netplan-v4/integrations/NetzfahrplanV4.php b/services/netplan-v4/integrations/NetzfahrplanV4.php new file mode 100644 index 0000000..5ffe258 --- /dev/null +++ b/services/netplan-v4/integrations/NetzfahrplanV4.php @@ -0,0 +1,71 @@ +getTimestamp(); + } + private static function number(mixed $value): float + { + if ((!is_float($value) && !is_int($value)) || !is_finite((float)$value)) { + throw new InvalidArgumentException('Finite numeric value required'); + } + return (float)$value; + } + public static function currentPoint(array $plan, int $now, int $revision, bool $shadowInspection=false): ?array + { + if (($plan['schemaVersion']??null)!==2 || ($plan['configRevision']??null)!==$revision + || ($plan['executable']??false)!==true || !is_array($plan['points']??null) + || !in_array($plan['status']??'', ['optimal','feasible_time_limit'], true) + || !is_string($plan['planId']??null) || $plan['planId']==='') return null; + if ($shadowInspection ? ($plan['runMode']??'')!=='shadow' + : (($plan['runMode']??'')!=='live' || ($plan['liveEnabled']??false)!==true)) return null; + try { + $age=$now-self::stamp($plan['generatedAt']); + if ($age < -30 || $age>900 || self::stamp($plan['validFrom'])>$now || self::stamp($plan['validUntil'])<=$now) return null; + $selected=null; $previousEnd=null; + foreach ($plan['points'] as $p) { + $start=self::stamp($p['time']); $end=self::stamp($p['validUntil']); + self::number($p['gridTargetW']); + if ($end<=$start || $end-$start>300 || ($previousEnd!==null && $previousEnd!==$start)) return null; + $previousEnd=$end; + if ($start<=$now && $now<$end) $selected=$p; + } + return $selected; + } catch (\Throwable $e) { return null; } + } + public static function correction(array $point, float $grid, float $battery, float $chargeAvailable, float $dischargeAvailable, ?float $hardImportLimit=null): array + { + foreach ([$grid,$battery,$chargeAvailable,$dischargeAvailable] as $v) self::number($v); + if ($chargeAvailable<0 || $dischargeAvailable<0) throw new InvalidArgumentException('Invalid available power'); + $target=self::number($point['gridTargetW']??null); + if ($hardImportLimit!==null) { + if (self::number($hardImportLimit)<0) throw new InvalidArgumentException('Negative hard limit'); + $target=min($target,$hardImportLimit); + } + $raw=$battery+$target-$grid; + if (abs($target)>1) { + $raw=match ($point['intent']??'') { + 'gridCharge'=>max(0.,$raw), + 'pvCharge'=>max(0.,min($battery-$grid,$raw)), + 'discharge','export'=>min(0.,$raw), + 'hold'=>0., + default=>throw new InvalidArgumentException('Unknown plan intent'), + }; + } + if ($hardImportLimit!==null) $raw=min($raw,$battery+$hardImportLimit-$grid); + $wanted=min($chargeAvailable,max(-$dischargeAvailable,$raw)); + $expectedGrid=$grid+$wanted-$battery; + return ['batteryTargetW'=>$wanted,'gridTargetW'=>$target,'limited'=>abs($wanted-$raw)>1, + 'expectedGridW'=>$expectedGrid,'trackingErrorW'=>$expectedGrid-$target]; + } +} diff --git a/services/netplan-v4/integrations/netplan-v4-bridge.mjs b/services/netplan-v4/integrations/netplan-v4-bridge.mjs new file mode 100644 index 0000000..1e8786e --- /dev/null +++ b/services/netplan-v4/integrations/netplan-v4-bridge.mjs @@ -0,0 +1,39 @@ +/** Scoped opt-in proxy. Never intercept the existing V1 schedule. */ +export function createPlannerV4Bridge({configuredPrognosisPlant,roleAllowed,bodyJson,json, + deviceActivation,checkDeviceRate,ownedLicenseState,serviceToken, + upstreamUrl='http://netplan-v4:9100',fetchImpl=fetch}){ + const base=new URL(upstreamUrl); + if(!['http:','https:'].includes(base.protocol)||base.username||base.password)throw new Error('Invalid V4 URL'); + async function forward(req,res,installation,suffix,method){ + if(!serviceToken){json(res,503,{error:'V4 ist nicht angebunden.'});return;} + const url=new URL(`/internal/v2/prognosis/${encodeURIComponent(installation)}/planner${suffix}`,base); + let response; + const payload=method==='GET'?undefined:await bodyJson(req); + try{response=await fetchImpl(url,{method,headers:{'X-Enelix-Service-Token':serviceToken,'Content-Type':'application/json'},body:payload===undefined?undefined:JSON.stringify(payload),signal:AbortSignal.timeout(10000),redirect:'error'});} + catch{json(res,503,{error:'V4 nicht erreichbar. Bestehende Regelung bleibt unveraendert.'});return;} + const data=await response.json().catch(()=>null); + if(!response.ok){const code=[400,401,403,404,409,413].includes(response.status)?response.status:503;json(res,code,{error:code===409?'Versionskonflikt: Ansicht neu laden.':'V4-Anfrage abgelehnt oder Dienst nicht verfuegbar.'});return;} + if(!data||typeof data!=='object'){json(res,503,{error:'Ungueltige V4-Antwort.'});return;} + json(res,200,data); + } + return async function(req,res,url){ + const c=url.pathname.match(/^\/api\/plants\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(settings|replan))?$/i); + if(c){ + const action=c[2]||'',method={'':'GET',settings:'PUT',replan:'POST'}[action]; + if(req.method!==method){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + const access=configuredPrognosisPlant(req,res,c[1],method!=='GET');if(!access)return true; + if(Number(access.license.quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + if(method!=='GET'&&!roleAllowed(access.session,['owner','admin','operator'])){json(res,403,{error:'Keine Aenderungsberechtigung.'});return true;} + await forward(req,res,access.plant.installation_id,action?`/${action}`:'',method);return true; + } + const d=url.pathname.match(/^\/api\/v1\/installations\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(operation|ack|measurements))?$/i); + if(d){ + const action=d[2]||'';if(req.method!==(action?'POST':'GET')){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + if(!checkDeviceRate(req,d[1],'planner-v4')){json(res,429,{error:'Zu viele Anfragen.'});return true;} + const activation=deviceActivation(req,res,d[1]);if(!activation)return true; + if(Number(ownedLicenseState(activation.plant_id).quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + await forward(req,res,d[1],action==='operation'?'/inputs/operation':action==='measurements'?'/measurements':action?'/ack':'',req.method);return true; + } + return false; + }; +} diff --git a/services/netplan-v4/integrations/netplan_v4_publisher.py b/services/netplan-v4/integrations/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/integrations/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/netplan_v4/__init__.py b/services/netplan-v4/netplan_v4/__init__.py new file mode 100644 index 0000000..21e0e8d --- /dev/null +++ b/services/netplan-v4/netplan_v4/__init__.py @@ -0,0 +1 @@ +"""ENELIX V4 shadow planner. No live actuator interface.""" diff --git a/services/netplan-v4/netplan_v4/battery_model.py b/services/netplan-v4/netplan_v4/battery_model.py new file mode 100644 index 0000000..0c7813e --- /dev/null +++ b/services/netplan-v4/netplan_v4/battery_model.py @@ -0,0 +1,44 @@ +"""SOC-aware battery model, including reserve recovery and hysteresis rearming.""" +from math import sqrt + + +class BatteryModel: + @staticmethod + def build(model,battery,steps,direction): + cap=battery.capacity_kwh;eta=sqrt(battery.roundtrip_efficiency) + reserve=cap*battery.min_soc_percent/100 + initial=cap*battery.soc_percent/100 + lower=min(initial,reserve) if battery.recovery_allowed else reserve + upper=cap*battery.max_soc_percent/100 + energies=[model.variable(lower,upper) for _ in range(len(steps)+1)] + model.constraint({energies[0]:1},initial,initial) + terminal=battery.terminal_soc_min_percent + if terminal is None:terminal=max(battery.soc_percent,battery.min_soc_percent) + model.constraint({energies[-1]:1},cap*terminal/100) + model.cost[energies[-1]]=-battery.terminal_value_chf_kwh + rearm=cap*(battery.rearm_soc_percent if battery.rearm_soc_percent is not None else battery.min_soc_percent)/100 + enabled=[model.variable(0,1,integer=True) for _ in steps] + # Only a numerical threshold, not an undisclosed extra operating reserve. + epsilon=min(1e-5,max(0.,upper-reserve)/1000) + initially_enabled=not battery.discharge_blocked and initial>=reserve+max(epsilon,1e-8) + model.constraint({enabled[0]:1},int(initially_enabled),int(initially_enabled)) + big=max(upper-lower+epsilon,1.) + charge,discharge=[],[] + for i,step in enumerate(steps): + dt=step.seconds/3600;cmax=battery.max_charge_w/1000;dmax=battery.max_discharge_w/1000 + c=model.variable(0,cmax,battery.throughput_chf_kwh*dt) + d=model.variable(0,dmax,battery.throughput_chf_kwh*dt) + charge.append(c);discharge.append(d) + model.constraint({c:1,direction[i]:-cmax},upper=0) + model.constraint({d:1,direction[i]:dmax},upper=dmax) + model.constraint({d:1,enabled[i]:-dmax},upper=0) + model.constraint({energies[i]:1,enabled[i]:-big},lower=reserve+epsilon-big) + # Once enabled, discharge may consume only energy above reserve. + model.constraint({energies[i+1]:1,enabled[i]:-big},lower=reserve-big) + if i: + # A disabled battery can rearm only AFTER prior charging has + # actually reached the configured hysteresis threshold. + model.constraint({energies[i]:1,enabled[i]:-big,enabled[i-1]:big},lower=rearm-big) + model.constraint({energies[i+1]:1,energies[i]:-1,c:-eta*dt,d:dt/eta},0,0) + return {'charge':charge,'discharge':discharge,'energy':energies, + 'terminal_min_percent':terminal,'discharge_enabled':enabled} diff --git a/services/netplan-v4/netplan_v4/controlled_trial.py b/services/netplan-v4/netplan_v4/controlled_trial.py new file mode 100644 index 0000000..cf9afcb --- /dev/null +++ b/services/netplan-v4/netplan_v4/controlled_trial.py @@ -0,0 +1,162 @@ +"""Explicit, time-limited commissioning authority; NEVER enables shadow execution. + +Only a reviewed internal operator call can arm a trial, and only for a separate +allowlist (empty by default). The manager and battery must additionally consent +locally. Existing shadow plans/acknowledgements keep their original meaning. +""" +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from uuid import UUID +import json + +MAX_SECONDS = 1800 +MAX_POWER_W = 5000 +AUTHORITY_TTL_SECONDS = 90 + + +def schema(con): + con.execute('''CREATE TABLE IF NOT EXISTS planner_controlled_trials( + plant TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, + value TEXT NOT NULL, revoked_at TEXT)''') + + +def timestamp(v): + if not isinstance(v, str): + raise ValueError('Explicit timestamp required') + t = datetime.fromisoformat(v.replace('Z', '+00:00')) + if t.tzinfo is None: + raise ValueError('Timezone required') + return t.astimezone(timezone.utc) + + +def uuid(v): + if not isinstance(v, str) or str(UUID(v)) != v: + raise ValueError('Canonical UUID required') + return v + + +def power(v): + if type(v) not in (int, float) or not 0 < v <= MAX_POWER_W: + raise ValueError('Pilot limit must be explicit and at most 5000 W') + return float(v) + + +def accounting(plan): + # A human checkbox alone must not relabel aggregate house consumption. + quality = plan.get('inputQuality', {}) + if quality.get('loadBasis') != 'base_load': + raise ValueError('Verified base-load/SDL adapter required before a control trial') + evidence = quality.get('accountingEvidenceId') + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Plan lacks traceable base-load accounting evidence') + return evidence + + +def eligibility(view, plant): + if view.get('installationId') != plant or view.get('liveEnabled') is not False: + raise ValueError('Wrong plant or service mode') + p = view.get('plan') + if view.get('fresh') is not True or not isinstance(p, dict): + raise ValueError('Fresh independently validated planning input required') + if (p.get('installationId') != plant or p.get('runMode') != 'shadow' + or p.get('liveEnabled') is not False or p.get('executable') is not True): + raise ValueError('Wrong source plan identity or mode') + settings = view.get('settings', {}) + if settings.get('family') == 'auto' or p.get('sourceFamily') != settings.get('family'): + raise ValueError('First controlled trial requires a fixed model family') + if p.get('configRevision') != settings.get('revision'): + raise ValueError('Configuration revision changed') + if len(p.get('controlContext', {}).get('batteries', {})) != 1: + raise ValueError('First controlled trial supports exactly one battery') + accounting(p) + return p + + +def arm(store, plant, request, view, now, allowed_plants): + """Caller is authenticated internal operator; not exposed via device proxy.""" + if plant not in allowed_plants: + raise ValueError('Controlled trial disabled by server allowlist') + fields = {'sessionId', 'expectedPlanId', 'expectedRevision', 'durationSeconds', + 'maxChargeW', 'maxDischargeW', 'assetId', 'managerId', 'batteryInstanceId', + 'acceptEstimatedPeak', 'actuatorWatchdogEvidenceId', 'confirmation'} + if set(request) != fields or request['confirmation'] != 'ARM_BOUNDED_CONTROL_TRIAL': + raise ValueError('Explicit reviewed commissioning request required') + session = uuid(request['sessionId']); p = eligibility(view, plant) + if type(request['expectedRevision']) is not int or request['expectedRevision'] < 0: + raise ValueError('Expected revision must be an integer') + if request['expectedPlanId'] != p['planId'] or request['expectedRevision'] != p['configRevision']: + raise ValueError('Source plan/revision changed; review again') + seconds = request['durationSeconds'] + if type(seconds) is not int or not 30 <= seconds <= MAX_SECONDS: + raise ValueError('Trial duration must be 30..1800 seconds') + for k in ('managerId', 'batteryInstanceId'): + if type(request[k]) is not int or not 1 <= request[k] <= 99999: + raise ValueError('Explicit local instance binding required') + if request['managerId'] == request['batteryInstanceId']: + raise ValueError('Manager and battery instance must differ') + if request['assetId'] not in p['controlContext']['batteries']: + raise ValueError('Wrong trial battery') + if type(request['acceptEstimatedPeak']) is not bool: + raise ValueError('Explicit estimated-peak policy required') + if p.get('peakCostIsEstimate') and not request['acceptEstimatedPeak']: + raise ValueError('Estimated peak has not been accepted for this trial') + evidence = request['actuatorWatchdogEvidenceId'] + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Device-side command-loss watchdog proof required') + value = {'kind': 'controlled_trial_grant', 'version': 1, 'sessionId': session, + 'installationId': plant, 'issuedAt': now.isoformat(), + 'expiresAt': (now + timedelta(seconds=seconds)).isoformat(), + 'revision': p['configRevision'], 'family': p['sourceFamily'], + 'assetId': request['assetId'], 'managerId': request['managerId'], + 'batteryInstanceId': request['batteryInstanceId'], + 'maxChargeW': power(request['maxChargeW']), + 'maxDischargeW': power(request['maxDischargeW']), + 'acceptEstimatedPeak': request['acceptEstimatedPeak'], + 'accountingEvidenceId': accounting(p), + 'actuatorWatchdogEvidenceId': evidence, + 'controlContext': deepcopy(p['controlContext'])} + with store.con: + existing = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if existing and existing[1] is None and timestamp(json.loads(existing[0])['expiresAt']) > now: + raise ValueError('Existing trial must first end; implicit extension forbidden') + # Reusing an expired/revoked session ID must never resurrect it. + used = store.con.execute("SELECT 1 FROM planner_audit WHERE plant=? AND kind='trial_arm' AND detail=?", (plant, session)).fetchone() + if used: + raise ValueError('Session ID already used') + store.con.execute('INSERT INTO planner_controlled_trials VALUES(?,?,?,NULL) ON CONFLICT(plant) DO UPDATE SET session_id=excluded.session_id,value=excluded.value,revoked_at=NULL', + (plant, session, json.dumps(value, sort_keys=True, allow_nan=False))) + store.con.execute("INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,'trial_arm',?)", (plant, now.isoformat(), session)) + return value + + +def revoke(store, plant, session, now): + uuid(session) + with store.con: + store.con.execute('UPDATE planner_controlled_trials SET revoked_at=? WHERE plant=? AND session_id=?', (now.isoformat(), plant, session)) + return {'status': 'revoked', 'sessionId': session, 'remoteRevocationMaxSeconds': AUTHORITY_TTL_SECONDS} + + +def authority(store, plant, view, now, allowed_plants): + """Separate authorization envelope, not a mutation of the shadow plan.""" + if plant not in allowed_plants: + return None + row = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if not row or row[1] is not None: + return None + grant = json.loads(row[0]) + try: + p = eligibility(view, plant) + if not timestamp(grant['issuedAt']) <= now < timestamp(grant['expiresAt']): + return None + if (p['configRevision'] != grant['revision'] or p['sourceFamily'] != grant['family'] + or p['controlContext'] != grant['controlContext'] + or accounting(p) != grant['accountingEvidenceId'] + or p.get('peakCostIsEstimate') and not grant['acceptEstimatedPeak']): + return None + except (ValueError, KeyError, TypeError): + return None + until = min(timestamp(grant['expiresAt']), timestamp(p['validUntil']), + now + timedelta(seconds=AUTHORITY_TTL_SECONDS)) + return {**grant, 'kind': 'controlled_trial_authority', + 'sourceShadowPlanId': p['planId'], 'checkedAt': now.isoformat(), + 'validUntil': until.isoformat(), 'sourcePlanRemainsShadow': True} diff --git a/services/netplan-v4/netplan_v4/domain.py b/services/netplan-v4/netplan_v4/domain.py new file mode 100644 index 0000000..140acb8 --- /dev/null +++ b/services/netplan-v4/netplan_v4/domain.py @@ -0,0 +1,170 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from math import isfinite +from typing import Mapping +from zoneinfo import ZoneInfo + +UTC = timezone.utc +ZURICH = ZoneInfo('Europe/Zurich') + +def utc(value): + if isinstance(value, str): + value = datetime.fromisoformat(value.replace('Z', '+00:00')) + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError('Timezone required') + return value.astimezone(UTC) + +def number(value, name, minimum=None, maximum=None): + if isinstance(value, bool) or not isinstance(value, (float, int)): + raise ValueError(f'{name}: finite number required') + value = float(value) + if not isfinite(value) or minimum is not None and value < minimum or maximum is not None and value > maximum: + raise ValueError(f'{name}: outside permitted range') + return value + +def quarter_start(value): + value = utc(value) + return value.replace(minute=value.minute // 15 * 15, second=0, microsecond=0) + +def month_key(value): + return utc(value).astimezone(ZURICH).strftime('%Y-%m') + +@dataclass(frozen=True) +class Family: + key: str + pv: str + load: str + schedule: str + label: str + +class FamilyRegistry: + def __init__(self, families=()): + self._families = {} + for family in families: + self.register(family) + def register(self, family): + if family.key in self._families or any(f.schedule == family.schedule for f in self._families.values()): + raise ValueError('Duplicate family or schedule identifier') + self._families[family.key] = family + def get(self, key): + if key not in self._families: + raise ValueError(f'Unknown model family: {key}') + return self._families[key] + def entries(self): + return tuple(self._families.values()) + +def default_registry(): + return FamilyRegistry([ + Family('3','prog_var_1','prog_var_2','prog_var_3','Variante 1 (1 / 2 / 3)'), + Family('13','prog_var_10','prog_var_11','prog_var_13','Variante 2 (10 / 11 / 13)'), + Family('23','prog_var_21','prog_var_22','prog_var_23','Variante 3 (21 / 22 / 23)'), + ]) + +@dataclass(frozen=True) +class Price: + chf_kwh: float + published_at: datetime | None = None + mode: str = 'static' + def known_at(self, at): + number(self.chf_kwh, 'energy price') + if self.mode not in ('static','dynamic'): + raise ValueError('Explicit static/dynamic price mode required') + return self.mode == 'static' or self.published_at is not None and utc(self.published_at) <= utc(at) + +@dataclass(frozen=True) +class Step: + start: datetime + base_load_w: float + pv_w: float + import_price: Price | None + export_price: Price | None + external_w: float = 0.0 + seconds: int = 300 + @property + def end(self): + return utc(self.start) + timedelta(seconds=self.seconds) + @property + def residual_w(self): + return self.base_load_w + self.external_w - self.pv_w + +def split_base_load(measured_house_w, flexible_w, external_w=0.0, external_already_removed=False): + measured = number(measured_house_w, 'measured_house_w') + flex = sum(number(v, 'flexible measurement') for v in flexible_w) + external = number(external_w, 'external measurement') + base = measured - flex - (0.0 if external_already_removed else external) + if base < -1.0: + raise ValueError('Negative base load: inconsistent measurement boundary or double subtraction') + return max(0.0, base) + +def priced_prefix(steps, at): + result = [] + for step in steps: + if not step.import_price or not step.export_price: + break + if not step.import_price.known_at(at) or not step.export_price.known_at(at): + break + result.append(step) + while result and result[-1].end != quarter_start(result[-1].end): + result.pop() + return result + +@dataclass(frozen=True) +class Battery: + asset_id: str + capacity_kwh: float + soc_percent: float + min_soc_percent: float + max_soc_percent: float + max_charge_w: float + max_discharge_w: float + measured_at: datetime + roundtrip_efficiency: float = 0.90 + grid_charging: bool = False + throughput_chf_kwh: float = 0.0 + terminal_soc_min_percent: float | None = None + terminal_value_chf_kwh: float = 0.0 + recovery_allowed: bool = False + physical_min_soc_percent: float = 0.0 + discharge_blocked: bool = False + rearm_soc_percent: float | None = None + def validate(self, at): + if not self.asset_id: + raise ValueError('Battery asset_id required') + number(self.capacity_kwh, 'capacity_kwh', 0.001) + lo = number(self.min_soc_percent, 'min_soc_percent', 0, 100) + hi = number(self.max_soc_percent, 'max_soc_percent', lo, 100) + physical=number(self.physical_min_soc_percent,'physical minimum SOC',0,lo) + if type(self.recovery_allowed) is not bool or type(self.discharge_blocked) is not bool: + raise ValueError('Explicit battery recovery and hysteresis flags required') + number(self.soc_percent, 'SOC', physical if self.recovery_allowed else lo, hi) + if self.rearm_soc_percent is not None:number(self.rearm_soc_percent,'hysteresis rearm SOC',lo,hi) + number(self.max_charge_w, 'max_charge_w', 0) + number(self.max_discharge_w, 'max_discharge_w', 0) + number(self.roundtrip_efficiency, 'roundtrip_efficiency', 0.01, 1) + number(self.throughput_chf_kwh, 'throughput_chf_kwh', 0) + number(self.terminal_value_chf_kwh, 'terminal_value_chf_kwh', 0) + if self.terminal_soc_min_percent is not None: + number(self.terminal_soc_min_percent, 'terminal_soc_min_percent', lo, hi) + age = (utc(at) - utc(self.measured_at)).total_seconds() + if age < -30 or age > 1800: + raise ValueError('SOC is stale or from the future') + +@dataclass(frozen=True) +class QuarterPast: + import_kwh: float + measured_seconds: int + +@dataclass(frozen=True) +class Limits: + export_w: float | None = None + import_w: float | None = None + manager_month_limits_w: Mapping[int,float] = field(default_factory=dict) + def import_limit(self, timestamp): + values = [] + if self.import_w is not None: + values.append(number(self.import_w, 'import limit', 0)) + m = utc(timestamp).astimezone(ZURICH).month + if m in self.manager_month_limits_w: + values.append(number(self.manager_month_limits_w[m], 'monthly manager limit', 0)) + return min(values) if values else None diff --git a/services/netplan-v4/netplan_v4/forecast_quality.py b/services/netplan-v4/netplan_v4/forecast_quality.py new file mode 100644 index 0000000..4a83446 --- /dev/null +++ b/services/netplan-v4/netplan_v4/forecast_quality.py @@ -0,0 +1,21 @@ +"""Minimal validity gates, not a claim of forecast accuracy or model ranking.""" +from math import isfinite + + +def assess_family(family, *, minimum_steps=3): + points=family.get('points',[]) + if len(points) physical load -> trained profiles. + +Lives in the existing planner service/database; no separate diagnostic service. +The device may append only to an operator-configured dataset. Original observations, +model revisions and prediction vintages are preserved. Output is never an actuator grant. +""" +from __future__ import annotations +from bisect import bisect_right +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from math import isfinite +from statistics import median +from zoneinfo import ZoneInfo +import json + +UTC = timezone.utc +LOCAL = ZoneInfo('Europe/Zurich') +FAMILIES = ('3', '13', '23') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond: + raise ValueError('Explicit whole-second UTC timestamp required') + return int(t.timestamp()) + + +def iso(t): + return datetime.fromtimestamp(t, UTC).isoformat() + + +def numeric(value, bound=1e12): + return type(value) in (int, float) and isfinite(value) and abs(value) <= bound + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_data_sets( + plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_observations( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,captured_at)); + CREATE TABLE IF NOT EXISTS planner_load_windows( + plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL, + available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,start)); + CREATE TABLE IF NOT EXISTS planner_load_models( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,model_id)); + CREATE TABLE IF NOT EXISTS planner_model_current( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_pipeline_state( + plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL, + status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_prediction_vintages( + plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL, + target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL, + load_w REAL NOT NULL, pv_w REAL NOT NULL, + PRIMARY KEY(plant,dataset,issued_at,target,family)); + CREATE INDEX IF NOT EXISTS planner_observation_window + ON planner_observations(plant,dataset,captured_at); + CREATE INDEX IF NOT EXISTS planner_prediction_target + ON planner_prediction_vintages(plant,dataset,target); + ''') + + +def validate_config(c): + fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', + 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', + 'minimumTrainingHours', 'historyDays'} + if not isinstance(c, dict) or set(c) != fields: + raise ValueError('Explicit dataset configuration required') + name = c['datasetId'] + if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): + raise ValueError('Invalid dataset ID') + for field in ('mappingSha256', 'inventorySha256'): + h = c[field] + if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h): + raise ValueError('Explicit mapping/inventory fingerprint required') + if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'): + raise ValueError('Unknown physical formula') + if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1: + raise ValueError('Coverage must be .90..1; recorded gaps remain visible') + for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)): + if type(c[field]) is not int or not lo <= c[field] <= hi: + raise ValueError('Invalid '+field) + sources = c['sources'] + if not isinstance(sources, list) or not 3 <= len(sources) <= 80: + raise ValueError('Source list required') + seen, ids = set(), set() + roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'} + for s in sources: + if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}: + raise ValueError('Explicit source definition required') + k = s['key'] + if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids: + raise ValueError('Duplicate/invalid source') + if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0: + raise ValueError('Source role/factor invalid') + if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300: + raise ValueError('Source lifetime invalid') + seen.add(k); ids.add(s['variableId']) + if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources): + raise ValueError('Grid and PV measurement sources required') + sr = c['solarReference'] + if c['formula'] == 'solar_terminal_v1': + if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}: + raise ValueError('Solar terminal sources required') + bykey = {s['key']: s['role'] for s in sources} + if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))): + raise ValueError('Solar origin roles mismatch') + elif sr is not None: + raise ValueError('No unused solar mapping allowed') + canonical(c) + return c + + +def register_dataset(con, plant, c, now): + """Operator endpoint only; device append endpoint cannot change units or limits.""" + validate_config(c) + value = canonical(c) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone() + if old and old[0] != value: + raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning') + con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False} + + +def configuration(con, plant, dataset): + row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone() + if row is None: + raise ValueError('Dataset not configured for this installation') + return json.loads(row[0]) + + +def project(record, c, plant, now): + if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: + raise ValueError('Wrong capture identity') + if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + raise ValueError('Wrong capture mapping or inventory') + t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) + if start > t or t > now+30 or t < now-90*86400: + raise ValueError('Capture timestamp outside permitted range') + if not isinstance(record.get('raw'), dict): + raise ValueError('Numeric raw observations required') + out = {}; issues = [] + for s in c['sources']: + r = record['raw'].get(s['key'], {}) + if not isinstance(r, dict): + r = {} + v, at = r.get('value'), r.get('sourceUpdatedAt') + good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == [] + if not good: + v = at = None + issues.append(s['key']) + # Unknown/free-text fields, credentials, client quality claims never persisted. + out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)} + return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues} + + +def ingest_batch(con, plant, payload, now): + if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: + raise ValueError('Measurement batch version/fields invalid') + c = configuration(con,plant,payload['datasetId']) + records = payload['records'] + if not isinstance(records,list) or not 1 <= len(records) <= 120: + raise ValueError('Batch requires 1..120 captures') + rows = [project(r,c,plant,now) for r in records] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): + raise ValueError('Batch must be in increasing capture order') + stored = duplicate = 0 + con.execute('BEGIN IMMEDIATE') + try: + for r in rows: + value = canonical(r); digest = sha256(value.encode()).hexdigest() + old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() + if old: + if old[0] != digest: + raise ValueError('Conflicting immutable observation') + duplicate += 1 + else: + con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + con.commit() + except Exception: + con.rollback(); raise + return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate, + 'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False} + + +def physical_value(values, c): + """Same explicit sign convention as configured acquisition. No virtual power in load.""" + total = defaultdict(float) + sr = c['solarReference'] + for s in c['sources']: + if s['role'] not in ('grid','pv','physical_storage','flexible_load'): + continue + if sr and s['key'] in (sr['pvKey'],sr['batteryKey']): + continue + val = values[s['key']]*s['factorToW'] + if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0): + raise ValueError('Invalid physical power') + total[s['role']] += val + solar = 0. + if sr: + raw, sf = values[sr['rawKey']], values[sr['scaleKey']] + if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6: + raise ValueError('Invalid solar power/scaling sentinel') + solar = raw*10**int(sf) + load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar + if not numeric(load,1e9) or load < 0: + raise ValueError('Negative/nonfinite physical load') + return load + + +def reconstruct(records, c): + """Bounded retrospective estimation, never a real-time feedback signal. + +Missing observations split support. Source timestamps are not refreshed. Small +uncovered portions remain quantified and are never filled with zero. +""" + if len(records) < 2: + return [] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])): + raise ValueError('Capture sequence not ordered') + sr = c['solarReference'] + primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')} + if sr: + primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) + for s in c['sources']: + if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + first,last = records[0]['capturedAt'],records[-1]['capturedAt'] + series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + pending = {k:None for k in primary}; high = {k:0 for k in primary} + edges = {first,last} + for a,b in zip(records,records[1:]): + if b['capturedAt']-a['capturedAt'] > 45: + gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1]) + for r in records: + at = r['capturedAt'] + for k in primary: + v = r['raw'].get(k,{}) + t = v.get('sourceUpdatedAt') + if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5: + if pending[k] is None: pending[k] = at + continue + high[k] = max(high[k],t) + if pending[k] is not None: + blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None + if t in series[k] and series[k][t] != v['value']: + series[k][t] = None + else: + series[k].setdefault(t,v['value']) + for k,s in primary.items(): + if pending[k] is not None: + blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) + for t in series[k]: edges.update((t,t+s['maxAgeSeconds'])) + edges.update(range(first//300*300+300,last,300)) + edges = sorted(x for x in edges if first <= x <= last) + knots = {k:sorted(v) for k,v in series.items()} + bins = {} + for a,b in zip(edges,edges[1:]): + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0}) + vals = {}; usable = not any(x <= a < y for x,y in gaps) + for k,s in primary.items(): + pos = bisect_right(knots[k],a)-1 + t = knots[k][pos] if pos >= 0 else None + if t is None or a >= t+s['maxAgeSeconds'] or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False + else: vals[k] = series[k][t] + load = None + if usable: + try: load = physical_value(vals,c) + except ValueError: usable = False + if usable: + item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + else: + item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + out=[] + for t,item in sorted(bins.items()): + # Partial beginning/end bins remain diagnostic and cannot train. + complete_extent = first <= t and last >= t+300 + coverage = item['seconds']/300 + eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds'] + out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], + 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, + 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, + 'meterBoundaryVerified':False,'method':c['formula']}) + return out + + +def slot(t): + local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL) + return local.hour*12+local.minute//5 + + +def build_profiles(rows): + samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)} + anchor=max(r['start'] for r in rows) + for r in rows: + i=slot(r['start']); v=r['loadW']; samples[i].append(v) + if anchor-r['start'] < 86400: recent[i].append(v) + weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v) + overall = median([r['loadW'] for r in rows]) + def profile(values): + # Missing calendar slots are a model estimate, not invented historical measurements. + result=[] + for i in range(288): + local=values.get(i,[]) + if not local: + local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])] + result.append(float(median(local)) if local else float(overall)) + return result + return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)}, + 'slotCoverage':len(samples)/288} + + +def predict(model, family, t): + p=model['profiles'][family] + if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday'] + return p[slot(t)] + + +def advance(con, plant, dataset, settings, now): + """Called by the existing worker; bounded data/model update once per five-minute tick.""" + c=configuration(con,plant,dataset); tick=now//300 + old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() + if old and old[0]==tick: return + fetched=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,dataset,now-172800-300,now,now)).fetchall() + records=[json.loads(r[0]) for r in fetched] + windows=reconstruct(records,c) + with con: + for w in windows: + if w['start']+300 > now-30: continue + con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', + (plant,dataset,w['start'],now,w['coverage'],canonical(w))) + rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] + good=[r for r in rows if r['profileUsable']] + active=current_model(con,plant,dataset,now) + cadence=86400 if settings['trainingCadence']=='daily' else 604800 + detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], + 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True} + state='collecting' + if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: + state='model_ready' if active else 'training' + attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0] + if attempted is None or now-attempted>=cadence: + profiles=build_profiles(good) + candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), + 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], + 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'measurementBoundaryVerified':False} + # Causal held-out validation: build validation profiles without the final day. + split=good[-1]['start']-86400 + train=[r for r in good if r['start']+300<=split]; test=[r for r in good if r['start']>=split] + if len(train)>=288 and len(test)>=240: + val={'profiles':build_profiles(train)} + errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} + candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors} + # Initial model is labelled bootstrap, never a production measurement proof. + # Existing model can be replaced only with held-out evidence and no aggregate regression. + promote=active is None + if active and candidate['validation']['status']=='causal_holdout': + past_model_eligible=active['trainedThrough']<=split + if past_model_eligible: + incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + promote=challenger<=incumbent + candidate['validation']['incumbentCompared']=True + else: + candidate['validation']['status']='holdout_overlaps_active_training' + ident=sha256(canonical(candidate).encode()).hexdigest() + with con: + con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate))) + if promote: + con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident)) + detail['candidateModelId']=ident;detail['candidatePromoted']=promote + state='model_ready' if promote or active else 'candidate_pending' + active=current_model(con,plant,dataset,now) + if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']}) + with con: + con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail', + (plant,dataset,tick,state,canonical(detail))) + + +def current_model(con,plant,dataset,at): + row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone() + return {**json.loads(row[1]),'modelId':row[0]} if row else None + + +def apply_load_forecast(con,plant,dataset,forecast,decision): + model=current_model(con,plant,dataset,decision) + if not model: + raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,dataset,decision,decision)).fetchone() + if not last: raise ValueError('No recent corrected observation') + last=json.loads(last[0]); c=configuration(con,plant,dataset) + if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') + sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] + if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') + s=sdl_sources[0];r=last['raw'][s['key']] + if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']: + raise ValueError('No current external SDL request for the persistence scenario') + sdl=r['value']*s['factorToW'] + result=json.loads(canonical(forecast)); result['families']={} + result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + for family,old in forecast['families'].items(): + if family not in FAMILIES: continue + points=[] + for p in old['points']: + t=epoch(p['time']) + points.append({**p,'loadW':predict(model,family,t),'externalW':sdl}) + result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']), + 'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'], + 'loadMethodVersion':model['methodVersion'],'loadVariant':family, + 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), + 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, + 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation']}} + return result + + +def pipeline_status(con,plant): + out=[] + for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): + ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], + 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds']}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False} diff --git a/services/netplan-v4/netplan_v4/meter_runtime.py b/services/netplan-v4/netplan_v4/meter_runtime.py new file mode 100644 index 0000000..db92eda --- /dev/null +++ b/services/netplan-v4/netplan_v4/meter_runtime.py @@ -0,0 +1,145 @@ +"""Persistent, explicitly estimated operational demand tracking. + +No inference from a configured cap. Samples are device-reception observations; +last-value integration is a labelled control estimate, not settlement metering. +No reset, stale sample or long communication gap is bridged silently. +""" +from datetime import datetime,timedelta,timezone +import json +from .domain import number,utc,month_key,quarter_start,ZURICH +from .peak_policy import basis_record + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_peak_assumptions( + plant TEXT NOT NULL,month TEXT NOT NULL,kw REAL NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_runtime_samples( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,at INTEGER NOT NULL, + power_w REAL NOT NULL,total_kwh REAL,policy_id TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,at)); + CREATE TABLE IF NOT EXISTS planner_runtime_quarters( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,start INTEGER NOT NULL, + import_kwh REAL NOT NULL,policy_id TEXT NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,start)); + CREATE INDEX IF NOT EXISTS idx_runtime_samples_time ON planner_runtime_samples(plant,at); + ''') + + +def validate_observation(obs, observed_at): + if not isinstance(obs,dict) or set(obs)!={'meterId','sampleAt','powerW','totalImportKwh','controlPolicyId'}: + raise ValueError('Explicit meter observation schema required') + import re + if not isinstance(obs['meterId'],str) or not re.fullmatch(r'symcon-active-import:[0-9a-f]{64}',obs['meterId']): + raise ValueError('Active import identity required') + if not isinstance(obs['controlPolicyId'],str) or not 1<=len(obs['controlPolicyId'])<=160: + raise ValueError('Control policy identity required') + t=utc(obs['sampleAt']) + if t.microsecond or not 0<=(utc(observed_at)-t).total_seconds()<=60: + raise ValueError('Fresh whole-second acquisition timestamp required') + number(obs['powerW'],'meter power',-1e9,1e9) + number(obs['totalImportKwh'],'active import total',0,1e12) + + +def integrate_power(samples,start,end,max_gap=120): + """samples sorted tuples (epoch,power,total,policy). Return None on gaps/reset. + + A source can keep the same value while receiving fresh telemetry. The caller + records all acquisitions, not only changes. A quarter is never labelled exact. + """ + if end<=start:return None + rows=sorted(samples,key=lambda r:r[0]) + if len(rows)<2:return None + covered=energy=0.;policies=set();largest_gap=0 + previous=None + for row in rows: + if previous is not None: + ta,pa,ea,pola=previous;tb,pb,eb,polb=row + if tb<=ta:return None + left=max(start,ta);right=min(end,tb) + if right>left: + if tb-ta>max_gap or pola!=polb:return None + if ea is not None and eb is not None and eb=validated['kw']:return old + con.execute('INSERT INTO planner_peak_assumptions VALUES(?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET kw=excluded.kw,value=excluded.value', + (plant,month,validated['kw'],json.dumps(validated,sort_keys=True,allow_nan=False))) + return validated + + +def assumptions(con,plant): + return {r[0]:json.loads(r[1]) for r in con.execute('SELECT month,value FROM planner_peak_assumptions WHERE plant=?',(plant,))} + + +def observe(con,plant,obs,at): + validate_observation(obs,at) + mid=obs['meterId'];stamp=int(utc(obs['sampleAt']).timestamp()) + existing=con.execute('SELECT power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at=?',(plant,mid,stamp)).fetchone() + values=(float(obs['powerW']),float(obs['totalImportKwh']),obs['controlPolicyId']) + if existing and tuple(existing)!=values:raise ValueError('Conflicting meter acquisition at same time') + con.execute('INSERT OR IGNORE INTO planner_runtime_samples VALUES(?,?,?,?,?,?)',(plant,mid,stamp,*values)) + # Only a just-completed quarter is finalised; late acquisition never promotes + # a historical gap to complete data without all supporting samples. + current=stamp//900*900 + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,mid,current-900-120,stamp))] + report=integrate_power(rows,current-900,current) + if report: + quarter=current-900 + previous=con.execute('SELECT import_kwh FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start=?',(plant,mid,quarter)).fetchone() + if previous is None: + con.execute('INSERT INTO planner_runtime_quarters VALUES(?,?,?,?,?,?)',(plant,mid,quarter,report['importKwh'],report['controlPolicyId'],json.dumps(report))) + m=month_key(datetime.fromtimestamp(quarter,timezone.utc)) + save_assumption(con,plant,m,{'kw':report['importKwh']/.25,'quality':'estimated','source':'sampled_power_estimate', + 'observedAt':utc(at).isoformat(),'meterId':mid,'notes':'Maximum of available sampled quarters; earlier month may be incomplete'},at) + # Samples are retained for reproducibility; production retention job required. + return report + + +def current_quarter(con,plant,obs,decision): + q=quarter_start(decision);start=int(q.timestamp());end=int(utc(decision).timestamp()) + if start==end:return None + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,obs['meterId'],start-120,end))] + report=integrate_power(rows,start,end) + if not report:return None + return {'start':q.isoformat(),'measuredSeconds':end-start,'importKwh':report['importKwh'], + 'quality':'estimated','source':'sampled_power_estimate','coverage':1.0,'notes':'Acquisition power estimate, not an exact billing counter boundary'} + + +def daily_peaks(con,plant,meter_id,now): + rows=con.execute('SELECT start,import_kwh,policy_id FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start>=? ORDER BY start', + (plant,meter_id,int((utc(now)-timedelta(days=91)).timestamp()))) + grouped={} + for start,energy,policy in rows: + t=datetime.fromtimestamp(start,timezone.utc).astimezone(ZURICH) + grouped.setdefault((t.strftime('%Y-%m-%d'),policy),[]).append((start,energy)) + result=[] + for (day,policy),items in grouped.items(): + begin=datetime.strptime(day,'%Y-%m-%d').replace(tzinfo=ZURICH);end=begin+timedelta(days=1) + if utc(end)>utc(now):continue + expected=set(range(int(begin.timestamp()),int(end.timestamp()),900)) + complete={t for t,e in items}==expected + result.append({'day':day,'peakKw':max(e/.25 for t,e in items),'controlPolicyId':policy,'complete':complete, + 'observedAt':utc(end).isoformat(),'quality':'estimated'}) + return result diff --git a/services/netplan-v4/netplan_v4/metering.py b/services/netplan-v4/netplan_v4/metering.py new file mode 100644 index 0000000..ecca015 --- /dev/null +++ b/services/netplan-v4/netplan_v4/metering.py @@ -0,0 +1,245 @@ +"""Auditable import-energy evidence. Pure calculations; no device/database writes. + +Raw change archives are NOT proof of uninterrupted meter communication. Boundaries +without an exact counter record yield intervals, never silently interpolated +billing facts. Only a separately verified source and chronology can be promoted +into the existing strict planner evidence contract. +""" +from __future__ import annotations +from bisect import bisect_left, bisect_right +from dataclasses import dataclass +from datetime import datetime, timedelta +import hashlib +import json +from math import isfinite +from .domain import utc, month_key, quarter_start, ZURICH + + +@dataclass(frozen=True) +class Reading: + at: datetime + kwh: float + + +class CounterSeries: + def __init__(self, source_id: str, rows, *, max_bracket_seconds=180, + quantity='active_import', unit='kWh', timestamp_verified=False, + identity_verified=False, chronology_verified=False): + if quantity != 'active_import' or unit != 'kWh': + raise ValueError('Active import energy in kWh required') + if not isinstance(source_id, str) or not source_id: + raise ValueError('Nonempty stable source identity required') + if type(max_bracket_seconds) is not int or max_bracket_seconds < 1: + raise ValueError('Positive bracket age required') + self.source_id = source_id + self.max_gap = max_bracket_seconds + self.verified = all(x is True for x in + (timestamp_verified, identity_verified, chronology_verified)) + values = {} + for row in rows: + at = utc(row.at) + value = row.kwh + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value < 0: + raise ValueError('Invalid cumulative active import value') + if at in values and abs(values[at] - value) > 1e-9: + raise ValueError('Conflicting duplicate timestamp') + values[at] = float(value) + self.times = sorted(values) + self.values = [values[t] for t in self.times] + self.resets = [self.times[i] for i in range(1,len(self.times)) + if self.values[i] < self.values[i-1] - 1e-9] + + def boundary(self, at): + at = utc(at) + pos = bisect_left(self.times, at) + if pos < len(self.times) and self.times[pos] == at: + value = self.values[pos] + return {'lowerKwh':value, 'upperKwh':value, 'exactRecord':True, + 'before':at.isoformat(), 'after':at.isoformat()} + if pos == 0 or pos == len(self.times): + return None + before, after = self.times[pos-1], self.times[pos] + if (after-before).total_seconds() > self.max_gap: + return None + if self.values[pos] < self.values[pos-1] - 1e-9: + return None + return {'lowerKwh':self.values[pos-1], 'upperKwh':self.values[pos], + 'exactRecord':False, 'before':before.isoformat(), 'after':after.isoformat()} + + def energy(self, start, end): + start, end = utc(start), utc(end) + if end <= start: + raise ValueError('Positive interval required') + left, right = self.boundary(start), self.boundary(end) + if not left or not right: + return None + support_start = utc(left['before']) + support_end = utc(right['after']) + if any(support_start < reset <= support_end for reset in self.resets): + return None + lower = max(0., right['lowerKwh'] - left['upperKwh']) + upper = right['upperKwh'] - left['lowerKwh'] + if upper < lower - 1e-9: + return None + exact = bool(left['exactRecord'] and right['exactRecord']) + return {'lowerKwh':lower, 'upperKwh':max(lower, upper), + 'exactRecords':exact, 'billingEvidence':exact and self.verified} + + +def native_meter_id(sources): + """Same ordered JSON and SHA256 as NetzfahrplanV4Bezugszaehler::identitaet.""" + import re + if not isinstance(sources,list) or not 1<=len(sources)<=8: + raise ValueError('Explicit native import sources required') + normalized=[];ids=set();idents=set();parents=set() + for source in sources: + if set(source)!={'VariableID','ElternID','Ident','FaktorZuKWh','Messgroesse'}: + raise ValueError('Unknown native source fields') + sid,pid,ident,factor=(source[k] for k in ('VariableID','ElternID','Ident','FaktorZuKWh')) + if (type(sid) is not int or sid<=0 or type(pid) is not int or pid<=0 + or sid in ids or ident in idents or not isinstance(ident,str) + or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}',ident) + or isinstance(factor,bool) or not isinstance(factor,(float,int)) + or not isfinite(factor) or not 0=0 else '')+str(power) + text=re.sub(r'"FaktorZuKWh":([0-9]+(?:\.[0-9]+)?)e([+-]?[0-9]+)',exponent,text) + return 'symcon-active-import:'+hashlib.sha256(text.encode()).hexdigest() + + +class MeterEvidence: + def __init__(self, series, *, native_sources=None): + self.series = list(series) + if not self.series or len({s.source_id for s in self.series}) != len(self.series): + raise ValueError('Unique, nonempty set of counter sources required') + self.native_bound = native_sources is not None + if self.native_bound: + self.meter_id=native_meter_id(native_sources) + if {s.source_id for s in self.series}!={'symcon:'+str(s['VariableID']) for s in native_sources}: + raise ValueError('Evidence stream does not match native source set') + else: + signature = json.dumps(sorted(s.source_id for s in self.series),separators=(',',':')) + self.meter_id = 'audit-only:' + hashlib.sha256(signature.encode()).hexdigest() + + def interval(self, start, end): + start, end = utc(start), utc(end) + parts = [s.energy(start,end) for s in self.series] + missing = [s.source_id for s,p in zip(self.series,parts) if p is None] + if missing: + return {'status':'missing', 'start':start.isoformat(),'end':end.isoformat(), + 'missingSources':missing, 'billingEvidence':False} + lo = sum(p['lowerKwh'] for p in parts) + hi = sum(p['upperKwh'] for p in parts) + seconds = (end-start).total_seconds() + exact = all(p['exactRecords'] for p in parts) + return {'status':'exact_records' if exact else 'bounded_records', + 'start':start.isoformat(),'end':end.isoformat(),'lowerKwh':lo,'upperKwh':hi, + 'lowerAverageKw':lo*3600/seconds,'upperAverageKw':hi*3600/seconds, + 'billingEvidence':all(p['billingEvidence'] for p in parts)} + + def month(self, at): + at = utc(at) + local = at.astimezone(ZURICH) + begin = utc(local.replace(day=1,hour=0,minute=0,second=0,microsecond=0)) + stop = quarter_start(at) + count = int((stop-begin).total_seconds())//900 + covered = exact = verified = 0 + lo = hi = 0. + gaps = [] + quarters = [] + for n in range(count): + start = begin + timedelta(minutes=15*n) + result = self.interval(start,start+timedelta(minutes=15)) + quarters.append(result) + if result['status'] == 'missing': + if len(gaps)<8:gaps.append(start.isoformat()) + continue + covered += 1 + exact += result['status']=='exact_records' + verified += result['billingEvidence'] + lo = max(lo,result['lowerAverageKw']) + hi = max(hi,result['upperAverageKw']) + complete = count>0 and covered==count + certified = count>0 and verified==count + return {'meterId':self.meter_id,'month':month_key(at),'completedQuarters':count, + 'coveredQuarters':covered,'exactRecordQuarters':exact,'verifiedQuarters':verified, + 'historyComplete':complete,'billingEvidence':certified, + 'observedPeakLowerKw':lo if covered else None, + 'observedPeakUpperKw':hi if covered else None, + 'monthPeakUpperKw':hi if complete else None, + 'verifiedMonthPeakKw':hi if certified else None, + 'firstMissingQuarters':gaps,'quarters':quarters} + + def strict_contract(self, at): + """Fail closed; do not promote archive gaps, estimated boundaries or a cap.""" + at = utc(at) + if not self.native_bound: + raise ValueError('Explicit native counter identity required') + result = self.month(at) + if not result['billingEvidence']: + raise ValueError('Full verified month history missing') + q = quarter_start(at) + past = None + if at != q: + partial = self.interval(q,at) + if not partial['billingEvidence']: + raise ValueError('Verified elapsed-quarter energy missing') + past = {'start':q.isoformat(),'measuredSeconds':int((at-q).total_seconds()), + 'importKwh':partial['upperKwh']} + return {'version':1,'meterId':self.meter_id,'measuredAt':at.isoformat(), + 'measuredPeaks':{month_key(at):{'kw':result['verifiedMonthPeakKw'], + 'source':'verified_month_history'}}, + 'quarterPast':past} + + +def audit_capture(payload): + """Read-only report from the dedicated Symcon export, never planner ingestion.""" + if payload.get('schemaVersion') != 1 or payload.get('kind') != 'v4_meter_capture': + raise ValueError('Unknown capture contract') + now = utc(payload['capturedAt']) + channels = payload['channels'] + series = [] + summaries = [] + for variable in ('59607','26620'): + channel = channels[variable] + rows = channel.get('history',[]) + readings = [Reading(utc(datetime.fromtimestamp(r['TimeStamp'],now.tzinfo)),r['Value']) for r in rows] + # An explicitly captured current value is usable from its capture, not + # from an earlier last-change timestamp, and never before logging began. + current = channel.get('snapshot') + if current and channel.get('snapshotConsistent'): + readings.append(Reading(utc(current['capturedAt']),current['value'])) + s = CounterSeries('symcon:'+variable, readings, timestamp_verified=False, + identity_verified=False, chronology_verified=False) + series.append(s) + summaries.append({'variableId':int(variable),'ident':channel['ident'], + 'logging':channel.get('logging'), 'rows':len(rows), + 'queryComplete':channel.get('queryComplete',False), + 'firstRecord':s.times[0].isoformat() if s.times else None, + 'lastRecord':s.times[-1].isoformat() if s.times else None, + 'counterDecreases':[t.isoformat() for t in s.resets]}) + meter = MeterEvidence(series) + month = meter.month(now) + current = None + q = quarter_start(now) + if now>q:current=meter.interval(q,now) + blockers=['Meter register identity/unit and acquisition timestamps still require verification.', + 'Change-only archive does not prove uninterrupted meter acquisition.'] + if any(not s['queryComplete'] for s in summaries):blockers.append('Archive export incomplete; do not infer complete history.') + if not month['historyComplete']:blockers.append('At least one completed billing quarter lacks bounded readings for every tariff.') + return {'capturedAt':now.isoformat(),'status':'audit_only','sourceSummary':summaries, + 'month':{k:v for k,v in month.items() if k!='quarters'}, + 'recentQuarters':month['quarters'][-8:], 'currentQuarter':current, + 'billingEvidence':False, 'liveEnabled':False,'blockers':blockers} diff --git a/services/netplan-v4/netplan_v4/optimizer.py b/services/netplan-v4/netplan_v4/optimizer.py new file mode 100644 index 0000000..8dabed5 --- /dev/null +++ b/services/netplan-v4/netplan_v4/optimizer.py @@ -0,0 +1,199 @@ +from __future__ import annotations +from math import sqrt +from uuid import uuid4 +import numpy as np +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import coo_matrix +from .domain import Battery, Limits, Step, month_key, number, quarter_start, utc +from .peak_policy import basis_record, RestMonthOutlook + +class Model: + def __init__(self): + self.lower,self.upper,self.cost,self.integer=[],[],[],[] + self.rows,self.row_lo,self.row_hi=[],[],[] + def variable(self, lower=0., upper=np.inf, cost=0., integer=False): + index=len(self.lower) + self.lower.append(lower);self.upper.append(upper);self.cost.append(cost);self.integer.append(int(integer)) + return index + def constraint(self, coefficients, lower=-np.inf, upper=np.inf): + self.rows.append(coefficients);self.row_lo.append(lower);self.row_hi.append(upper) + def matrices(self): + rr,cc,vv=[],[],[] + for row,coefficients in enumerate(self.rows): + for col,value in coefficients.items(): + rr.append(row);cc.append(col);vv.append(value) + matrix=coo_matrix((vv,(rr,cc)),shape=(len(self.rows),len(self.lower))).tocsr() + return matrix,np.array(self.row_lo),np.array(self.row_hi) + +from .battery_model import BatteryModel + +class AssetRegistry: + """Reviewed asset classes only; future EV/thermal models add their own constraints.""" + def __init__(self):self.models={Battery:BatteryModel} + def register(self,asset_type,implementation): + if asset_type in self.models:raise ValueError('Asset model already registered') + self.models[asset_type]=implementation + def build(self,model,asset,steps,direction): + if type(asset) not in self.models:raise ValueError('Unsupported asset model') + return self.models[type(asset)].build(model,asset,steps,direction) + +def _error(reason,status='invalid_inputs'): + return {'schemaVersion':2,'status':status,'executable':False,'points':[],'reason':str(reason)} + +def _validate(steps,assets,at,past,observed_peaks,peak_prices,limits): + if not steps or len(steps)>576:raise ValueError('Need 1..576 steps') + for i,step in enumerate(steps): + start=utc(step.start) + if type(step.seconds) is not int or not 1<=step.seconds<=300:raise ValueError('Interval duration must be 1..300 seconds') + if start.microsecond or step.end.second or step.end.microsecond or step.end.minute%5:raise ValueError('Intervals must end on a 5-minute boundary') + if i and (step.seconds!=300 or start.second or start.minute%5):raise ValueError('Only first interval may be shortened') + if i and start!=steps[i-1].end:raise ValueError('Missing/duplicated/overlapping interval') + number(step.base_load_w,'base load',0);number(step.pv_w,'PV',0);number(step.external_w,'external flow') + if not step.import_price or not step.export_price or not step.import_price.known_at(at) or not step.export_price.known_at(at):raise ValueError('Unknown or unpublished prices') + limits.import_limit(start) + if utc(at)!=utc(steps[0].start):raise ValueError('First interval must start at decision time') + if steps[-1].end!=quarter_start(steps[-1].end):raise ValueError('End horizon on complete billing quarter') + if limits.export_w is not None:number(limits.export_w,'export limit',0) + ids=set() + for asset in assets: + asset.validate(at) + if asset.asset_id in ids:raise ValueError('Duplicate asset_id') + ids.add(asset.asset_id) + q=quarter_start(steps[0].start);elapsed=int((utc(steps[0].start)-q).total_seconds()) + if any(key!=q for key in past):raise ValueError('Only elapsed energy of first quarter permitted') + if not elapsed and q in past and (past[q].import_kwh!=0 or past[q].measured_seconds!=0):raise ValueError('No elapsed energy at quarter boundary') + if elapsed: + if q not in past or past[q].measured_seconds!=elapsed:raise ValueError('Actual elapsed quarter import energy missing') + number(past[q].import_kwh,'quarter import energy',0) + for month in {month_key(s.start) for s in steps}: + if month not in observed_peaks or month not in peak_prices:raise ValueError(f'Measured peak state or tariff missing for {month}') + number(observed_peaks[month],'measured peak',0);number(peak_prices[month],'peak tariff',0) + +def optimize(steps,batteries,*,at,limits=None,observed_peaks=None,peak_prices=None,quarter_history=None,config_revision=1,family='3',timeout_seconds=30.,asset_registry=None,peak_context=None,peak_outlooks=None): + """Pure MILP. Peak state is measured, never a configured cap. No device calls.""" + limits=limits or Limits();observed_peaks=observed_peaks or {};peak_prices=peak_prices or {} + past={utc(k):v for k,v in (quarter_history or {}).items()} + try: + _validate(steps,batteries,at,past,observed_peaks,peak_prices,limits) + number(timeout_seconds,'solver timeout',.01,600) + contexts={} + for month in {month_key(s.start) for s in steps}: + if peak_context is None: + contexts[month]={'kw':observed_peaks[month],'quality':'verified','source':'legacy_verified_contract','observedAt':utc(at).isoformat()} + else: + contexts[month]=basis_record(peak_context[month],month,at,allow_estimates=True) + if abs(contexts[month]['kw']-observed_peaks[month])>1e-8:raise ValueError('Peak context differs from numerical basis') + outlooks=peak_outlooks or {} + if set(outlooks)-set(contexts):raise ValueError('Outlook for month outside horizon') + for month,outlook in outlooks.items(): + if not isinstance(outlook,RestMonthOutlook) or outlook.month!=month:raise ValueError('Invalid peak outlook') + outlook.validate(at,steps[-1].end) + except (ValueError,TypeError,KeyError,AttributeError) as exc:return _error(exc) + model=Model();direction=[model.variable(0,1,integer=True) for _ in steps];registry=asset_registry or AssetRegistry() + try:handles={b.asset_id:registry.build(model,b,steps,direction) for b in batteries} + except ValueError as exc:return _error(exc) + total_charge=sum(b.max_charge_w for b in batteries)/1000 + total_discharge=sum(b.max_discharge_w for b in batteries)/1000 + imp,exp,curtail=[],[],[];quarters={} + for i,step in enumerate(steps): + residual=step.residual_w/1000;dt=step.seconds/3600 + imax=max(0.,(step.base_load_w+step.external_w)/1000)+total_charge + emax=max(0.,-residual)+total_discharge;limit=limits.import_limit(step.start) + if limit is not None:imax=min(imax,limit/1000) + if limits.export_w is not None:emax=min(emax,limits.export_w/1000) + pi=model.variable(0,imax,step.import_price.chf_kwh*dt) + pe=model.variable(0,emax,-step.export_price.chf_kwh*dt) + pc=model.variable(0,step.pv_w/1000) + imp.append(pi);exp.append(pe);curtail.append(pc) + gm=model.variable(0,1,integer=True) + model.constraint({pi:1,gm:-imax},upper=0);model.constraint({pe:1,gm:emax},upper=emax) + balance={pi:1,pe:-1,pc:-1};pv_only={} + for b in batteries: + h=handles[b.asset_id];balance[h['charge'][i]]=-1;balance[h['discharge'][i]]=1 + if not b.grid_charging:pv_only[h['charge'][i]]=1 + model.constraint(balance,residual,residual) + if pv_only: + model.constraint(pv_only,upper=max(0.,-residual)) + no_grid_max=sum(b.max_charge_w for b in batteries if not b.grid_charging)/1000 + model.constraint({**pv_only,gm:no_grid_max},upper=no_grid_max) + quarters.setdefault(quarter_start(step.start),[]).append(i) + months=sorted({month_key(s.start) for s in steps}) + peaks={m:model.variable(observed_peaks[m]) for m in months} + for m in months: + if m in outlooks:outlooks[m].add_to_model(model,peaks[m],observed_peaks[m],peak_prices[m]) + else:model.cost[peaks[m]]=peak_prices[m] + for quarter,positions in quarters.items(): + coefficients={imp[i]:steps[i].seconds/900 for i in positions};coefficients[peaks[month_key(quarter)]]=-1 + used=past[quarter].import_kwh if quarter in past else 0. + model.constraint(coefficients,upper=-used/.25) + matrix,row_lo,row_hi=model.matrices() + try: + result=milp(np.array(model.cost),integrality=np.array(model.integer),bounds=Bounds(model.lower,model.upper),constraints=LinearConstraint(matrix,row_lo,row_hi),options={'time_limit':float(timeout_seconds),'mip_rel_gap':1e-4}) + except Exception as exc:return _error(f'Solver exception: {type(exc).__name__}','solver_error') + x=result.x + if result.status not in (0,1) or x is None:return _error(result.message,'no_feasible_plan') + if not np.isfinite(x).all():return _error('Non-finite solver result','validation_failed') + ax=matrix@x;tol=1e-6 + if (np.any(xnp.array(model.upper)+tol) or np.any(axrow_hi+tol) or any(abs(x[i]-round(x[i]))>tol for i,flag in enumerate(model.integer) if flag)): + return _error('Solver incumbent violates constraints','validation_failed') + running_peaks=dict(observed_peaks);baseline_peaks=dict(observed_peaks);peak_deltas={};baseline_peak_deltas={} + for quarter,positions in quarters.items(): + used=past[quarter].import_kwh if quarter in past else 0.;m=month_key(quarter) + demand=(used+sum(x[imp[i]]*steps[i].seconds/3600 for i in positions))/.25 + baseline=(used+sum(max(0.,steps[i].residual_w)*steps[i].seconds/3600000 for i in positions))/.25 + before=running_peaks[m];running_peaks[m]=max(before,demand);peak_deltas[positions[-1]]=(running_peaks[m]-before)*peak_prices[m] + before=baseline_peaks[m];baseline_peaks[m]=max(before,baseline);baseline_peak_deltas[positions[-1]]=(baseline_peaks[m]-before)*peak_prices[m] + points=[];cumulative_energy=cumulative_cash=cumulative_baseline_energy=cumulative_baseline_cash=cumulative_throughput=0. + for i,step in enumerate(steps): + dt=step.seconds/3600;p_import=max(0.,float(x[imp[i]]));p_export=max(0.,float(x[exp[i]])) + energy_cost=(p_import*step.import_price.chf_kwh-p_export*step.export_price.chf_kwh)*dt + baseline_import=max(0.,step.residual_w)/1000;baseline_export=max(0.,-step.residual_w)/1000 + if limits.export_w is not None:baseline_export=min(baseline_export,limits.export_w/1000) + baseline_cost=(baseline_import*step.import_price.chf_kwh-baseline_export*step.export_price.chf_kwh)*dt + targets,soc_end={},{};throughput_cost=0. + for b in batteries: + h=handles[b.asset_id];targets[b.asset_id]=float((x[h['charge'][i]]-x[h['discharge'][i]])*1000) + soc_end[b.asset_id]=float(100*x[h['energy'][i+1]]/b.capacity_kwh) + throughput_cost+=float((x[h['charge'][i]]+x[h['discharge'][i]])*dt*b.throughput_chf_kwh) + cumulative_energy+=energy_cost;cumulative_cash+=energy_cost+peak_deltas.get(i,0.) + cumulative_baseline_energy+=baseline_cost;cumulative_baseline_cash+=baseline_cost+baseline_peak_deltas.get(i,0.) + cumulative_throughput+=throughput_cost;battery_w=sum(targets.values()) + if battery_w>1:intent='gridCharge' if p_import>.001 else 'pvCharge' + elif battery_w< -1:intent='export' if p_export>.001 else 'discharge' + else:intent='hold' + points.append({'time':utc(step.start).isoformat(),'validUntil':step.end.isoformat(), + 'baselineGridW':float(step.residual_w),'gridTargetW':(p_import-p_export)*1000, + 'batteryTargetW':battery_w,'assetTargetsW':targets,'socEndPercent':soc_end, + 'pvCurtailmentW':float(x[curtail[i]]*1000),'intent':intent, + 'importLimitW':limits.import_limit(step.start),'exportLimitW':limits.export_w, + 'importPriceChfKwh':step.import_price.chf_kwh,'exportPriceChfKwh':step.export_price.chf_kwh, + 'energyCostChf':energy_cost,'additionalPeakCostChf':peak_deltas.get(i,0.), + 'throughputCostChf':throughput_cost,'cumulativeEnergyCostChf':cumulative_energy, + 'cumulativeCashCostChf':cumulative_cash,'cumulativeBaselineEnergyCostChf':cumulative_baseline_energy, + 'cumulativeBaselineCashCostChf':cumulative_baseline_cash,'baselineAdditionalPeakCostChf':baseline_peak_deltas.get(i,0.)}) + end_value=sum(float(x[handles[b.asset_id]['energy'][-1]])*b.terminal_value_chf_kwh for b in batteries) + def planning_cost(chosen): + return sum(outlooks[m].incremental_cost(observed_peaks[m],chosen[m],peak_prices[m]) if m in outlooks + else max(0.,chosen[m]-observed_peaks[m])*peak_prices[m] for m in months) + planning_peak=planning_cost(running_peaks) + full_peak=cumulative_cash-cumulative_energy + estimated=any(c['quality']=='estimated' for c in contexts.values()) + return {'schemaVersion':2,'planId':str(uuid4()),'configRevision':config_revision,'sourceFamily':family, + 'status':'optimal' if result.status==0 else 'feasible_time_limit','executable':True, + 'generatedAt':utc(at).isoformat(),'validFrom':points[0]['time'],'validUntil':points[-1]['validUntil'], + 'intervalMinutes':5,'points':points,'solverGap':float(result.mip_gap) if getattr(result,'mip_gap',None) is not None else None, + 'measuredPeaksKw':{m:observed_peaks[m] for m in months if contexts[m]['quality']=='verified'}, + 'peakBasisKw':{m:observed_peaks[m] for m in months},'peakBasis':contexts, + 'peakCostIsEstimate':estimated,'planningPeakCostChf':planning_peak, + 'restMonthAdjustmentChf':planning_peak-full_peak, + 'peakScenarioOutlook':{m:o.as_dict() for m,o in outlooks.items()}, + 'plannedPeaksKw':{m:running_peaks[m] for m in months}, + 'additionalPeakCostChf':cumulative_cash-cumulative_energy,'energyCostChf':cumulative_energy,'cashCostChf':cumulative_cash, + 'baselineEnergyCostChf':cumulative_baseline_energy,'baselineCashCostChf':cumulative_baseline_cash, + 'baselineAdditionalPeakCostChf':sum(baseline_peak_deltas.values()),'baselinePeaksKw':baseline_peaks, + 'throughputCostChf':cumulative_throughput,'terminalValueChf':end_value, + 'objectiveChf':cumulative_energy+planning_peak+cumulative_throughput-end_value, + 'baselinePlanningPeakCostChf':planning_cost(baseline_peaks), + 'cashCostMeaning':'Projected horizon energy plus full incremental monthly tariff, relative to the labelled peak basis; not an invoice', + 'terminalMinSocPercent':{b.asset_id:handles[b.asset_id]['terminal_min_percent'] for b in batteries}, + 'peakOutlook':'rest_month_scenarios' if outlooks else 'full_incremental_tariff'} diff --git a/services/netplan-v4/netplan_v4/peak_policy.py b/services/netplan-v4/netplan_v4/peak_policy.py new file mode 100644 index 0000000..30dad71 --- /dev/null +++ b/services/netplan-v4/netplan_v4/peak_policy.py @@ -0,0 +1,173 @@ +"""Economic peak policy. Planning assumptions NEVER become metering facts. + +Monthly marginal cost is convex. A rest-month scenario is a peak expected after +this horizon under a comparable control policy, not a configured limit, free +allowance, or a promise of savings. With insufficient evidence use full tariff. +""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime, timedelta +from math import isclose +from .domain import number, utc, month_key, ZURICH + +VERIFIED_SOURCES = frozenset({'meter_month_register','verified_month_history','verified_new_month'}) +ESTIMATE_SOURCES = frozenset({'power_history_estimate','sampled_power_estimate','counter_interval_estimate','operator_estimate','new_month'}) + + +def valid_month(value): + if not isinstance(value,str) or len(value)!=7: + raise ValueError('Calendar month YYYY-MM required') + if datetime.strptime(value,'%Y-%m').strftime('%Y-%m') != value: + raise ValueError('Invalid calendar month') + return value + + +def basis_record(value, month, at, allow_estimates=False): + """Strict provenance; missing stays missing. Caller decides explicit opt-in.""" + valid_month(month) + if not isinstance(value,dict) or set(value)-{'kw','quality','source','observedAt','notes','coverage','meterId'}: + raise ValueError('Invalid peak-basis schema') + kw=number(value.get('kw'),'peak basis kW',0,1e6) + quality=value.get('quality') + source=value.get('source') + if quality=='verified': + if source not in VERIFIED_SOURCES:raise ValueError('Not a verified peak source') + elif quality=='estimated': + if not allow_estimates or source not in ESTIMATE_SOURCES: + raise ValueError('Estimated planning basis not permitted or source invalid') + elif quality=='new_month': + if source!='new_month' or kw!=0 or month<=month_key(at): + raise ValueError('Zero future-month state is not a past measured peak') + else:raise ValueError('Peak quality must be explicit') + stamp=utc(value.get('observedAt')) + if stamp>utc(at):raise ValueError('Peak evidence from the future') + if quality!='new_month' and month>month_key(at): + raise ValueError('Future peak is an outlook, not historical evidence') + notes=value.get('notes','') + if not isinstance(notes,str) or len(notes)>500:raise ValueError('Invalid peak notes') + coverage=value.get('coverage') + if coverage is not None:number(coverage,'peak coverage',0,1) + mid=value.get('meterId') + if mid is not None and (not isinstance(mid,str) or not 1<=len(mid)<=160):raise ValueError('Invalid meter identity') + return {**value,'kw':kw,'quality':quality,'source':source,'observedAt':stamp.isoformat()} + + +@dataclass(frozen=True) +class PeakScenario: + future_peak_kw: float + probability: float + + +@dataclass(frozen=True) +class RestMonthOutlook: + month: str + scenarios: tuple[PeakScenario, ...] + issued_at: datetime + future_from: datetime + history_until: datetime + control_policy_id: str + method: str + evidence_id: str + reliance: float = 0.5 + + def validate(self, at, horizon_end): + valid_month(self.month) + at=utc(at);end=utc(horizon_end) + if utc(self.issued_at)>at or utc(self.history_until)>utc(self.issued_at): + raise ValueError('Rest-month outlook contains future information') + if at-utc(self.issued_at)>timedelta(days=2):raise ValueError('Stale rest-month outlook') + if utc(self.future_from)=end:return None + good={} + for r in daily_records: + if r.get('controlPolicyId')!=control_policy_id or r.get('complete') is not True:continue + if r.get('quality') not in ('verified','estimated'):continue + day=datetime.strptime(r['day'],'%Y-%m-%d').replace(tzinfo=ZURICH) + finished=utc(day+timedelta(days=1)) + observed=utc(r['observedAt']) + if finished>utc(at) or observed>utc(at) or observedtimedelta(days=90):continue + kw=number(r['peakKw'],'historical daily peak',0,1e6) + if r['day'] in good and good[r['day']]!=kw:raise ValueError('Conflicting daily peak evidence') + good[r['day']]=kw + if len(good)len(longest):longest=segment + segment=[] + segment.append(day) + if len(segment)>len(longest):longest=segment + if len(longest)=utc(self.last_decision) and self.terminal_normalized and self.constraint_breaches==0) + +def choose_family(setting,current,scores,*,registry=None,lookback_days=14,minimum_days=7,minimum_coverage=.9,margin_chf=1.,now): + registry=registry or default_registry() + if setting!='auto': + registry.get(setting) + return {'family':setting,'mode':'configured','reason':'Explicit installation setting'} + registry.get(current);number(margin_chf,'margin',0);valid=[] + for score in scores: + registry.get(score.family) + if (score.valid() and score.coverage>=minimum_coverage and score.days>=minimum_days + and utc(score.first_decision)>=utc(now)-timedelta(days=lookback_days) + and utc(score.last_decision)<=utc(now) and utc(score.actual_available_at)<=utc(now)): + valid.append(score) + groups={} + for score in valid: + key=(score.comparison_key,utc(score.first_decision),utc(score.last_decision),score.days) + groups.setdefault(key,{})[score.family]=score + complete=[g for g in groups.values() if len(g)==len(registry.entries())] + if not complete:return {'family':current,'mode':'collecting','reason':'Insufficient comparable out-of-sample replay evidence'} + group=max(complete,key=lambda g:utc(next(iter(g.values())).last_decision)) + best=min(group,key=lambda f:(group[f].cost_chf,f));improvement=group[current].cost_chf-group[best].cost_chf + chosen=best if improvement>margin_chf else current + return {'family':chosen,'mode':'economic_replay','improvementChf':improvement,'reason':'Matched historical cost replay; switching margin applied','costByFamilyChf':{k:v.cost_chf for k,v in group.items()}} + +def training_due(last_trained_at,now,cadence='daily'): + if cadence not in ('daily','weekly'):raise ValueError('Unknown training cadence') + if last_trained_at is None:return True + now,last=utc(now).astimezone(ZURICH),utc(last_trained_at).astimezone(ZURICH) + return (now.date()-last.date()).days >= (1 if cadence=='daily' else 7) + +def promote_candidate(*,active_cost,candidate_cost,valid_coverage,no_data_leakage,constraints_passed): + number(active_cost,'active cost');number(candidate_cost,'candidate cost') + return bool(valid_coverage and no_data_leakage and constraints_passed and candidate_cost20 or len({b.asset_id for b in result})!=len(result):raise ValueError('Duplicate/too many batteries') + return result + +def validate(kind,value,now,registry): + if kind not in KINDS or type(value.get('version')) is not int or value['version']!=1:raise ValueError('Unsupported event version/kind') + identifier=value['eventId'] + if not isinstance(identifier,str) or not 1<=len(identifier)<=160:raise ValueError('Event ID required') + observed=utc(value['observedAt']) + if observed>now+timedelta(seconds=30):raise ValueError('Observation from future') + fields={'version','eventId','observedAt'} + if kind=='forecast': + fields|={'families','modelVersions'} + if not value['families']:raise ValueError('No forecast families') + for key,family in value['families'].items(): + registry.get(key) + if family['loadBasis'] not in ('base_load','house_total'):raise ValueError('Explicit metering basis required') + evidence=family.get('accountingEvidenceId') + if evidence is not None and (family['loadBasis']!='base_load' or not isinstance(evidence,str) or not 8<=len(evidence)<=160):raise ValueError('Explicit base-load accounting evidence required') + if family.get('trainedUntil') and utc(family['trainedUntil'])>observed:raise ValueError('Training leakage') + if not 1<=len(family['points'])<=576:raise ValueError('Need 1..576 forecast intervals') + previous=None + for p in family['points']: + t=utc(p['time']) + if t.second or t.microsecond or t.minute%5:raise ValueError('Forecast interval alignment') + if previous and t-previous!=timedelta(minutes=5):raise ValueError('Forecast gap/overlap') + previous=t;number(p['pvW'],'PV',0,1e9);number(p['loadW'],'load',0,1e9);number(p.get('externalW',0.),'external',-1e9,1e9) + elif kind=='operation': + fields|={'gridW','meteringBoundary','batteries','limits','measuredPeaks','quarterPast','planningPeaks','quarterEstimate','meterObservation'} + if value['meteringBoundary']!='common_pcc':raise ValueError('Common metering boundary required') + number(value['gridW'],'grid W',-1e9,1e9) + for b in batteries(value['batteries']):b.validate(observed) + limits=value['limits'] + if set(limits)!={'importW','exportW','managerMonthLimitsW'}:raise ValueError('Explicit limits required; null unlimited, zero zero') + for k in ('importW','exportW'): + if limits[k] is not None:number(limits[k],k,0,1e9) + for k,v in limits['managerMonthLimitsW'].items(): + if str(int(k))!=k or not 1<=int(k)<=12:raise ValueError('Invalid manager month') + number(v,'manager limit',0,1e9) + for m,p in value.get('measuredPeaks',{}).items(): + datetime.strptime(m,'%Y-%m');number(p['kw'],'peak kW',0) + if m>month_key(observed) or p['source'] not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Measured peak source invalid; cap is not paid peak') + past=value.get('quarterPast') + if past: + q=quarter_start(observed) + if utc(past['start'])!=q or type(past['measuredSeconds']) is not int or past['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Quarter measurement timestamp mismatch') + number(past['importKwh'],'quarter energy',0) + for m,p in value.get('planningPeaks',{}).items(): + record=basis_record(p,m,observed,allow_estimates=True) + if record['quality']!='estimated':raise ValueError('planningPeaks contains estimates only') + estimate=value.get('quarterEstimate') + if estimate: + if set(estimate)-{'start','measuredSeconds','importKwh','quality','source','coverage','notes'}:raise ValueError('Unknown quarter estimate field') + if estimate.get('quality')!='estimated' or estimate.get('source') not in ('power_history_estimate','sampled_power_estimate','counter_interval_estimate'):raise ValueError('Explicit quarter estimate provenance required') + q=quarter_start(observed) + if utc(estimate['start'])!=q or type(estimate['measuredSeconds']) is not int or estimate['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Estimated quarter timing mismatch') + number(estimate['importKwh'],'estimated quarter energy',0) + if number(estimate.get('coverage'),'quarter estimate coverage',0,1)<1.:raise ValueError('Missing current-quarter coverage') + if value.get('meterObservation') is not None:meter_runtime.validate_observation(value['meterObservation'],observed) + elif kind=='planning_basis': + fields|={'peaks'} + if not isinstance(value.get('peaks'),dict) or not value['peaks']:raise ValueError('Explicit peak estimates required') + for m,p in value['peaks'].items(): + if basis_record(p,m,observed,allow_estimates=True)['quality']!='estimated':raise ValueError('Planning basis is not a metering import') + elif kind=='peak_outlook': + fields|={'outlooks'} + for m,v in value['outlooks'].items(): + outlook=RestMonthOutlook.from_dict(v) + if m!=outlook.month:raise ValueError('Outlook month mismatch') + outlook.validate(observed,observed) + elif kind=='tariffs': + fields|={'import','export','peakChfKwMonth'} + for side in ('import','export'): + p=value[side] + if p['mode'] not in ('static','dynamic') or not p['tariffId']:raise ValueError('Explicit price mode/id required') + if p['mode']=='static':number(p['staticChfKwh'],'static price') + peaks=value['peakChfKwMonth'] + if isinstance(peaks,dict): + for m,v in peaks.items():datetime.strptime(m,'%Y-%m');number(v,'peak tariff',0) + else:number(peaks,'peak tariff',0) + else: + fields|={'periods'} + if len(value['periods'])>3000:raise ValueError('Too many price intervals') + for p in value['periods']: + if p['unit'] not in ('CHF/kWh','CHF_kWh','Rp/kWh','CHF/MWh') or p['side'] not in ('import','export'):raise ValueError('Explicit price unit/direction required') + if p['sourceKind'] not in ('published_interval','estimate','carried_forward'):raise ValueError('Explicit price provenance required') + number(p['value'],'price') + if utc(p['end'])<=utc(p['start']) or utc(p['observedAt'])>observed:raise ValueError('Invalid price interval or observation') + if p.get('publishedAt') and utc(p['publishedAt'])>utc(p['observedAt']):raise ValueError('Price not published when observed') + if set(value)-fields:raise ValueError('Unknown fields: extra device data/credentials must not be submitted') + canonical(value) + +def ingest(store,plant,kind,value,now): + validate(kind,value,now,store.registry);data=canonical(value);con=store.con;con.execute('BEGIN IMMEDIATE') + try: + old=con.execute('SELECT value FROM planner_inputs WHERE plant=? AND kind=? AND event_id=?',(plant,kind,value['eventId'])).fetchone() + if old: + if old[0]!=data:raise ValueError('Immutable event conflict') + con.commit();return {'status':'duplicate','queued':False} + current=latest(store,plant,kind) + if current and utc(current['observedAt'])==utc(value['observedAt']): + left,right=dict(current),dict(value);left.pop('eventId');right.pop('eventId') + if canonical(left)!=canonical(right):raise ValueError('Conflicting simultaneous observations') + con.execute('INSERT INTO planner_inputs VALUES(?,?,?,?,?)',(plant,kind,value['eventId'],utc(value['observedAt']).isoformat(timespec='microseconds'),data)) + newer=not current or utc(current['observedAt'])<=utc(value['observedAt']) + if newer: + if kind=='operation': + known=store.peaks(plant) + for m,p in value.get('measuredPeaks',{}).items(): + if m in known and p['kw']=end and utc(p['observedAt'])<=now): + factor={'CHF/kWh':1.,'CHF_kWh':1.,'Rp/kWh':.01,'CHF/MWh':.001}[p['unit']] + candidates.append((utc(p['observedAt']),p['value']*factor)) + if not candidates:return None + last=max(t for t,v in candidates);values={v for t,v in candidates if t==last} + if len(values)!=1:raise ValueError('Conflicting published price intervals') + return Price(values.pop(),last,'dynamic') + +class AwaitingInput(ValueError):pass + +def assemble(store,plant,family,now): + values={k:latest(store,plant,k) for k in ('operation','forecast','tariffs')} + for k,v in values.items(): + if not v:raise AwaitingInput('Missing '+k+' input') + op,forecast,tariffs=(values[k] for k in ('operation','forecast','tariffs')) + decision=utc(op['observedAt']).replace(microsecond=0) + settings=store.settings(plant);allow_estimates=settings['measurementPolicy']=='allow_estimates' + input_quality={'quarter':'verified','warnings':[]} + if not 0<=(now-decision).total_seconds()<=120:raise AwaitingInput('Fresh manager observation required (120s maximum)') + if (now-utc(forecast['observedAt'])).total_seconds()>5400:raise AwaitingInput('Forecast older than 90 minutes') + if utc(forecast['observedAt'])>decision or utc(tariffs['observedAt'])>decision:raise AwaitingInput('New data awaiting fresh manager observation') + if settings['forecastSource']=='corrected_profile': + try:forecast=measurement_pipeline.apply_load_forecast(store.con,plant,settings['measurementDataset'],forecast,int(decision.timestamp())) + except ValueError as exc:raise AwaitingInput(str(exc)) from exc + if family not in forecast['families']:raise AwaitingInput('Chosen family is unavailable; no silent switch') + source=forecast['families'][family];steps=[] + future_source={**source,'points':[p for p in source['points'] if utc(p['time'])+timedelta(minutes=5)>decision]} + assessment=assess_family(future_source) + if not assessment['valid']:raise AwaitingInput(assessment['reason']) + input_quality['forecastAssessment']=assessment + if source.get('dataPipeline'): + input_quality['dataPipeline']=source['dataPipeline'] + input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule') + for p in source['points']: + start=utc(p['time']);end=start+timedelta(minutes=5) + if end<=decision:continue + start=max(start,decision) + external=p.get('externalW',0.) if source['loadBasis']=='base_load' else 0. + steps.append(Step(start,p['loadW'],p['pvW'],price_at(store,plant,'import',tariffs['import'],start,end,decision),price_at(store,plant,'export',tariffs['export'],start,end,decision),external,int((end-start).total_seconds()))) + if not steps or steps[0].start!=decision:raise AwaitingInput('Forecast has no current interval') + full_end=steps[-1].end;steps=priced_prefix(steps,decision) + if not steps:raise AwaitingInput('No complete published-price billing quarter') + q=quarter_start(decision);elapsed=int((decision-q).total_seconds());past={} + if elapsed: + p=op.get('quarterPast') + if not p and allow_estimates: + p=op.get('quarterEstimate') + if not p and op.get('meterObservation'):p=meter_runtime.current_quarter(store.con,plant,op['meterObservation'],decision) + if p: + input_quality['quarter']='estimated' + input_quality['warnings'].append('Current quarter uses an explicitly estimated energy value, not a billing measurement') + if not p or utc(p['start'])!=q or p['measuredSeconds']!=elapsed:raise AwaitingInput('Current-quarter energy missing (measured or explicitly permitted estimate)') + past[q]=QuarterPast(p['importKwh'],elapsed) + peaks=store.peaks(plant);months={month_key(s.start) for s in steps};contexts={} + estimates=meter_runtime.assumptions(store.con,plant) + for m in months: + if m>month_key(decision): + peaks[m]=0.;contexts[m]={'kw':0.,'quality':'new_month','source':'new_month','observedAt':decision.isoformat()} + elif m in peaks: + r=store.con.execute('SELECT source,updated_at FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone() + known_at=op['observedAt'] if m in op.get('measuredPeaks',{}) else r['updated_at'] + contexts[m]={'kw':peaks[m],'quality':'verified','source':r['source'],'observedAt':known_at} + # A later acquired larger quarter may raise an older verified baseline, + # but the resulting combined planning basis must then say estimated. + if allow_estimates and m in estimates and estimates[m]['kw']>peaks[m] and utc(estimates[m]['observedAt'])>utc(known_at): + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True) + peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('A newer sampled quarter increased the earlier verified peak baseline; current planning maximum is estimated') + elif allow_estimates and m in estimates: + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True);peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('Monthly peak '+m+' is a planning estimate, not an authoritative billing maximum') + else:raise AwaitingInput('Peak basis missing; supply a verified maximum or explicitly permit a labelled estimate') + peak_prices=tariffs['peakChfKwMonth'] + if not isinstance(peak_prices,dict):peak_prices={m:peak_prices for m in months} + lim=op['limits'];limits=Limits(lim['exportW'],lim['importW'],{int(k):v for k,v in lim['managerMonthLimitsW'].items()}) + outlooks={};horizon_end=steps[-1].end + if settings['peakOutlookPolicy']=='empirical_if_available': + explicit=latest(store,plant,'peak_outlook') + for m in months: + if explicit and m in explicit['outlooks']: + candidate=RestMonthOutlook.from_dict(explicit['outlooks'][m]) + try: + candidate.validate(decision,horizon_end) + policy=op.get('meterObservation',{}).get('controlPolicyId') + if not policy or candidate.control_policy_id!=policy:raise ValueError('Different or unknown control policy') + except ValueError:input_quality['warnings'].append('Stale, overlapping or incomparable rest-month outlook ignored; full incremental tariff used') + else:outlooks[m]=replace(candidate,reliance=min(candidate.reliance,settings['peakOutlookReliance'])) + elif op.get('meterObservation'): + observation=op['meterObservation'] + candidate=empirical_rest_month(meter_runtime.daily_peaks(store.con,plant,observation['meterId'],decision), + month=m,at=decision,horizon_end=horizon_end,control_policy_id=observation['controlPolicyId'], + reliance=settings['peakOutlookReliance']) + if candidate is not None:outlooks[m]=candidate + if not outlooks:input_quality['warnings'].append('Insufficient comparable rest-month history; full incremental peak tariff used, no arbitrary free peak allowance') + data={'steps':steps,'batteries':batteries(op['batteries']),'limits':limits,'observed_peaks':peaks,'peak_prices':peak_prices, + 'quarter_history':past,'at':decision,'peak_context':contexts,'peak_outlooks':outlooks} + if source['loadBasis']=='base_load' and source.get('accountingEvidenceId'): + input_quality['accountingEvidenceId']=source['accountingEvidenceId'] + return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)} + +def run_once(store,now): + stamp=int(now.timestamp())//300 + plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')] + for plant in plants: + config=store.settings(plant) + datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))] + for dataset in datasets: + try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp())) + except ValueError as exc:logging.getLogger(__name__).warning('Data pipeline unavailable for configured dataset: %s',type(exc).__name__) + with store.con: + old=store.con.execute('SELECT tick FROM planner_ticks WHERE plant=?',(plant,)).fetchone() + if not old or old[0]!=stamp: + store._request(plant,store.settings(plant)['revision'],'five_minute_tick',now) + store.con.execute('INSERT INTO planner_ticks VALUES(?,?) ON CONFLICT(plant) DO UPDATE SET tick=excluded.tick',(plant,stamp)) + claim=store.claim(now) + if not claim:return {'status':'idle'} + plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]} + try: + settings=store.settings(plant);previous=store.current(plant) + current=previous['sourceFamily'] if previous else store.registry.entries()[0].key + # Productive replay ingestion is intentionally not fabricated from R2 metrics. + selection=choose_family(settings['family'],current,(),registry=store.registry,now=now) + data,full_end,quality=assemble(store,plant,selection['family'],now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + result=optimize(**data,config_revision=settings['revision'],family=selection['family']) + if result['executable']: + result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])}) + store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token']) + return result + except (ValueError,TypeError,KeyError) as exc: + result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]} + return result + finally: + detail={k:v for k,v in result.items() if k in ('status','reason','planId','configRevision','sourceFamily')} + with store.con:store.con.execute('INSERT INTO planner_run_status VALUES(?,?,?,?) ON CONFLICT(plant) DO UPDATE SET updated_at=excluded.updated_at,status=excluded.status,detail=excluded.detail',(plant,now.isoformat(),result['status'],canonical(detail))) + store.finish(claim) + +def status(store,plant,now): + plan=store.current(plant);settings=store.settings(plant) + row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone() + pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone() + ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone() + fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit')) + return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)} + +def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()): + allowed={str(UUID(p)) for p in plants} + trial_allowed={str(UUID(p)) for p in controlled_trial_plants} + if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist') + if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required') + path=Path(db_path).resolve() + if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required') + path.parent.mkdir(parents=True,exist_ok=True);stop=Event() + def factory():return PlannerStore(str(path)) + def loop(): + while not stop.is_set(): + s=factory() + try:run_once(s,datetime.now(timezone.utc).replace(microsecond=0)) + except Exception as exc:logging.getLogger(__name__).error('V4 worker error: %s',type(exc).__name__) + finally:s.close() + stop.wait(1.) + @asynccontextmanager + async def lifespan(app): + thread=Thread(target=loop,name='v4-shadow',daemon=True) + if start_worker:thread.start() + yield + stop.set() + if start_worker:thread.join(35) + app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan) + app.state.store_factory=factory + def authorize(plant,token): + if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized') + try:plant=str(UUID(plant)) + except ValueError:raise HTTPException(400,'Invalid installation ID') + if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial') + return factory() + @app.get('/health') + def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1} + @app.get('/internal/v2/prognosis/{plant}/planner') + def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc);view=status(s,plant,now) + view['controlledTrial']=controlled_trial.authority(s,plant,view,now,trial_allowed) + view['controlledTrialAuthorized']=view['controlledTrial'] is not None + return view + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/arm') + def arm_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + return controlled_trial.arm(s,plant,payload,status(s,plant,now),now,trial_allowed) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409,str(exc)[:300]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/revoke') + def revoke_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if set(payload)!={'sessionId'}:raise ValueError('Session ID only') + return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/settings') + def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return s.save_settings(plant,payload['changes'],payload['expectedRevision'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409 if 'Revision conflict' in str(exc) else 400,str(exc)) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/replan') + def replan(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.request(plant,'manual',datetime.now(timezone.utc));return {'status':'queued','liveEnabled':False} + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/inputs/{kind}') + def input_event(plant:str,kind:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return ingest(s,plant,kind,payload,datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}') + def configure_dataset(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if payload.get('datasetId')!=dataset:raise ValueError('Dataset path/payload mismatch') + return measurement_pipeline.register_dataset(s.con,plant,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/measurements') + def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp())) + # Existing five-minute worker handles rollup/training; no per-record optimizer flood. + return result + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/ack') + def ack(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.acknowledge(plant,payload['planId'],payload['revision'],datetime.now(timezone.utc),payload.get('step'),payload.get('status','shadow_seen'));return {'status':'recorded'} + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + from starlette.responses import JSONResponse + class BodyLimit: + def __init__(self,app):self.app=app + async def __call__(self,scope,receive,send): + if scope['type']!='http' or scope['method'] not in ('POST','PUT'):return await self.app(scope,receive,send) + chunks=[];total=0 + while True: + msg=await receive() + if msg['type']=='http.disconnect':return + total+=len(msg.get('body',b'')) + if total>2000000:return await JSONResponse({'detail':'Request too large'},status_code=413)(scope,receive,send) + chunks.append(msg) + if not msg.get('more_body',False):break + async def replay():return chunks.pop(0) if chunks else await receive() + return await self.app(scope,replay,send) + app.add_middleware(BodyLimit) + return app + +def from_environment(): + return create_app(os.environ.get('NETPLAN_V4_DB','/data/netplan-v4.sqlite'),os.environ.get('PROGNOSIS_SERVICE_TOKEN',''),[p.strip() for p in os.environ.get('NETPLAN_V4_PLANTS','').split(',') if p.strip()], controlled_trial_plants=[p.strip() for p in os.environ.get('NETPLAN_V4_CONTROL_TRIAL_PLANTS','').split(',') if p.strip()]) diff --git a/services/netplan-v4/netplan_v4/store.py b/services/netplan-v4/netplan_v4/store.py new file mode 100644 index 0000000..07d5254 --- /dev/null +++ b/services/netplan-v4/netplan_v4/store.py @@ -0,0 +1,147 @@ +from __future__ import annotations +import json +import sqlite3 +from datetime import datetime,timedelta +from uuid import uuid4 +from .domain import default_registry,month_key,number,quarter_start,utc +from . import meter_runtime, controlled_trial, measurement_pipeline + +def canonical(value): + return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False) + +DEFAULT_SETTINGS={'family':'3','autoLookbackDays':14,'autoMinimumDays':7,'autoMinimumCoverage':.9,'autoSwitchMarginChf':1.,'tariffPolicy':'published_only','trainingCadence':'daily','trainingPromotion':'validated_only','runMode':'shadow','roundtripEfficiency':.90,'measurementPolicy':'verified_only','peakOutlookPolicy':'empirical_if_available','peakOutlookReliance':.5,'forecastSource':'legacy','measurementDataset':''} + +class PlannerStore: + """Own SQLite file, no mutation of legacy application databases.""" + def __init__(self,path,registry=None): + self.registry=registry or default_registry();self.con=sqlite3.connect(path,timeout=10) + self.con.row_factory=sqlite3.Row;self.con.execute('PRAGMA foreign_keys=ON') + self.con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_settings(plant TEXT PRIMARY KEY,revision INTEGER NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_audit(id INTEGER PRIMARY KEY,plant TEXT NOT NULL,at TEXT NOT NULL,kind TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_work(plant TEXT PRIMARY KEY,sequence INTEGER NOT NULL,revision INTEGER NOT NULL,reasons TEXT NOT NULL,requested_at TEXT NOT NULL,lease_until TEXT,lease_token TEXT); + CREATE TABLE IF NOT EXISTS planner_measurements(plant TEXT NOT NULL,start TEXT NOT NULL,import_kwh REAL NOT NULL,PRIMARY KEY(plant,start)); + CREATE TABLE IF NOT EXISTS planner_month_peaks(plant TEXT NOT NULL,month TEXT NOT NULL,peak_kw REAL NOT NULL,source TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_snapshots(id TEXT PRIMARY KEY,plant TEXT NOT NULL,issued_at TEXT NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_plans(plan_id TEXT PRIMARY KEY,plant TEXT NOT NULL,revision INTEGER NOT NULL,mode TEXT NOT NULL,value TEXT NOT NULL,created_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_current(plant TEXT NOT NULL,mode TEXT NOT NULL,plan_id TEXT NOT NULL REFERENCES planner_plans(plan_id),PRIMARY KEY(plant,mode)); + CREATE TABLE IF NOT EXISTS planner_ack(plant TEXT PRIMARY KEY,plan_id TEXT NOT NULL,revision INTEGER NOT NULL,received_at TEXT NOT NULL,applied_step TEXT,status TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_inputs(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,observed_at TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(plant,kind,event_id)); + CREATE TABLE IF NOT EXISTS planner_input_current(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,PRIMARY KEY(plant,kind)); + CREATE TABLE IF NOT EXISTS planner_run_status(plant TEXT PRIMARY KEY,updated_at TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_ticks(plant TEXT PRIMARY KEY,tick INTEGER NOT NULL); + ''') + meter_runtime.schema(self.con) + controlled_trial.schema(self.con) + measurement_pipeline.schema(self.con) + def close(self):self.con.close() + def settings(self,plant): + row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone() + return {'revision':row['revision'],**DEFAULT_SETTINGS,**json.loads(row['value'])} if row else {'revision':0,**DEFAULT_SETTINGS} + def _validate_settings(self,value): + if set(value)!=set(DEFAULT_SETTINGS):raise ValueError('Unknown or missing setting') + if value['family']!='auto':self.registry.get(value['family']) + for key in ('autoLookbackDays','autoMinimumDays'): + if type(value[key]) is not int:raise ValueError('Days must be integers') + number(value['autoLookbackDays'],'lookback',7,90);number(value['autoMinimumDays'],'minimum days',1,value['autoLookbackDays']) + number(value['autoMinimumCoverage'],'coverage',.5,1);number(value['autoSwitchMarginChf'],'margin',0) + if value['tariffPolicy']!='published_only':raise ValueError('Only published-price policy implemented') + if value['trainingCadence'] not in ('daily','weekly') or value['trainingPromotion']!='validated_only':raise ValueError('Training must use validated promotion') + if value['runMode']!='shadow':raise ValueError('Shadow-only: live release requires separate validation') + number(value['roundtripEfficiency'],'roundtrip efficiency',.01,1) + if value['measurementPolicy'] not in ('verified_only','allow_estimates'):raise ValueError('Invalid measurement policy') + if value['peakOutlookPolicy'] not in ('full_incremental','empirical_if_available'):raise ValueError('Invalid peak outlook policy') + number(value['peakOutlookReliance'],'peak outlook reliance',0,1) + if value['forecastSource'] not in ('legacy','corrected_profile'):raise ValueError('Unknown forecast source') + if not isinstance(value['measurementDataset'],str) or len(value['measurementDataset'])>80:raise ValueError('Invalid measurement dataset') + if value['forecastSource']=='corrected_profile' and not value['measurementDataset']:raise ValueError('Corrected forecast requires an explicit dataset') + def _request(self,plant,revision,reason,now): + row=self.con.execute('SELECT * FROM planner_work WHERE plant=?',(plant,)).fetchone() + reasons=set(json.loads(row['reasons'])) if row else set();reasons.add(reason) + seq=row['sequence']+1 if row else 1 + self.con.execute('''INSERT INTO planner_work(plant,sequence,revision,reasons,requested_at) VALUES(?,?,?,?,?) + ON CONFLICT(plant) DO UPDATE SET sequence=excluded.sequence,revision=excluded.revision,reasons=excluded.reasons,requested_at=excluded.requested_at''',(plant,seq,revision,canonical(sorted(reasons)),utc(now).isoformat())) + def save_settings(self,plant,changes,expected_revision,now): + if type(expected_revision) is not int or expected_revision<0:raise ValueError('Invalid expected revision') + self.con.execute('BEGIN IMMEDIATE') + try: + current=self.settings(plant) + if current.pop('revision')!=expected_revision:raise ValueError('Revision conflict; reload before saving') + current.update(changes);self._validate_settings(current);revision=expected_revision+1 + self.con.execute('''INSERT INTO planner_settings VALUES(?,?,?) ON CONFLICT(plant) DO UPDATE SET revision=excluded.revision,value=excluded.value''',(plant,revision,canonical(current))) + self._request(plant,revision,'configuration_changed',now) + self.con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',(plant,utc(now).isoformat(),'settings',canonical({'revision':revision,'changes':changes}))) + self.con.commit();return {'revision':revision,**current} + except Exception:self.con.rollback();raise + def request(self,plant,reason,now): + if reason not in ('prices_changed','telemetry_changed','five_minute_tick','manual','model_promoted','forecast_changed','operation_changed','tariffs_changed'):raise ValueError('Unknown trigger') + with self.con:self._request(plant,self.settings(plant)['revision'],reason,now) + def claim(self,now,lease_seconds=120): + at=utc(now);self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT * FROM planner_work WHERE lease_until IS NULL OR lease_until < ? ORDER BY requested_at LIMIT 1',(at.isoformat(),)).fetchone() + if not row:self.con.commit();return None + token=str(uuid4());self.con.execute('UPDATE planner_work SET lease_until=?,lease_token=? WHERE plant=?',((at+timedelta(seconds=lease_seconds)).isoformat(),token,row['plant'])) + self.con.commit();return {**dict(row),'lease_token':token} + except Exception:self.con.rollback();raise + def finish(self,claim): + self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(claim['plant'],)).fetchone() + if not row or row['lease_token']!=claim['lease_token']:self.con.commit();return False + if row['sequence']==claim['sequence']:self.con.execute('DELETE FROM planner_work WHERE plant=?',(claim['plant'],)) + else:self.con.execute('UPDATE planner_work SET lease_until=NULL,lease_token=NULL WHERE plant=?',(claim['plant'],)) + self.con.commit();return True + except Exception:self.con.rollback();raise + def initialize_peak(self,plant,month,peak_kw,source,now): + number(peak_kw,'authoritative measured peak',0) + if source not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Configured cap is NOT measured peak') + datetime.strptime(month,'%Y-%m') + if month>month_key(now):raise ValueError('Future month cannot have a measured peak') + with self.con: + old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,month)).fetchone() + if old and peak_kwreceived_at:raise ValueError('Completed aligned intervals required') + with self.con: + old=self.con.execute('SELECT import_kwh FROM planner_measurements WHERE plant=? AND start=?',(plant,start.isoformat())).fetchone() + if old and abs(old[0]-import_kwh)>1e-9:raise ValueError('Conflicting metering fact') + self.con.execute('INSERT OR IGNORE INTO planner_measurements VALUES(?,?,?)',(plant,start.isoformat(),import_kwh)) + q=quarter_start(start);rows=self.con.execute('SELECT start,import_kwh FROM planner_measurements WHERE plant=? AND start>=? AND start8000000:raise RuntimeError('Probe execution failed or oversized') + probe=json.loads(result.stdout) + save(reports/'native-inputs.json',probe) + report['nativeInputStatus']=probe.get('status') + report['forecastSummary']=probe.get('forecastSummary',{}) + report['dataBlockers']=[] + for key,value in report['forecastSummary'].items(): + if key in ('prog_var_2','prog_var_11','prog_var_22') and (not value.get('points') or value.get('allZero')): + report['dataBlockers'].append(key+': missing or unconfirmed all-zero load forecast') + recent=probe.get('inputFrames',{}).get('df_recent_raw',{}).get('columns',{}).get('Hausverbrauch',{}).get('lastFiniteAt') + if not recent or (datetime.now(timezone.utc)-datetime.fromisoformat(recent)).total_seconds()>1800: + report['dataBlockers'].append('Recent raw load telemetry missing or older than 30 minutes') + if probe.get('status')!='read_only_acquired' or report['dataBlockers']:success=False + except (ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + report['nativeInputStatus']='failed';report['nativeInputErrorType']=type(error).__name__;success=False + try: + verify_native_bundle() + report['sourceStableDuringTests']=(source_manifest()==report['sourceHashes'] and verify_forecast_bundle(ROOT/'acceptance/forecast-src',PROJECT/'forecast_engine')==forecast_hashes) + except (OSError,ValueError): + report['sourceStableDuringTests']=False + if not report['sourceStableDuringTests']: + report.setdefault('dataBlockers',[]).append('Source changed during checks; results cannot certify current candidate') + success=False + report['preflightChecksPassed']=success + save(reports/'REPORT.json',report) + save(ROOT/'acceptance-reports'/'LATEST.json',{'report':str(reports/'REPORT.json')}) + print('\nPRECHECK '+('PASSED' if success else 'NEEDS REVIEW')+'; NOT a production release.') + print('REPORT: '+str(reports/'REPORT.json')) + for key,value in report.get('forecastSummary',{}).items():print(key,json.dumps(value)) + for blocker in report.get('dataBlockers',[]):print('DATA BLOCKER:',blocker) + print('Existing containers, Symcon settings, timers and actuators unchanged.') + return 0 if success else 1 + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda v:str(UUID(v))) + parser.add_argument('--show-commands',action='store_true') + args=parser.parse_args() + if args.show_commands: + for name,command,cwd,timeout in command_list(args.plant):print(name,json.dumps(command)) + print('Native input probe: read-only docker compose exec forecast-engine python with reviewed stdin script') + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not alter Docker socket permissions.') + try:raise SystemExit(run(args.plant)) + except (OSError,ValueError) as error:raise SystemExit('Preflight stopped safely: '+str(error)) diff --git a/services/netplan-v4/requirements.txt b/services/netplan-v4/requirements.txt new file mode 100644 index 0000000..595f8e7 --- /dev/null +++ b/services/netplan-v4/requirements.txt @@ -0,0 +1,5 @@ +numpy>=2.2,<3 +scipy>=1.15,<1.18 +fastapi>=0.115,<0.129 +httpx>=0.27,<0.29 +uvicorn>=0.30,<0.41 diff --git a/services/netplan-v4/run_tests.py b/services/netplan-v4/run_tests.py new file mode 100644 index 0000000..4876825 --- /dev/null +++ b/services/netplan-v4/run_tests.py @@ -0,0 +1,21 @@ +"""Run against the exact server working tree; no live data or credentials used.""" +from pathlib import Path +import hashlib,json,os,sys,unittest +ROOT=Path(__file__).resolve().parent +DEPS=Path('/home/agent/services/qa/netplan-v4-candidate-20261001/python-dependencies') +if DEPS.exists():sys.path.insert(0,str(DEPS)) +sys.path.insert(0,str(ROOT)) +if __name__=='__main__': + import numpy,scipy,fastapi,httpx + versions={'python':sys.version.split()[0],**{m.__name__:m.__version__ for m in (numpy,scipy,fastapi,httpx)}} + print('Runtime',json.dumps(versions),flush=True) + reports=Path(os.getenv('NETPLAN_V4_TEST_REPORT_DIR',str(ROOT))) + reports.mkdir(parents=True,exist_ok=True) + suite=unittest.defaultTestLoader.discover(str(ROOT/'tests')) + with (reports/'TEST_RESULTS.txt').open('w') as output: + output.write('Actual server/target tests; synthetic data only.\n'+json.dumps(versions)+'\n\n') + result=unittest.TextTestRunner(stream=output,verbosity=2).run(suite) + print((reports/'TEST_RESULTS.txt').read_text()) + hashes={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(ROOT.rglob('*.py')) if '__pycache__' not in p.parts} + (reports/'TESTED_SOURCE_HASHES.json').write_text(json.dumps(hashes,indent=2)+'\n') + raise SystemExit(0 if result.wasSuccessful() else 1) diff --git a/services/netplan-v4/runtime_preflight.py b/services/netplan-v4/runtime_preflight.py new file mode 100644 index 0000000..8991424 --- /dev/null +++ b/services/netplan-v4/runtime_preflight.py @@ -0,0 +1,46 @@ +"""Check packaged code readability and writable temporary test output as non-root.""" +import os +from pathlib import Path +import tempfile + +SOURCE_DIRS = ("netplan_v4", "tests", "integrations", "gui") +SOURCE_FILES = ("run_tests.py", "install_hooks.py", "runtime_preflight.py", "requirements.txt") + + +def verify_access(root, reports): + root, reports = Path(root).resolve(), Path(reports).resolve() + if reports == root or root in reports.parents: + raise ValueError("Test reports must be outside the packaged application tree") + if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK): + raise PermissionError(f"Application directory is not accessible: {root}") + paths = [root / name for name in SOURCE_FILES] + for name in SOURCE_DIRS: + directory = root / name + if not directory.is_dir(): + raise FileNotFoundError(f"Missing packaged directory: {directory}") + def on_error(error): + raise error + for parent, directories, files in os.walk(directory, onerror=on_error): + base = Path(parent) + if not os.access(base, os.R_OK | os.X_OK): + raise PermissionError(f"Packaged directory is not accessible: {base}") + if any((base / child).is_symlink() for child in directories + files): + raise ValueError(f"Unexpected symlink in packaged source: {base}") + paths.extend(base / name for name in files) + for path in paths: + with path.open("rb") as handle: + handle.read(1) + reports.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryFile(dir=reports) as probe: + probe.write(b"permission check") + probe.flush() + return len(paths) + + +if __name__ == "__main__": + if os.geteuid() == 0: + raise SystemExit("Runtime preflight must run as the unprivileged container user") + reports = Path(os.getenv("NETPLAN_V4_TEST_REPORT_DIR", "/tmp/test-results")) + count = verify_access(Path(__file__).resolve().parent, reports) + import numpy, scipy, fastapi, httpx + print(f"Non-root runtime check OK: uid={os.geteuid()}, readable_files={count}, reports={reports}") diff --git a/services/netplan-v4/tests/manager_protocol.php b/services/netplan-v4/tests/manager_protocol.php new file mode 100644 index 0000000..9a56099 --- /dev/null +++ b/services/netplan-v4/tests/manager_protocol.php @@ -0,0 +1,24 @@ +'2026-10-01T12:00:00Z','validUntil'=>'2026-10-01T12:05:00Z','gridTargetW'=>0.,'intent'=>'pvCharge']; +$plan=['schemaVersion'=>2,'planId'=>'test','configRevision'=>7,'status'=>'optimal','executable'=>true, + 'runMode'=>'shadow','liveEnabled'=>false,'generatedAt'=>$point['time'],'validFrom'=>$point['time'],'validUntil'=>'2026-10-01T12:10:00Z', + 'points'=>[$point,['time'=>'2026-10-01T12:05:00Z','validUntil'=>'2026-10-01T12:10:00Z','gridTargetW'=>9000.,'intent'=>'gridCharge']]]; +$now=strtotime('2026-10-01T12:03:00Z'); +check(NetzfahrplanV4::currentPoint($plan,$now,7)===null,'Shadow never live'); +check(NetzfahrplanV4::currentPoint($plan,$now,7,true)['gridTargetW']===0.,'Current interval, not nearest future point'); +check(NetzfahrplanV4::currentPoint($plan,$now,8,true)===null,'Revision mismatch'); +check(NetzfahrplanV4::currentPoint($plan,$now+3600,7,true)===null,'Expired plan'); +$bad=$plan;$bad['points'][1]['time']=$point['time'];check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Overlapping intervals'); +$bad=$plan;$bad['points'][0]['gridTargetW']=NAN;check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'NaN rejected'); +$bad=$plan;$bad['generatedAt']='2026-10-01T12:00:00';check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Timezone required'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,39000.,0.,15000.);check($r['batteryTargetW']===5332.,'Surplus charges');check($r['gridTargetW']===0.,'Peak cap never replaces zero target');check($r['expectedGridW']===0.,'Expected grid balanced'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,3000.,0.);check($r['batteryTargetW']===3000.&&$r['limited'],'Charging availability enforced'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],1000.,0.,39000.,0.);check($r['batteryTargetW']===0.,'No inadvertent grid charge from load forecast'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],7000.,0.,39000.,1000.,5000.);check($r['batteryTargetW']===-1000.&&$r['limited'],'Safety override still respects available discharge'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>-40000.,'intent'=>'pvCharge'],-10000.,0.,39000.,10000.);check($r['batteryTargetW']===0.,'No unintended discharge in PV charge intent');check($r['trackingErrorW']===30000.,'Intent saturation reports unmet target'); +echo "Netzfahrplan V4: {$count} assertions passed. No IPS or device calls.\n"; diff --git a/services/netplan-v4/tests/portal.test.mjs b/services/netplan-v4/tests/portal.test.mjs new file mode 100644 index 0000000..05bb0fa --- /dev/null +++ b/services/netplan-v4/tests/portal.test.mjs @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createPlannerV4Bridge} from '../integrations/netplan-v4-bridge.mjs'; +const PLANT='00000000-0000-4000-8000-000000000001',INSTALL='e3a08f9e-af12-4695-99bd-8b51c0520021'; +function fixture(options={}){ + const sent=[],auth=[],upstream=[]; + const bridge=createPlannerV4Bridge({ + configuredPrognosisPlant:(req,res,id,csrf)=>{auth.push({id,csrf});return options.denied?null:{plant:{installation_id:INSTALL},license:{quantities:{grid_schedule:options.unlicensed?0:1}},session:{}};}, + roleAllowed:()=>!options.viewer,bodyJson:async()=>({expectedRevision:0,changes:{family:'23'}}), + json:(res,status,payload)=>sent.push({status,payload}),deviceActivation:()=>options.deviceDenied?null:{plant_id:PLANT}, + checkDeviceRate:()=>!options.ratelimited,ownedLicenseState:()=>({quantities:{grid_schedule:1}}),serviceToken:'synthetic-test-only', + fetchImpl:async(url,args)=>{upstream.push({url:String(url),args});if(options.down)throw new Error('secret-host-detail');return {ok:!options.conflict,status:options.conflict?409:200,json:async()=>options.conflict?{detail:'internal-field'}:{liveEnabled:false,plan:{planId:'p1'}}};} + });return {bridge,sent,auth,upstream}; +} +const url=suffix=>new URL(`https://portal.test/api/plants/${PLANT}/prognosis/planner-v4${suffix}`); +test('read scoped by customer plant, upstream uses installation id',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},url('')),true);assert.equal(f.auth[0].csrf,false);assert.match(f.upstream[0].url,new RegExp(INSTALL));assert.equal(f.sent[0].status,200);}); +test('save requires existing CSRF checks',async()=>{const f=fixture();await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.auth[0].csrf,true);assert.equal(f.upstream[0].args.method,'PUT');}); +test('viewer cannot mutate',async()=>{const f=fixture({viewer:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,403);assert.equal(f.upstream.length,0);}); +test('unowned plant blocked',async()=>{const f=fixture({denied:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.upstream.length,0);}); +test('license required',async()=>{const f=fixture({unlicensed:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,403);}); +test('revision conflict preserved without internal error disclosure',async()=>{const f=fixture({conflict:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,409);assert.doesNotMatch(JSON.stringify(f.sent),/internal-field/);}); +test('service failure neither exposes details nor changes live plan',async()=>{const f=fixture({down:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,503);assert.doesNotMatch(JSON.stringify(f.sent),/secret-host-detail|synthetic-test-only/);}); +test('V1 live schedule is NOT intercepted',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/schedule`)),false);assert.equal(f.upstream.length,0);}); +test('device telemetry mapped only to operation ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/operation`));assert.match(f.upstream[0].url,/\/inputs\/operation$/);}); +test('unauthenticated device blocked',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.upstream.length,0);}); +test('device rate limit reused',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); +test('unexpected method rejected',async()=>{const f=fixture();await f.bridge({method:'DELETE'},{},url(''));assert.equal(f.sent[0].status,405);assert.equal(f.upstream.length,0);}); + +test('measurement batches use authenticated dedicated data ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.match(f.upstream[0].url,/\/planner\/measurements$/);assert.equal(f.upstream[0].args.method,'POST');}); +test('unauthenticated measurement upload cannot reach storage',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.upstream.length,0);}); +test('device cannot change dataset mapping through public proxy',async()=>{const f=fixture();assert.equal(await f.bridge({method:'PUT'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/datasets/physical-v1`)),false);assert.equal(f.upstream.length,0);}); +test('measurement upload keeps device rate protection',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); diff --git a/services/netplan-v4/tests/test_application_deployment.py b/services/netplan-v4/tests/test_application_deployment.py new file mode 100644 index 0000000..44b2d8e --- /dev/null +++ b/services/netplan-v4/tests/test_application_deployment.py @@ -0,0 +1,39 @@ +import importlib.util +from pathlib import Path +import sqlite3,tempfile,unittest +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('app_deploy',ROOT/'commissioning/deploy_application.py');d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) +class ApplicationDeploymentTest(unittest.TestCase): + def test_backup_consistent_and_original_not_changed(self): + with tempfile.TemporaryDirectory() as tmp: + src=Path(tmp)/'source.sqlite';out=Path(tmp)/'backup.sqlite' + with sqlite3.connect(src) as c:c.execute('CREATE TABLE sample(value)');c.execute('INSERT INTO sample VALUES(3)') + before=src.read_bytes();d.backup_database(src,out) + with sqlite3.connect(out) as c:self.assertEqual(c.execute('SELECT value FROM sample').fetchone()[0],3) + self.assertEqual(before,src.read_bytes()) + def test_backup_cannot_invent_missing_database(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError):d.backup_database(Path(tmp)/'missing.sqlite',Path(tmp)/'new.sqlite') + def test_atomic_refuses_symlink(self): + with tempfile.TemporaryDirectory() as tmp: + a=Path(tmp)/'a';b=Path(tmp)/'b';a.write_text('original');b.symlink_to(a) + with self.assertRaises(ValueError):d.atomic(b,b'changed') + self.assertEqual(a.read_text(),'original') + def test_only_two_reviewed_portal_files(self): + self.assertEqual(set(d.PORTAL_FILES.values()),{'netplan-v4-bridge.mjs','public/netplan-v4.js'}) + def test_build_tests_backup_before_runtime_replacement(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertLess(s.index("'target_python_and_portal_tests_passed'"),s.index('v4_replaced=True')) + self.assertLess(s.index("'consistent_database_backup'"),s.index('v4_replaced=True')) + def test_does_not_change_optimizer_or_device_permissions(self): + self.assertNotIn('/settings',d.BOOTSTRAP) + self.assertNotIn('trial/arm',d.BOOTSTRAP) + self.assertNotIn('IPS_',d.BOOTSTRAP) + self.assertIn("state['liveEnabled'] is False",d.BOOTSTRAP) + def test_image_contains_application_installer_imported_by_tests(self): + self.assertIn('COPY commissioning/deploy_application.py ./commissioning/deploy_application.py',(ROOT/'Dockerfile').read_text()) + def test_file_bind_mount_recreated_not_only_restarted(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertIn("'--force-recreate','--wait','license-portal'",s) + self.assertIn('additive_data_retained',s) +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_battery_recovery.py b/services/netplan-v4/tests/test_battery_recovery.py new file mode 100644 index 0000000..e82974c --- /dev/null +++ b/services/netplan-v4/tests/test_battery_recovery.py @@ -0,0 +1,43 @@ +from dataclasses import replace +import unittest +from netplan_v4.optimizer import optimize +from netplan_v4.domain import Limits +from test_v4 import AT,seq,batt + +class BatteryRecoveryTest(unittest.TestCase): + def plan(self,b,steps=None): + steps=steps or seq([0.]*3+[1000.]*3,pv=[1000.]*3+[0.]*3,sell=[0.]*6) + return optimize(steps,[b],at=AT,observed_peaks={'2026-10':1.},peak_prices={'2026-10':0.}) + def test_reserve_recovery_preserves_real_initial_energy(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + energy=.05 + for x in p['points']: + target=x['batteryTargetW'];energy+=target/1000/12*(.9**.5 if target>=0 else 1/(.9**.5)) + self.assertAlmostEqual(100*energy,x['socEndPercent']['b'],places=5) + self.assertGreaterEqual(energy,.05-1e-6) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],15.-1e-5) + def test_below_hardware_min_not_fabricated(self): + b=batt(soc_percent=1.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True) + self.assertFalse(self.plan(b)['executable']) + def test_disabled_can_rearm_after_charging(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + self.assertAlmostEqual(min(0.,p['points'][0]['batteryTargetW']),0.) + self.assertTrue(any(x['batteryTargetW']<-1. for x in p['points'][3:])) + first=next(i for i,x in enumerate(p['points']) if x['batteryTargetW']<-1.) + self.assertGreaterEqual(p['points'][first-1]['socEndPercent']['b'],17.-1e-5) + def test_cannot_discharge_below_rearm_while_blocked(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False) + p=self.plan(b,seq([1000.]*6));self.assertTrue(p['executable']) + self.assertTrue(all(x['batteryTargetW']>=-1e-5 for x in p['points'])) + def test_unavailable_power_does_not_get_invented(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,max_charge_w=0.,max_discharge_w=0.) + p=self.plan(b);self.assertFalse(p['executable']);self.assertEqual(p['status'],'no_feasible_plan') + def test_invalid_hysteresis_range_rejected(self): + b=batt(rearm_soc_percent=95.) + self.assertFalse(self.plan(b)['executable']) + def test_no_recovery_permission_keeps_strict_behavior(self): + self.assertFalse(self.plan(batt(soc_percent=5.))['executable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_container_access.py b/services/netplan-v4/tests/test_container_access.py new file mode 100644 index 0000000..8a24ebc --- /dev/null +++ b/services/netplan-v4/tests/test_container_access.py @@ -0,0 +1,60 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import unittest +from runtime_preflight import SOURCE_DIRS, SOURCE_FILES, verify_access + + +class ContainerAccessTest(unittest.TestCase): + def fixture(self, tmp): + root = Path(tmp) / "app" + root.mkdir() + for name in SOURCE_DIRS: + (root / name).mkdir() + (root / name / "sample.txt").write_text("readable") + for name in SOURCE_FILES: + (root / name).write_text("readable") + return root, Path(tmp) / "reports" + + def test_code_readable_and_reports_outside_readonly_tree(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + self.assertEqual(verify_access(root, reports), 8) + self.assertEqual(list(reports.iterdir()), []) + + def test_missing_runner_is_reported(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + (root / "run_tests.py").unlink() + with self.assertRaises(FileNotFoundError): + verify_access(root, reports) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_unreadable_source_fails(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "netplan_v4" / "sample.txt" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o600) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_untraversable_directory_is_not_silently_skipped(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "tests" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o700) + + def test_reports_cannot_target_application_tree(self): + with TemporaryDirectory() as tmp: + root, _ = self.fixture(tmp) + with self.assertRaises(ValueError): + verify_access(root, root / "test-results") diff --git a/services/netplan-v4/tests/test_controlled_trial.py b/services/netplan-v4/tests/test_controlled_trial.py new file mode 100644 index 0000000..ddb1f2a --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial.py @@ -0,0 +1,120 @@ +import copy +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app +from netplan_v4 import controlled_trial as t + +AT=datetime(2026,10,2,12,tzinfo=timezone.utc) +PLANT='00000000-0000-4000-8000-000000000001' +SESSION='00000000-0000-4000-8000-000000000002' +PLAN='00000000-0000-4000-8000-000000000003' +TOKEN='synthetic-only-control-test-token' + +def fixture(): + context={'batteries':{'b':{'capacityKwh':10.}},'limits':{'importW':10000.,'exportW':None,'managerMonthLimitsW':{}}} + plan={'planId':PLAN,'installationId':PLANT,'sourceFamily':'3','configRevision':1, + 'runMode':'shadow','liveEnabled':False,'executable':True,'validUntil':(AT+timedelta(hours=1)).isoformat(), + 'controlContext':context,'inputQuality':{'loadBasis':'base_load','accountingEvidenceId':'synthetic-meter-boundary-v1'}, + 'peakCostIsEstimate':True} + view={'installationId':PLANT,'liveEnabled':False,'fresh':True,'plan':plan,'settings':{'family':'3','revision':1}} + request={'sessionId':SESSION,'expectedPlanId':PLAN,'expectedRevision':1,'durationSeconds':300, + 'maxChargeW':5000.,'maxDischargeW':5000.,'assetId':'b','managerId':11111,'batteryInstanceId':22222, + 'acceptEstimatedPeak':True,'actuatorWatchdogEvidenceId':'synthetic-hardware-watchdog', + 'confirmation':'ARM_BOUNDED_CONTROL_TRIAL'} + return view,request + +class ControlledTrialTest(unittest.TestCase): + def setUp(self): + self.s=PlannerStore(':memory:');self.v,self.r=fixture() + def tearDown(self): self.s.close() + def arm(self,**kw): + return t.arm(self.s,PLANT,kw.get('request',self.r),kw.get('view',self.v),kw.get('now',AT),kw.get('allowed',{PLANT})) + def test_default_has_no_authority(self): + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,set())) + with self.assertRaises(ValueError):self.arm(allowed=set()) + def test_shadow_plan_never_becomes_live(self): + before=copy.deepcopy(self.v);self.arm() + a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(self.v,before);self.assertEqual(a['kind'],'controlled_trial_authority') + self.assertTrue(a['sourcePlanRemainsShadow']);self.assertFalse(self.v['plan']['liveEnabled']) + def test_short_authority_lease(self): + self.arm();a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(t.timestamp(a['validUntil'])-AT,timedelta(seconds=90)) + def test_session_expiry(self): + self.arm();self.assertIsNone(t.authority(self.s,PLANT,self.v,AT+timedelta(seconds=300),{PLANT})) + def test_revoke(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT);self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_expired_or_revoked_session_not_resurrected(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT) + with self.assertRaises(ValueError):self.arm() + def test_active_session_cannot_be_extended(self): + self.arm() + with self.assertRaises(ValueError):self.arm(now=AT+timedelta(seconds=30)) + def test_other_plant(self): + self.arm();self.assertIsNone(t.authority(self.s,SESSION,self.v,AT,{SESSION})) + def test_revocation_mismatched_id_leaves_real_session(self): + self.arm();t.revoke(self.s,PLANT,PLAN,AT);self.assertIsNotNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_stale_source_plan_cannot_arm(self): + self.v['fresh']=False + with self.assertRaises(ValueError):self.arm() + def test_stale_source_removes_authority(self): + self.arm();self.v['fresh']=False;self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_revision_bool_not_an_integer(self): + self.r['expectedRevision']=True + with self.assertRaises(ValueError):self.arm() + def test_wrong_revision(self): + self.r['expectedRevision']=2 + with self.assertRaises(ValueError):self.arm() + def test_revision_change_invalidates_authority(self): + self.arm();self.v['settings']['revision']=2;self.v['plan']['configRevision']=2 + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_policy_change_invalidates_authority(self): + self.arm();self.v['plan']['controlContext']['limits']['importW']=12000. + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_newer_plan_same_policy_supported(self): + self.arm();self.v['plan']['planId']=SESSION + self.assertEqual(t.authority(self.s,PLANT,self.v,AT,{PLANT})['sourceShadowPlanId'],SESSION) + def test_auto_family_not_enabled_for_first_trial(self): + self.v['settings']['family']='auto' + with self.assertRaises(ValueError):self.arm() + def test_aggregate_house_refused(self): + self.v['plan']['inputQuality']['loadBasis']='house_total' + with self.assertRaises(ValueError):self.arm() + def test_no_accounting_evidence_no_trial(self): + del self.v['plan']['inputQuality']['accountingEvidenceId'] + with self.assertRaises(ValueError):self.arm() + def test_no_watchdog_evidence_no_trial(self): + self.r['actuatorWatchdogEvidenceId']='' + with self.assertRaises(ValueError):self.arm() + def test_no_implicit_estimate_acceptance(self): + self.r['acceptEstimatedPeak']=False + with self.assertRaises(ValueError):self.arm() + def test_power_cap_and_nonfinite(self): + for v in (5001, float('nan'),float('inf'),True,0,'5000'): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'maxChargeW':v}) + def test_duration_bound(self): + for v in (0,1801,True,30.5): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'durationSeconds':v}) + def test_cannot_replace_asset(self): + self.r['assetId']='other' + with self.assertRaises(ValueError):self.arm() + def test_two_assets_not_supported(self): + self.v['plan']['controlContext']['batteries']['other']={} + with self.assertRaises(ValueError):self.arm() + def test_operator_endpoint_requires_auth(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False) + with TestClient(app) as c: + self.assertEqual(c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',json=self.r).status_code,401) + r=c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',headers={'X-Enelix-Service-Token':TOKEN},json=self.r) + self.assertEqual(r.status_code,409);self.assertIn('allowlist',r.text) + self.assertIsNone(c.get(f'/internal/v2/prognosis/{PLANT}/planner',headers={'X-Enelix-Service-Token':TOKEN}).json()['controlledTrial']) + def test_trial_allowlist_cannot_expand_installation_access(self): + with tempfile.TemporaryDirectory() as td,self.assertRaises(ValueError): + create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[SESSION]) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_controlled_trial_pipeline.py b/services/netplan-v4/tests/test_controlled_trial_pipeline.py new file mode 100644 index 0000000..894242e --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial_pipeline.py @@ -0,0 +1,44 @@ +import copy +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch +from fastapi.testclient import TestClient +from netplan_v4.service import create_app, ingest, run_once +from test_v4 import inputs, AT +from test_controlled_trial import fixture, PLANT, SESSION, TOKEN + +class ControlledTrialPipelineTest(unittest.TestCase): + def test_real_planner_to_internal_trial_api_and_revoke(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'trial.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[PLANT]) + store=app.state.store_factory() + op,fc,tar=inputs() + for f in fc['families'].values():f['accountingEvidenceId']='synthetic-accounting-review' + for k,v in [('operation',op),('forecast',fc),('tariffs',tar)]:ingest(store,PLANT,k,v,AT) + plan=run_once(store,AT);self.assertEqual(plan['status'],'optimal');store.close() + headers={'X-Enelix-Service-Token':TOKEN};prefix=f'/internal/v2/prognosis/{PLANT}/planner' + with patch('netplan_v4.service.datetime',wraps=datetime) as clock,TestClient(app) as c: + clock.now.return_value=AT + view=c.get(prefix,headers=headers).json() + self.assertIsNone(view['controlledTrial']);self.assertTrue(view['fresh']) + _,request=fixture();request.update(expectedPlanId=plan['planId'],expectedRevision=0,acceptEstimatedPeak=False) + grant=c.post(prefix+'/trial/arm',headers=headers,json=request) + self.assertEqual(grant.status_code,200,grant.text) + view=c.get(prefix,headers=headers).json() + self.assertEqual(view['controlledTrial']['sourceShadowPlanId'],plan['planId']) + self.assertFalse(view['liveEnabled']);self.assertFalse(view['plan']['liveEnabled']) + self.assertEqual(view['plan']['runMode'],'shadow') + self.assertEqual(c.post(prefix+'/ack',headers=headers,json={'planId':plan['planId'],'revision':0,'status':'applied'}).status_code,400) + self.assertEqual(c.post(prefix+'/trial/revoke',headers=headers,json={'sessionId':SESSION}).status_code,200) + self.assertIsNone(c.get(prefix,headers=headers).json()['controlledTrial']) + self.assertEqual(c.post(prefix+'/trial/arm',headers=headers,json=request).status_code,409) + def test_accounting_marker_cannot_label_aggregate_load_verified(self): + from netplan_v4.store import PlannerStore + s=PlannerStore(':memory:');op,fc,tar=inputs() + f=fc['families']['3'];f['loadBasis']='house_total';f['accountingEvidenceId']='synthetic-incorrect-label' + with self.assertRaises(ValueError):ingest(s,PLANT,'forecast',fc,AT) + s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_delivery.py b/services/netplan-v4/tests/test_delivery.py new file mode 100644 index 0000000..3cbe890 --- /dev/null +++ b/services/netplan-v4/tests/test_delivery.py @@ -0,0 +1,82 @@ +import importlib.util +from pathlib import Path +from tempfile import TemporaryDirectory +from datetime import datetime,timedelta,timezone +import unittest +ROOT=Path(__file__).resolve().parents[1] +def module(name,path): + spec=importlib.util.spec_from_file_location(name,path);value=importlib.util.module_from_spec(spec);spec.loader.exec_module(value);return value +publisher=module('publisher',ROOT/'integrations/netplan_v4_publisher.py') +hooks=module('hooks',ROOT/'install_hooks.py') +AT=datetime(2026,10,1,12,0,tzinfo=timezone.utc) + +class PublisherTest(unittest.TestCase): + def test_tariff_null_preserved_and_mode_explicit(self): + cfg={'tarif_bezug_modus':'statisch','tarif_bezug':'custom','tarif_bezug_fest':0.,'tarif_einspeisung_modus':'dynamisch','tarif_einspeisung':'dynamic','tarif_peak_fest':0.} + p=publisher.tariff_payload(cfg,AT);self.assertEqual(p['import']['staticChfKwh'],0.);self.assertEqual(p['peakChfKwMonth'],0.) + del cfg['tarif_bezug_modus'] + with self.assertRaises(ValueError):publisher.tariff_payload(cfg,AT) + def test_native_ckw_unit_zero_and_delivery_range(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'CHF_kWh','value':0.}]}] + p=publisher.ckw_payload('CKW Dynamisch Home',rows,AT-timedelta(hours=1),AT) + self.assertEqual(p['periods'][0]['value'],0.);self.assertEqual(p['periods'][0]['unit'],'CHF/kWh');self.assertEqual(p['periods'][0]['sourceKind'],'published_interval') + del rows[0]['end_timestamp'] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + def test_unknown_unit_and_future_publication_rejected(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'unknown','value':100.}]}] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + rows[0]['integrated'][0]['unit']='CHF_kWh' + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,AT+timedelta(minutes=1),AT) + def test_forecast_not_relabelled_as_base_load(self): + p=publisher.forecast_payload([(23,{AT:5000.},{AT:2000.})],AT) + self.assertEqual(p['families']['23']['loadBasis'],'house_total');self.assertIsNone(p['families']['23']['trainedUntil']) + +class InstallerTest(unittest.TestCase): + def fixtures(self,root): + files={'prognosis-manager-enelix2/api/main.py':'def f():\n _portal_store_configuration(anlagen_id, configuration)\n', + 'prognosis-manager-enelix2/forecast_engine/main.py':'def f():\n for a in []:\n for key, default in []:\n d[key] = float(d.get(key) or default)\n for c in []:\n p_3 = v3.predict(data_obj, p_1, p_2)\n', + 'prognosis-manager-enelix2/tariff_importer/main.py':'def f():\n for a in []:\n for b in []:\n for c in []:\n if True:\n return candidate_points\n', + 'license/server.mjs':'const server = createServer(async (req, res) => {\n if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {status:"ok"});\n});', + 'license/public/app.js':'// Parallel GUI work. Must remain byte-identical.'} + for name,data in files.items():p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(data) + return files + def test_install_idempotent_and_verified_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);originals=self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root) + self.assertFalse(hooks.plan(root));self.assertEqual((root/'license/public/app.js').read_text(),originals['license/public/app.js']) + hooks.rollback(receipt) + for name,data in originals.items():self.assertEqual((root/name).read_text(),data) + self.assertFalse((root/'license/public/netplan-v4.html').exists()) + def test_concurrent_changes_never_overwritten(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);entries=hooks.plan(root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Concurrent') + with self.assertRaises(ValueError):hooks.apply(entries,root) + self.assertTrue(p.read_text().endswith('// Concurrent')) + def test_later_changes_block_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Later') + with self.assertRaises(ValueError):hooks.rollback(receipt) + self.assertTrue(p.read_text().endswith('// Later')) + def test_changed_or_ambiguous_anchor_fails_closed(self): + with self.assertRaises(ValueError):hooks.once('x x','x','y') + with self.assertRaises(ValueError):hooks.patch_api('def f():pass') + +class HorizonValidationTest(unittest.TestCase): + def test_synthetic_48h_energy_soc_limits(self): + import math + from netplan_v4.domain import Battery,Step,Price,Limits + from netplan_v4.optimizer import optimize + steps=[] + for i in range(576): + solar=max(0.,math.sin(math.pi*((i%288)/12-6)/12))*10000. + steps.append(Step(AT+timedelta(minutes=5*i),3000.,solar,Price(.15 if i%288<72 else .35),Price(.1))) + b=Battery('synthetic',40.,20.,10.,90.,10000.,8000.,AT,grid_charging=True) + p=optimize(steps,[b],at=AT,limits=Limits(import_w=12000.,export_w=10000.),observed_peaks={'2026-10':8.},peak_prices={'2026-10':5.}) + self.assertTrue(p['executable']);self.assertEqual(len(p['points']),576) + for s,x in zip(steps,p['points']): + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=3) + self.assertLessEqual(x['gridTargetW'],12000.001);self.assertGreaterEqual(x['gridTargetW'],-10000.001) + self.assertTrue(9.999<=x['socEndPercent']['synthetic']<=90.001) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['synthetic'],19.999) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_acceptance.py b/services/netplan-v4/tests/test_forecast_acceptance.py new file mode 100644 index 0000000..eb89b2e --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_acceptance.py @@ -0,0 +1,80 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from forecast_acceptance import source_paths, verify_forecast_bundle + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('forecast_release_test', ROOT / 'release_preflight.py') +preflight = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preflight) + + +class ForecastAcceptanceTest(unittest.TestCase): + def fixture(self, root): + src, bundle = root / 'source', root / 'bundle' + for path in (src, bundle): + path.mkdir() + for name, value in {'main.py': 'x=1\n', 'telemetry_quality.py': 'x=2\n', 'requirements.txt': 'pandas\n'}.items(): + (path / name).write_text(value) + manifest = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in source_paths(src)} + (bundle / 'SOURCE_MANIFEST.json').write_text(json.dumps(manifest)) + return src, bundle + + def test_exact_source_is_verified(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_modified_development_source_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (src / 'main.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_extra_staged_python_file_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'surprise.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_symlink_is_rejected(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'main.py').unlink() + (bundle / 'main.py').symlink_to(src / 'main.py') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_models_secrets_and_databases_not_staged(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + for name in ('.env', 'model.pkl', 'users.db'): + (src / name).write_text('not a source file') + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_forecast_checks_have_no_network_or_live_mounts(self): + args = next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='forecast_candidate_python311') + self.assertEqual(args[args.index('--network')+1], 'none') + self.assertEqual(args[args.index('--user')+1], '1000:1000') + self.assertIn('--read-only', args) + for forbidden in ('--volume', '-v', '--env-file', '--privileged', 'up', 'restart'): + self.assertNotIn(forbidden, args) + + def test_runner_does_not_import_application_main(self): + source = (ROOT / 'acceptance/check_forecast.py').read_text() + tree = ast.parse(source) + imports = {a.name for node in ast.walk(tree) if isinstance(node, ast.Import) for a in node.names} + self.assertNotIn('main', imports) + self.assertNotIn('run_forecast(', source) + self.assertNotIn('get_configs(', source) + + +if __name__ == '__main__': + unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_quality.py b/services/netplan-v4/tests/test_forecast_quality.py new file mode 100644 index 0000000..0a81f63 --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_quality.py @@ -0,0 +1,31 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from netplan_v4.forecast_quality import assess_family +from netplan_v4.service import ingest,run_once +from netplan_v4.store import PlannerStore +from test_v4 import inputs,AT,AID + +class ForecastQualityTest(unittest.TestCase): + def test_zero_load_not_free_energy_assumption(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_confirmed_pv_only_plant_is_supported(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)],'zeroLoadConfirmed':True} + self.assertTrue(assess_family(f)['valid']) + def test_boolean_and_nan_are_not_forecast_powers(self): + for value in (float('nan'),True,-1.,None): + f={'points':[{'loadW':value,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_selected_invalid_family_not_silently_replaced(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));op,fc,tar=inputs() + for p in fc['families']['3']['points']:p['loadW']=0. + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(s,AID,k,v,AT) + plan=run_once(s,AT) + self.assertEqual(plan['status'],'awaiting_inputs');self.assertIn('all-zero',plan['reason']);self.assertIsNone(s.current(AID));s.close() + def test_nonzero_family_remains_usable(self): + f={'points':[{'loadW':50.,'pvW':0.} for _ in range(3)]} + self.assertTrue(assess_family(f)['valid']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_integrated_deployment.py b/services/netplan-v4/tests/test_integrated_deployment.py new file mode 100644 index 0000000..d9ed2d7 --- /dev/null +++ b/services/netplan-v4/tests/test_integrated_deployment.py @@ -0,0 +1,71 @@ +import ast +import hashlib +import importlib.util +import io +import contextlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('integrated_deployment_test',ROOT/'deploy_integrated_shadow.py') +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class IntegratedDeploymentTest(unittest.TestCase): + def test_exact_server_scope_and_no_symcon(self): + scope=d.compose_groups() + self.assertEqual(set(scope),{'v4','forecast','portal'}) + self.assertEqual(scope['forecast'][2],['api','forecast-engine','tariff-importer']) + self.assertNotIn('license-admin',scope['portal'][2]) + self.assertNotIn('/var/lib/symcon',(ROOT/'deploy_integrated_shadow.py').read_text()) + def test_sqlite_backup_is_consistent_and_source_unchanged(self): + with tempfile.TemporaryDirectory() as temp: + source=Path(temp)/'before.sqlite';dest=Path(temp)/'backup.sqlite' + con=sqlite3.connect(source);con.execute('CREATE TABLE test(value INTEGER)') + con.execute('INSERT INTO test VALUES (37)');con.commit();con.close() + before=source.read_bytes() + with patch.object(d.os,'geteuid',return_value=1000):result=d.backup_database(source,dest) + self.assertEqual(result['integrity'],'ok');self.assertEqual(before,source.read_bytes()) + check=sqlite3.connect(dest);self.assertEqual(check.execute('SELECT value FROM test').fetchone()[0],37);check.close() + self.assertEqual(result['sha256'],hashlib.sha256(dest.read_bytes()).hexdigest()) + def test_missing_database_not_fabricated(self): + with tempfile.TemporaryDirectory() as temp: + dst=Path(temp)/'backup' + self.assertEqual(d.backup_database(Path(temp)/'missing',dst),{'exists':False}) + self.assertFalse(dst.exists()) + def test_scope_uses_no_environment_dump_or_control_activation(self): + text=(ROOT/'deploy_integrated_shadow.py').read_text() + self.assertNotIn('.Config.Env',text) + self.assertNotIn('NetzfahrplanAktiv',text) + self.assertNotIn('IPS_RequestAction',text) + self.assertNotIn('PROGNOSIS_SERVICE_TOKEN',text) + ast.parse(text) + def test_build_and_tests_precede_all_service_recreation(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp)/'netplan';root.mkdir();project=Path(temp)/'forecast';project.mkdir() + portal=Path(temp)/'portal';portal.mkdir();(root/'acceptance').mkdir() + commands=[] + def fake(args,**kwargs): + commands.append(list(args));out='' + if 'ps' in args:out='0123456789abcdef\n' + elif 'inspect' in args: + out='sha256:'+('a'*64) if '{{.Image}}' in args else '/some/compose.yaml' + elif 'exec' in args: + out=json.dumps({'health':{'mode':'shadow','liveEnabled':False},'inputEventsByKind':{},'lastRunStatus':None}) + return SimpleNamespace(returncode=0,stdout=out) + with patch.object(d,'ROOT',root),patch.object(d,'PROJECT',project),patch.object(d,'PORTAL',portal),patch.object(d,'verify_sources',return_value={}),patch.object(d.os,'geteuid',return_value=1000),patch.object(d.subprocess,'run',side_effect=fake),patch('install_hooks.plan',return_value={}),patch('install_hooks.apply',return_value=None),contextlib.redirect_stdout(io.StringIO()): + d.run('e3a08f9e-af12-4695-99bd-8b51c0520021') + ups=[i for i,c in enumerate(commands) if 'up' in c] + self.assertEqual(len(ups),3) + test_at=next(i for i,c in enumerate(commands) if '--test-only' in c) + build_at=max(i for i,c in enumerate(commands) if 'build' in c) + self.assertLess(test_at,min(ups));self.assertLess(build_at,min(ups)) + self.assertTrue(all('--no-build' in commands[i] and '--no-deps' in commands[i] for i in ups)) + report=json.loads(next((root/'deployment-reports').glob('*/DEPLOYMENT.json')).read_text()) + self.assertEqual(report['status'],'server_shadow_installed') + self.assertFalse(report['liveEnabled']);self.assertFalse(report['productionReady']) diff --git a/services/netplan-v4/tests/test_measurement_pipeline.py b/services/netplan-v4/tests/test_measurement_pipeline.py new file mode 100644 index 0000000..ab8cfa6 --- /dev/null +++ b/services/netplan-v4/tests/test_measurement_pipeline.py @@ -0,0 +1,161 @@ +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once, status +from test_v4 import inputs, AID, TOKEN + +NOW=int(datetime(2026,10,2,12,tzinfo=timezone.utc).timestamp()) + +def config(**changes): + c={'datasetId':'physical-v1','mappingSha256':'a'*64,'inventorySha256':'b'*64, + 'formula':'physical_sum_v1','solarReference':None,'minimumCoverage':.95, + 'maximumGapSeconds':10,'minimumTrainingHours':1,'historyDays':14,'sources':[]} + for i,(k,role) in enumerate((('grid','grid'),('pv','pv'),('battery','physical_storage'),('sdl','sdl_request'))): + c['sources'].append({'key':k,'role':role,'variableId':101+i,'factorToW':1,'maxAgeSeconds':60}) + c.update(changes);return c + +def record(t, c=None, **values): + c=c or config(); v={'grid':3000.,'pv':5000.,'battery':2000.,'sdl':0.,**values} + return {'schemaVersion':1,'kind':'raw_accounting_capture','installationId':AID, + 'capturedAt':m.iso(t),'captureStartedAt':m.iso(t),'mappingSha256':c['mappingSha256'], + 'reportedInventorySha256':c['inventorySha256'],'raw':{s['key']:{'variableId':s['variableId'],'value':v.get(s['key'],0), + 'sourceUpdatedAt':t,'issues':[]} for s in c['sources']}} + +class MeasurementPipelineTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.path=str(Path(self.tmp.name)/'pipeline.sqlite') + self.s=PlannerStore(self.path);self.c=config();m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self):self.s.close();self.tmp.cleanup() + def batch(self,rows,now=NOW,plant=AID):return m.ingest_batch(self.s.con,plant,{'version':1,'datasetId':'physical-v1','records':rows},now) + def history(self,hours=2): + rows=[record(t) for t in range(NOW-hours*3600,NOW+1,30)] + for i in range(0,len(rows),120):self.batch(rows[i:i+120]) + def test_append_ack_idempotence(self): + r=record(NOW);self.assertEqual(self.batch([r])['stored'],1) + a=self.batch([r]);self.assertEqual(a['duplicates'],1);self.assertEqual(a['acceptedThrough'],m.iso(NOW)) + def test_conflict_rolls_back_whole_batch(self): + self.batch([record(NOW)]) + with self.assertRaises(ValueError):self.batch([record(NOW-30),record(NOW,grid=9000)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_registration_immutable(self): + self.assertEqual(m.register_dataset(self.s.con,AID,self.c,NOW)['status'],'configured') + with self.assertRaises(ValueError):m.register_dataset(self.s.con,AID,{**self.c,'minimumCoverage':1.},NOW) + def test_plants_isolated(self): + with self.assertRaises(ValueError):self.batch([record(NOW)],plant='30509683-4569-49e4-848f-4905e4cc813a') + def test_mapping_is_not_device_supplied(self): + r=record(NOW);r['mappingSha256']='c'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_no_unknown_text_persisted(self): + r=record(NOW);r['unknown']='do-not-store';r['raw']['grid']['value']='do-not-store';r['assessment']={'status':'verified'} + self.batch([r]);data=self.s.con.execute('SELECT value FROM planner_observations').fetchone()[0] + self.assertNotIn('do-not-store',data);self.assertIn('null',data);self.assertNotIn('verified',data) + def test_missing_zero_not_invented(self): + r=record(NOW);r['raw'].pop('grid');p=m.project(r,self.c,AID,NOW) + self.assertIsNone(p['raw']['grid']['value']);self.assertFalse(p['raw']['grid']['valid']) + def test_boolean_not_numeric(self): + p=m.project(record(NOW,grid=True),self.c,AID,NOW);self.assertFalse(p['raw']['grid']['valid']) + def test_batch_order_required(self): + with self.assertRaises(ValueError):self.batch([record(NOW),record(NOW-30)]) + def test_batch_size_limit(self): + with self.assertRaises(ValueError):self.batch([record(NOW)]*121) + def test_no_naive_time(self): + r=record(NOW);r['capturedAt']='2026-10-02T12:00:00' + with self.assertRaises(ValueError):self.batch([r]) + def test_future_source_not_rejuvenated(self): + r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1 + self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid']) + def test_profile_uses_physical_not_virtual(self): + r=record(NOW);r['raw']['virtual_ev']={'value':-90000} + self.assertEqual(m.physical_value({'grid':3000,'pv':5000,'battery':2000},self.c),6000) + def test_zero_numeric_valid(self): + p=m.project(record(NOW,grid=0,pv=0,battery=0),self.c,AID,NOW);self.assertTrue(p['raw']['grid']['valid']) + def projected(self,rows):return [m.project(r,self.c,AID,NOW) for r in rows] + def test_rollup_time_weighted(self): + rows=[record(t,grid=3000 if t=a-1e-10 for a,b in zip(slopes,slopes[1:])));self.assertAlmostEqual(slopes[-1],5.) + self.assertAlmostEqual(o.incremental_cost(1.,.8,5.),0.) + def test_hard_cap_beats_outlook(self): + p=self.solve(limits=Limits(import_w=1000.),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(max(x['gridTargetW'] for x in p['points']),1000.001) + def test_no_fixed_cap_is_supported(self): + p=self.solve(limits=Limits(),peak_outlooks={'2026-10':outlook()}) + self.assertTrue(p['executable']);self.assertTrue(all(x['importLimitW'] is None for x in p['points'])) + def test_monthly_cap_beats_outlook(self): + p=self.solve(limits=Limits(manager_month_limits_w={10:1000.}),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_estimated_basis_not_in_measured_map(self): + p=self.solve(peak_context={'2026-10':estimate(1.)}) + self.assertEqual(p['measuredPeaksKw'],{});self.assertEqual(p['peakBasisKw']['2026-10'],1.) + self.assertTrue(p['peakCostIsEstimate']) + def test_context_cannot_lie_about_numerical_basis(self): + self.assertFalse(self.solve(peak_context={'2026-10':estimate(3.)})['executable']) + def test_no_free_cap_source(self): + with self.assertRaises(ValueError):basis_record(estimate(source='manager_cap'),'2026-10',AT,True) + def test_overlap_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),future_from=AT)})['executable']) + def test_future_observation_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),issued_at=AT+timedelta(seconds=1))})['executable']) + def test_wrong_month_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-11':outlook()})['executable']) + def test_invalid_probabilities_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':outlook(scenarios=((5.,.8),))})['executable']) + def test_no_history_no_outlook_not_fake_allowance(self): + self.assertIsNone(empirical_rest_month([],month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p')) + def test_empirical_requires_comparable_complete_history(self): + records=[] + for n in range(20): + d=AT.astimezone(ZURICH).replace(hour=0,minute=0)-timedelta(days=n+1) + records.append({'day':d.strftime('%Y-%m-%d'),'peakKw':3.+n%4,'observedAt':(d+timedelta(days=1)).isoformat(),'quality':'estimated','complete':True,'controlPolicyId':'p'}) + args=dict(month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p') + o=empirical_rest_month(records,**args);self.assertIsNotNone(o);o.validate(AT,args['horizon_end']) + for r in records:r['controlPolicyId']='different' + self.assertIsNone(empirical_rest_month(records,**args)) + + +class EstimatedBasisServiceTest(unittest.TestCase): + def setUp(self):self.tmp=TemporaryDirectory();self.path=str(Path(self.tmp.name)/'planner.sqlite');self.s=PlannerStore(self.path) + def tearDown(self):self.s.close();self.tmp.cleanup() + def load(self,when=AT,quarter=False): + op,fc,tar=inputs(when);op['measuredPeaks']={};op['planningPeaks']={'2026-10':estimate(at=when)} + if quarter: + op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.2,'measuredSeconds':int((when-AT).total_seconds()),'quality':'estimated','source':'power_history_estimate','coverage':1.} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,when) + return op + def allow(self):self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},self.s.settings(AID)['revision'],AT) + def test_opt_in_required(self): + self.load();self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + self.allow();p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertTrue(p['peakCostIsEstimate']) + self.assertEqual(self.s.peaks(AID),{}) + def test_no_peak_not_invented_even_when_opted_in(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.allow() + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + def test_historical_seed_independent_from_live_operation(self): + self.allow();op,fc,tar=inputs();op['measuredPeaks']={} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + ingest(self.s,AID,'planning_basis',{'version':1,'eventId':'seed','observedAt':AT.isoformat(),'peaks':{'2026-10':estimate()}},AT) + p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertEqual(p['measuredPeaksKw'],{}) + def test_estimates_persist_but_plans_do_not_increase_them(self): + self.load();self.allow();p=run_once(self.s,AT) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['kw'],5.) + self.s.close();self.s=PlannerStore(self.path) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['quality'],'estimated') + def test_verified_source_has_precedence_even_if_lower_than_estimate(self): + self.load();self.allow();self.s.initialize_peak(AID,'2026-10',4.,'verified_month_history',AT) + p=run_once(self.s,AT);self.assertEqual(p['measuredPeaksKw'],{'2026-10':4.});self.assertFalse(p['peakCostIsEstimate']) + def test_quarter_estimate_never_disguised(self): + self.load(AT+timedelta(minutes=5),True);self.allow();p=run_once(self.s,AT+timedelta(minutes=5)) + self.assertTrue(p['executable']);self.assertEqual(p['inputQuality']['quarter'],'estimated') + def test_incomplete_quarter_refused(self): + op,_,_=inputs();op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.1,'measuredSeconds':0,'quality':'estimated','source':'power_history_estimate','coverage':.8} + with self.assertRaises(ValueError):ingest(self.s,AID,'operation',op,AT) + def test_new_settings_load_old_rows(self): + old=self.s.settings(AID);old.pop('revision');old.pop('measurementPolicy');old.pop('peakOutlookPolicy');old.pop('peakOutlookReliance') + with self.s.con:self.s.con.execute('INSERT INTO planner_settings VALUES(?,?,?)',(AID,1,json.dumps(old))) + self.assertEqual(self.s.settings(AID)['measurementPolicy'],'verified_only') + self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},1,AT) + def test_http_explicit_estimate_optin_and_no_live(self): + app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as c: + url=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':0,'changes':{'measurementPolicy':'allow_estimates'}}).status_code,200) + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + + +class RuntimeMeterTest(unittest.TestCase): + def test_exact_samples_still_estimates(self): + r=integrate_power([(0,1000.,0.,'p'),(60,1000.,.02,'p'),(120,1000.,.04,'p')],0,120) + self.assertAlmostEqual(r['importKwh'],1/30);self.assertFalse(r['billingEvidence']) + def test_gap_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,0.,'p'),(180,1000.,.1,'p')],0,180)) + def test_reset_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,1.,'p'),(60,1000.,0.,'p')],0,60)) + def test_control_policy_change_not_comparable(self):self.assertIsNone(integrate_power([(0,1000.,0.,'a'),(60,1000.,1.,'b')],0,60)) + def test_only_completed_quarters_update_peak(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));mid='symcon-active-import:'+'a'*64 + for i in range(16): + at=AT+timedelta(minutes=i) + obs={'meterId':mid,'sampleAt':at.isoformat(),'powerW':2000.,'totalImportKwh':10+i/30,'controlPolicyId':'p'} + with s.con:observe(s.con,AID,obs,at) + if i<15:self.assertEqual(assumptions(s.con,AID),{}) + self.assertAlmostEqual(assumptions(s.con,AID)['2026-10']['kw'],2.) + self.assertEqual(s.peaks(AID),{});s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_receiver_contract.py b/services/netplan-v4/tests/test_receiver_contract.py new file mode 100644 index 0000000..1377ae8 --- /dev/null +++ b/services/netplan-v4/tests/test_receiver_contract.py @@ -0,0 +1,45 @@ +import copy +from datetime import timedelta +import tempfile +from pathlib import Path +import unittest +from netplan_v4.receiver_contract import control_context, provenance +from netplan_v4.store import PlannerStore +from netplan_v4.service import ingest, run_once, status +from test_v4 import inputs, AT, AID + +class ReceiverContractTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'qa.sqlite')) + op,fc,tar=inputs() + for kind,value in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,kind,value,AT) + def tearDown(self):self.s.close();self.tmp.cleanup() + def test_published_plan_and_response_bound_to_same_plant(self): + p=run_once(self.s,AT);r=status(self.s,AID,AT) + self.assertEqual(p['installationId'],AID);self.assertEqual(r['installationId'],AID) + self.assertEqual(r['receiverProtocolVersion'],1);self.assertTrue(r['fresh']) + self.assertFalse(r['liveEnabled']);self.assertFalse(p['liveEnabled']) + self.assertEqual(p['inputRefs'],{'operation':'op1','forecast':'fc1','tariffs':'tar1'}) + def test_control_context_tracks_stable_policy_not_volatile_soc(self): + op=inputs()[0];a=control_context(op);op['batteries'][0]['socPercent']=80.;op['gridW']=1000. + self.assertEqual(a,control_context(op));op['batteries'][0]['minSocPercent']=15. + self.assertNotEqual(a,control_context(op)) + def test_context_is_snapshot_not_reference_to_future_changes(self): + op,fc,tar=inputs();p=provenance({'operation':op,'forecast':fc,'tariffs':tar}) + op['limits']['importW']=100.;self.assertEqual(p['controlContext']['limits']['importW'],10000.) + def test_no_plan_for_another_installation(self): + run_once(self.s,AT);r=status(self.s,'00000000-0000-4000-8000-000000000002',AT) + self.assertIsNone(r['plan']);self.assertFalse(r['fresh']) + def test_shadow_ack_is_not_execution(self): + p=run_once(self.s,AT) + self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,step=p['points'][0]['time'],status='shadow_seen') + self.assertEqual(status(self.s,AID,AT)['acknowledgement']['status'],'shadow_seen') + with self.assertRaises(ValueError):self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,status='applied') + def test_new_configuration_invalidates_fresh_flag(self): + run_once(self.s,AT);self.s.save_settings(AID,{'family':'23'},0,AT+timedelta(seconds=1)) + self.assertFalse(status(self.s,AID,AT+timedelta(seconds=2))['fresh']) + def test_rearm_default_matches_php_contract(self): + op=inputs()[0];op['batteries'][0]['rearmSocPercent']=None + self.assertEqual(control_context(op)['batteries']['b']['rearmSocPercent'],10.) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_release_preflight.py b/services/netplan-v4/tests/test_release_preflight.py new file mode 100644 index 0000000..977a292 --- /dev/null +++ b/services/netplan-v4/tests/test_release_preflight.py @@ -0,0 +1,50 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +import unittest + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('release_preflight_test',ROOT/'release_preflight.py') +preflight=importlib.util.module_from_spec(spec);spec.loader.exec_module(preflight) + +class ReleasePreflightTest(unittest.TestCase): + def test_preflight_never_restarts_existing_services(self): + for name,args,cwd,timeout in preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021'): + self.assertNotIn('up',args);self.assertNotIn('restart',args);self.assertNotIn('--with-bridges',args) + self.assertGreater(timeout,0) + first=preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021')[0][1] + self.assertIn('--test-only',first) + def test_php_runs_without_network_or_writeable_root(self): + args=next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='php83_offline') + self.assertIn('--read-only',args);self.assertEqual(args[args.index('--network')+1],'none') + self.assertEqual(args[args.index('--user')+1],'1000:1000') + def test_native_bundle_requires_same_development_source(self): + with TemporaryDirectory() as d: + root=Path(d)/'root';repo=Path(d)/'repo';out=root/'acceptance/php-src' + out.mkdir(parents=True);repo.mkdir() + source=b'=-1e-5 for x in p['points'][:3])) + def test_roundtrip_90_percent(self): + p=solve(seq([0.]*3+[5000.]*3,pv=[5000.]*3+[0.]*3,buy=[.3]*6,sell=[0.]*6),[batt(soc_percent=10.,grid_charging=False,max_discharge_w=5000.)]) + c=sum(max(0,x['batteryTargetW']) for x in p['points']);d=sum(max(0,-x['batteryTargetW']) for x in p['points']);self.assertAlmostEqual(d/c,.9,places=5) + def test_nulleinspeisung(self): + p=solve(seq([0.]*3,pv=[10000.]*3),limits=Limits(export_w=0.));self.assertTrue(p['executable']);self.assertTrue(all(abs(x['gridTargetW'])<1e-4 for x in p['points'])) + def test_nulltariffs(self): + p=solve(seq([1000.]*3,buy=[0.]*3,sell=[0.]*3));self.assertEqual(p['cashCostChf'],0.) + def test_negative_prices(self): + p=solve(seq([1000.]*3,buy=[-.1]*3));self.assertLess(p['energyCostChf'],0.) + def test_asymmetric_limits_and_balance(self): + steps=seq([0.]*3+[3000.]*3,pv=[5000.]*3+[0.]*3) + p=solve(steps,[batt(max_charge_w=700.,max_discharge_w=300.)]);self.assertTrue(p['executable']) + for s,x in zip(steps,p['points']): + self.assertLessEqual(x['batteryTargetW'],700.001);self.assertGreaterEqual(x['batteryTargetW'],-300.001) + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=4) + self.assertTrue(9.999<=x['socEndPercent']['b']<=90.001) + def test_stale_soc_rejected(self):self.assertFalse(solve(seq([0.]*3),[batt(measured_at=AT-timedelta(hours=1))])['executable']) + def test_soc_not_fabricated(self):self.assertFalse(solve(seq([0.]*3),[batt(soc_percent=5.)])['executable']) + def test_peak_increment_only_and_baseline(self): + p=solve(seq([2000.]*3),observed_peaks={'2026-10':1.5},peak_prices={'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],2.5);self.assertAlmostEqual(p['baselineCashCostChf'],2.65) + def test_peak_blocks_unprofitable_charging(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.)],observed_peaks={'2026-10':1.},peak_prices={'2026-10':20.}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_month_switch(self): + at=datetime(2026,9,30,21,45,tzinfo=timezone.utc) + p=solve(seq([2000.]*6,at=at),observed_peaks={'2026-09':2.,'2026-10':.5},peak_prices={'2026-09':5.,'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],7.5) + def test_manager_cap_is_not_measured_peak(self): + p=solve(seq([1000.]*3),limits=Limits(manager_month_limits_w={10:1500.}),observed_peaks={'2026-10':0.},peak_prices={'2026-10':5.}) + self.assertEqual(p['measuredPeaksKw']['2026-10'],0.);self.assertAlmostEqual(p['additionalPeakCostChf'],5.) + def test_infeasible_not_fake_residual(self): + p=solve(seq([10000.]*3),limits=Limits(import_w=1000.));self.assertFalse(p['executable']);self.assertEqual(p['points'],[]) + def test_elapsed_quarter_required(self): + s=seq([1000.],at=AT+timedelta(minutes=10));self.assertFalse(solve(s)['executable']) + p=solve(s,quarter_history={AT:QuarterPast(.5,600)},peak_prices={'2026-10':5.},observed_peaks={'2026-10':0.}) + self.assertAlmostEqual(p['plannedPeaksKw']['2026-10'],7/3) + def test_partial_first_step(self): + s=seq([1000.]*3);s[0]=replace(s[0],start=AT+timedelta(seconds=10),seconds=290) + p=solve(s,quarter_history={AT:QuarterPast(.02,10)});self.assertTrue(p['executable']);self.assertEqual(p['validFrom'],s[0].start.isoformat()) + def test_terminal_guard(self): + p=solve(seq([0.]*3,sell=[10.]*3),[batt(soc_percent=50.,grid_charging=False)]);self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],49.999) + def test_grid_charging_opt_in(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.,grid_charging=False)]);self.assertTrue(all(x['batteryTargetW']<=1e-5 for x in p['points'])) + +class StoreAndSelectionTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def test_settings_revision_and_shadow_only(self): + s=self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(s['revision'],1) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'family':'3'},0,AT) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'runMode':'live'},1,AT) + def test_request_during_calculation_survives(self): + self.store.request(AID,'manual',AT);claim=self.store.claim(AT);self.store.request(AID,'prices_changed',AT) + p=solve(seq([0.]*3),config_revision=0) + with self.assertRaises(ValueError):self.store.publish_shadow(AID,p,0,AT,claim['sequence'],claim['lease_token']) + self.store.finish(claim);self.assertIsNotNone(self.store.claim(AT)) + def test_month_peak_persistent_and_monotonic(self): + self.store.initialize_peak(AID,'2026-10',20.,'meter_month_register',AT) + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',15.,'meter_month_register',AT) + self.store.close();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'));self.assertEqual(self.store.peaks(AID)['2026-10'],20.) + def test_cap_is_not_valid_peak_source(self): + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',20.,'configured_limit',AT) + def test_meter_peak_requires_complete_quarter(self): + self.store.initialize_peak(AID,'2026-10',0.,'verified_new_month',AT) + for i in range(3):r=self.store.record_import_interval(AID,AT+timedelta(minutes=5*i),.5,AT+timedelta(hours=1)) + self.assertEqual(r['quarterPeakKw'],6.);self.assertEqual(self.store.peaks(AID)['2026-10'],6.) + def test_shadow_ack_cannot_claim_applied(self): + p=solve(seq([0.]*3),config_revision=0);self.store.publish_shadow(AID,p,0,AT) + with self.assertRaises(ValueError):self.store.acknowledge(AID,p['planId'],0,AT,status='applied') + self.store.acknowledge(AID,p['planId'],0,AT,status='shadow_seen') + def test_family_registry_extensible(self): + r=default_registry();r.register(Family('42','p42','l42','g42','Future'));self.assertEqual(r.get('42').pv,'p42') + def test_sdl_removed_once(self): + self.assertEqual(split_base_load(15000,[3000],2000),10000);self.assertEqual(split_base_load(13000,[3000],2000,True),10000) + with self.assertRaises(ValueError):split_base_load(100,[1000],0) + def test_auto_uses_comparable_cost_and_margin(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT,c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['family'],'23');self.assertEqual(choose_family('auto','3',scores,now=AT,margin_chf=6.)['family'],'3') + def test_auto_no_future_outcomes(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT+timedelta(hours=1),c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['mode'],'collecting') + def test_training_schedule_and_validation(self): + self.assertTrue(training_due(AT-timedelta(days=1),AT));self.assertFalse(training_due(AT-timedelta(days=1),AT,'weekly')) + self.assertFalse(promote_candidate(active_cost=10,candidate_cost=1,valid_coverage=True,no_data_leakage=False,constraints_passed=True)) + +class ServiceIntegrationTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def load(self,values=None,now=AT): + values=values or inputs() + for kind,value in zip(('operation','forecast','tariffs'),values):ingest(self.store,AID,kind,value,now) + def test_pipeline_shadow_plan(self): + self.load();p=run_once(self.store,AT);self.assertTrue(p['executable']);self.assertEqual(p['runMode'],'shadow');self.assertTrue(status(self.store,AID,AT)['fresh']);self.assertEqual(self.store.current(AID)['planId'],p['planId']) + def test_exact_model_selection(self): + self.load();self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(run_once(self.store,AT)['sourceFamily'],'23') + def test_one_tick_not_every_poll(self): + self.load();run_once(self.store,AT);self.assertEqual(run_once(self.store,AT)['status'],'idle') + def test_stale_operation_visible(self): + self.load();p=run_once(self.store,AT+timedelta(minutes=5));self.assertEqual(p['status'],'awaiting_inputs');self.assertIsNone(self.store.current(AID)) + def test_missing_peak_not_invented(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_static_zero_roundtrip(self): + op,fc,tar=inputs();tar['import']['staticChfKwh']=0.;tar['export']['staticChfKwh']=0.;tar['peakChfKwMonth']=0.;self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['cashCostChf'],0.) + def test_immutable_id_conflict(self): + op,fc,tar=inputs();self.load((op,fc,tar));self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'duplicate');op['gridW']=123. + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_older_input_never_overwrites_current(self): + op,fc,tar=inputs();self.load((op,fc,tar));op['observedAt']=(AT-timedelta(seconds=1)).isoformat();op['eventId']='old' + self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'archived_older');self.assertEqual(latest(self.store,AID,'operation')['eventId'],'op1') + def test_unpublished_dynamic_tail_not_executed(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_published_negative_price_and_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(minutes=15)).isoformat(),'value':-2.,'unit':'Rp/kWh','observedAt':AT.isoformat(),'sourceKind':'published_interval'}]} + ingest(self.store,AID,'prices',price,AT);p=run_once(self.store,AT);self.assertEqual(len(p['points']),3);self.assertEqual(p['points'][0]['importPriceChfKwh'],-.02) + def test_carried_price_does_not_extend_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(hours=48)).isoformat(),'value':.1,'unit':'CHF/kWh','observedAt':AT.isoformat(),'sourceKind':'carried_forward'}]} + ingest(self.store,AID,'prices',price,AT);self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_aggregate_load_warning(self): + op,fc,tar=inputs();fc['families']['3']['loadBasis']='house_total';self.load((op,fc,tar));self.assertTrue(run_once(self.store,AT)['warnings']) + def test_extra_credentials_not_stored(self): + op,fc,tar=inputs();op['password']='do-not-store' + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_http_auth_allowlist_and_revision_conflict(self): + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + path=f'/internal/v2/prognosis/{AID}/planner';headers={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(client.get(path).status_code,401);self.assertEqual(client.get(path,headers=headers).status_code,200) + other='30509683-4569-49e4-848f-4905e4cc813a';self.assertEqual(client.get(path.replace(AID,other),headers=headers).status_code,403) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'23'}}).status_code,200) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'3'}}).status_code,409) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + def test_body_limit_and_no_legacy_db(self): + with self.assertRaises(ValueError):create_app(str(Path(self.temp.name)/'users.db'),TOKEN,[AID]) + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + r=client.post(f'/internal/v2/prognosis/{AID}/planner/inputs/forecast',json={'x':'a'*2000001});self.assertEqual(r.status_code,413) + +if __name__=='__main__':unittest.main() diff --git a/tests/NetzfahrplanV4DatenTest.php b/tests/NetzfahrplanV4DatenTest.php new file mode 100644 index 0000000..d4c69b9 --- /dev/null +++ b/tests/NetzfahrplanV4DatenTest.php @@ -0,0 +1,21 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($process);fclose($pipes[0]); + $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($process),$out.$err); + self::assertStringContainsString('TOTAL 21 application-data checks passed',$out); + } +} diff --git a/tests/V4ApplicationData/checks.php b/tests/V4ApplicationData/checks.php new file mode 100644 index 0000000..5f018e9 --- /dev/null +++ b/tests/V4ApplicationData/checks.php @@ -0,0 +1,83 @@ +2,'VariableUpdated'=>$sensor[$id]['updated'],'VariableChanged'=>$sensor[$id]['changed']];} +function IPS_GetObject($id):array {return ['ParentID'=>200,'ObjectIdent'=>'s'.$id];} +function GetValue($id) {global $sensor;return $sensor[$id]['value'];} +foreach(['CURLOPT_POST','CURLOPT_RETURNTRANSFER','CURLOPT_FOLLOWLOCATION','CURLOPT_CONNECTTIMEOUT','CURLOPT_TIMEOUT','CURLOPT_SSL_VERIFYPEER','CURLOPT_SSL_VERIFYHOST','CURLOPT_HTTPHEADER','CURLOPT_POSTFIELDS','CURLINFO_HTTP_CODE'] as $i=>$k)if(!defined($k))define($k,$i+100); +function curl_init($url){$o=new \stdClass();$o->url=$url;return $o;} +function curl_setopt_array($h,$options){$h->options=$options;return true;} +function curl_exec($h){global $sent,$mode;$p=json_decode($h->options[CURLOPT_POSTFIELDS],true);$sent[]=$p; + return json_encode(['status'=>'stored','datasetId'=>$p['datasetId'],'acceptedThrough'=>$mode==='wrong_ack'?'2000-01-01T00:00:00+00:00':end($p['records'])['capturedAt']]);} +function curl_getinfo($h,$info){global $mode;return $mode==='429'?429:200;} +function curl_close($h):void {} +class Lizenzpruefung {const NETZFAHRPLAN='test-only';} +class FakeManager { + use ManagerNetzfahrplanV4DatenTrait; + public int $InstanceID=17004;public array $p=[],$a=[],$vars=[],$timers=[];public bool $licensed=true; + public function boot():void {$this->registriereNetzfahrplanV4Daten();} + public function configure():void {$this->konfiguriereNetzfahrplanV4Daten();} + public function folder():string {return $this->v4DatenVerzeichnis();} + public function send():array {return $this->v4DatenUebertragen($this->folder());} + public function RegisterPropertyBoolean($k,$v){$this->p[$k]??=$v;} + public function RegisterPropertyString($k,$v){$this->p[$k]??=$v;} + public function RegisterAttributeString($k,$v){$this->a[$k]??=$v;} + public function RegisterAttributeInteger($k,$v){$this->a[$k]??=$v;} + public function RegisterVariableString($k,...$args){$this->vars[$k]='';} + public function RegisterTimer($k,$v,$code){$this->timers[$k]=$v;} + public function SetTimerInterval($k,$v){$this->timers[$k]=$v;} + public function ReadPropertyBoolean($k){return $this->p[$k];} + public function ReadPropertyString($k){return $this->p[$k];} + public function ReadAttributeString($k){return $this->a[$k];} + public function ReadAttributeInteger($k){return $this->a[$k];} + public function WriteAttributeString($k,$v){$this->a[$k]=$v;} + public function WriteAttributeInteger($k,$v){$this->a[$k]=$v;} + public function SetValue($k,$v){if($k!=='NetzfahrplanV4Datenstatus')throw new \RuntimeException('Unexpected variable write');$this->vars[$k]=$v;} + private function berechtigungLizenziert($k){return $this->licensed;} +} +$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004, + 'reportedInventorySha256'=>str_repeat('a',64),'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]],'extraSources'=>[]]; +foreach(['grid','pv','physical_storage','controlled_virtual','external_virtual'] as $i=>$role){$id=101+$i; + $c['accounting']['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'s'.$id,'factorToW'=>1,'maxAgeSeconds'=>60,'dependsOn'=>[]]; + $sensor[$id]=['value'=>[3000,5000,2000,2000,0][$i],'updated'=>$clock,'changed'=>$clock];} +$m=new FakeManager();$m->boot();$m->a['LizenzInstallationID']=$c['installationId'];$m->a['PrognoseInstallationsToken']='synthetic-not-a-real-token'; +try { + $m->ErfasseNetzfahrplanV4Daten();check($sent===[],'disabled by default makes no network request'); + $m->p['NetzfahrplanV4MessdatenAktiv']=true;$m->p['NetzfahrplanV4Datensatz']='physical-v1';$m->p['NetzfahrplanV4Messkonfiguration']=json_encode($c); + $m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===30000,'only application acquisition timer enabled'); + $m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'native capture delivered to existing device API'); + $cursor=$m->a['NetzfahrplanV4DatenCursor'];check(json_decode($cursor,true)['offset']>0,'matching ack commits durable byte cursor'); + check($sent[0]['records'][0]['raw']['s101']['value']===3000,'physical reading retained'); + check($sent[0]['records'][0]['raw']['s101']['sourceUpdatedAt']===$clock,'original timestamp retained'); + check(!str_contains(json_encode($sent),'synthetic-not-a-real-token'),'token never included in data payload'); + check($sent[0]['records'][0]['controlEligible']===false,'data delivery is not actuator permission'); + $clock+=30;$m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'30-second collection does not double request rate'); + check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'unsent rows remain behind cursor'); + $clock+=30;$mode='429';$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'429 never advances acknowledgement cursor'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['dataRetained'],'offline data retained'); + $mode='wrong_ack';$clock+=120;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'wrong ack timestamp rejected'); + $mode='ok';$clock+=240;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']!==$cursor,'later success sends the preserved backlog'); + check(count(end($sent)['records'])===4,'all previously unacknowledged rows included once'); + $file=glob($m->folder().'/raw-*.jsonl')[0];$count=count(file($file));check($count===5,'raw journal never cleared by upload'); + $lastCursor=$m->a['NetzfahrplanV4DatenCursor'];file_put_contents($file,'incomplete',FILE_APPEND); + check($m->send()['status']==='up_to_date','in-progress tail not transmitted');check($m->a['NetzfahrplanV4DatenCursor']===$lastCursor,'partial tail not acknowledged'); + $m->p['NetzfahrplanV4MessdatenAktiv']=false;$m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===0,'disable affects acquisition only'); + check(count($m->vars)===1,'only own manager data-status variable written'); + $text=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4DatenTrait.php'); + check(!preg_match('/\b(?:RequestAction\(|IPS_ApplyChanges\(|MC_Reload\(|schreibeRegister\()/',$text),'data path contains no actuator/module reload calls'); + echo "TOTAL $checks application-data checks passed; mocked IPS and HTTP only.\n"; +} finally { + foreach(new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base,\FilesystemIterator::SKIP_DOTS),\RecursiveIteratorIterator::CHILD_FIRST)as$p){$p->isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base); +} From fe3f46ba0ea6975441234783cd5b8142f411d7cf Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 21:01:24 +0000 Subject: [PATCH 10/22] fix(release): include forecast test dependencies in versioned application --- services/netplan-v4/SOURCE_MANIFEST.json | 3 ++- .../acceptance/forecast-src/requirements.txt | 11 +++++++++++ .../commissioning/application-source/RELEASE.json | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 services/netplan-v4/acceptance/forecast-src/requirements.txt diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json index 878da98..f32a7a7 100644 --- a/services/netplan-v4/SOURCE_MANIFEST.json +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -20,6 +20,7 @@ "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", @@ -88,6 +89,6 @@ "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" }, - "packagedAt": "2026-10-02T20:57:00.387145+00:00", + "packagedAt": "2026-10-02T21:00:35.231591+00:00", "runtimeChanged": false } diff --git a/services/netplan-v4/acceptance/forecast-src/requirements.txt b/services/netplan-v4/acceptance/forecast-src/requirements.txt new file mode 100644 index 0000000..69cb44c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/requirements.txt @@ -0,0 +1,11 @@ +schedule +pandas +numpy +influxdb-client +pytz +holidays +pvlib +geopy +scikit-learn +scipy +joblib diff --git a/services/netplan-v4/commissioning/application-source/RELEASE.json b/services/netplan-v4/commissioning/application-source/RELEASE.json index 596cf5c..0395956 100644 --- a/services/netplan-v4/commissioning/application-source/RELEASE.json +++ b/services/netplan-v4/commissioning/application-source/RELEASE.json @@ -22,6 +22,7 @@ "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", @@ -94,6 +95,6 @@ "netplan-v4-bridge.mjs": "f58c652b04cd2ac20f2bec11e1161c44b72c26267b4dfb42096a1fad36a3c9c2", "public/netplan-v4.js": "6cd1d887bb08c73a815c92eb684a0992e8cb092f748770cc94debcfd9705a628" }, - "createdAt": "2026-10-02T20:57:00.387145+00:00", + "createdAt": "2026-10-02T21:00:35.231591+00:00", "controlEnabled": false } From 6d9aba74b038c24d97cd5fe09f922424b4976a73 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sat, 3 Oct 2026 07:34:44 +0000 Subject: [PATCH 11/22] fix(installer): avoid staged and installed capture class collision --- docs/netplan-v4-application.md | 34 +++++++++ .../application-class-loading.patch | 71 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 examples/V4ApplicationData/application-class-loading.patch diff --git a/docs/netplan-v4-application.md b/docs/netplan-v4-application.md index bf964b0..452f504 100644 --- a/docs/netplan-v4-application.md +++ b/docs/netplan-v4-application.md @@ -95,3 +95,37 @@ commissioning. Economic replay-based automatic family selection, full corrected feedback/control integration, long-term outbox/server retention and real live/ failure acceptance remain open. Existing trial gates are unchanged. No sensor, accounting or hardware evidence identifier is fabricated by data ingestion. + +## Installer class-path correction (2026-10-03) + +The original commissioning installer loaded `source/libs/NetzfahrplanV4Messaufnahme.php` +while importing observation history. A later Manager callback loads the installed +copy at `modules/Enelix-EMS/libs/NetzfahrplanV4Messaufnahme.php`. PHP's once-only file +inclusion does not deduplicate class declarations across those different files. +The dependent `NetzfahrplanV4Bilanzierung` class is subject to the same collision. + +The tested installer-only fix is preserved in +`examples/V4ApplicationData/application-class-loading.patch` (SHA256 +`6d53d74d4059bb1ae3a446350637a4f4a4a73cc2fd64d2fc2f0967bd23ec7173`). +When building a commissioning package, load both classes only from the installed +module tree, after validating their expected source hashes. Before any require, +check both already-loaded origins without autoload. Reject a foreign staged origin +with a controlled error; do not hide it by wrapping the class declaration in a guard. +Include `installed_capture_bootstrap.php` in the package checksum map. Do not change +the installed-module manifest or bless unrelated source changes. + +The Lihrenmoos staging entrypoint has this fix applied, with backups of its original +installer and checksum map. No running modules, data files, timers, native callbacks, +server components or actuator settings were changed by the repair. Actual completion +still requires the user's fresh Symcon invocation of the corrected installer. + +Validation: the original two-path failure was reproduced with the real PHP classes. +Thirteen additional isolated installer scenarios passed, including callback/reload +loading order, preloaded dependencies, history import, repeat/finished import, +foreign-class refusal, partial-import preservation, source drift, registration wait +and first installation. These are PHP CLI tests with mocked IPS in temporary trees, +not a completed test in the actual Symcon kernel. + +Evidence on the test host: +`/srv/agent/netplan-v4-application-build/class-loading-fix/TEST_RESULTS.json` and +`FIX_RESULT.json`. The test harness and exact candidate source are retained there. diff --git a/examples/V4ApplicationData/application-class-loading.patch b/examples/V4ApplicationData/application-class-loading.patch new file mode 100644 index 0000000..5b39d69 --- /dev/null +++ b/examples/V4ApplicationData/application-class-loading.patch @@ -0,0 +1,71 @@ +--- a/install.php ++++ b/install.php +@@ -19,6 +19,8 @@ + } + $before=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $report['files']=v4StageInstall(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); ++ require_once __DIR__.'/installed_capture_bootstrap.php'; ++ $report['classLoading']=v4ApplicationLoadInstalledCapture('/var/lib/symcon/modules/Enelix-EMS',__DIR__.'/MANIFEST.json'); + $controls=[]; + foreach (IPS_GetModuleList() as $mid) if ((IPS_GetModule($mid)['Prefix']??'')==='MC') $controls=array_merge($controls,IPS_GetInstanceListByModuleID($mid)); + if (count($controls)!==1||!function_exists('MC_ReloadModule')) throw new RuntimeException('Module Control nicht eindeutig.'); +@@ -38,7 +40,7 @@ + $marker=$dir.'/observer-import.json'; + if (!is_file($marker)&&!IPS_GetProperty($manager,'NetzfahrplanV4MessdatenAktiv')) { + if (glob($dir.'/raw-*.jsonl')!==[]) throw new RuntimeException('Messdaten ohne Importabschluss vorhanden; nicht doppelt importieren.'); +- require_once __DIR__.'/source/libs/NetzfahrplanV4Messaufnahme.php'; ++ // Reuse the hash-checked installed class; never load the staged copy. + $count=0;$last=0;$cutoff=time()-172800; + $files=glob('/srv/agent/netplan-v4-separated-observer-stage/data/raw-*.jsonl');sort($files,SORT_STRING); + foreach ($files as $file) { +--- /dev/null ++++ b/installed_capture_bootstrap.php +@@ -0,0 +1,48 @@ ++ 'libs/NetzfahrplanV4Bilanzierung.php', ++ 'Belevo\\EnelixEMS\\NetzfahrplanV4Messaufnahme' => 'libs/NetzfahrplanV4Messaufnahme.php', ++ ]; ++ // Check BOTH origins before loading either file, including the transitive dependency. ++ foreach ($classes as $class => $relative) { ++ $path = $target . '/' . $relative; ++ $expected = $manifest['files'][$relative]['after'] ?? null; ++ if (!is_string($expected) || !preg_match('/^[a-f0-9]{64}$/D', $expected) ++ || is_link($path) || !is_file($path) || realpath($path) !== $path ++ || hash_file('sha256', $path) !== $expected) { ++ throw new RuntimeException('Installierte Messbibliothek weicht vom geprueften Paket ab: ' . $relative); ++ } ++ if (class_exists($class, false)) { ++ $origin = (new ReflectionClass($class))->getFileName(); ++ if ($origin !== $path) { ++ throw new RuntimeException('Messbibliothek bereits aus anderem Pfad geladen. Installationsaufruf separat ausfuehren, ohne weitere Sammler-Includes.'); ++ } ++ } ++ } ++ foreach ($classes as $class => $relative) { ++ $path = $target . '/' . $relative; ++ if (!class_exists($class, false)) { ++ require_once $path; ++ } ++ if (!class_exists($class, false) || (new ReflectionClass($class))->getFileName() !== $path) { ++ throw new RuntimeException('Installierte Messbibliothek konnte nicht eindeutig geladen werden.'); ++ } ++ } ++ return ['source' => 'installed_module_only', 'stagedClassesLoaded' => false, ++ 'files' => array_values($classes)]; ++} From 5890093b8006b23ee011ce31db648ffc75fd397d Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sat, 3 Oct 2026 08:04:28 +0000 Subject: [PATCH 12/22] fix(history): model sparse publication without relaxing live freshness --- docs/netplan-v4-history-timing.md | 90 ++++ services/netplan-v4/Dockerfile | 1 + services/netplan-v4/SOURCE_MANIFEST.json | 12 +- .../commissioning/deploy_history_timing.py | 146 +++++ .../history-timing-source/RELEASE.json | 29 + .../history-timing-source/dataset.json | 206 ++++++++ .../history-timing-source/source/Dockerfile | 24 + .../source/netplan_v4/history_timing.py | 58 ++ .../source/netplan_v4/measurement_pipeline.py | 500 ++++++++++++++++++ .../source/tests/test_history_timing.py | 204 +++++++ .../tests/test_history_timing_deployment.py | 94 ++++ .../netplan-v4/netplan_v4/history_timing.py | 58 ++ .../netplan_v4/measurement_pipeline.py | 89 +++- .../netplan-v4/tests/test_history_timing.py | 204 +++++++ .../tests/test_history_timing_deployment.py | 94 ++++ 15 files changed, 1787 insertions(+), 22 deletions(-) create mode 100644 docs/netplan-v4-history-timing.md create mode 100644 services/netplan-v4/commissioning/deploy_history_timing.py create mode 100644 services/netplan-v4/commissioning/history-timing-source/RELEASE.json create mode 100644 services/netplan-v4/commissioning/history-timing-source/dataset.json create mode 100644 services/netplan-v4/commissioning/history-timing-source/source/Dockerfile create mode 100644 services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py create mode 100644 services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py create mode 100644 services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py create mode 100644 services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py create mode 100644 services/netplan-v4/netplan_v4/history_timing.py create mode 100644 services/netplan-v4/tests/test_history_timing.py create mode 100644 services/netplan-v4/tests/test_history_timing_deployment.py diff --git a/docs/netplan-v4-history-timing.md b/docs/netplan-v4-history-timing.md new file mode 100644 index 0000000..b96f8d4 --- /dev/null +++ b/docs/netplan-v4-history-timing.md @@ -0,0 +1,90 @@ +# Historical publication timing (2026-10-03) + +## Implemented scope + +`services/netplan-v4/netplan_v4/history_timing.py` implements optional, bounded +retrospective estimates between consecutive identical source values at DIFFERENT +original publication timestamps. It is used by the existing measurement pipeline, +model training and forecast source substitution. It is NOT a real-time estimator, +a measurement certificate, an actuator authority or a repair of device communications. + +No source `maxAgeSeconds` is changed. When a regular hold expires, only its tail +until the next qualifying original publication may be estimated. Strict processing +remains the default. The following forbid the estimate: unequal endpoints, +nonfinite/missing observations, conflicting same-timestamp values, collector gaps, +invalid source intervals or a span beyond the explicit source-specific bound. +There is no extrapolation past the last observation and no fabricated zero. + +Each window reports `publicationEstimatedSeconds`, per-source estimated portions, +uncovered-source seconds and `availableNotBefore`. Training/validation must not use +a confirmation or receipt before it was actually available. Backfilled windows +cannot establish a supposedly causal holdout at a date before their availability. +All physical windows remain configured estimates; even full support is not proof +that the physical waveform stayed constant between the two endpoint readings. + +## Evidence and model assumptions + +Read-only source configuration projection on the test server, snapshot +2026-10-03T07:47:24Z: GoodWe 1 instance 19742 (ModBus Device) has Poller 60000 ms; +GoodWe 2 instance 57658 has Poller 5000 ms; SolarEdge scale instance 48996 +(ModBus Address) has Poller 20000 ms. No polling setting was changed. +Projection program: /srv/agent/netplan-v4-application-build/inspect_source_update_policy.py. + +Official Symcon documentation accessed on 2026-10-03: +https://www.symcon.de/de/service/dokumentation/modulreferenz/geraete/modbus-rtu-tcp/vorlagen/ +It documents value publication for ModBus Device on changes or when the variable is +older than 60 seconds, including ordinary and virtual addresses. VariableUpdated +is therefore not automatically the last successful device poll. The documentation +is not a proof that a particular installed device link was healthy. + +Explicit Lihrenmoos HISTORICAL estimate bounds: +- GoodWe 1 PV/physical storage: equal original publications at most 130 s apart + (60 s publication suppression + configured 60 s poll + 10 s scheduling allowance). +- GoodWe 2 PV/physical storage: at most 75 s (60 + 5 + 10). +- SolarEdge scale: equal scale publications at most 90 s apart. This is a modelling + bound based on the configured 20 s polling and observed sparse scale publication, + not a manufacturer guarantee and not proof of the same internal implementation + as ModBus Device. Raw AC power remains under the original strict limit. +The 10 s margins are explicit scheduling assumptions, NOT measured timing guarantees. +Gaps of many minutes (e.g. the previously observed GoodWe gaps above 600 s) are NOT +bridged. Nonzero changes are never interpolated by this policy. + +## Versioned dataset without restarting collection + +`lihrenmoos-physical-published-v2` references immutable observations in +`lihrenmoos-physical-v1`. No observations are copied, deleted, relabelled or resent. +Registration requires the same installation and identical original mapping, +formula, coverage and training configuration. Cycles/reference chains and device +uploads directly into a derived dataset are rejected. Original ingestion continues +unchanged in the Manager. The 24 usable-hour minimum stays in force. + +Models/windows are stored separately for v1/v2 and record their timing policy. An +existing selected model/family is not automatically switched by this release. +The current SDL request still uses the strict current-state requirements; no +historical bridge grants a future SDL schedule or a live battery permission. + +## Tests and release + +254 isolated Python tests passed (219 previous + 35 new including simulated +installation/recovery). Includes the actual service pipeline from referenced +observations to model and optimizer using synthetic records. No new target-container +or field-history evaluation has been run with this release yet; do not infer an +improved real coverage percentage or production acceptance from unit tests. +Evidence: /home/agent/services/qa/history-timing-20261003/ALL_TEST_RESULTS.txt. + +Prepared command on enelix-services (root required for Docker): + + python3 /home/agent/services/netplan-v4-shadow/commissioning/deploy_history_timing.py \ + --plant e3a08f9e-af12-4695-99bd-8b51c0520021 --apply + +Default without --apply validates staged source only. The apply path backs up source +files, builds and runs the exact target tests, backs up the V4 SQLite database, and +recreates ONLY the V4 service. A failed test restores source without a service +restart; failed post-deploy validation attempts the previous image and preserves +additive data. Concurrently modified files are not silently overwritten. + +No portal/forecast/tariff/Symcon restart, no new sensor requests, no change to +161.44 kWh / 39 kW, SOC accounting, old raw samplers or actuator permissions. +The new dataset is registered but not selected for the active forecast. A root +execution and successful report are still required to install it. Previous release +manifests remain historical records; do not bypass them to reapply an older build. diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile index b98928f..ceff456 100644 --- a/services/netplan-v4/Dockerfile +++ b/services/netplan-v4/Dockerfile @@ -10,6 +10,7 @@ COPY gui ./gui COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ COPY acceptance ./acceptance COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py # Host sources can be 0600/0700. COPY makes them root-owned. # Normalize only packaged application code; never change host secrets or sockets. RUN find /app -type d -exec chmod 0755 {} + \ diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json index f32a7a7..34ed6eb 100644 --- a/services/netplan-v4/SOURCE_MANIFEST.json +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -1,7 +1,7 @@ { "sourceHashes": { ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", - "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "Dockerfile": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", @@ -58,7 +58,7 @@ "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", - "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/measurement_pipeline.py": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", @@ -87,7 +87,13 @@ "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", - "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", + "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", + "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "commissioning/history-timing-source/dataset.json": "671216e1cc04148f4f553103077bcb4500bf0120cb74780703bfcf26cd1121d3", + "commissioning/history-timing-source/RELEASE.json": "14cb690a357c40f698c2e9b4190933318396b9c08e15cc4ad9f9f4439d6054ca" }, "packagedAt": "2026-10-02T21:00:35.231591+00:00", "runtimeChanged": false diff --git a/services/netplan-v4/commissioning/deploy_history_timing.py b/services/netplan-v4/commissioning/deploy_history_timing.py new file mode 100644 index 0000000..c4c5914 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_history_timing.py @@ -0,0 +1,146 @@ +"""Install the tested historical-publication correction in the V4 service only. + +Default is source validation. --apply requires root. No Symcon changes, polling, +portal restart, old-data rewrite, forecast-source switch or actuator permission. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import hashlib +import json +import os +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = Path(__file__).with_name('history-timing-source') +TARGETS = ('Dockerfile', 'netplan_v4/history_timing.py', 'netplan_v4/measurement_pipeline.py', + 'tests/test_history_timing.py', 'tests/test_history_timing_deployment.py') + + +def digest(path): + if path.is_symlink(): + raise ValueError('Source symlink refused') + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +def verify(): + manifest = json.loads((PACKAGE/'RELEASE.json').read_text()) + if manifest.get('scope') != 'historical_publication_only' or tuple(manifest.get('files', {})) != TARGETS: + raise ValueError('Unexpected timing release scope') + for name, versions in manifest['files'].items(): + if digest(PACKAGE/'source'/name) != versions['after']: + raise ValueError('Timing candidate changed: '+name) + if digest(ROOT/name) not in (versions['before'], versions['after']): + raise ValueError('Concurrent application change: '+name) + if digest(PACKAGE/'dataset.json') != manifest['datasetSha256']: + raise ValueError('Dataset candidate changed') + if digest(ROOT/'commissioning/deploy_application.py') != manifest['existingDeploymentHelperSha256']: + raise ValueError('Existing deployment helper changed') + if digest(Path(__file__).resolve()) != manifest['installerSha256']: + raise ValueError('Timing installer changed') + return manifest + + +BOOTSTRAP = r'''import json,os,sys,time,urllib.request +payload=json.load(sys.stdin) +base='http://127.0.0.1:9100/internal/v2/prognosis/'+payload['plant']+'/planner' +headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +def get(): + return json.load(urllib.request.urlopen(urllib.request.Request(base,headers=headers),timeout=10)) +before=get() +assert before['dataPipeline'].get('historyTimingVersion')==1 +assert before['liveEnabled'] is False +req=urllib.request.Request(base+'/datasets/'+payload['dataset']['datasetId'], + data=json.dumps(payload['dataset']).encode(),headers=headers,method='PUT') +receipt=json.load(urllib.request.urlopen(req,timeout=10)) +after=get() +assert before['settings']==after['settings'] and after['liveEnabled'] is False +view=next(d for d in after['dataPipeline']['datasets'] if d['datasetId']==payload['dataset']['datasetId']) +print(json.dumps({'dataset':receipt,'originalObservationDataset':view['observationDatasetId'], + 'availableRecords':view['records'],'pipelineStatus':view['status'], + 'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'], + 'liveEnabled':False})) +''' + + +def run(plant, apply=False): + manifest = verify() + if plant != manifest['installationId']: + raise ValueError('Use the reviewed installation') + if not apply: + print('HISTORY TIMING SOURCE CHECK PASSED; no runtime changes.') + return + if os.geteuid() != 0: + raise ValueError('Run as root; do not change Docker permissions') + # Reuse the reviewed atomic-write and consistent-backup implementation. + from deploy_application import atomic, backup_database + folder = ROOT/'history-timing-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ') + folder.mkdir(parents=True, mode=0o700) + result={'scope':'historical_publication_only','startedAt':datetime.now(timezone.utc).isoformat(), + 'steps':[],'liveEnabled':False,'symconChanged':False,'pollingChanged':False, + 'sourceFreshnessLimitsChanged':False,'forecastSelectionChanged':False} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')] + env={**os.environ,'NETPLAN_V4_PLANTS':plant} + def cmd(args, timeout=180, capture=False, data=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data, + stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + changed={}; previous_image=None; replaced=False + try: + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1: raise ValueError('Expected one running V4 container') + previous_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip() + result['previousImage']=previous_image + for name, versions in manifest['files'].items(): + target=ROOT/name; content=(PACKAGE/'source'/name).read_bytes() + before=target.read_bytes() if target.is_file() else None + if digest(target)==versions['after']: continue + if digest(target)!=versions['before']: raise ValueError('Concurrent source change') + backup=folder/'source-before'/name; backup.parent.mkdir(parents=True,exist_ok=True) + if before is not None: backup.write_bytes(before) + atomic(target,content); changed[name]=(before,content) + verify() + cmd(compose+['build','netplan-v4'],timeout=900) + cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240) + result['steps'].append('target_tests_passed'); verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite') + result['steps'].append('consistent_database_backup') + rollback=folder/'rollback.yaml' + rollback.write_text('services:\n netplan-v4:\n image: '+previous_image+'\n') + replaced=True + cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4']) + payload=json.dumps({'plant':plant,'dataset':json.loads((PACKAGE/'dataset.json').read_text())}) + response=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=payload) + result['application']=json.loads(response.stdout) + result['status']='historical_timing_installed_no_actuation' + result['steps'].append('versioned_dataset_registered_original_observations_retained') + except Exception as exc: + result['status']='needs_review';result['errorType']=type(exc).__name__ + conflicts=[] + for name,(before,after) in changed.items(): + path=ROOT/name + if digest(path)!=hashlib.sha256(after).hexdigest(): conflicts.append(name);continue + if before is None:path.unlink() + else:atomic(path,before) + result['concurrentFilesNotOverwritten']=conflicts + if replaced and previous_image: + try: + cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + result['rollback']='previous_image_restored_additive_dataset_retained' + except Exception: result['rollback']='manual_review_required' + raise + finally: + result['finishedAt']=datetime.now(timezone.utc).isoformat() + report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n') + uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid + for path in (folder.parent,folder,report):os.chown(path,uid,gid) + report.chmod(0o640) + print('HISTORY TIMING REPORT:',report) + print('Only V4 updated. Original data, manager, polling, forecast selection and actuator permissions unchanged.') + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True);parser.add_argument('--apply',action='store_true') + args=parser.parse_args() + try:run(args.plant,args.apply) + except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. See HISTORY TIMING REPORT; do not enable actuators.') diff --git a/services/netplan-v4/commissioning/history-timing-source/RELEASE.json b/services/netplan-v4/commissioning/history-timing-source/RELEASE.json new file mode 100644 index 0000000..65f5a18 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/RELEASE.json @@ -0,0 +1,29 @@ +{ + "scope": "historical_publication_only", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "files": { + "Dockerfile": { + "before": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "after": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90" + }, + "netplan_v4/history_timing.py": { + "before": null, + "after": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f" + }, + "netplan_v4/measurement_pipeline.py": { + "before": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "after": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d" + }, + "tests/test_history_timing.py": { + "before": null, + "after": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2" + }, + "tests/test_history_timing_deployment.py": { + "before": null, + "after": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e" + } + }, + "datasetSha256": "671216e1cc04148f4f553103077bcb4500bf0120cb74780703bfcf26cd1121d3", + "installerSha256": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "existingDeploymentHelperSha256": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e" +} diff --git a/services/netplan-v4/commissioning/history-timing-source/dataset.json b/services/netplan-v4/commissioning/history-timing-source/dataset.json new file mode 100644 index 0000000..bf153a3 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/dataset.json @@ -0,0 +1,206 @@ +{ + "datasetId": "lihrenmoos-physical-published-v2", + "mappingSha256": "517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b", + "inventorySha256": "0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc", + "sources": [ + { + "key": "grid", + "variableId": 40348, + "factorToW": 1000, + "maxAgeSeconds": 60, + "role": "grid" + }, + { + "key": "pv_goodwe1", + "variableId": 48459, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "pv_goodwe2", + "variableId": 53802, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "pv_solaredge", + "variableId": 20335, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "factorToW": -1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "factorToW": -1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "ev_account", + "variableId": 52020, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "reference" + }, + { + "key": "sdl_account", + "variableId": 25085, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "reference" + }, + { + "key": "grid_display", + "variableId": 49301, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_goodwe1", + "variableId": 27361, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_goodwe2", + "variableId": 23109, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_solaredge", + "variableId": 51938, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_ev", + "variableId": 32871, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_sdl", + "variableId": 23879, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_available_charge", + "variableId": 50230, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_available_discharge", + "variableId": 43899, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "energy_t1", + "variableId": 59607, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "energy_t2", + "variableId": 26620, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_requested", + "variableId": 19651, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "sdl_requested", + "variableId": 38943, + "role": "sdl_request", + "factorToW": 1, + "maxAgeSeconds": 120 + }, + { + "key": "solar_ac_value", + "variableId": 37975, + "role": "solar_raw", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "solar_ac_scale", + "variableId": 41853, + "role": "solar_scale", + "factorToW": 1, + "maxAgeSeconds": 60 + } + ], + "formula": "solar_terminal_v1", + "solarReference": { + "pvKey": "pv_solaredge", + "batteryKey": "physical_solaredge", + "rawKey": "solar_ac_value", + "scaleKey": "solar_ac_scale" + }, + "minimumCoverage": 0.95, + "maximumGapSeconds": 10, + "minimumTrainingHours": 24, + "historyDays": 28, + "sourceDatasetId": "lihrenmoos-physical-v1", + "historyTimingPolicy": { + "version": 1, + "method": "equal_endpoint_v1", + "sources": { + "pv_goodwe1": { + "maxSpanSeconds": 130, + "evidenceId": "symcon-modbus-publish60-poll60-jitter10" + }, + "physical_goodwe1": { + "maxSpanSeconds": 130, + "evidenceId": "symcon-modbus-publish60-poll60-jitter10" + }, + "pv_goodwe2": { + "maxSpanSeconds": 75, + "evidenceId": "symcon-modbus-publish60-poll5-jitter10" + }, + "physical_goodwe2": { + "maxSpanSeconds": 75, + "evidenceId": "symcon-modbus-publish60-poll5-jitter10" + }, + "solar_ac_scale": { + "maxSpanSeconds": 90, + "evidenceId": "configured-scale-poll20-bounded-equal-value-estimate" + } + } + } +} diff --git a/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile b/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile new file mode 100644 index 0000000..ceff456 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY netplan_v4 ./netplan_v4 +COPY tests ./tests +COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile . +COPY integrations ./integrations +COPY gui ./gui +COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ +COPY acceptance ./acceptance +COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py +# Host sources can be 0600/0700. COPY makes them root-owned. +# Normalize only packaged application code; never change host secrets or sockets. +RUN find /app -type d -exec chmod 0755 {} + \ + && find /app -type f -exec chmod 0644 {} + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results +USER 1000:1000 +# Fail the build early if the actual unprivileged runtime cannot read the code. +RUN python /app/runtime_preflight.py +CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"] diff --git a/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py new file mode 100644 index 0000000..6205d84 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py @@ -0,0 +1,58 @@ +"""Explicit, retrospective equal-endpoint estimates for sparse state publication. + +Never changes a source timestamp, real-time freshness limit, meter evidence or +actuator lease. A repeated value after a bounded gap supports a MODEL estimate, +not proof that the physical signal was constant between the observations. +""" +from math import isfinite +import re + + +def validate_policy(policy, sources): + if policy is None: + return None + if (not isinstance(policy, dict) or set(policy) != {'version', 'method', 'sources'} + or type(policy['version']) is not int or policy['version'] != 1 + or policy['method'] != 'equal_endpoint_v1' + or not isinstance(policy['sources'], dict) or not policy['sources']): + raise ValueError('Explicit historical timing policy required') + defined = {s['key']: s for s in sources} + for key, rule in policy['sources'].items(): + if key not in defined or defined[key]['role'] not in ('pv', 'physical_storage', 'solar_scale'): + raise ValueError('Timing estimates limited to explicit power/scale sources') + if not isinstance(rule, dict) or set(rule) != {'maxSpanSeconds', 'evidenceId'}: + raise ValueError('Timing bound and evidence reference required') + if type(rule['maxSpanSeconds']) is not int or not defined[key]['maxAgeSeconds'] <= rule['maxSpanSeconds'] <= 180: + raise ValueError('Historical endpoint span outside reviewed bounds') + if not isinstance(rule['evidenceId'], str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,100}', rule['evidenceId']): + raise ValueError('Invalid publication evidence reference') + return policy + + +def endpoint_bridges(series, first_observed, blocks, capture_gaps, policy, source_specs): + """Index strictly consecutive, equal original timestamps; never invent endpoints. + +Only the stale tail of a normal hold is filled. A missing/invalid capture or a +source conflict forbids bridging. The right endpoint must have been observed; +its availability is retained by the caller for causal training and replay. +""" + result = {key: {} for key in series} + if policy is None: + return result + for key, rule in policy['sources'].items(): + if key not in series: + continue # e.g. an unused DC channel in an AC-terminal formula + values = series[key] + times = sorted(values) + for left, right in zip(times, times[1:]): + value, next_value = values[left], values[right] + if (value is None or next_value is None or type(value) not in (int, float) + or not isfinite(value) or value != next_value): + continue + if not source_specs[key]['maxAgeSeconds'] < right-left <= rule['maxSpanSeconds']: + continue + if any(start < right and end > left for start, end in (*blocks[key], *capture_gaps)): + continue + result[key][left] = {'end': right, 'availableAt': first_observed[key][right], + 'spanSeconds': right-left} + return result diff --git a/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py new file mode 100644 index 0000000..0579467 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py @@ -0,0 +1,500 @@ +"""Application data path: versioned numeric observations -> physical load -> trained profiles. + +Lives in the existing planner service/database; no separate diagnostic service. +The device may append only to an operator-configured dataset. Original observations, +model revisions and prediction vintages are preserved. Output is never an actuator grant. +""" +from __future__ import annotations +from bisect import bisect_right +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from math import isfinite +from statistics import median +from zoneinfo import ZoneInfo +import json +from .history_timing import validate_policy, endpoint_bridges + +UTC = timezone.utc +LOCAL = ZoneInfo('Europe/Zurich') +FAMILIES = ('3', '13', '23') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond: + raise ValueError('Explicit whole-second UTC timestamp required') + return int(t.timestamp()) + + +def iso(t): + return datetime.fromtimestamp(t, UTC).isoformat() + + +def numeric(value, bound=1e12): + return type(value) in (int, float) and isfinite(value) and abs(value) <= bound + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_data_sets( + plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_observations( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,captured_at)); + CREATE TABLE IF NOT EXISTS planner_load_windows( + plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL, + available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,start)); + CREATE TABLE IF NOT EXISTS planner_load_models( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,model_id)); + CREATE TABLE IF NOT EXISTS planner_model_current( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_pipeline_state( + plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL, + status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_prediction_vintages( + plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL, + target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL, + load_w REAL NOT NULL, pv_w REAL NOT NULL, + PRIMARY KEY(plant,dataset,issued_at,target,family)); + CREATE INDEX IF NOT EXISTS planner_observation_window + ON planner_observations(plant,dataset,captured_at); + CREATE INDEX IF NOT EXISTS planner_prediction_target + ON planner_prediction_vintages(plant,dataset,target); + ''') + + +def validate_config(c): + fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', + 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', + 'minimumTrainingHours', 'historyDays'} + optional = {'sourceDatasetId', 'historyTimingPolicy'} + if not isinstance(c, dict) or not fields <= set(c) or set(c)-fields-optional: + raise ValueError('Explicit dataset configuration required') + name = c['datasetId'] + if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): + raise ValueError('Invalid dataset ID') + for field in ('mappingSha256', 'inventorySha256'): + h = c[field] + if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h): + raise ValueError('Explicit mapping/inventory fingerprint required') + if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'): + raise ValueError('Unknown physical formula') + if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1: + raise ValueError('Coverage must be .90..1; recorded gaps remain visible') + for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)): + if type(c[field]) is not int or not lo <= c[field] <= hi: + raise ValueError('Invalid '+field) + sources = c['sources'] + if not isinstance(sources, list) or not 3 <= len(sources) <= 80: + raise ValueError('Source list required') + seen, ids = set(), set() + roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'} + for s in sources: + if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}: + raise ValueError('Explicit source definition required') + k = s['key'] + if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids: + raise ValueError('Duplicate/invalid source') + if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0: + raise ValueError('Source role/factor invalid') + if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300: + raise ValueError('Source lifetime invalid') + seen.add(k); ids.add(s['variableId']) + if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources): + raise ValueError('Grid and PV measurement sources required') + sr = c['solarReference'] + if c['formula'] == 'solar_terminal_v1': + if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}: + raise ValueError('Solar terminal sources required') + bykey = {s['key']: s['role'] for s in sources} + if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))): + raise ValueError('Solar origin roles mismatch') + elif sr is not None: + raise ValueError('No unused solar mapping allowed') + validate_policy(c.get('historyTimingPolicy'), sources) + source = c.get('sourceDatasetId') + if source is not None and (not isinstance(source, str) or not 1 <= len(source) <= 80 or source == name or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in source)): + raise ValueError('Invalid source dataset reference') + canonical(c) + return c + + +def register_dataset(con, plant, c, now): + """Operator endpoint only; device append endpoint cannot change units or limits.""" + validate_config(c) + if c.get('sourceDatasetId'): + origin = configuration(con, plant, c['sourceDatasetId']) + if origin.get('sourceDatasetId'): + raise ValueError('Dataset reference chains are not allowed') + comparable = lambda v: {k:x for k,x in v.items() if k not in ('datasetId','sourceDatasetId','historyTimingPolicy')} + if canonical(comparable(origin)) != canonical(comparable(c)): + raise ValueError('Referenced observations must retain identical measurement meaning') + value = canonical(c) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone() + if old and old[0] != value: + raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning') + con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False} + + +def configuration(con, plant, dataset): + row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone() + if row is None: + raise ValueError('Dataset not configured for this installation') + return json.loads(row[0]) + + +def project(record, c, plant, now): + if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: + raise ValueError('Wrong capture identity') + if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + raise ValueError('Wrong capture mapping or inventory') + t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) + if start > t or t > now+30 or t < now-90*86400: + raise ValueError('Capture timestamp outside permitted range') + if not isinstance(record.get('raw'), dict): + raise ValueError('Numeric raw observations required') + out = {}; issues = [] + for s in c['sources']: + r = record['raw'].get(s['key'], {}) + if not isinstance(r, dict): + r = {} + v, at = r.get('value'), r.get('sourceUpdatedAt') + good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == [] + if not good: + v = at = None + issues.append(s['key']) + # Unknown/free-text fields, credentials, client quality claims never persisted. + out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)} + return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues} + + +def ingest_batch(con, plant, payload, now): + if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: + raise ValueError('Measurement batch version/fields invalid') + c = configuration(con,plant,payload['datasetId']) + if c.get('sourceDatasetId'): + raise ValueError('Derived dataset is read-only; append to the original measurement dataset') + records = payload['records'] + if not isinstance(records,list) or not 1 <= len(records) <= 120: + raise ValueError('Batch requires 1..120 captures') + rows = [project(r,c,plant,now) for r in records] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): + raise ValueError('Batch must be in increasing capture order') + stored = duplicate = 0 + con.execute('BEGIN IMMEDIATE') + try: + for r in rows: + value = canonical(r); digest = sha256(value.encode()).hexdigest() + old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() + if old: + if old[0] != digest: + raise ValueError('Conflicting immutable observation') + duplicate += 1 + else: + con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + con.commit() + except Exception: + con.rollback(); raise + return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate, + 'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False} + + +def physical_value(values, c): + """Same explicit sign convention as configured acquisition. No virtual power in load.""" + total = defaultdict(float) + sr = c['solarReference'] + for s in c['sources']: + if s['role'] not in ('grid','pv','physical_storage','flexible_load'): + continue + if sr and s['key'] in (sr['pvKey'],sr['batteryKey']): + continue + val = values[s['key']]*s['factorToW'] + if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0): + raise ValueError('Invalid physical power') + total[s['role']] += val + solar = 0. + if sr: + raw, sf = values[sr['rawKey']], values[sr['scaleKey']] + if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6: + raise ValueError('Invalid solar power/scaling sentinel') + solar = raw*10**int(sf) + load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar + if not numeric(load,1e9) or load < 0: + raise ValueError('Negative/nonfinite physical load') + return load + + +def reconstruct(records, c): + """Bounded retrospective estimation, never a real-time feedback signal. + +Missing observations split support. Source timestamps are not refreshed. Small +uncovered portions remain quantified and are never filled with zero. +""" + if len(records) < 2: + return [] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])): + raise ValueError('Capture sequence not ordered') + sr = c['solarReference'] + primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')} + if sr: + primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) + for s in c['sources']: + if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) + first,last = records[0]['capturedAt'],records[-1]['capturedAt'] + series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + first_observed = {k:{} for k in primary} + pending = {k:None for k in primary}; high = {k:0 for k in primary} + edges = {first,last} + for a,b in zip(records,records[1:]): + if b['capturedAt']-a['capturedAt'] > 45: + gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1]) + for r in records: + at = r['capturedAt'] + for k in primary: + v = r['raw'].get(k,{}) + t = v.get('sourceUpdatedAt') + if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5: + if pending[k] is None: pending[k] = at + continue + high[k] = max(high[k],t) + if pending[k] is not None: + blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None + if t in series[k] and series[k][t] != v['value']: + series[k][t] = None + else: + series[k].setdefault(t,v['value']) + first_observed[k].setdefault(t, max(at, r.get('_receivedAt', at))) + for k,s in primary.items(): + if pending[k] is not None: + blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) + for t in series[k]: edges.update((t,t+s['maxAgeSeconds'])) + edges.update(range(first//300*300+300,last,300)) + edges = sorted(x for x in edges if first <= x <= last) + knots = {k:sorted(v) for k,v in series.items()} + bridges = endpoint_bridges(series, first_observed, blocks, gaps, timing_policy, primary) + bins = {} + for a,b in zip(edges,edges[1:]): + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0, + 'publicationEstimatedSeconds':0,'publicationEstimatedBySource':{},'knownAt':0,'missingSourceSeconds':{}}) + vals = {}; usable = not any(x <= a < y for x,y in gaps) + extended = []; unavailable = []; known_at = 0 + for k,s in primary.items(): + pos = bisect_right(knots[k],a)-1 + t = knots[k][pos] if pos >= 0 else None + bridge = bridges[k].get(t) + expired = t is None or a >= t+s['maxAgeSeconds'] + supported_tail = bool(bridge and a < bridge['end']) + if t is None or (expired and not supported_tail) or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False; unavailable.append(k) + else: + vals[k] = series[k][t] + known_at = max(known_at, first_observed[k][t]) + if expired: + extended.append(k); known_at = max(known_at, bridge['availableAt']) + load = None + if usable: + try: load = physical_value(vals,c) + except ValueError: usable = False + if usable: + item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + item['knownAt'] = max(item['knownAt'], known_at) + if extended: item['publicationEstimatedSeconds'] += b-a + for key in extended: item['publicationEstimatedBySource'][key] = item['publicationEstimatedBySource'].get(key,0)+b-a + else: + item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + for key in unavailable: item['missingSourceSeconds'][key] = item['missingSourceSeconds'].get(key,0)+b-a + out=[] + for t,item in sorted(bins.items()): + # Partial beginning/end bins remain diagnostic and cannot train. + complete_extent = first <= t and last >= t+300 + coverage = item['seconds']/300 + eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds'] + out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], + 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, + 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, + 'meterBoundaryVerified':False,'method':c['formula'], + 'historyTimingMethod':(timing_policy or {}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':item['publicationEstimatedSeconds'], + 'publicationEstimatedBySourceSeconds':item['publicationEstimatedBySource'], + 'missingSourceSeconds':item['missingSourceSeconds'], + 'availableNotBefore':max(t+300,item['knownAt'])}) + return out + + +def slot(t): + local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL) + return local.hour*12+local.minute//5 + + +def build_profiles(rows): + samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)} + anchor=max(r['start'] for r in rows) + for r in rows: + i=slot(r['start']); v=r['loadW']; samples[i].append(v) + if anchor-r['start'] < 86400: recent[i].append(v) + weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v) + overall = median([r['loadW'] for r in rows]) + def profile(values): + # Missing calendar slots are a model estimate, not invented historical measurements. + result=[] + for i in range(288): + local=values.get(i,[]) + if not local: + local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])] + result.append(float(median(local)) if local else float(overall)) + return result + return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)}, + 'slotCoverage':len(samples)/288} + + +def predict(model, family, t): + p=model['profiles'][family] + if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday'] + return p[slot(t)] + + +def advance(con, plant, dataset, settings, now): + """Called by the existing worker; bounded data/model update once per five-minute tick.""" + c=configuration(con,plant,dataset); tick=now//300 + old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() + if old and old[0]==tick: return + observation_dataset=c.get('sourceDatasetId',dataset) + fetched=con.execute('SELECT value,received_at FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,observation_dataset,now-172800-300,now,now)).fetchall() + records=[{**json.loads(r[0]),'_receivedAt':r[1]} for r in fetched] + windows=reconstruct(records,c) + with con: + for w in windows: + if w['start']+300 > now-30 or w.get('availableNotBefore',0)>now: continue + con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', + (plant,dataset,w['start'],now,w['coverage'],canonical(w))) + rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] + good=[r for r in rows if r['profileUsable']] + active=current_model(con,plant,dataset,now) + cadence=86400 if settings['trainingCadence']=='daily' else 604800 + detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], + 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True, + 'observationDatasetId':observation_dataset, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':sum(r.get('publicationEstimatedSeconds',0) for r in good), + 'originalFreshnessLimitsChanged':False, + 'lastCapture':iso(records[-1]['capturedAt']) if records else None} + state='collecting' + if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: + state='model_ready' if active else 'training' + attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0] + if attempted is None or now-attempted>=cadence: + profiles=build_profiles(good) + candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), + 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], + 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'timingPolicySha256':sha256(canonical(c.get('historyTimingPolicy')).encode()).hexdigest(), + 'observationDatasetId':observation_dataset, + 'measurementBoundaryVerified':False} + # Causal held-out validation: build validation profiles without the final day. + split=good[-1]['start']-86400 + train=[r for r in good if r['start']+300<=split and r.get('availableNotBefore',r['start']+300)<=split]; test=[r for r in good if r['start']>=split] + if len(train)>=288 and len(test)>=240: + val={'profiles':build_profiles(train)} + errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} + candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors} + # Initial model is labelled bootstrap, never a production measurement proof. + # Existing model can be replaced only with held-out evidence and no aggregate regression. + promote=active is None + if active and candidate['validation']['status']=='causal_holdout': + past_model_eligible=active['trainedThrough']<=split and active['trainedAt']<=split + if past_model_eligible: + incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + promote=challenger<=incumbent + candidate['validation']['incumbentCompared']=True + else: + candidate['validation']['status']='holdout_overlaps_active_training' + ident=sha256(canonical(candidate).encode()).hexdigest() + with con: + con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate))) + if promote: + con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident)) + detail['candidateModelId']=ident;detail['candidatePromoted']=promote + state='model_ready' if promote or active else 'candidate_pending' + active=current_model(con,plant,dataset,now) + if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']}) + with con: + con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail', + (plant,dataset,tick,state,canonical(detail))) + + +def current_model(con,plant,dataset,at): + row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone() + return {**json.loads(row[1]),'modelId':row[0]} if row else None + + +def apply_load_forecast(con,plant,dataset,forecast,decision): + model=current_model(con,plant,dataset,decision) + if not model: + raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') + c=configuration(con,plant,dataset) + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,c.get('sourceDatasetId',dataset),decision,decision)).fetchone() + if not last: raise ValueError('No recent corrected observation') + last=json.loads(last[0]) + if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') + sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] + if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') + s=sdl_sources[0];r=last['raw'][s['key']] + if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']: + raise ValueError('No current external SDL request for the persistence scenario') + sdl=r['value']*s['factorToW'] + result=json.loads(canonical(forecast)); result['families']={} + result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + for family,old in forecast['families'].items(): + if family not in FAMILIES: continue + points=[] + for p in old['points']: + t=epoch(p['time']) + points.append({**p,'loadW':predict(model,family,t),'externalW':sdl}) + result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']), + 'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'], + 'loadMethodVersion':model['methodVersion'],'loadVariant':family, + 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), + 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, + 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation'], + 'historyTimingMethod':model.get('historyTimingMethod','strict_expiry'), + 'observationDatasetId':model.get('observationDatasetId',dataset)}} + return result + + +def pipeline_status(con,plant): + out=[] + for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): + ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() + out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], + 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], + 'observationDatasetId':c.get('sourceDatasetId',ds), + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry')}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False,'historyTimingVersion':1} diff --git a/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py new file mode 100644 index 0000000..b05b046 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py @@ -0,0 +1,204 @@ +"""Synthetic publication gaps plus full source-reference/application integration.""" +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.history_timing import validate_policy +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once +from test_measurement_pipeline import config, record, NOW, AID +from test_v4 import inputs, TOKEN + + +def timed_config(**kw): + return config(historyTimingPolicy={'version':1,'method':'equal_endpoint_v1', + 'sources':{'pv':{'maxSpanSeconds':130,'evidenceId':'synthetic-publication-policy'}}}, **kw) + + +def projected(span=360, period=120, c=None): + c = c or timed_config() + result=[] + for delta in range(0,span+1,30): + r=record(NOW+delta,c) + r['raw']['pv']['sourceUpdatedAt']=NOW+(delta//period)*period + result.append(m.project(r,c,AID,NOW+span)) + return result + + +class HistoricalTimingTest(unittest.TestCase): + def test_strict_policy_unchanged(self): + rows=projected(); result=m.reconstruct(rows,config()) + self.assertEqual(result[0]['coveredSeconds'],180) + self.assertEqual(result[0]['publicationEstimatedSeconds'],0) + + def test_equal_endpoint_bounded_tail_estimate(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertEqual(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],120) + self.assertEqual(w['publicationEstimatedBySourceSeconds'],{'pv':120}) + self.assertEqual(w['loadW'],6000) + self.assertFalse(w['fullPhysicalIntervalMeasured']) + self.assertFalse(w['meterBoundaryVerified']) + + def test_no_extension_at_open_end(self): + rows=projected(span=300) + for r in rows: r['raw']['pv']['sourceUpdatedAt']=NOW + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['coveredSeconds'],60) + + def test_equal_zero_requires_new_original_timestamp(self): + rows=projected() + for r in rows:r['raw']['pv']['value']=0 + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],300) + for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],60) + + def test_changed_endpoints_not_interpolated(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=r['raw']['pv']['sourceUpdatedAt']-NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_no_floating_tolerance(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=1e-9*(r['raw']['pv']['sourceUpdatedAt']-NOW) + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_long_gap_not_recovered_by_equal_zero(self): + rows=projected(span=720,period=660) + for r in rows:r['raw']['pv']['value']=0 + out=m.reconstruct(rows,timed_config()) + self.assertEqual(sum(w['publicationEstimatedSeconds'] for w in out),0) + self.assertEqual(out[0]['coveredSeconds'],60) + + def test_collector_gap_not_bridged(self): + rows=projected(); rows=[r for r in rows if r['capturedAt']!=NOW+90] + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_source_blocks_equality_inference(self): + rows=projected();rows[3]['raw']['pv']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + self.assertLess(w['coverage'],1) + + def test_conflicting_same_timestamp_blocks_equality(self): + rows=projected();rows[1]['raw']['pv']['value']+=10 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_other_physical_input_still_vetoes(self): + rows=projected();rows[3]['raw']['battery']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coverage'],1) + + def test_future_receipt_not_available_early(self): + rows=projected() + for r in rows:r['_receivedAt']=NOW+900 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['availableNotBefore'],NOW+900) + + def test_unrelated_virtual_failure_does_not_veto(self): + rows=projected() + for r in rows:r['raw']['sdl']['valid']=False + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coverage'],1) + + def test_no_input_mutation_or_source_age_change(self): + rows=projected();c=timed_config();before=copy.deepcopy((rows,c)) + m.reconstruct(rows,c) + self.assertEqual((rows,c),before) + self.assertEqual(c['sources'][1]['maxAgeSeconds'],60) + + def test_policy_requires_trusted_explicit_bounds(self): + for change in ({'version':True},{'method':'always_fill'},{'sources':{'pv':{'maxSpanSeconds':600,'evidenceId':'test-policy'}}}, + {'sources':{'grid':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}, + {'sources':{'sdl':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}): + c=timed_config();c['historyTimingPolicy'].update(change) + with self.subTest(change=change),self.assertRaises(ValueError):m.validate_config(c) + + def test_profile_values_stay_estimates(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertTrue(w['estimated']) + self.assertNotIn('controlEnabled',w) + self.assertNotIn('accountingEvidenceId',w) + + +class HistoryReferenceIntegrationTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.tmp.name)/'test.sqlite')) + self.c=config();self.new=timed_config(datasetId='physical-v2',sourceDatasetId='physical-v1') + m.register_dataset(self.store.con,AID,self.c,NOW) + def tearDown(self):self.store.close();self.tmp.cleanup() + def register(self):m.register_dataset(self.store.con,AID,self.new,NOW) + def fill(self): + raw=[] + for t in range(NOW-7200,NOW+1,30): + r=record(t);r['raw']['pv']['sourceUpdatedAt']=t-(t-(NOW-7200))%120;raw.append(r) + for i in range(0,len(raw),120):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':raw[i:i+120]},NOW) + return len(raw) + def test_reference_shares_original_immutable_observations(self): + n=self.fill();self.register() + self.assertEqual(m.pipeline_status(self.store.con,AID)['datasets'][1]['records'],n) + self.assertEqual(self.store.con.execute('SELECT count(*) FROM planner_observations').fetchone()[0],n) + self.assertEqual(m.configuration(self.store.con,AID,'physical-v1'),self.c) + def test_no_append_into_derived_dataset(self): + self.register() + with self.assertRaises(ValueError):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v2','records':[record(NOW)]},NOW) + def test_source_mapping_cannot_change_in_reference(self): + for key,value in [('mappingSha256','c'*64),('formula','solar_terminal_v1'),('minimumCoverage',.9)]: + c=copy.deepcopy(self.new);c[key]=value + with self.subTest(key=key),self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_no_reference_to_other_plant(self): + with self.assertRaises(ValueError):m.register_dataset(self.store.con,'00000000-0000-4000-8000-000000000099',self.new,NOW) + def test_no_cycles_or_reference_chains(self): + self.register(); c={**self.new,'datasetId':'physical-v3','sourceDatasetId':'physical-v2'} + with self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_training_model_is_versioned_and_policy_annotated(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v1',settings,NOW+60) + m.advance(self.store.con,AID,'physical-v2',settings,NOW+60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v1',NOW+60)) + model=m.current_model(self.store.con,AID,'physical-v2',NOW+60) + self.assertIsNotNone(model);self.assertEqual(model['historyTimingMethod'],'equal_endpoint_v1') + self.assertFalse(model['measurementBoundaryVerified']) + state=m.pipeline_status(self.store.con,AID)['datasets'][1]['detail'] + self.assertGreater(state['publicationEstimatedSeconds'],0) + self.assertFalse(state['originalFreshnessLimitsChanged']) + def test_no_future_receipt_leaks_into_model(self): + self.fill();self.register() + m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW-60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v2',NOW-60)) + def test_existing_sdl_freshness_not_relaxed_by_history(self): + self.fill();self.register();m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW+30) + op,fc,tar=inputs(datetime.fromtimestamp(NOW+180,timezone.utc)) + with self.assertRaises(ValueError):m.apply_load_forecast(self.store.con,AID,'physical-v2',fc,NOW+180) + def test_full_application_plan_uses_reference_model(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v2',settings,NOW) + op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc)) + for kind,val in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.store,AID,kind,val,datetime.fromtimestamp(NOW,timezone.utc)) + self.store.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v2'},0,datetime.fromtimestamp(NOW,timezone.utc)) + out=run_once(self.store,datetime.fromtimestamp(NOW,timezone.utc)) + self.assertTrue(out['executable'],out);self.assertFalse(out['liveEnabled']) + self.assertEqual(out['inputQuality']['dataPipeline']['historyTimingMethod'],'equal_endpoint_v1') + def test_backfilled_or_late_confirmed_training_is_not_causal_holdout(self): + # Completed synthetic windows learned only NOW cannot validate a model at a past split. + with self.store.con: + for t in range(NOW-3*86400,NOW,300): + window={'start':t,'profileUsable':True,'coverage':1.,'loadW':5000.,'availableNotBefore':NOW} + self.store.con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?)',(AID,'physical-v1',t,NOW,1.,m.canonical(window))) + m.advance(self.store.con,AID,'physical-v1',self.store.settings(AID),NOW) + model=m.current_model(self.store.con,AID,'physical-v1',NOW) + self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout') + + def test_v1_raw_ingest_works_after_reference_added(self): + self.register() + r=m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':[record(NOW)]},NOW) + self.assertEqual(r['stored'],1) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py new file mode 100644 index 0000000..2acf194 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py @@ -0,0 +1,94 @@ +"""Installer ordering/recovery tests; subprocesses, privileges and DB backup are simulated.""" +import hashlib +import importlib.util +import json +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPT=Path(__file__).resolve().parents[1]/'commissioning/deploy_history_timing.py' +spec=importlib.util.spec_from_file_location('history_deploy_tests',SCRIPT) +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class TimingDeploymentTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name) + self.package=self.root/'commissioning/history-timing-source';self.package.mkdir(parents=True) + files={} + for index,name in enumerate(d.TARGETS): + original=(b'old:'+name.encode()) if index%2==0 else None + target=self.root/name;target.parent.mkdir(parents=True,exist_ok=True) + if original is not None:target.write_bytes(original) + candidate=self.package/'source'/name;candidate.parent.mkdir(parents=True,exist_ok=True);candidate.write_bytes(b'new:'+name.encode()) + files[name]={'before':hashlib.sha256(original).hexdigest() if original else None,'after':d.digest(candidate)} + (self.package/'dataset.json').write_text('{}') + (self.root/'commissioning/deploy_application.py').write_text('synthetic helper') + self.manifest={'scope':'historical_publication_only','installationId':'plant','files':files, + 'existingDeploymentHelperSha256':d.digest(self.root/'commissioning/deploy_application.py'), + 'datasetSha256':d.digest(self.package/'dataset.json'),'installerSha256':d.digest(SCRIPT)} + (self.package/'RELEASE.json').write_text(json.dumps(self.manifest)) + self.context=patch.multiple(d,ROOT=self.root,PACKAGE=self.package);self.context.start() + self.calls=[] + def tearDown(self):self.context.stop();self.tmp.cleanup() + def fake_run(self,args,**kwargs): + self.calls.append(args) + if 'inspect' in args:out='sha256:test-old-image' + elif 'ps' in args:out='test-container' + elif 'exec' in args:out=json.dumps({'settingsUnchanged':True,'liveEnabled':False}) + else:out='' + return subprocess.CompletedProcess(args,0,out,'') + def execute(self,fail_at=None): + def run(args,**kwargs): + out=self.fake_run(args,**kwargs) + if fail_at and fail_at in args:raise subprocess.CalledProcessError(1,args) + return out + def atomic(path,data,*args,**kwargs):path.write_bytes(data) + def backup(source,dest):self.calls.append(['test-backup']);dest.write_text('synthetic backup') + helpers=types.SimpleNamespace(atomic=atomic,backup_database=backup) + with patch.object(d.os,'geteuid',return_value=0),patch.object(d.os,'chown'),patch.object(d.subprocess,'run',side_effect=run),patch.dict(sys.modules,{'deploy_application':helpers}): + d.run('plant',True) + def report(self):return json.loads(next(self.root.glob('history-timing-releases/*/REPORT.json')).read_text()) + def test_default_checks_only(self): + d.run('plant');self.assertEqual(self.calls,[]) + self.assertFalse((self.root/'history-timing-releases').exists()) + def test_order_tests_backup_then_only_v4_replacement(self): + self.execute() + test=next(i for i,c in enumerate(self.calls) if '/app/run_tests.py' in c) + backup=next(i for i,c in enumerate(self.calls) if c==['test-backup']) + up=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(test,backup);self.assertLess(backup,up) + self.assertTrue(all('license-portal' not in c and 'forecast-engine' not in c for c in self.calls)) + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_failed_tests_restore_host_source_without_restart(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('/app/run_tests.py') + self.assertFalse(any('up' in c for c in self.calls)) + for name,v in self.manifest['files'].items():self.assertEqual(d.digest(self.root/name),v['before']) + def test_failed_registration_rolls_image_back(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('exec') + self.assertIn('previous_image_restored',self.report()['rollback']) + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_source_drift_refused_before_actions(self): + (self.root/d.TARGETS[0]).write_text('parallel change') + with self.assertRaises(ValueError):d.verify() + self.assertEqual(self.calls,[]) + def test_source_already_installed_idempotent(self): + for name in d.TARGETS:(self.root/name).write_bytes((self.package/'source'/name).read_bytes()) + self.assertEqual(d.verify()['scope'],'historical_publication_only') + self.execute() + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_no_production_selection_or_permission_in_bootstrap(self): + self.assertNotIn("'/settings'",d.BOOTSTRAP) + self.assertNotIn('/trial/arm',d.BOOTSTRAP) + self.assertIn("before['settings']==after['settings']",d.BOOTSTRAP) + self.assertIn("historyTimingVersion",d.BOOTSTRAP) + def test_target_image_contains_installer_test_dependency(self): + dockerfile=(SCRIPT.parents[1]/'Dockerfile').read_text() + self.assertIn('COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py',dockerfile) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/netplan_v4/history_timing.py b/services/netplan-v4/netplan_v4/history_timing.py new file mode 100644 index 0000000..6205d84 --- /dev/null +++ b/services/netplan-v4/netplan_v4/history_timing.py @@ -0,0 +1,58 @@ +"""Explicit, retrospective equal-endpoint estimates for sparse state publication. + +Never changes a source timestamp, real-time freshness limit, meter evidence or +actuator lease. A repeated value after a bounded gap supports a MODEL estimate, +not proof that the physical signal was constant between the observations. +""" +from math import isfinite +import re + + +def validate_policy(policy, sources): + if policy is None: + return None + if (not isinstance(policy, dict) or set(policy) != {'version', 'method', 'sources'} + or type(policy['version']) is not int or policy['version'] != 1 + or policy['method'] != 'equal_endpoint_v1' + or not isinstance(policy['sources'], dict) or not policy['sources']): + raise ValueError('Explicit historical timing policy required') + defined = {s['key']: s for s in sources} + for key, rule in policy['sources'].items(): + if key not in defined or defined[key]['role'] not in ('pv', 'physical_storage', 'solar_scale'): + raise ValueError('Timing estimates limited to explicit power/scale sources') + if not isinstance(rule, dict) or set(rule) != {'maxSpanSeconds', 'evidenceId'}: + raise ValueError('Timing bound and evidence reference required') + if type(rule['maxSpanSeconds']) is not int or not defined[key]['maxAgeSeconds'] <= rule['maxSpanSeconds'] <= 180: + raise ValueError('Historical endpoint span outside reviewed bounds') + if not isinstance(rule['evidenceId'], str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,100}', rule['evidenceId']): + raise ValueError('Invalid publication evidence reference') + return policy + + +def endpoint_bridges(series, first_observed, blocks, capture_gaps, policy, source_specs): + """Index strictly consecutive, equal original timestamps; never invent endpoints. + +Only the stale tail of a normal hold is filled. A missing/invalid capture or a +source conflict forbids bridging. The right endpoint must have been observed; +its availability is retained by the caller for causal training and replay. +""" + result = {key: {} for key in series} + if policy is None: + return result + for key, rule in policy['sources'].items(): + if key not in series: + continue # e.g. an unused DC channel in an AC-terminal formula + values = series[key] + times = sorted(values) + for left, right in zip(times, times[1:]): + value, next_value = values[left], values[right] + if (value is None or next_value is None or type(value) not in (int, float) + or not isfinite(value) or value != next_value): + continue + if not source_specs[key]['maxAgeSeconds'] < right-left <= rule['maxSpanSeconds']: + continue + if any(start < right and end > left for start, end in (*blocks[key], *capture_gaps)): + continue + result[key][left] = {'end': right, 'availableAt': first_observed[key][right], + 'spanSeconds': right-left} + return result diff --git a/services/netplan-v4/netplan_v4/measurement_pipeline.py b/services/netplan-v4/netplan_v4/measurement_pipeline.py index ead16bc..0579467 100644 --- a/services/netplan-v4/netplan_v4/measurement_pipeline.py +++ b/services/netplan-v4/netplan_v4/measurement_pipeline.py @@ -13,6 +13,7 @@ from math import isfinite from statistics import median from zoneinfo import ZoneInfo import json +from .history_timing import validate_policy, endpoint_bridges UTC = timezone.utc LOCAL = ZoneInfo('Europe/Zurich') @@ -79,7 +80,8 @@ def validate_config(c): fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', 'minimumTrainingHours', 'historyDays'} - if not isinstance(c, dict) or set(c) != fields: + optional = {'sourceDatasetId', 'historyTimingPolicy'} + if not isinstance(c, dict) or not fields <= set(c) or set(c)-fields-optional: raise ValueError('Explicit dataset configuration required') name = c['datasetId'] if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): @@ -122,6 +124,10 @@ def validate_config(c): raise ValueError('Solar origin roles mismatch') elif sr is not None: raise ValueError('No unused solar mapping allowed') + validate_policy(c.get('historyTimingPolicy'), sources) + source = c.get('sourceDatasetId') + if source is not None and (not isinstance(source, str) or not 1 <= len(source) <= 80 or source == name or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in source)): + raise ValueError('Invalid source dataset reference') canonical(c) return c @@ -129,6 +135,13 @@ def validate_config(c): def register_dataset(con, plant, c, now): """Operator endpoint only; device append endpoint cannot change units or limits.""" validate_config(c) + if c.get('sourceDatasetId'): + origin = configuration(con, plant, c['sourceDatasetId']) + if origin.get('sourceDatasetId'): + raise ValueError('Dataset reference chains are not allowed') + comparable = lambda v: {k:x for k,x in v.items() if k not in ('datasetId','sourceDatasetId','historyTimingPolicy')} + if canonical(comparable(origin)) != canonical(comparable(c)): + raise ValueError('Referenced observations must retain identical measurement meaning') value = canonical(c) con.execute('BEGIN IMMEDIATE') try: @@ -178,6 +191,8 @@ def ingest_batch(con, plant, payload, now): if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: raise ValueError('Measurement batch version/fields invalid') c = configuration(con,plant,payload['datasetId']) + if c.get('sourceDatasetId'): + raise ValueError('Derived dataset is read-only; append to the original measurement dataset') records = payload['records'] if not isinstance(records,list) or not 1 <= len(records) <= 120: raise ValueError('Batch requires 1..120 captures') @@ -244,8 +259,10 @@ uncovered portions remain quantified and are never filled with zero. primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) for s in c['sources']: if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) first,last = records[0]['capturedAt'],records[-1]['capturedAt'] series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + first_observed = {k:{} for k in primary} pending = {k:None for k in primary}; high = {k:0 for k in primary} edges = {first,last} for a,b in zip(records,records[1:]): @@ -266,6 +283,7 @@ uncovered portions remain quantified and are never filled with zero. series[k][t] = None else: series[k].setdefault(t,v['value']) + first_observed[k].setdefault(t, max(at, r.get('_receivedAt', at))) for k,s in primary.items(): if pending[k] is not None: blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) @@ -273,24 +291,38 @@ uncovered portions remain quantified and are never filled with zero. edges.update(range(first//300*300+300,last,300)) edges = sorted(x for x in edges if first <= x <= last) knots = {k:sorted(v) for k,v in series.items()} + bridges = endpoint_bridges(series, first_observed, blocks, gaps, timing_policy, primary) bins = {} for a,b in zip(edges,edges[1:]): - start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0}) + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0, + 'publicationEstimatedSeconds':0,'publicationEstimatedBySource':{},'knownAt':0,'missingSourceSeconds':{}}) vals = {}; usable = not any(x <= a < y for x,y in gaps) + extended = []; unavailable = []; known_at = 0 for k,s in primary.items(): pos = bisect_right(knots[k],a)-1 t = knots[k][pos] if pos >= 0 else None - if t is None or a >= t+s['maxAgeSeconds'] or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): - usable = False - else: vals[k] = series[k][t] + bridge = bridges[k].get(t) + expired = t is None or a >= t+s['maxAgeSeconds'] + supported_tail = bool(bridge and a < bridge['end']) + if t is None or (expired and not supported_tail) or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False; unavailable.append(k) + else: + vals[k] = series[k][t] + known_at = max(known_at, first_observed[k][t]) + if expired: + extended.append(k); known_at = max(known_at, bridge['availableAt']) load = None if usable: try: load = physical_value(vals,c) except ValueError: usable = False if usable: item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + item['knownAt'] = max(item['knownAt'], known_at) + if extended: item['publicationEstimatedSeconds'] += b-a + for key in extended: item['publicationEstimatedBySource'][key] = item['publicationEstimatedBySource'].get(key,0)+b-a else: item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + for key in unavailable: item['missingSourceSeconds'][key] = item['missingSourceSeconds'].get(key,0)+b-a out=[] for t,item in sorted(bins.items()): # Partial beginning/end bins remain diagnostic and cannot train. @@ -300,7 +332,12 @@ uncovered portions remain quantified and are never filled with zero. out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, - 'meterBoundaryVerified':False,'method':c['formula']}) + 'meterBoundaryVerified':False,'method':c['formula'], + 'historyTimingMethod':(timing_policy or {}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':item['publicationEstimatedSeconds'], + 'publicationEstimatedBySourceSeconds':item['publicationEstimatedBySource'], + 'missingSourceSeconds':item['missingSourceSeconds'], + 'availableNotBefore':max(t+300,item['knownAt'])}) return out @@ -341,13 +378,14 @@ def advance(con, plant, dataset, settings, now): c=configuration(con,plant,dataset); tick=now//300 old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() if old and old[0]==tick: return - fetched=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', - (plant,dataset,now-172800-300,now,now)).fetchall() - records=[json.loads(r[0]) for r in fetched] + observation_dataset=c.get('sourceDatasetId',dataset) + fetched=con.execute('SELECT value,received_at FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,observation_dataset,now-172800-300,now,now)).fetchall() + records=[{**json.loads(r[0]),'_receivedAt':r[1]} for r in fetched] windows=reconstruct(records,c) with con: for w in windows: - if w['start']+300 > now-30: continue + if w['start']+300 > now-30 or w.get('availableNotBefore',0)>now: continue con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', (plant,dataset,w['start'],now,w['coverage'],canonical(w))) rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] @@ -356,7 +394,12 @@ def advance(con, plant, dataset, settings, now): cadence=86400 if settings['trainingCadence']=='daily' else 604800 detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], - 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True} + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True, + 'observationDatasetId':observation_dataset, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':sum(r.get('publicationEstimatedSeconds',0) for r in good), + 'originalFreshnessLimitsChanged':False, + 'lastCapture':iso(records[-1]['capturedAt']) if records else None} state='collecting' if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: state='model_ready' if active else 'training' @@ -366,10 +409,13 @@ def advance(con, plant, dataset, settings, now): candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'timingPolicySha256':sha256(canonical(c.get('historyTimingPolicy')).encode()).hexdigest(), + 'observationDatasetId':observation_dataset, 'measurementBoundaryVerified':False} # Causal held-out validation: build validation profiles without the final day. split=good[-1]['start']-86400 - train=[r for r in good if r['start']+300<=split]; test=[r for r in good if r['start']>=split] + train=[r for r in good if r['start']+300<=split and r.get('availableNotBefore',r['start']+300)<=split]; test=[r for r in good if r['start']>=split] if len(train)>=288 and len(test)>=240: val={'profiles':build_profiles(train)} errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} @@ -378,7 +424,7 @@ def advance(con, plant, dataset, settings, now): # Existing model can be replaced only with held-out evidence and no aggregate regression. promote=active is None if active and candidate['validation']['status']=='causal_holdout': - past_model_eligible=active['trainedThrough']<=split + past_model_eligible=active['trainedThrough']<=split and active['trainedAt']<=split if past_model_eligible: incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) @@ -409,9 +455,10 @@ def apply_load_forecast(con,plant,dataset,forecast,decision): model=current_model(con,plant,dataset,decision) if not model: raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') - last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,dataset,decision,decision)).fetchone() + c=configuration(con,plant,dataset) + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,c.get('sourceDatasetId',dataset),decision,decision)).fetchone() if not last: raise ValueError('No recent corrected observation') - last=json.loads(last[0]); c=configuration(con,plant,dataset) + last=json.loads(last[0]) if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') @@ -433,7 +480,9 @@ def apply_load_forecast(con,plant,dataset,forecast,decision): 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), - 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation']}} + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation'], + 'historyTimingMethod':model.get('historyTimingMethod','strict_expiry'), + 'observationDatasetId':model.get('observationDatasetId',dataset)}} return result @@ -441,9 +490,11 @@ def pipeline_status(con,plant): out=[] for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() - count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, - 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds']}) - return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False} + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], + 'observationDatasetId':c.get('sourceDatasetId',ds), + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry')}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False,'historyTimingVersion':1} diff --git a/services/netplan-v4/tests/test_history_timing.py b/services/netplan-v4/tests/test_history_timing.py new file mode 100644 index 0000000..b05b046 --- /dev/null +++ b/services/netplan-v4/tests/test_history_timing.py @@ -0,0 +1,204 @@ +"""Synthetic publication gaps plus full source-reference/application integration.""" +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.history_timing import validate_policy +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once +from test_measurement_pipeline import config, record, NOW, AID +from test_v4 import inputs, TOKEN + + +def timed_config(**kw): + return config(historyTimingPolicy={'version':1,'method':'equal_endpoint_v1', + 'sources':{'pv':{'maxSpanSeconds':130,'evidenceId':'synthetic-publication-policy'}}}, **kw) + + +def projected(span=360, period=120, c=None): + c = c or timed_config() + result=[] + for delta in range(0,span+1,30): + r=record(NOW+delta,c) + r['raw']['pv']['sourceUpdatedAt']=NOW+(delta//period)*period + result.append(m.project(r,c,AID,NOW+span)) + return result + + +class HistoricalTimingTest(unittest.TestCase): + def test_strict_policy_unchanged(self): + rows=projected(); result=m.reconstruct(rows,config()) + self.assertEqual(result[0]['coveredSeconds'],180) + self.assertEqual(result[0]['publicationEstimatedSeconds'],0) + + def test_equal_endpoint_bounded_tail_estimate(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertEqual(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],120) + self.assertEqual(w['publicationEstimatedBySourceSeconds'],{'pv':120}) + self.assertEqual(w['loadW'],6000) + self.assertFalse(w['fullPhysicalIntervalMeasured']) + self.assertFalse(w['meterBoundaryVerified']) + + def test_no_extension_at_open_end(self): + rows=projected(span=300) + for r in rows: r['raw']['pv']['sourceUpdatedAt']=NOW + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['coveredSeconds'],60) + + def test_equal_zero_requires_new_original_timestamp(self): + rows=projected() + for r in rows:r['raw']['pv']['value']=0 + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],300) + for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],60) + + def test_changed_endpoints_not_interpolated(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=r['raw']['pv']['sourceUpdatedAt']-NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_no_floating_tolerance(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=1e-9*(r['raw']['pv']['sourceUpdatedAt']-NOW) + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_long_gap_not_recovered_by_equal_zero(self): + rows=projected(span=720,period=660) + for r in rows:r['raw']['pv']['value']=0 + out=m.reconstruct(rows,timed_config()) + self.assertEqual(sum(w['publicationEstimatedSeconds'] for w in out),0) + self.assertEqual(out[0]['coveredSeconds'],60) + + def test_collector_gap_not_bridged(self): + rows=projected(); rows=[r for r in rows if r['capturedAt']!=NOW+90] + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_source_blocks_equality_inference(self): + rows=projected();rows[3]['raw']['pv']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + self.assertLess(w['coverage'],1) + + def test_conflicting_same_timestamp_blocks_equality(self): + rows=projected();rows[1]['raw']['pv']['value']+=10 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_other_physical_input_still_vetoes(self): + rows=projected();rows[3]['raw']['battery']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coverage'],1) + + def test_future_receipt_not_available_early(self): + rows=projected() + for r in rows:r['_receivedAt']=NOW+900 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['availableNotBefore'],NOW+900) + + def test_unrelated_virtual_failure_does_not_veto(self): + rows=projected() + for r in rows:r['raw']['sdl']['valid']=False + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coverage'],1) + + def test_no_input_mutation_or_source_age_change(self): + rows=projected();c=timed_config();before=copy.deepcopy((rows,c)) + m.reconstruct(rows,c) + self.assertEqual((rows,c),before) + self.assertEqual(c['sources'][1]['maxAgeSeconds'],60) + + def test_policy_requires_trusted_explicit_bounds(self): + for change in ({'version':True},{'method':'always_fill'},{'sources':{'pv':{'maxSpanSeconds':600,'evidenceId':'test-policy'}}}, + {'sources':{'grid':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}, + {'sources':{'sdl':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}): + c=timed_config();c['historyTimingPolicy'].update(change) + with self.subTest(change=change),self.assertRaises(ValueError):m.validate_config(c) + + def test_profile_values_stay_estimates(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertTrue(w['estimated']) + self.assertNotIn('controlEnabled',w) + self.assertNotIn('accountingEvidenceId',w) + + +class HistoryReferenceIntegrationTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.tmp.name)/'test.sqlite')) + self.c=config();self.new=timed_config(datasetId='physical-v2',sourceDatasetId='physical-v1') + m.register_dataset(self.store.con,AID,self.c,NOW) + def tearDown(self):self.store.close();self.tmp.cleanup() + def register(self):m.register_dataset(self.store.con,AID,self.new,NOW) + def fill(self): + raw=[] + for t in range(NOW-7200,NOW+1,30): + r=record(t);r['raw']['pv']['sourceUpdatedAt']=t-(t-(NOW-7200))%120;raw.append(r) + for i in range(0,len(raw),120):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':raw[i:i+120]},NOW) + return len(raw) + def test_reference_shares_original_immutable_observations(self): + n=self.fill();self.register() + self.assertEqual(m.pipeline_status(self.store.con,AID)['datasets'][1]['records'],n) + self.assertEqual(self.store.con.execute('SELECT count(*) FROM planner_observations').fetchone()[0],n) + self.assertEqual(m.configuration(self.store.con,AID,'physical-v1'),self.c) + def test_no_append_into_derived_dataset(self): + self.register() + with self.assertRaises(ValueError):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v2','records':[record(NOW)]},NOW) + def test_source_mapping_cannot_change_in_reference(self): + for key,value in [('mappingSha256','c'*64),('formula','solar_terminal_v1'),('minimumCoverage',.9)]: + c=copy.deepcopy(self.new);c[key]=value + with self.subTest(key=key),self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_no_reference_to_other_plant(self): + with self.assertRaises(ValueError):m.register_dataset(self.store.con,'00000000-0000-4000-8000-000000000099',self.new,NOW) + def test_no_cycles_or_reference_chains(self): + self.register(); c={**self.new,'datasetId':'physical-v3','sourceDatasetId':'physical-v2'} + with self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_training_model_is_versioned_and_policy_annotated(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v1',settings,NOW+60) + m.advance(self.store.con,AID,'physical-v2',settings,NOW+60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v1',NOW+60)) + model=m.current_model(self.store.con,AID,'physical-v2',NOW+60) + self.assertIsNotNone(model);self.assertEqual(model['historyTimingMethod'],'equal_endpoint_v1') + self.assertFalse(model['measurementBoundaryVerified']) + state=m.pipeline_status(self.store.con,AID)['datasets'][1]['detail'] + self.assertGreater(state['publicationEstimatedSeconds'],0) + self.assertFalse(state['originalFreshnessLimitsChanged']) + def test_no_future_receipt_leaks_into_model(self): + self.fill();self.register() + m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW-60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v2',NOW-60)) + def test_existing_sdl_freshness_not_relaxed_by_history(self): + self.fill();self.register();m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW+30) + op,fc,tar=inputs(datetime.fromtimestamp(NOW+180,timezone.utc)) + with self.assertRaises(ValueError):m.apply_load_forecast(self.store.con,AID,'physical-v2',fc,NOW+180) + def test_full_application_plan_uses_reference_model(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v2',settings,NOW) + op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc)) + for kind,val in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.store,AID,kind,val,datetime.fromtimestamp(NOW,timezone.utc)) + self.store.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v2'},0,datetime.fromtimestamp(NOW,timezone.utc)) + out=run_once(self.store,datetime.fromtimestamp(NOW,timezone.utc)) + self.assertTrue(out['executable'],out);self.assertFalse(out['liveEnabled']) + self.assertEqual(out['inputQuality']['dataPipeline']['historyTimingMethod'],'equal_endpoint_v1') + def test_backfilled_or_late_confirmed_training_is_not_causal_holdout(self): + # Completed synthetic windows learned only NOW cannot validate a model at a past split. + with self.store.con: + for t in range(NOW-3*86400,NOW,300): + window={'start':t,'profileUsable':True,'coverage':1.,'loadW':5000.,'availableNotBefore':NOW} + self.store.con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?)',(AID,'physical-v1',t,NOW,1.,m.canonical(window))) + m.advance(self.store.con,AID,'physical-v1',self.store.settings(AID),NOW) + model=m.current_model(self.store.con,AID,'physical-v1',NOW) + self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout') + + def test_v1_raw_ingest_works_after_reference_added(self): + self.register() + r=m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':[record(NOW)]},NOW) + self.assertEqual(r['stored'],1) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_history_timing_deployment.py b/services/netplan-v4/tests/test_history_timing_deployment.py new file mode 100644 index 0000000..2acf194 --- /dev/null +++ b/services/netplan-v4/tests/test_history_timing_deployment.py @@ -0,0 +1,94 @@ +"""Installer ordering/recovery tests; subprocesses, privileges and DB backup are simulated.""" +import hashlib +import importlib.util +import json +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPT=Path(__file__).resolve().parents[1]/'commissioning/deploy_history_timing.py' +spec=importlib.util.spec_from_file_location('history_deploy_tests',SCRIPT) +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class TimingDeploymentTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name) + self.package=self.root/'commissioning/history-timing-source';self.package.mkdir(parents=True) + files={} + for index,name in enumerate(d.TARGETS): + original=(b'old:'+name.encode()) if index%2==0 else None + target=self.root/name;target.parent.mkdir(parents=True,exist_ok=True) + if original is not None:target.write_bytes(original) + candidate=self.package/'source'/name;candidate.parent.mkdir(parents=True,exist_ok=True);candidate.write_bytes(b'new:'+name.encode()) + files[name]={'before':hashlib.sha256(original).hexdigest() if original else None,'after':d.digest(candidate)} + (self.package/'dataset.json').write_text('{}') + (self.root/'commissioning/deploy_application.py').write_text('synthetic helper') + self.manifest={'scope':'historical_publication_only','installationId':'plant','files':files, + 'existingDeploymentHelperSha256':d.digest(self.root/'commissioning/deploy_application.py'), + 'datasetSha256':d.digest(self.package/'dataset.json'),'installerSha256':d.digest(SCRIPT)} + (self.package/'RELEASE.json').write_text(json.dumps(self.manifest)) + self.context=patch.multiple(d,ROOT=self.root,PACKAGE=self.package);self.context.start() + self.calls=[] + def tearDown(self):self.context.stop();self.tmp.cleanup() + def fake_run(self,args,**kwargs): + self.calls.append(args) + if 'inspect' in args:out='sha256:test-old-image' + elif 'ps' in args:out='test-container' + elif 'exec' in args:out=json.dumps({'settingsUnchanged':True,'liveEnabled':False}) + else:out='' + return subprocess.CompletedProcess(args,0,out,'') + def execute(self,fail_at=None): + def run(args,**kwargs): + out=self.fake_run(args,**kwargs) + if fail_at and fail_at in args:raise subprocess.CalledProcessError(1,args) + return out + def atomic(path,data,*args,**kwargs):path.write_bytes(data) + def backup(source,dest):self.calls.append(['test-backup']);dest.write_text('synthetic backup') + helpers=types.SimpleNamespace(atomic=atomic,backup_database=backup) + with patch.object(d.os,'geteuid',return_value=0),patch.object(d.os,'chown'),patch.object(d.subprocess,'run',side_effect=run),patch.dict(sys.modules,{'deploy_application':helpers}): + d.run('plant',True) + def report(self):return json.loads(next(self.root.glob('history-timing-releases/*/REPORT.json')).read_text()) + def test_default_checks_only(self): + d.run('plant');self.assertEqual(self.calls,[]) + self.assertFalse((self.root/'history-timing-releases').exists()) + def test_order_tests_backup_then_only_v4_replacement(self): + self.execute() + test=next(i for i,c in enumerate(self.calls) if '/app/run_tests.py' in c) + backup=next(i for i,c in enumerate(self.calls) if c==['test-backup']) + up=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(test,backup);self.assertLess(backup,up) + self.assertTrue(all('license-portal' not in c and 'forecast-engine' not in c for c in self.calls)) + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_failed_tests_restore_host_source_without_restart(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('/app/run_tests.py') + self.assertFalse(any('up' in c for c in self.calls)) + for name,v in self.manifest['files'].items():self.assertEqual(d.digest(self.root/name),v['before']) + def test_failed_registration_rolls_image_back(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('exec') + self.assertIn('previous_image_restored',self.report()['rollback']) + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_source_drift_refused_before_actions(self): + (self.root/d.TARGETS[0]).write_text('parallel change') + with self.assertRaises(ValueError):d.verify() + self.assertEqual(self.calls,[]) + def test_source_already_installed_idempotent(self): + for name in d.TARGETS:(self.root/name).write_bytes((self.package/'source'/name).read_bytes()) + self.assertEqual(d.verify()['scope'],'historical_publication_only') + self.execute() + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_no_production_selection_or_permission_in_bootstrap(self): + self.assertNotIn("'/settings'",d.BOOTSTRAP) + self.assertNotIn('/trial/arm',d.BOOTSTRAP) + self.assertIn("before['settings']==after['settings']",d.BOOTSTRAP) + self.assertIn("historyTimingVersion",d.BOOTSTRAP) + def test_target_image_contains_installer_test_dependency(self): + dockerfile=(SCRIPT.parents[1]/'Dockerfile').read_text() + self.assertIn('COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py',dockerfile) + + +if __name__=='__main__':unittest.main() From 6c91d6bcc6dff83909777ada82921f7ffdbcb243 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sat, 3 Oct 2026 09:40:02 +0000 Subject: [PATCH 13/22] fix(accounting): stabilize numeric mapping identity across property storage --- docs/netplan-v4-numeric-mapping-fix.md | 52 ++++++++++++++++++++++++++ libs/NetzfahrplanV4Bilanzierung.php | 3 +- tests/V4Accounting/checks.php | 10 +++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 docs/netplan-v4-numeric-mapping-fix.md diff --git a/docs/netplan-v4-numeric-mapping-fix.md b/docs/netplan-v4-numeric-mapping-fix.md new file mode 100644 index 0000000..a834702 --- /dev/null +++ b/docs/netplan-v4-numeric-mapping-fix.md @@ -0,0 +1,52 @@ +# Numeric mapping identity defect - 2026-10-03 + +## Confirmed from installed code and read-only evidence + +The pending batch contains 120 observations: zero duplicate timestamps, zero +backwards pairs, but 75 identity failures. The initial 45 observations in this +batch are imported observer records; later native captures use the saved manager +configuration. Evidence: test-host `PENDING_DELIVERY_REVIEW.json`. + +The configured mapping fingerprint is +`517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b`. +Reproducing native configuration loading generates +`3a1a9cf42aae49f55385d2350da74cae780ae12f170e4976849591e48a393994`. +The sole difference in the compared configurations is numeric representation of +`accounting.splitToleranceW`: prepared float 100.0 versus saved integer 100. +Both inventory fingerprints and all measurement definitions match. +`NetzfahrplanV4Bilanzierung::konfigurieren` validated this field using a function +which returns a float, but discarded the normalized return value. Other numeric +factors were already assigned from that validator. + +## Tested source correction + +Assign the returned float to `splitToleranceW`. Equal numeric settings now retain +the original fingerprint through a plain JSON property round trip. Actual numeric +changes still produce distinct fingerprints, and boolean/string values are still +rejected. Input configurations are not mutated. + +42 isolated PHP checks passed on the test host: 34 accounting regressions plus +8 numeric identity regressions. A separate check with the actual prepared capture +configuration reproduces the configured fingerprint after the property round trip. +Evidence: `/srv/agent/netplan-v4-application-build/NUMERIC_MAPPING_TEST_RESULTS.json` +and `NUMERIC_MAPPING_TEST_RESULTS.txt`. Library SHA256: +`ffda2a03dbb226a4405a2ad66704c080a9dc475ce64282ad3f0d7d60034ab199`. + +## Not deployed / recovery still incomplete + +No installed library, original outbox, cursor, server validator, settings, dataset +registration or actuator permission was changed in this turn. Future-capture +normalization alone does not recover already stored native observations. The +server correctly rejects an entire batch containing a mismatched mapping. + +An untested proposal to send distinct immutable dataset versions in separate +batches was removed from the development source and preserved only as a draft at +`/home/agent/services/qa/numeric-mapping-20261003/UNTESTED_SENDER_DRAFT.patch`. +Metadata preparation files are not an installation or permission to ingest into +an unregistered dataset. Tool writes for the complete server recovery package were +blocked; no alternative deployment or identity-check bypass was performed. + +Next recovery must preserve original records and provenance, retain exact +per-dataset identity validation, acknowledge only accepted data and include tests +for the mixed 45/75 batch, retries, partial ACKs and no cursor/data loss. Do not +skip the rejected block, rewrite hashes in raw journals or re-run old installers. diff --git a/libs/NetzfahrplanV4Bilanzierung.php b/libs/NetzfahrplanV4Bilanzierung.php index 38f61a2..a468dc4 100644 --- a/libs/NetzfahrplanV4Bilanzierung.php +++ b/libs/NetzfahrplanV4Bilanzierung.php @@ -34,7 +34,8 @@ final class NetzfahrplanV4Bilanzierung || !is_int($c['maxSkewSeconds']) || $c['maxSkewSeconds'] < 0 || $c['maxSkewSeconds'] > 60) { throw new InvalidArgumentException('Bilanzkonfiguration unvollstaendig.'); } - self::number($c['splitToleranceW'], 'Zuordnungstoleranz W', 0, 500); + // Preserve a stable numeric representation across JSON property round-trips. + $c['splitToleranceW'] = self::number($c['splitToleranceW'], 'Zuordnungstoleranz W', 0, 500); $seen=[];$keys=[];$roles=[]; foreach ($c['sources'] as &$s) { if (!is_array($s) || count($s)!==8 || array_diff(['key','role','variableId','parentId','ident','factorToW','maxAgeSeconds','dependsOn'],array_keys($s))) { diff --git a/tests/V4Accounting/checks.php b/tests/V4Accounting/checks.php index 2d9c806..64084f7 100644 --- a/tests/V4Accounting/checks.php +++ b/tests/V4Accounting/checks.php @@ -53,4 +53,14 @@ $c['sources'][0]['factorToW']=2.;check(take($c,$m,$n)['mappingId']!==$a,'factor [$c,$m,$n]=fixture();$calls=[]; fails(function()use($c,$m,$n,&$calls){B::aufnehmen($c,function($id)use($m,&$calls){$calls[$id]=($calls[$id]??0)+1;$v=$m[$id];if($calls[$id]>1)$v['value']++;return $v;},$n);},'same-second concurrent value change rejected'); $source=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Bilanzierung.php');check(!preg_match('/\b(?:SetValue|IPS_RequestAction|RequestAction|curl_exec|IPS_SetProperty|AC_SetLoggingStatus)\s*\(/',$source),'adapter has no runtime or external write calls'); +[$c,$m,$n]=fixture();$integer=$c;$integer['splitToleranceW']=100; +check(B::konfigurieren($c)===B::konfigurieren($integer),'integer and float tolerance normalize identically'); +check(take($c,$m,$n)['mappingId']===take($integer,$m,$n)['mappingId'],'equal tolerance gives identical mapping hash'); +$roundtrip=json_decode(json_encode($c,JSON_THROW_ON_ERROR),true,64,JSON_THROW_ON_ERROR); +check(take($roundtrip,$m,$n)['mappingId']===take($c,$m,$n)['mappingId'],'plain JSON property roundtrip preserves mapping'); +$normalized=B::konfigurieren($c);check(B::konfigurieren($normalized)===$normalized,'normalization remains idempotent'); +check(is_int($integer['splitToleranceW']) && is_float($c['splitToleranceW']),'input configurations are not mutated'); +$changed=$c;$changed['splitToleranceW']=101; +check(take($changed,$m,$n)['mappingId']!==take($c,$m,$n)['mappingId'],'real tolerance change still changes identity'); +foreach([true,'100'] as $invalid){$bad=$c;$bad['splitToleranceW']=$invalid;fails(fn()=>B::konfigurieren($bad),'non-numeric tolerance remains rejected '.gettype($invalid));} echo "TOTAL $count checks passed; synthetic snapshots only.\n"; From 8eb9688942f723238d1a48e322068a8bcaa3a5e4 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sat, 3 Oct 2026 10:40:18 +0000 Subject: [PATCH 14/22] feat(v4): consolidate audited data recovery economic replay and optional archival --- docs/netplan-v4-unified-release.md | 77 ++++ examples/V4UnifiedRelease/file_installer.php | 35 ++ examples/V4UnifiedRelease/install.php | 33 ++ libs/ManagerNetzfahrplanV4DatenTrait.php | 48 +- libs/NetzfahrplanV4Datenarchiv.php | 78 ++++ services/netplan-v4/Dockerfile | 1 + services/netplan-v4/SOURCE_MANIFEST.json | 36 +- .../commissioning/deploy_unified_release.py | 124 ++++++ .../commissioning/unified-release/.gitignore | 1 + .../unified-release/RELEASE.json | 414 ++++++++++++++++++ .../unified-release/build_context.py | 28 ++ .../unified-release/mapping-proof.json | 5 + services/netplan-v4/gui/netplan-v4.js | 10 +- .../netplan-v4/netplan_v4/economic_replay.py | 207 +++++++++ .../netplan-v4/netplan_v4/mapping_identity.py | 100 +++++ .../netplan_v4/measurement_pipeline.py | 27 +- services/netplan-v4/netplan_v4/retention.py | 106 +++++ services/netplan-v4/netplan_v4/service.py | 25 +- services/netplan-v4/netplan_v4/store.py | 4 +- .../netplan-v4/tests/test_economic_replay.py | 144 ++++++ .../netplan-v4/tests/test_mapping_recovery.py | 102 +++++ .../tests/test_release_preflight.py | 3 +- services/netplan-v4/tests/test_retention.py | 56 +++ .../netplan-v4/tests/test_unified_release.py | 81 ++++ tests/V4ApplicationData/archive_checks.php | 27 ++ tests/V4ApplicationData/checks.php | 13 + 26 files changed, 1747 insertions(+), 38 deletions(-) create mode 100644 docs/netplan-v4-unified-release.md create mode 100644 examples/V4UnifiedRelease/file_installer.php create mode 100644 examples/V4UnifiedRelease/install.php create mode 100644 libs/NetzfahrplanV4Datenarchiv.php create mode 100644 services/netplan-v4/commissioning/deploy_unified_release.py create mode 100644 services/netplan-v4/commissioning/unified-release/.gitignore create mode 100644 services/netplan-v4/commissioning/unified-release/RELEASE.json create mode 100644 services/netplan-v4/commissioning/unified-release/build_context.py create mode 100644 services/netplan-v4/commissioning/unified-release/mapping-proof.json create mode 100644 services/netplan-v4/netplan_v4/economic_replay.py create mode 100644 services/netplan-v4/netplan_v4/mapping_identity.py create mode 100644 services/netplan-v4/netplan_v4/retention.py create mode 100644 services/netplan-v4/tests/test_economic_replay.py create mode 100644 services/netplan-v4/tests/test_mapping_recovery.py create mode 100644 services/netplan-v4/tests/test_retention.py create mode 100644 services/netplan-v4/tests/test_unified_release.py create mode 100644 tests/V4ApplicationData/archive_checks.php diff --git a/docs/netplan-v4-unified-release.md b/docs/netplan-v4-unified-release.md new file mode 100644 index 0000000..09bd7e0 --- /dev/null +++ b/docs/netplan-v4-unified-release.md @@ -0,0 +1,77 @@ +# V4 Unified RC1 - actual scope and release boundary + +This release consolidates application data recovery, native delivery status, economic +family comparison, and optional verified archival. It is NOT a production actuation +release. No claim that all original product requirements or hardware acceptance are +complete. Sources are in the existing develop branch, not feature branches. + +## Delivered code + +* Audited recovery for the exact 100.0/100 splitToleranceW serialization defect. + The internal operator endpoint validates the two complete JSON representations, + their exact SHA256, installation, inventory, and every field. Only that numeric + representation may differ. Device-supplied arbitrary aliases are never accepted. + Existing rows/measurements are not rewritten. The original received mapping and + proof reference are recorded with the observation. A mixed 45/75 block and retries + are regression tested. Adoption requires an explicit deployment flag. +* Native normalization prevents recurrence. The sender preserves a separate last + successful receipt even during scheduled/backoff states, reports safe numeric HTTP + status (no response or credential text), and continues trying to deliver a durable + prefix if the current capture cannot be appended. No cursor reset or skipped block. +* Economic comparison is actually connected, not an empty score list. At the start + of a local calendar day the worker freezes comparable plans for families 3/13/23 + with the then-known forecasts, prices and common battery/limit state. Completed + days are evaluated against subsequently acquired physical residual data. Initial + energy, losses, common terminal energy valuation and 15-minute demand charge are + included; monthly maxima are charged once over a comparison period, not per day. + GUI lookback/minimum days/coverage/switch margin are applied by the selector. +* v1 economic scope: one grid-charge-enabled battery, frozen day-ahead plans, with + local grid-target tracking simulated during the day. It is NOT a full receding- + horizon replay. Historical external SDL uses the then-recorded SDL request and is + clearly marked as an estimate, not independently measured delivery. Missing + actuals, unmatched contexts and breaches cannot become zero cost or a successful + comparison. Broader device topologies stay collecting; no fake equivalence. +* Lossless archive implementation and tests are present for acknowledged old native + outbox days and old server observations/plans. Archives are verified before plain + rows/files are retired. Active plans, acknowledged plans and unacknowledged data + are excluded. Archiving is OFF by default; enable only with an explicit storage + policy (native NetzfahrplanV4ArchivAktiv; server NETPLAN_V4_ARCHIVE_ENABLED=1). + Compressed archives are retained: this is not an external backup or infinite disk. + +## Deployment is not data/model/actuator acceptance + +Server entry: commissioning/deploy_unified_release.py. Default checks sources only. +--apply builds and tests a separate candidate context with no network/production +mounts before replacing the V4 service. It preserves the running plant allowlist, +backs up SQLite consistently, and uses a pinned built image. Only the V4 planner JS +asset is updated; unrelated GUI work, portal runtime, tariffs, forecasting, Symcon +and actuators are untouched. Rollback restores previous source/image; additive audit +rows remain. Database schema changes do not overwrite old observations. + +--approve-numeric-mapping-compatibility explicitly authorizes the demonstrated pair +of equivalent configuration representations. Without it, strict original validation +remains and backlog containing the legacy serialization is still rejected. + +Lihrenmoos native stage: /srv/agent/netplan-v4-unified-release/install.php. +This patches three data-only libraries, reloads the existing library, compares all +pre-existing properties and lets the existing data timer resume. No raw replay, +force-skip or actuator callback from the installer. Archiving stays disabled by default. + +## Remaining product work - do not label as complete + +1. The full production dispatcher is not installed. Earlier controlled-trial code + still has bounded pilot grants; continuous production authority and complete + corrected physical/virtual feedback integration are NOT delivered by this RC. +2. The actual host/container/kernel and field acceptance of this release are not + replaced by CLI mocks. Real source validity and independent command-loss behavior + are not asserted by a measurement algebra or an override. +3. The replay limitation above must be accepted as a beta feature, or extended to + continuous rolling reoptimization and additional asset types before advertising + the original full automatic-selection requirement. +4. Native multi-plant commissioning, external backups, retained archive lifecycle, + model quality on field data and publication authentication remain operational + release criteria. The tests do not demonstrate actual cost savings. + +Do not re-run obsolete piecemeal installers after this release, bypass their source +checks, turn the previous grid schedule on, or turn 'productionReady' true manually. +The retained EV capacity is 161.44 kWh / 39 kW; SDL reserve is already outside EV. diff --git a/examples/V4UnifiedRelease/file_installer.php b/examples/V4UnifiedRelease/file_installer.php new file mode 100644 index 0000000..b19ffe3 --- /dev/null +++ b/examples/V4UnifiedRelease/file_installer.php @@ -0,0 +1,35 @@ +'already_installed']; + foreach($names as$name)if(($old[$name]===null?null:hash('sha256',$old[$name]))!==$m['files'][$name]['before'])throw new RuntimeException('Teilstand zuerst pruefen.'); + $backup=$stage.'/backups/'.gmdate('YmdTHis').'-'.bin2hex(random_bytes(4)); + if(!mkdir($backup.'/libs',0700,true))throw new RuntimeException('Sicherung fehlt.'); + foreach($names as$name)if($old[$name]!==null)v4UnifiedWrite($backup.'/'.$name,$old[$name],0600); + v4UnifiedWrite($backup.'/MANIFEST.json',json_encode($m,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0600); + $written=[]; + try {foreach($names as$name){$p=$target.'/'.$name;if((is_file($p)?file_get_contents($p):null)!==$old[$name])throw new RuntimeException('Parallele Dateiaenderung.');v4UnifiedWrite($p,$new[$name]);$written[]=$name;}} + catch(Throwable $e){foreach(array_reverse($written)as$name){$p=$target.'/'.$name;if(hash_file('sha256',$p)!==$m['files'][$name]['after'])continue;if($old[$name]===null)unlink($p);else v4UnifiedWrite($p,$old[$name]);}throw $e;} + return ['status'=>'installed','backup'=>$backup]; +} diff --git a/examples/V4UnifiedRelease/install.php b/examples/V4UnifiedRelease/install.php new file mode 100644 index 0000000..6062f0c --- /dev/null +++ b/examples/V4UnifiedRelease/install.php @@ -0,0 +1,33 @@ +'unified_native_data_only','startedAt'=>gmdate('c'),'newActuatorPermission'=>false,'cursorReset'=>false,'outboxRewritten'=>false]; +try { + $package=json_decode(file_get_contents(__DIR__.'/PACKAGE_HASHES.json'),true,64,JSON_THROW_ON_ERROR); + foreach($package as$name=>$hash)if(str_contains($name,'..')||str_starts_with($name,'/')||is_link(__DIR__.'/'.$name)||hash_file('sha256',__DIR__.'/'.$name)!==$hash)throw new RuntimeException('Installationspaket wurde geaendert.'); + if(!IPS_InstanceExists($manager)||IPS_GetInstance($manager)['ModuleInfo']['ModuleID']!=='{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}')throw new RuntimeException('Erwarteter Manager fehlt.'); + if(IPS_HasChanges($manager))throw new RuntimeException('Offene Manageraenderungen zuerst speichern oder verwerfen.'); + if(IPS_GetProperty($manager,'NetzfahrplanAktiv')!==false||IPS_GetProperty($manager,'NetzfahrplanV4MessdatenAktiv')!==true)throw new RuntimeException('Erwartet: Datenaufnahme EIN, alter Netzfahrplan AUS.'); + $controls=[];foreach(IPS_GetModuleList()as$mid)if((IPS_GetModule($mid)['Prefix']??'')==='MC')$controls=array_merge($controls,IPS_GetInstanceListByModuleID($mid)); + if(count($controls)!==1||!function_exists('MC_ReloadModule'))throw new RuntimeException('Module Control nicht eindeutig.'); + $lock=IPS_SemaphoreEnter('ENELIX.V4.UnifiedRelease',3000);if(!$lock)throw new RuntimeException('Installation bereits aktiv.'); + $before=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $report['files']=v4UnifiedFiles(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); + // No staged application classes are loaded into this script context. + if(MC_ReloadModule($controls[0],'Enelix-EMS')===false)throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + $after=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + foreach($before as$key=>$value)if(!array_key_exists($key,$after)||$after[$key]!==$value)throw new RuntimeException('Bestehender Parameter unerwartet geaendert: '.$key); + $report['status']='native_data_candidate_installed';$report['parametersUnchanged']=true; + echo "OK: Einheitliche Messdatenkennung und dauerhafte Versandbestaetigung installiert. Archivierung vorbereitet, standardmaessig AUS.\n"; + echo "Der bestehende Datentimer setzt den Versand fort. Sendezeiger und Originalbestand wurden nicht umgeschrieben.\n"; + echo "Keine V4-Stellfreigabe und kein Wechsel der Prognosequelle.\n"; +} catch(Throwable $e) { + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,250); + echo 'FEHLER: '.$report['reason']."\nKeine neue Stellfreigabe.\n"; +} finally { + $report['finishedAt']=gmdate('c');v4UnifiedWrite(__DIR__.'/INSTALL_RESULT.json',json_encode($report,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0644); + if($lock)IPS_SemaphoreLeave('ENELIX.V4.UnifiedRelease'); + echo 'UNIFIED MANAGER REPORT: '.__DIR__."/INSTALL_RESULT.json\n"; +} diff --git a/libs/ManagerNetzfahrplanV4DatenTrait.php b/libs/ManagerNetzfahrplanV4DatenTrait.php index 16001c6..a718525 100644 --- a/libs/ManagerNetzfahrplanV4DatenTrait.php +++ b/libs/ManagerNetzfahrplanV4DatenTrait.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Belevo\EnelixEMS; require_once __DIR__ . '/NetzfahrplanV4Messaufnahme.php'; +require_once __DIR__ . '/NetzfahrplanV4Datenarchiv.php'; /** Application measurement acquisition + acknowledged durable delivery. No device writes. */ trait ManagerNetzfahrplanV4DatenTrait @@ -11,9 +12,13 @@ trait ManagerNetzfahrplanV4DatenTrait private function registriereNetzfahrplanV4Daten(): void { $this->RegisterPropertyBoolean('NetzfahrplanV4MessdatenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4ArchivAktiv', false); $this->RegisterPropertyString('NetzfahrplanV4Messkonfiguration', '{}'); $this->RegisterPropertyString('NetzfahrplanV4Datensatz', ''); $this->RegisterAttributeString('NetzfahrplanV4DatenCursor', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4LetzteDatenquittung', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4ArchivTag', ''); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersion', 0); $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersuch', 0); $this->RegisterAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); $this->RegisterVariableString('NetzfahrplanV4Datenstatus', 'V4 Messdaten und Modellversorgung', '', 199); @@ -27,6 +32,12 @@ trait ManagerNetzfahrplanV4DatenTrait if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; try { $this->v4DatenKonfiguration(); + if ($this->ReadAttributeInteger('NetzfahrplanV4DatenVersion')<1) { + // One retry after the data-only upgrade, never a cursor reset or a device write. + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersion', 1); + } $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 30000); } catch (\Throwable $e) { $this->v4DatenStatus(['status'=>'configuration_error','errorType'=>get_class($e)]); @@ -60,6 +71,8 @@ trait ManagerNetzfahrplanV4DatenTrait private function v4DatenStatus(array $data): void { + $receipt=json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteDatenquittung'),true,32,JSON_THROW_ON_ERROR); + $data['lastAcknowledgement']=$receipt; $this->SetValue('NetzfahrplanV4Datenstatus',json_encode($data+['checkedAt'=>gmdate('c'), 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'controlEnabled'=>false],JSON_THROW_ON_ERROR)); } @@ -78,11 +91,25 @@ trait ManagerNetzfahrplanV4DatenTrait return ['value'=>GetValue($id),'updated'=>(int)$v['VariableUpdated'],'changed'=>(int)$v['VariableChanged'], 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; }; - $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); $dir=$this->v4DatenVerzeichnis(); - NetzfahrplanV4Messaufnahme::append($dir,$r); - $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), - 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + $archiveResult=null; + if ($this->ReadPropertyBoolean('NetzfahrplanV4ArchivAktiv') && $this->ReadAttributeString('NetzfahrplanV4ArchivTag')!==gmdate('Y-m-d')) { + try { + $archiveCursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $archiveResult=NetzfahrplanV4Datenarchiv::run($dir,$archiveCursor,time()); + if ($archiveResult['archived']<2 && ($archiveResult['status']??null)!=='writer_busy')$this->WriteAttributeString('NetzfahrplanV4ArchivTag',gmdate('Y-m-d')); + } catch (\Throwable $e) { $archiveResult=['status'=>'archive_error','errorType'=>get_class($e),'originalsRetained'=>true]; } + } + try { + $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); + NetzfahrplanV4Messaufnahme::append($dir,$r); + $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), + 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + } catch (\Throwable $e) { + // Storage/capture failure must not prevent retrying the already durable outbox. + $result=['status'=>'capture_error','errorType'=>get_class($e),'dataRetained'=>true]; + } + if ($archiveResult!==null)$result['archive']=$archiveResult; $last=$this->ReadAttributeInteger('NetzfahrplanV4DatenVersuch'); $failures=min(5,$this->ReadAttributeInteger('NetzfahrplanV4DatenFehlerzahl')); $retry=min(900,60*(2**$failures)); @@ -90,10 +117,16 @@ trait ManagerNetzfahrplanV4DatenTrait $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch',time()); try { $result['delivery']=$this->v4DatenUebertragen($dir); + if (($result['delivery']['status']??null)==='acknowledged') { + $this->WriteAttributeString('NetzfahrplanV4LetzteDatenquittung',json_encode($result['delivery']+['receivedAt'=>gmdate('c')],JSON_THROW_ON_ERROR)); + } $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',0); } catch (\Throwable $e) { $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',$failures+1); - $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e),'dataRetained'=>true]; + $httpCode=(int)$e->getCode(); + $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e), + 'httpStatus'=>$httpCode>=100&&$httpCode<=599?$httpCode:null,'dataRetained'=>true, + 'nextAttemptAt'=>gmdate('c',time()+min(900,60*(2**min(5,$failures+1))))]; } } else $result['delivery']=['status'=>'scheduled','nextAttemptAt'=>gmdate('c',$last+$retry)]; $this->v4DatenStatus($result); @@ -147,13 +180,14 @@ trait ManagerNetzfahrplanV4DatenTrait CURLOPT_HTTPHEADER=>['Content-Type: application/json','Accept: application/json','Authorization: Bearer '.$token], CURLOPT_POSTFIELDS=>$payload]); $answer=curl_exec($h);$code=(int)curl_getinfo($h,CURLINFO_HTTP_CODE); - if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged'); + if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged',$code); $ack=json_decode($answer,true,32,JSON_THROW_ON_ERROR); if (!in_array($ack['status']??null,['stored','duplicate'],true) ||($ack['datasetId']??null)!==$this->ReadPropertyString('NetzfahrplanV4Datensatz') ||($ack['acceptedThrough']??null)!==$lastTime) throw new \RuntimeException('Measurement acknowledgement does not match batch'); $this->WriteAttributeString('NetzfahrplanV4DatenCursor',json_encode($candidate,JSON_THROW_ON_ERROR)); - return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime]; + return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime, + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz')]; } finally { curl_close($h); } } } diff --git a/libs/NetzfahrplanV4Datenarchiv.php b/libs/NetzfahrplanV4Datenarchiv.php new file mode 100644 index 0000000..3211530 --- /dev/null +++ b/libs/NetzfahrplanV4Datenarchiv.php @@ -0,0 +1,78 @@ +0]; + if (!is_string($day)||!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day) + ||!is_int($cursor['offset']??null)||$cursor['offset']<0 + ||is_link($directory)||realpath($directory)!==$directory) { + throw new \RuntimeException('Invalid archive cursor/directory'); + } + $cutoff='raw-'.gmdate('Ymd',$now-7*86400).'.jsonl'; + $archive=$directory.'/archive'; + if (is_link($archive)) throw new \RuntimeException('Archive symlink refused'); + if (!is_dir($archive)&&!mkdir($archive,0700)) throw new \RuntimeException('Archive directory unavailable'); + if ((fileperms($archive)&0007)!==0) throw new \RuntimeException('Archive must remain private'); + $lockPath=$directory.'/.writer.lock'; + if (is_link($lockPath)) throw new \RuntimeException('Writer lock symlink refused'); + $lock=fopen($lockPath,'c+b'); + if ($lock===false) throw new \RuntimeException('Archive lock unavailable'); + $done=[]; + try { + if (!flock($lock,LOCK_EX|LOCK_NB)) return ['archived'=>0,'status'=>'writer_busy']; + foreach (glob($directory.'/raw-*.jsonl')?:[] as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||$name>=$day||$name>=$cutoff) continue; + if (is_link($file)||realpath($file)!==$file||!is_file($file)) throw new \RuntimeException('Unexpected journal'); + $before=stat($file); + if (!$before||$before['size']>67108864) throw new \RuntimeException('Archive input bounds'); + $digest=hash_file('sha256',$file); + if (!is_string($digest)) throw new \RuntimeException('Cannot hash journal'); + $target=$archive.'/'.$name.'.'.$digest.'.gz'; + if (is_link($target)) throw new \RuntimeException('Archive target symlink'); + if (!is_file($target)) { + $tmp=tempnam($archive,'.pending-'); + if ($tmp===false) throw new \RuntimeException('Cannot create archive'); + try { + chmod($tmp,0600);$in=fopen($file,'rb');$gz=gzopen($tmp,'wb6'); + if ($in===false||$gz===false) {if(is_resource($in))fclose($in);if(is_resource($gz))gzclose($gz);throw new \RuntimeException('Cannot open archive streams');} + try { + while (!feof($in)) { + $block=fread($in,65536); + if ($block===false) throw new \RuntimeException('Archive read failed'); + $offset=0; + while($offset$before['size'])throw new \RuntimeException('Oversized decompressed archive');hash_update($hash,$block);}} + finally {gzclose($gz);} + clearstatcache(true,$file);$after=stat($file); + if ($bytes!==$before['size']||hash_final($hash)!==$digest||!$after + ||$after['size']!==$before['size']||$after['ino']!==$before['ino']||$after['mtime']!==$before['mtime'] + ||hash_file('sha256',$file)!==$digest) throw new \RuntimeException('Archive mismatch; original retained'); + $directoryHandle=fopen($archive,'r'); + if ($directoryHandle===false) throw new \RuntimeException('Cannot sync archive directory'); + try {if(!fsync($directoryHandle))throw new \RuntimeException('Archive directory sync failed');} finally {fclose($directoryHandle);} + if (!unlink($file)) throw new \RuntimeException('Verified original could not be retired'); + $done[]=$name; + if (count($done)>=2) break; + } + } finally {flock($lock,LOCK_UN);fclose($lock);} + return ['archived'=>count($done),'compressedOriginalsRetained'=>true,'unacknowledgedDaysTouched'=>false]; + } +} diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile index ceff456..33e5701 100644 --- a/services/netplan-v4/Dockerfile +++ b/services/netplan-v4/Dockerfile @@ -11,6 +11,7 @@ COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py app COPY acceptance ./acceptance COPY commissioning/deploy_application.py ./commissioning/deploy_application.py COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py +COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py # Host sources can be 0600/0700. COPY makes them root-owned. # Normalize only packaged application code; never change host secrets or sockets. RUN find /app -type d -exec chmod 0755 {} + \ diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json index 34ed6eb..9cbe60b 100644 --- a/services/netplan-v4/SOURCE_MANIFEST.json +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -1,7 +1,7 @@ { "sourceHashes": { ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", - "Dockerfile": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", + "Dockerfile": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33", "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", @@ -42,13 +42,15 @@ "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "commissioning/deploy_unified_release.py": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", - "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "gui/netplan-v4.js": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9", "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", @@ -57,16 +59,20 @@ "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/economic_replay.py": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302", "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", - "netplan_v4/measurement_pipeline.py": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", + "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "netplan_v4/mapping_identity.py": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d", + "netplan_v4/measurement_pipeline.py": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f", "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/retention.py": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3", "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", - "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", - "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "netplan_v4/service.py": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f", + "netplan_v4/store.py": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2", "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", @@ -79,22 +85,22 @@ "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_economic_replay.py": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3", "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_mapping_recovery.py": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e", "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", - "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", - "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", - "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", - "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", - "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", - "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", - "commissioning/history-timing-source/dataset.json": "671216e1cc04148f4f553103077bcb4500bf0120cb74780703bfcf26cd1121d3", - "commissioning/history-timing-source/RELEASE.json": "14cb690a357c40f698c2e9b4190933318396b9c08e15cc4ad9f9f4439d6054ca" + "tests/test_release_preflight.py": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68", + "tests/test_retention.py": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51", + "tests/test_unified_release.py": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" }, - "packagedAt": "2026-10-02T21:00:35.231591+00:00", - "runtimeChanged": false + "scope": "application_source_only", + "productionReady": false } diff --git a/services/netplan-v4/commissioning/deploy_unified_release.py b/services/netplan-v4/commissioning/deploy_unified_release.py new file mode 100644 index 0000000..edf2f93 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_unified_release.py @@ -0,0 +1,124 @@ +"""Install the reviewed V4 release candidate as one transaction-like deployment. + +Tests run in an isolated candidate image BEFORE replacing any running service. +No device control permission, forecast-source switch, telemetry rewrite or cursor +jump. Exactly proven numeric representations are registered for backlog recovery. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse,hashlib,json,os,subprocess,sys,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PACKAGE=ROOT/'commissioning/unified-release' +GUI=ROOT.parent/'license/public/netplan-v4.js' + + +def digest(p): + if p.is_symlink():raise ValueError('Symlink refused') + return hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None + + +def verify(): + m=json.loads((PACKAGE/'RELEASE.json').read_text()) + if m.get('scope')!='unified_v4_candidate' or not m.get('files'):raise ValueError('Unexpected release') + if digest(Path(__file__).resolve())!=m['installerSha256']:raise ValueError('Installer changed') + for n,v in m['files'].items(): + if Path(n).is_absolute() or '..' in Path(n).parts:raise ValueError('Unsafe release path') + if digest(PACKAGE/'context'/n)!=v['after']:raise ValueError('Candidate drift: '+n) + if digest(ROOT/n) not in (v['before'],v['after']):raise ValueError('Concurrent runtime source edit: '+n) + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent GUI edit') + if digest(PACKAGE/'mapping-proof.json')!=m['proofSha256']:raise ValueError('Compatibility proof changed') + if digest(ROOT/'commissioning/deploy_application.py')!=m['backupHelperSha256']:raise ValueError('Backup helper changed') + return m + + +BOOTSTRAP=r'''import json,os,sys,urllib.request +x=json.load(sys.stdin);base='http://127.0.0.1:9100' +health=json.load(urllib.request.urlopen(base+'/health',timeout=5)) +assert health.get('applicationRelease')=='unified-rc1' and health.get('liveEnabled') is False +headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +url=base+'/internal/v2/prognosis/'+x['plant']+'/planner' +def get():return json.load(urllib.request.urlopen(urllib.request.Request(url,headers=headers),timeout=20)) +before=get() +receipt={'status':'not_registered_requires_explicit_approval'} +if x['approveMapping']: + request=urllib.request.Request(url+'/datasets/'+x['dataset']+'/mapping-compatibility',data=json.dumps(x['proof']).encode(),headers=headers,method='PUT') + receipt=json.load(urllib.request.urlopen(request,timeout=15)) +after=get();assert before['settings']==after['settings'] and after['liveEnabled'] is False +print(json.dumps({'health':health,'mappingRecovery':receipt,'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'],'economicReplayConnected':after['economicComparison']['connected'],'dataSets':[{'datasetId':d['datasetId'],'records':d['records'],'lastCapture':d['lastCapture'],'lastReceived':d.get('lastReceived'),'status':d['status']} for d in after['dataPipeline']['datasets']]})) +''' + + +def run(plant,apply=False,approve_mapping=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Wrong prepared installation') + if not apply: + print('UNIFIED RELEASE SOURCE CHECK PASSED; no installation.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker socket permissions') + from deploy_application import atomic,backup_database + folder=ROOT/'unified-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ');folder.mkdir(parents=True,mode=0o700) + report={'scope':'unified_v4_candidate','releaseId':m['releaseId'],'startedAt':datetime.now(timezone.utc).isoformat(),'liveEnabled':False,'productionCommissioned':False,'steps':[]} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')];env=dict(os.environ) + def cmd(args,timeout=180,capture=False,data=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data,stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + changed={};old_gui=None;new_gui=None;replaced=False;old_image=None + try: + # Read only this non-secret allowlist; never print or persist the complete environment. + env['NETPLAN_V4_PLANTS']=plant + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + allowed=cmd(['docker','exec',ids[0],'python','-c',"import os;print(os.getenv('NETPLAN_V4_PLANTS',''))"],capture=True).stdout.strip() + if plant not in allowed.split(','):raise ValueError('Installation not in running allowlist') + env['NETPLAN_V4_PLANTS']=allowed + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip();report['previousImage']=old_image + tag='enelix-netplan-v4:rc-'+m['releaseId'][:16] + cmd(['docker','build','-t',tag,str(PACKAGE/'context')],timeout=900) + candidate=cmd(['docker','image','inspect','--format','{{.Id}}',tag],capture=True).stdout.strip() + cmd(['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--tmpfs','/tmp:rw,nosuid,nodev,noexec,size=256m','--entrypoint','python',candidate,'/app/run_tests.py'],timeout=600) + cmd(['node','--test',str(PACKAGE/'context/tests/portal.test.mjs')]);cmd(['node','--check',str(PACKAGE/'context/gui/netplan-v4.js')]) + report['steps'].append('isolated_candidate_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite');report['steps'].append('consistent_backup') + for name,v in m['files'].items(): + target=ROOT/name;after=(PACKAGE/'context'/name).read_bytes();before=target.read_bytes() if target.is_file() else None + if before==after:continue + if digest(target)!=v['before']:raise ValueError('Source changed during tests') + backup=folder/'source-before'/name;backup.parent.mkdir(parents=True,exist_ok=True) + if before is not None:backup.write_bytes(before) + target.parent.mkdir(parents=True,exist_ok=True);atomic(target,after);changed[name]=(before,after) + override=folder/'candidate.yaml';override.write_text('services:\n netplan-v4:\n image: '+candidate+'\n') + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + replaced=True;cmd(compose+['-f',str(override),'up','-d','--no-deps','--no-build','--wait','netplan-v4'],timeout=180) + data=json.dumps({'plant':plant,'dataset':'lihrenmoos-physical-v1','proof':json.loads((PACKAGE/'mapping-proof.json').read_text()),'approveMapping':approve_mapping}) + result=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=data) + report['application']=json.loads(result.stdout) + old_gui=GUI.read_bytes();new_gui=(PACKAGE/'context/gui/netplan-v4.js').read_bytes() + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent portal UI edit') + (folder/'gui-before.js').write_bytes(old_gui);atomic(GUI,new_gui) + report['steps'].append('audited_backlog_compatibility_and_application_verified');report['candidateImage']=candidate + report['status']='unified_candidate_installed_no_actuation' + except Exception as exc: + report['status']='needs_review';report['errorType']=type(exc).__name__ + if old_gui is not None and GUI.read_bytes()==new_gui:atomic(GUI,old_gui) + for name,(before,after) in reversed(list(changed.items())): + target=ROOT/name + if target.read_bytes()!=after:continue + if before is None:target.unlink() + else:atomic(target,before) + if replaced and old_image: + try: + cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + report['rollback']='previous_image_restored_additive_audit_tables_retained' + except Exception:report['rollback']='manual_review_required' + raise + finally: + report['finishedAt']=datetime.now(timezone.utc).isoformat();p=folder/'REPORT.json';p.write_text(json.dumps(report,indent=2)+'\n') + uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid + for x in (folder.parent,folder,p):os.chown(x,uid,gid) + p.chmod(0o640);print('UNIFIED RELEASE REPORT:',p) + print('V4 candidate and planner UI installed. Mapping equivalence is activated only with explicit approval. No data/cursor rewrite and no device-control grant.') + + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True);p.add_argument('--apply',action='store_true');p.add_argument('--approve-numeric-mapping-compatibility',action='store_true',help='Explicitly approve only the audited 100 versus 100.0 mapping equivalence; never rewrite original records');a=p.parse_args() + try:run(a.plant,a.apply,a.approve_numeric_mapping_compatibility) + except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. Review UNIFIED RELEASE REPORT; no actuator permission enabled.') diff --git a/services/netplan-v4/commissioning/unified-release/.gitignore b/services/netplan-v4/commissioning/unified-release/.gitignore new file mode 100644 index 0000000..2b3f20c --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/.gitignore @@ -0,0 +1 @@ +context/ diff --git a/services/netplan-v4/commissioning/unified-release/RELEASE.json b/services/netplan-v4/commissioning/unified-release/RELEASE.json new file mode 100644 index 0000000..611d7e0 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/RELEASE.json @@ -0,0 +1,414 @@ +{ + "scope": "unified_v4_candidate", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "files": { + ".dockerignore": { + "before": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "after": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790" + }, + "Dockerfile": { + "before": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", + "after": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33" + }, + "acceptance/Dockerfile.php": { + "before": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "after": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062" + }, + "acceptance/check_forecast.py": { + "before": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "after": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5" + }, + "acceptance/forecast-src/SOURCE_MANIFEST.json": { + "before": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "after": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5" + }, + "acceptance/forecast-src/main.py": { + "before": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "after": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c" + }, + "acceptance/forecast-src/methods/__init__.py": { + "before": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "after": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "acceptance/forecast-src/methods/battery_optimizer.py": { + "before": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "after": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6" + }, + "acceptance/forecast-src/methods/common.py": { + "before": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "after": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8" + }, + "acceptance/forecast-src/methods/var_1.py": { + "before": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "after": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4" + }, + "acceptance/forecast-src/methods/var_10.py": { + "before": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "after": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7" + }, + "acceptance/forecast-src/methods/var_11.py": { + "before": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "after": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc" + }, + "acceptance/forecast-src/methods/var_13.py": { + "before": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "after": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9" + }, + "acceptance/forecast-src/methods/var_2.py": { + "before": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "after": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a" + }, + "acceptance/forecast-src/methods/var_21.py": { + "before": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "after": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15" + }, + "acceptance/forecast-src/methods/var_22.py": { + "before": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "after": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6" + }, + "acceptance/forecast-src/methods/var_23.py": { + "before": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "after": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678" + }, + "acceptance/forecast-src/methods/var_3.py": { + "before": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "after": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984" + }, + "acceptance/forecast-src/model_isolation.py": { + "before": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "after": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5" + }, + "acceptance/forecast-src/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "acceptance/forecast-src/requirements.txt": { + "before": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "after": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083" + }, + "acceptance/forecast-src/shared_utils.py": { + "before": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "after": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57" + }, + "acceptance/forecast-src/soc_diagnostics.py": { + "before": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "after": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a" + }, + "acceptance/forecast-src/telemetry_quality.py": { + "before": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "after": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810" + }, + "acceptance/forecast-src/tests/test_battery_optimizer.py": { + "before": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "after": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8" + }, + "acceptance/forecast-src/tests/test_load_forecast.py": { + "before": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "after": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c" + }, + "acceptance/forecast-src/tests/test_model_isolation.py": { + "before": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "after": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30" + }, + "acceptance/forecast-src/tests/test_telemetry_integrity.py": { + "before": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "after": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" + }, + "acceptance/php-src/SOURCE_MANIFEST.json": { + "before": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "after": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50" + }, + "acceptance/php-src/check.php": { + "before": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "after": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473" + }, + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": { + "before": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "after": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7" + }, + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": { + "before": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "after": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9" + }, + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": { + "before": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "after": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a" + }, + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": { + "before": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "after": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" + }, + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": { + "before": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "after": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c" + }, + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": { + "before": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "after": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6" + }, + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": { + "before": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "after": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2" + }, + "acceptance/read_native_forecasts.py": { + "before": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "after": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f" + }, + "approved_previous_assets.json": { + "before": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "after": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0" + }, + "commissioning/APPLICATION_STATUS.md": { + "before": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "after": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc" + }, + "commissioning/application-source/server-dataset.json": { + "before": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "after": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d" + }, + "commissioning/deploy_application.py": { + "before": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "after": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e" + }, + "commissioning/deploy_history_timing.py": { + "before": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "after": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb" + }, + "commissioning/deploy_unified_release.py": { + "before": null, + "after": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e" + }, + "compose.portal-bridge.yaml": { + "before": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "after": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197" + }, + "compose.yaml": { + "before": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "after": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607" + }, + "deploy_integrated_shadow.py": { + "before": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "after": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898" + }, + "forecast_acceptance.py": { + "before": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "after": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9" + }, + "gui/netplan-v4.css": { + "before": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "after": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd" + }, + "gui/netplan-v4.html": { + "before": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "after": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755" + }, + "gui/netplan-v4.js": { + "before": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "after": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9" + }, + "install_hooks.py": { + "before": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "after": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3" + }, + "integrations/NetzfahrplanV4.php": { + "before": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "after": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3" + }, + "integrations/netplan-v4-bridge.mjs": { + "before": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "after": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7" + }, + "integrations/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "netplan_v4/__init__.py": { + "before": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "after": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48" + }, + "netplan_v4/battery_model.py": { + "before": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "after": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332" + }, + "netplan_v4/controlled_trial.py": { + "before": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "after": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4" + }, + "netplan_v4/domain.py": { + "before": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "after": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b" + }, + "netplan_v4/economic_replay.py": { + "before": null, + "after": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302" + }, + "netplan_v4/forecast_quality.py": { + "before": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "after": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840" + }, + "netplan_v4/history_timing.py": { + "before": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "after": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f" + }, + "netplan_v4/mapping_identity.py": { + "before": null, + "after": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d" + }, + "netplan_v4/measurement_pipeline.py": { + "before": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", + "after": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f" + }, + "netplan_v4/meter_runtime.py": { + "before": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "after": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547" + }, + "netplan_v4/metering.py": { + "before": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "after": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987" + }, + "netplan_v4/optimizer.py": { + "before": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "after": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063" + }, + "netplan_v4/peak_policy.py": { + "before": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "after": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576" + }, + "netplan_v4/receiver_contract.py": { + "before": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "after": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c" + }, + "netplan_v4/retention.py": { + "before": null, + "after": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3" + }, + "netplan_v4/selection.py": { + "before": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "after": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad" + }, + "netplan_v4/service.py": { + "before": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "after": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f" + }, + "netplan_v4/store.py": { + "before": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "after": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2" + }, + "release_preflight.py": { + "before": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "after": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5" + }, + "requirements.txt": { + "before": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "after": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464" + }, + "run_tests.py": { + "before": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "after": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4" + }, + "runtime_preflight.py": { + "before": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "after": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1" + }, + "tests/manager_protocol.php": { + "before": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "after": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93" + }, + "tests/portal.test.mjs": { + "before": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "after": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a" + }, + "tests/test_application_deployment.py": { + "before": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "after": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5" + }, + "tests/test_battery_recovery.py": { + "before": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "after": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3" + }, + "tests/test_container_access.py": { + "before": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "after": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9" + }, + "tests/test_controlled_trial.py": { + "before": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "after": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64" + }, + "tests/test_controlled_trial_pipeline.py": { + "before": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "after": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d" + }, + "tests/test_delivery.py": { + "before": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "after": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09" + }, + "tests/test_economic_replay.py": { + "before": null, + "after": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3" + }, + "tests/test_forecast_acceptance.py": { + "before": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "after": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76" + }, + "tests/test_forecast_quality.py": { + "before": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "after": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82" + }, + "tests/test_history_timing.py": { + "before": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "after": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2" + }, + "tests/test_history_timing_deployment.py": { + "before": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", + "after": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e" + }, + "tests/test_integrated_deployment.py": { + "before": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "after": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883" + }, + "tests/test_mapping_recovery.py": { + "before": null, + "after": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e" + }, + "tests/test_measurement_pipeline.py": { + "before": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "after": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4" + }, + "tests/test_metering.py": { + "before": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "after": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d" + }, + "tests/test_peak_release.py": { + "before": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "after": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d" + }, + "tests/test_receiver_contract.py": { + "before": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "after": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e" + }, + "tests/test_release_preflight.py": { + "before": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "after": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68" + }, + "tests/test_retention.py": { + "before": null, + "after": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51" + }, + "tests/test_unified_release.py": { + "before": null, + "after": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236" + }, + "tests/test_v4.py": { + "before": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", + "after": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + } + }, + "installerSha256": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", + "guiBefore": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "proofSha256": "11cb827ab65611bbb06bf8a181d0f44a16202ae957d82f56e0a18a916325128d", + "backupHelperSha256": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "autoArchiveDefault": false, + "mappingRegistrationRequiresExplicitApproval": true, + "productionReady": false, + "releaseId": "aeff5f094f6c8fbd7dc962d2d091d364a14c45e10fc7bd2f4847dd6dba329a22" +} diff --git a/services/netplan-v4/commissioning/unified-release/build_context.py b/services/netplan-v4/commissioning/unified-release/build_context.py new file mode 100644 index 0000000..ae11ab4 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/build_context.py @@ -0,0 +1,28 @@ +"""Reproduce the frozen candidate context from the versioned application sources. + +No Docker/runtime/secret reads. Existing output is verified, never silently replaced. +""" +from pathlib import Path +import hashlib,json,shutil + +PACKAGE=Path(__file__).resolve().parent +ROOT=PACKAGE.parents[1] + +def build(): + manifest=json.loads((PACKAGE/'RELEASE.json').read_text()) + context=PACKAGE/'context' + for name,v in manifest['files'].items(): + if Path(name).is_absolute() or '..' in Path(name).parts:raise ValueError('Unsafe source path') + source=ROOT/name;target=context/name + if source.is_symlink() or not source.is_file() or hashlib.sha256(source.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Checked-out source does not match release: '+name) + if target.is_symlink():raise ValueError('Target symlink refused') + if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Existing context differs: '+name) + for name in manifest['files']: + target=context/name + if target.exists():continue + target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(ROOT/name,target);target.chmod(0o644) + print('Frozen build context verified:',len(manifest['files']),'source files; no installation.') + +if __name__=='__main__':build() diff --git a/services/netplan-v4/commissioning/unified-release/mapping-proof.json b/services/netplan-v4/commissioning/unified-release/mapping-proof.json new file mode 100644 index 0000000..382cdca --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/mapping-proof.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "canonicalJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100.0},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}", + "legacyJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}" +} diff --git a/services/netplan-v4/gui/netplan-v4.js b/services/netplan-v4/gui/netplan-v4.js index a67ac55..7e76572 100644 --- a/services/netplan-v4/gui/netplan-v4.js +++ b/services/netplan-v4/gui/netplan-v4.js @@ -78,8 +78,14 @@ async function render() { replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } }); refresh.addEventListener('click', render); panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.')); - panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik wartet weiterhin auf vergleichbare Kosten-Replays, nicht bloss auf gute R2-Werte.')); - for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik erstellt vergleichbare Kosten-Replays aus eingefrorenen Tagesplaenen und spaeter eingegangenen Messwerten. Ohne ausreichende Vergleichstage bleibt die bisherige Familie aktiv.')); + for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Letzter Serverempfang', time(d.lastReceived)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + if (state.economicComparison?.connected) { + table('Wirtschaftlicher Modellvergleich', [['Gespeicherte Tagesvergleiche', state.economicComparison.cohorts], ['Abgeschlossene Vergleiche', state.economicComparison.completedComparisons.length], ['Methode', 'Eingefrorener Tagesplan mit simulierter Netzregelung'], ['Grenze dieses Vergleichs', 'Eine netzladefaehige Batterie; keine vollstaendige Nachbildung laufender Neuoptimierungen'], ['Bewertung', 'Energiekosten + zusaetzlicher Peak + Verluste + gemeinsame Restenergiebewertung']]); + panel.append(element('p', 'Der SDL-Anteil der historischen Bilanz ist eine gekennzeichnete Schaetzung aus dem damaligen Auftrag. Die Vergleichskosten sind keine nachgewiesenen Einsparungen auf einer Stromrechnung.')); + for (const item of state.economicComparison.completedComparisons) table('Vergleich ' + item.day, Object.entries(item.results).map(([family, result]) => ['Familie ' + family, num(result.costChf) + ' CHF; Datenabdeckung ' + num(100 * result.coverage, 1) + ' %'])); + } + if (state.maintenance) table('Datenaufbewahrung', [['Archivierung', state.maintenance.status], ['Originale', 'Nach verifizierter Kompression weiterhin wiederherstellbar; externe Sicherung separat betreiben']]); panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.')); if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.')); for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]); diff --git a/services/netplan-v4/netplan_v4/economic_replay.py b/services/netplan-v4/netplan_v4/economic_replay.py new file mode 100644 index 0000000..239647f --- /dev/null +++ b/services/netplan-v4/netplan_v4/economic_replay.py @@ -0,0 +1,207 @@ +"""Causal daily economic family comparison using frozen plans and later physical data. + +This is a labelled *simulation*, not savings on an invoice. A cohort is captured +near local midnight, uses only prices/forecasts then available, and all families +share initial storage, physical outcomes, tariff and constraints. No device calls. +Supported v1: one grid-charge-enabled battery; other topologies remain explicit. +""" +from dataclasses import asdict, replace +from datetime import datetime, timedelta +from hashlib import sha256 +from math import sqrt +import json +from .domain import ZURICH, utc, month_key, quarter_start +from .selection import ReplayScore +from . import measurement_pipeline as m + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_economic_cohorts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, issued_at INTEGER NOT NULL, + ends_at INTEGER NOT NULL, policy_id TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_results( + plant TEXT NOT NULL, local_day TEXT NOT NULL, evaluated_at INTEGER NOT NULL, + policy_id TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_attempts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, checked_at INTEGER NOT NULL, + reason TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + ''') + + +def _policy(data, cfg): + batteries=[] + for b in data['batteries']: + v=asdict(b) + for k in ('soc_percent','measured_at','discharge_blocked'):v.pop(k) + batteries.append(v) + return sha256(m.canonical({'battery':batteries,'limits':asdict(data['limits']), + 'mapping':cfg['mappingSha256'],'formula':cfg['formula'],'dataset':cfg['datasetId'], + 'method':'frozen_day_ahead_grid_tracking_v1','external':'sdl_request_estimate','peakTariffs':data['peak_prices']}).encode()).hexdigest() + + +def capture(store, plant, now, assemble, optimizer): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile': return + local=utc(now).astimezone(ZURICH); day=local.date().isoformat() + # Freeze only at the beginning of a local day. Never reconstruct an old forecast from hindsight. + if local.hour!=0 or local.minute>=15:return + if store.con.execute('SELECT 1 FROM planner_economic_cohorts WHERE plant=? AND local_day=?',(plant,day)).fetchone():return + attempted=store.con.execute('SELECT checked_at FROM planner_economic_attempts WHERE plant=? AND local_day=?',(plant,day)).fetchone() + if attempted and now.timestamp()-attempted[0]<300:return + reason='awaiting_comparable_snapshot' + try: + cfg=m.configuration(store.con,plant,settings['measurementDataset']);plans={}; common=None;policy=None + end=datetime.combine(local.date()+timedelta(days=1),datetime.min.time(),tzinfo=ZURICH).astimezone(utc(now).tzinfo) + for family in store.registry.entries(): + data,_,quality=assemble(store,plant,family.key,now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + if len(data['batteries'])!=1 or not data['batteries'][0].grid_charging: + raise ValueError('unsupported_replay_topology') + if data['steps'][-1].end=rearm-1e-9:blocked=False + discharge=0. if blocked else min(b['max_discharge_w'],max(0.,(energy-low)*eta/dt*1000)) + target=p['gridTargetW'];cap=limits['import_w'];monthly=limits['manager_month_limits_w'].get(str(start.astimezone(ZURICH).month),limits['manager_month_limits_w'].get(start.astimezone(ZURICH).month)) + if monthly is not None:cap=monthly if cap is None else min(cap,monthly) + if cap is not None:target=min(target,cap) + if limits['export_w'] is not None:target=max(target,-limits['export_w']) + battery=min(charge,max(-discharge,target-residual));grid=residual+battery + energy+=battery/1000*dt*(eta if battery>=0 else 1/eta) + if energyhigh+1e-6:raise ValueError('Replay SOC invariant') + if energy<=low+1e-9:blocked=True + if cap is not None and grid>cap+1.:breaches+=1 + if limits['export_w'] is not None and grid < -limits['export_w']-1.:breaches+=1 + money+=(max(grid,0)*p['importPriceChfKwh']-max(-grid,0)*p['exportPriceChfKwh'])/1000*dt + wear+=abs(battery)/1000*dt*b['throughput_chf_kwh'] + q=quarter_start(start).isoformat();v=quarters.setdefault(q,[0.,0]);v[0]+=max(grid,0)/1000*dt;v[1]+=seconds + if end==quarter_start(start)+timedelta(minutes=15): + if v[1]!=900:raise ValueError('Incomplete simulated billing quarter') + month=month_key(start);peaks[month]=max(peaks[month],v[0]/.25);quarter_max[month]=max(quarter_max.get(month,0.),v[0]/.25) + additional=sum(max(0,v-ctx['peaks'][month])*ctx['peakTariffs'][month] for month,v in peaks.items()) + # Identical terminal valuation for all families, known at cohort creation; separate from cash. + last=plan['points'][-1];buy=max(0.,last['importPriceChfKwh']);sell=max(0.,min(buy,last['exportPriceChfKwh'])) + terminal=max(initial-energy,0)/eta*buy-max(energy-initial,0)*eta*sell + return {'cashCostChf':money+additional,'energyCostChf':money,'peakCostChf':additional,'throughputCostChf':wear, + 'terminalAdjustmentChf':terminal,'costChf':money+additional+wear+terminal, + 'initialEnergyKwh':initial,'finalEnergyKwh':energy,'coverage':min(coverage), + 'quarterMaximaKw':quarter_max,'initialPeaksKw':ctx['peaks'],'peakTariffs':ctx['peakTariffs'], + 'constraintBreaches':breaches,'terminalNormalized':True, + 'terminalMethod':'common_known_end_price_inventory_valuation_not_physical_restoration'} + + +def advance(store,plant,now): + timestamp=int(now.timestamp()) + rows=store.con.execute('SELECT c.* FROM planner_economic_cohorts c LEFT JOIN planner_economic_results r USING(plant,local_day) WHERE c.plant=? AND c.ends_at<=? AND c.ends_at>=? AND r.local_day IS NULL AND NOT EXISTS(SELECT 1 FROM planner_economic_attempts a WHERE a.plant=c.plant AND a.local_day=c.local_day AND a.checked_at>?) ORDER BY c.issued_at LIMIT 1',(plant,timestamp-120,timestamp-90*86400,timestamp-300)).fetchall() + for row in rows: + with store.con:store.con.execute('INSERT INTO planner_economic_attempts VALUES(?,?,?,?) ON CONFLICT(plant,local_day) DO UPDATE SET checked_at=excluded.checked_at,reason=excluded.reason',(plant,row['local_day'],timestamp,'evaluating_actuals')) + try: + cohort=json.loads(row['value']);cfg=m.configuration(store.con,plant,cohort['datasetId']);source=cfg.get('sourceDatasetId',cfg['datasetId']);start=m.epoch(cohort['start']);end=m.epoch(cohort['end']) + observed=store.con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',(plant,source,start-600,end+600,timestamp)) + windows=actuals([json.loads(r[0]) for r in observed],cfg);outcomes={w['start']:w for w in windows} + results={f:simulate(cohort,f,outcomes) for f in cohort['plans']} + value={'status':'evaluated','results':results,'start':cohort['start'],'end':cohort['end'], + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','method':cohort['method'], + 'actualCashSavings':False,'controlEnabled':False} + with store.con:store.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(plant,row['local_day'],timestamp,row['policy_id'],m.canonical(value))) + except (ValueError,KeyError,TypeError): + # Retain unscored cohort. Lack of actual data must never become a zero cost. + pass + + +def scores(store,plant,now): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile':return [] + since=int((utc(now)-timedelta(days=settings['autoLookbackDays'])).timestamp()) + recent=store.con.execute('SELECT policy_id,value FROM planner_economic_cohorts WHERE plant=? ORDER BY issued_at DESC LIMIT 1',(plant,)).fetchone() + if not recent or json.loads(recent[1])['datasetId']!=settings.get('measurementDataset'):return [] + rows=store.con.execute('SELECT c.issued_at,c.ends_at,r.evaluated_at,r.value FROM planner_economic_results r JOIN planner_economic_cohorts c USING(plant,local_day) WHERE r.plant=? AND r.policy_id=? AND c.issued_at>=? AND r.evaluated_at<=? ORDER BY c.issued_at',(plant,recent[0],since,int(now.timestamp()))).fetchall() + if not rows:return [] + keys=[f.key for f in store.registry.entries()];valid=[] + for r in rows: + v=json.loads(r['value']);out=v['results'] + if set(out)!=set(keys) or any(out[k]['constraintBreaches'] for k in keys):continue + valid.append((r,v)) + if not valid:return [] + result=[] + for k in keys: + # Monthly demand cost is paid ONCE for the maximum, not once per replay day. + total=sum(v['results'][k]['energyCostChf']+v['results'][k]['throughputCostChf']+v['results'][k]['terminalAdjustmentChf'] for _,v in valid) + bases={}; maxima={}; rates={} + for _,v in valid: + d=v['results'][k] + for month,peak in d['quarterMaximaKw'].items(): + bases.setdefault(month,d['initialPeaksKw'][month]) + maxima[month]=max(maxima.get(month,0.),peak);rates[month]=d['peakTariffs'][month] + total+=sum(max(0.,maxima[month]-bases[month])*rates[month] for month in maxima) + cover=min(v['results'][k]['coverage'] for _,v in valid) + first=utc(valid[0][1]['start']);end=utc(valid[-1][1]['end']);available=datetime.fromtimestamp(max(r['evaluated_at'] for r,_ in valid),utc(now).tzinfo) + result.append(ReplayScore(k,recent[0],first,end,first,available,total,cover,len(valid))) + return result + + +def status(store,plant): + rows=store.con.execute('SELECT local_day,value FROM planner_economic_results WHERE plant=? ORDER BY local_day DESC LIMIT 14',(plant,)) + evaluated=[{'day':r[0],**json.loads(r[1])} for r in rows] + count=store.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts WHERE plant=?',(plant,)).fetchone()[0] + return {'connected':True,'method':'frozen_day_ahead_grid_tracking_v1','cohorts':count,'completedComparisons':evaluated, + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','isBillingEvidence':False, + 'limitations':['one_grid_charging_battery','frozen_daily_plan_not_receding_horizon_field_replay']} diff --git a/services/netplan-v4/netplan_v4/mapping_identity.py b/services/netplan-v4/netplan_v4/mapping_identity.py new file mode 100644 index 0000000..255f5f1 --- /dev/null +++ b/services/netplan-v4/netplan_v4/mapping_identity.py @@ -0,0 +1,100 @@ +"""Audited numeric-representation compatibility, not relaxed device identity checks. + +Only the internal operator API can register equivalence. Raw journals are never +rewritten, and the original received fingerprint is retained with each receipt. +""" +from hashlib import sha256 +import json +import math +import re + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_mapping_compatibility( + plant TEXT NOT NULL, dataset TEXT NOT NULL, alias TEXT NOT NULL, + canonical TEXT NOT NULL, inventory TEXT NOT NULL, evidence TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset,alias)); + CREATE TABLE IF NOT EXISTS planner_observation_origins( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_mapping TEXT NOT NULL, inventory TEXT NOT NULL, + evidence_id TEXT, received_at INTEGER NOT NULL, + PRIMARY KEY(plant,dataset,captured_at,received_mapping)); + ''') + + +def _decode(text): + if not isinstance(text, str) or len(text.encode()) > 131072: + raise ValueError('Bounded configuration evidence required') + def pairs(items): + d = {} + for k, v in items: + if k in d: raise ValueError('Duplicate configuration key') + d[k] = v + return d + def constant(_): raise ValueError('Nonfinite configuration') + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + + +def _same(a, b): + if type(a) is not type(b): return False + if isinstance(a, dict): + return list(a) == list(b) and all(_same(a[k], b[k]) for k in a) + if isinstance(a, list): return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b)) + return a == b + + +def validate_evidence(plant, config, payload): + if not isinstance(payload, dict) or set(payload) != {'version', 'canonicalJson', 'legacyJson'} or type(payload['version']) is not int or payload['version'] != 1: + raise ValueError('Explicit versioned representation evidence required') + ca, le = payload['canonicalJson'], payload['legacyJson'] + a, b = _decode(ca), _decode(le) + canonical_hash, alias = sha256(ca.encode()).hexdigest(), sha256(le.encode()).hexdigest() + if canonical_hash != config['mappingSha256'] or alias == canonical_hash: + raise ValueError('Evidence does not match configured mapping') + for c in (a, b): + if not isinstance(c, dict) or c.get('installationId') != plant or c.get('reportedInventorySha256') != config['inventorySha256']: + raise ValueError('Evidence belongs to another installation or inventory') + x = a.get('accounting', {}).get('splitToleranceW') + y = b.get('accounting', {}).get('splitToleranceW') + if type(x) is not float or type(y) is not int or not math.isfinite(x) or x != y or not 0 <= x <= 500: + raise ValueError('Only demonstrated float/integer tolerance representation is compatible') + b['accounting']['splitToleranceW'] = float(y) + if not _same(a, b): + raise ValueError('Other configuration differences are not representation compatibility') + # Proof includes the exact hashed JSON strings; no arbitrary labels as evidence. + evidence = json.dumps(payload, sort_keys=True, separators=(',', ':'), allow_nan=False) + return {'alias': alias, 'canonical': canonical_hash, 'inventory': config['inventorySha256'], + 'evidenceId': sha256(evidence.encode()).hexdigest(), 'evidence': evidence} + + +def register(con, plant, config, payload, now): + if config.get('sourceDatasetId'): raise ValueError('Register compatibility on original dataset only') + v = validate_evidence(plant, config, payload) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT canonical,inventory,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND alias=?', + (plant, config['datasetId'], v['alias'])).fetchone() + expected = (v['canonical'], v['inventory'], v['evidence']) + if old and tuple(old) != expected: raise ValueError('Immutable mapping compatibility conflict') + con.execute('INSERT OR IGNORE INTO planner_mapping_compatibility VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], v['alias'], *expected, now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status': 'registered', 'datasetId': config['datasetId'], 'evidenceId': v['evidenceId'], + 'compatibleMapping': v['alias'], 'canonicalMapping': v['canonical'], 'controlEnabled': False} + + +def approved(con, plant, config): + rows = con.execute('SELECT alias,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND canonical=? AND inventory=?', + (plant, config['datasetId'], config['mappingSha256'], config['inventorySha256'])) + return {r['alias']: sha256(r['evidence'].encode()).hexdigest() for r in rows} + + +def save_origin(con, plant, config, record, captured_at, received_at, aliases): + source = record['mappingSha256'] + if source != config['mappingSha256'] and source not in aliases: + raise ValueError('Unknown mapping; no receipt written') + con.execute('INSERT OR IGNORE INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], captured_at, source, config['inventorySha256'], aliases.get(source), received_at)) diff --git a/services/netplan-v4/netplan_v4/measurement_pipeline.py b/services/netplan-v4/netplan_v4/measurement_pipeline.py index 0579467..61ca5a9 100644 --- a/services/netplan-v4/netplan_v4/measurement_pipeline.py +++ b/services/netplan-v4/netplan_v4/measurement_pipeline.py @@ -14,6 +14,7 @@ from statistics import median from zoneinfo import ZoneInfo import json from .history_timing import validate_policy, endpoint_bridges +from . import mapping_identity UTC = timezone.utc LOCAL = ZoneInfo('Europe/Zurich') @@ -42,6 +43,7 @@ def numeric(value, bound=1e12): def schema(con): + mapping_identity.schema(con) con.executescript(''' CREATE TABLE IF NOT EXISTS planner_data_sets( plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, @@ -162,10 +164,11 @@ def configuration(con, plant, dataset): return json.loads(row[0]) -def project(record, c, plant, now): +def project(record, c, plant, now, approved_mappings=()): if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: raise ValueError('Wrong capture identity') - if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + incoming = record.get('mappingSha256') + if not isinstance(incoming, str) or (incoming != c['mappingSha256'] and incoming not in approved_mappings) or record.get('reportedInventorySha256') != c['inventorySha256']: raise ValueError('Wrong capture mapping or inventory') t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) if start > t or t > now+30 or t < now-90*86400: @@ -196,13 +199,14 @@ def ingest_batch(con, plant, payload, now): records = payload['records'] if not isinstance(records,list) or not 1 <= len(records) <= 120: raise ValueError('Batch requires 1..120 captures') - rows = [project(r,c,plant,now) for r in records] + aliases = mapping_identity.approved(con, plant, c) + rows = [project(r,c,plant,now,aliases) for r in records] if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): raise ValueError('Batch must be in increasing capture order') stored = duplicate = 0 con.execute('BEGIN IMMEDIATE') try: - for r in rows: + for original, r in zip(records, rows): value = canonical(r); digest = sha256(value.encode()).hexdigest() old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() if old: @@ -211,6 +215,7 @@ def ingest_batch(con, plant, payload, now): duplicate += 1 else: con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + mapping_identity.save_origin(con, plant, c, original, r['capturedAt'], now, aliases) con.commit() except Exception: con.rollback(); raise @@ -243,7 +248,7 @@ def physical_value(values, c): return load -def reconstruct(records, c): +def reconstruct(records, c, *, projection=None): """Bounded retrospective estimation, never a real-time feedback signal. Missing observations split support. Source timestamps are not refreshed. Small @@ -260,6 +265,11 @@ uncovered portions remain quantified and are never filled with zero. for s in c['sources']: if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) + if projection is not None: + configured={s['key']:s for s in c['sources']} + if set(projection)!= {'keys','calculate'} or not callable(projection['calculate']) or not projection['keys'] or not set(projection['keys']) <= set(configured): + raise ValueError('Invalid internal projection') + primary={k:configured[k] for k in projection['keys']} first,last = records[0]['capturedAt'],records[-1]['capturedAt'] series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] first_observed = {k:{} for k in primary} @@ -313,7 +323,9 @@ uncovered portions remain quantified and are never filled with zero. extended.append(k); known_at = max(known_at, bridge['availableAt']) load = None if usable: - try: load = physical_value(vals,c) + try: + load = physical_value(vals,c) if projection is None else projection['calculate'](vals,c) + if not numeric(load,1e9):raise ValueError('Invalid historical projection') except ValueError: usable = False if usable: item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 @@ -490,9 +502,10 @@ def pipeline_status(con,plant): out=[] for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() - count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at),MAX(received_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'lastReceived':iso(count[3]) if count[3] else None, 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], 'observationDatasetId':c.get('sourceDatasetId',ds), diff --git a/services/netplan-v4/netplan_v4/retention.py b/services/netplan-v4/netplan_v4/retention.py new file mode 100644 index 0000000..219a9a8 --- /dev/null +++ b/services/netplan-v4/netplan_v4/retention.py @@ -0,0 +1,106 @@ +"""Bounded online tables with verified lossless archives; never prune unarchived data. + +Runs at most hourly. Raw input history: 120 days (longer than 90-day training and +comparison limits). Ordinary plans: 7 days, excluding active/acknowledged plans. +Archives are retained; external backup/long-term archive lifecycle is operational. +""" +from pathlib import Path +import gzip +import hashlib +import json +import os +import sqlite3 +import tempfile + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_maintenance( + name TEXT PRIMARY KEY, checked_at INTEGER NOT NULL, status TEXT NOT NULL, detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_archives( + id TEXT PRIMARY KEY, created_at INTEGER NOT NULL, filename TEXT NOT NULL, + sha256 TEXT NOT NULL, records INTEGER NOT NULL, kind TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS planner_plan_created ON planner_plans(created_at); + ''') + + +def archive_batch(con,directory,now,kind,limit=100): + if kind not in ('observations','plans') or type(limit) is not int or not 1<=limit<=1000: + raise ValueError('Invalid archival scope') + directory=Path(directory) + if directory.is_symlink():raise ValueError('Archive symlink refused') + directory.mkdir(mode=0o700,parents=True,exist_ok=True) + if directory.stat().st_mode & 0o007:raise ValueError('Archive directory must not be public') + from datetime import datetime,timezone + with_context=False + con.execute('BEGIN IMMEDIATE') + try: + if kind=='observations': + rows=con.execute('SELECT * FROM planner_observations WHERE captured_at67108864:raise ValueError('Archive batch exceeds memory budget') + ident=hashlib.sha256(raw).hexdigest();name=kind+'-'+ident+'.jsonl.gz';path=directory/name + if path.is_symlink():raise ValueError('Archive target symlink refused') + if not path.exists(): + fd,tmp=tempfile.mkstemp(prefix='.archive-',dir=directory) + try: + os.fchmod(fd,0o600) + with os.fdopen(fd,'wb') as out: + with gzip.GzipFile(fileobj=out,mode='wb',mtime=0) as zipped:zipped.write(raw) + out.flush();os.fsync(out.fileno()) + os.replace(tmp,path) + dfd=os.open(directory,os.O_RDONLY) + try:os.fsync(dfd) + finally:os.close(dfd) + finally: + if os.path.exists(tmp):os.unlink(tmp) + # Read back exact bytes before removing any database row. + with gzip.open(path,'rb') as f:verified=f.read(len(raw)+1) + if verified!=raw:raise ValueError('Archive verification failed; original rows retained') + for r in rows: + if kind=='observations': + key=(r['plant'],r['dataset'],r['captured_at']) + con.execute('DELETE FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',key) + con.execute('DELETE FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=? AND fingerprint=?',(*key,r['fingerprint'])) + else:con.execute('DELETE FROM planner_plans WHERE plan_id=?',(r['plan_id'],)) + con.execute('INSERT OR IGNORE INTO planner_archives VALUES(?,?,?,?,?,?)',(ident,now,name,ident,len(rows),kind)) + con.commit() + return {'archived':len(rows),'kind':kind,'file':name,'sha256':ident,'sourceRecoverable':True} + except Exception: + con.rollback();raise + + +def maintain(store,now): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return + con=store.con;stamp=int(now.timestamp());old=con.execute("SELECT checked_at FROM planner_maintenance WHERE name='archive'").fetchone() + if old and stamp-old[0]<3600:return + db=con.execute('PRAGMA database_list').fetchone()[2] + if not db:return # In-memory test/ephemeral databases have no archival location. + try: + directory=Path(db).resolve().parent/'archives' + results=[archive_batch(con,directory,stamp,kind) for kind in ('plans','observations')] + if any(r['archived']>=100 for r in results):stamp-=3300 + state='ok';detail={'results':results,'archivesRetained':True,'rawRetentionDays':120,'ordinaryPlanRetentionDays':7} + except (OSError,ValueError,sqlite3.Error) as exc: + state='archive_error';detail={'errorType':type(exc).__name__,'unverifiedDataNotDeleted':True} + with con:con.execute('INSERT INTO planner_maintenance VALUES(?,?,?,?) ON CONFLICT(name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,detail=excluded.detail', + ('archive',stamp,state,json.dumps(detail,separators=(',',':')))) + + +def status(con): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return {'status':'disabled_requires_operator_opt_in'} + r=con.execute("SELECT checked_at,status,detail FROM planner_maintenance WHERE name='archive'").fetchone() + # Public per-plant state must not expose archival filenames/rows of other tenants. + return {'checkedAtEpoch':r[0],'status':r[1],'rawRetentionDays':120,'ordinaryPlanRetentionDays':7,'archivesRetained':True} if r else {'status':'not_yet_run'} diff --git a/services/netplan-v4/netplan_v4/service.py b/services/netplan-v4/netplan_v4/service.py index ceabaee..0143c79 100644 --- a/services/netplan-v4/netplan_v4/service.py +++ b/services/netplan-v4/netplan_v4/service.py @@ -16,7 +16,7 @@ from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_star from .store import PlannerStore,canonical from .selection import choose_family from .optimizer import optimize -from . import meter_runtime, controlled_trial, measurement_pipeline +from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline from .forecast_quality import assess_family from .receiver_contract import provenance from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month @@ -267,10 +267,12 @@ def assemble(store,plant,family,now): return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)} def run_once(store,now): + retention.maintain(store,now) stamp=int(now.timestamp())//300 plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')] for plant in plants: config=store.settings(plant) + economic_replay.advance(store,plant,now) datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))] for dataset in datasets: try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp())) @@ -286,14 +288,16 @@ def run_once(store,now): try: settings=store.settings(plant);previous=store.current(plant) current=previous['sourceFamily'] if previous else store.registry.entries()[0].key - # Productive replay ingestion is intentionally not fabricated from R2 metrics. - selection=choose_family(settings['family'],current,(),registry=store.registry,now=now) + selection=choose_family(settings['family'],current,economic_replay.scores(store,plant,now),registry=store.registry,now=now, + lookback_days=settings['autoLookbackDays'],minimum_days=settings['autoMinimumDays'], + minimum_coverage=settings['autoMinimumCoverage'],margin_chf=settings['autoSwitchMarginChf']) data,full_end,quality=assemble(store,plant,selection['family'],now) data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] result=optimize(**data,config_revision=settings['revision'],family=selection['family']) if result['executable']: result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])}) store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token']) + economic_replay.capture(store,plant,now,assemble,optimize) return result except (ValueError,TypeError,KeyError) as exc: result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]} @@ -309,7 +313,7 @@ def status(store,plant,now): pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone() ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone() fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit')) - return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)} + return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)} def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()): allowed={str(UUID(p)) for p in plants} @@ -343,7 +347,7 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial') return factory() @app.get('/health') - def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1} + def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1,'applicationRelease':'unified-rc1','mappingCompatibilityVersion':1,'economicReplayVersion':1,'archiveVersion':1} @app.get('/internal/v2/prognosis/{plant}/planner') def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): s=authorize(plant,token) @@ -369,6 +373,17 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc)) except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}/mapping-compatibility') + def mapping_compatibility(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + from .mapping_identity import register + cfg=measurement_pipeline.configuration(s.con,plant,dataset) + return register(s.con,plant,cfg,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError,AttributeError): + raise HTTPException(400, 'Mapping compatibility proof rejected') + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/settings') def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): s=authorize(plant,token) diff --git a/services/netplan-v4/netplan_v4/store.py b/services/netplan-v4/netplan_v4/store.py index 07d5254..236e376 100644 --- a/services/netplan-v4/netplan_v4/store.py +++ b/services/netplan-v4/netplan_v4/store.py @@ -4,7 +4,7 @@ import sqlite3 from datetime import datetime,timedelta from uuid import uuid4 from .domain import default_registry,month_key,number,quarter_start,utc -from . import meter_runtime, controlled_trial, measurement_pipeline +from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline def canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False) @@ -33,6 +33,8 @@ class PlannerStore: ''') meter_runtime.schema(self.con) controlled_trial.schema(self.con) + economic_replay.schema(self.con) + retention.schema(self.con) measurement_pipeline.schema(self.con) def close(self):self.con.close() def settings(self,plant): diff --git a/services/netplan-v4/tests/test_economic_replay.py b/services/netplan-v4/tests/test_economic_replay.py new file mode 100644 index 0000000..3b5dc43 --- /dev/null +++ b/services/netplan-v4/tests/test_economic_replay.py @@ -0,0 +1,144 @@ +import copy +import json +import tempfile +import unittest +from dataclasses import asdict +from datetime import datetime,timedelta,timezone +from pathlib import Path +from netplan_v4 import economic_replay as e, measurement_pipeline as m +from netplan_v4.domain import Battery,Limits,Step,Price +from netplan_v4.store import PlannerStore +from netplan_v4.selection import choose_family +from test_measurement_pipeline import config,record +from test_v4 import AID + +AT=datetime(2026,10,2,22,tzinfo=timezone.utc) +END=AT+timedelta(days=1) +TS=int(AT.timestamp()) + + +def fixture(): + battery=Battery('b',20.,50.,10.,90.,5000.,5000.,AT,grid_charging=True) + steps=[Step(AT+timedelta(minutes=5*i),1000.,0.,Price(.1 if i<144 else .4),Price(.03)) for i in range(288)] + data={'steps':steps,'batteries':[battery],'limits':Limits(import_w=10000.),'observed_peaks':{'2026-10':5.},'peak_prices':{'2026-10':5.},'quarter_history':{},'at':AT} + def optimizer(**kw): + points=[] + family=kw['family'] + for i,s in enumerate(kw['steps']): + target=1000. if family in ('3','13') else (1600. if i<144 else 500.) + points.append({'time':s.start.isoformat(),'validUntil':s.end.isoformat(),'gridTargetW':target, + 'importPriceChfKwh':s.import_price.chf_kwh,'exportPriceChfKwh':s.export_price.chf_kwh}) + return {'executable':True,'points':points} + def assemble(*args):return copy.deepcopy(data),END,{} + return data,assemble,optimizer + + +class EconomicReplayTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'));self.c=config() + m.register_dataset(self.s.con,AID,self.c,TS) + self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':self.c['datasetId'],'autoMinimumDays':1},0,AT) + self.data,self.assemble,self.optimizer=fixture() + def tearDown(self):self.s.close();self.tmp.cleanup() + def capture(self):e.capture(self.s,AID,AT,self.assemble,self.optimizer) + def cohort(self):return json.loads(self.s.con.execute('SELECT value FROM planner_economic_cohorts').fetchone()[0]) + def populate(self): + now=int(END.timestamp())+180 + rows=[record(t,self.c) for t in range(TS-60,now+1,30)] + for i in range(0,len(rows),120):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows[i:i+120]},now) + def outcomes(self,value=1000.):return {TS+300*i:{'loadW':value,'coverage':1.,'profileUsable':True} for i in range(288)} + def test_snapshot_frozen_once(self): + self.capture();first=self.cohort();self.capture();self.assertEqual(first,self.cohort()) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_no_retroactive_midday_snapshot(self): + e.capture(self.s,AID,AT+timedelta(hours=12),self.assemble,self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_silent_legacy_replay(self): + self.s.save_settings(AID,{'forecastSource':'legacy'},1,AT);self.capture() + self.assertEqual(e.scores(self.s,AID,END),[]) + def test_all_families_same_initial_energy(self): + self.capture();c=self.cohort();values=[e.simulate(c,f,self.outcomes()) for f in c['plans']] + self.assertEqual(len({v['initialEnergyKwh'] for v in values}),1) + for v in values:self.assertGreaterEqual(v['finalEnergyKwh'],2.-1e-6);self.assertLessEqual(v['finalEnergyKwh'],18.00001) + def test_profitable_time_shift_has_lower_normalized_cost(self): + self.capture();c=self.cohort();a=e.simulate(c,'3',self.outcomes());b=e.simulate(c,'23',self.outcomes()) + self.assertLess(b['costChf'],a['costChf']);self.assertTrue(b['terminalNormalized']) + def test_end_energy_not_free_savings(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=0.;p['importPriceChfKwh']=.4 + d=e.simulate(c,'3',self.outcomes());self.assertGreater(d['terminalAdjustmentChf'],0) + def test_missing_actual_not_zero(self): + self.capture();out=self.outcomes();out.pop(TS+300) + with self.assertRaises(ValueError):e.simulate(self.cohort(),'3',out) + def test_export_and_import_limits_not_ignored(self): + self.capture();c=self.cohort();c['context']['limits']['import_w']=500. + result=e.simulate(c,'3',self.outcomes(20000.));self.assertGreater(result['constraintBreaches'],0) + def test_peak_increment_not_total_billed_twice(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=6000. + result=e.simulate(c,'3',self.outcomes(6000.));self.assertAlmostEqual(result['peakCostChf'],5.) + def test_no_actual_results_before_day_end(self): + self.capture();self.populate();e.advance(self.s,AID,AT+timedelta(hours=12)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + def test_real_pipeline_from_frozen_plans_to_actuals_to_selection(self): + self.capture();self.populate();now=END+timedelta(minutes=4) + e.advance(self.s,AID,now);scores=e.scores(self.s,AID,now) + self.assertEqual(len(scores),3);self.assertEqual(scores[0].days,1) + chosen=choose_family('auto','3',scores,minimum_days=1,margin_chf=0,now=now) + self.assertEqual(chosen['family'],'23');self.assertEqual(chosen['mode'],'economic_replay') + self.assertTrue(e.status(self.s,AID)['connected']);self.assertFalse(e.status(self.s,AID)['isBillingEvidence']) + def test_evaluation_idempotent(self): + self.capture();self.populate();now=END+timedelta(minutes=4);e.advance(self.s,AID,now);e.advance(self.s,AID,now) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],1) + def test_missing_outcomes_preserve_cohort(self): + self.capture();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_changed_tariffs_between_families_reject_cohort(self): + def changed(**kwargs): + v=self.optimizer(**kwargs) + if kwargs['family']=='23':v['points'][0]['importPriceChfKwh']+=.5 + return v + e.capture(self.s,AID,AT,self.assemble,changed) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_grid_charging_unsupported_is_explicit(self): + self.data['batteries'][0]=__import__('dataclasses').replace(self.data['batteries'][0],grid_charging=False) + e.capture(self.s,AID,AT,lambda *args:(self.data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_short_price_horizon_no_fake_extension(self): + data=copy.deepcopy(self.data);data['steps']=data['steps'][:12] + e.capture(self.s,AID,AT,lambda *args:(data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_other_plant_no_results(self): + self.capture();self.populate();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(e.scores(self.s,'30509683-4569-49e4-848f-4905e4cc813a',END+timedelta(minutes=4)),[]) + def test_residual_projection_does_not_change_load_profile(self): + rows=[m.project(record(t,self.c),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1000.) + self.assertEqual(m.reconstruct(rows,self.c)[0]['loadW'],6000.) + def test_monthly_peak_not_charged_again_every_comparison_day(self): + self.capture();c=self.cohort() + for family in c['plans']: + for point in c['plans'][family]['points']:point['gridTargetW']=6000. + result={f:e.simulate(c,f,self.outcomes(6000.)) for f in c['plans']} + self.assertEqual(result['3']['peakCostChf'],5.) + policy=self.s.con.execute('SELECT policy_id FROM planner_economic_cohorts').fetchone()[0] + with self.s.con: + for n in (0,1): + start=AT+timedelta(days=n);end=END+timedelta(days=n);day=start.astimezone(__import__('zoneinfo').ZoneInfo('Europe/Zurich')).date().isoformat() + if n: + self.s.con.execute('INSERT INTO planner_economic_cohorts VALUES(?,?,?,?,?,?)',(AID,day,int(start.timestamp()),int(end.timestamp()),policy,json.dumps(c))) + v={'results':result,'start':start.isoformat(),'end':end.isoformat()} + self.s.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(AID,day,int(end.timestamp())+120,policy,json.dumps(v))) + scores=e.scores(self.s,AID,END+timedelta(days=1,minutes=4)) + value=next(x.cost_chf for x in scores if x.family=='3') + expected=2*(result['3']['energyCostChf']+result['3']['throughputCostChf']+result['3']['terminalAdjustmentChf'])+5. + self.assertAlmostEqual(value,expected) + + def test_external_sdl_not_assumed_zero(self): + rows=[m.project(record(t,self.c,sdl=250.),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1250.) + for r in rows:r['raw']['sdl']['valid']=False + self.assertFalse(e.actuals(rows,self.c)[0]['profileUsable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_mapping_recovery.py b/services/netplan-v4/tests/test_mapping_recovery.py new file mode 100644 index 0000000..1532535 --- /dev/null +++ b/services/netplan-v4/tests/test_mapping_recovery.py @@ -0,0 +1,102 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path +from hashlib import sha256 +from netplan_v4.store import PlannerStore +from netplan_v4 import mapping_identity as mi, measurement_pipeline as m +from netplan_v4.service import create_app +from fastapi.testclient import TestClient +from test_measurement_pipeline import config, record, NOW +from test_v4 import AID, TOKEN + + +def proof(c): + a = {'version':1, 'installationId':AID, 'reportedInventorySha256':c['inventorySha256'], + 'accounting':{'splitToleranceW':100.0,'sources':copy.deepcopy(c['sources'])}} + b=copy.deepcopy(a); b['accounting']['splitToleranceW']=100 + p={'version':1, 'canonicalJson':json.dumps(a,separators=(',',':')), 'legacyJson':json.dumps(b,separators=(',',':'))} + c['mappingSha256']=sha256(p['canonicalJson'].encode()).hexdigest() + return p + + +class MappingRecoveryTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(); self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.c=config(); self.proof=proof(self.c); self.alias=sha256(self.proof['legacyJson'].encode()).hexdigest() + m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self): self.s.close(); self.tmp.cleanup() + def register(self): return mi.register(self.s.con,AID,self.c,self.proof,NOW) + def batch(self,rows): return m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows},NOW) + def r(self,t,legacy=False): + r=record(t,self.c) + if legacy:r['mappingSha256']=self.alias + return r + def test_original_mixed_batch_defect_and_atomic_recovery(self): + rows=[self.r(NOW-3600+i*30,i>=45) for i in range(120)]; before=copy.deepcopy(rows) + with self.assertRaises(ValueError):self.batch(rows) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],0) + self.register(); result=self.batch(rows) + self.assertEqual(result['stored'],120); self.assertEqual(rows,before) + self.assertEqual(result['acceptedThrough'],rows[-1]['capturedAt']) + origins=self.s.con.execute('SELECT received_mapping,COUNT(*) FROM planner_observation_origins GROUP BY received_mapping').fetchall() + self.assertEqual(dict(origins)[self.alias],75) + def test_retry_does_not_duplicate(self): + self.register(); self.batch([self.r(NOW,True)]) + self.assertEqual(self.batch([self.r(NOW,True)])['duplicates'],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1) + def test_canonical_retry_preserves_both_origins(self): + self.register(); self.batch([self.r(NOW,True)]); self.batch([self.r(NOW)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],2) + def test_unknown_alias_rolls_back_entire_block(self): + self.register(); bad=self.r(NOW,True);bad['mappingSha256']='f'*64 + with self.assertRaises(ValueError): self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],0) + def test_conflicting_measurement_keeps_existing_and_does_not_ack_prefix(self): + self.register(); self.batch([self.r(NOW)]) + bad=self.r(NOW,True);bad['raw']['grid']['value']+=500 + with self.assertRaises(ValueError):self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_proof_wrong_canonical_hash_rejected(self): + p=copy.deepcopy(self.proof);p['canonicalJson']+=' ' + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_numeric_change_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":101,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_boolean_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":true,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_other_field_change_rejected(self): + p=copy.deepcopy(self.proof);b=json.loads(p['legacyJson']);b['accounting']['sources'][0]['factorToW']=2;p['legacyJson']=json.dumps(b) + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_inventory_remains_strict(self): + self.register();r=self.r(NOW,True);r['reportedInventorySha256']='d'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_other_plant_does_not_inherit_proof(self): + self.register();other='30509683-4569-49e4-848f-4905e4cc813a';m.register_dataset(self.s.con,other,self.c,NOW) + r=self.r(NOW,True);r['installationId']=other + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,other,{'version':1,'datasetId':self.c['datasetId'],'records':[r]},NOW) + def test_other_dataset_does_not_inherit_proof(self): + self.register();c=copy.deepcopy(self.c);c['datasetId']='unrelated';m.register_dataset(self.s.con,AID,c,NOW) + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':'unrelated','records':[self.r(NOW,True)]},NOW) + def test_direct_project_still_strict(self): + self.register() + with self.assertRaises(ValueError):m.project(self.r(NOW,True),self.c,AID,NOW) + def test_registration_idempotent_and_persistent(self): + a=self.register();b=self.register();self.assertEqual(a,b) + self.s.close();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.assertEqual(self.batch([self.r(NOW,True)])['stored'],1) + def test_malformed_proof_no_registration(self): + for x in ({}, {'version':True,'canonicalJson':'{}','legacyJson':'{}'}): + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,x,NOW) + def test_internal_api_authentication(self): + app=create_app(str(Path(self.tmp.name)/'db.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + url=f'/internal/v2/prognosis/{AID}/planner/datasets/{self.c["datasetId"]}/mapping-compatibility' + self.assertEqual(client.put(url,json=self.proof).status_code,401) + response=client.put(url,json=self.proof,headers={'X-Enelix-Service-Token':TOKEN}) + self.assertEqual(response.status_code,200,response.text);self.assertFalse(response.json()['controlEnabled']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_release_preflight.py b/services/netplan-v4/tests/test_release_preflight.py index 977a292..d4942a9 100644 --- a/services/netplan-v4/tests/test_release_preflight.py +++ b/services/netplan-v4/tests/test_release_preflight.py @@ -42,7 +42,8 @@ class ReleasePreflightTest(unittest.TestCase): text=(ROOT/'gui/netplan-v4.js').read_text() self.assertIn('SCHATTENBETRIEB',text) self.assertIn('measurementPolicy',text) - self.assertIn('wartet weiterhin auf vergleichbare Kosten-Replays',text) + self.assertIn('erstellt vergleichbare Kosten-Replays',text) + self.assertIn('keine vollstaendige Nachbildung laufender Neuoptimierungen',text) self.assertIn('forecastSource',text) self.assertIn('trainingCadence',text) self.assertIn('Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe',text) diff --git a/services/netplan-v4/tests/test_retention.py b/services/netplan-v4/tests/test_retention.py new file mode 100644 index 0000000..910b56e --- /dev/null +++ b/services/netplan-v4/tests/test_retention.py @@ -0,0 +1,56 @@ +import gzip,hashlib,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +from netplan_v4.store import PlannerStore +from netplan_v4 import retention +from test_v4 import AID + +NOW=1791028800 + +class RetentionTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name);self.s=PlannerStore(str(self.root/'db.sqlite'));self.archive=self.root/'archive' + def tearDown(self):self.s.close();self.tmp.cleanup() + def old(self,ago=121,received_ago=10): + v='{"numeric":123}';fp=hashlib.sha256(v.encode()).hexdigest() + with self.s.con: + self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,NOW-received_ago*86400,fp,v)) + self.s.con.execute('INSERT INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,'m','i','e',NOW-received_ago*86400)) + def count(self):return self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0] + def test_lossless_archive_before_removal(self): + self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations');self.assertEqual(r['archived'],1);self.assertEqual(self.count(),0) + raw=gzip.decompress((self.archive/r['file']).read_bytes());row=json.loads(raw) + self.assertEqual(row['row']['value'],'{"numeric":123}');self.assertEqual(row['origins'][0]['received_mapping'],'m') + self.assertEqual(hashlib.sha256(raw).hexdigest(),r['sha256']) + def test_recent_and_late_received_rows_not_pruned(self): + self.old(ago=100);self.old(ago=121,received_ago=1) + self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0);self.assertEqual(self.count(),2) + def test_failed_durability_keeps_original(self): + self.old() + with patch('netplan_v4.retention.os.fsync',side_effect=OSError('synthetic')): + with self.assertRaises(OSError):retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(self.count(),1);self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1) + def test_symlink_archive_rejected(self): + self.old();other=self.root/'other';other.mkdir();self.archive.symlink_to(other) + with self.assertRaises(ValueError):retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(self.count(),1) + def test_current_and_acknowledged_plans_retained(self): + with self.s.con: + for key in ('old','current','ack'): + self.s.con.execute('INSERT INTO planner_plans VALUES(?,?,?,?,?,?)',(key,AID,0,'shadow','{}','2020-01-01T00:00:00+00:00')) + self.s.con.execute('INSERT INTO planner_current VALUES(?,?,?)',(AID,'shadow','current')) + self.s.con.execute('INSERT INTO planner_ack VALUES(?,?,?,?,?,?)',(AID,'ack',0,'2020-01-01T00:00:00+00:00',None,'shadow_seen')) + r=retention.archive_batch(self.s.con,self.archive,NOW,'plans');self.assertEqual(r['archived'],1) + self.assertEqual({r[0] for r in self.s.con.execute('SELECT plan_id FROM planner_plans')},{'current','ack'}) + def test_archived_rows_can_be_restored_exactly(self): + self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations') + obj=json.loads(gzip.decompress((self.archive/r['file']).read_bytes()));row=obj['row'] + with self.s.con:self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',tuple(row[k] for k in ('plant','dataset','captured_at','received_at','fingerprint','value'))) + self.assertEqual(self.count(),1) + self.assertEqual(self.s.con.execute('SELECT fingerprint FROM planner_observations').fetchone()[0],hashlib.sha256(row['value'].encode()).hexdigest()) + def test_no_duplicate_archive_after_repeat(self): + self.old();retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0) + self.assertEqual(len(list(self.archive.glob('*.gz'))),1) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_unified_release.py b/services/netplan-v4/tests/test_unified_release.py new file mode 100644 index 0000000..e010e34 --- /dev/null +++ b/services/netplan-v4/tests/test_unified_release.py @@ -0,0 +1,81 @@ +import importlib.util,json,hashlib,tempfile,unittest,sqlite3,subprocess,sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from test_v4 import AID + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'commissioning')) +import deploy_unified_release as dep + + +class UnifiedReleaseTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name)/'service';self.root.mkdir() + self.package=self.root/'commissioning/unified-release';(self.package/'context/netplan_v4').mkdir(parents=True) + (self.package/'context/gui').mkdir();(self.root/'netplan_v4').mkdir();(self.root/'gui').mkdir();(self.root/'data').mkdir() + self.gui=Path(self.tmp.name)/'portal.js';self.gui.write_text('old ui') + self.proof=self.package/'mapping-proof.json';self.proof.write_text('{}') + helper=self.root/'commissioning/deploy_application.py';helper.write_bytes((ROOT/'commissioning/deploy_application.py').read_bytes()) + self.files={} + for name,old,new in [('netplan_v4/code.py','old','new'),('gui/netplan-v4.js','old ui','new ui')]: + (self.root/name).write_text(old);(self.package/'context'/name).write_text(new) + self.files[name]={'before':hashlib.sha256(old.encode()).hexdigest(),'after':hashlib.sha256(new.encode()).hexdigest()} + with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:c.execute('CREATE TABLE original(value TEXT)');c.execute("INSERT INTO original VALUES('kept')") + manifest={'scope':'unified_v4_candidate','releaseId':'a'*64,'installationId':AID,'files':self.files, + 'installerSha256':dep.digest(Path(dep.__file__).resolve()),'guiBefore':dep.digest(self.gui),'proofSha256':dep.digest(self.proof),'backupHelperSha256':dep.digest(helper)} + (self.package/'RELEASE.json').write_text(json.dumps(manifest));self.calls=[];self.fail=None + self.patches=[patch.object(dep,'ROOT',self.root),patch.object(dep,'PACKAGE',self.package),patch.object(dep,'GUI',self.gui),patch.object(dep.os,'geteuid',return_value=0),patch.object(dep.os,'chown'),patch.object(dep.subprocess,'run',side_effect=self.command)] + for p in self.patches:p.start() + def tearDown(self): + for p in reversed(self.patches):p.stop() + self.tmp.cleanup() + def command(self,args,**kwargs): + self.calls.append(args) + if self.fail=='build' and args[:2]==['docker','build']:raise subprocess.CalledProcessError(1,args) + if args[0]=='node':return SimpleNamespace(stdout='') + if args[:2]==['docker','exec']:return SimpleNamespace(stdout=AID+'\n') + if args[:2]==['docker','inspect']:return SimpleNamespace(stdout='sha256:'+'1'*64+'\n') + if args[:3]==['docker','image','inspect']:return SimpleNamespace(stdout='sha256:'+'2'*64+'\n') + if 'ps' in args:return SimpleNamespace(stdout='container\n') + if 'exec' in args: + if self.fail=='bootstrap':raise subprocess.CalledProcessError(1,args) + return SimpleNamespace(stdout=json.dumps({'settingsUnchanged':True,'health':{'liveEnabled':False}})) + return SimpleNamespace(stdout='') + def test_source_only_never_deploys(self): + dep.run(AID);self.assertEqual(self.calls,[]);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old') + def test_drift_stops_before_docker(self): + (self.root/'netplan_v4/code.py').write_text('other work') + with self.assertRaises(ValueError):dep.run(AID,True) + self.assertEqual(self.calls,[]) + def test_gui_parallel_change_preserved(self): + self.gui.write_text('parallel gui') + with self.assertRaises(ValueError):dep.run(AID,True) + self.assertEqual(self.gui.read_text(),'parallel gui') + def test_tests_before_replacement_and_no_other_service(self): + dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new') + tests=next(i for i,c in enumerate(self.calls) if c[:2]==['docker','run']) + start=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(tests,start);self.assertIn('none',self.calls[tests]);self.assertIn('--read-only',self.calls[tests]) + for c in self.calls: + if 'up' in c:self.assertEqual(c[-1],'netplan-v4') + self.assertEqual(self.gui.read_text(),'new ui') + with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:self.assertEqual(c.execute('SELECT value FROM original').fetchone()[0],'kept') + def test_failed_build_changes_no_running_sources(self): + self.fail='build' + with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True) + self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertFalse(any('up' in c for c in self.calls)) + def test_bootstrap_failure_restores_previous_runtime_sources(self): + self.fail='bootstrap' + with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True) + self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertEqual(self.gui.read_text(),'old ui') + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_repeat_is_safe(self): + dep.run(AID,True);dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new') + def test_no_test_or_production_dispatch_grant(self): + text=Path(dep.__file__).read_text() + self.assertNotIn('/trial/arm',text);self.assertNotIn("'runMode':'live'",text);self.assertNotIn('SET NetzfahrplanV4DatenCursor',text) + def test_dockerfile_copies_installers_needed_by_test_suite(self): + self.assertIn('COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py',(ROOT/'Dockerfile').read_text()) + +if __name__=='__main__':unittest.main() diff --git a/tests/V4ApplicationData/archive_checks.php b/tests/V4ApplicationData/archive_checks.php new file mode 100644 index 0000000..d5654ef --- /dev/null +++ b/tests/V4ApplicationData/archive_checks.php @@ -0,0 +1,27 @@ +isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($dir);} +$base=sys_get_temp_dir().'/v4-archive-'.bin2hex(random_bytes(6));mkdir($base,0700); +try { + $old='raw-'.gmdate('Ymd',$now-10*86400).'.jsonl';$recent='raw-'.gmdate('Ymd',$now-2*86400).'.jsonl';$day='raw-'.gmdate('Ymd',$now).'.jsonl'; + $raw="{\"sample\":1}\n{\"sample\":2}\n";file_put_contents($base.'/'.$old,$raw);file_put_contents($base.'/'.$recent,$raw);file_put_contents($base.'/'.$day,$raw); + checkArchive(A::run($base,[],$now)['archived']===0,'no acknowledgement means no archival'); + $r=A::run($base,['day'=>$old,'offset'=>5],$now);checkArchive($r['archived']===0&&is_file($base.'/'.$old),'partially acknowledged day retained'); + $r=A::run($base,['day'=>$day,'offset'=>5],$now);checkArchive($r['archived']===1&&!is_file($base.'/'.$old),'only completely acknowledged old day retired'); + $files=glob($base.'/archive/*.gz');checkArchive(count($files)===1&&gzdecode(file_get_contents($files[0]))===$raw,'compressed original verified byte for byte'); + checkArchive(is_file($base.'/'.$recent)&&is_file($base.'/'.$day),'recent and current outbox preserved'); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===0,'repeat is idempotent'); + file_put_contents($base.'/'.$old,$raw);file_put_contents($files[0],gzencode('corrupt')); + try {A::run($base,['day'=>$day,'offset'=>5],$now);throw new LogicException('expected refusal');} catch(RuntimeException $e) {checkArchive(is_file($base.'/'.$old),'archive corruption never loses original');} + unlink($files[0]);$lock=fopen($base.'/.writer.lock','c+b');flock($lock,LOCK_EX); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['status']==='writer_busy','archive serializes with actual capture writer');flock($lock,LOCK_UN);fclose($lock); + foreach(range(11,15)as$d)file_put_contents($base.'/raw-'.gmdate('Ymd',$now-$d*86400).'.jsonl',$raw); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===2,'bounded archive work per invocation'); + $text=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Datenarchiv.php');checkArchive(!preg_match('/\b(?:RequestAction|IPS_SetProperty|curl_exec)\s*\(/',$text),'archive has no control or network calls'); + echo "TOTAL $n archive checks passed; only temporary test data.\n"; +} finally {cleanupArchive($base);} diff --git a/tests/V4ApplicationData/checks.php b/tests/V4ApplicationData/checks.php index 5f018e9..318bbb3 100644 --- a/tests/V4ApplicationData/checks.php +++ b/tests/V4ApplicationData/checks.php @@ -63,16 +63,29 @@ try { check($sent[0]['records'][0]['raw']['s101']['sourceUpdatedAt']===$clock,'original timestamp retained'); check(!str_contains(json_encode($sent),'synthetic-not-a-real-token'),'token never included in data payload'); check($sent[0]['records'][0]['controlEligible']===false,'data delivery is not actuator permission'); + $expected=hash('sha256',json_encode(NetzfahrplanV4Messaufnahme::configuration($c),JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + check($sent[0]['records'][0]['mappingSha256']===$expected,'property roundtrip preserves approved mapping identity'); + check(json_decode($m->a['NetzfahrplanV4LetzteDatenquittung'],true)['acceptedThrough']===end($sent[0]['records'])['capturedAt'],'successful acknowledgement persisted independently'); $clock+=30;$m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'30-second collection does not double request rate'); check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'unsent rows remain behind cursor'); + check(isset(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['lastAcknowledgement']['acceptedThrough']),'scheduled status retains last successful receipt'); $clock+=30;$mode='429';$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'429 never advances acknowledgement cursor'); check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['dataRetained'],'offline data retained'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['httpStatus']===429,'safe HTTP error code visible without credentials'); $mode='wrong_ack';$clock+=120;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'wrong ack timestamp rejected'); $mode='ok';$clock+=240;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']!==$cursor,'later success sends the preserved backlog'); check(count(end($sent)['records'])===4,'all previously unacknowledged rows included once'); $file=glob($m->folder().'/raw-*.jsonl')[0];$count=count(file($file));check($count===5,'raw journal never cleared by upload'); $lastCursor=$m->a['NetzfahrplanV4DatenCursor'];file_put_contents($file,'incomplete',FILE_APPEND); check($m->send()['status']==='up_to_date','in-progress tail not transmitted');check($m->a['NetzfahrplanV4DatenCursor']===$lastCursor,'partial tail not acknowledged'); + // Only temporary test journal: turn the prior incomplete tail into an additional complete queued record. + $buf=file_get_contents($file);$buf=substr($buf,0,-strlen('incomplete')); + $extra=$sent[0]['records'][0];$clock+=60;$extra['capturedAt']=gmdate('c',$clock);$extra['captureStartedAt']=$extra['capturedAt']; + file_put_contents($file,$buf.json_encode($extra)."\n".'crash-tail'); + $clock+=60;$beforeCount=count($sent);$m->ErfasseNetzfahrplanV4Daten(); + $err=json_decode($m->vars['NetzfahrplanV4Datenstatus'],true); + check($err['status']==='capture_error'&&$err['delivery']['status']==='acknowledged','capture failure does not deadlock pending delivery'); + check(count($sent)===$beforeCount+1&&str_ends_with(file_get_contents($file),'crash-tail'),'complete prefix sent while original incomplete tail retained'); $m->p['NetzfahrplanV4MessdatenAktiv']=false;$m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===0,'disable affects acquisition only'); check(count($m->vars)===1,'only own manager data-status variable written'); $text=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4DatenTrait.php'); From 9878197aeae59c48fd69a65a49742c6dbf6ab090 Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sat, 3 Oct 2026 11:40:54 +0000 Subject: [PATCH 15/22] feat(control): connect physical feedback to bounded V4 command and fallback paths --- Batterie/module.php | 20 ++- Manager/module.php | 6 + docs/netplan-v4-corrected-feedback.md | 108 ++++++++++++++ examples/V4CorrectedFeedback/MANIFEST.json | 65 ++++++++ examples/V4CorrectedFeedback/build_stage.py | 25 ++++ .../V4CorrectedFeedback/feedback-config.json | 71 +++++++++ .../V4CorrectedFeedback/file_installer.php | 66 ++++++++ examples/V4CorrectedFeedback/install.php | 68 +++++++++ ...atterieNetzfahrplanV4RueckmeldungTrait.php | 38 +++++ libs/BatterieNetzfahrplanV4TestTrait.php | 13 ++ libs/ManagerNetzfahrplanV4EmpfangTrait.php | 28 ++++ libs/ManagerNetzfahrplanV4TestTrait.php | 3 + libs/NetzfahrplanV4Regeltest.php | 26 ++++ libs/NetzfahrplanV4Rueckmeldung.php | 141 ++++++++++++++++++ tests/NetzfahrplanV4BilanzierungTest.php | 2 +- tests/NetzfahrplanV4DatenTest.php | 2 +- tests/NetzfahrplanV4RegeltestTest.php | 2 +- tests/NetzfahrplanV4RueckmeldungTest.php | 25 ++++ tests/V4ControlTrial/checks.php | 20 ++- tests/V4ControlTrial/feedback_checks.php | 22 +++ tests/V4Feedback/checks.php | 58 +++++++ tests/V4Feedback/fallback_checks.php | 23 +++ tests/V4Feedback/message_checks.php | 23 +++ tests/V4Feedback/receiver_checks.php | 43 ++++++ 24 files changed, 891 insertions(+), 7 deletions(-) create mode 100644 docs/netplan-v4-corrected-feedback.md create mode 100644 examples/V4CorrectedFeedback/MANIFEST.json create mode 100644 examples/V4CorrectedFeedback/build_stage.py create mode 100644 examples/V4CorrectedFeedback/feedback-config.json create mode 100644 examples/V4CorrectedFeedback/file_installer.php create mode 100644 examples/V4CorrectedFeedback/install.php create mode 100644 libs/BatterieNetzfahrplanV4RueckmeldungTrait.php create mode 100644 libs/NetzfahrplanV4Rueckmeldung.php create mode 100644 tests/NetzfahrplanV4RueckmeldungTest.php create mode 100644 tests/V4ControlTrial/feedback_checks.php create mode 100644 tests/V4Feedback/checks.php create mode 100644 tests/V4Feedback/fallback_checks.php create mode 100644 tests/V4Feedback/message_checks.php create mode 100644 tests/V4Feedback/receiver_checks.php diff --git a/Batterie/module.php b/Batterie/module.php index 8510535..d5e35e1 100644 --- a/Batterie/module.php +++ b/Batterie/module.php @@ -9,6 +9,7 @@ require_once __DIR__ . '/../libs/BatterieRegler.php'; require_once __DIR__ . '/../libs/BatterieNetzfahrplanV4TestTrait.php'; use Belevo\EnelixEMS\BatterieRegler; +use Belevo\EnelixEMS\NetzfahrplanV4Rueckmeldung; use Belevo\EnelixEMS\Nachrichtenvertrag; use Belevo\EnelixEMS\VerbraucherBasisTrait; use Belevo\EnelixEMS\VerbraucherSchnittstelle; @@ -197,6 +198,10 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle !$this->ReadPropertyBoolean('NachladenMitMaximalleistung') ); + $formular['elements'][]=['type'=>'ExpansionPanel','caption'=>'V4 physische Rueckmeldung (keine Freigabe)', 'items'=>[ + ['type'=>'Label','caption'=>'Quellenzuordnung fuer Vorschau und separaten Regeltest. Eine Modellaufteilung ist keine separate EV-/SDL-Messung.'], + ['type'=>'ValidationTextBox','name'=>'NetzfahrplanV4RueckmeldungKonfiguration','caption'=>'Versionierte Quellenzuordnung (JSON)'] + ]]; return json_encode($formular, JSON_THROW_ON_ERROR); } @@ -392,6 +397,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->aktualisiereEnergie($jetzt); $messwerte = $this->leseMesswerte(); $messwerteGueltig = $messwerte['Gueltig']; + $this->SetBuffer('V4FeedbackEstimated', ($messwerte['V4FeedbackEstimated'] ?? false) ? '1' : '0'); $this->setzeZustand('Ladezustand', $messwerte['Ladezustand']); $this->setzeZustand('LadezustandGueltig', $messwerteGueltig); @@ -402,7 +408,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->setzeSichtbareVariable( 'Leistungsquelle', $messwerteGueltig - ? Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN + ? (($messwerte['V4FeedbackEstimated'] ?? false) ? Nachrichtenvertrag::LEISTUNGSQUELLE_BERECHNET : Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) : Nachrichtenvertrag::LEISTUNGSQUELLE_NICHT_VORHANDEN ); $this->aktualisiereLadestatus( @@ -527,6 +533,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $gueltig = true; foreach ($zuordnung as $property => $schluessel) { + if ($this->v4BatterieTestAktiv() && in_array($schluessel,['Netzleistung','Istleistung'],true)) continue; $variablenID = $this->ReadPropertyInteger($property); if (!$this->istMesswertAktuell($variablenID)) { $gueltig = false; @@ -535,6 +542,15 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $werte[$schluessel] = (float) GetValue($variablenID); } + if ($this->v4BatterieTestAktiv()) { + try { + $f=$this->v4BatterieRueckmeldung(); + NetzfahrplanV4Rueckmeldung::requireTrial($f,$this->v4BatterieTestSitzung(),time()); + $werte['Istleistung']=$f['batteryW'];$werte['Netzleistung']=$f['gridW']; + $werte['V4FeedbackEstimated']=$f['estimated']; + } catch (Throwable $e) { $gueltig=false; } + } + if ($werte['MaxLaden'] <= 0.0 || $werte['MaxEntladen'] <= 0.0 || $werte['Ladezustand'] < 0.0 @@ -714,7 +730,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ? (float) $this->leseZustand('Istleistung') : null, 'Leistungsquelle' => $messwerteGueltig - ? Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN + ? ($this->GetBuffer('V4FeedbackEstimated') === '1' ? Nachrichtenvertrag::LEISTUNGSQUELLE_BERECHNET : Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) : Nachrichtenvertrag::LEISTUNGSQUELLE_NICHT_VORHANDEN, 'Zustand' => [ [ diff --git a/Manager/module.php b/Manager/module.php index 82ab561..c12c6c6 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -724,7 +724,13 @@ class Manager extends IPSModule implements ManagerSchnittstelle $lock = 'ENELIX.V4.Control.' . $this->InstanceID; if (!IPS_SemaphoreEnter($lock, 0)) return; try { + $trialBefore=(bool)$this->v4ManagerTestSession(); $this->regelnIntern($erneuern); + if ($trialBefore && !$this->v4ManagerTestSession() && $this->GetValue('Aktiv') + && IPS_GetInstance($this->InstanceID)['InstanceStatus'] === self::STATUS_AKTIV) { + $this->WriteAttributeString('LetzteSollwerte','{}'); + $this->regelnIntern(true); // exactly one fresh fallback, no cached pre-trial target + } if ($this->v4ManagerTestSession() && (!$this->GetValue('Aktiv') || IPS_GetInstance($this->InstanceID)['InstanceStatus'] !== self::STATUS_AKTIV)) { $this->v4ManagerTestAbbrechen('Manager nicht mehr betriebsbereit'); diff --git a/docs/netplan-v4-corrected-feedback.md b/docs/netplan-v4-corrected-feedback.md new file mode 100644 index 0000000..527d837 --- /dev/null +++ b/docs/netplan-v4-corrected-feedback.md @@ -0,0 +1,108 @@ +# V4 corrected physical feedback -> local bounded control + +## Scope of this delivery + +The shared physical feedback reader is now connected to the actual Manager preview, +the existing explicit bounded-trial command path, and an independent reread in the +Battery module immediately before hardware output. This is not a new observer. +The installer leaves both local trial permissions disabled and cannot arm a server +trial. Normal operation continues through the existing local controller. + +Lihrenmoos binding: Manager 17004, Battery 44234, virtual asset +`anlage01-virtual-ev`. Effective 161.44 kWh / 39 kW unchanged. Source definitions +come from the existing versioned capture config and current saved topology, not +from new guessed device models. No old histories, sender cursors, virtual energy +accounts, SDL requests, archive policies or inverter polling are changed. + +## Feedback and meaning + +`NetzfahrplanV4Rueckmeldung` checks original value timestamps and object identities +in two read passes. Live max age is bounded to 60s and inter-source skew to 30s. +History interpolation/publication estimates are not accepted as live feedback. +The mapping fingerprint normalizes numbers and ordering, so a JSON property round +trip does not create a new identity. A mapping change cancels an active session. + +Two adapters exist: sum of explicitly assigned physical meters, and a virtual +EV/SDL partition model. For the latter, gateway state and current requests are +read but unchanged zero commands are not mistaken for failed sensor heartbeats. +When SDL request is exactly zero, current physical battery power is used rather +than the filtered virtual EV display. A new EV request does not invent an immediate +physical response or force the measured sign toward the desired sign. + +When SDL is nonzero, source timestamps must cover the latest request change and +physical tracking error must remain within the configured small tolerance. The +partition is explicitly estimated, not separately measured. Opposing EV/SDL flows +cannot be uniquely identified; they remain ineligible for trial control. Unknown +or stale physical sources never fall back to held filtered EV values. + +`allowEstimatedForTrial` is false in the prepared Lihrenmoos config. Configuring +feedback or seeing a valid preview therefore does NOT accept the estimate for a +trial and does NOT satisfy accounting/device-watchdog evidence or other gates. +This adapter does not claim to resolve arbitrary simultaneous SDL activity. + +## Actual control integration + +The same coherent grid/battery snapshot is used in the Manager. Battery command +is a TOTAL power, not a delta added to the existing command. Planned changes in +other consumers are counted once and remain explicitly separate from measured grid. + +A separate explicit server authority, both local consents, accounting evidence, +device-watchdog evidence, a deliberate session start and fresh plan remain required. +The existing trial is limited to 1800s / 5000W per direction; individual command +leases remain <=10s and cannot be renewed by replay or ordinary manager messages. +The local feedback mapping fingerprint and grid caps are bound to the session. + +Immediately before writing, Battery rereads physical feedback and applies current +SOC, reserve, hysteresis, availability and change-lock constraints. Its resulting +command must still meet the bound grid limits; an unreachable target revokes the +trial rather than claiming that the planned grid value was achieved. It cannot +claim physical performance merely because a register call returned successfully. + +Abort revokes the lease and attempts zero, and the Manager wrapper then runs at +most one fresh ordinary allocation, clearing cached pre-trial targets. It does +not recurse forever or revive an old plan. A failed stop stays explicitly failed. +The watchdog still requires a functioning kernel. Independent hardware failure +behavior is NOT certified by software tests. + +Outside a V4 session the existing battery power path is unchanged. The compatible +BatterieRegler library includes the already developed optional reserve charging +limit; with its default (maximal charge) 13,824 old/new ordinary-offer test cases +match exactly. That comparison is not an on-device dynamics test. + +## Installation + +A single prepared Symcon script is the next runtime action: + + require '/srv/agent/netplan-v4-feedback-stage/install.php'; + +It hash-checks nine changed files and all version-matched dependencies, backs up +existing source, installs dependencies before modules and reloads the ENELIX +library. Reload/ApplyChanges may execute existing initialization routines; this +is not promised to be a zero-effect library reload. It configures only the new +feedback mapping on Battery. No server redeployment is required. No shared classes +are included from staging, avoiding the earlier duplicate-class problem. + +A delayed module-registration result requests one repeat. A valid installation +can still report an unavailable feedback sample; that is not reinterpreted as zero. +Any preexisting trial permission, unsaved change, or concurrent source change stops +the installer. Source write failure rolls back its own changed files; reload errors +are reported for review and do not automatically authorize anything. + +## Validation and remaining acceptance + +125 isolated PHP functional checks cover the reader, real receiver, actual trial +traits with a simulated register driver, the extracted actual Manager fallback +wrapper, and the real Battery message builder with optional diagnostic variables absent. +Source quality is stored internally; a last partition estimate is not relabelled as a +measured value just because a trial ends or a diagnostic variable is hidden. Four full module linkage checks and ten isolated installer scenarios also +pass. Full PHP syntax and dependency hashes are checked. Neither these fixtures nor +the ordinary-offer comparison run the real Symcon kernel or an inverter. + +Evidence: test host `/srv/agent/netplan-v4-feedback-stage/PREPARATION.json`, +`TEST_RESULTS.txt`, `INSTALLER_TEST_RESULTS.json`, `ORDINARY_OFFER_TEST.json`. + +Runtime installation, real feedback reception, and a separately authorized field +trial remain outstanding. This finishes the code connection for bounded control; +it is NOT an unrestricted continuous-production controller or a hardware +commissioning certificate. Corrected model selection and actual real-world savings +must be checked against their own data. Existing safety gates are not bypassed. diff --git a/examples/V4CorrectedFeedback/MANIFEST.json b/examples/V4CorrectedFeedback/MANIFEST.json new file mode 100644 index 0000000..a9e7c3b --- /dev/null +++ b/examples/V4CorrectedFeedback/MANIFEST.json @@ -0,0 +1,65 @@ +{ + "scope": "corrected_feedback_trial_disabled", + "managerId": 17004, + "batteryInstanceId": 44234, + "files": { + "libs/BatterieNetzfahrplanV4RueckmeldungTrait.php": { + "before": null, + "after": "24cf0654f1a6bd545d64ce5f115f309b16d8ffca5172ec64fb37c390c6f001a7" + }, + "libs/BatterieNetzfahrplanV4TestTrait.php": { + "before": null, + "after": "80474ff24cf9b2d32dfe481f0ce26bb94584c458b0fa3de232dec3897b64a4b2" + }, + "libs/BatterieRegler.php": { + "before": "035988c54d4d0cc3a59d234f1c08a8edbf5fcac42e09a6bcea61d3dd8270771b", + "after": "beecc9c817b722336367e1c574c1a38afc8f3f5346442170d67c101b980adf7b" + }, + "libs/ManagerNetzfahrplanV4EmpfangTrait.php": { + "before": "d083cfa6ae8e8026c134d2b53f34bb6fbe342fb7332a3e7da1b881de33e13e97", + "after": "936939a3e4a5a5d54fdff68f6fc689a9400e8406aecd5ac09ebc8a6b569c0b66" + }, + "libs/ManagerNetzfahrplanV4TestTrait.php": { + "before": null, + "after": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18" + }, + "libs/NetzfahrplanV4Regeltest.php": { + "before": null, + "after": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249" + }, + "libs/NetzfahrplanV4Rueckmeldung.php": { + "before": null, + "after": "a7459a208c4dd84ffe3a166d266dcd00216f3660fe60e5ee61d5042d0b840631" + }, + "Batterie/module.php": { + "before": "34503645f7e21dd1e5bf0a3f754f6592175dd67f9a263bf431f3801209555401", + "after": "3d5fcd83e4e7d999d5ac949fe357b492d5661ffa76639d2ca57a3f3934eabdbe" + }, + "Manager/module.php": { + "before": "83e80883c8f9b817c0af5e239b20bcb019c7cf7f7943ee5bcc6493ac5a743532", + "after": "cbbb33a8f71dda8cc80a4fc01cd3d6154da8153b5523306b2da723bb44f2dd79" + } + }, + "configurationSha256": "f66cfb9ebf5e02259b902f876a4c78756a51b3f2f2c240fd92c0f01161268c56", + "dependencies": { + "libs/NetzfahrplanV4Planpruefung.php": "24b935d80bad4ce2d9047b6777aaf74a18773236ca91a2844a0907012f4fadf4", + "libs/Nachrichtenvertrag.php": "215ad7d4c5240b39b48cb7672f5057afa532aafdb34039862a1a22985a9ee839", + "libs/VerbraucherBasisTrait.php": "10fd9f54283c378104bce11d623d5581558355d26fc2f85bf04b316542874507", + "libs/VerbraucherSchnittstelle.php": "6329d423dca7b551f01fd884204be45be43f92cd6da6e4faedf2c3c9b4654a15", + "libs/ManagerNetzfahrplanV4Trait.php": "e27a3595e4ed8cdb62e2561149ef6a50adb9194e0239eb25c1216aee2716fa88", + "libs/ManagerNetzfahrplanV4DatenTrait.php": "59ae0ae283839ca82c58db9c071dcf28d0ab95e44f7a74c6473aca837ce416c4", + "libs/NetzfahrplanV4Datenarchiv.php": "57ef7a60c273c5270b3e93ff5528d7d294e708725c9edfe81ac7ed819587f983", + "libs/NetzfahrplanV4Messaufnahme.php": "ebe873bb714041e505a4f500b34c9bae3a5a0b93ac459161584ced758507a6e9", + "libs/NetzfahrplanV4Bilanzierung.php": "ffda2a03dbb226a4405a2ad66704c080a9dc475ce64282ad3f0d7d60034ab199", + "libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "libs/ManagerEnergieTrait.php": "2f89b0ad31ee6dd99180a7225173036eb6212b2ea7157e57fee23f0f8b441a51", + "libs/EnergieMessung.php": "16c64e1cbefde81c25f913aaca6546f0d30cc8f2fddecea0768a3760a98e97d2", + "libs/StoerungsSnapshot.php": "c871c1dd58d7fb1ef6c06985de066518688ef080f3cbdadda192d66cedcf49e1", + "libs/Anlagentopologie.php": "1b7e95d103a3048ed38c464a424f2f12821a147b269e149fd6b89bca84878c47", + "libs/Lizenzpruefung.php": "cb93cb49b93ed3ebb2060dba28b5b2dec2f29548d6ccff1afd9c7cb4624ad77e", + "libs/EinspeiseRegler.php": "8b982be5b95275c6dd6611a2bf2b1f89b991ba53cf94e46df0eed3249add65e0", + "libs/ManagerRegler.php": "608a4abebbe45da37a16316a530fa3f5ab21cb880bb341acbdcd3a9aaa93c6ee", + "libs/ManagerSchnittstelle.php": "0aae8c8c85eb252c577fcb0b5a50a12f3247e4a0bb083579122854f6c35878b3" + } +} diff --git a/examples/V4CorrectedFeedback/build_stage.py b/examples/V4CorrectedFeedback/build_stage.py new file mode 100644 index 0000000..e9ab97c --- /dev/null +++ b/examples/V4CorrectedFeedback/build_stage.py @@ -0,0 +1,25 @@ +"""Recreate a reviewed Lihrenmoos package from this commit; never install in Symcon.""" +from pathlib import Path +import hashlib,json,sys +here=Path(__file__).resolve().parent;repo=here.parents[1] +if len(sys.argv)!=2:raise SystemExit('Usage: python3 build_stage.py NEW_PRIVATE_STAGING_DIRECTORY') +target=Path(sys.argv[1]).resolve() +allowed=[Path('/srv/agent'),Path('/home/agent/services/qa')] +if not any(target!=p and target.is_relative_to(p) for p in allowed):raise SystemExit('Use a private agent staging directory, not live module paths') +if target.exists():raise SystemExit('Destination must not exist') +m=json.loads((here/'MANIFEST.json').read_text());contents={} +for n,v in m['files'].items(): + p=repo/n + if p.is_symlink() or not p.resolve().is_relative_to(repo):raise SystemExit('Unsafe source') + data=p.read_bytes() + if hashlib.sha256(data).hexdigest()!=v['after']:raise SystemExit('Reviewed source changed: '+n) + contents['source/'+n]=data +for n,v in m['dependencies'].items(): + if hashlib.sha256((repo/n).read_bytes()).hexdigest()!=v:raise SystemExit('Dependency changed: '+n) +for n in ['install.php','file_installer.php','MANIFEST.json','feedback-config.json']:contents[n]=(here/n).read_bytes() +if hashlib.sha256(contents['feedback-config.json']).hexdigest()!=m['configurationSha256']:raise SystemExit('Configuration changed') +target.mkdir(parents=True,mode=0o700) +for n,data in contents.items(): + p=target/n;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(data);p.chmod(0o640) +(target/'PACKAGE_HASHES.json').write_text(json.dumps({n:hashlib.sha256(v).hexdigest() for n,v in contents.items()},indent=2)+'\n') +print('Private staging package created:',target,'; no running module, settings or control changes.') diff --git a/examples/V4CorrectedFeedback/feedback-config.json b/examples/V4CorrectedFeedback/feedback-config.json new file mode 100644 index 0000000..4b5f04a --- /dev/null +++ b/examples/V4CorrectedFeedback/feedback-config.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "assetId": "anlage01-virtual-ev", + "managerId": 17004, + "batteryInstanceId": 44234, + "mode": "virtual_split", + "allowEstimatedForTrial": false, + "maxSkewSeconds": 30, + "maxAgeSeconds": 60, + "idleToleranceW": 50.0, + "trackingToleranceW": 200.0, + "sources": [ + { + "key": "grid", + "variableId": 40348, + "parentId": 11490, + "ident": "Power_8", + "factorToW": 1000, + "role": "grid" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "parentId": 19742, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "parentId": 57658, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "parentId": 30789, + "ident": "Value", + "factorToW": 1, + "role": "physical" + }, + { + "key": "ev_requested", + "variableId": 19651, + "parentId": 58448, + "ident": "Nennleistung_Soll_EV", + "factorToW": 1.0, + "role": "ev_request" + }, + { + "key": "sdl_requested", + "variableId": 38943, + "parentId": 58448, + "ident": "Nennleistung_Soll_SDL", + "factorToW": 1.0, + "role": "sdl_request" + }, + { + "key": "gateway_active", + "role": "gateway_active", + "variableId": 23483, + "parentId": 58448, + "ident": "State", + "factorToW": 1.0 + } + ] +} diff --git a/examples/V4CorrectedFeedback/file_installer.php b/examples/V4CorrectedFeedback/file_installer.php new file mode 100644 index 0000000..a5ba371 --- /dev/null +++ b/examples/V4CorrectedFeedback/file_installer.php @@ -0,0 +1,66 @@ +$hash) { + if (!preg_match('~^libs/[A-Za-z0-9]+\.php$~D',$name) || is_link($target.'/'.$name) || hash_file('sha256',$target.'/'.$name)!==$hash) throw new RuntimeException('Abhaengigkeit wurde parallel geaendert: '.$name); + } + $before=[];$source=[];$already=true; + // Explicit dependency order. Both top-level modules are replaced only after all dependencies. + foreach($allowed as$name) { + $path=$target.'/'.$name;$candidate=$stage.'/source/'.$name; + if (is_link($path)||is_link($candidate)||realpath(dirname($path))!==$target.'/'.dirname($name)) throw new RuntimeException('Unerwarteter Dateipfad.'); + $source[$name]=(string)file_get_contents($candidate); + if (hash('sha256',$source[$name])!==$m['files'][$name]['after']) throw new RuntimeException('Paketpruefsumme geaendert: '.$name); + token_get_all($source[$name],TOKEN_PARSE); + $before[$name]=is_file($path)?file_get_contents($path):null; + $hash=$before[$name]===null?null:hash('sha256',$before[$name]); + if (!in_array($hash,[$m['files'][$name]['before'],$m['files'][$name]['after']],true)) throw new RuntimeException('Paralleler Modulstand; nichts ueberschrieben: '.$name); + if($hash!==$m['files'][$name]['after'])$already=false; + } + if($already)return ['status'=>'already_installed','backup'=>null]; + $backup=$stage.'/backups/'.gmdate('Ymd\THis\Z').'-'.bin2hex(random_bytes(4)); + if(!mkdir($backup,0700,true))throw new RuntimeException('Sicherung fehlgeschlagen.'); + foreach($allowed as$name) { + if($before[$name]!==null) { + $dest=$backup.'/'.$name;if(!is_dir(dirname($dest)))mkdir(dirname($dest),0700,true); + v4FeedbackWrite($dest,$before[$name],0600); + } + } + v4FeedbackWrite($backup.'/MANIFEST.json',json_encode($m,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0600); + $written=[]; + try { + foreach($allowed as$name) { + if((is_file($target.'/'.$name)?file_get_contents($target.'/'.$name):null)!==$before[$name])throw new RuntimeException('Parallele Aenderung waehrend Installation.'); + if($before[$name]===$source[$name])continue; + v4FeedbackWrite($target.'/'.$name,$source[$name]);$written[]=$name; + } + } catch(Throwable $e) { + foreach(array_reverse($written)as$name) { + $p=$target.'/'.$name; + if(!is_link($p)&&is_file($p)&&hash_file('sha256',$p)===$m['files'][$name]['after']) { + if($before[$name]===null)unlink($p);else v4FeedbackWrite($p,$before[$name]); + } + } + throw $e; + } + return ['status'=>'installed','backup'=>$backup,'filesChanged'=>$written]; +} diff --git a/examples/V4CorrectedFeedback/install.php b/examples/V4CorrectedFeedback/install.php new file mode 100644 index 0000000..7f01294 --- /dev/null +++ b/examples/V4CorrectedFeedback/install.php @@ -0,0 +1,68 @@ +'corrected_feedback_trial_disabled','startedAt'=>gmdate('c'),'actuatorPermissionGranted'=>false,'serverUpdateRequired'=>false]; +try { + $hashes=json_decode((string)file_get_contents(__DIR__.'/PACKAGE_HASHES.json'),true,32,JSON_THROW_ON_ERROR); + foreach($hashes as$name=>$hash) { + if(str_contains($name,'..')||str_starts_with($name,'/')||is_link(__DIR__.'/'.$name)||hash_file('sha256',__DIR__.'/'.$name)!==$hash)throw new RuntimeException('Vorbereitetes Paket veraendert.'); + } + foreach([$manager=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}',$battery=>'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}']as$id=>$module) { + if(!IPS_InstanceExists($id)||IPS_GetInstance($id)['ModuleInfo']['ModuleID']!==$module)throw new RuntimeException('Modulzuordnung ungueltig.'); + if(IPS_HasChanges($id))throw new RuntimeException('Offene Aenderungen zuerst speichern oder verwerfen.'); + } + $beforeM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $beforeB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + if(($beforeM['NetzfahrplanAktiv']??null)!==false || ($beforeM['NetzfahrplanV4RegeltestErlaubt']??false)!==false || ($beforeB['NetzfahrplanV4RegeltestErlaubt']??false)!==false)throw new RuntimeException('Alte Fahrplanausgabe und beide Regeltestfreigaben muessen AUS bleiben.'); + $c=json_decode((string)file_get_contents(__DIR__.'/feedback-config.json'),true,32,JSON_THROW_ON_ERROR); + $manifest=json_decode((string)file_get_contents(__DIR__.'/MANIFEST.json'),true,32,JSON_THROW_ON_ERROR); + if(($c['managerId']??null)!==$manager || ($c['batteryInstanceId']??null)!==$battery || ($c['allowEstimatedForTrial']??null)!==false + || hash_file('sha256',__DIR__.'/feedback-config.json')!==$manifest['configurationSha256'])throw new RuntimeException('Falsche Rueckmeldekonfiguration.'); + $desired=json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION); + $oldMapping=$beforeB['NetzfahrplanV4RueckmeldungKonfiguration']??'{}'; + if(!in_array($oldMapping,['','{}',$desired],true))throw new RuntimeException('Abweichende Rueckmeldekonfiguration nicht ueberschrieben.'); + $controls=[];foreach(IPS_GetModuleList()as$module)if((IPS_GetModule($module)['Prefix']??'')==='MC')$controls=array_merge($controls,IPS_GetInstanceListByModuleID($module)); + if(count($controls)!==1||!function_exists('MC_ReloadModule'))throw new RuntimeException('Module Control nicht eindeutig.'); + $locked=IPS_SemaphoreEnter('ENELIX.V4.FeedbackInstall',3000);if(!$locked)throw new RuntimeException('Installation bereits aktiv.'); + $report['files']=v4FeedbackFiles(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); + if(MC_ReloadModule($controls[0],'Enelix-EMS')===false)throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + $afterM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $afterB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + foreach([[$beforeM,$afterM],[$beforeB,$afterB]]as$pair)foreach($pair[0]as$key=>$value)if(!array_key_exists($key,$pair[1])||$pair[1][$key]!==$value)throw new RuntimeException('Bestehender Parameter beim Reload veraendert: '.$key); + if(!array_key_exists('NetzfahrplanV4RueckmeldungKonfiguration',$afterB)||!array_key_exists('NetzfahrplanV4RegeltestErlaubt',$afterM) + ||!function_exists('ENELIX_GetV4BatterieRueckmeldung')||!function_exists('ENELIX_GetV4ManagerTestStatus')) { + $report['status']='waiting_for_registration'; + echo "Dateien installiert. Modulregistrierung wird abgeschlossen; diesen Aufruf einmal wiederholen. Keine Stellfreigabe.\n"; + } else { + if($afterM['NetzfahrplanV4RegeltestErlaubt']!==false||$afterB['NetzfahrplanV4RegeltestErlaubt']!==false)throw new RuntimeException('Unerwartete Testfreigabe.'); + if($afterB['NetzfahrplanV4RueckmeldungKonfiguration']!==$desired) { + IPS_SetProperty($battery,'NetzfahrplanV4RueckmeldungKonfiguration',$desired);$changedMapping=true;IPS_ApplyChanges($battery); + } + $finalM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $finalB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + foreach([[$beforeM,$finalM],[$beforeB,$finalB]]as$pair)foreach($pair[0]as$key=>$value)if($key!=='NetzfahrplanV4RueckmeldungKonfiguration'&&(!array_key_exists($key,$pair[1])||$pair[1][$key]!==$value))throw new RuntimeException('Bestehender Parameter unerwartet veraendert: '.$key); + if(($finalM['NetzfahrplanAktiv']??null)!==false||($finalM['NetzfahrplanV4RegeltestErlaubt']??null)!==false||($finalB['NetzfahrplanV4RegeltestErlaubt']??null)!==false)throw new RuntimeException('Unerwartete Stellfreigabe nach Installation.'); + $f=json_decode(ENELIX_GetV4BatterieRueckmeldung($battery),true,32,JSON_THROW_ON_ERROR); + $report['feedback']=array_intersect_key($f,array_flip(['status','kind','method','reason','reasons','estimated','batteryW','gridW','sourceOldestAt','checkedAt','configHash','usableForTrial','canDispatch'])); + $report['status']='corrected_feedback_installed_trial_disabled';$report['originalParametersPreserved']=true; + echo "OK: Korrigierte physische Rueckmeldung mit Manager-Vorschau und Batterietreiber verbunden.\n"; + echo 'Rueckmeldung: '.($f['status']??'available').'; Modellanteil: '.(($f['estimated']??false)?'ja':'nein').".\n"; + if(isset($f['reason']))echo 'Aktueller Hinweis: '.$f['reason']."\n"; + echo "Beide Regeltestfreigaben AUS. Keine V4-Ausgabe aktiviert; vorhandene Regelung, Energiekonten und Datensender bleiben bestehen.\n"; + } +} catch(Throwable $e) { + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,260); + if($changedMapping && $oldMapping!==null) { + try{IPS_SetProperty($battery,'NetzfahrplanV4RueckmeldungKonfiguration',$oldMapping);IPS_ApplyChanges($battery);$report['mappingRestored']=true;}catch(Throwable $ignored){$report['mappingRestored']=false;} + } + echo 'FEHLER: '.$report['reason']."\nKeine V4-Stellfreigabe erteilt.\n"; +} finally { + $report['finishedAt']=gmdate('c');v4FeedbackWrite(__DIR__.'/INSTALL_RESULT.json',json_encode($report,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0644); + if($locked)IPS_SemaphoreLeave('ENELIX.V4.FeedbackInstall'); + echo 'FEEDBACK INSTALL REPORT: '.__DIR__."/INSTALL_RESULT.json\n"; +} diff --git a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php new file mode 100644 index 0000000..da72811 --- /dev/null +++ b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php @@ -0,0 +1,38 @@ +RegisterPropertyString('NetzfahrplanV4RueckmeldungKonfiguration', '{}'); + } + + private function v4BatterieRueckmeldung(): array + { + $json=$this->ReadPropertyString('NetzfahrplanV4RueckmeldungKonfiguration'); + if (strlen($json)>65536) throw new \RuntimeException('feedback_configuration_too_large'); + $c=json_decode($json,true,32,JSON_THROW_ON_ERROR); + if (!is_array($c) || ($c['batteryInstanceId']??null)!==$this->InstanceID) throw new \RuntimeException('feedback_battery_binding_invalid'); + if (!in_array($c['managerId']??0,$this->zugeordneteManagerIDs(),true)) throw new \RuntimeException('feedback_manager_binding_invalid'); + return NetzfahrplanV4Rueckmeldung::read($c,static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('feedback_variable_missing'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id);$value=GetValue($id); + return ['value'=>$value,'updated'=>$v['VariableUpdated'],'changed'=>$v['VariableChanged'],'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + },time()); + } + + /** Read only. A returned current value is NOT actuator permission. */ + public function GetV4BatterieRueckmeldung(): string + { + try { $r=$this->v4BatterieRueckmeldung(); } + catch (\Throwable $e) { $r=['kind'=>'local_physical_feedback','status'=>'unavailable','reason'=>substr($e->getMessage(),0,200), + 'batteryW'=>null,'gridW'=>null,'usableForTrial'=>false,'canDispatch'=>false]; } + return json_encode($r,JSON_THROW_ON_ERROR); + } +} diff --git a/libs/BatterieNetzfahrplanV4TestTrait.php b/libs/BatterieNetzfahrplanV4TestTrait.php index 56a20d2..f12a22b 100644 --- a/libs/BatterieNetzfahrplanV4TestTrait.php +++ b/libs/BatterieNetzfahrplanV4TestTrait.php @@ -8,12 +8,15 @@ use RuntimeException; use Throwable; require_once __DIR__ . '/NetzfahrplanV4Regeltest.php'; +require_once __DIR__ . '/BatterieNetzfahrplanV4RueckmeldungTrait.php'; /** Dedicated short-lease battery input. Disabled until explicitly armed on BOTH modules. */ trait BatterieNetzfahrplanV4TestTrait { + use BatterieNetzfahrplanV4RueckmeldungTrait; private function registriereV4BatterieTest(): void { + $this->registriereV4Rueckmeldung(); $this->RegisterPropertyBoolean('NetzfahrplanV4RegeltestErlaubt', false); $this->RegisterAttributeString('NetzfahrplanV4LetzteTestsitzung', ''); $this->RegisterPropertyString('NetzfahrplanV4GeraeteWatchdogNachweis', ''); @@ -69,6 +72,7 @@ trait BatterieNetzfahrplanV4TestTrait if (!$this->GetValue('Aktiv') || $this->ReadPropertyInteger('Batteriemanagement') !== 2) { throw new RuntimeException('Batterie nicht unter ENELIX-Regelung.'); } + NetzfahrplanV4Rueckmeldung::requireTrial($this->v4BatterieRueckmeldung(), $s, $now); $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $s['sessionId']); $s['active'] = true;$s['lastSequence'] = 0;$s['lastWallClockAt'] = $now; $s['monotonicDeadline'] = hrtime(true) / 1e9 + $end - $now; @@ -157,6 +161,8 @@ trait BatterieNetzfahrplanV4TestTrait $s = $this->v4BatterieTestSitzung();$mono = hrtime(true) / 1e9; try { if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt')) throw new RuntimeException('Lokale Testfreigabe entzogen.'); + $feedback = $this->v4BatterieRueckmeldung(); + NetzfahrplanV4Rueckmeldung::requireTrial($feedback, $s, $now); $c = json_decode($this->GetBuffer('V4BatteryTrialCommand') ?: '{}', true); if ($now >= $s['expiresAt'] || $mono >= $s['monotonicDeadline'] || $now < $s['lastWallClockAt']) throw new RuntimeException('Testsitzung abgelaufen/Zeitsprung.'); if (!$c) { @@ -175,6 +181,13 @@ trait BatterieNetzfahrplanV4TestTrait if ($last * $w < 0) $w = 0; elseif (abs($w) > abs($last)) $w = $last; } + // Final hardware-policy/ramp clipping must not invalidate the binding or grid limits. + if (($c['feedbackConfigHash'] ?? null) !== $feedback['configHash']) throw new RuntimeException('feedback_configuration_changed'); + $expectedGrid = $feedback['gridW'] - $feedback['batteryW'] + $w; + if (($c['importLimitW'] !== null && $expectedGrid > $c['importLimitW'] + 1) + || ($c['exportLimitW'] !== null && -$expectedGrid > $c['exportLimitW'] + 1)) { + throw new RuntimeException('final_battery_clamp_cannot_meet_grid_limit'); + } $s['lastWallClockAt'] = $now; $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); return $w; diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php index 0761567..a6fde3f 100644 --- a/libs/ManagerNetzfahrplanV4EmpfangTrait.php +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -8,6 +8,7 @@ use RuntimeException; use Throwable; require_once __DIR__ . '/NetzfahrplanV4Planpruefung.php'; +require_once __DIR__ . '/NetzfahrplanV4Rueckmeldung.php'; /** Independent preview only: no SendData, consumer RequestAction or legacy plan writes. */ trait ManagerNetzfahrplanV4EmpfangTrait @@ -158,6 +159,31 @@ trait ManagerNetzfahrplanV4EmpfangTrait $topology = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 64, JSON_THROW_ON_ERROR); $actual = [];$now = time(); foreach ($topology as $asset) { + $controllers=[]; + foreach ($this->aktiveVerbraucherIDs() as $controller) { + if (IPS_InstanceExists($controller) + && IPS_GetInstance($controller)['ModuleInfo']['ModuleID']==='{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}' + && IPS_GetProperty($controller,'IstleistungVariableID')===($asset['LeistungVariableID']??null) + && IPS_GetProperty($controller,'LadezustandVariableID')===($asset['SOCVariableID']??null)) $controllers[]=$controller; + } + if (count($controllers)>1) throw new RuntimeException('feedback_controller_ambiguous'); + if (count($controllers)===1 && function_exists('ENELIX_GetV4BatterieRueckmeldung')) { + $controller=$controllers[0];$configured=IPS_GetProperty($controller,'NetzfahrplanV4RueckmeldungKonfiguration'); + if (is_string($configured) && !in_array(trim($configured),['','{}'],true)) { + if (count($topology)!==1) throw new RuntimeException('feedback_multi_asset_adapter_required'); + $f=json_decode(ENELIX_GetV4BatterieRueckmeldung($controller),true,32,JSON_THROW_ON_ERROR); + if (($f['installationId']??null)!==$this->ReadAttributeString('LizenzInstallationID') + || ($f['assetId']??null)!==$asset['ID'] || ($f['managerId']??null)!==$this->InstanceID + || ($f['batteryInstanceId']??null)!==$controller || !is_numeric($f['batteryW']??null) + || !is_numeric($f['gridW']??null) || !is_int($f['checkedAt']??null) || $now-$f['checkedAt']>2 || $f['checkedAt']>$now) { + throw new RuntimeException('feedback_invalid_no_legacy_fallback'); + } + $actual[$asset['ID']]=$f['batteryW'];$d['operation']['gridW']=$f['gridW']; + $d['operation']['localFeedback'][$asset['ID']]=$f; + continue; + } + } + // Legacy value remains display-only until an explicit physical feedback mapping exists. $id = (int) ($asset['LeistungVariableID'] ?? 0); if (!IPS_VariableExists($id)) throw new RuntimeException('Batterie-Istleistung fehlt.'); $before = IPS_GetVariable($id);$v = GetValue($id);$after = IPS_GetVariable($id); @@ -189,6 +215,8 @@ trait ManagerNetzfahrplanV4EmpfangTrait $checked = NetzfahrplanV4Planpruefung::pruefen($envelope['response'], $this->ReadAttributeString('LizenzInstallationID'), $operation, time(), $envelope['receivedAt']); $out = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); + $out['localFeedback'] = $operation['localFeedback'] ?? []; + $out['feedbackMeaning'] = isset($operation['localFeedback']) ? 'fresh_physical_or_explicit_partition_model' : 'legacy_display_only_not_for_trial'; $out['reception'] = json_decode($this->ReadAttributeString('NetzfahrplanV4EmpfangStatus'), true); } catch (Throwable $error) { $out = ['status' => 'unavailable', 'reason' => substr($error->getMessage(), 0, 300)]; diff --git a/libs/ManagerNetzfahrplanV4TestTrait.php b/libs/ManagerNetzfahrplanV4TestTrait.php index 7a052e4..aaf419e 100644 --- a/libs/ManagerNetzfahrplanV4TestTrait.php +++ b/libs/ManagerNetzfahrplanV4TestTrait.php @@ -78,6 +78,8 @@ trait ManagerNetzfahrplanV4TestTrait 'expiresAt' => $until, 'maxChargeW' => NetzfahrplanV4Regeltest::number($request['maxChargeW'] ?? null, 'Testladegrenze', 1, 5000), 'maxDischargeW' => NetzfahrplanV4Regeltest::number($request['maxDischargeW'] ?? null, 'Testentladegrenze', 1, 5000), 'actuatorWatchdogEvidenceId' => IPS_GetProperty($batteryID, 'NetzfahrplanV4GeraeteWatchdogNachweis'), + 'feedbackConfigHash' => $op['localFeedback'][$a['assetId']]['configHash'] ?? '', + 'gridLimits' => NetzfahrplanV4Planpruefung::kontext($op)['limits'], 'sequence' => 1, 'lastWallClockAt' => $now, 'monotonicDeadline' => hrtime(true) / 1e9 + $until - $now]; // Validate everything BEFORE arming either module. This rejects all normal shadow replies. @@ -156,6 +158,7 @@ trait ManagerNetzfahrplanV4TestTrait } if (!$found) throw new RuntimeException('Testbatterie nicht mehr in aktiver Zuordnung.'); $op['gridW'] += $otherDelta; + $op['localPredictedOtherDeltaW']=$otherDelta; if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { throw new RuntimeException('Kein gueltiger empfangener Testkontext.'); } diff --git a/libs/NetzfahrplanV4Regeltest.php b/libs/NetzfahrplanV4Regeltest.php index a377053..00e6d4f 100644 --- a/libs/NetzfahrplanV4Regeltest.php +++ b/libs/NetzfahrplanV4Regeltest.php @@ -7,6 +7,7 @@ namespace Belevo\EnelixEMS; use InvalidArgumentException; require_once __DIR__ . '/NetzfahrplanV4Planpruefung.php'; +require_once __DIR__ . '/NetzfahrplanV4Rueckmeldung.php'; /** Separate, explicitly authorized commissioning commands. A preview is NEVER permission. */ final class NetzfahrplanV4Regeltest @@ -87,6 +88,11 @@ final class NetzfahrplanV4Regeltest if (($a['controlContext'] ?? null) != $plan['controlContext']) { throw new InvalidArgumentException('Testkonfiguration stimmt nicht mehr.'); } + $feedback = $operation['localFeedback'][$a['assetId']] ?? []; + NetzfahrplanV4Rueckmeldung::requireTrial($feedback, $local, $now); + if (abs(self::number($actual[$a['assetId']] ?? null,'Batterie Ist',-1e9,1e9) - self::number($feedback['batteryW'],'Rueckmeldung',-1e9,1e9)) > 0.000001) throw new InvalidArgumentException('feedback_power_not_from_verified_snapshot'); + $otherDelta=self::number($operation['localPredictedOtherDeltaW']??0.0,'Andere Verbraucherabweichung',-1e9,1e9); + if (abs($operation['gridW']-$feedback['gridW']-$otherDelta)>0.001) throw new InvalidArgumentException('feedback_grid_not_from_same_snapshot'); $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, $now); $charge = min(self::number($a['maxChargeW'] ?? null, 'Server-Ladegrenze', 1, self::MAX_TEST_POWER_W), self::number($local['maxChargeW'] ?? null, 'Lokale Ladegrenze', 1, self::MAX_TEST_POWER_W)); @@ -109,6 +115,9 @@ final class NetzfahrplanV4Regeltest 'watts' => $watts, 'maxChargeW' => (int) $charge, 'maxDischargeW' => (int) $discharge, 'sourceShadowPlanId' => $plan['planId'], 'sourceStep' => $p['time'], 'revision' => $plan['configRevision'], 'expectedGridW' => $grid, + 'feedbackConfigHash' => $feedback['configHash'], 'feedbackEstimated' => $feedback['estimated'], + 'feedbackSourceOldestAt' => $feedback['sourceOldestAt'], + 'importLimitW' => $p['importLimitW'], 'exportLimitW' => $p['exportLimitW'], 'executionMeaning' => 'separate_local_control_trial_not_shadow_plan_execution']; } @@ -139,6 +148,23 @@ final class NetzfahrplanV4Regeltest if ($c > $session['maxChargeW'] || $d > $session['maxDischargeW'] || $w > $c || $w < -$d) { throw new InvalidArgumentException('Befehl ueberschreitet lokale Testgrenze.'); } + if (!is_string($command['feedbackConfigHash'] ?? null) || !preg_match('/^[a-f0-9]{64}$/D', $command['feedbackConfigHash']) + || $command['feedbackConfigHash'] !== ($session['feedbackConfigHash'] ?? null)) throw new InvalidArgumentException('feedback_command_binding_invalid'); + foreach (['importLimitW','exportLimitW'] as $limit) { + if (!array_key_exists($limit,$command)) throw new InvalidArgumentException('feedback_command_limits_missing'); + if ($command[$limit] !== null) self::number($command[$limit],$limit,0,1e9); + } + $l=$session['gridLimits'] ?? null; + if (!is_array($l) || !array_key_exists('importW',$l) || !array_key_exists('exportW',$l) || !is_array($l['managerMonthLimitsW']??null)) throw new InvalidArgumentException('session_grid_limits_missing'); + $month=(int)(new \DateTimeImmutable('@'.$now))->setTimezone(new \DateTimeZone('Europe/Zurich'))->format('n'); + $import=$l['importW'];$monthly=$l['managerMonthLimitsW'][$month]??null; + if ($monthly!==null) $import=$import===null?$monthly:min($import,$monthly); + foreach (['importLimitW'=>$import,'exportLimitW'=>$l['exportW']] as $key=>$limit) { + if ($limit!==null) { + self::number($limit,$key,0,1e9); + if ($command[$key]===null || $command[$key]>$limit) throw new InvalidArgumentException('command_relaxes_local_grid_limit'); + } + } self::uuid($command['sourceShadowPlanId'] ?? null); } diff --git a/libs/NetzfahrplanV4Rueckmeldung.php b/libs/NetzfahrplanV4Rueckmeldung.php new file mode 100644 index 0000000..1ecd110 --- /dev/null +++ b/libs/NetzfahrplanV4Rueckmeldung.php @@ -0,0 +1,141 @@ +$hi) { + throw new InvalidArgumentException('feedback_numeric_invalid'); + } + return (float)$v; + } + + private static function ordered(array $v): array + { + if (!array_is_list($v)) ksort($v, SORT_STRING); + foreach ($v as &$x) if (is_array($x)) $x=self::ordered($x); + unset($x);return $v; + } + + public static function configuration(array $c): array + { + $keys=['version','installationId','assetId','managerId','batteryInstanceId','mode','allowEstimatedForTrial', + 'maxSkewSeconds','maxAgeSeconds','idleToleranceW','trackingToleranceW','sources']; + if (count($c)!==count($keys) || array_diff($keys,array_keys($c)) || $c['version']!==1 + || !is_string($c['installationId']) || !preg_match('/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/D',$c['installationId']) + || !is_string($c['assetId']) || $c['assetId']==='' || strlen($c['assetId'])>100 + || !in_array($c['mode'],['physical_sum','virtual_split'],true) || !is_bool($c['allowEstimatedForTrial'])) { + throw new InvalidArgumentException('feedback_configuration_invalid'); + } + foreach (['managerId','batteryInstanceId'] as $k) if (!is_int($c[$k]) || $c[$k]<1 || $c[$k]>99999) throw new InvalidArgumentException('feedback_binding_invalid'); + if ($c['managerId']===$c['batteryInstanceId']) throw new InvalidArgumentException('feedback_binding_invalid'); + foreach (['maxSkewSeconds'=>30,'maxAgeSeconds'=>60] as $k=>$max) { + if (!is_int($c[$k]) || $c[$k]<1 || $c[$k]>$max) throw new InvalidArgumentException('feedback_freshness_invalid'); + } + $c['idleToleranceW']=self::number($c['idleToleranceW'],0,100); + $c['trackingToleranceW']=self::number($c['trackingToleranceW'],0,500); + if (!is_array($c['sources']) || !array_is_list($c['sources']) || count($c['sources'])<2 || count($c['sources'])>40) throw new InvalidArgumentException('feedback_sources_invalid'); + $ids=[];$names=[];$roles=[]; + foreach ($c['sources'] as &$s) { + $fields=['key','role','variableId','parentId','ident','factorToW']; + if (!is_array($s) || count($s)!==count($fields) || array_diff($fields,array_keys($s)) + || !is_string($s['key']) || !preg_match('/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/D',$s['key']) + || isset($names[$s['key']]) || !in_array($s['role'],['grid','physical','ev_request','sdl_request','gateway_active'],true) + || !is_int($s['variableId']) || $s['variableId']<1 || $s['variableId']>99999 || isset($ids[$s['variableId']]) + || !is_int($s['parentId']) || $s['parentId']<1 || !is_string($s['ident']) || strlen($s['ident'])>100) throw new InvalidArgumentException('feedback_source_identity_invalid'); + $s['factorToW']=self::number($s['factorToW'],-1e6,1e6); + if ($s['factorToW']==0 || ($s['role']==='gateway_active' && $s['factorToW']!==1.0)) throw new InvalidArgumentException('feedback_factor_invalid'); + $ids[$s['variableId']]=true;$names[$s['key']]=true;$roles[$s['role']]=($roles[$s['role']]??0)+1; + } + unset($s); + if (($roles['grid']??0)!==1 || ($roles['physical']??0)<1) throw new InvalidArgumentException('feedback_measurements_required'); + foreach (['ev_request','sdl_request','gateway_active'] as $role) { + if (($roles[$role]??0)!==($c['mode']==='virtual_split'?1:0)) throw new InvalidArgumentException('feedback_partition_sources_required'); + } + usort($c['sources'],static fn($a,$b)=>strcmp($a['key'],$b['key'])); + return self::ordered($c); + } + + public static function fingerprint(array $c): string + { + return hash('sha256',json_encode(self::configuration($c),JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + } + + /** Reader supplies value, original updated/changed, parentID, ident. Never trusts collector timestamp alone. */ + public static function read(array $config, callable $reader, int $now): array + { + $c=self::configuration($config); + if ($now<1) throw new InvalidArgumentException('feedback_time_invalid'); + $first=[];$values=[];$times=[];$physicalTimes=[];$changed=[]; + foreach ($c['sources'] as $s) { + $r=$reader($s['variableId']); + if (!is_array($r) || ($r['parentID']??null)!==$s['parentId'] || ($r['ident']??null)!==$s['ident'] + || !is_int($r['updated']??null) || $r['updated']<1 || $r['updated']>$now + || !is_int($r['changed']??null) || $r['changed']<0 || $r['changed']>$r['updated']) throw new InvalidArgumentException('feedback_origin_invalid:'.$s['key']); + $first[$s['key']]=array_intersect_key($r,array_flip(['value','updated','changed','parentID','ident'])); + if ($s['role']==='gateway_active') { + if (($r['value']??null)!==true) throw new InvalidArgumentException('feedback_gateway_inactive'); + } else { + $w=self::number($r['value']??null,-1e9,1e9)*$s['factorToW']; + $values[$s['role']]=($values[$s['role']]??0)+self::number($w,-1e9,1e9); + } + // An unchanged command is held STATE, not proof of a fresh sensor or an actuator watchdog. + if (in_array($s['role'],['grid','physical'],true)) { + if ($now-$r['updated']>$c['maxAgeSeconds']) throw new InvalidArgumentException('feedback_stale_physical:'.$s['key']); + $times[]=$r['updated'];if ($s['role']==='physical') $physicalTimes[]=$r['updated']; + } else $changed[$s['role']]=$r['changed']; + } + foreach ($c['sources'] as $s) { + $second=$reader($s['variableId']); + if (!is_array($second) || array_intersect_key($second,array_flip(['value','updated','changed','parentID','ident']))!==$first[$s['key']]) { + throw new InvalidArgumentException('feedback_changed_during_read:'.$s['key']); + } + } + if (max($times)-min($times)>$c['maxSkewSeconds']) throw new InvalidArgumentException('feedback_source_skew'); + $physical=(float)$values['physical'];$power=$physical;$external=0.0;$estimated=$c['mode']==='virtual_split';$reasons=[];$tracking=null; + if ($estimated) { + if ($changed['gateway_active']>min($physicalTimes) || $changed['sdl_request']>min($physicalTimes)) throw new InvalidArgumentException('feedback_external_transition_not_observed'); + $ev=(float)$values['ev_request'];$sdl=(float)$values['sdl_request'];$tracking=$physical-$ev-$sdl; + if ($sdl===0.0) { + // No external commanded contribution. Physical response may lag a NEW EV command. + $power=$physical; + if ($ev===0.0 && abs($physical)>$c['idleToleranceW']) throw new InvalidArgumentException('feedback_unassigned_idle_power'); + } else { + if ($changed['ev_request']>min($physicalTimes)) throw new InvalidArgumentException('feedback_ev_transition_not_observed'); + if (abs($tracking)>$c['trackingToleranceW']) throw new InvalidArgumentException('feedback_partition_not_tracking'); + $power=$ev+$tracking*abs($ev)/(abs($ev)+abs($sdl));$external=$physical-$power; + if ($ev*$sdl<0) $reasons[]='opposing_virtual_flows_not_identifiable'; + } + if (!$c['allowEstimatedForTrial']) $reasons[]='modelled_feedback_not_accepted_for_trial'; + } + return ['version'=>1,'kind'=>'local_physical_feedback','installationId'=>$c['installationId'],'assetId'=>$c['assetId'], + 'managerId'=>$c['managerId'],'batteryInstanceId'=>$c['batteryInstanceId'],'checkedAt'=>$now, + 'sourceOldestAt'=>min($times),'sourceNewestAt'=>max($times),'physicalOldestAt'=>min($physicalTimes), + 'sourceSkewSeconds'=>max($times)-min($times),'maxAgeSeconds'=>$c['maxAgeSeconds'], + 'configHash'=>self::fingerprint($c),'gridW'=>(float)$values['grid'],'physicalStorageW'=>$physical, + 'batteryW'=>$power,'externalModelW'=>$external,'trackingErrorW'=>$tracking, + 'estimated'=>$estimated,'separatelyMeasured'=>!$estimated,'usableForTrial'=>$reasons===[], + 'reasons'=>$reasons,'method'=>$estimated?'physical_request_partition_unfiltered':'physical_sum', + 'canDispatch'=>false,'gatewayWatchdogVerified'=>false,'heldFilterUsed'=>false]; + } + + public static function requireTrial(array $f, array $binding, int $now): void + { + foreach (['installationId','assetId','managerId','batteryInstanceId'] as $key) { + if (!array_key_exists($key,$binding) || ($f[$key]??null)!==$binding[$key]) throw new InvalidArgumentException('feedback_wrong_binding'); + } + if (($f['kind']??null)!=='local_physical_feedback' || ($f['version']??null)!==1 || ($f['usableForTrial']??null)!==true + || !is_int($f['checkedAt']??null) || $f['checkedAt']>$now || $now-$f['checkedAt']>2 + || !is_int($f['sourceOldestAt']??null) || !is_int($f['maxAgeSeconds']??null) + || $f['maxAgeSeconds']<1 || $f['maxAgeSeconds']>60 || $f['sourceOldestAt']>$now || $now-$f['sourceOldestAt']>$f['maxAgeSeconds'] + || !is_string($f['configHash']??null) || !preg_match('/^[a-f0-9]{64}$/D',$f['configHash'])) throw new InvalidArgumentException('feedback_not_usable_for_trial'); + if (isset($binding['feedbackConfigHash']) && $binding['feedbackConfigHash']!==$f['configHash']) throw new InvalidArgumentException('feedback_configuration_changed'); + self::number($f['batteryW']??null,-1e9,1e9);self::number($f['gridW']??null,-1e9,1e9); + } +} diff --git a/tests/NetzfahrplanV4BilanzierungTest.php b/tests/NetzfahrplanV4BilanzierungTest.php index 6e1145a..2f716db 100644 --- a/tests/NetzfahrplanV4BilanzierungTest.php +++ b/tests/NetzfahrplanV4BilanzierungTest.php @@ -11,6 +11,6 @@ final class NetzfahrplanV4BilanzierungTest extends TestCase $p=proc_open([PHP_BINARY,__DIR__.'/V4Accounting/checks.php'],[0=>['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); fclose($pipes[1]);fclose($pipes[2]);self::assertSame(0,proc_close($p),$out.$err);self::assertSame('',$err); - self::assertStringContainsString('TOTAL 34 checks passed',$out); + self::assertStringContainsString('TOTAL 42 checks passed',$out); } } diff --git a/tests/NetzfahrplanV4DatenTest.php b/tests/NetzfahrplanV4DatenTest.php index d4c69b9..3de873c 100644 --- a/tests/NetzfahrplanV4DatenTest.php +++ b/tests/NetzfahrplanV4DatenTest.php @@ -16,6 +16,6 @@ final class NetzfahrplanV4DatenTest extends TestCase $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); fclose($pipes[1]);fclose($pipes[2]); self::assertSame(0,proc_close($process),$out.$err); - self::assertStringContainsString('TOTAL 21 application-data checks passed',$out); + self::assertStringContainsString('TOTAL 27 application-data checks passed',$out); } } diff --git a/tests/NetzfahrplanV4RegeltestTest.php b/tests/NetzfahrplanV4RegeltestTest.php index a49725f..64dee03 100644 --- a/tests/NetzfahrplanV4RegeltestTest.php +++ b/tests/NetzfahrplanV4RegeltestTest.php @@ -19,6 +19,6 @@ final class NetzfahrplanV4RegeltestTest extends TestCase fclose($pipes[1]);fclose($pipes[2]); self::assertSame(0, proc_close($process), $out . $err); self::assertSame('', $err); - self::assertStringContainsString('TOTAL 59 checks passed.', $out); + self::assertStringContainsString('TOTAL 71 checks passed.', $out); } } diff --git a/tests/NetzfahrplanV4RueckmeldungTest.php b/tests/NetzfahrplanV4RueckmeldungTest.php new file mode 100644 index 0000000..d975f5b --- /dev/null +++ b/tests/NetzfahrplanV4RueckmeldungTest.php @@ -0,0 +1,25 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]); + $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err); + self::assertSame('',$err);self::assertStringContainsString('TOTAL ',$out); + } + public function testPhysicalFeedback(): void {$this->checkScript('checks.php');} + public function testActualReceiverIntegration(): void {$this->checkScript('receiver_checks.php');} + public function testActualFallbackWrapper(): void {$this->checkScript('fallback_checks.php');} + public function testMessageWithoutVisibleDiagnostics(): void {$this->checkScript('message_checks.php');} +} diff --git a/tests/V4ControlTrial/checks.php b/tests/V4ControlTrial/checks.php index eb0d903..d9cfae7 100644 --- a/tests/V4ControlTrial/checks.php +++ b/tests/V4ControlTrial/checks.php @@ -14,6 +14,11 @@ namespace Belevo\EnelixEMS { function IPS_GetInstance($id) { return ['InstanceStatus'=>$GLOBALS['objects'][$id]->ready?102:201, 'ModuleInfo'=>['ModuleID'=>$id===44234?'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}':'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']]; } function IPS_GetProperty($id,$key) { return $GLOBALS['objects'][$id]->props[$key]; } + function IPS_VariableExists($id) { return in_array($id,[50001,50002],true); } + function IPS_GetVariable($id) { $t=$GLOBALS['feedbackAt'][$id]??$GLOBALS['clock'];return ['VariableUpdated'=>$t,'VariableChanged'=>$t]; } + function IPS_GetObject($id) { return ['ParentID'=>200,'ObjectIdent'=>'source'.$id]; } + function GetValue($id) { return $id===50001?$GLOBALS['objects'][17004]->op['gridW']:$GLOBALS['objects'][17004]->actual['b']; } + function IPS_RequestAction($id,$key,$value) { $GLOBALS['calls'][]=[$id,$key];$b=$GLOBALS['objects'][$id]; if ($key==='NetzfahrplanV4TestStart') $b->V4BatterieTestStarten($value); @@ -76,7 +81,10 @@ namespace Belevo\EnelixEMS { private function berechtigungLizenziert($k) { return $this->licensed; } private function actieveDummy() {} private function aktiveVerbraucherIDs() { return [44234]; } - private function netzfahrplanV4LokaleVorschauwerte() { return [$this->op,$this->actual]; } + private function netzfahrplanV4LokaleVorschauwerte() { + $op=$this->op;$f=json_decode($GLOBALS['objects'][44234]->GetV4BatterieRueckmeldung(),true); + $op['localFeedback']=['b'=>$f];return [$op,$this->actual]; + } private function regeln($renew) { $this->fallback++; } public function tick(&$allocation,$consumers=null,$sync=true) { $this->v4ManagerTestUebergabe($allocation,$consumers??[['InstanzID'=>44234,'Daten'=>[]]],$sync); @@ -105,7 +113,14 @@ namespace { 'maxChargeW'=>5000.,'maxDischargeW'=>5000.,'actuatorWatchdogEvidenceId'=>'synthetic-hardware-watchdog', 'accountingEvidenceId'=>'synthetic-meter-boundary-v1','acceptEstimatedPeak'=>true,'controlContext'=>$r['plan']['controlContext']]; $r['controlledTrial']=$serverAuthority?$a:null; - $m=new ManagerHarness($r,$op);$b=new BatteryHarness();$GLOBALS['objects']=[17004=>$m,44234=>$b]; + $m=new ManagerHarness($r,$op);$b=new BatteryHarness();$GLOBALS['objects']=[17004=>$m,44234=>$b];$GLOBALS['feedbackAt']=[]; + $fc=['version'=>1,'installationId'=>$id,'assetId'=>'b','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>'physical_sum','allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60, + 'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0,'sources'=>[]]; + foreach ([50001=>'grid',50002=>'physical'] as $vid=>$role) $fc['sources'][]=['key'=>'source'.$vid,'role'=>$role,'variableId'=>$vid,'parentId'=>200,'ident'=>'source'.$vid,'factorToW'=>1.0]; + $b->props['NetzfahrplanV4RueckmeldungKonfiguration']=json_encode($fc); + $op['localFeedback']=['b'=>json_decode($b->GetV4BatterieRueckmeldung(),true)]; + $m->props['NetzfahrplanV4RegeltestErlaubt']=$permissions;$b->props['NetzfahrplanV4RegeltestErlaubt']=$permissions; $b->props['NetzfahrplanV4GeraeteWatchdogNachweis']='synthetic-hardware-watchdog'; $start=['confirmation'=>'START_BOUNDED_V4_CONTROL_TRIAL','sessionId'=>$a['sessionId'],'durationSeconds'=>300,'maxChargeW'=>5000.,'maxDischargeW'=>5000.]; @@ -157,5 +172,6 @@ namespace { $bad=$r;$bad['controlledTrial']['validUntil']=gmdate('c',$now+120);rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'oversized server authorization lease'); $bad=$r;$bad['controlledTrial']['acceptEstimatedPeak']=false;rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'peak uncertainty needs explicit acceptance'); require __DIR__ . '/additional_checks.php'; + require __DIR__ . '/feedback_checks.php'; echo "TOTAL $count checks passed. Mocked IPS, clocks and register writes only.\n"; } diff --git a/tests/V4ControlTrial/feedback_checks.php b/tests/V4ControlTrial/feedback_checks.php new file mode 100644 index 0000000..00c384e --- /dev/null +++ b/tests/V4ControlTrial/feedback_checks.php @@ -0,0 +1,22 @@ +props['NetzfahrplanV4RueckmeldungKonfiguration']='{}'; +rejects(fn()=>start($m,$s),'trial cannot use old filtered feedback when mapping missing');check($b->writes===[],'missing corrected feedback sends no command'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock-61; +$a=tick($m);check(!$m->active()&&!$b->active()&&isset($a['Sollwerte'][44234]),'physical feedback expiry returns battery to normal allocation'); +check(end($b->writes)===0,'expired physical feedback revokes prior command'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock-61;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'independent battery read catches staleness without manager tick'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=json_decode($b->props['NetzfahrplanV4RueckmeldungKonfiguration'],true);$c['trackingToleranceW']=250; +$b->props['NetzfahrplanV4RueckmeldungKonfiguration']=json_encode($c);$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'mapping revision change revokes active battery session'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();$c['sequence']++;$c['feedbackConfigHash']=str_repeat('f',64); +rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'changed feedback command binding rejected'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();check(isset($c['feedbackConfigHash'],$c['feedbackSourceOldestAt'])&&array_key_exists('importLimitW',$c),'command includes source provenance and limits'); +$c['sequence']++;unset($c['exportLimitW']);rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'missing explicit grid limit not interpreted as unlimited'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();$c['sequence']++;$c['importLimitW']=1000.;$c['watts']=0; +$m->op['gridW']=1500.;$b->measurements['Ladezustand']=15.; +rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'final device clipping cannot silently violate accepted grid boundary'); +check(!$b->active()&&end($b->writes)===0,'unreachable final limit aborts software trial'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock+1;tick($m); +check(!$m->active(),'future physical sample is not smoothed into control'); diff --git a/tests/V4Feedback/checks.php b/tests/V4Feedback/checks.php new file mode 100644 index 0000000..06cb6e7 --- /dev/null +++ b/tests/V4Feedback/checks.php @@ -0,0 +1,58 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','assetId'=>'ev','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>$mode,'allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60,'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0,'sources'=>[]]; + $roles=['grid','physical','physical'];if($mode==='virtual_split')$roles=array_merge($roles,['ev_request','sdl_request','gateway_active']);$values=[3000.0,-2000.0,-1000.0,-3000.0,0.0,true];$r=[]; + foreach($roles as$i=>$role){$id=100+$i;$c['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'q'.$id,'factorToW'=>1];$r[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-60,'parentID'=>200,'ident'=>'q'.$id];} + return [$c,$r,$now]; +} +function readit($c,$r,$n){return F::read($c,fn($id)=>$r[$id],$n);} +[$c,$r,$n]=fx();$f=readit($c,$r,$n); +check($f['batteryW']===-3000.0&&$f['gridW']===3000.0,'physical total and grid signs retained'); +check($f['usableForTrial']&&!$f['estimated']&&!$f['canDispatch'],'usable feedback alone never grants dispatch'); +F::requireTrial($f,$c,$n);check(true,'matching trial binding accepted'); +check(F::fingerprint($c)===F::fingerprint(json_decode(json_encode($c),true)),'configuration hash stable over numeric property roundtrip'); +$d=$c;$d['sources']=array_reverse($d['sources']);check(F::fingerprint($c)===F::fingerprint($d),'source list order canonical'); +$d['trackingToleranceW']=201;check(F::fingerprint($c)!==F::fingerprint($d),'changed tolerance invalidates session hash'); +foreach([true,'1',NAN,INF,null]as$bad){$x=$r;$x[101]['value']=$bad;fails(fn()=>readit($c,$x,$n),'invalid physical number '.gettype($bad));} +$x=$r;$x[101]['updated']=$n-61;$x[101]['changed']=$n-90;fails(fn()=>readit($c,$x,$n),'physical age not refreshed by read'); +$x=$r;$x[101]['updated']=$n-31;fails(fn()=>readit($c,$x,$n),'incoherent grid battery timestamps rejected'); +$x=$r;$x[101]['updated']=$n+1;fails(fn()=>readit($c,$x,$n),'future original sample rejected'); +$x=$r;$x[101]['parentID']=201;fails(fn()=>readit($c,$x,$n),'reused variable wrong parent rejected'); +$x=$r;$x[101]['ident']='other';fails(fn()=>readit($c,$x,$n),'reused variable wrong ident rejected'); +$d=$c;$d['sources'][1]['variableId']=$d['sources'][0]['variableId'];fails(fn()=>F::configuration($d),'duplicate physical source rejected'); +$d=$c;$d['maxAgeSeconds']=300;fails(fn()=>F::configuration($d),'historical freshness not accepted for live control'); +$calls=[];fails(function()use($c,$r,$n,&$calls){F::read($c,function($id)use($r,&$calls){$calls[$id]=($calls[$id]??0)+1;$x=$r[$id];if($calls[$id]>1)$x['value']+=1;return$x;},$n);},'same-second change during double read rejected'); +$d=$c;$d['sources'][1]['factorToW']=-1;check(readit($d,$r,$n)['batteryW']===1000.0,'vendor sign conversion once'); +$f=readit($c,$r,$n);fails(fn()=>F::requireTrial($f,$c,$n+3),'old validated snapshot not reused'); +$d=$c;$d['managerId']=20000;fails(fn()=>F::requireTrial($f,$d,$n),'different manager cannot use feedback'); +$d=$c;$d['feedbackConfigHash']=str_repeat('a',64);fails(fn()=>F::requireTrial($f,$d,$n),'changed mapping aborts trial'); +[$c,$r,$n]=fx('virtual_split');$r[103]['updated']=$r[104]['updated']=$r[105]['updated']=$n-86400;$r[103]['changed']=$r[104]['changed']=$r[105]['changed']=$n-172800; +$r[101]['value']=$r[102]['value']=$r[103]['value']=$r[104]['value']=0.0;$f=readit($c,$r,$n); +check($f['batteryW']===0.0&&$f['estimated'],'unchanged idle commands need no artificial heartbeat'); +check(!$f['usableForTrial'],'virtual estimate requires explicit acceptance'); +$c['allowEstimatedForTrial']=true;$f=readit($c,$r,$n);check($f['usableForTrial']&&!$f['gatewayWatchdogVerified'],'idle acceptance is not gateway watchdog proof'); +$r[101]['value']=20.0;$f=readit($c,$r,$n);check($f['batteryW']===20.0,'idle residual not clipped to zero'); +$r[101]['value']=60.0;fails(fn()=>readit($c,$r,$n),'unexplained idle power refused'); +$r[101]['value']=-2000.0;$r[102]['value']=-1000.0;$r[103]['value']=2000.0;$r[103]['updated']=$n;$r[103]['changed']=$n; +$f=readit($c,$r,$n);check($f['batteryW']===-3000.0,'single EV feedback follows physics not desired sign or held filter'); +$r[104]['value']=1000.0;$r[104]['updated']=$n;$r[104]['changed']=$n;$r[101]['updated']=$n-1; +fails(fn()=>readit($c,$r,$n),'SDL transition after physical sample cannot be guessed'); +[$c,$r,$n]=fx('virtual_split');$c['allowEstimatedForTrial']=true;$r[103]['value']=3000.0;$r[104]['value']=1000.0;$r[101]['value']=1900.0;$r[102]['value']=2000.0;$f=readit($c,$r,$n); +check($f['batteryW']===2925.0&&$f['externalModelW']===975.0,'small tracking error allocated but explicitly estimated'); +check(!$f['separatelyMeasured']&&$f['heldFilterUsed']===false,'partition never labelled separately measured'); +$r[101]['value']=1000.0;fails(fn()=>readit($c,$r,$n),'large tracking error prevents stale virtual correction'); +$r[103]['value']=42000.0;$r[104]['value']=-40000.0;$r[101]['value']=0.0;$r[102]['value']=2000.0;$f=readit($c,$r,$n); +check(!$f['usableForTrial']&&in_array('opposing_virtual_flows_not_identifiable',$f['reasons'],true),'counterflow cannot be independently attributed'); +$r[105]['value']=false;fails(fn()=>readit($c,$r,$n),'disabled gateway blocks feedback'); +$src=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Rueckmeldung.php'); +check(!preg_match('/\b(?:SetValue|IPS_RequestAction|curl_exec|IPS_SetProperty|AC_SetLoggingStatus)\s*\(/',$src),'pure feedback has no control side effects'); +echo "TOTAL $count physical-feedback checks passed. Synthetic source readings only.\n"; diff --git a/tests/V4Feedback/fallback_checks.php b/tests/V4Feedback/fallback_checks.php new file mode 100644 index 0000000..efe7d87 --- /dev/null +++ b/tests/V4Feedback/fallback_checks.php @@ -0,0 +1,23 @@ +$GLOBALS['instance']->status];} +eval('class FallbackHarness { const STATUS_AKTIV=102; public $InstanceID=17004,$trial=true,$active=true,$status=102,$abortFirst=true,$throwFirst=false,$passes=[],$attributes=[],$stops=[];'. + 'public function run(){ $this->regeln(false); } private function v4ManagerTestSession(){return $this->trial?[1]:[];} private function GetValue($k){return $this->active;}'. + 'private function WriteAttributeString($k,$v){$this->attributes[$k]=$v;} private function v4ManagerTestAbbrechen($s){$this->stops[]=$s;$this->trial=false;}'. + 'private function regelnIntern(bool $r):void { $this->passes[]=$r;if(count($this->passes)==1){if($this->throwFirst)throw new RuntimeException("synthetic_failure");if($this->abortFirst)$this->trial=false;}}'. $method.'}'); +$count=0;function ck($v,$s){global$count;if(!$v)throw new RuntimeException('FAIL '.$s);$count++;echo "PASS $s\n";} +$instance=new FallbackHarness();$instance->run();ck($instance->passes===[false,true],'trial abort performs exactly one fresh ordinary recomputation'); +ck($instance->attributes['LetzteSollwerte']==='{}','pre-trial cached target is discarded before fallback');ck(!$locked,'control semaphore released after fallback'); +$instance=new FallbackHarness();$instance->trial=false;$instance->run();ck($instance->passes===[false],'ordinary control path does not gain extra passes'); +$instance=new FallbackHarness();$instance->abortFirst=false;$instance->run();ck($instance->passes===[false],'valid trial not overridden by ordinary pass'); +$instance=new FallbackHarness();$instance->status=201;$instance->run();ck($instance->passes===[false],'invalid measurement status prevents forced ordinary recomputation'); +$instance=new FallbackHarness();$instance->throwFirst=true;try{$instance->run();}catch(RuntimeException $e){} +ck(!$locked&&!$instance->trial&&count($instance->stops)===1,'exception revokes trial and releases semaphore without endless retries'); +echo "TOTAL $count actual manager fallback-wrapper checks passed. No real IPS runtime.\n"; diff --git a/tests/V4Feedback/message_checks.php b/tests/V4Feedback/message_checks.php new file mode 100644 index 0000000..0c85b13 --- /dev/null +++ b/tests/V4Feedback/message_checks.php @@ -0,0 +1,23 @@ +'1'];public bool $valid=true; + protected function GetBuffer($k){return $this->buffers[$k]??'';} + protected function GetValue($k){throw new RuntimeException('No diagnostic variables exist: '.$k);} + protected function ReadAttributeBoolean($k){return $k==='ZustandLadezustandGueltig'?$this->valid:false;} + protected function ReadAttributeInteger($k){return 0;} + protected function ReadAttributeFloat($k){return 0.0;} + protected function ReadAttributeString($k){return $k==='Leistungsangebot'?'[0]':($k==='Betriebsart'?'Auto':'');} + protected function ReadPropertyInteger($k){return 1;} +} +require __DIR__.'/../../Batterie/module.php'; +use Belevo\EnelixEMS\Nachrichtenvertrag as N; +$b=new Batterie();$read=new ReflectionMethod(Batterie::class,'baueVerbraucherdaten');$count=0; +function ck($v,$s){global$count;if(!$v)throw new RuntimeException('FAIL '.$s);$count++;echo "PASS $s\n";} +$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_BERECHNET,'partition source quality independent of hidden diagnostic variables'); +$b->buffers['V4FeedbackEstimated']='0';$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_GEMESSEN,'ordinary physical source marked measured'); +$b->valid=false;$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_NICHT_VORHANDEN&&$d['Istleistung_W']===null,'invalid sample never published as measured'); +$b->valid=true;$b->buffers['V4FeedbackEstimated']='1';$b->buffers['V4BatteryTrialSession']='{}';$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_BERECHNET,'last estimate not relabelled measured immediately after trial ends'); +$source=file_get_contents(__DIR__.'/../../Batterie/module.php');ck(str_contains($source,"SetBuffer('V4FeedbackEstimated', (\$messwerte['V4FeedbackEstimated'] ?? false) ? '1' : '0')"),'fresh measurement updates internal source quality'); +echo "TOTAL $count actual message-builder checks passed. No real variables or writes.\n"; diff --git a/tests/V4Feedback/receiver_checks.php b/tests/V4Feedback/receiver_checks.php new file mode 100644 index 0000000..f617f4b --- /dev/null +++ b/tests/V4Feedback/receiver_checks.php @@ -0,0 +1,43 @@ +['ModuleID'=>'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}']];} + function IPS_GetProperty($id,$k){return ['IstleistungVariableID'=>1002,'LadezustandVariableID'=>1001,'NetzfahrplanV4RueckmeldungKonfiguration'=>$GLOBALS['configured']?'configured':'{}'][$k];} + function IPS_VariableExists($id){return $id===1002;} + function IPS_GetVariable($id){return ['VariableUpdated'=>$GLOBALS['now']];} + function GetValue($id){return -9000.;} // deliberately held/incorrect legacy virtual display + class FeedbackReceiver { + use ManagerNetzfahrplanV4EmpfangTrait; + public int $InstanceID=17004;public array $props,$attrs,$op; + public function __construct($r,$op){$this->op=$op;$this->props=['NetzfahrplanV4EmpfangAktiv'=>true,'NetzfahrplanV4SchattenAktiv'=>true,'NetzfahrplanAktiv'=>false,'MesswertMaxAlter'=>60,'AnlagenBatterien'=>json_encode([['ID'=>'b','LeistungVariableID'=>1002,'SOCVariableID'=>1001,'Leistungsfaktor'=>1]])];$this->attrs=['LizenzInstallationID'=>$r['installationId'],'NetzfahrplanV4Empfang'=>json_encode(['response'=>$r,'receivedAt'=>time()]),'NetzfahrplanV4EmpfangStatus'=>'{}'];} + private function ReadPropertyBoolean($k){return $this->props[$k];} + private function ReadPropertyString($k){return $this->props[$k];} + private function ReadPropertyInteger($k){return $this->props[$k];} + private function ReadAttributeString($k){return $this->attrs[$k];} + private function aktiveVerbraucherIDs(){return [44234];} + private function berechtigungLizenziert($k){return true;} + public function GetNetzfahrplanV4Diagnose(){return json_encode(['operation'=>$this->op]);} + } +} +namespace { + function ENELIX_GetV4BatterieRueckmeldung($id){return json_encode($GLOBALS['feedback']);} + require_once __DIR__.'/../V4Receiver/fixture.php'; + use Belevo\EnelixEMS\FeedbackReceiver; + [$response,$id,$op,$now]=receiverFixture();$configured=true;$count=0; + $feedback=['kind'=>'local_physical_feedback','version'=>1,'installationId'=>$id,'assetId'=>'b','managerId'=>17004,'batteryInstanceId'=>44234,'batteryW'=>0.,'gridW'=>$op['gridW'],'checkedAt'=>$now,'estimated'=>true,'usableForTrial'=>false,'canDispatch'=>false,'configHash'=>str_repeat('a',64)]; + function ck($v,$t){global$count;if(!$v)throw new RuntimeException('FAIL '.$t);$count++;echo "PASS $t\n";} + $m=new FeedbackReceiver($response,$op);$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='preview_only'&&$p['actualBatteryW']==0,'real receiver uses corrected physical feedback rather than held -9kW'); + ck($p['previewBatteryW']==3000,'net target translated once to total battery demand'); + ck($p['canDispatch']===false&&$p['localFeedback']['b']['estimated']===true,'model feedback remains marked and preview grants no dispatch'); + $feedback=['status'=>'unavailable','batteryW'=>null,'gridW'=>null];$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='unavailable'&&$p['previewBatteryW']===null,'configured feedback failure never falls back to held virtual display'); + $configured=false;$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='preview_only'&&$p['feedbackMeaning']==='legacy_display_only_not_for_trial','unconfigured old preview stays compatible but explicitly not for control'); + ck($p['canDispatch']===false,'legacy display cannot gain actuation permission'); + echo "TOTAL $count actual receiver checks passed; synthetic read-only controller API.\n"; +} From 913a4cd5f3910b20adb2fa3bd1a7f4761fa6143e Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Sun, 4 Oct 2026 07:06:50 +0000 Subject: [PATCH 16/22] docs(v4): preserve conversation decisions and current agent handoff --- AGENTS.md | 25 +++ docs/NETPLAN_V4_AGENT_HANDOFF.md | 302 +++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 AGENTS.md create mode 100644 docs/NETPLAN_V4_AGENT_HANDOFF.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..26e35b8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,25 @@ +# Agenten-Anweisungen für Enelix-EMS + +## Netzfahrplan V4: zuerst den Gesprächskontext übernehmen + +Bei Aufgaben zu Netzfahrplan, Prognosen, V4, Batterieoptimierung oder Lihrenmoos **vor Änderungen** die vollständige Übergabe lesen: + +**[`docs/NETPLAN_V4_AGENT_HANDOFF.md`](docs/NETPLAN_V4_AGENT_HANDOFF.md)** + +Sie enthält Daniels Anforderungen und Korrekturen, installierte versus nur entwickelte Komponenten, Quellen-/Instanzzuordnung, bekannte behobene Fehler, Grenzen der Testfreigabe und den nächsten konkreten Arbeitsschritt. Die Runtime-Berichte stammen zuletzt vom 03.10.2026; Zeitpunkt und aktuellen Zustand prüfen. Alte Entwicklungsdokumente können noch „nicht installiert“ sagen, obwohl ein jüngerer Installationsbericht den Abschluss belegt. + +Kurzstand bei Erstellung dieser Anweisung: Unified RC1 und korrigierter Rückmelde-/begrenzter Testpfad installiert; Versandproblem behoben. Nächste offene Umsetzung sind bestätigte Geräte-Lesezeitpunkte statt pauschaler Verwendung von `VariableUpdated` (`feedback_source_skew`). EV **161.44 kWh / 39 kW**, nicht die zurückgenommenen 160/30. Dauerproduktion ist noch nicht fertig; keine automatische Stellfreigabe. + +## Arbeits- und Git-Regeln + +- Vor Arbeit Host-`/srv/agent/AGENT_CONTEXT.md`, Branch und Arbeitsverzeichnis prüfen. Servercheckout ist nicht gleich laufendes Symcon-Modulverzeichnis. +- Neue Entwicklung auf **`develop`**, keine Feature-Branches. Nur eigene/zugehörige Änderungen committen. `develop` darf im vereinbarten Rahmen gepusht werden; `beta` nur gemäss ausdrücklicher Freigabe und nach Tests, `main` nach ausdrücklichem Auftrag/Feldabnahme. Keine Schutzregeln oder fehlende Authentifizierung umgehen. +- Daniel möchte zusammenhängende produktionsgeeignete Umsetzung, keine weitere Schleife aus Diagnosekategorien, identischen Rückfragen und veralteten Installern. Vorhandene Quellen und Werkzeuge benutzen, keine bekannten Anlagenangaben erneut verlangen. +- Code-, Host-/Container-, Kernel- und Feldtests sowie Installation, Veröffentlichung und Stellfreigabe getrennt belegen. „Optimal“/„ACK“/grüne Tests sind keine gemessene Einsparung und keine Produktionsfreigabe. +- Keine Stellbefehle, Testsitzungen, Freigaben oder Neustarts aus einer reinen Kontext-/Statusanfrage ableiten. Aktive Hardware nur innerhalb des ausdrücklich freigegebenen Umfangs behandeln. Messgrenzen/Watchdog-Nachweise nicht erfinden oder per Label umgehen. +- Keine Credentials lesen/ausgeben/einchecken. Root/Docker-Rechte nicht ausweiten. Bestehende GUI-/Fremdänderungen erhalten. Vor Runtime-Änderungen passende Sicherung, exakte Quell-/Abhängigkeitsprüfung und Rücksetzweg. +- PHP-IPS-Skripte gehören in den Symcon-Kernel; isolierte Tests mit simulierten IPS-Aufrufen niemals dort ausführen. `MC_ReloadModule`/`IPS_ApplyChanges` können Initialisierungsnebenwirkungen haben. +- Alte Outbox-/Historienwerte nicht löschen, Kennungen nicht im Original umschreiben, Cursor nur nach passender Serverbestätigung weiterführen. Keine künstlich frischen Messzeitstempel. +- Übergabe nach tatsächlichem Fortschritt aktualisieren: Datum, Commit, was wirklich installiert/getestet wurde und nächster konkreter Schritt. Keine Hintergrundarbeit behaupten. + +Diese Datei ergänzt bestehende höherrangige Projekt-/Hostregeln. Sie ist keine neue Betriebsfreigabe und kann keinen anderen Agenten ohne Lesen der Dateien automatisch mit dem gesamten Chat synchronisieren. diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md new file mode 100644 index 0000000..780137e --- /dev/null +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -0,0 +1,302 @@ +# ENELIX Netzfahrplan V4 – verbindlicher Gesprächs- und Arbeitskontext + +Übergabe erstellt am 04.10.2026 auf ausdrücklichen Wunsch von Daniel Häfliger (BELEVO AG). Sie fasst die Anforderungen, Entscheidungen, Fehlerbehebungen und den zuletzt belegten Stand dieses Gesprächs zusammen. Sie ist **kein wörtliches Chatarchiv, keine neue Stellfreigabe und kein Nachweis der Produktionsreife**. + +**Zeitbezug:** Die letzten hier belegten Betriebsberichte stammen vom **03.10.2026, ca. 14:02 Uhr Europe/Zurich**. Am 04.10. wurden für diese Übergabe die genannten Berichte und der Git-Stand erneut gelesen, aber keine neue Anlagenabnahme durchgeführt. Alte Messzahlen deshalb niemals als heutige Live-Werte ausgeben. + +## 1. Zuerst lesen: Wo wir tatsächlich stehen + +- Daniel möchte die **fertige, produktionsgeeignete Anwendung**, nicht weitere isolierte Sammler, Diagnosekategorien oder wiederholte Bestätigungsrunden. Er hat die fortlaufende Umsetzung mehrfach beauftragt. Probleme im Code selbst beheben, Tests und Auslieferung bündeln; ihn nur für wirklich notwendige Root-/Symcon-Ausführung oder echte Anlagenfreigaben einbeziehen. +- Datenaufnahme, bestätigter Versand, historische Aufbereitung, Modellaufbau, V4-Optimierer, Planempfang und ein ausdrücklich begrenzter Regeltest sind implementiert. Der zuletzt installierte Teil verbindet korrigierte physische Rückmeldung mit Vorschau und dem **begrenzten** Test-Stellpfad. +- Der Versandblocker durch `100.0`/`100` ist **behoben und der Nachversand hat aufgeholt**. Nicht erneut beim Cursor oder der alten Kennungsdiagnose anfangen. +- Die letzte Feedback-Installation ist **erfolgreich abgeschlossen**, nicht mehr `waiting_for_registration`. Bericht: `corrected_feedback_installed_trial_disabled`, Rückmeldung `unavailable`, Grund **`feedback_source_skew`**. +- **Nächster konkreter Entwicklungsauftrag:** bestätigte, zum tatsächlich gelesenen Wert passende Geräte-Lesezeitpunkte an den lokalen Rückmeldepfad anschliessen. Bisher wird `VariableUpdated` als Live-Zeitbasis verwendet; unveränderte Modbus-Werte werden aber nicht zwingend bei jedem erfolgreichen Abruf erneut publiziert. Geräteabruf, Variablenpublikation und aktueller Lesezugriff müssen auseinandergehalten werden. +- Keine künstliche Zeitstempelverjüngung, pauschale Toleranzerhöhung, Nullersetzung, historische Interpolation als Live-Messung oder Rückkehr zum festgehaltenen virtuellen EV-Wert. +- **Weiterhin keine produktive V4-Batterieausführung:** alter intelligenter Netzfahrplan AUS; beide lokalen Regeltestfreigaben AUS; kein gestarteter Stellversuch. Bestehende lokale Regelung und SDL laufen unabhängig weiter. +- Ein uneingeschränkter Produktions-Dauerregler ist **noch offene Umsetzung**, nicht nur ein fehlender Haken. Automatik, reale Modellqualität, Geräteausfallverhalten und Mehranlagenfreigabe haben ebenfalls verbleibende Grenzen. +- Letzter fachlicher EMS-Commit: **`9878197`**, `develop`, bei Übergabeprüfung sauber und **15 Commits vor lokalem `origin/develop`**. Kein Remote-Fetch in dieser Prüfung. Vorherige Pushversuche scheiterten an Authentifizierung. Dokumentations-Commits dieser Übergabe können danach folgen. + +## 2. Arbeitsweise und Freigaben + +Daniel ist technisch versiert, will aber keine unnötige manuelle Kleinarbeit. Deutsch, Schweizer Schreibweise, konkrete nutzbare Befehle. Ein abgeschlossener Test ist nicht automatisch Installation, Installation nicht Stellfreigabe, `optimal` nicht reale Einsparung. Immer getrennt berichten: implementiert / isoliert getestet / Zielcontainer getestet / installiert / im Kernel geprüft / am Gerät erprobt / veröffentlicht. + +**Nicht wieder in die alte Schleife zurückfallen:** + +1. Keine neuen Diagnosekategorien als Ersatz für die eigentliche Integration. +2. Keine erneuten Fragen nach bereits bestätigten EV-Kapazitäten oder pauschal immer wieder nach Wechselrichtertypen. Bestehende Quellen nutzen. Eine tatsächlich fehlende sicherheitsrelevante Eigenschaft darf trotzdem nicht erfunden werden. +3. Nicht einfach „24 Stunden warten“, solange ein Software- oder Übertragungsfehler das Training verhindert. +4. Keine alten Einzelinstaller erneut ausführen; neueren Quellstand und parallele GUI-Arbeit nicht überschreiben. +5. Nicht behaupten, im Hintergrund weiterzuarbeiten. Nur tatsächlich ausgeführte Arbeit und belegte Ergebnisse berichten. +6. Die Regel „ein einziger konkreter Punkt“ betraf Einträge in **Offene Punkte**, nicht Entwicklungsaufträge. Implementierungsaufträge dürfen zusammenhängende Anforderungen bündeln. + +**Git:** Keine Feature-Branches. Entwicklung auf `develop`; nach Tests `beta`; nach Feldtest `main`. Daniel hat im Gespräch Commit/Push für `develop` und damals auch `beta` ausdrücklich freigegeben. Diese Freigabe nicht als Auftrag interpretieren, einen aktuell ungetesteten Kandidaten blind auf `beta`/`main` zu schieben. `develop`-Push ohne erneute Nachfrage im bestehenden Rahmen erlaubt. Nicht force-pushen, nicht unzusammenhängende Änderungen übernehmen, keine Zugangsdaten verlangen/ausgeben. Historische Agenten-Commitidentität war `ENELIX Agent `; keine globale Git-Konfiguration ungeprüft ändern. + +Die aktuelle Nachricht beauftragt **Kontextübergabe**. Sie erteilt keine zusätzliche Anlagen-, Runtime- oder Stellfreigabe. Bei späterer Fortsetzung greifen die bestehenden Sicherheits- und Branchregeln weiter. + +## 3. Ursprüngliches fachliches Ziel + +### 3.1 Prognosefamilien und Erweiterbarkeit + +Im GUI manuell eine Familie wählen oder `auto`: + +| Familienschlüssel / Fahrplan | PV | Last | Fahrplanvariable | +|---|---|---|---| +| `3` | `prog_var_1` | `prog_var_2` | `prog_var_3` | +| `13` | `prog_var_10` | `prog_var_11` | `prog_var_13` | +| `23` | `prog_var_21` | `prog_var_22` | `prog_var_23` | + +Die neun Zahlen sind nicht neun unabhängige Fahrplanmodelle. Daniel akzeptierte die Auswahl der vollständigen Modellfamilie. Registry und Datenadapter so gestalten, dass weitere Familien und später weitere flexible Anlagen ergänzt werden können. Nicht behaupten, drei bereits vorhandene Lastmethoden seien identisch mit den neu aufgebauten physischen Profilen; Datenquelle und Modellversion müssen sichtbar sein. + +`auto` soll jene Familie wählen, die in der jüngeren Vergangenheit **wirtschaftlich am besten abgeschnitten hätte**, mit gleichen Anfangsbedingungen, Tarifen, SOC-/Restenergiebewertung und Randbedingungen. Keine Auswahl allein nach R², keine Zukunftsinformation im historischen Replay. Mindestabdeckung, Mindestdauer, Rückblick und Wechselmarge verhindern häufige/schlecht belegte Wechsel. + +### 3.2 Netzfahrplan / Optimierung + +- Bis zu **48 Stunden**, grundsätzlich **5-Minuten-Schritte** (höchstens 576), rollende Neuberechnung. +- Ausgangsverlauf: korrekt abgegrenzte Verbraucherlast minus PV plus separat berücksichtigte externe Flüsse. Darauf flexible Verbraucher optimieren; aktuell EV-Batterie, später Ladestationen, Boiler usw. +- Batterie: aktueller virtueller EV-SOC, dessen zugehörige EV-Gesamtkapazität, aktuelle maximale Lade-/Entladeleistung, Min-/Max-SOC, Reserve, Hysterese, Verfügbarkeit und lokale Einschränkungen. Keine zukünftige Energie vor dem Laden nutzen. +- **90 % Roundtrip-Wirkungsgrad**, nicht zweimal 90 %. Implementierung verteilt Verluste mit `sqrt(0.9)` auf Laden/Entladen. +- Netzladen ist ausdrücklich erlaubt/gewünscht, wenn freigegeben und wirtschaftlich sinnvoll. Nicht generell Nachtbezug verbieten oder tagsüber Entladung verbieten. +- Kosten pro Intervall: Bezugsenergie mal Bezugspreis minus Einspeiseenergie mal Einspeisepreis. Leistungswerte in W/kW korrekt über Zeit in kWh umrechnen. Gesamtkosten minimieren, nicht mathematisch auf exakt null zwingen; negative Kosten sind möglich, nicht garantiert. +- Statische oder dynamische, in der Anlage konfigurierte Bezugspreise und Einspeisevergütungen verwenden. GUI-Preisänderungen speichern und Neuberechnung anfordern. PV-Vergütung, Netzladekosten, Verluste und Peakpreis gehören in dieselbe Wirtschaftlichkeitsbetrachtung. +- Wenn dynamische Preise noch nicht veröffentlicht sind, den **ausführbaren wirtschaftlichen Horizont auf den bekannten Preisbereich begrenzen** (`published_only`); kein frei erfundener Preis für den Rest der 48 h. Volle Prognose und ausführbaren Preishorizont unterscheiden; Ende passend zur vollständigen Abrechnungsviertelstunde. +- Restenergie am Horizont fair bewerten/absichern, damit die Batterie nicht am Planende oder zu fast wertlosen Mittagstarifen grundlos leerverkauft wird. +- Bezug/Einspeisung sowie Laden/Entladen nicht als unphysikalische gleichzeitige Arbitrage zulassen. +- Einspeisebegrenzung, zulässige Abregelung/entgangene Vergütung und lokale harte Netzgrenzen berücksichtigen. Einen unerreichbaren Plan nicht als eingehalten darstellen. + +### 3.3 Monatspeak + +- Leistungstarif auf den relevanten **15-Minuten-Netzbezug**; persistenter Monatsmaximalwert, nicht momentane Spitzenleistung. Für den laufenden Monat zusätzliche Kosten nur für Erhöhung über den bereits erreichten Peak; Monatswechsel korrekt behandeln. +- Bereits verstrichene Energie der aktuellen Viertelstunde mitnehmen. Fehlende Messbasis nicht durch eine konfigurierte Grenze ersetzen. +- Am Monatsanfang nicht künstlich jede Aktivität verhindern: Restmonatsbewertung/empirische Aussicht ist vorgesehen, mit ausgewiesener Unsicherheit. Ein erwarteter künftiger Peak ist **keine bereits bezahlte kostenlose Freigabe**. +- Anlagen können **keine feste Bezugsgrenze** haben. Falls im Manager statische oder monatliche Peakgrenzen konfiguriert sind, diese zusätzlich respektieren. +- Tarifeinheiten und tatsächliche Abrechnungsdefinition prüfen; gemessene, geschätzte und nur konfigurierte Peakwerte getrennt kennzeichnen. + +### 3.4 Training, Reaktion und Validierung + +Tägliches oder wöchentliches Nachtraining nach GUI-Einstellung; Parameter-/Preisänderungen sollen Neuberechnung auslösen. Getrennte versionierte Historien und Modelle, keine alten belasteten Daten als bereinigt umbenennen. Ein Cold-Start-Modell nach Mindestdatenbasis ist noch keine validierte Feldprognose. Vergleich gegen die bisherige Batterie-Betriebsweise, nicht nur gegen „Anlage ohne Batterie“. + +## 4. Lihrenmoos: bestätigte Anlage und massgebliche Korrekturen + +- Installation: **`e3a08f9e-af12-4695-99bd-8b51c0520021`**. +- Symcon Manager **`17004`**, ENELIX-Batterieinstanz **`44234`**, virtuelles Asset **`anlage01-virtual-ev`**, bestehendes EV-/SDL-Gateway **`58448`**. +- **Massgeblich bestätigt: EV-Kapazität `161.44 kWh`, EV-Leistung `39 kW`.** Die vorher genannten `160 kWh`, `30 kW` und „je 10 kW pro WR“ hat Daniel ausdrücklich als Erinnerungskorrektur zurückgenommen. Daraus keine neue Verteilung auf Einzelgeräte ableiten. +- Virtueller EV-SOC und die EV-Kapazität bilden ein gemeinsames Energiekonto. Keine SOC-/Energie-Resets oder Umrechnung auf 322 kWh. **SDL-Reserve ist bereits ausgegliedert** und darf nicht nochmals von EV abgezogen werden. +- Vom Nutzer genannte physische Ausstattung: zwei GoodWe à **50 kW AC**, je **156 kWh** Batterie; SolarEdge **10 kW AC / 10 kWh** Batterie. Summe physische Kapazitäten rechnerisch 322 kWh, nicht die EV-Kapazität. AC-Nennleistung ist kein unabhängiger Nachweis jeder aktuell zulässigen Batterie-Ladeleistung. +- Dach-PV ungefähr **20 kWp Ost + 20 kWp West**. +- Exakte Gerätetypen/Firmware, jede AC-/DC-Messgrenze und autonomes Verhalten beim Befehlsausfall sind durch diese Nennwerte nicht automatisch bestätigt. Vorhandene Dokumente/Quellen verwenden, statt Angaben zu erfinden oder Daniel wiederholt pauschal zu befragen. +- Historischer Tarif-Snapshot: Bezug `CKW Dynamisch Home`, statischer Fallbackwert 0.246 CHF/kWh, Einspeisung „Preis selber eingeben“ 0.10 CHF/kWh, Peakparameter 5.0. **Nicht als aktuelle Tarifvorgabe neu setzen**, vor Verwendung aktuelle Konfiguration lesen. + +## 5. Bilanzierung und SDL: nicht wieder die alte Fehlerquelle einbauen + +Der alte Prognosesender verwendete sinngemäss `PV + Netz − virtuelle EV-Batterieleistung`. Weil nur die virtuelle EV-Batterie in der Topologie stand, konnte **SDL als Hausverbrauch gelernt** werden. Zusätzlich hält das bestehende Gateway unter bestimmten Bedingungen gefilterte virtuelle EV-/SDL-Istleistungen fest und schreibt sie erneut. Neuer Variablenzeitstempel allein machte diese Werte nicht zur frischen physikalischen Messung. + +**Neue Grundidee, bei passender elektrischer Messgrenze:** + +`Verbraucherlast = Netzbezug + PV-Erzeugung − gesamte physische Batterieladung` + +`Grundlast = Verbraucherlast − separat geplante flexible Verbraucher` + +Grundlast bedeutet hier zeitabhängiger nicht separat geplanter Verbrauch, **keine Konstante**. Nicht separat geplante Boiler/Wallboxen bleiben vorerst darin. Später beim Übergang zu eigenständiger Planung genau einmal herausrechnen. + +**SolarEdge-Sonderfall:** PV-Variable 20335 war bereits aus skaliertem AC-Ausgang plus Batterieleistung berechnet und auf null begrenzt. Neuer Lastadapter `solar_terminal_v1` verwendet den AC-Ausgang aus 37975/41853 direkt und zieht den SolarEdge-Batteriesaldo nicht nochmals ab. Herkunft, Vorzeichen, AC/DC und Null-Clipping beachten. Algebraischer Bilanzschluss allein ist kein unabhängiger Zählernachweis. + +SDL wird **nicht vom V4-Flexibilitätsoptimierer gesteuert**. Bekannten externen Fahrplan berücksichtigen; bei unbekanntem Verlauf nicht still null einsetzen. Die vorhandene Schattenplanung kann den aktuellen SDL-Auftrag als ausdrücklich gekennzeichnete Fortschreibung verwenden. Das ist kein veröffentlichter Zukunftsfahrplan. Für Dauerproduktion sind belastbare externe Grenzen/Reservebehandlung und Konfliktauflösung noch zu prüfen. Ein aktueller SDL-Wert null garantiert keine SDL-freie kommende halbe Stunde. + +Bestehendes Gateway: `/var/lib/symcon/modules/Symcon_Belevo_Energiemanagement_testing/Bat_EV_SDL_V4/module.php`. Aktionsbrücke historisch `scripts/31800.ips.php`. `State=false` war dort **kein nachgewiesener Nullstell-/Notstopp**. EV-Stopp darf nicht pauschal unabhängige SDL-Aufträge löschen. Virtuelle Energiekonten/Filter nicht ungeprüft umstellen. + +## 6. Quellzuordnung für die nächste Arbeit + +Werte/Identitäten mit aktueller Quellenkonfiguration abgleichen; Tabelle ist zuletzt geprüfte Zuordnung, keine Erlaubnis, Register umzulegen. + +| Grösse | Variable | Ursprung/Umrechnung | +|---|---:|---| +| Original-Netzleistung | 40348 | Parent 11490, `Power_8`, kW ×1000 → W, Bezug positiv | +| Netz-Anzeige | 49301 | Abgeleitet; nicht mit Originalquelle verwechseln | +| GoodWe 1 PV | 48459 | Parent 19742, `A_3_3_35301`, W | +| GoodWe 2 PV | 53802 | Parent 57658, `A_3_3_35301`, W | +| GoodWe 1 Batterie physisch | 47725 | Parent 19742, `A_6_3_35182`, Faktor −1 → Laden positiv | +| GoodWe 2 Batterie physisch | 35724 | Parent 57658, `A_6_3_35182`, Faktor −1 | +| SolarEdge Batterie physisch | 21447 | Parent 30789, `Value`, Faktor +1 | +| SolarEdge abgeleitete PV | 20335 | Parent 36915; nicht unabhängige PV-Messung | +| SolarEdge AC-Rohwert | 37975 | Parent 35514, `Value`, ReadAddress 40083 | +| SolarEdge AC-Skalierung | 41853 | Parent 48996, `Value`, ReadAddress 40084 | +| Virtuelle EV-Istleistung | 52020 | Parent 58448, `Aktuelle_Leistung_EV`; Filterhaltewert möglich | +| Virtuelle SDL-Istleistung | 25085 | Parent 58448, `Aktuelle_Leistung_SDL` | +| EV-Auftrag | 19651 | Parent 58448, `Nennleistung_Soll_EV` | +| SDL-Auftrag | 38943 | Parent 58448, `Nennleistung_Soll_SDL` | +| Gateway aktiv | 23483 | Parent 58448, `State`; kein Gerätewatchdog-Nachweis | +| EV-SOC | 32871 | Parent 58448, `SoC_EV` | +| SDL-SOC | 23879 | Parent 58448, `SDL_Pos` | +| EV verfügbare Ladeleistung | 50230 | Parent 58448, `P_EV_laden` | +| EV verfügbare Entladeleistung | 43899 | Parent 58448, `P_EV_entladen` | +| GoodWe 1 / 2 / SolarEdge SOC | 27361 / 23109 / 51938 | Originalzuordnung siehe Capture-Konfiguration | +| Wirkenergie Bezug T1 | 59607 | Parent 11490, `Energy_0`, kWh | +| Wirkenergie Bezug T2 | 26620 | Parent 11490, `Energy_1`; Zählerarchivierung aktiviert | + +**Nicht wieder 53476 als nachgewiesenen Bezugszähler verwenden:** Im Gespräch zeigte 53476 / Quelle 32797 (`Energy_6`) einen unplausibel anderen Verlauf gegenüber Netzleistung und T1. Alte Historie 53476 wurde bewusst nicht überschrieben. Archivinstanz 16207. Neuzeitlicher Peak muss aus geeigneter T1/T2-Summe bzw. ausdrücklich zugelassener vollständiger Viertelstundenschätzung stammen. + +Poller laut zuletzt gelesenem Snapshot: GoodWe 1 Parent 19742 `60000 ms`, GoodWe 2 Parent 57658 `5000 ms`; SolarEdge Batterie Parent 30789 `2000 ms`, AC Parent 35514 `1000 ms`, Skalierung Parent 48996 `20000 ms`. M-Bus Parent 11490 hatte `Interval=0`, trotzdem laufend frische Originalwerte. Nicht daraus schliessen, die Erfassung sei aus; zusätzliche bestehende Aufrufwege prüfen. Poller nie allein als erfolgreichen Geräteabruf interpretieren. + +## 7. Hosts, Code, Dienste und Berechtigungen + +### enelix-services / Connector Server_89 + +- Host `enelix-services`, IPv4 `87.106.60.89`, Agentbenutzer `agent`. Erlaubte Wurzeln `/home/agent/services`, `/srv/agent`. +- Entwicklung: **`/srv/agent/repos/Enelix-EMS`**, Gitea `ENELIX/Enelix-EMS` auf `git.belevo.ch`. Weiteres Projekt `ENELIX/Enelix-Utils`; historische Referenz `dh/Symcon_Belevo_Energiemanagement_testing`. Andere Hosts/Klone nicht ohne Nachweis gleichsetzen. +- Versionierter neuer Servercode: **`services/netplan-v4/`** im EMS-Repo. +- Runtime/Build-Staging: **`/home/agent/services/netplan-v4-shadow`**, SQLite `data/netplan-v4.sqlite`, Compose `compose.yaml`, interner V4-Port 9100. +- Bisheriger Forecast/API-Dienst: `/home/agent/services/prognosis-manager-enelix2`; Forecast Engine interner Port 9000. Nicht durch neue V4-Arbeit unbemerkt ersetzen. +- Portal: `/home/agent/services/license`, `server.mjs`, `public/`, `compose.yaml`. Parallele GUI-Arbeit möglich. Unified-RC1 aktualisierte nur V4 und zugehöriges Planner-JS, nicht pauschal alle Portalprozesse. +- Vor Veränderungen `/srv/agent/AGENT_CONTEXT.md` lesen. Agent hatte zuletzt keinen Docker-Socket-/Root-Zugriff; den Nutzer Root-Ausführung nur für tatsächlich benötigte Deployment-Schritte erledigen lassen. Keine Berechtigungen auf Docker-Socket ausweiten. + +### iot-symcon01 / Connector Testanlage_Lihrenmoos + +- IP-Symcon 8.0, Dienst `symcon.service`, Kerneldateien `/var/lib/symcon`, Logs `/var/log/symcon`, Programme `/usr/share/symcon`. +- Installiertes EMS-Repo: `/var/lib/symcon/modules/Enelix-EMS`. Das ist produktionsnah laufender Code, nicht der Entwicklungscheckout auf Server_89. +- Staging, Tests, Diagnose: `/srv/agent/netplan-v4-...`. +- Hostregeln `/srv/agent/AGENT_CONTEXT.md` lesen. Keine allgemeinen Settings-/Credential-Exports. Für gezielte schreibgeschützte Zustandsprüfung nur erlaubte Felder extrahieren, niemals ganze secret-bearing Konfiguration ausgeben. +- **PHP mit IPS-Funktionen gehört in Symcon**, nicht in die Linux-Bash und nicht in gewöhnliches CLI-PHP. Isolierte CLI-Tests mit simulierten IPS-Funktionen umgekehrt niemals im echten Kernel ausführen. +- `MC_ReloadModule`/`IPS_ApplyChanges` können bestehende Initialisierungen und deren Nebenwirkungen ausführen. Nicht pauschal als ohne Auswirkungen versprechen. Code- und Settings-Sicherung, Hash-/Driftprüfung, Rücksetzweg nötig. +- Bei verzögerter Modulregistrierung wurde ein Wiederholen angefordert. Die letzte Installation ist inzwischen fertig: **jetzt nicht wiederholen**. + +## 8. Installierter Daten-/Prognosepfad + +1. Manager erfasst 23 numerische Quellen alle 30 s, mit originalen Zeitstempeln und Qualitätsmerkmalen. Keine zusätzlichen Modbus-Abfragen durch den bisherigen Sammler. +2. Lokale append-only Tagesdateien/Outbox. Versand regulär höchstens einmal/min, bis 120 Aufnahmen pro Batch; geordneter, bestätigter Cursor. Backoff bei Fehlern; Originale bleiben erhalten. `scheduled` kann reguläre Sendepause oder Backoff bedeuten; getrennte letzte erfolgreiche Bestätigung beachten. +3. Server-Datensatz **`lihrenmoos-physical-v1`** speichert Originalaufnahmen. Abgeleiteter Datensatz **`lihrenmoos-physical-published-v2`** referenziert dieselben Daten für die verbesserte historische Zeitbehandlung. +4. `equal_endpoint_v1`: begrenzte rückblickende Schätzung bei gleichen Werten an getrennten Originalzeitpunkten. Verfügbarkeit der späteren Bestätigung wird mitgeführt; keine Leckage in damalige Entscheidungen. Unterschiedliche Werte, lange Ausfälle, widersprüchliche Quellen oder Sammellücken nicht unbemerkt überbrücken. Live-Grenzen unverändert. +5. Worker verarbeitet im 5-Minuten-Takt. Mindestbasis 24 **verwertbare äquivalente Stunden**, nicht bloss seit Start verstrichene Zeit. Neue Tagesprofile, versionierte Modelle, konfigurierbare tägliche/wöchentliche Neubildung. Kaltstart/Validierung sind getrennt. +6. Neue Datenquelle muss als `corrected_profile` mit passendem `measurementDataset` gewählt werden, sobald verwendbar; zuletzt blieb **`forecastSource=legacy`, `family=3`**. Nicht bei Installation automatisch umgeschaltet. `optimal` auf der Legacy-Quelle ist kein Nachweis der neuen Prognosequalität. +7. Wirtschaftlicher Vergleich Unified-RC1 ist angeschlossen, aber v1 ist **eingefrorener Tagesplan + simulierte lokale Netzzielnachführung**, nicht vollständiges rollendes MPC-Replay. Eine netzladefähige Batterie; gleiche Anfangsbedingungen, Restenergie, Verluste und einmalige Monatspeakabrechnung. Historischer SDL-Auftrag als gekennzeichnete Schätzung. Vollständige Modellautomatik/Produktversprechen nicht über diesen Scope hinaus behaupten. +8. Archivierung ist implementiert, **AUS als Standard** (native `NetzfahrplanV4ArchivAktiv`, serverseitig `NETPLAN_V4_ARCHIVE_ENABLED`). Verifizierte komprimierte Archive ersetzen kein externes Backup und lösen keine unendliche Speicherkapazität. Keine unbestätigten Outbox-Daten löschen. + +## 9. Behobene Fehler – nicht erneut aufrollen + +- Früherer numpy-Fehler `assignment destination is read-only`, falscher Import `battery_optimizer_new`, Dateirechte im unprivilegierten Container: damalige Deployment-/Codefehler, nicht aktuelle Blocker. +- Direkt nach Start `Connection refused`, danach HTTP 409 „bereits ein Lauf aktiv“: Start-/Parallelitätszustand, nicht durch wiederholte Neustarts lösen. +- Native Betriebsdaten fehlten; anschließend Sender installiert. Tarifimporter und Portal-Rate-Limit wurden repariert. Alte 429-Meldungen nicht ohne neue Evidenz zur heutigen Ursache erklären. +- Klasse `NetzfahrplanV4Messaufnahme` doppelt aus Staging und Modulpfad geladen: Installer auf eindeutige installierte Bibliothek umgestellt. `require_once` verhindert nicht verschiedene Dateien mit derselben Klasse. +- Gemischter 120er-Block: 45 importierte/75 native Aufnahmen, keine doppelten/rückwärtslaufenden Zeiten, aber andere Kennung allein durch `splitToleranceW:100.0` vs `100`. Normalisierung korrigiert; beide vollständigen Konfigurationen exakt geprüft. Kompatibilität nur für diese anlagen- und datensatzgebundene Gleichwertigkeit registriert; Ursprungskennung/Evidence gespeichert. Keine Hashs in Originaljournalen ersetzt, kein Cursor-Sprung. +- **Nachweis der Reparatur:** am 03.10. 12:58 Schweizer Zeit 1'782 statt 1'320 Aufnahmen, jüngste Aufnahme 12:58:09, 25 s alt, vollständiger ehemals blockierter Batch vorhanden, Manager `acknowledged`/Fehlerzahl 0. Quelle `[F]` unten. Keine Garantie über den danach liegenden Zeitraum ableiten. +- Historische Datenqualität verbesserte sich von 0 brauchbaren Fenstern auf 104/8.66 h, nach Nachversand 130/10.82 h; zuletzt Worker 140/11.65 h. Prozentangaben zur zeitlichen Abdeckung sind **keine Messgenauigkeit und keine Prognosegüte**. + +## 10. Installierter Rückmelde- und begrenzter Stellpfad + +Gemeinsamer Code: `NetzfahrplanV4Rueckmeldung.php`, `BatterieNetzfahrplanV4RueckmeldungTrait.php`, `ManagerNetzfahrplanV4EmpfangTrait.php`, `ManagerNetzfahrplanV4TestTrait.php`, `BatterieNetzfahrplanV4TestTrait.php`, `NetzfahrplanV4Regeltest.php`, Manager-/Batterie-Module. + +- Zweifaches Lesen mit Identitäts- und Originalzeitprüfung; momentan Live-Alter maximal 60 s, Quellenversatz maximal 30 s. +- Modus in Lihrenmoos **`virtual_split`**, **`allowEstimatedForTrial=false`**. Auch gültige Rückmeldung oder Vorschau erteilt keine Freigabe. +- Wenn SDL-Auftrag exakt null, physische Batteriesumme statt gefilterter virtueller EV-Anzeige verwenden; unveränderter Nullauftrag ist Zustand, kein Messgeräte-Heartbeat. Eine neue EV-Vorgabe bedeutet noch keine sofortige physische Reaktion. +- Bei SDL ungleich null: zeitliche Abdeckung von Auftragsänderung, Tracking-Toleranz und explizite Schätzkennzeichnung. Gegenläufige EV-/SDL-Flüsse sind nicht eindeutig identifizierbar und bleiben für diesen Test gesperrt. +- Netzziel wird in **Gesamt-Batterieleistung**, nicht zusätzlichen Ladeauftrag übersetzt. Erwartete Änderungen anderer Verbraucher genau einmal berücksichtigen. +- Separate interne Server-Testsitzung, leere Default-Allowlist `NETPLAN_V4_CONTROL_TRIAL_PLANTS`, manuelle Familie, Anlagen-/Asset-/Instanz-/Revisionsbindung, beide lokalen Freigaben `NetzfahrplanV4RegeltestErlaubt`, bewusster lokaler Start, gültiger Plan, Bilanz- und Gerätewatchdog-Nachweis. +- Test max. 30 min / 5 kW je Richtung; Serverauthority maximal 90 s; Batteriebefehl maximal 10 s; Batterie-Softwarewatchdog 1 s, Manager-Testtakt 2 s. Nicht einfach Limits für Produktion hochsetzen. +- Batterie liest vor Ausgabe unabhängig neu und prüft SOC, Reserve, Hysterese, Verfügbarkeit, Sperrzeiten und Netzgrenzen. Unmögliches Netzziel → Testabbruch, keine Erfolgsbehauptung. +- Abbruch widerruft Sitzung vor Nullstellversuch; Stoppfehler bleibt sichtbar. Danach maximal eine **frisch berechnete** normale Zuteilung, keine Rückkehr zu gecachtem Test-/Vor-Testwert. Keine Wiederbelebung nach Neustart oder doppeltem Befehl. +- `shadow_seen` bedeutet Empfang, **nicht ausgeführt**. `canDispatch=false` in Vorschau bedeutet keine Vorschau-Freigabe. Bei einer separat gestarteten Testsitzung wäre `runMode=shadow` alleine trotzdem kein hinreichender Nachweis für null Stellbefehle. +- Softwarewatchdog funktioniert nur bei laufendem Kernel/Kommunikation. Kein Nachweis für Rechnerabsturz, Stromausfall oder unerreichbaren Wechselrichter. Bereits vorhandenes Verhalten prüfen, nicht erfinden und nicht Schutzgates entfernen. + +## 11. Aktueller Blocker: bestätigter Geräteabruf statt falscher Zeitbasis + +Letzter Installationsbericht `[A]`: + +``` +status: corrected_feedback_installed_trial_disabled +feedback.status: unavailable +feedback.reason: feedback_source_skew +batteryW: null +gridW: null +usableForTrial: false +canDispatch: false +``` + +Das ist **kein Installationsfehler**. Die Meldung „Modellanteil: nein“ im Installer ist im Fehlerfall irreführend: fehlendes Feld wird als false dargestellt, obwohl die Berechnung gar nicht erfolgreich war. Kleine UI-/Diagnosekorrektur mitnehmen. + +Letzte Zeituntersuchung `[B]` (237 Aufnahmen / 2 h): 48 Aufnahmen mit Versatz >30 s. Netz median 1 s/max 2 s, GoodWe 1 median 1 s/max 48 s, GoodWe 2 median 1 s/max 46 s, SolarEdge median 12 s/max 59 s; SolarEdge in 223 Aufnahmen unter den ältesten Quellen. Keine der vier Quellen in genau diesem Zeitfenster über 60 s. Beispiel 13:58:38: drei Quellen neu, SolarEdge 31 s alt → Ablehnung. Frühere mehrminütige Lücken trotzdem nicht damit wegargumentieren. + +**Nächste Umsetzung, ohne neue Diagnosekategorie:** + +1. Installierten Gerätemodultyp und unterstützten Leseweg verifizieren. Im Gespräch wurde `ModBus_RequestRead()` als möglicher Ansatz genannt. API-/Erfolgs-/Zeitsemantik und Unterschied `ModBus Address` vs `ModBus Device` anhand offizieller aktueller Dokumentation und vorhandenem Code prüfen; nicht voraussetzen, dass ein boolescher Erfolg alle Register atomar aktualisiert oder einen unabhängigen Gerätezeitpunkt liefert. +2. Erfolgreiche vollständige Antwort mit genau den daraus stammenden Werten, Zuordnung, Status und Bestätigungszeit binden. `VariableUpdated`, Geräteabrufbestätigung und Sammlerzeit getrennt erhalten. +3. Begrenzte Abfragefrequenz, Timeout, Serialisierung, Kommunikationslast und Lesekohärenz berücksichtigen. Keine Netz-I/O in jedem Vorschau-/Regelcallback vervielfachen, keine blockierenden Abfragen unter dem Stellregler-Lock. Existierende erfolgreiche Polls nach Möglichkeit nutzen. +4. Wenn keine echte Bestätigung vorliegt, unsicher/gesperrt bleiben. Keine künstlichen `SetValue`-Herzschläge und keine permissive Freigabe bloss wegen des Pollerintervalls. +5. In bestehenden Batterie-/Managerpfad integrieren; Tests: unveränderte Werte mit erfolgreichem Abruf, fehlgeschlagene/partielle Antwort, Zeitversatz, Race, Wiederholung, Neustart, erneute unabhängige Batterieprüfung und Rückfall. +6. Zusammenhängenden Kandidaten bauen und testen; neue Runtime-Installation nur im bekannten gesicherten Verfahren. Bisherige Installer nicht nochmals ausführen. + +Diese Arbeit wurde am Gesprächsende **noch nicht implementiert oder installiert**. Sie ist nicht durch die vorliegende Dokumentationsübergabe erledigt. + +## 12. Produktionsreife: verbleibende echte Arbeiten + +- Verlässlicher bestätigter Live-Messpfad wie oben; unklares AC/DC/externes SDL nicht durch Labels als verifiziert deklarieren. +- Neue Lastprofile mit tatsächlicher ausreichender Datenbasis/Validierung aufbauen und gezielt zur Schattenrechnung schalten. Aktuelle Einstellungen/Modelle vorher lesen, nicht alte Momentaufnahmen fortschreiben. +- **Dauerbetrieb** mit sauberem Authority-/Zustands-/Neustart-/Fallback-Verhalten implementieren. Bisher nur begrenzter Pilot, keine fertige kontinuierliche Produktionsfreigabe. +- Reale Callback-/Registerstrecke und separat freigegebener Stellversuch; korrekte Vorzeichen, verfügbare Leistungen, Netzziel/Peak-/Exportgrenzen, Abschalten/Verbindungsausfall, lokale Priorität und SDL-Konflikte prüfen. +- Modellautomatik über den deklarierten Frozen-Day-v1-Scope hinaus nur nach Erweiterung/Nachweis bewerben; Grenzen transparent halten. +- Lebenszyklus/Backups/Recovery/Observability, Mehranlagenisolierung und parametrisierte Inbetriebnahme. Einige Installer sind bewusst auf Lihrenmoos gebunden; für andere Anlagen nicht IDs blind übernehmen. +- Git-Authentifizierung/Veröffentlichung lösen; `develop` → geprüfte `beta` → Feldabnahme `main`. Browser-Login, Tailscale oder Portforwarding stellen nicht automatisch Git-Schreibauthentifizierung bereit. Keine Veröffentlichung behaupten, die nicht passiert ist. + +## 13. Wichtige Meilensteine und nicht erneut auszuführende Installer + +| Paket | Status bis Gesprächsende | +|---|---| +| V4-Schattenserver / Native Operation / Tarife | Installiert, Betrieb historisch belegt | +| Passiver Empfänger `/srv/agent/netplan-v4-receiver-stage/install.php` | Installiert, automatische `shadow_seen`-ACKs belegt | +| Rohsammler `/srv/agent/netplan-v4-data-capture-stage` | Separater früherer Sammler; Originaldateien erhalten | +| Leseexport `/srv/agent/netplan-v4-data-review-stage` | Einmaliger Export wegen restriktiver Originalrechte; heute keine erneute allgemeine Exportpflicht | +| Getrennter Beobachter `/srv/agent/netplan-v4-separated-observer-stage` | 23 Quellen, Dateien gezielt lesbar; weiterhin Originalbestand | +| Managerdaten `/srv/agent/netplan-v4-application-build/install.php` | Installiert, Klasse-/Kennungsfehler später behoben | +| `deploy_history_timing.py` | Installiert, neue Historienversion v2 | +| `deploy_unified_release.py --approve-numeric-mapping-compatibility` + `/srv/agent/netplan-v4-unified-release/install.php` | Installiert, Aliasfreigabe exakt registriert, Nachversand aufgeholt | +| `/srv/agent/netplan-v4-feedback-stage/install.php` | **Fertig installiert nach einmaliger Registrierungsschleife**, aktuell Zeitversatzprüfung | + +Fachliche Commitfolge (lokales `develop`; nicht als Remote-/Runtime-Gleichstand missverstehen): +`b5c1fa6` Empfänger, `9394fe3` begrenzter Test, `e0e52d2` Bilanzierung, `76f8b06` Capture, `44362e4` getrennte Beobachtung, `fe3f46b` Datenanwendung, `6d9aba7` Klassenladen, `5890093` Historienzeit, `6c91d6b` Zahlennormalisierung, `8eb9688` Unified RC1, **`9878197` korrigierte Rückmelde-/Testkette**. Einige weitere Commits können dazwischenliegen; `git log` ist massgeblich. + +## 14. Quellenindex und Tests – zuerst Nachweise lesen, nicht neu installieren + +**[A] Letzte erfolgreiche Rückmeldeinstallation (Testhost):** +`/srv/agent/netplan-v4-feedback-stage/INSTALL_RESULT.json` – Ende `2026-10-03T11:59:58Z`. Der frühere `waiting_for_registration`-Bericht ist überholt. Die Sicherung stammt vom ersten Dateidurchlauf, nicht vom zweiten `already_installed`-Durchlauf. + +**[B] Letzte Zeit-/Pipeline-Auswertung (Server):** +`/home/agent/services/qa/feedback-timing-20261003/TIMING_REVIEW.json` – `2026-10-03T12:02:13Z`, 140 brauchbare Fenster/11.646 h, Quelle v2 noch `collecting`. + +**[C] Unified-Serverinstallation:** +`/home/agent/services/netplan-v4-shadow/unified-releases/20261003T105407.036202Z/REPORT.json`; Runtime `unified-rc1`, Mappingkompatibilität registriert, Einstellungen unverändert. + +**[D] Unified-Managerinstallation (Testhost):** +`/srv/agent/netplan-v4-unified-release/INSTALL_RESULT.json`. + +**[E] Historieninstallation:** +`/home/agent/services/netplan-v4-shadow/history-timing-releases/20261003T092245.175100Z/REPORT.json`. + +**[F] Erfolgreiches Aufholen des Datenwegs:** +`/home/agent/services/qa/unified-release-20261003/FLOW-20261003T105834.324945Z.json`. + +**[G] Installiertes und versioniertes Verhalten:** +`docs/netplan-v4-corrected-feedback.md`, `docs/netplan-v4-unified-release.md`, `docs/netplan-v4-history-timing.md`, `docs/netplan-v4-numeric-mapping-fix.md`, `docs/netplan-v4-application.md`, `docs/netplan-v4-controlled-trial.md`, `docs/netplan-v4-accounting.md`. Alte „noch nicht installiert“-Abschnitte dieser Entwicklungsdokumente durch aktuelle Installationsberichte einordnen; nicht mit echten Runtime-Nachweisen verwechseln. + +**[H] Rückmelde-/Testsuite (Testhost):** +`/srv/agent/netplan-v4-feedback-stage/PREPARATION.json`, `TEST_RESULTS.txt`, `INSTALLER_TEST_RESULTS.json`, `ORDINARY_OFFER_TEST.json`, `MANIFEST.json`, `PACKAGE_HASHES.json`, `feedback-config.json`. +125 isolierte Funktionsprüfungen, 4 vollständige Modul-Linkageprüfungen, 10 Installationsszenarien, 13'824 übereinstimmende normale Leistungsangebotsfälle. Keine Aussage, dass dieselbe Zahl Tests im echten Kernel/Wechselrichter gelaufen sei. + +**[I] Unified-Suite (Server):** +`/home/agent/services/qa/unified-release-20261003/PYTHON_TEST_RESULTS.txt`, `NODE_TEST_RESULTS.txt`, `PUBLICATION.json`, `DELIVERY_STATUS.json`. +306 Python- und 16 Portaltests; native separate Tests. Spätere Änderungen brauchen erneute relevante Tests, nicht bloss Übernahme dieser Zahlen. + +**[J] Wiederaufbau und Code:** +EMS `examples/V4CorrectedFeedback/`, `examples/V4UnifiedRelease/`, `services/netplan-v4/commissioning/`. Staging ist nicht dieselbe Quelle wie das laufende Modul. Dateien, Hashs und Abhängigkeiten vor Änderungen vergleichen; Paralleländerungen abbrechen statt überschreiben. + +**[K] Quellen-/Konfigurationsdiagnose (Testhost):** +`/srv/agent/netplan-v4-application-build/inspect_source_update_policy.py`, `inspect_delivery_state.py`, `MAPPING_IDENTITY_DIAGNOSIS.json`, `MAPPING_EQUIVALENCE.json`; `netplan-v4-feedback-stage/review_current_feedback_state.py`. Diese Skripte zuerst lesen; manche erzeugen nur einen Bericht, manche lesen gespeicherte statt live aktuelle Settings. Alte Settings-Snapshots können eine bereits installierte Eigenschaft noch nicht enthalten. + +**Sicherheitsnotiz:** In einem alten Upload-Skript wurden Klartextzugangsdaten bemerkt. Nicht lesen, verbreiten, in Dokumentation/Git übernehmen oder für neue Integration verwenden. Separate autorisierte Rotation/Secret-Auslagerung bleibt Aufgabe; die Werte gehören nicht in diese Übergabe. + +## 15. Übernahmeroutine für einen neuen Agenten + +1. Host-`AGENT_CONTEXT.md`, Repo-`AGENTS.md` und diese Datei vollständig lesen; bei fehlendem Remote-Stand zuerst den Servercheckout prüfen. +2. `git status`/Branch/letzte Commits sowie `[A]`, `[B]`, `[C]` lesen. Zeitstempel nennen; keine alten Zähler als heutige Werte ausgeben. Diese Übernahme selbst ändert keine Anlage. +3. Nur notwendige gezielte Read-only-Prüfungen. Kein weiterer vollständiger Neustart der Analyse oder Wiederholung bereits erledigter Installer. +4. Bei Fortsetzungsauftrag direkt Abschnitt 11 umsetzen und anschliessend echte Restarbeiten aus Abschnitt 12 abarbeiten. Anforderungen, Testfall, Implementierung und Rollout in einem Paket halten. +5. Nach jedem Abschluss diese Übergabe mit Datum, Codecommit, tatsächlichem Installations-/Teststatus und nächstem konkreten Arbeitsschritt ergänzen. Keine gescheiterten Schritte in „fertig“ umdeuten. Bei Widerspruch aktuelle geprüfte Laufzeit und ausdrückliche spätere Nutzerkorrektur nennen. + +**Kurzübernahme:** „Netzfahrplan V4 in ENELIX fortsetzen. Datenkennung/Versand repariert, Unified RC1 und korrigierter Feedback-/Testpfad installiert. Aktuell `feedback_source_skew`; bestätigte Geräte-Lesezeitpunkte in die bestehende lokale Rückmeldung integrieren. EV 161.44 kWh/39 kW. Keine neue Diagnosekategorie, keine alten Installer, kein automatischer Stelltest. Dauerproduktion ist noch nicht fertig; Tests/Abnahme/Veröffentlichung sauber getrennt halten.“ From 30f7652803ad15471145d130f611f0bf536ec239 Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Sun, 4 Oct 2026 09:07:30 +0000 Subject: [PATCH 17/22] feat(forecast): commission corrected Lihrenmoos profile --- docs/NETPLAN_V4_AGENT_HANDOFF.md | 2 + .../activate_corrected_forecast.py | 256 ++++++++++++++++++ .../netplan_v4/measurement_pipeline.py | 14 +- .../test_corrected_forecast_commissioning.py | 116 ++++++++ .../tests/test_measurement_pipeline.py | 8 +- 5 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 services/netplan-v4/commissioning/activate_corrected_forecast.py create mode 100644 services/netplan-v4/tests/test_corrected_forecast_commissioning.py diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md index 780137e..2bae37f 100644 --- a/docs/NETPLAN_V4_AGENT_HANDOFF.md +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -4,6 +4,8 @@ **Zeitbezug:** Die letzten hier belegten Betriebsberichte stammen vom **03.10.2026, ca. 14:02 Uhr Europe/Zurich**. Am 04.10. wurden für diese Übergabe die genannten Berichte und der Git-Stand erneut gelesen, aber keine neue Anlagenabnahme durchgeführt. Alte Messzahlen deshalb niemals als heutige Live-Werte ausgeben. +**Fortsetzung Prognose am 04.10.2026:** Der aktuelle Runtime-Stand wurde gelesen, ohne die alten Diagnose- oder Installationsschritte zu wiederholen. Der abgeleitete Datensatz `lihrenmoos-physical-published-v2` ist `model_ready` und erfüllt mit mehr als 24 nutzbaren äquivalenten Stunden die Trainingsschwelle. Die V4-Einstellungen stehen seit Revision 2 in `shadow` auf `forecastSource=corrected_profile` und `measurementDataset=lihrenmoos-physical-published-v2`; `liveEnabled` und Stellfreigabe bleiben aus. Der erste echte Lauf zeigte einen kompatibilitätsbedingten Stopp bei mikrosekundengenauen `observedAt`-Zeitstempeln. Die Korrektur rundet ausschliesslich die kausale Verfügbarkeit von Prognoseereignissen auf die nächste volle Sekunde auf; Rohmessungen bleiben streng ganzsekündlich. 310 Python- und 16 Portal-Tests sind erfolgreich. Repository und Runtime-Build-Kontext enthalten bytegleich den geprüften Fix; der laufende Container enthält ihn noch nicht, weil beide Agentzugänge weder Docker-Socket noch `sudo` erhalten. Der vorbereitete, syntaktisch geprüfte Root-Rollout liegt unter `/home/agent/services/netplan-v4-shadow/commissioning/finish_corrected_forecast_rollout.sh` und baut/testet vor dem Austausch, hält das vorige Image als Rollback fest und ändert keine Stellfreigabe. Daniel hat danach einen Aktivbetrieb angefragt; der aktuelle V4-Kern ist jedoch technisch `shadow-only`, und die weiterhin unbelegte Rückmeldung (`feedback_source_skew`, `usableForTrial=false`, `canDispatch=false`) verbietet ein blosses Umstellen. Keine Live-Freigabe wurde erteilt oder eingebaut. Git-Klarstellung durch Daniel: Commit-Autor `dh_Agent `, Gitea-Konto `dh`; Commit/Push auf `develop` und `beta` am 04.10.2026 ausdrücklich freigegeben. Der Agentzugang besitzt derzeit keine HTTPS-Schreibanmeldung für `dh`. + ## 1. Zuerst lesen: Wo wir tatsächlich stehen - Daniel möchte die **fertige, produktionsgeeignete Anwendung**, nicht weitere isolierte Sammler, Diagnosekategorien oder wiederholte Bestätigungsrunden. Er hat die fortlaufende Umsetzung mehrfach beauftragt. Probleme im Code selbst beheben, Tests und Auslieferung bündeln; ihn nur für wirklich notwendige Root-/Symcon-Ausführung oder echte Anlagenfreigaben einbeziehen. diff --git a/services/netplan-v4/commissioning/activate_corrected_forecast.py b/services/netplan-v4/commissioning/activate_corrected_forecast.py new file mode 100644 index 0000000..f3b251a --- /dev/null +++ b/services/netplan-v4/commissioning/activate_corrected_forecast.py @@ -0,0 +1,256 @@ +"""Commission the corrected Lihrenmoos forecast in shadow mode. + +The command is a dry run unless --apply is supplied. It never enables live control, +changes Symcon actuator flags, or reads/prints service credentials. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sqlite3 +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +SERVICE_ROOT = Path(__file__).resolve().parents[1] +if str(SERVICE_ROOT) not in sys.path: + sys.path.insert(0, str(SERVICE_ROOT)) +RUNTIME_ROOT = Path("/home/agent/services/netplan-v4-shadow") +PLANT = "e3a08f9e-af12-4695-99bd-8b51c0520021" +DATASET = "lihrenmoos-physical-published-v2" +SOURCE_FILES = ( + "netplan_v4/store.py", + "netplan_v4/measurement_pipeline.py", + "netplan_v4/service.py", +) + + +def digest(path: Path) -> str: + if path.is_symlink() or not path.is_file(): + raise ValueError("Source file missing or is a symlink") + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify_runtime_source(runtime_root: Path) -> None: + for name in SOURCE_FILES: + if digest(SERVICE_ROOT / name) != digest(runtime_root / name): + raise ValueError("Runtime source differs from reviewed repository: " + name) + + +def read_state(db: Path, plant: str, dataset: str, now: int) -> dict: + stamp = db.stat() + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + con.row_factory = sqlite3.Row + try: + from netplan_v4.store import DEFAULT_SETTINGS + + row = con.execute( + "SELECT revision,value FROM planner_settings WHERE plant=?", (plant,) + ).fetchone() + settings = {**DEFAULT_SETTINGS, **(json.loads(row["value"]) if row else {})} + settings["revision"] = row["revision"] if row else 0 + ds = con.execute( + "SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?", + (plant, dataset), + ).fetchone() + if not ds: + raise ValueError("Corrected dataset is not registered") + config = json.loads(ds["config"]) + state = con.execute( + "SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?", + (plant, dataset), + ).fetchone() + model = con.execute( + """SELECT m.model_id,m.trained_at,m.trained_through,m.value + FROM planner_model_current c JOIN planner_load_models m + ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id + WHERE c.plant=? AND c.dataset=?""", + (plant, dataset), + ).fetchone() + observation_dataset = config.get("sourceDatasetId", dataset) + observation = con.execute( + """SELECT COUNT(*) records,MAX(captured_at) last_capture, + MAX(received_at) last_received + FROM planner_observations WHERE plant=? AND dataset=?""", + (plant, observation_dataset), + ).fetchone() + plan = con.execute( + """SELECT p.value FROM planner_current c JOIN planner_plans p + ON p.plan_id=c.plan_id WHERE c.plant=? AND c.mode='shadow'""", + (plant,), + ).fetchone() + detail = json.loads(state["detail"]) if state else {} + model_value = json.loads(model["value"]) if model else None + return { + "settings": settings, + "dataset": dataset, + "observationDataset": observation_dataset, + "pipelineStatus": state["status"] if state else None, + "pipelineDetail": detail, + "records": observation["records"], + "lastCapture": observation["last_capture"], + "lastReceived": observation["last_received"], + "activeModel": { + "modelId": model["model_id"], + "trainedAt": model["trained_at"], + "trainedThrough": model["trained_through"], + "validation": model_value.get("validation"), + } if model else None, + "currentPlan": json.loads(plan["value"]) if plan else None, + "checkedAt": now, + "databaseMtimeNs": stamp.st_mtime_ns, + } + finally: + con.close() + + +def require_ready(state: dict, now: int) -> None: + if state["settings"]["runMode"] != "shadow": + raise ValueError("Only shadow mode may be commissioned") + if state["pipelineStatus"] != "model_ready" or not state["activeModel"]: + raise ValueError("Corrected profile model is not ready") + if state["records"] < 1 or not isinstance(state["lastCapture"], int): + raise ValueError("Corrected observations are missing") + if not 0 <= now - state["lastCapture"] <= 600: + raise ValueError("Corrected observations are not current") + if not 0 <= now - state["lastReceived"] <= 600: + raise ValueError("Corrected delivery is not current") + detail = state["pipelineDetail"] + if detail.get("modelId") != state["activeModel"]["modelId"]: + raise ValueError("Pipeline and active model disagree") + if detail.get("usableEquivalentHours", 0) < detail.get("requiredEquivalentHours", 24): + raise ValueError("Minimum usable equivalent hours not reached") + + +def backup_database(source: Path, destination: Path) -> None: + if destination.exists() or source.is_symlink(): + raise ValueError("Backup destination must be new and source must be regular") + with sqlite3.connect(f"file:{source}?mode=ro", uri=True) as src, sqlite3.connect(destination) as dst: + src.backup(dst) + if dst.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise ValueError("Database backup integrity check failed") + destination.chmod(0o600) + + +def plan_uses_dataset(db: Path, plant: str, dataset: str, revision: int) -> dict | None: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + row = con.execute( + """SELECT p.value FROM planner_current c JOIN planner_plans p + ON p.plan_id=c.plan_id WHERE c.plant=? AND c.mode='shadow'""", + (plant,), + ).fetchone() + if not row: + return None + plan = json.loads(row[0]) + pipeline = plan.get("inputQuality", {}).get("dataPipeline", {}) + if plan.get("configRevision") == revision and pipeline.get("datasetId") == dataset: + return plan + return None + finally: + con.close() + + +def activate(db: Path, plant: str, dataset: str, now: datetime) -> dict: + from netplan_v4.store import PlannerStore + + store = PlannerStore(str(db)) + try: + before = store.settings(plant) + if before["runMode"] != "shadow": + raise ValueError("Runtime left shadow mode") + if before["forecastSource"] == "corrected_profile" and before["measurementDataset"] == dataset: + store.request(plant, "manual", now) + return {"changed": False, "revision": before["revision"]} + result = store.save_settings( + plant, + {"forecastSource": "corrected_profile", "measurementDataset": dataset}, + before["revision"], + now, + ) + return {"changed": True, "revision": result["revision"]} + finally: + store.close() + + +def commission(runtime_root: Path, plant: str, dataset: str, apply: bool, wait_seconds: int = 90) -> dict: + plant = str(UUID(plant)) + if runtime_root.resolve() != RUNTIME_ROOT: + raise ValueError("Unexpected runtime root") + verify_runtime_source(runtime_root) + db = runtime_root / "data/netplan-v4.sqlite" + now = datetime.now(timezone.utc).replace(microsecond=0) + before = read_state(db, plant, dataset, int(now.timestamp())) + require_ready(before, int(now.timestamp())) + report = { + "scope": "corrected_forecast_shadow_commissioning", + "startedAt": now.isoformat(), + "installationId": plant, + "datasetId": dataset, + "liveEnabled": False, + "actuatorPermissionGranted": False, + "before": { + "settings": before["settings"], + "pipelineStatus": before["pipelineStatus"], + "records": before["records"], + "lastCapture": before["lastCapture"], + "activeModel": before["activeModel"], + "usableEquivalentHours": before["pipelineDetail"].get("usableEquivalentHours"), + }, + } + if not apply: + report["status"] = "ready_no_changes" + return report + + folder = runtime_root / "corrected-forecast-releases" / now.strftime("%Y%m%dT%H%M%SZ") + folder.mkdir(parents=True, mode=0o700) + backup_database(db, folder / "before.sqlite") + activated = activate(db, plant, dataset, now) + report["activation"] = activated + deadline = time.monotonic() + wait_seconds + plan = None + while time.monotonic() < deadline: + plan = plan_uses_dataset(db, plant, dataset, activated["revision"]) + if plan: + break + time.sleep(2) + after = read_state(db, plant, dataset, int(datetime.now(timezone.utc).timestamp())) + settings = after["settings"] + if settings["runMode"] != "shadow" or settings["forecastSource"] != "corrected_profile" or settings["measurementDataset"] != dataset: + raise ValueError("Corrected shadow settings were not retained") + report["after"] = { + "settings": settings, + "pipelineStatus": after["pipelineStatus"], + "activeModel": after["activeModel"], + "planId": plan.get("planId") if plan else None, + "planStatus": plan.get("status") if plan else None, + "sourceFamily": plan.get("sourceFamily") if plan else None, + "dataPipeline": plan.get("inputQuality", {}).get("dataPipeline") if plan else None, + } + report["status"] = "corrected_forecast_running_shadow" if plan else "corrected_forecast_activated_plan_pending" + report["finishedAt"] = datetime.now(timezone.utc).isoformat() + output = folder / "REPORT.json" + output.write_text(json.dumps(report, indent=2) + "\n") + output.chmod(0o640) + report["reportPath"] = str(output) + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plant", default=PLANT) + parser.add_argument("--dataset", default=DATASET) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--wait-seconds", type=int, default=90) + args = parser.parse_args() + if not 1 <= args.wait_seconds <= 300: + raise SystemExit("wait-seconds must be 1..300") + try: + result = commission(RUNTIME_ROOT, args.plant, args.dataset, args.apply, args.wait_seconds) + except Exception as exc: + raise SystemExit("Stopped: " + type(exc).__name__ + ". No actuator permission was changed.") from exc + print(json.dumps(result, indent=2)) diff --git a/services/netplan-v4/netplan_v4/measurement_pipeline.py b/services/netplan-v4/netplan_v4/measurement_pipeline.py index 61ca5a9..e5af4da 100644 --- a/services/netplan-v4/netplan_v4/measurement_pipeline.py +++ b/services/netplan-v4/netplan_v4/measurement_pipeline.py @@ -9,7 +9,7 @@ from bisect import bisect_right from collections import defaultdict from datetime import datetime, timedelta, timezone from hashlib import sha256 -from math import isfinite +from math import ceil, isfinite from statistics import median from zoneinfo import ZoneInfo import json @@ -34,6 +34,16 @@ def epoch(value): return int(t.timestamp()) +def availability_epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0: + raise ValueError('Explicit UTC timestamp required') + # Never claim an input existed before its sub-second observation completed. + return int(ceil(t.timestamp())) + + def iso(t): return datetime.fromtimestamp(t, UTC).isoformat() @@ -479,7 +489,7 @@ def apply_load_forecast(con,plant,dataset,forecast,decision): raise ValueError('No current external SDL request for the persistence scenario') sdl=r['value']*s['factorToW'] result=json.loads(canonical(forecast)); result['families']={} - result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + result['observedAt']=iso(max(availability_epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) for family,old in forecast['families'].items(): if family not in FAMILIES: continue points=[] diff --git a/services/netplan-v4/tests/test_corrected_forecast_commissioning.py b/services/netplan-v4/tests/test_corrected_forecast_commissioning.py new file mode 100644 index 0000000..dda7d69 --- /dev/null +++ b/services/netplan-v4/tests/test_corrected_forecast_commissioning.py @@ -0,0 +1,116 @@ +import importlib.util +import json +import sqlite3 +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "activate_corrected_forecast", ROOT / "commissioning/activate_corrected_forecast.py" +) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + +from netplan_v4 import measurement_pipeline +from netplan_v4.store import PlannerStore + +PLANT = module.PLANT +DATASET = module.DATASET +NOW = 1791099600 + + +class CorrectedForecastCommissioningTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db = Path(self.tmp.name) / "planner.sqlite" + self.store = PlannerStore(str(self.db)) + config = { + "datasetId": DATASET, + "mappingSha256": "a" * 64, + "inventorySha256": "b" * 64, + "formula": "physical_sum_v1", + "solarReference": None, + "minimumCoverage": 0.95, + "maximumGapSeconds": 10, + "minimumTrainingHours": 24, + "historyDays": 14, + "sources": [ + {"key": "grid", "variableId": 1, "role": "grid", "factorToW": 1, "maxAgeSeconds": 60}, + {"key": "pv", "variableId": 2, "role": "pv", "factorToW": 1, "maxAgeSeconds": 60}, + {"key": "battery", "variableId": 3, "role": "physical_storage", "factorToW": 1, "maxAgeSeconds": 60}, + ], + } + measurement_pipeline.register_dataset(self.store.con, PLANT, config, NOW - 100) + model = { + "version": 1, + "datasetId": DATASET, + "methodVersion": "profile_v1", + "trainedAt": NOW - 60, + "trainedThrough": NOW - 300, + "profiles": {}, + "validation": {"status": "bootstrap_insufficient_holdout"}, + } + model_id = "c" * 64 + with self.store.con: + self.store.con.execute( + "INSERT INTO planner_load_models VALUES(?,?,?,?,?,?)", + (PLANT, DATASET, model_id, NOW - 60, NOW - 300, json.dumps(model)), + ) + self.store.con.execute( + "INSERT INTO planner_model_current VALUES(?,?,?)", (PLANT, DATASET, model_id) + ) + self.store.con.execute( + "INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?)", + ( + PLANT, + DATASET, + 1, + "model_ready", + json.dumps( + { + "modelId": model_id, + "usableEquivalentHours": 24.1, + "requiredEquivalentHours": 24, + } + ), + ), + ) + self.store.con.execute( + "INSERT INTO planner_observations VALUES(?,?,?,?,?,?)", + (PLANT, DATASET, NOW - 30, NOW - 20, "d" * 64, "{}"), + ) + self.store.close() + + def tearDown(self): + self.tmp.cleanup() + + def test_ready_state_can_activate_only_corrected_shadow_settings(self): + state = module.read_state(self.db, PLANT, DATASET, NOW) + module.require_ready(state, NOW) + result = module.activate( + self.db, PLANT, DATASET, datetime.fromtimestamp(NOW, timezone.utc) + ) + self.assertTrue(result["changed"]) + store = PlannerStore(str(self.db)) + settings = store.settings(PLANT) + self.assertEqual(settings["forecastSource"], "corrected_profile") + self.assertEqual(settings["measurementDataset"], DATASET) + self.assertEqual(settings["runMode"], "shadow") + store.close() + + def test_stale_observations_refused(self): + state = module.read_state(self.db, PLANT, DATASET, NOW + 700) + with self.assertRaises(ValueError): + module.require_ready(state, NOW + 700) + + def test_backup_is_consistent(self): + backup = Path(self.tmp.name) / "backup.sqlite" + module.backup_database(self.db, backup) + with sqlite3.connect(backup) as con: + self.assertEqual(con.execute("PRAGMA integrity_check").fetchone()[0], "ok") + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/tests/test_measurement_pipeline.py b/services/netplan-v4/tests/test_measurement_pipeline.py index ab8cfa6..85b4f38 100644 --- a/services/netplan-v4/tests/test_measurement_pipeline.py +++ b/services/netplan-v4/tests/test_measurement_pipeline.py @@ -67,6 +67,9 @@ class MeasurementPipelineTest(unittest.TestCase): def test_no_naive_time(self): r=record(NOW);r['capturedAt']='2026-10-02T12:00:00' with self.assertRaises(ValueError):self.batch([r]) + def test_measurement_time_remains_whole_second(self): + r=record(NOW);r['capturedAt']='2026-10-02T12:00:00.000001+00:00' + with self.assertRaises(ValueError):self.batch([r]) def test_future_source_not_rejuvenated(self): r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1 self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid']) @@ -115,7 +118,10 @@ class MeasurementPipelineTest(unittest.TestCase): with self.assertRaises(ValueError):m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW) def test_external_is_labelled_persistence_not_published(self): self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW) - op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc));out=m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW) + op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc)) + fc['observedAt']=datetime.fromtimestamp(NOW-1,timezone.utc).replace(microsecond=1).isoformat() + out=m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW) + self.assertEqual(out['observedAt'],m.iso(NOW)) self.assertEqual(out['families']['3']['points'][0]['loadW'],6000) self.assertFalse(out['families']['3']['dataPipeline']['futureSdlPublished']) self.assertNotIn('accountingEvidenceId',out['families']['3']) From 13e49ff60cfec37ddcad5466df3ad8c72ab37da5 Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Sun, 4 Oct 2026 15:34:36 +0000 Subject: [PATCH 18/22] feat(v4): confirm feedback with device reads --- docs/NETPLAN_V4_AGENT_HANDOFF.md | 2 + examples/V4ConfirmedFeedback/MANIFEST.json | 41 +++++ examples/V4ConfirmedFeedback/README.md | 18 ++ examples/V4ConfirmedFeedback/build_stage.py | 49 ++++++ .../V4ConfirmedFeedback/feedback-config.json | 71 ++++++++ .../V4ConfirmedFeedback/file_installer.php | 127 ++++++++++++++ examples/V4ConfirmedFeedback/install.php | 161 ++++++++++++++++++ ...atterieNetzfahrplanV4RueckmeldungTrait.php | 45 ++++- libs/NetzfahrplanV4Geraeteabruf.php | 101 +++++++++++ libs/NetzfahrplanV4Rueckmeldung.php | 22 ++- tests/NetzfahrplanV4RueckmeldungTest.php | 1 + tests/V4ControlTrial/checks.php | 12 +- tests/V4ControlTrial/feedback_checks.php | 10 +- tests/V4Feedback/checks.php | 11 +- tests/V4Feedback/device_read_checks.php | 51 ++++++ 15 files changed, 693 insertions(+), 29 deletions(-) create mode 100644 examples/V4ConfirmedFeedback/MANIFEST.json create mode 100644 examples/V4ConfirmedFeedback/README.md create mode 100644 examples/V4ConfirmedFeedback/build_stage.py create mode 100644 examples/V4ConfirmedFeedback/feedback-config.json create mode 100644 examples/V4ConfirmedFeedback/file_installer.php create mode 100644 examples/V4ConfirmedFeedback/install.php create mode 100644 libs/NetzfahrplanV4Geraeteabruf.php create mode 100644 tests/V4Feedback/device_read_checks.php diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md index 2bae37f..989d7d3 100644 --- a/docs/NETPLAN_V4_AGENT_HANDOFF.md +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -6,6 +6,8 @@ **Fortsetzung Prognose am 04.10.2026:** Der aktuelle Runtime-Stand wurde gelesen, ohne die alten Diagnose- oder Installationsschritte zu wiederholen. Der abgeleitete Datensatz `lihrenmoos-physical-published-v2` ist `model_ready` und erfüllt mit mehr als 24 nutzbaren äquivalenten Stunden die Trainingsschwelle. Die V4-Einstellungen stehen seit Revision 2 in `shadow` auf `forecastSource=corrected_profile` und `measurementDataset=lihrenmoos-physical-published-v2`; `liveEnabled` und Stellfreigabe bleiben aus. Der erste echte Lauf zeigte einen kompatibilitätsbedingten Stopp bei mikrosekundengenauen `observedAt`-Zeitstempeln. Die Korrektur rundet ausschliesslich die kausale Verfügbarkeit von Prognoseereignissen auf die nächste volle Sekunde auf; Rohmessungen bleiben streng ganzsekündlich. 310 Python- und 16 Portal-Tests sind erfolgreich. Repository und Runtime-Build-Kontext enthalten bytegleich den geprüften Fix; der laufende Container enthält ihn noch nicht, weil beide Agentzugänge weder Docker-Socket noch `sudo` erhalten. Der vorbereitete, syntaktisch geprüfte Root-Rollout liegt unter `/home/agent/services/netplan-v4-shadow/commissioning/finish_corrected_forecast_rollout.sh` und baut/testet vor dem Austausch, hält das vorige Image als Rollback fest und ändert keine Stellfreigabe. Daniel hat danach einen Aktivbetrieb angefragt; der aktuelle V4-Kern ist jedoch technisch `shadow-only`, und die weiterhin unbelegte Rückmeldung (`feedback_source_skew`, `usableForTrial=false`, `canDispatch=false`) verbietet ein blosses Umstellen. Keine Live-Freigabe wurde erteilt oder eingebaut. Git-Klarstellung durch Daniel: Commit-Autor `dh_Agent `, Gitea-Konto `dh`; Commit/Push auf `develop` und `beta` am 04.10.2026 ausdrücklich freigegeben. Der Agentzugang besitzt derzeit keine HTTPS-Schreibanmeldung für `dh`. +**Fortsetzung bestätigte Rückmeldung am 04.10.2026:** Daniels Freigabe des modellierten `virtual_split` gilt für den ausdrücklich begrenzten Feldtest. Die physische Rückmeldung wurde um synchrone, durch den Gerätetreiber bestätigte Lesevorgänge erweitert: M-Bus verwendet `MBUS_UpdateValues`, ModBus Device und ModBus Address verwenden `ModBus_RequestRead`. Nur ein erfolgreicher Geräteaufruf erzeugt `confirmedAt`; `VariableUpdated` bleibt getrennte Herkunftsinformation. Die Regeltest-Gates verlangen `deviceReadConfirmed=true`. Das reproduzierbare inkrementelle Paket liegt auf der Testanlage unter `/srv/agent/netplan-v4-confirmed-feedback-20261004`; 18 Paketprüfsummen, 37 Rückmeldetests, 7 Geräteabruf-Tests und 71 Regeltest-Prüfungen sind erfolgreich. Es ist noch **nicht im IP-Symcon-Kernel installiert**, weil der lokale JSON-RPC-Zugang ohne Anmeldung mit HTTP 401 antwortet. Die Installation selbst lässt alten Netzfahrplan und beide Regeltestfreigaben aus. Ein realer Stellversuch bleibt zusätzlich durch den nicht belegten unabhängigen Geräte-Watchdog und die fehlende Serverfreigabe blockiert; der vorhandene Symcon-`VorgabeTimeout` ist kein unabhängiger Hardware-Nachweis. Auf `enelix-services` und der Testanlage ist kein nutzbarer Git-HTTPS-Credential-Helper hinterlegt; die Prüfung des Tunnel-Servers lief zweimal in eine externe Connector-Freigabezeitüberschreitung. Tokeninhalt wurde weder gelesen noch ausgegeben. Commit/Push dieses Folgepakets stehen deshalb noch aus. + ## 1. Zuerst lesen: Wo wir tatsächlich stehen - Daniel möchte die **fertige, produktionsgeeignete Anwendung**, nicht weitere isolierte Sammler, Diagnosekategorien oder wiederholte Bestätigungsrunden. Er hat die fortlaufende Umsetzung mehrfach beauftragt. Probleme im Code selbst beheben, Tests und Auslieferung bündeln; ihn nur für wirklich notwendige Root-/Symcon-Ausführung oder echte Anlagenfreigaben einbeziehen. diff --git a/examples/V4ConfirmedFeedback/MANIFEST.json b/examples/V4ConfirmedFeedback/MANIFEST.json new file mode 100644 index 0000000..0984be8 --- /dev/null +++ b/examples/V4ConfirmedFeedback/MANIFEST.json @@ -0,0 +1,41 @@ +{ + "scope": "confirmed_feedback_trial_disabled", + "managerId": 17004, + "batteryInstanceId": 44234, + "files": { + "libs/NetzfahrplanV4Rueckmeldung.php": { + "before": "a7459a208c4dd84ffe3a166d266dcd00216f3660fe60e5ee61d5042d0b840631", + "after": "0b261f6534c5ca9864018ea4c2caf97e6ce6913d92f13dcc926383b4f401a673" + }, + "libs/NetzfahrplanV4Geraeteabruf.php": { + "before": null, + "after": "2b69a2ed27554139db6c996c906a4afe430918381e6359d23e8bb3369cd9be47" + }, + "libs/BatterieNetzfahrplanV4RueckmeldungTrait.php": { + "before": "24cf0654f1a6bd545d64ce5f115f309b16d8ffca5172ec64fb37c390c6f001a7", + "after": "1de81499da07ad814c98ba778ca728e34817bdfcf29cc7e518f7b5f8fd7733b7" + } + }, + "testSupport": { + "libs/BatterieNetzfahrplanV4TestTrait.php": "80474ff24cf9b2d32dfe481f0ce26bb94584c458b0fa3de232dec3897b64a4b2", + "libs/NetzfahrplanV4Regeltest.php": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249", + "libs/ManagerNetzfahrplanV4TestTrait.php": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18", + "libs/NetzfahrplanV4Planpruefung.php": "24b935d80bad4ce2d9047b6777aaf74a18773236ca91a2844a0907012f4fadf4" + }, + "tests": { + "tests/V4Feedback/checks.php": "b1aacc540ba11adabdd140aaded9e2968d1328f261ca15756e6002db3427330a", + "tests/V4Feedback/device_read_checks.php": "5c5d9e50dd1d0f80cb01a2d95e63018eeec558ba78c07afde208e3c0ef304cc3", + "tests/V4ControlTrial/checks.php": "2f12e94aeb207e6939a645f2f45b5f54e57c18530360ed3b16aa52ff89a1ef39", + "tests/V4ControlTrial/additional_checks.php": "df01119f2b9c0befe1d14ae6b6ddf39c75d3687b669142e9b24b5f9f4821f2c3", + "tests/V4ControlTrial/feedback_checks.php": "e87d41dff8e3ecd414a59b3dd7a234b34183d37acca8cddaf847b82eae92e60c", + "tests/V4Receiver/fixture.php": "0eeb65e63250aff5ca364a82c2b2a71f9f229be2d9d412e3037014979deb793a" + }, + "configurationSha256": "e393c1f01b30febc1d0785233474be43e98d849b152b4b2450b72efa6b73d540", + "dependencies": { + "libs/NetzfahrplanV4Regeltest.php": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249", + "libs/ManagerNetzfahrplanV4TestTrait.php": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18", + "libs/ManagerNetzfahrplanV4EmpfangTrait.php": "936939a3e4a5a5d54fdff68f6fc689a9400e8406aecd5ac09ebc8a6b569c0b66", + "Batterie/module.php": "3d5fcd83e4e7d999d5ac949fe357b492d5661ffa76639d2ca57a3f3934eabdbe", + "Manager/module.php": "cbbb33a8f71dda8cc80a4fc01cd3d6154da8153b5523306b2da723bb44f2dd79" + } +} diff --git a/examples/V4ConfirmedFeedback/README.md b/examples/V4ConfirmedFeedback/README.md new file mode 100644 index 0000000..7c00342 --- /dev/null +++ b/examples/V4ConfirmedFeedback/README.md @@ -0,0 +1,18 @@ +# V4 confirmed feedback for Lihrenmoos + +This incremental package adds synchronous device reads before V4 feedback is evaluated. + +- Grid input 40348 is refreshed through its M-Bus device. +- GoodWe and SolarEdge inputs are refreshed through their ModBus parent instances. +- A successful driver call is required; cached variable timestamps alone are not accepted. +- Daniel Haefliger accepted the existing `virtual_split` model for the bounded field test on 2026-10-04. +- The installer keeps the legacy schedule and both V4 trial permissions disabled. +- A real control trial still requires separate server authorization and evidence of an independent device-side watchdog. + +Build a private staging directory: + +```sh +python3 examples/V4ConfirmedFeedback/build_stage.py /srv/agent/netplan-v4-confirmed-feedback +``` + +Run the staged PHP checks before copying the package to the test plant. Execute `install.php` only from the IP-Symcon script editor on Lihrenmoos. diff --git a/examples/V4ConfirmedFeedback/build_stage.py b/examples/V4ConfirmedFeedback/build_stage.py new file mode 100644 index 0000000..55a129f --- /dev/null +++ b/examples/V4ConfirmedFeedback/build_stage.py @@ -0,0 +1,49 @@ +"""Create the reviewed Lihrenmoos confirmed-feedback package; never installs into Symcon.""" +from pathlib import Path +import hashlib +import json +import sys + +here = Path(__file__).resolve().parent +repo = here.parents[1] +if len(sys.argv) != 2: + raise SystemExit("Usage: python3 build_stage.py NEW_PRIVATE_STAGING_DIRECTORY") +target = Path(sys.argv[1]).resolve() +allowed_roots = [Path("/srv/agent"), Path("/home/agent/services/qa")] +if not any(target != root and target.is_relative_to(root) for root in allowed_roots): + raise SystemExit("Use a private agent staging directory, not live module paths") +if target.exists(): + raise SystemExit("Destination must not exist") + +manifest = json.loads((here / "MANIFEST.json").read_text()) +contents = {} +for group in ("files", "testSupport", "tests"): + for name, record in manifest[group].items(): + expected = record["after"] if group == "files" else record + source = repo / name + if source.is_symlink() or not source.resolve().is_relative_to(repo): + raise SystemExit("Unsafe source: " + name) + data = source.read_bytes() + if hashlib.sha256(data).hexdigest() != expected: + raise SystemExit("Reviewed source changed: " + name) + contents["source/" + name] = data + +for name, expected in manifest["dependencies"].items(): + if hashlib.sha256((repo / name).read_bytes()).hexdigest() != expected: + raise SystemExit("Dependency changed: " + name) + +for name in ("install.php", "file_installer.php", "MANIFEST.json", "feedback-config.json", "README.md"): + contents[name] = (here / name).read_bytes() +if hashlib.sha256(contents["feedback-config.json"]).hexdigest() != manifest["configurationSha256"]: + raise SystemExit("Configuration changed") + +target.mkdir(parents=True, mode=0o700) +for name, data in contents.items(): + path = target / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o640) +(target / "PACKAGE_HASHES.json").write_text( + json.dumps({name: hashlib.sha256(data).hexdigest() for name, data in contents.items()}, indent=2) + "\n" +) +print("Private confirmed-feedback package created:", target, "; both trial permissions remain disabled.") diff --git a/examples/V4ConfirmedFeedback/feedback-config.json b/examples/V4ConfirmedFeedback/feedback-config.json new file mode 100644 index 0000000..73a9b4e --- /dev/null +++ b/examples/V4ConfirmedFeedback/feedback-config.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "assetId": "anlage01-virtual-ev", + "managerId": 17004, + "batteryInstanceId": 44234, + "mode": "virtual_split", + "allowEstimatedForTrial": true, + "maxSkewSeconds": 30, + "maxAgeSeconds": 60, + "idleToleranceW": 50.0, + "trackingToleranceW": 200.0, + "sources": [ + { + "key": "grid", + "variableId": 40348, + "parentId": 11490, + "ident": "Power_8", + "factorToW": 1000, + "role": "grid" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "parentId": 19742, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "parentId": 57658, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "parentId": 30789, + "ident": "Value", + "factorToW": 1, + "role": "physical" + }, + { + "key": "ev_requested", + "variableId": 19651, + "parentId": 58448, + "ident": "Nennleistung_Soll_EV", + "factorToW": 1.0, + "role": "ev_request" + }, + { + "key": "sdl_requested", + "variableId": 38943, + "parentId": 58448, + "ident": "Nennleistung_Soll_SDL", + "factorToW": 1.0, + "role": "sdl_request" + }, + { + "key": "gateway_active", + "role": "gateway_active", + "variableId": 23483, + "parentId": 58448, + "ident": "State", + "factorToW": 1.0 + } + ] +} diff --git a/examples/V4ConfirmedFeedback/file_installer.php b/examples/V4ConfirmedFeedback/file_installer.php new file mode 100644 index 0000000..92b37cd --- /dev/null +++ b/examples/V4ConfirmedFeedback/file_installer.php @@ -0,0 +1,127 @@ + $hash) { + if (!preg_match('~^(libs/[A-Za-z0-9]+\\.php|Batterie/module\\.php|Manager/module\\.php)$~D', $name) + || is_link($target . '/' . $name) + || hash_file('sha256', $target . '/' . $name) !== $hash) { + throw new RuntimeException('Abhaengigkeit wurde parallel geaendert: ' . $name); + } + } + + $before = []; + $source = []; + $already = true; + foreach ($allowed as $name) { + $path = $target . '/' . $name; + $candidate = $stage . '/source/' . $name; + if (is_link($path) || is_link($candidate) || realpath(dirname($path)) !== $target . '/' . dirname($name)) { + throw new RuntimeException('Unerwarteter Dateipfad.'); + } + $source[$name] = (string) file_get_contents($candidate); + if (hash('sha256', $source[$name]) !== $manifest['files'][$name]['after']) { + throw new RuntimeException('Paketpruefsumme geaendert: ' . $name); + } + token_get_all($source[$name], TOKEN_PARSE); + $before[$name] = is_file($path) ? file_get_contents($path) : null; + $hash = $before[$name] === null ? null : hash('sha256', $before[$name]); + if (!in_array($hash, [$manifest['files'][$name]['before'], $manifest['files'][$name]['after']], true)) { + throw new RuntimeException('Paralleler Modulstand; nichts ueberschrieben: ' . $name); + } + if ($hash !== $manifest['files'][$name]['after']) { + $already = false; + } + } + if ($already) { + return ['status' => 'already_installed', 'backup' => null, 'filesChanged' => []]; + } + + $backup = $stage . '/backups/' . gmdate('Ymd\\THis\\Z') . '-' . bin2hex(random_bytes(4)); + if (!mkdir($backup, 0700, true)) { + throw new RuntimeException('Sicherung fehlgeschlagen.'); + } + foreach ($allowed as $name) { + if ($before[$name] !== null) { + $destination = $backup . '/' . $name; + if (!is_dir(dirname($destination))) { + mkdir(dirname($destination), 0700, true); + } + v4ConfirmedWrite($destination, $before[$name], 0600); + } + } + v4ConfirmedWrite( + $backup . '/MANIFEST.json', + json_encode($manifest, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT) . "\n", + 0600 + ); + + $written = []; + try { + foreach ($allowed as $name) { + if ((is_file($target . '/' . $name) ? file_get_contents($target . '/' . $name) : null) !== $before[$name]) { + throw new RuntimeException('Parallele Aenderung waehrend Installation.'); + } + if ($before[$name] === $source[$name]) { + continue; + } + v4ConfirmedWrite($target . '/' . $name, $source[$name]); + $written[] = $name; + } + } catch (Throwable $error) { + foreach (array_reverse($written) as $name) { + $path = $target . '/' . $name; + if (!is_link($path) && is_file($path) + && hash_file('sha256', $path) === $manifest['files'][$name]['after']) { + if ($before[$name] === null) { + unlink($path); + } else { + v4ConfirmedWrite($path, $before[$name]); + } + } + } + throw $error; + } + + return ['status' => 'installed', 'backup' => $backup, 'filesChanged' => $written]; +} diff --git a/examples/V4ConfirmedFeedback/install.php b/examples/V4ConfirmedFeedback/install.php new file mode 100644 index 0000000..4463d31 --- /dev/null +++ b/examples/V4ConfirmedFeedback/install.php @@ -0,0 +1,161 @@ + 'confirmed_feedback_trial_disabled', + 'startedAt' => gmdate('c'), + 'actuatorPermissionGranted' => false, + 'serverUpdateRequired' => true, + 'acceptedModel' => 'virtual_split', +]; + +try { + $hashes = json_decode((string) file_get_contents(__DIR__ . '/PACKAGE_HASHES.json'), true, 32, JSON_THROW_ON_ERROR); + foreach ($hashes as $name => $hash) { + if (str_contains($name, '..') || str_starts_with($name, '/') || is_link(__DIR__ . '/' . $name) + || hash_file('sha256', __DIR__ . '/' . $name) !== $hash) { + throw new RuntimeException('Vorbereitetes Paket veraendert.'); + } + } + foreach ([ + $manager => '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}', + $battery => '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}', + ] as $id => $module) { + if (!IPS_InstanceExists($id) || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== $module) { + throw new RuntimeException('Modulzuordnung ungueltig.'); + } + if (IPS_HasChanges($id)) { + throw new RuntimeException('Offene Aenderungen zuerst speichern oder verwerfen.'); + } + } + + $beforeManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $beforeBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + if (($beforeManager['NetzfahrplanAktiv'] ?? null) !== false + || ($beforeManager['NetzfahrplanV4RegeltestErlaubt'] ?? false) !== false + || ($beforeBattery['NetzfahrplanV4RegeltestErlaubt'] ?? false) !== false) { + throw new RuntimeException('Alte Fahrplanausgabe und beide Regeltestfreigaben muessen AUS bleiben.'); + } + + $configuration = json_decode((string) file_get_contents(__DIR__ . '/feedback-config.json'), true, 32, JSON_THROW_ON_ERROR); + $manifest = json_decode((string) file_get_contents(__DIR__ . '/MANIFEST.json'), true, 32, JSON_THROW_ON_ERROR); + if (($configuration['managerId'] ?? null) !== $manager + || ($configuration['batteryInstanceId'] ?? null) !== $battery + || ($configuration['mode'] ?? null) !== 'virtual_split' + || ($configuration['allowEstimatedForTrial'] ?? null) !== true + || hash_file('sha256', __DIR__ . '/feedback-config.json') !== $manifest['configurationSha256']) { + throw new RuntimeException('Falsche Rueckmeldekonfiguration.'); + } + $desired = json_encode($configuration, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION); + $previousConfiguration = $configuration; + $previousConfiguration['allowEstimatedForTrial'] = false; + $previousDesired = json_encode($previousConfiguration, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION); + $oldMapping = $beforeBattery['NetzfahrplanV4RueckmeldungKonfiguration'] ?? '{}'; + if (!in_array($oldMapping, [$previousDesired, $desired], true)) { + throw new RuntimeException('Abweichende Rueckmeldekonfiguration nicht ueberschrieben.'); + } + + $controls = []; + foreach (IPS_GetModuleList() as $module) { + if ((IPS_GetModule($module)['Prefix'] ?? '') === 'MC') { + $controls = array_merge($controls, IPS_GetInstanceListByModuleID($module)); + } + } + if (count($controls) !== 1 || !function_exists('MC_ReloadModule')) { + throw new RuntimeException('Module Control nicht eindeutig.'); + } + $locked = IPS_SemaphoreEnter('ENELIX.V4.ConfirmedFeedbackInstall', 3000); + if (!$locked) { + throw new RuntimeException('Installation bereits aktiv.'); + } + + $report['files'] = v4ConfirmedFiles(__DIR__, '/var/lib/symcon/modules/Enelix-EMS'); + if (MC_ReloadModule($controls[0], 'Enelix-EMS') === false) { + throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + } + + $afterManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $afterBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + foreach ([[$beforeManager, $afterManager], [$beforeBattery, $afterBattery]] as $pair) { + foreach ($pair[0] as $key => $value) { + if (!array_key_exists($key, $pair[1]) || $pair[1][$key] !== $value) { + throw new RuntimeException('Bestehender Parameter beim Reload veraendert: ' . $key); + } + } + } + + if (!function_exists('ENELIX_GetV4BatterieRueckmeldung')) { + $report['status'] = 'waiting_for_registration'; + echo "Dateien installiert. Modulregistrierung wird abgeschlossen; diesen Aufruf einmal wiederholen. Keine Stellfreigabe.\n"; + } else { + if ($afterBattery['NetzfahrplanV4RueckmeldungKonfiguration'] !== $desired) { + IPS_SetProperty($battery, 'NetzfahrplanV4RueckmeldungKonfiguration', $desired); + $changedMapping = true; + IPS_ApplyChanges($battery); + } + $finalManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $finalBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + foreach ([[$beforeManager, $finalManager], [$beforeBattery, $finalBattery]] as $pair) { + foreach ($pair[0] as $key => $value) { + if ($key !== 'NetzfahrplanV4RueckmeldungKonfiguration' + && (!array_key_exists($key, $pair[1]) || $pair[1][$key] !== $value)) { + throw new RuntimeException('Bestehender Parameter unerwartet veraendert: ' . $key); + } + } + } + if (($finalManager['NetzfahrplanAktiv'] ?? null) !== false + || ($finalManager['NetzfahrplanV4RegeltestErlaubt'] ?? null) !== false + || ($finalBattery['NetzfahrplanV4RegeltestErlaubt'] ?? null) !== false) { + throw new RuntimeException('Unerwartete Stellfreigabe nach Installation.'); + } + + $feedback = json_decode(ENELIX_GetV4BatterieRueckmeldung($battery), true, 32, JSON_THROW_ON_ERROR); + $report['feedback'] = array_intersect_key($feedback, array_flip([ + 'status', 'kind', 'method', 'reason', 'reasons', 'estimated', 'batteryW', 'gridW', + 'sourceOldestAt', 'sourceVariableOldestAt', 'confirmedAt', 'checkedAt', 'configHash', + 'deviceReadConfirmed', 'usableForTrial', 'canDispatch', + ])); + $report['status'] = 'confirmed_feedback_installed_trial_disabled'; + $report['originalParametersPreserved'] = true; + echo "OK: Geraeteabfrage bestaetigt die physischen Messwerte; virtual_split ist fuer den begrenzten Regeltest akzeptiert.\n"; + echo 'Rueckmeldung: ' . ($feedback['status'] ?? 'available') + . '; Geraeteabfrage: ' . (($feedback['deviceReadConfirmed'] ?? false) ? 'bestaetigt' : 'nicht bestaetigt') + . '; Modellanteil: ' . (($feedback['estimated'] ?? false) ? 'ja' : 'nein') . ".\n"; + if (isset($feedback['reason'])) { + echo 'Aktueller Hinweis: ' . $feedback['reason'] . "\n"; + } + echo "Beide Regeltestfreigaben AUS. Kein Stellbefehl ohne separat nachgewiesenen Geraete-Watchdog und Serverfreigabe.\n"; + } +} catch (Throwable $error) { + $report['status'] = 'needs_review'; + $report['reason'] = substr($error->getMessage(), 0, 260); + if ($changedMapping && $oldMapping !== null) { + try { + IPS_SetProperty($battery, 'NetzfahrplanV4RueckmeldungKonfiguration', $oldMapping); + IPS_ApplyChanges($battery); + $report['mappingRestored'] = true; + } catch (Throwable $ignored) { + $report['mappingRestored'] = false; + } + } + echo 'FEHLER: ' . $report['reason'] . "\nKeine V4-Stellfreigabe erteilt.\n"; +} finally { + $report['finishedAt'] = gmdate('c'); + v4ConfirmedWrite(__DIR__ . '/INSTALL_RESULT.json', json_encode($report, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT) . "\n", 0644); + if ($locked) { + IPS_SemaphoreLeave('ENELIX.V4.ConfirmedFeedbackInstall'); + } + echo 'CONFIRMED FEEDBACK INSTALL REPORT: ' . __DIR__ . "/INSTALL_RESULT.json\n"; +} diff --git a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php index da72811..0ddfcdd 100644 --- a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php +++ b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php @@ -4,8 +4,9 @@ declare(strict_types=1); namespace Belevo\EnelixEMS; require_once __DIR__.'/NetzfahrplanV4Rueckmeldung.php'; +require_once __DIR__.'/NetzfahrplanV4Geraeteabruf.php'; -/** Shared physical reading used by receiver and battery immediately before a trial command. */ +/** Shared confirmed physical reading used by receiver and battery before a trial command. */ trait BatterieNetzfahrplanV4RueckmeldungTrait { private function registriereV4Rueckmeldung(): void @@ -20,11 +21,41 @@ trait BatterieNetzfahrplanV4RueckmeldungTrait $c=json_decode($json,true,32,JSON_THROW_ON_ERROR); if (!is_array($c) || ($c['batteryInstanceId']??null)!==$this->InstanceID) throw new \RuntimeException('feedback_battery_binding_invalid'); if (!in_array($c['managerId']??0,$this->zugeordneteManagerIDs(),true)) throw new \RuntimeException('feedback_manager_binding_invalid'); - return NetzfahrplanV4Rueckmeldung::read($c,static function(int $id):array { - if (!IPS_VariableExists($id)) throw new \RuntimeException('feedback_variable_missing'); - $v=IPS_GetVariable($id);$o=IPS_GetObject($id);$value=GetValue($id); - return ['value'=>$value,'updated'=>$v['VariableUpdated'],'changed'=>$v['VariableChanged'],'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; - },time()); + + $lock='ENELIX.V4.Feedback.'.$this->InstanceID; + if (!IPS_SemaphoreEnter($lock,5000)) throw new \RuntimeException('feedback_refresh_busy'); + try { + $reader=static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('feedback_variable_missing'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id);$value=GetValue($id); + return ['value'=>$value,'updated'=>$v['VariableUpdated'],'changed'=>$v['VariableChanged'], + 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + }; + $result=NetzfahrplanV4Geraeteabruf::snapshot( + $c, + static function(int $id):string { + if (!IPS_InstanceExists($id)) throw new \RuntimeException('feedback_parent_instance_missing'); + return (string)(IPS_GetInstance($id)['ModuleInfo']['ModuleID']??''); + }, + static function(int $id,string $module):bool { + if ($module===NetzfahrplanV4Geraeteabruf::MBUS_DEVICE) { + if (!function_exists('MBUS_UpdateValues') && !function_exists(__NAMESPACE__.'\\MBUS_UpdateValues')) throw new \RuntimeException('feedback_mbus_refresh_unavailable'); + return MBUS_UpdateValues($id)===true; + } + if (in_array($module,[NetzfahrplanV4Geraeteabruf::MODBUS_DEVICE,NetzfahrplanV4Geraeteabruf::MODBUS_ADDRESS],true)) { + if (!function_exists('ModBus_RequestRead') && !function_exists(__NAMESPACE__.'\\ModBus_RequestRead')) throw new \RuntimeException('feedback_modbus_refresh_unavailable'); + return ModBus_RequestRead($id)===true; + } + return false; + }, + $reader, + static fn():int=>time() + ); + $this->SetBuffer('V4ConfirmedFeedback',json_encode($result,JSON_THROW_ON_ERROR)); + return $result; + } finally { + IPS_SemaphoreLeave($lock); + } } /** Read only. A returned current value is NOT actuator permission. */ @@ -32,7 +63,7 @@ trait BatterieNetzfahrplanV4RueckmeldungTrait { try { $r=$this->v4BatterieRueckmeldung(); } catch (\Throwable $e) { $r=['kind'=>'local_physical_feedback','status'=>'unavailable','reason'=>substr($e->getMessage(),0,200), - 'batteryW'=>null,'gridW'=>null,'usableForTrial'=>false,'canDispatch'=>false]; } + 'batteryW'=>null,'gridW'=>null,'usableForTrial'=>false,'canDispatch'=>false,'deviceReadConfirmed'=>false]; } return json_encode($r,JSON_THROW_ON_ERROR); } } diff --git a/libs/NetzfahrplanV4Geraeteabruf.php b/libs/NetzfahrplanV4Geraeteabruf.php new file mode 100644 index 0000000..097943e --- /dev/null +++ b/libs/NetzfahrplanV4Geraeteabruf.php @@ -0,0 +1,101 @@ + $sources) { + $module = $moduleId($parentId); + if (!in_array($module, [self::MBUS_DEVICE, self::MODBUS_DEVICE, self::MODBUS_ADDRESS], true)) { + throw new InvalidArgumentException('feedback_refresh_module_unsupported:' . $sources[0]['key']); + } + if ($refresh($parentId, $module) !== true) { + throw new InvalidArgumentException('feedback_device_read_failed:' . $sources[0]['key']); + } + $confirmedAt = $clock(); + if (!is_int($confirmedAt) || $confirmedAt < 1) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + foreach ($sources as $source) { + $record = $reader($source['variableId']); + if (!is_array($record)) { + throw new InvalidArgumentException('feedback_reader_invalid:' . $source['key']); + } + $record['confirmedAt'] = $confirmedAt; + $record['confirmedModuleId'] = $module; + $snapshot[$source['variableId']] = $record; + } + } + + foreach ($config['sources'] as $source) { + if (!isset($snapshot[$source['variableId']])) { + $record = $reader($source['variableId']); + if (!is_array($record)) { + throw new InvalidArgumentException('feedback_reader_invalid:' . $source['key']); + } + $snapshot[$source['variableId']] = $record; + } + } + + foreach ($snapshot as $variableId => $record) { + $again = $reader($variableId); + $keys = ['value', 'updated', 'changed', 'parentID', 'ident']; + if (!is_array($again) || array_intersect_key($again, array_flip($keys)) + !== array_intersect_key($record, array_flip($keys))) { + throw new InvalidArgumentException( + 'feedback_changed_after_confirmed_read:' . $sourcesById[$variableId]['key'] + ); + } + } + + $checkedAt = $clock(); + if (!is_int($checkedAt) || $checkedAt < 1) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + foreach ($snapshot as $record) { + if (isset($record['confirmedAt']) && $record['confirmedAt'] > $checkedAt) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + } + + return NetzfahrplanV4Rueckmeldung::read( + $config, + static fn(int $variableId): array => $snapshot[$variableId], + $checkedAt + ); + } +} diff --git a/libs/NetzfahrplanV4Rueckmeldung.php b/libs/NetzfahrplanV4Rueckmeldung.php index 1ecd110..df89be3 100644 --- a/libs/NetzfahrplanV4Rueckmeldung.php +++ b/libs/NetzfahrplanV4Rueckmeldung.php @@ -72,28 +72,32 @@ final class NetzfahrplanV4Rueckmeldung { $c=self::configuration($config); if ($now<1) throw new InvalidArgumentException('feedback_time_invalid'); - $first=[];$values=[];$times=[];$physicalTimes=[];$changed=[]; + $first=[];$values=[];$times=[];$physicalTimes=[];$variableTimes=[];$changed=[]; foreach ($c['sources'] as $s) { $r=$reader($s['variableId']); if (!is_array($r) || ($r['parentID']??null)!==$s['parentId'] || ($r['ident']??null)!==$s['ident'] || !is_int($r['updated']??null) || $r['updated']<1 || $r['updated']>$now - || !is_int($r['changed']??null) || $r['changed']<0 || $r['changed']>$r['updated']) throw new InvalidArgumentException('feedback_origin_invalid:'.$s['key']); - $first[$s['key']]=array_intersect_key($r,array_flip(['value','updated','changed','parentID','ident'])); + || !is_int($r['changed']??null) || $r['changed']<0 || $r['changed']>$r['updated'] + || (in_array($s['role'],['grid','physical'],true) + && (!is_int($r['confirmedAt']??null) || $r['confirmedAt']<1 || $r['confirmedAt']>$now + || $r['updated']>$r['confirmedAt']))) throw new InvalidArgumentException('feedback_origin_invalid:'.$s['key']); + $first[$s['key']]=array_intersect_key($r,array_flip(['value','updated','changed','parentID','ident','confirmedAt'])); if ($s['role']==='gateway_active') { if (($r['value']??null)!==true) throw new InvalidArgumentException('feedback_gateway_inactive'); } else { $w=self::number($r['value']??null,-1e9,1e9)*$s['factorToW']; $values[$s['role']]=($values[$s['role']]??0)+self::number($w,-1e9,1e9); } - // An unchanged command is held STATE, not proof of a fresh sensor or an actuator watchdog. + // VariableUpdated is retained as provenance. Freshness comes only from a successful device response. if (in_array($s['role'],['grid','physical'],true)) { - if ($now-$r['updated']>$c['maxAgeSeconds']) throw new InvalidArgumentException('feedback_stale_physical:'.$s['key']); - $times[]=$r['updated'];if ($s['role']==='physical') $physicalTimes[]=$r['updated']; + if ($now-$r['confirmedAt']>$c['maxAgeSeconds']) throw new InvalidArgumentException('feedback_stale_physical:'.$s['key']); + $times[]=$r['confirmedAt'];$variableTimes[]=$r['updated']; + if ($s['role']==='physical') $physicalTimes[]=$r['confirmedAt']; } else $changed[$s['role']]=$r['changed']; } foreach ($c['sources'] as $s) { $second=$reader($s['variableId']); - if (!is_array($second) || array_intersect_key($second,array_flip(['value','updated','changed','parentID','ident']))!==$first[$s['key']]) { + if (!is_array($second) || array_intersect_key($second,array_flip(['value','updated','changed','parentID','ident','confirmedAt']))!==$first[$s['key']]) { throw new InvalidArgumentException('feedback_changed_during_read:'.$s['key']); } } @@ -117,7 +121,8 @@ final class NetzfahrplanV4Rueckmeldung return ['version'=>1,'kind'=>'local_physical_feedback','installationId'=>$c['installationId'],'assetId'=>$c['assetId'], 'managerId'=>$c['managerId'],'batteryInstanceId'=>$c['batteryInstanceId'],'checkedAt'=>$now, 'sourceOldestAt'=>min($times),'sourceNewestAt'=>max($times),'physicalOldestAt'=>min($physicalTimes), - 'sourceSkewSeconds'=>max($times)-min($times),'maxAgeSeconds'=>$c['maxAgeSeconds'], + 'sourceSkewSeconds'=>max($times)-min($times),'sourceVariableOldestAt'=>min($variableTimes), + 'sourceVariableNewestAt'=>max($variableTimes),'deviceReadConfirmed'=>true,'maxAgeSeconds'=>$c['maxAgeSeconds'], 'configHash'=>self::fingerprint($c),'gridW'=>(float)$values['grid'],'physicalStorageW'=>$physical, 'batteryW'=>$power,'externalModelW'=>$external,'trackingErrorW'=>$tracking, 'estimated'=>$estimated,'separatelyMeasured'=>!$estimated,'usableForTrial'=>$reasons===[], @@ -131,6 +136,7 @@ final class NetzfahrplanV4Rueckmeldung if (!array_key_exists($key,$binding) || ($f[$key]??null)!==$binding[$key]) throw new InvalidArgumentException('feedback_wrong_binding'); } if (($f['kind']??null)!=='local_physical_feedback' || ($f['version']??null)!==1 || ($f['usableForTrial']??null)!==true + || ($f['deviceReadConfirmed']??null)!==true || !is_int($f['checkedAt']??null) || $f['checkedAt']>$now || $now-$f['checkedAt']>2 || !is_int($f['sourceOldestAt']??null) || !is_int($f['maxAgeSeconds']??null) || $f['maxAgeSeconds']<1 || $f['maxAgeSeconds']>60 || $f['sourceOldestAt']>$now || $now-$f['sourceOldestAt']>$f['maxAgeSeconds'] diff --git a/tests/NetzfahrplanV4RueckmeldungTest.php b/tests/NetzfahrplanV4RueckmeldungTest.php index d975f5b..9b1852c 100644 --- a/tests/NetzfahrplanV4RueckmeldungTest.php +++ b/tests/NetzfahrplanV4RueckmeldungTest.php @@ -19,6 +19,7 @@ final class NetzfahrplanV4RueckmeldungTest extends TestCase self::assertSame('',$err);self::assertStringContainsString('TOTAL ',$out); } public function testPhysicalFeedback(): void {$this->checkScript('checks.php');} + public function testConfirmedDeviceReads(): void {$this->checkScript('device_read_checks.php');} public function testActualReceiverIntegration(): void {$this->checkScript('receiver_checks.php');} public function testActualFallbackWrapper(): void {$this->checkScript('fallback_checks.php');} public function testMessageWithoutVisibleDiagnostics(): void {$this->checkScript('message_checks.php');} diff --git a/tests/V4ControlTrial/checks.php b/tests/V4ControlTrial/checks.php index d9cfae7..37cb0bd 100644 --- a/tests/V4ControlTrial/checks.php +++ b/tests/V4ControlTrial/checks.php @@ -10,9 +10,13 @@ namespace Belevo\EnelixEMS { function hrtime($asNumber=true) { return (int)($GLOBALS['mono']*1e9); } function IPS_SemaphoreEnter($key,$wait) { if (!empty($GLOBALS['locks'][$key])) return false;$GLOBALS['locks'][$key]=true;return true; } function IPS_SemaphoreLeave($key) { unset($GLOBALS['locks'][$key]); } - function IPS_InstanceExists($id) { return isset($GLOBALS['objects'][$id]); } - function IPS_GetInstance($id) { return ['InstanceStatus'=>$GLOBALS['objects'][$id]->ready?102:201, - 'ModuleInfo'=>['ModuleID'=>$id===44234?'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}':'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']]; } + function IPS_InstanceExists($id) { return $id===200 || isset($GLOBALS['objects'][$id]); } + function IPS_GetInstance($id) { + if ($id===200) return ['InstanceStatus'=>102,'ModuleInfo'=>['ModuleID'=>'{90BF9234-B44C-7EB3-280B-FBD6075ADDAA}']]; + return ['InstanceStatus'=>$GLOBALS['objects'][$id]->ready?102:201, + 'ModuleInfo'=>['ModuleID'=>$id===44234?'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}':'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']]; + } + function ModBus_RequestRead($id) { return $id===200 && ($GLOBALS['deviceReadOk']??true); } function IPS_GetProperty($id,$key) { return $GLOBALS['objects'][$id]->props[$key]; } function IPS_VariableExists($id) { return in_array($id,[50001,50002],true); } function IPS_GetVariable($id) { $t=$GLOBALS['feedbackAt'][$id]??$GLOBALS['clock'];return ['VariableUpdated'=>$t,'VariableChanged'=>$t]; } @@ -102,7 +106,7 @@ namespace { function check($ok,$name) { global $count;if (!$ok) throw new RuntimeException('FAIL '.$name);$count++;echo "PASS $name\n"; } function rejects(callable $call,$name) { try {$call();}catch(Throwable $e){check(true,$name);return;}throw new RuntimeException('FAIL did not reject '.$name); } function fixture($permissions=true,$serverAuthority=true): array { - [$r,$id,$op,$now]=receiverFixture();$GLOBALS['clock']=$now;$GLOBALS['mono']=1000.;$GLOBALS['locks']=[];$GLOBALS['calls']=[]; + [$r,$id,$op,$now]=receiverFixture();$GLOBALS['clock']=$now;$GLOBALS['mono']=1000.;$GLOBALS['locks']=[];$GLOBALS['calls']=[];$GLOBALS['deviceReadOk']=true; $r['plan']['inputQuality']=['loadBasis'=>'base_load','accountingEvidenceId'=>'synthetic-meter-boundary-v1']; $r['plan']['peakCostIsEstimate']=true; $a=['kind'=>'controlled_trial_authority','version'=>1,'sourcePlanRemainsShadow'=>true, diff --git a/tests/V4ControlTrial/feedback_checks.php b/tests/V4ControlTrial/feedback_checks.php index 00c384e..7b23e76 100644 --- a/tests/V4ControlTrial/feedback_checks.php +++ b/tests/V4ControlTrial/feedback_checks.php @@ -2,11 +2,11 @@ // Run through actual Manager/Battery trial traits, fresh read helper, and simulated register driver. [$m,$b,$s]=fixture();$b->props['NetzfahrplanV4RueckmeldungKonfiguration']='{}'; rejects(fn()=>start($m,$s),'trial cannot use old filtered feedback when mapping missing');check($b->writes===[],'missing corrected feedback sends no command'); -[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock-61; -$a=tick($m);check(!$m->active()&&!$b->active()&&isset($a['Sollwerte'][44234]),'physical feedback expiry returns battery to normal allocation'); -check(end($b->writes)===0,'expired physical feedback revokes prior command'); -[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock-61;$b->watchdog(); -check(!$b->active()&&end($b->writes)===0,'independent battery read catches staleness without manager tick'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$deviceReadOk=false; +$a=tick($m);check(!$m->active()&&!$b->active()&&isset($a['Sollwerte'][44234]),'failed confirmed read returns battery to normal allocation'); +check(end($b->writes)===0,'failed confirmed read revokes prior command'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$deviceReadOk=false;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'independent battery read catches device failure without manager tick'); [$m,$b,$s]=fixture();start($m,$s);tick($m);$c=json_decode($b->props['NetzfahrplanV4RueckmeldungKonfiguration'],true);$c['trackingToleranceW']=250; $b->props['NetzfahrplanV4RueckmeldungKonfiguration']=json_encode($c);$b->watchdog(); check(!$b->active()&&end($b->writes)===0,'mapping revision change revokes active battery session'); diff --git a/tests/V4Feedback/checks.php b/tests/V4Feedback/checks.php index 06cb6e7..59ba0db 100644 --- a/tests/V4Feedback/checks.php +++ b/tests/V4Feedback/checks.php @@ -11,7 +11,7 @@ function fx($mode='physical_sum') { $now=1791032400;$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','assetId'=>'ev','managerId'=>17004,'batteryInstanceId'=>44234, 'mode'=>$mode,'allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60,'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0,'sources'=>[]]; $roles=['grid','physical','physical'];if($mode==='virtual_split')$roles=array_merge($roles,['ev_request','sdl_request','gateway_active']);$values=[3000.0,-2000.0,-1000.0,-3000.0,0.0,true];$r=[]; - foreach($roles as$i=>$role){$id=100+$i;$c['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'q'.$id,'factorToW'=>1];$r[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-60,'parentID'=>200,'ident'=>'q'.$id];} + foreach($roles as$i=>$role){$id=100+$i;$c['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'q'.$id,'factorToW'=>1];$r[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-60,'parentID'=>200,'ident'=>'q'.$id,'confirmedAt'=>$now];} return [$c,$r,$now]; } function readit($c,$r,$n){return F::read($c,fn($id)=>$r[$id],$n);} @@ -23,8 +23,9 @@ check(F::fingerprint($c)===F::fingerprint(json_decode(json_encode($c),true)),'co $d=$c;$d['sources']=array_reverse($d['sources']);check(F::fingerprint($c)===F::fingerprint($d),'source list order canonical'); $d['trackingToleranceW']=201;check(F::fingerprint($c)!==F::fingerprint($d),'changed tolerance invalidates session hash'); foreach([true,'1',NAN,INF,null]as$bad){$x=$r;$x[101]['value']=$bad;fails(fn()=>readit($c,$x,$n),'invalid physical number '.gettype($bad));} -$x=$r;$x[101]['updated']=$n-61;$x[101]['changed']=$n-90;fails(fn()=>readit($c,$x,$n),'physical age not refreshed by read'); -$x=$r;$x[101]['updated']=$n-31;fails(fn()=>readit($c,$x,$n),'incoherent grid battery timestamps rejected'); +$x=$r;$x[101]['updated']=$n-61;$x[101]['changed']=$n-90;$f=readit($c,$x,$n);check($f['sourceVariableOldestAt']===$n-61,'confirmed read does not manufacture VariableUpdated'); +$x=$r;unset($x[101]['confirmedAt']);fails(fn()=>readit($c,$x,$n),'physical source without confirmed device read rejected'); +$x=$r;$x[101]['confirmedAt']=$n-31;fails(fn()=>readit($c,$x,$n),'incoherent confirmed device timestamps rejected'); $x=$r;$x[101]['updated']=$n+1;fails(fn()=>readit($c,$x,$n),'future original sample rejected'); $x=$r;$x[101]['parentID']=201;fails(fn()=>readit($c,$x,$n),'reused variable wrong parent rejected'); $x=$r;$x[101]['ident']='other';fails(fn()=>readit($c,$x,$n),'reused variable wrong ident rejected'); @@ -44,8 +45,8 @@ $r[101]['value']=20.0;$f=readit($c,$r,$n);check($f['batteryW']===20.0,'idle resi $r[101]['value']=60.0;fails(fn()=>readit($c,$r,$n),'unexplained idle power refused'); $r[101]['value']=-2000.0;$r[102]['value']=-1000.0;$r[103]['value']=2000.0;$r[103]['updated']=$n;$r[103]['changed']=$n; $f=readit($c,$r,$n);check($f['batteryW']===-3000.0,'single EV feedback follows physics not desired sign or held filter'); -$r[104]['value']=1000.0;$r[104]['updated']=$n;$r[104]['changed']=$n;$r[101]['updated']=$n-1; -fails(fn()=>readit($c,$r,$n),'SDL transition after physical sample cannot be guessed'); +$r[104]['value']=1000.0;$r[104]['updated']=$n;$r[104]['changed']=$n;$r[101]['confirmedAt']=$n-1; +fails(fn()=>readit($c,$r,$n),'SDL transition after physical confirmation cannot be guessed'); [$c,$r,$n]=fx('virtual_split');$c['allowEstimatedForTrial']=true;$r[103]['value']=3000.0;$r[104]['value']=1000.0;$r[101]['value']=1900.0;$r[102]['value']=2000.0;$f=readit($c,$r,$n); check($f['batteryW']===2925.0&&$f['externalModelW']===975.0,'small tracking error allocated but explicitly estimated'); check(!$f['separatelyMeasured']&&$f['heldFilterUsed']===false,'partition never labelled separately measured'); diff --git a/tests/V4Feedback/device_read_checks.php b/tests/V4Feedback/device_read_checks.php new file mode 100644 index 0000000..6b00aca --- /dev/null +++ b/tests/V4Feedback/device_read_checks.php @@ -0,0 +1,51 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','assetId'=>'ev','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>'physical_sum','allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60,'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0, + 'sources'=>[ + ['key'=>'grid','role'=>'grid','variableId'=>10,'parentId'=>20,'ident'=>'grid','factorToW'=>1.0], + ['key'=>'battery_a','role'=>'physical','variableId'=>11,'parentId'=>21,'ident'=>'a','factorToW'=>1.0], + ['key'=>'battery_b','role'=>'physical','variableId'=>12,'parentId'=>21,'ident'=>'b','factorToW'=>1.0], + ]]; + $records=[ + 10=>['value'=>3000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>20,'ident'=>'grid'], + 11=>['value'=>-1000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>21,'ident'=>'a'], + 12=>['value'=>-2000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>21,'ident'=>'b'], + ]; + return [$config,$records,$now]; +} +[$c,$r,$n]=dfixture();$calls=[];$times=[$n-1,$n-1,$n]; +$f=D::snapshot($c,fn($id)=>$id===20?D::MBUS_DEVICE:D::MODBUS_DEVICE, + function($id,$module)use(&$calls){$calls[]=$id;return true;}, + fn($id)=>$r[$id],function()use(&$times){return array_shift($times);}); +sort($calls);dcheck($calls===[20,21],'one refresh per physical parent'); +dcheck($f['deviceReadConfirmed']===true&&$f['sourceOldestAt']===$n-1,'successful read confirmation is freshness basis'); +dcheck($f['sourceVariableOldestAt']===$n-100,'original VariableUpdated retained separately'); +dcheck($f['batteryW']===-3000.0&&$f['usableForTrial'],'unchanged values accepted after confirmed device response'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n]; +dfails(fn()=>D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>false,fn($id)=>$r[$id],function()use(&$times){return array_shift($times);}), + 'failed device read rejected'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n];$reads=0; +dfails(fn()=>D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>true, + function($id)use(&$r,&$reads){$reads++;$x=$r[$id];if($reads>3&&$id===10)$x['value']++;return $x;}, + function()use(&$times){return array_shift($times);}), 'change after confirmed response rejected'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n]; +dfails(fn()=>D::snapshot($c,fn($id)=>'{00000000-0000-0000-0000-000000000000}',fn()=>true,fn($id)=>$r[$id], + function()use(&$times){return array_shift($times);}), 'unsupported device module rejected'); + +echo "TOTAL $count confirmed-device-read checks passed. Synthetic calls only.\n"; From f4a3c5fbe7b11d5bb0f91662253efe9b1550ca25 Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Sun, 4 Oct 2026 15:35:35 +0000 Subject: [PATCH 19/22] docs(v4): record deployment blockers --- docs/NETPLAN_V4_AGENT_HANDOFF.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md index 989d7d3..a370441 100644 --- a/docs/NETPLAN_V4_AGENT_HANDOFF.md +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -6,7 +6,7 @@ **Fortsetzung Prognose am 04.10.2026:** Der aktuelle Runtime-Stand wurde gelesen, ohne die alten Diagnose- oder Installationsschritte zu wiederholen. Der abgeleitete Datensatz `lihrenmoos-physical-published-v2` ist `model_ready` und erfüllt mit mehr als 24 nutzbaren äquivalenten Stunden die Trainingsschwelle. Die V4-Einstellungen stehen seit Revision 2 in `shadow` auf `forecastSource=corrected_profile` und `measurementDataset=lihrenmoos-physical-published-v2`; `liveEnabled` und Stellfreigabe bleiben aus. Der erste echte Lauf zeigte einen kompatibilitätsbedingten Stopp bei mikrosekundengenauen `observedAt`-Zeitstempeln. Die Korrektur rundet ausschliesslich die kausale Verfügbarkeit von Prognoseereignissen auf die nächste volle Sekunde auf; Rohmessungen bleiben streng ganzsekündlich. 310 Python- und 16 Portal-Tests sind erfolgreich. Repository und Runtime-Build-Kontext enthalten bytegleich den geprüften Fix; der laufende Container enthält ihn noch nicht, weil beide Agentzugänge weder Docker-Socket noch `sudo` erhalten. Der vorbereitete, syntaktisch geprüfte Root-Rollout liegt unter `/home/agent/services/netplan-v4-shadow/commissioning/finish_corrected_forecast_rollout.sh` und baut/testet vor dem Austausch, hält das vorige Image als Rollback fest und ändert keine Stellfreigabe. Daniel hat danach einen Aktivbetrieb angefragt; der aktuelle V4-Kern ist jedoch technisch `shadow-only`, und die weiterhin unbelegte Rückmeldung (`feedback_source_skew`, `usableForTrial=false`, `canDispatch=false`) verbietet ein blosses Umstellen. Keine Live-Freigabe wurde erteilt oder eingebaut. Git-Klarstellung durch Daniel: Commit-Autor `dh_Agent `, Gitea-Konto `dh`; Commit/Push auf `develop` und `beta` am 04.10.2026 ausdrücklich freigegeben. Der Agentzugang besitzt derzeit keine HTTPS-Schreibanmeldung für `dh`. -**Fortsetzung bestätigte Rückmeldung am 04.10.2026:** Daniels Freigabe des modellierten `virtual_split` gilt für den ausdrücklich begrenzten Feldtest. Die physische Rückmeldung wurde um synchrone, durch den Gerätetreiber bestätigte Lesevorgänge erweitert: M-Bus verwendet `MBUS_UpdateValues`, ModBus Device und ModBus Address verwenden `ModBus_RequestRead`. Nur ein erfolgreicher Geräteaufruf erzeugt `confirmedAt`; `VariableUpdated` bleibt getrennte Herkunftsinformation. Die Regeltest-Gates verlangen `deviceReadConfirmed=true`. Das reproduzierbare inkrementelle Paket liegt auf der Testanlage unter `/srv/agent/netplan-v4-confirmed-feedback-20261004`; 18 Paketprüfsummen, 37 Rückmeldetests, 7 Geräteabruf-Tests und 71 Regeltest-Prüfungen sind erfolgreich. Es ist noch **nicht im IP-Symcon-Kernel installiert**, weil der lokale JSON-RPC-Zugang ohne Anmeldung mit HTTP 401 antwortet. Die Installation selbst lässt alten Netzfahrplan und beide Regeltestfreigaben aus. Ein realer Stellversuch bleibt zusätzlich durch den nicht belegten unabhängigen Geräte-Watchdog und die fehlende Serverfreigabe blockiert; der vorhandene Symcon-`VorgabeTimeout` ist kein unabhängiger Hardware-Nachweis. Auf `enelix-services` und der Testanlage ist kein nutzbarer Git-HTTPS-Credential-Helper hinterlegt; die Prüfung des Tunnel-Servers lief zweimal in eine externe Connector-Freigabezeitüberschreitung. Tokeninhalt wurde weder gelesen noch ausgegeben. Commit/Push dieses Folgepakets stehen deshalb noch aus. +**Fortsetzung bestätigte Rückmeldung am 04.10.2026:** Daniels Freigabe des modellierten `virtual_split` gilt für den ausdrücklich begrenzten Feldtest. Die physische Rückmeldung wurde um synchrone, durch den Gerätetreiber bestätigte Lesevorgänge erweitert: M-Bus verwendet `MBUS_UpdateValues`, ModBus Device und ModBus Address verwenden `ModBus_RequestRead`. Nur ein erfolgreicher Geräteaufruf erzeugt `confirmedAt`; `VariableUpdated` bleibt getrennte Herkunftsinformation. Die Regeltest-Gates verlangen `deviceReadConfirmed=true`. Das reproduzierbare inkrementelle Paket liegt auf der Testanlage unter `/srv/agent/netplan-v4-confirmed-feedback-20261004`; 18 Paketprüfsummen, 37 Rückmeldetests, 7 Geräteabruf-Tests und 71 Regeltest-Prüfungen sind erfolgreich. Es ist noch **nicht im IP-Symcon-Kernel installiert**, weil der lokale JSON-RPC-Zugang ohne Anmeldung mit HTTP 401 antwortet. Die Installation selbst lässt alten Netzfahrplan und beide Regeltestfreigaben aus. Ein realer Stellversuch bleibt zusätzlich durch den nicht belegten unabhängigen Geräte-Watchdog und die fehlende Serverfreigabe blockiert; der vorhandene Symcon-`VorgabeTimeout` ist kein unabhängiger Hardware-Nachweis. Auf `enelix-services` und der Testanlage ist kein nutzbarer Git-HTTPS-Credential-Helper hinterlegt; die Prüfung des Tunnel-Servers lief zweimal in eine externe Connector-Freigabezeitüberschreitung. Tokeninhalt wurde weder gelesen noch ausgegeben. Der Commit des Folgepakets ist lokal mit der vereinbarten Identitaet `dh_Agent ` erstellt. Der atomare Push auf `develop` und `beta` scheiterte ohne Teilaktualisierung, weil fuer `https://dh@git.belevo.ch` kein Passwort/Token bereitstand. ## 1. Zuerst lesen: Wo wir tatsächlich stehen From b6e7353086258ae81759ea6c8b2ea4f5f9147f26 Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Sun, 4 Oct 2026 17:18:14 +0000 Subject: [PATCH 20/22] docs(v4): record active-operation safety findings --- docs/NETPLAN_V4_AGENT_HANDOFF.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md index a370441..0d726e7 100644 --- a/docs/NETPLAN_V4_AGENT_HANDOFF.md +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -8,6 +8,8 @@ **Fortsetzung bestätigte Rückmeldung am 04.10.2026:** Daniels Freigabe des modellierten `virtual_split` gilt für den ausdrücklich begrenzten Feldtest. Die physische Rückmeldung wurde um synchrone, durch den Gerätetreiber bestätigte Lesevorgänge erweitert: M-Bus verwendet `MBUS_UpdateValues`, ModBus Device und ModBus Address verwenden `ModBus_RequestRead`. Nur ein erfolgreicher Geräteaufruf erzeugt `confirmedAt`; `VariableUpdated` bleibt getrennte Herkunftsinformation. Die Regeltest-Gates verlangen `deviceReadConfirmed=true`. Das reproduzierbare inkrementelle Paket liegt auf der Testanlage unter `/srv/agent/netplan-v4-confirmed-feedback-20261004`; 18 Paketprüfsummen, 37 Rückmeldetests, 7 Geräteabruf-Tests und 71 Regeltest-Prüfungen sind erfolgreich. Es ist noch **nicht im IP-Symcon-Kernel installiert**, weil der lokale JSON-RPC-Zugang ohne Anmeldung mit HTTP 401 antwortet. Die Installation selbst lässt alten Netzfahrplan und beide Regeltestfreigaben aus. Ein realer Stellversuch bleibt zusätzlich durch den nicht belegten unabhängigen Geräte-Watchdog und die fehlende Serverfreigabe blockiert; der vorhandene Symcon-`VorgabeTimeout` ist kein unabhängiger Hardware-Nachweis. Auf `enelix-services` und der Testanlage ist kein nutzbarer Git-HTTPS-Credential-Helper hinterlegt; die Prüfung des Tunnel-Servers lief zweimal in eine externe Connector-Freigabezeitüberschreitung. Tokeninhalt wurde weder gelesen noch ausgegeben. Der Commit des Folgepakets ist lokal mit der vereinbarten Identitaet `dh_Agent ` erstellt. Der atomare Push auf `develop` und `beta` scheiterte ohne Teilaktualisierung, weil fuer `https://dh@git.belevo.ch` kein Passwort/Token bereitstand. +**Fortsetzung Aktivbereitschaft am 04.10.2026:** Die reale Stellkette wurde in den gespeicherten Testanlagen-Settings bis zu den Aktionszielen geprüft. Die ENELIX-Batterieinstanz 44234 schreibt über das Aktionsskript 31800 auf den EV-Sollwerteingang 19651 des Gateways 58448. Dieses Gateway verteilt im Zwei-Sekunden-Takt auf zwei GoodWe-Pfade über reine SetValue-Aktionsskripte und auf drei SolarEdge-Modbus-Adressen. Der ENELIX-`VorgabeTimeout` setzt bei laufendem Symcon nach 30 Sekunden auf 0 W zurück. Weder im Gateway noch in den drei GoodWe-Aktionsskripten wurde jedoch ein vom Symcon-Kernel unabhängiger Geräte-Watchdog gefunden; auch die SolarEdge-Modbus-Konfiguration weist nur Poll-/Write-Parameter, keinen Geräte-Timeout aus. Ein unbeaufsichtigter 1-2-Tage-Stellbetrieb bleibt deshalb gesperrt. Beide lokalen Regeltestfreigaben sind weiterhin aus, es wurde kein Stellbefehl ausgelöst. Der bestätigte Rückmeldeinstaller ist für einen gestoppten Kernel vorbereitet und geprüft, konnte mangels privilegierter `systemctl`-Berechtigung des Connector-Benutzers aber noch nicht ausgeführt werden. Auf `tunel100` ist ein Credential-Helper für das Konto `dh` vorhanden und ein `develop`-Dry-Run war erfolgreich; die danach nötige Connector-Freigabe lief erneut ab, bevor der lokale 19-Commit-Stand übertragen und veröffentlicht werden konnte. + ## 1. Zuerst lesen: Wo wir tatsächlich stehen - Daniel möchte die **fertige, produktionsgeeignete Anwendung**, nicht weitere isolierte Sammler, Diagnosekategorien oder wiederholte Bestätigungsrunden. Er hat die fortlaufende Umsetzung mehrfach beauftragt. Probleme im Code selbst beheben, Tests und Auslieferung bündeln; ihn nur für wirklich notwendige Root-/Symcon-Ausführung oder echte Anlagenfreigaben einbeziehen. From e6aa50bb444c0eb57f55c8e6c6963d4c610268cc Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Mon, 5 Oct 2026 14:50:48 +0000 Subject: [PATCH 21/22] fix(v4): retry transient confirmed device reads --- libs/BatterieNetzfahrplanV4RueckmeldungTrait.php | 3 ++- libs/ManagerNetzfahrplanV4EmpfangTrait.php | 10 +++++++--- libs/NetzfahrplanV4Geraeteabruf.php | 15 +++++++++++++-- tests/V4Feedback/device_read_checks.php | 6 ++++++ tests/V4Feedback/receiver_checks.php | 3 ++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php index 0ddfcdd..d2fd5dc 100644 --- a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php +++ b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php @@ -49,7 +49,8 @@ trait BatterieNetzfahrplanV4RueckmeldungTrait return false; }, $reader, - static fn():int=>time() + static fn():int=>time(), + static function(int $microseconds):void { usleep($microseconds); } ); $this->SetBuffer('V4ConfirmedFeedback',json_encode($result,JSON_THROW_ON_ERROR)); return $result; diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php index a6fde3f..0785d88 100644 --- a/libs/ManagerNetzfahrplanV4EmpfangTrait.php +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -47,10 +47,10 @@ trait ManagerNetzfahrplanV4EmpfangTrait /** Appended dynamically; existing form.json and parallel portal GUI stay untouched. */ private function netzfahrplanV4EmpfangFormular(): array { - return ['type' => 'ExpansionPanel', 'caption' => 'V4 Planempfang / Sollwert-Vorschau', 'items' => [ - ['type' => 'Label', 'caption' => 'NUR BEOBACHTEN. Kein Lade-/Entladebefehl. Der alte intelligente Netzfahrplan muss AUS bleiben.'], + return ['type' => 'ExpansionPanel', 'caption' => 'V4 Planempfang / Prognosemanager', 'items' => [ + ['type' => 'Label', 'caption' => 'Der alte intelligente Netzfahrplan muss AUS bleiben. Die separate Laufzeitvariable unter dem Manager schaltet den zeitlich begrenzten Testbetrieb ein oder aus.'], ['type' => 'CheckBox', 'name' => 'NetzfahrplanV4EmpfangAktiv', 'caption' => 'V4 Schattenplan empfangen und Sollwert anzeigen (keine Steuerung)'], - ['type' => 'Label', 'caption' => 'Diagnose und Sollwerte stehen in den beiden V4-Vorschauvariablen unterhalb des Managers.'], + ['type' => 'Label', 'caption' => 'Schalter: V4 Prognosemanager aktiv (Testbetrieb). Ausschalten beendet V4 und fordert sofort 0 W an. Maximale Laufzeit ab dem ersten Einschalten: 48 Stunden.'], ['type' => 'Button', 'caption' => 'Plan jetzt pruefen (Abruf begrenzt auf 1/min)', 'onClick' => "IPS_RequestAction(\$id, 'NetzfahrplanV4Empfangen', true);"], ]]; @@ -172,6 +172,10 @@ trait ManagerNetzfahrplanV4EmpfangTrait if (is_string($configured) && !in_array(trim($configured),['','{}'],true)) { if (count($topology)!==1) throw new RuntimeException('feedback_multi_asset_adapter_required'); $f=json_decode(ENELIX_GetV4BatterieRueckmeldung($controller),true,32,JSON_THROW_ON_ERROR); + if (($f['status']??null)==='unavailable') { + $reason=is_string($f['reason']??null)?$f['reason']:'unknown'; + throw new RuntimeException('feedback_unavailable:'.substr($reason,0,180)); + } if (($f['installationId']??null)!==$this->ReadAttributeString('LizenzInstallationID') || ($f['assetId']??null)!==$asset['ID'] || ($f['managerId']??null)!==$this->InstanceID || ($f['batteryInstanceId']??null)!==$controller || !is_numeric($f['batteryW']??null) diff --git a/libs/NetzfahrplanV4Geraeteabruf.php b/libs/NetzfahrplanV4Geraeteabruf.php index 097943e..af58290 100644 --- a/libs/NetzfahrplanV4Geraeteabruf.php +++ b/libs/NetzfahrplanV4Geraeteabruf.php @@ -25,7 +25,8 @@ final class NetzfahrplanV4Geraeteabruf callable $moduleId, callable $refresh, callable $reader, - callable $clock + callable $clock, + ?callable $pause = null ): array { $config = NetzfahrplanV4Rueckmeldung::configuration($config); $groups = []; @@ -43,7 +44,17 @@ final class NetzfahrplanV4Geraeteabruf if (!in_array($module, [self::MBUS_DEVICE, self::MODBUS_DEVICE, self::MODBUS_ADDRESS], true)) { throw new InvalidArgumentException('feedback_refresh_module_unsupported:' . $sources[0]['key']); } - if ($refresh($parentId, $module) !== true) { + $confirmed = false; + for ($attempt = 1; $attempt <= 4; ++$attempt) { + if ($refresh($parentId, $module) === true) { + $confirmed = true; + break; + } + if ($attempt < 4 && $pause !== null) { + $pause(200000); + } + } + if (!$confirmed) { throw new InvalidArgumentException('feedback_device_read_failed:' . $sources[0]['key']); } $confirmedAt = $clock(); diff --git a/tests/V4Feedback/device_read_checks.php b/tests/V4Feedback/device_read_checks.php index 6b00aca..bb6f455 100644 --- a/tests/V4Feedback/device_read_checks.php +++ b/tests/V4Feedback/device_read_checks.php @@ -35,6 +35,12 @@ dcheck($f['deviceReadConfirmed']===true&&$f['sourceOldestAt']===$n-1,'successful dcheck($f['sourceVariableOldestAt']===$n-100,'original VariableUpdated retained separately'); dcheck($f['batteryW']===-3000.0&&$f['usableForTrial'],'unchanged values accepted after confirmed device response'); +[$c,$r,$n]=dfixture();$times=[$n,$n,$n];$attempts=0;$pauses=[]; +$f=D::snapshot($c,fn($id)=>D::MODBUS_DEVICE, + function()use(&$attempts){$attempts++;return $attempts>=2;},fn($id)=>$r[$id], + function()use(&$times){return array_shift($times);},function($us)use(&$pauses){$pauses[]=$us;}); +dcheck($attempts===3&&$pauses===[200000],'transient busy read is retried once without relaxing confirmation'); + [$c,$r,$n]=dfixture();$times=[$n,$n,$n]; dfails(fn()=>D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>false,fn($id)=>$r[$id],function()use(&$times){return array_shift($times);}), 'failed device read rejected'); diff --git a/tests/V4Feedback/receiver_checks.php b/tests/V4Feedback/receiver_checks.php index f617f4b..62887d5 100644 --- a/tests/V4Feedback/receiver_checks.php +++ b/tests/V4Feedback/receiver_checks.php @@ -34,8 +34,9 @@ namespace { ck($p['status']==='preview_only'&&$p['actualBatteryW']==0,'real receiver uses corrected physical feedback rather than held -9kW'); ck($p['previewBatteryW']==3000,'net target translated once to total battery demand'); ck($p['canDispatch']===false&&$p['localFeedback']['b']['estimated']===true,'model feedback remains marked and preview grants no dispatch'); - $feedback=['status'=>'unavailable','batteryW'=>null,'gridW'=>null];$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + $feedback=['status'=>'unavailable','reason'=>'feedback_device_read_failed:grid','batteryW'=>null,'gridW'=>null];$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); ck($p['status']==='unavailable'&&$p['previewBatteryW']===null,'configured feedback failure never falls back to held virtual display'); + ck(str_contains($p['reason'],'feedback_unavailable:feedback_device_read_failed:grid'),'configured feedback exposes bounded root cause without credentials'); $configured=false;$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); ck($p['status']==='preview_only'&&$p['feedbackMeaning']==='legacy_display_only_not_for_trial','unconfigured old preview stays compatible but explicitly not for control'); ck($p['canDispatch']===false,'legacy display cannot gain actuation permission'); From ce525a1ca071af9a569a3c93c244dbc8d3e159f2 Mon Sep 17 00:00:00 2001 From: dh_Agent Date: Mon, 5 Oct 2026 15:10:25 +0000 Subject: [PATCH 22/22] fix(v4): reset receiver throttle on reinit --- libs/ManagerNetzfahrplanV4EmpfangTrait.php | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php index 0785d88..7047efc 100644 --- a/libs/ManagerNetzfahrplanV4EmpfangTrait.php +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -32,6 +32,7 @@ trait ManagerNetzfahrplanV4EmpfangTrait $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); $this->WriteAttributeString('NetzfahrplanV4LetzteBestaetigung', '{}'); $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', '{"status":"not_received"}'); + $this->WriteAttributeInteger('NetzfahrplanV4NaechsterAbruf', 0); $this->zeigeNetzfahrplanV4Vorschau(['status' => 'not_received', 'reason' => 'Neu initialisiert; keinen alten Sollwert weiterverwenden.']); }