diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a3c9c9d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# Agenten-Anweisungen für Enelix-EMS + +## Aktueller V4-Fortsetzungseinstieg: 06.10.2026 + +Neuester Quellstand: Bedienintegration und Zusammenfuehrung sind in +docs/prognose-bedienintegration.md und im obersten Abschnitt der folgenden +Uebergabe beschrieben. Daniel hat Commit/Push auf develop und beta ausdruecklich +freigegeben; dies ist kein Anlagenrollout und keine neue Stellfreigabe. + +Die vollstaendige Uebergabe liegt auf **enelix-services / Server 89** unter +`/srv/agent/repos/Enelix-EMS/docs/NETPLAN_V4_AGENT_HANDOFF.md`. +Vor Fortsetzung zuerst den neuesten datierten Abschnitt lesen. Die Datei ist +im integrierten Repository versioniert; alte Serverkopien koennen abweichen. + +Letzter hier belegter Anlagenzustand: **06.10.2026, 12:01:11 UTC / +14:01:11 Europe/Zurich**, aus `/srv/agent/v4-recovery-20261006/finished.json` +auf **iot-symcon01**. Dies ist keine neue Live-Abnahme: +V4 und alter Netzfahrplan AUS, Batterie 44234 gestoppt, EV-Auftrag 19651 = 0 W. +Der unabhaengige SDL-Auftrag blieb unangetastet. +Rueckmelde-/Cache-/Empfangskorrekturen sind installiert; **offen ist die +instanzbezogene Batterietimerblockade (LastRun=0)**. Kein nachgewiesenes +physisches Tracking, kein >90-s-Dauerbetrieb und kein automatischer +Batterie-Watchdog-Lauf. Alte Installer nicht erneut ausfuehren und keine +externen Tick-Hooks als Umgehung einsetzen. Zuerst Timerursache beheben und +automatische Ausfuehrung belegen, danach nur innerhalb gueltiger Freigabe testen. + +EV bleibt **161.44 kWh / 39 kW**. Die bestehende befristete Testfreigabe endet +**07.10.2026, 17:38:07 Europe/Zurich**; diese Notiz erneuert oder verlaengert +sie nicht. Wegfall des zusaetzlichen +/-5-kW-Limits hebt Geraete-, SOC-, +Netzgrenzen und die dokumentierten Grenzen der Testanlagen-Ausnahme nicht auf. +Kein Aktivstart oder Dienstneustart aus der Bitte um Kontext-Sicherung ableiten. + +Aeltere V4-Statusangaben und damalige "naechste Schritte" weiter unten sind +historisch, soweit die neuere Uebergabe sie ersetzt. Sonstige Projekt- und +Sicherheitsregeln sowie separate Manager-Leistungsverteilungsnotizen bleiben +unveraendert. Aktuellen Zustand und Freigaben vor operativen Arbeiten erneut +pruefen. Ein neuer Agent muss diese Quellen lesen; kein automatischer Chattransfer. + +## Netzfahrplan V4: zuerst den Gesprächskontext übernehmen + +Bei Aufgaben zu Netzfahrplan, Prognosen, V4, Batterieoptimierung oder Lihrenmoos **vor Änderungen** die vollständige Übergabe lesen: + +**[`docs/NETPLAN_V4_AGENT_HANDOFF.md`](docs/NETPLAN_V4_AGENT_HANDOFF.md)** + +Sie enthält Daniels Anforderungen und Korrekturen, installierte versus nur entwickelte Komponenten, Quellen-/Instanzzuordnung, bekannte behobene Fehler, Grenzen der Testfreigabe und den nächsten konkreten Arbeitsschritt. Die Runtime-Berichte stammen zuletzt vom 03.10.2026; Zeitpunkt und aktuellen Zustand prüfen. Alte Entwicklungsdokumente können noch „nicht installiert“ sagen, obwohl ein jüngerer Installationsbericht den Abschluss belegt. + +Kurzstand bei Erstellung dieser Anweisung: Unified RC1 und korrigierter Rückmelde-/begrenzter Testpfad installiert; Versandproblem behoben. Nächste offene Umsetzung sind bestätigte Geräte-Lesezeitpunkte statt pauschaler Verwendung von `VariableUpdated` (`feedback_source_skew`). EV **161.44 kWh / 39 kW**, nicht die zurückgenommenen 160/30. Dauerproduktion ist noch nicht fertig; keine automatische Stellfreigabe. + +## Arbeits- und Git-Regeln + +- Vor Arbeit Host-`/srv/agent/AGENT_CONTEXT.md`, Branch und Arbeitsverzeichnis prüfen. Servercheckout ist nicht gleich laufendes Symcon-Modulverzeichnis. +- Neue Entwicklung auf **`develop`**, keine Feature-Branches. Nur eigene/zugehörige Änderungen committen. `develop` darf im vereinbarten Rahmen gepusht werden; `beta` nur gemäss ausdrücklicher Freigabe und nach Tests, `main` nach ausdrücklichem Auftrag/Feldabnahme. Keine Schutzregeln oder fehlende Authentifizierung umgehen. +- Daniel möchte zusammenhängende produktionsgeeignete Umsetzung, keine weitere Schleife aus Diagnosekategorien, identischen Rückfragen und veralteten Installern. Vorhandene Quellen und Werkzeuge benutzen, keine bekannten Anlagenangaben erneut verlangen. +- Code-, Host-/Container-, Kernel- und Feldtests sowie Installation, Veröffentlichung und Stellfreigabe getrennt belegen. „Optimal“/„ACK“/grüne Tests sind keine gemessene Einsparung und keine Produktionsfreigabe. +- Keine Stellbefehle, Testsitzungen, Freigaben oder Neustarts aus einer reinen Kontext-/Statusanfrage ableiten. Aktive Hardware nur innerhalb des ausdrücklich freigegebenen Umfangs behandeln. Messgrenzen/Watchdog-Nachweise nicht erfinden oder per Label umgehen. +- Keine Credentials lesen/ausgeben/einchecken. Root/Docker-Rechte nicht ausweiten. Bestehende GUI-/Fremdänderungen erhalten. Vor Runtime-Änderungen passende Sicherung, exakte Quell-/Abhängigkeitsprüfung und Rücksetzweg. +- PHP-IPS-Skripte gehören in den Symcon-Kernel; isolierte Tests mit simulierten IPS-Aufrufen niemals dort ausführen. `MC_ReloadModule`/`IPS_ApplyChanges` können Initialisierungsnebenwirkungen haben. +- Alte Outbox-/Historienwerte nicht löschen, Kennungen nicht im Original umschreiben, Cursor nur nach passender Serverbestätigung weiterführen. Keine künstlich frischen Messzeitstempel. +- Übergabe nach tatsächlichem Fortschritt aktualisieren: Datum, Commit, was wirklich installiert/getestet wurde und nächster konkreter Schritt. Keine Hintergrundarbeit behaupten. + +Diese Datei ergänzt bestehende höherrangige Projekt-/Hostregeln. Sie ist keine neue Betriebsfreigabe und kann keinen anderen Agenten ohne Lesen der Dateien automatisch mit dem gesamten Chat synchronisieren. diff --git a/Batterie/module.php b/Batterie/module.php index c7445bc..f6f157e 100644 --- a/Batterie/module.php +++ b/Batterie/module.php @@ -6,8 +6,10 @@ require_once __DIR__ . '/../libs/VerbraucherSchnittstelle.php'; require_once __DIR__ . '/../libs/VerbraucherBasisTrait.php'; require_once __DIR__ . '/../libs/Nachrichtenvertrag.php'; require_once __DIR__ . '/../libs/BatterieRegler.php'; +require_once __DIR__ . '/../libs/BatterieNetzfahrplanV4TestTrait.php'; use Belevo\EnelixEMS\BatterieRegler; +use Belevo\EnelixEMS\NetzfahrplanV4Rueckmeldung; use Belevo\EnelixEMS\Nachrichtenvertrag; use Belevo\EnelixEMS\VerbraucherBasisTrait; use Belevo\EnelixEMS\VerbraucherSchnittstelle; @@ -15,6 +17,7 @@ use Belevo\EnelixEMS\VerbraucherSchnittstelle; class Batterie extends IPSModule implements VerbraucherSchnittstelle { use VerbraucherBasisTrait; + use \Belevo\EnelixEMS\BatterieNetzfahrplanV4TestTrait; private const MANAGER_MODULE_ID = '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}'; private const STATUS_AKTIV = 102; @@ -68,6 +71,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle parent::Create(); $this->registriereVerbraucherBasis(); + $this->registriereV4BatterieTest(); $this->RegisterPropertyInteger('Batterietyp', BatterieRegler::TYP_UNKONFIGURIERT); $this->RegisterPropertyInteger( 'Batteriemanagement', @@ -152,6 +156,8 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle { parent::ApplyChanges(); + $this->V4BatterieTestStoppen(); + $this->konfiguriereV4Rueckmeldung(); $this->aktualisiereVariablen(); $this->bereinigeUngueltigenSollwert(); $this->registriereMesswertmeldungen(); @@ -249,7 +255,25 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->aktualisiere(true); return; + case 'NetzfahrplanV4TestStart': + if (!is_string($wert)) throw new InvalidArgumentException('JSON-Testanmeldung erforderlich.'); + $this->V4BatterieTestStarten($wert); + return; + case 'NetzfahrplanV4RueckmeldungAktualisieren': + $this->aktualisiereV4Rueckmeldung(); + return; + case 'NetzfahrplanV4TestCommand': + if (!is_string($wert)) throw new InvalidArgumentException('JSON-Testbefehl erforderlich.'); + $this->V4BatterieTestBefehl($wert); + return; + case 'NetzfahrplanV4TestStop': + $this->V4BatterieTestStoppen(); + return; + case 'NetzfahrplanV4TestWatchdog': + $this->aktualisiere(true); + return; case 'SichererZustand': + $this->V4BatterieTestStoppen(); $this->verwerfeSollwert(); $this->schreibeRegister(0, true); $this->aktualisiere(true); @@ -288,6 +312,14 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle /** @param array $daten */ public function ManagerdatenEmpfangen(array $daten): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterieannahme beschaeftigt.'); + try { $this->ManagerdatenEmpfangenIntern($daten); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function ManagerdatenEmpfangenIntern(array $daten): void { Nachrichtenvertrag::pruefeManagerdaten($daten); if ($daten['Kopf']['EmpfaengerID'] !== $this->InstanceID) { @@ -303,8 +335,13 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ); } + if ($this->v4BatterieTestAktiv()) { + // Ordinary manager messages cannot overwrite or renew the independent short test lease. + $this->planeMeldung(); + return; + } $this->WriteAttributeString('Betriebsart', $daten['Betriebsart']); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); $sollleistung = $daten['Sollleistung_W']; if ($sollleistung === null) { @@ -315,7 +352,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ) ) { $this->verwerfeSollwert(); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); } $this->planeMeldung(); return; @@ -342,17 +379,26 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->setzeZustand('Sollleistung', $sollleistung); $this->setzeZustand('SollwertGueltig', true); $this->planeVorgabeTimeout(time()); - $this->aktualisiere(false); + $this->aktualisiereIntern(false); $this->planeMeldung(); } private function aktualisiere(bool $meldungPlanen): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { $this->aktualisiereIntern($meldungPlanen); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function aktualisiereIntern(bool $meldungPlanen): void { try { $jetzt = time(); $this->aktualisiereEnergie($jetzt); $messwerte = $this->leseMesswerte(); $messwerteGueltig = $messwerte['Gueltig']; + $this->SetBuffer('V4FeedbackEstimated', ($messwerte['V4FeedbackEstimated'] ?? false) ? '1' : '0'); $this->setzeZustand('Ladezustand', $messwerte['Ladezustand']); $this->setzeZustand('LadezustandGueltig', $messwerteGueltig); @@ -363,7 +409,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $this->setzeSichtbareVariable( 'Leistungsquelle', $messwerteGueltig - ? Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN + ? (($messwerte['V4FeedbackEstimated'] ?? false) ? Nachrichtenvertrag::LEISTUNGSQUELLE_BERECHNET : Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) : Nachrichtenvertrag::LEISTUNGSQUELLE_NICHT_VORHANDEN ); $this->aktualisiereLadestatus( @@ -427,6 +473,8 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $ziel = $verfuegbar && (bool) $this->leseZustand('SollwertGueltig') ? (int) $this->leseZustand('Sollleistung') : 0; + $v4Ziel = $this->v4BatterieTestZiel($messwerte, $verfuegbar, $jetzt); + if ($v4Ziel !== null) $ziel = $v4Ziel; $this->schreibeRegister($ziel, false); $this->setzeZustand('Sollleistung', $ziel); $this->setzeZustand('Verfuegbar', $verfuegbar); @@ -486,6 +534,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $gueltig = true; foreach ($zuordnung as $property => $schluessel) { + if ($this->v4BatterieTestAktiv() && in_array($schluessel,['Netzleistung','Istleistung'],true)) continue; $variablenID = $this->ReadPropertyInteger($property); if (!$this->istMesswertAktuell($variablenID)) { $gueltig = false; @@ -494,8 +543,17 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle $werte[$schluessel] = (float) GetValue($variablenID); } - if ($werte['MaxLaden'] <= 0.0 - || $werte['MaxEntladen'] <= 0.0 + if ($this->v4BatterieTestAktiv()) { + try { + $f=$this->v4BatterieRueckmeldung(); + NetzfahrplanV4Rueckmeldung::requireTrial($f,$this->v4BatterieTestSitzung(),time()); + $werte['Istleistung']=$f['batteryW'];$werte['Netzleistung']=$f['gridW']; + $werte['V4FeedbackEstimated']=$f['estimated']; + } catch (Throwable $e) { $gueltig=false; } + } + + if ($werte['MaxLaden'] < 0.0 + || $werte['MaxEntladen'] < 0.0 || $werte['Ladezustand'] < 0.0 || $werte['Ladezustand'] > 100.0 ) { @@ -536,7 +594,9 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle try { foreach ($werte as $register => $wert) { $property = self::REGISTER_PROPERTIES[$register]; - RequestAction($this->ReadPropertyInteger($property), $wert); + if (RequestAction($this->ReadPropertyInteger($property), $wert) !== true) { + throw new RuntimeException('Registeraktion nicht bestaetigt: ' . $register); + } } } catch (Throwable $fehler) { throw new RuntimeException( @@ -673,7 +733,7 @@ class Batterie extends IPSModule implements VerbraucherSchnittstelle ? (float) $this->leseZustand('Istleistung') : null, 'Leistungsquelle' => $messwerteGueltig - ? Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN + ? ($this->GetBuffer('V4FeedbackEstimated') === '1' ? Nachrichtenvertrag::LEISTUNGSQUELLE_BERECHNET : Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) : Nachrichtenvertrag::LEISTUNGSQUELLE_NICHT_VORHANDEN, 'Zustand' => [ [ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b61f23..f7dfadd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Alle wesentlichen Aenderungen an Enelix EMS werden in dieser Datei dokumentiert. ## Unreleased +- Manager: V4-Planempfang und Prognosebedienung im vorhandenen Bereich + Prognose / Forecast zusammengefasst. Explizite Start-/Stoppaktionen pruefen + Lizenz und bestehende Testfreigaben. Kein automatischer Start, keine neue + Dauerbetriebsfreigabe und keine Loeschung bestehender Laufzeitobjekte. +- Integration des bisherigen V4-Entwicklungsstands mit den aktuellen Manager-, + SDL- und Sollwertkorrekturen; V4-Stopp und Sollwertverwerfen bleiben erhalten. +- Optionaler nativer V4-Betriebsdatensender fuer den Schattenversuch (standardmaessig aus), separate Diagnose und sichere Batterie-/Messquellenzuordnung. Kein neuer Fahrplan wird lokal ausgefuehrt; vollstaendige Monatspeak-/Viertelstunden-Messnachweise und deren Historienadapter bleiben Voraussetzung. - Verbraucher: Beim Verwerfen einer Manager-Vorgabe wird auch die gespeicherte und sichtbare Sollleistung auf 0 W gesetzt. Gilt fuer Batterie, beide Ladestationen, Pufferspeicher, Verbraucher 1-Stufig, Waermepumpe und diff --git a/Manager/form.json b/Manager/form.json index a837a63..54d92cc 100644 --- a/Manager/form.json +++ b/Manager/form.json @@ -799,6 +799,7 @@ }, { "type": "ExpansionPanel", + "name": "PrognoseForecast", "caption": "Prognose / Forecast", "items": [ { diff --git a/Manager/module.php b/Manager/module.php index 3e56a06..b0a5863 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -12,6 +12,11 @@ require_once __DIR__ . '/../libs/StoerungsSnapshot.php'; require_once __DIR__ . '/../libs/EnergieMessung.php'; require_once __DIR__ . '/../libs/ManagerEnergieTrait.php'; require_once __DIR__ . '/../libs/ManagerSDLTrait.php'; +require_once __DIR__ . '/../libs/NetzfahrplanV4Betriebsdaten.php'; +require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4Trait.php'; +require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4TestTrait.php'; +require_once __DIR__ . '/../libs/ManagerNetzfahrplanV4AktivtestTrait.php'; +require_once __DIR__ . '/../libs/ManagerPrognoseFormular.php'; use Belevo\EnelixEMS\Anlagentopologie; use Belevo\EnelixEMS\EinspeiseRegler; @@ -19,6 +24,7 @@ use Belevo\EnelixEMS\EnergieMessung; use Belevo\EnelixEMS\ManagerEnergieTrait; use Belevo\EnelixEMS\Lizenzpruefung; use Belevo\EnelixEMS\ManagerRegler; +use Belevo\EnelixEMS\ManagerPrognoseFormular; use Belevo\EnelixEMS\ManagerSchnittstelle; use Belevo\EnelixEMS\Nachrichtenvertrag; use Belevo\EnelixEMS\StoerungsSnapshot; @@ -27,6 +33,9 @@ class Manager extends IPSModule implements ManagerSchnittstelle { use ManagerEnergieTrait; use \Belevo\EnelixEMS\ManagerSDLTrait; + use \Belevo\EnelixEMS\ManagerNetzfahrplanV4Trait; + use \Belevo\EnelixEMS\ManagerNetzfahrplanV4TestTrait; + use \Belevo\EnelixEMS\ManagerNetzfahrplanV4AktivtestTrait; private const STATUS_AKTIV = 102; private const STATUS_MESSWERT_UNGUELTIG = 201; @@ -118,6 +127,9 @@ class Manager extends IPSModule implements ManagerSchnittstelle public function Create(): void { parent::Create(); + $this->registriereNetzfahrplanV4(); + $this->registriereV4ManagerTest(); + $this->registriereV4ManagerAktivtest(); $this->RegisterPropertyInteger('Rolle', 0); $this->RegisterPropertyInteger('NetzleistungVariableID', 0); @@ -257,8 +269,12 @@ class Manager extends IPSModule implements ManagerSchnittstelle public function ApplyChanges(): void { parent::ApplyChanges(); + $this->v4ManagerTestAbbrechen('Manager neu initialisiert'); + $this->SetTimerInterval('NetzfahrplanV4Senden', 0); + $this->stoppeNetzfahrplanV4Vorschau(); $this->initialisiereLizenzInstallationID(); + $this->konfiguriereNetzfahrplanV4Daten(); $this->aktualisiereVariablen(); try { @@ -315,6 +331,10 @@ class Manager extends IPSModule implements ManagerSchnittstelle return; } $this->SetStatus(self::STATUS_AKTIV); + if ($this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->SetTimerInterval('NetzfahrplanV4Senden', 90000); + } + $this->konfiguriereNetzfahrplanV4Vorschau(); $this->synchronisiereAnlagentopologie(); if ($this->prognoseTelemetrieKonfiguriert()) { $this->SetTimerInterval( @@ -451,6 +471,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->ReadPropertyBoolean('SDLAktiv') ); + $formular = ManagerPrognoseFormular::integrieren($formular, $this->prognoseBedienzustand()); return json_encode($formular, JSON_THROW_ON_ERROR); } @@ -489,6 +510,48 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->regeln(false); return; + case 'NetzfahrplanV4Empfangen': + $this->empfangeNetzfahrplanV4(); + return; + + case 'NetzfahrplanV4VorschauAktualisieren': + $this->aktualisiereNetzfahrplanV4Vorschau(); + return; + + case 'NetzfahrplanV4TestStart': + if (!is_string($wert)) throw new InvalidArgumentException('Explizite JSON-Testfreigabe erforderlich.'); + $this->V4ManagerTestStarten($wert); + return; + case 'NetzfahrplanV4TestStop': + $this->V4ManagerTestStoppen(); + return; + + case 'NetzfahrplanV4Aktivtest': + $this->V4ManagerAktivtestSchalten((bool) $wert); + return; + + case 'FormNetzfahrplanSchalten': + if (!is_bool($wert)) { + throw new InvalidArgumentException('Netzfahrplan-Schaltung muss ein boolescher Wert sein.'); + } + if ($wert && !ManagerPrognoseFormular::startErlaubt($this->prognoseBedienzustand())) { + throw new RuntimeException('Netzfahrplan nicht freigegeben. Gespeicherte Konfiguration, Lizenz und Testfreigabe pruefen.'); + } + try { + $this->V4ManagerAktivtestSchalten($wert); + } finally { + $this->aktualisierePrognoseBedienung(); + } + return; + + case 'NetzfahrplanV4DatenErfassen': + $this->ErfasseNetzfahrplanV4Daten(); + return; + + case 'NetzfahrplanV4Senden': + $this->sendeNetzfahrplanV4(); + return; + case 'PrognoseSenden': $this->sendePrognoseTelemetrie(); return; @@ -520,6 +583,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle if (!$aktiv) { $this->UpdateFormField('NetzfahrplanAktiv', 'value', false); } + $this->aktualisierePrognoseBedienung($aktiv); return; case 'FormSDLAktiv': @@ -727,6 +791,28 @@ class Manager extends IPSModule implements ManagerSchnittstelle } private function regeln(bool $erneuern): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { + $trialBefore=(bool)$this->v4ManagerTestSession(); + $this->regelnIntern($erneuern); + if ($trialBefore && !$this->v4ManagerTestSession() && $this->GetValue('Aktiv') + && IPS_GetInstance($this->InstanceID)['InstanceStatus'] === self::STATUS_AKTIV) { + $this->WriteAttributeString('LetzteSollwerte','{}'); + $this->regelnIntern(true); // exactly one fresh fallback, no cached pre-trial target + } + if ($this->v4ManagerTestSession() + && (!$this->GetValue('Aktiv') || IPS_GetInstance($this->InstanceID)['InstanceStatus'] !== self::STATUS_AKTIV)) { + $this->v4ManagerTestAbbrechen('Manager nicht mehr betriebsbereit'); + } + } catch (Throwable $e) { + $this->v4ManagerTestAbbrechen('Regelungsfehler'); + throw $e; + } finally { IPS_SemaphoreLeave($lock); } + } + + private function regelnIntern(bool $erneuern): void { if (!$this->aktualisiereLizenzfreigabe($this->ReadPropertyString('Lizenzcode'), false)) { $this->SetValue('Betriebsart', ManagerRegler::BETRIEBSART_INAKTIV); @@ -831,6 +917,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle : $this->nichtGeregelteGeraeteText($nichtGeregelt) ); + $this->v4ManagerTestUebergabe($ergebnis, $verbraucher, $synchronisiert); $letzteSollwerte = $this->leseJsonAttribut('LetzteSollwerte'); foreach ($ergebnis['Sollwerte'] as $instanzID => $sollleistung) { if (!$erneuern && ($letzteSollwerte[(string) $instanzID] ?? null) === $sollleistung) { @@ -840,6 +927,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->sendeManagerdaten((int) $instanzID, $betriebsart, $sollleistung); $letzteSollwerte[(string) $instanzID] = $sollleistung; } catch (Throwable $fehler) { + $this->v4ManagerTestAbbrechen('Andere Verbraucher-Stellwertuebergabe fehlgeschlagen'); $stoerungen[] = 'Sollwert an Instanz ' . $instanzID . ': ' . $fehler->getMessage(); } } @@ -2600,6 +2688,35 @@ class Manager extends IPSModule implements ManagerSchnittstelle $aktiv && $netzfahrplanLizenziert && $this->peakShavingLizenziert($lizenzcode) ); + $this->aktualisierePrognoseBedienung(null, $lizenzcode); + } + + private function prognoseBedienzustand(?bool $aktiv = null, ?string $lizenzcode = null): array + { + return [ + 'prognoseAktiv' => ($aktiv ?? true) && $this->ReadPropertyBoolean('PrognoseAktiv'), + 'lizenziert' => $this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN, $lizenzcode), + 'senderAktiv' => $this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv'), + 'empfangAktiv' => $this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv'), + 'messdatenAktiv' => $this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv'), + 'legacyAktiv' => $this->ReadPropertyBoolean('NetzfahrplanAktiv'), + 'managerAktiv' => $this->GetValue('Aktiv') + && IPS_GetInstance($this->InstanceID)['InstanceStatus'] === self::STATUS_AKTIV, + 'testErlaubt' => $this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt'), + 'aktivtestErlaubt' => $this->ReadPropertyBoolean('NetzfahrplanV4AktivtestErlaubt'), + 'watchdogVerzichtErlaubt' => $this->ReadPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt'), + 'aktivtestAktiv' => (bool) $this->GetValue('NetzfahrplanV4Aktivtest'), + 'ende' => $this->ReadAttributeInteger('NetzfahrplanV4AktivtestEnde'), + 'jetzt' => time(), + ]; + } + + private function aktualisierePrognoseBedienung(?bool $aktiv = null, ?string $lizenzcode = null): void + { + $zustand = $this->prognoseBedienzustand($aktiv, $lizenzcode); + $this->UpdateFormField('PrognoseNetzfahrplanStart', 'enabled', ManagerPrognoseFormular::startErlaubt($zustand)); + $this->UpdateFormField('PrognoseNetzfahrplanStatus', 'caption', ManagerPrognoseFormular::statustext($zustand)); + $this->UpdateFormField('NetzfahrplanV4EmpfangAktiv', 'enabled', $zustand['lizenziert'] || $zustand['empfangAktiv']); } private function prognoseLizenzStatustext(bool $aktiv, bool $lizenziert): string diff --git a/docs/NETPLAN_V4_AGENT_HANDOFF.md b/docs/NETPLAN_V4_AGENT_HANDOFF.md new file mode 100644 index 0000000..b2209b5 --- /dev/null +++ b/docs/NETPLAN_V4_AGENT_HANDOFF.md @@ -0,0 +1,387 @@ +# ENELIX Netzfahrplan V4 – verbindlicher Gesprächs- und Arbeitskontext + +## Integration 06.10.2026: Freigabe fuer develop und beta, kein Anlagenrollout + +Daniel hat nach der Bedienintegration Merge, Commit und Push auf **develop und +beta** ausdruecklich beauftragt. Der getestete Integrationsstand verbindet die +22 lokalen V4-Commits bis ce525a1 mit origin/develop bis 5437f3f und den +zugehoerigen V4-/Formularkorrekturen. Originalarbeitsverzeichnisse bleiben +unangetastet; Integration in /srv/agent/prognose-integration-20261006/repo auf +ubuntu. Keine Feature-Branches, kein Force-Push, main bleibt unveraendert. + +Die Bedienung liegt im bisherigen Bereich Prognose / Forecast; der separate +V4-Bereich entfaellt. Einzelheiten und verbleibender Umfang stehen in +docs/prognose-bedienintegration.md. Es bleibt der vorhandene begrenzte Testpfad, +kein regulaerer Dauerbetrieb. Keine Laufzeitordner geloescht, keine Dienste +neu gestartet, keine Module geladen und kein neuer Stellversuch ausgefuehrt. +Die unten dokumentierte Batterietimerblockade bleibt offen. Die ausdrueckliche +Branchfreigabe ersetzt keine physische Anlagenabnahme oder Stellfreigabe. + +Isoliert bestanden: 357 PHPUnit-Tests / 1758 Assertions, Syntax aller 150 PHP- +Dateien, 310 Backend-Tests, 16 Portal-Tests und die Bedien-/V4-Einzelchecks. +Kein neuer Symcon-Kernel-/GUI-Test, weil Docker im Agentzugang nicht nutzbar war. +Eine lokale Backend-Aenderung, die ungepruefte Messkonfigurationen automatisch +mit accountingEvidenceId versah, wurde nach fehlgeschlagenem Schutztest nicht +uebernommen; Originalarbeitskopie und laufendes Backend bleiben erhalten. + +Commit-Autor ist gemaess bestehender Teamklarstellung dh_Agent , +konfiguriertes Gitea-Konto dh. Die Repository-Abfrage bestaetigte Pushrecht; +die Profilabfrage war mangels read:user-Scope nicht verfuegbar. Fuer den +tatsaechlichen Veroeffentlichungsstand die Remote-Refs und den Merge-Commit +pruefen, nicht historische Ahead-/Behind-Angaben weiter unten verwenden. + +## Kontext-Sicherung 06.10.2026: Lesereihenfolge und Quellen + +Auf Daniels ausdruecklichen Wunsch wird der Arbeitsstand dauerhaft verlinkt. +Fuer den Laufzeitstand ist der unmittelbar folgende Abschnitt **06.10.2026, +12:01 UTC** massgeblich, nicht die aelteren Abschnitte 1, 10, 11 oder 15. +Insbesondere ist der bestaetigte Geraeteabruf bereits implementiert/installiert; +der naechste Schritt ist die dort belegte instanzbezogene Batterietimerblockade. +Der Abschlussnachweis `/srv/agent/v4-recovery-20261006/finished.json` auf +iot-symcon01 wurde erneut gelesen, aber kein neuer Stellversuch ausgefuehrt. + +Git-Nachkontrolle in dieser Kontext-Sicherung: nach erfolgreichem Fetch steht +das Server-89-develop **22 Commits vor / 3 hinter origin/develop**; Remote-Stand +bei Fetch `58f91e4`. Kein sicherer Fast-Forward. Bestehende fremde/lokale +Aenderungen bleiben erhalten; kein Merge, Projekt-Commit oder Projekt-Push. +Das ist vom anderen Checkout auf ubuntu und dessen separat dokumentierter +Manager-Leistungsverteilung zu unterscheiden. + +Die mitgesendeten Screenshots haben kein bestaetigtes Aufnahmedatum und sind +historische Anschauung, kein neuer Laufzeitnachweis. Der Aktualisierungsdialog +ist keine Freigabe, lokale Modulaenderungen zu verwerfen. Es wurden keine +Module aktualisiert, Dienste neu gestartet oder Anlagenfreigaben veraendert. + +## Fortsetzung 06.10.2026, 12:01 UTC: Code installiert, Batterietimer blockieren Live-Abnahme + +- Daniel hat nach der ausdruecklichen Rueckfrage den Test ohne zusaetzliches +/-5-kW-Limit bestaetigt. Dynamische Geraetegrenzen (hier bis 39 kW), SOC- und Netzgrenzen bleiben verbindlich. Die urspruengliche Freigabe endet unveraendert am 07.10.2026 um 17:38:07 Europe/Zurich (1791387487). Kein Neustart der 48-h-Frist. Der fehlende unabhaengige Hardware-Watchdog bleibt nur durch die bereits genehmigte Testanlagen-Ausnahme abgedeckt. +- Zusaetzliche allgemeine Arbeitsfreigabe: Routinemaessige Schreibzugriffe auf Lihrenmoos, tunel100 und Server89 benoetigen keine erneute Rueckfrage. Das hebt Sicherheitsgrenzen, technische Berechtigungen, Git-Regeln und konkrete Anlagenfreigaben nicht auf. +- Der vorstehend beschriebene Kandidat wurde inzwischen installiert und getestet. Weitere eng begrenzte Korrekturen dieser Fortsetzung: (1) Batteriebefehl meldet den konkreten Ablehnungsgrund; (2) physische Geraeteabfrage aus dem Batterie-Stellmutex getrennt, eigenstaendiger 1-s-Samplingtimer mit nicht wartender Lesesperre; (3) Stellpfad nutzt nur unveraenderten bestaetigten Cache, maximal 2 s alt nach Wand- und monotoner Zeit, mit Konfigurations-/Quellenbindungspruefung; (4) passendes Server-Pending fuehrt auch vor Aktivstart zu 15-s-Retry statt zum Verpassen des fertigen Plans, HTTP-429 behaelt normalen Backoff. Cache-90-s-Grenze und physische Zeitstempel wurden nicht verlaengert oder erfunden. +- Live-Installationen mit SHA-256-Pruefung und Backups: Fehlerdiagnose 11:31:18 UTC; Sampler 11:38:43 UTC; Empfang um 11:50 UTC. Arbeits- und Sicherungsordner: /srv/agent/v4-recovery-20261006 auf iot-symcon01. Berichte deployed.json, sampler-deployed.json, receiver-deployed.json und test-results.json. Geaenderte Quellen und Tests sind auch im bestehenden develop-Arbeitsverzeichnis auf enelix-services, fremde Aenderungen sind erhalten. +- Letzter Aktivversuch: 11:50:52 UTC, Session 13395dbe-0dbb-42b8-9063-511bd7166f41. Manager meldete danach controller_command_sent (seq 5, 18693 W); Batterie wurde wegen feedback_confirmed_cache_stale_or_invalid gestoppt. Danach Managerabbruch wegen ungueltiger lokaler Betriebsdaten, controllerStopAccepted=true. Kein belastbarer Nachweis von physischem Tracking oder >90-s-Dauerbetrieb. Ein gesendeter Befehl ist ausdruecklich keine Anlagenabnahme. +- KONKRETE OFFENE BLOCKADE: Saemtliche Timer der Batterieinstanz 44234 zeigen LastRun=0. Meldezyklus (5000 ms) und neuer Rueckmeldungstimer (1000 ms) sind lange ueberfaellig, Running=false. Manager- und uebrige Anlagentimer laufen. Instanzstatus 102; Instanz und alle Elternobjekte nicht deaktiviert; keine dauerhaft belegten Skriptthreads nachgewiesen. IPS_ApplyChanges(44234) bei bestaetigtem Null-Stopp um 11:57:31 UTC gab TRUE zurueck, hat die Timer aber nicht repariert. Die Ursache ist NICHT geklaert. Auch der Batterie-Watchdog hat bisher keinen automatischen Lauf nachgewiesen. Deshalb kein weiterer Aktivstart und keine Umgehung durch einen externen Tick-Hook. +- Weitere beobachtete Abbruchbedingung: Durch Reloads/Messluecken fehlte zeitweise laufende Viertelstundenenergie; der Planner verwirft Luecken >120 s weiterhin korrekt. Nach einem Quartalswechsel wurden wieder optimale Plaene empfangen. Zum Schluss meldete der Betriebssender zudem veraltete Verbraucher-Rueckmeldung, passend zum nicht laufenden Batterie-Meldezyklus. Keine Energiestaende, Cursor oder Datenbankwerte wurden in dieser Fortsetzung umgeschrieben. +- Endzustand 12:01:11 UTC: Schalter 46716=false, alter Netzfahrplan=false, Batterie status=stopped, EV-Sollwert 19651=0 W. SDL-Sollwert war 7600 W aus dem unabhaengigen bestehenden Betrieb und wurde nicht auf null gezwungen. Kein physischer V4-Regelungsnachweis. Verbindlicher Abschlussbericht: /srv/agent/v4-recovery-20261006/finished.json. +- Alle temporaeren Start-, Deploy- und Diagnosehooks wurden aus 12555.ips.php entfernt. Original aus /srv/agent/12555.ips.php.v4hook-backup wiederhergestellt, SHA-256 c702a4947ae200b23c3eecccabeb843ad6a55e97509292584ca2de779d5e4940. Nur v4viewsRun des urspruenglichen getrennten Beobachters verbleibt. Inhaltsuche in /var/lib/symcon/scripts fand keine Verweise auf die V4-Recovery-/Continuation-Hooks. Keine unbeobachtete Wiederanlaufautomatik hinterlassen. +- Tests erneut erfolgreich: 96 Regel-/Cache-/Lockpruefungen, 32 Empfangspruefungen, 8 Rueckmeldungs-Empfangspruefungen, 9 Geraeteabrufpruefungen und 1 Test der echten Registermethode mit synthetischen Aktionen, insgesamt 146. Syntaxpruefungen erfolgreich; git diff --check sauber. Das sind automatisierte Tests, kein Hardwareabnahmetest. +- Git bleibt develop, ahead 22 / behind 1 gegen origin/develop, kein sicherer Fast-Forward. Keine Projekt-Commits, Merges oder Pushes. Konto nicht fuer Schreibaktion verifiziert; weder beta noch main geaendert. Die fehlende Live-Abnahme schliesst eine Beta-Freigabe zusaetzlich aus. +- Naechster fachlicher Schritt: Instanzbezogene Symcon-Timerblockade beheben, automatische Sampler-/Melde-/Watchdog-Ausfuehrung belegen und erst dann einen kontrollierten Neustart des V4-Tests versuchen. Ein etwa erforderlicher Symcon-Dienstneustart betrifft weitere Anlagenfunktionen und wurde hier NICHT ausgefuehrt. Danach physisches Tracking, Planwechsel >90 s und bestaetigten Null-Stopp nachweisen. Alte Startmarker oder Installationshooks keinesfalls einfach loeschen/reaktivieren. + +## Fortsetzung 06.10.2026: gepruefter Kandidat, Live-Eingriff noch gesperrt + +- Daniel hat die Fortsetzung und die Aufhebung der 5-kW-Testgrenze beauftragt. Die bestehende 48-h-Frist, SOC-/Netz-/Geraetegrenzen und der bereits bekannte fehlende unabhaengige Hardware-Watchdog bleiben unveraendert. +- Aktuell auf iot-symcon01 gelesen: Refresh-Deployment 07:43:14 UTC; Full-Power-Deployment 08:46:59 UTC. Diese waren schon vor dieser Fortsetzung installiert. Snapshot 09:30:41 UTC: Schalter AUS, Batterie disabled, Manager stopped_fallback_pending wegen ungueltiger lokaler Betriebsdaten. Keine physische V4-Regelung belegt. +- Neuer Kandidat korrigiert ignorierte FALSE-Rueckgaben von IPS_RequestAction/RequestAction (Start, Command, Stop und Register), den Vor-I/O-Zeitvergleich beim Batterie-Scharfschalten und den blockierenden HTTP-Refresh im Regel-Tick. Separater Empfangstimer: 10 s Prueftakt, aktiver Abruf fruehestens nach 45 s, Fehler-Retry 15 s; Cache-Lebensdauer und Serverzeit bleiben maximal 90 s. +- Kandidat liegt auf develop im Arbeitsverzeichnis und isoliert unter /srv/agent/v4-continuation-20261006 auf Lihrenmoos. 87 Regeltest-, 29 Empfangs-, 8 Rueckmeldungspruefungen plus 1 echter Modulmethoden-Test mit synthetischen Aktionen bestanden; 5 geaenderte Live-PHP-Dateien syntaxgeprueft. Keine Hardwaretests daraus ableiten. Testbericht: test-results.json im Stage. +- Neue Tests decken insbesondere FALSE ohne Exception, Sekundengrenze beim Start, bytegleich behaltenen aktiven Cache, 15-s-Retry, 90-s-Ablauf und zukuenftige Empfangszeit ab. git diff --check erfolgreich. +- Die automatische Sicherheitspruefung hat die Ausfuehrung von /srv/agent/activate_v4_continuation.py abgelehnt, weil diese Live-Installation und Aktivstart koppelt. Der Benutzer wurde um ausdrueckliche Freigabe fuer Installation und Start ohne 5-kW-Limit bis zum bisherigen Ende (07.10.2026 17:38 Europe/Zurich) gebeten. NICHT ueber andere Werkzeuge umgehen. Die Aktivierung wurde nicht ausgefuehrt; 12555.ips.php ist unveraendert. +- Achtung: Der bestehende temporaere v4_direct_battery_command_hook.php sendet alle 30 s den letzten Managerbefehl erneut. Das kann den Replay-Schutz ausloesen. Vor freigegebenem Neustart kontrolliert entfernen. Vorbereitete Aktivierung sichert den alten Timer und ersetzt alte Schreib-/Diagnosehooks durch eine einmalige gesicherte Installation, genau einen Startversuch und reine Beobachtung. Sie ist noch nicht ausgefuehrt. Bei Live-Drift abbrechen. +- Git nach erfolgreichem Fetch: develop ist 22 Commits vor und 1 Commit hinter origin/develop (fremder Commit 37addf4). Fremde Lade-/GUI-Aenderungen sind erhalten. Kein Fast-Forward moeglich; kein Commit, Merge oder Push in dieser Fortsetzung. Kandidatenpatch: /srv/agent/v4-continuation-20261006.patch auf enelix-services. +- Naechster Schritt erst nach Freigabe: Stage/Live-Hashes und Zustand erneut pruefen, Kandidat installieren, genau einen Startversuch beobachten und dessen echte Batterie-Antwort pruefen. Danach >90 s mit Planwechsel, reale Leistungs-/Netzreaktion und sicheren Nullstell-Stopp nachweisen. Erst anschliessend regulaeren Testlauf belassen und temporaere Hooks entfernen. Kein stabiler Aktivbetrieb zugesichert. + +Übergabe erstellt am 04.10.2026 auf ausdrücklichen Wunsch von Daniel Häfliger (BELEVO AG). Sie fasst die Anforderungen, Entscheidungen, Fehlerbehebungen und den zuletzt belegten Stand dieses Gesprächs zusammen. Sie ist **kein wörtliches Chatarchiv, keine neue Stellfreigabe und kein Nachweis der Produktionsreife**. + +**Zeitbezug:** Die letzten hier belegten Betriebsberichte stammen vom **03.10.2026, ca. 14:02 Uhr Europe/Zurich**. Am 04.10. wurden für diese Übergabe die genannten Berichte und der Git-Stand erneut gelesen, aber keine neue Anlagenabnahme durchgeführt. Alte Messzahlen deshalb niemals als heutige Live-Werte ausgeben. + +**Fortsetzung Prognose am 04.10.2026:** Der aktuelle Runtime-Stand wurde gelesen, ohne die alten Diagnose- oder Installationsschritte zu wiederholen. Der abgeleitete Datensatz `lihrenmoos-physical-published-v2` ist `model_ready` und erfüllt mit mehr als 24 nutzbaren äquivalenten Stunden die Trainingsschwelle. Die V4-Einstellungen stehen seit Revision 2 in `shadow` auf `forecastSource=corrected_profile` und `measurementDataset=lihrenmoos-physical-published-v2`; `liveEnabled` und Stellfreigabe bleiben aus. Der erste echte Lauf zeigte einen kompatibilitätsbedingten Stopp bei mikrosekundengenauen `observedAt`-Zeitstempeln. Die Korrektur rundet ausschliesslich die kausale Verfügbarkeit von Prognoseereignissen auf die nächste volle Sekunde auf; Rohmessungen bleiben streng ganzsekündlich. 310 Python- und 16 Portal-Tests sind erfolgreich. Repository und Runtime-Build-Kontext enthalten bytegleich den geprüften Fix; der laufende Container enthält ihn noch nicht, weil beide Agentzugänge weder Docker-Socket noch `sudo` erhalten. Der vorbereitete, syntaktisch geprüfte Root-Rollout liegt unter `/home/agent/services/netplan-v4-shadow/commissioning/finish_corrected_forecast_rollout.sh` und baut/testet vor dem Austausch, hält das vorige Image als Rollback fest und ändert keine Stellfreigabe. Daniel hat danach einen Aktivbetrieb angefragt; der aktuelle V4-Kern ist jedoch technisch `shadow-only`, und die weiterhin unbelegte Rückmeldung (`feedback_source_skew`, `usableForTrial=false`, `canDispatch=false`) verbietet ein blosses Umstellen. Keine Live-Freigabe wurde erteilt oder eingebaut. Git-Klarstellung durch Daniel: Commit-Autor `dh_Agent `, Gitea-Konto `dh`; Commit/Push auf `develop` und `beta` am 04.10.2026 ausdrücklich freigegeben. Der Agentzugang besitzt derzeit keine HTTPS-Schreibanmeldung für `dh`. + +**Fortsetzung bestätigte Rückmeldung am 04.10.2026:** Daniels Freigabe des modellierten `virtual_split` gilt für den ausdrücklich begrenzten Feldtest. Die physische Rückmeldung wurde um synchrone, durch den Gerätetreiber bestätigte Lesevorgänge erweitert: M-Bus verwendet `MBUS_UpdateValues`, ModBus Device und ModBus Address verwenden `ModBus_RequestRead`. Nur ein erfolgreicher Geräteaufruf erzeugt `confirmedAt`; `VariableUpdated` bleibt getrennte Herkunftsinformation. Die Regeltest-Gates verlangen `deviceReadConfirmed=true`. Das reproduzierbare inkrementelle Paket liegt auf der Testanlage unter `/srv/agent/netplan-v4-confirmed-feedback-20261004`; 18 Paketprüfsummen, 37 Rückmeldetests, 7 Geräteabruf-Tests und 71 Regeltest-Prüfungen sind erfolgreich. Es ist noch **nicht im IP-Symcon-Kernel installiert**, weil der lokale JSON-RPC-Zugang ohne Anmeldung mit HTTP 401 antwortet. Die Installation selbst lässt alten Netzfahrplan und beide Regeltestfreigaben aus. Ein realer Stellversuch bleibt zusätzlich durch den nicht belegten unabhängigen Geräte-Watchdog und die fehlende Serverfreigabe blockiert; der vorhandene Symcon-`VorgabeTimeout` ist kein unabhängiger Hardware-Nachweis. Auf `enelix-services` und der Testanlage ist kein nutzbarer Git-HTTPS-Credential-Helper hinterlegt; die Prüfung des Tunnel-Servers lief zweimal in eine externe Connector-Freigabezeitüberschreitung. Tokeninhalt wurde weder gelesen noch ausgegeben. Der Commit des Folgepakets ist lokal mit der vereinbarten Identitaet `dh_Agent ` erstellt. Der atomare Push auf `develop` und `beta` scheiterte ohne Teilaktualisierung, weil fuer `https://dh@git.belevo.ch` kein Passwort/Token bereitstand. + +**Fortsetzung Aktivbereitschaft am 04.10.2026:** Die reale Stellkette wurde in den gespeicherten Testanlagen-Settings bis zu den Aktionszielen geprüft. Die ENELIX-Batterieinstanz 44234 schreibt über das Aktionsskript 31800 auf den EV-Sollwerteingang 19651 des Gateways 58448. Dieses Gateway verteilt im Zwei-Sekunden-Takt auf zwei GoodWe-Pfade über reine SetValue-Aktionsskripte und auf drei SolarEdge-Modbus-Adressen. Der ENELIX-`VorgabeTimeout` setzt bei laufendem Symcon nach 30 Sekunden auf 0 W zurück. Weder im Gateway noch in den drei GoodWe-Aktionsskripten wurde jedoch ein vom Symcon-Kernel unabhängiger Geräte-Watchdog gefunden; auch die SolarEdge-Modbus-Konfiguration weist nur Poll-/Write-Parameter, keinen Geräte-Timeout aus. Ein unbeaufsichtigter 1-2-Tage-Stellbetrieb bleibt deshalb gesperrt. Beide lokalen Regeltestfreigaben sind weiterhin aus, es wurde kein Stellbefehl ausgelöst. Der bestätigte Rückmeldeinstaller ist für einen gestoppten Kernel vorbereitet und geprüft, konnte mangels privilegierter `systemctl`-Berechtigung des Connector-Benutzers aber noch nicht ausgeführt werden. Auf `tunel100` ist ein Credential-Helper für das Konto `dh` vorhanden und ein `develop`-Dry-Run war erfolgreich; die danach nötige Connector-Freigabe lief erneut ab, bevor der lokale 19-Commit-Stand übertragen und veröffentlicht werden konnte. + +## 1. Zuerst lesen: Wo wir tatsächlich stehen + +- Daniel möchte die **fertige, produktionsgeeignete Anwendung**, nicht weitere isolierte Sammler, Diagnosekategorien oder wiederholte Bestätigungsrunden. Er hat die fortlaufende Umsetzung mehrfach beauftragt. Probleme im Code selbst beheben, Tests und Auslieferung bündeln; ihn nur für wirklich notwendige Root-/Symcon-Ausführung oder echte Anlagenfreigaben einbeziehen. +- Datenaufnahme, bestätigter Versand, historische Aufbereitung, Modellaufbau, V4-Optimierer, Planempfang und ein ausdrücklich begrenzter Regeltest sind implementiert. Der zuletzt installierte Teil verbindet korrigierte physische Rückmeldung mit Vorschau und dem **begrenzten** Test-Stellpfad. +- Der Versandblocker durch `100.0`/`100` ist **behoben und der Nachversand hat aufgeholt**. Nicht erneut beim Cursor oder der alten Kennungsdiagnose anfangen. +- Die letzte Feedback-Installation ist **erfolgreich abgeschlossen**, nicht mehr `waiting_for_registration`. Bericht: `corrected_feedback_installed_trial_disabled`, Rückmeldung `unavailable`, Grund **`feedback_source_skew`**. +- **Nächster konkreter Entwicklungsauftrag:** bestätigte, zum tatsächlich gelesenen Wert passende Geräte-Lesezeitpunkte an den lokalen Rückmeldepfad anschliessen. Bisher wird `VariableUpdated` als Live-Zeitbasis verwendet; unveränderte Modbus-Werte werden aber nicht zwingend bei jedem erfolgreichen Abruf erneut publiziert. Geräteabruf, Variablenpublikation und aktueller Lesezugriff müssen auseinandergehalten werden. +- Keine künstliche Zeitstempelverjüngung, pauschale Toleranzerhöhung, Nullersetzung, historische Interpolation als Live-Messung oder Rückkehr zum festgehaltenen virtuellen EV-Wert. +- **Weiterhin keine produktive V4-Batterieausführung:** alter intelligenter Netzfahrplan AUS; beide lokalen Regeltestfreigaben AUS; kein gestarteter Stellversuch. Bestehende lokale Regelung und SDL laufen unabhängig weiter. +- Ein uneingeschränkter Produktions-Dauerregler ist **noch offene Umsetzung**, nicht nur ein fehlender Haken. Automatik, reale Modellqualität, Geräteausfallverhalten und Mehranlagenfreigabe haben ebenfalls verbleibende Grenzen. +- Letzter fachlicher EMS-Commit: **`9878197`**, `develop`, bei Übergabeprüfung sauber und **15 Commits vor lokalem `origin/develop`**. Kein Remote-Fetch in dieser Prüfung. Vorherige Pushversuche scheiterten an Authentifizierung. Dokumentations-Commits dieser Übergabe können danach folgen. + +## 2. Arbeitsweise und Freigaben + +Daniel ist technisch versiert, will aber keine unnötige manuelle Kleinarbeit. Deutsch, Schweizer Schreibweise, konkrete nutzbare Befehle. Ein abgeschlossener Test ist nicht automatisch Installation, Installation nicht Stellfreigabe, `optimal` nicht reale Einsparung. Immer getrennt berichten: implementiert / isoliert getestet / Zielcontainer getestet / installiert / im Kernel geprüft / am Gerät erprobt / veröffentlicht. + +**Nicht wieder in die alte Schleife zurückfallen:** + +1. Keine neuen Diagnosekategorien als Ersatz für die eigentliche Integration. +2. Keine erneuten Fragen nach bereits bestätigten EV-Kapazitäten oder pauschal immer wieder nach Wechselrichtertypen. Bestehende Quellen nutzen. Eine tatsächlich fehlende sicherheitsrelevante Eigenschaft darf trotzdem nicht erfunden werden. +3. Nicht einfach „24 Stunden warten“, solange ein Software- oder Übertragungsfehler das Training verhindert. +4. Keine alten Einzelinstaller erneut ausführen; neueren Quellstand und parallele GUI-Arbeit nicht überschreiben. +5. Nicht behaupten, im Hintergrund weiterzuarbeiten. Nur tatsächlich ausgeführte Arbeit und belegte Ergebnisse berichten. +6. Die Regel „ein einziger konkreter Punkt“ betraf Einträge in **Offene Punkte**, nicht Entwicklungsaufträge. Implementierungsaufträge dürfen zusammenhängende Anforderungen bündeln. + +**Git:** Keine Feature-Branches. Entwicklung auf `develop`; nach Tests `beta`; nach Feldtest `main`. Daniel hat im Gespräch Commit/Push für `develop` und damals auch `beta` ausdrücklich freigegeben. Diese Freigabe nicht als Auftrag interpretieren, einen aktuell ungetesteten Kandidaten blind auf `beta`/`main` zu schieben. `develop`-Push ohne erneute Nachfrage im bestehenden Rahmen erlaubt. Nicht force-pushen, nicht unzusammenhängende Änderungen übernehmen, keine Zugangsdaten verlangen/ausgeben. Historische Agenten-Commitidentität war `ENELIX Agent `; keine globale Git-Konfiguration ungeprüft ändern. + +Die aktuelle Nachricht beauftragt **Kontextübergabe**. Sie erteilt keine zusätzliche Anlagen-, Runtime- oder Stellfreigabe. Bei späterer Fortsetzung greifen die bestehenden Sicherheits- und Branchregeln weiter. + +## 3. Ursprüngliches fachliches Ziel + +### 3.1 Prognosefamilien und Erweiterbarkeit + +Im GUI manuell eine Familie wählen oder `auto`: + +| Familienschlüssel / Fahrplan | PV | Last | Fahrplanvariable | +|---|---|---|---| +| `3` | `prog_var_1` | `prog_var_2` | `prog_var_3` | +| `13` | `prog_var_10` | `prog_var_11` | `prog_var_13` | +| `23` | `prog_var_21` | `prog_var_22` | `prog_var_23` | + +Die neun Zahlen sind nicht neun unabhängige Fahrplanmodelle. Daniel akzeptierte die Auswahl der vollständigen Modellfamilie. Registry und Datenadapter so gestalten, dass weitere Familien und später weitere flexible Anlagen ergänzt werden können. Nicht behaupten, drei bereits vorhandene Lastmethoden seien identisch mit den neu aufgebauten physischen Profilen; Datenquelle und Modellversion müssen sichtbar sein. + +`auto` soll jene Familie wählen, die in der jüngeren Vergangenheit **wirtschaftlich am besten abgeschnitten hätte**, mit gleichen Anfangsbedingungen, Tarifen, SOC-/Restenergiebewertung und Randbedingungen. Keine Auswahl allein nach R², keine Zukunftsinformation im historischen Replay. Mindestabdeckung, Mindestdauer, Rückblick und Wechselmarge verhindern häufige/schlecht belegte Wechsel. + +### 3.2 Netzfahrplan / Optimierung + +- Bis zu **48 Stunden**, grundsätzlich **5-Minuten-Schritte** (höchstens 576), rollende Neuberechnung. +- Ausgangsverlauf: korrekt abgegrenzte Verbraucherlast minus PV plus separat berücksichtigte externe Flüsse. Darauf flexible Verbraucher optimieren; aktuell EV-Batterie, später Ladestationen, Boiler usw. +- Batterie: aktueller virtueller EV-SOC, dessen zugehörige EV-Gesamtkapazität, aktuelle maximale Lade-/Entladeleistung, Min-/Max-SOC, Reserve, Hysterese, Verfügbarkeit und lokale Einschränkungen. Keine zukünftige Energie vor dem Laden nutzen. +- **90 % Roundtrip-Wirkungsgrad**, nicht zweimal 90 %. Implementierung verteilt Verluste mit `sqrt(0.9)` auf Laden/Entladen. +- Netzladen ist ausdrücklich erlaubt/gewünscht, wenn freigegeben und wirtschaftlich sinnvoll. Nicht generell Nachtbezug verbieten oder tagsüber Entladung verbieten. +- Kosten pro Intervall: Bezugsenergie mal Bezugspreis minus Einspeiseenergie mal Einspeisepreis. Leistungswerte in W/kW korrekt über Zeit in kWh umrechnen. Gesamtkosten minimieren, nicht mathematisch auf exakt null zwingen; negative Kosten sind möglich, nicht garantiert. +- Statische oder dynamische, in der Anlage konfigurierte Bezugspreise und Einspeisevergütungen verwenden. GUI-Preisänderungen speichern und Neuberechnung anfordern. PV-Vergütung, Netzladekosten, Verluste und Peakpreis gehören in dieselbe Wirtschaftlichkeitsbetrachtung. +- Wenn dynamische Preise noch nicht veröffentlicht sind, den **ausführbaren wirtschaftlichen Horizont auf den bekannten Preisbereich begrenzen** (`published_only`); kein frei erfundener Preis für den Rest der 48 h. Volle Prognose und ausführbaren Preishorizont unterscheiden; Ende passend zur vollständigen Abrechnungsviertelstunde. +- Restenergie am Horizont fair bewerten/absichern, damit die Batterie nicht am Planende oder zu fast wertlosen Mittagstarifen grundlos leerverkauft wird. +- Bezug/Einspeisung sowie Laden/Entladen nicht als unphysikalische gleichzeitige Arbitrage zulassen. +- Einspeisebegrenzung, zulässige Abregelung/entgangene Vergütung und lokale harte Netzgrenzen berücksichtigen. Einen unerreichbaren Plan nicht als eingehalten darstellen. + +### 3.3 Monatspeak + +- Leistungstarif auf den relevanten **15-Minuten-Netzbezug**; persistenter Monatsmaximalwert, nicht momentane Spitzenleistung. Für den laufenden Monat zusätzliche Kosten nur für Erhöhung über den bereits erreichten Peak; Monatswechsel korrekt behandeln. +- Bereits verstrichene Energie der aktuellen Viertelstunde mitnehmen. Fehlende Messbasis nicht durch eine konfigurierte Grenze ersetzen. +- Am Monatsanfang nicht künstlich jede Aktivität verhindern: Restmonatsbewertung/empirische Aussicht ist vorgesehen, mit ausgewiesener Unsicherheit. Ein erwarteter künftiger Peak ist **keine bereits bezahlte kostenlose Freigabe**. +- Anlagen können **keine feste Bezugsgrenze** haben. Falls im Manager statische oder monatliche Peakgrenzen konfiguriert sind, diese zusätzlich respektieren. +- Tarifeinheiten und tatsächliche Abrechnungsdefinition prüfen; gemessene, geschätzte und nur konfigurierte Peakwerte getrennt kennzeichnen. + +### 3.4 Training, Reaktion und Validierung + +Tägliches oder wöchentliches Nachtraining nach GUI-Einstellung; Parameter-/Preisänderungen sollen Neuberechnung auslösen. Getrennte versionierte Historien und Modelle, keine alten belasteten Daten als bereinigt umbenennen. Ein Cold-Start-Modell nach Mindestdatenbasis ist noch keine validierte Feldprognose. Vergleich gegen die bisherige Batterie-Betriebsweise, nicht nur gegen „Anlage ohne Batterie“. + +## 4. Lihrenmoos: bestätigte Anlage und massgebliche Korrekturen + +- Installation: **`e3a08f9e-af12-4695-99bd-8b51c0520021`**. +- Symcon Manager **`17004`**, ENELIX-Batterieinstanz **`44234`**, virtuelles Asset **`anlage01-virtual-ev`**, bestehendes EV-/SDL-Gateway **`58448`**. +- **Massgeblich bestätigt: EV-Kapazität `161.44 kWh`, EV-Leistung `39 kW`.** Die vorher genannten `160 kWh`, `30 kW` und „je 10 kW pro WR“ hat Daniel ausdrücklich als Erinnerungskorrektur zurückgenommen. Daraus keine neue Verteilung auf Einzelgeräte ableiten. +- Virtueller EV-SOC und die EV-Kapazität bilden ein gemeinsames Energiekonto. Keine SOC-/Energie-Resets oder Umrechnung auf 322 kWh. **SDL-Reserve ist bereits ausgegliedert** und darf nicht nochmals von EV abgezogen werden. +- Vom Nutzer genannte physische Ausstattung: zwei GoodWe à **50 kW AC**, je **156 kWh** Batterie; SolarEdge **10 kW AC / 10 kWh** Batterie. Summe physische Kapazitäten rechnerisch 322 kWh, nicht die EV-Kapazität. AC-Nennleistung ist kein unabhängiger Nachweis jeder aktuell zulässigen Batterie-Ladeleistung. +- Dach-PV ungefähr **20 kWp Ost + 20 kWp West**. +- Exakte Gerätetypen/Firmware, jede AC-/DC-Messgrenze und autonomes Verhalten beim Befehlsausfall sind durch diese Nennwerte nicht automatisch bestätigt. Vorhandene Dokumente/Quellen verwenden, statt Angaben zu erfinden oder Daniel wiederholt pauschal zu befragen. +- Historischer Tarif-Snapshot: Bezug `CKW Dynamisch Home`, statischer Fallbackwert 0.246 CHF/kWh, Einspeisung „Preis selber eingeben“ 0.10 CHF/kWh, Peakparameter 5.0. **Nicht als aktuelle Tarifvorgabe neu setzen**, vor Verwendung aktuelle Konfiguration lesen. + +## 5. Bilanzierung und SDL: nicht wieder die alte Fehlerquelle einbauen + +Der alte Prognosesender verwendete sinngemäss `PV + Netz − virtuelle EV-Batterieleistung`. Weil nur die virtuelle EV-Batterie in der Topologie stand, konnte **SDL als Hausverbrauch gelernt** werden. Zusätzlich hält das bestehende Gateway unter bestimmten Bedingungen gefilterte virtuelle EV-/SDL-Istleistungen fest und schreibt sie erneut. Neuer Variablenzeitstempel allein machte diese Werte nicht zur frischen physikalischen Messung. + +**Neue Grundidee, bei passender elektrischer Messgrenze:** + +`Verbraucherlast = Netzbezug + PV-Erzeugung − gesamte physische Batterieladung` + +`Grundlast = Verbraucherlast − separat geplante flexible Verbraucher` + +Grundlast bedeutet hier zeitabhängiger nicht separat geplanter Verbrauch, **keine Konstante**. Nicht separat geplante Boiler/Wallboxen bleiben vorerst darin. Später beim Übergang zu eigenständiger Planung genau einmal herausrechnen. + +**SolarEdge-Sonderfall:** PV-Variable 20335 war bereits aus skaliertem AC-Ausgang plus Batterieleistung berechnet und auf null begrenzt. Neuer Lastadapter `solar_terminal_v1` verwendet den AC-Ausgang aus 37975/41853 direkt und zieht den SolarEdge-Batteriesaldo nicht nochmals ab. Herkunft, Vorzeichen, AC/DC und Null-Clipping beachten. Algebraischer Bilanzschluss allein ist kein unabhängiger Zählernachweis. + +SDL wird **nicht vom V4-Flexibilitätsoptimierer gesteuert**. Bekannten externen Fahrplan berücksichtigen; bei unbekanntem Verlauf nicht still null einsetzen. Die vorhandene Schattenplanung kann den aktuellen SDL-Auftrag als ausdrücklich gekennzeichnete Fortschreibung verwenden. Das ist kein veröffentlichter Zukunftsfahrplan. Für Dauerproduktion sind belastbare externe Grenzen/Reservebehandlung und Konfliktauflösung noch zu prüfen. Ein aktueller SDL-Wert null garantiert keine SDL-freie kommende halbe Stunde. + +Bestehendes Gateway: `/var/lib/symcon/modules/Symcon_Belevo_Energiemanagement_testing/Bat_EV_SDL_V4/module.php`. Aktionsbrücke historisch `scripts/31800.ips.php`. `State=false` war dort **kein nachgewiesener Nullstell-/Notstopp**. EV-Stopp darf nicht pauschal unabhängige SDL-Aufträge löschen. Virtuelle Energiekonten/Filter nicht ungeprüft umstellen. + +## 6. Quellzuordnung für die nächste Arbeit + +Werte/Identitäten mit aktueller Quellenkonfiguration abgleichen; Tabelle ist zuletzt geprüfte Zuordnung, keine Erlaubnis, Register umzulegen. + +| Grösse | Variable | Ursprung/Umrechnung | +|---|---:|---| +| Original-Netzleistung | 40348 | Parent 11490, `Power_8`, kW ×1000 → W, Bezug positiv | +| Netz-Anzeige | 49301 | Abgeleitet; nicht mit Originalquelle verwechseln | +| GoodWe 1 PV | 48459 | Parent 19742, `A_3_3_35301`, W | +| GoodWe 2 PV | 53802 | Parent 57658, `A_3_3_35301`, W | +| GoodWe 1 Batterie physisch | 47725 | Parent 19742, `A_6_3_35182`, Faktor −1 → Laden positiv | +| GoodWe 2 Batterie physisch | 35724 | Parent 57658, `A_6_3_35182`, Faktor −1 | +| SolarEdge Batterie physisch | 21447 | Parent 30789, `Value`, Faktor +1 | +| SolarEdge abgeleitete PV | 20335 | Parent 36915; nicht unabhängige PV-Messung | +| SolarEdge AC-Rohwert | 37975 | Parent 35514, `Value`, ReadAddress 40083 | +| SolarEdge AC-Skalierung | 41853 | Parent 48996, `Value`, ReadAddress 40084 | +| Virtuelle EV-Istleistung | 52020 | Parent 58448, `Aktuelle_Leistung_EV`; Filterhaltewert möglich | +| Virtuelle SDL-Istleistung | 25085 | Parent 58448, `Aktuelle_Leistung_SDL` | +| EV-Auftrag | 19651 | Parent 58448, `Nennleistung_Soll_EV` | +| SDL-Auftrag | 38943 | Parent 58448, `Nennleistung_Soll_SDL` | +| Gateway aktiv | 23483 | Parent 58448, `State`; kein Gerätewatchdog-Nachweis | +| EV-SOC | 32871 | Parent 58448, `SoC_EV` | +| SDL-SOC | 23879 | Parent 58448, `SDL_Pos` | +| EV verfügbare Ladeleistung | 50230 | Parent 58448, `P_EV_laden` | +| EV verfügbare Entladeleistung | 43899 | Parent 58448, `P_EV_entladen` | +| GoodWe 1 / 2 / SolarEdge SOC | 27361 / 23109 / 51938 | Originalzuordnung siehe Capture-Konfiguration | +| Wirkenergie Bezug T1 | 59607 | Parent 11490, `Energy_0`, kWh | +| Wirkenergie Bezug T2 | 26620 | Parent 11490, `Energy_1`; Zählerarchivierung aktiviert | + +**Nicht wieder 53476 als nachgewiesenen Bezugszähler verwenden:** Im Gespräch zeigte 53476 / Quelle 32797 (`Energy_6`) einen unplausibel anderen Verlauf gegenüber Netzleistung und T1. Alte Historie 53476 wurde bewusst nicht überschrieben. Archivinstanz 16207. Neuzeitlicher Peak muss aus geeigneter T1/T2-Summe bzw. ausdrücklich zugelassener vollständiger Viertelstundenschätzung stammen. + +Poller laut zuletzt gelesenem Snapshot: GoodWe 1 Parent 19742 `60000 ms`, GoodWe 2 Parent 57658 `5000 ms`; SolarEdge Batterie Parent 30789 `2000 ms`, AC Parent 35514 `1000 ms`, Skalierung Parent 48996 `20000 ms`. M-Bus Parent 11490 hatte `Interval=0`, trotzdem laufend frische Originalwerte. Nicht daraus schliessen, die Erfassung sei aus; zusätzliche bestehende Aufrufwege prüfen. Poller nie allein als erfolgreichen Geräteabruf interpretieren. + +## 7. Hosts, Code, Dienste und Berechtigungen + +### enelix-services / Connector Server_89 + +- Host `enelix-services`, IPv4 `87.106.60.89`, Agentbenutzer `agent`. Erlaubte Wurzeln `/home/agent/services`, `/srv/agent`. +- Entwicklung: **`/srv/agent/repos/Enelix-EMS`**, Gitea `ENELIX/Enelix-EMS` auf `git.belevo.ch`. Weiteres Projekt `ENELIX/Enelix-Utils`; historische Referenz `dh/Symcon_Belevo_Energiemanagement_testing`. Andere Hosts/Klone nicht ohne Nachweis gleichsetzen. +- Versionierter neuer Servercode: **`services/netplan-v4/`** im EMS-Repo. +- Runtime/Build-Staging: **`/home/agent/services/netplan-v4-shadow`**, SQLite `data/netplan-v4.sqlite`, Compose `compose.yaml`, interner V4-Port 9100. +- Bisheriger Forecast/API-Dienst: `/home/agent/services/prognosis-manager-enelix2`; Forecast Engine interner Port 9000. Nicht durch neue V4-Arbeit unbemerkt ersetzen. +- Portal: `/home/agent/services/license`, `server.mjs`, `public/`, `compose.yaml`. Parallele GUI-Arbeit möglich. Unified-RC1 aktualisierte nur V4 und zugehöriges Planner-JS, nicht pauschal alle Portalprozesse. +- Vor Veränderungen `/srv/agent/AGENT_CONTEXT.md` lesen. Agent hatte zuletzt keinen Docker-Socket-/Root-Zugriff; den Nutzer Root-Ausführung nur für tatsächlich benötigte Deployment-Schritte erledigen lassen. Keine Berechtigungen auf Docker-Socket ausweiten. + +### iot-symcon01 / Connector Testanlage_Lihrenmoos + +- IP-Symcon 8.0, Dienst `symcon.service`, Kerneldateien `/var/lib/symcon`, Logs `/var/log/symcon`, Programme `/usr/share/symcon`. +- Installiertes EMS-Repo: `/var/lib/symcon/modules/Enelix-EMS`. Das ist produktionsnah laufender Code, nicht der Entwicklungscheckout auf Server_89. +- Staging, Tests, Diagnose: `/srv/agent/netplan-v4-...`. +- Hostregeln `/srv/agent/AGENT_CONTEXT.md` lesen. Keine allgemeinen Settings-/Credential-Exports. Für gezielte schreibgeschützte Zustandsprüfung nur erlaubte Felder extrahieren, niemals ganze secret-bearing Konfiguration ausgeben. +- **PHP mit IPS-Funktionen gehört in Symcon**, nicht in die Linux-Bash und nicht in gewöhnliches CLI-PHP. Isolierte CLI-Tests mit simulierten IPS-Funktionen umgekehrt niemals im echten Kernel ausführen. +- `MC_ReloadModule`/`IPS_ApplyChanges` können bestehende Initialisierungen und deren Nebenwirkungen ausführen. Nicht pauschal als ohne Auswirkungen versprechen. Code- und Settings-Sicherung, Hash-/Driftprüfung, Rücksetzweg nötig. +- Bei verzögerter Modulregistrierung wurde ein Wiederholen angefordert. Die letzte Installation ist inzwischen fertig: **jetzt nicht wiederholen**. + +## 8. Installierter Daten-/Prognosepfad + +1. Manager erfasst 23 numerische Quellen alle 30 s, mit originalen Zeitstempeln und Qualitätsmerkmalen. Keine zusätzlichen Modbus-Abfragen durch den bisherigen Sammler. +2. Lokale append-only Tagesdateien/Outbox. Versand regulär höchstens einmal/min, bis 120 Aufnahmen pro Batch; geordneter, bestätigter Cursor. Backoff bei Fehlern; Originale bleiben erhalten. `scheduled` kann reguläre Sendepause oder Backoff bedeuten; getrennte letzte erfolgreiche Bestätigung beachten. +3. Server-Datensatz **`lihrenmoos-physical-v1`** speichert Originalaufnahmen. Abgeleiteter Datensatz **`lihrenmoos-physical-published-v2`** referenziert dieselben Daten für die verbesserte historische Zeitbehandlung. +4. `equal_endpoint_v1`: begrenzte rückblickende Schätzung bei gleichen Werten an getrennten Originalzeitpunkten. Verfügbarkeit der späteren Bestätigung wird mitgeführt; keine Leckage in damalige Entscheidungen. Unterschiedliche Werte, lange Ausfälle, widersprüchliche Quellen oder Sammellücken nicht unbemerkt überbrücken. Live-Grenzen unverändert. +5. Worker verarbeitet im 5-Minuten-Takt. Mindestbasis 24 **verwertbare äquivalente Stunden**, nicht bloss seit Start verstrichene Zeit. Neue Tagesprofile, versionierte Modelle, konfigurierbare tägliche/wöchentliche Neubildung. Kaltstart/Validierung sind getrennt. +6. Neue Datenquelle muss als `corrected_profile` mit passendem `measurementDataset` gewählt werden, sobald verwendbar; zuletzt blieb **`forecastSource=legacy`, `family=3`**. Nicht bei Installation automatisch umgeschaltet. `optimal` auf der Legacy-Quelle ist kein Nachweis der neuen Prognosequalität. +7. Wirtschaftlicher Vergleich Unified-RC1 ist angeschlossen, aber v1 ist **eingefrorener Tagesplan + simulierte lokale Netzzielnachführung**, nicht vollständiges rollendes MPC-Replay. Eine netzladefähige Batterie; gleiche Anfangsbedingungen, Restenergie, Verluste und einmalige Monatspeakabrechnung. Historischer SDL-Auftrag als gekennzeichnete Schätzung. Vollständige Modellautomatik/Produktversprechen nicht über diesen Scope hinaus behaupten. +8. Archivierung ist implementiert, **AUS als Standard** (native `NetzfahrplanV4ArchivAktiv`, serverseitig `NETPLAN_V4_ARCHIVE_ENABLED`). Verifizierte komprimierte Archive ersetzen kein externes Backup und lösen keine unendliche Speicherkapazität. Keine unbestätigten Outbox-Daten löschen. + +## 9. Behobene Fehler – nicht erneut aufrollen + +- Früherer numpy-Fehler `assignment destination is read-only`, falscher Import `battery_optimizer_new`, Dateirechte im unprivilegierten Container: damalige Deployment-/Codefehler, nicht aktuelle Blocker. +- Direkt nach Start `Connection refused`, danach HTTP 409 „bereits ein Lauf aktiv“: Start-/Parallelitätszustand, nicht durch wiederholte Neustarts lösen. +- Native Betriebsdaten fehlten; anschließend Sender installiert. Tarifimporter und Portal-Rate-Limit wurden repariert. Alte 429-Meldungen nicht ohne neue Evidenz zur heutigen Ursache erklären. +- Klasse `NetzfahrplanV4Messaufnahme` doppelt aus Staging und Modulpfad geladen: Installer auf eindeutige installierte Bibliothek umgestellt. `require_once` verhindert nicht verschiedene Dateien mit derselben Klasse. +- Gemischter 120er-Block: 45 importierte/75 native Aufnahmen, keine doppelten/rückwärtslaufenden Zeiten, aber andere Kennung allein durch `splitToleranceW:100.0` vs `100`. Normalisierung korrigiert; beide vollständigen Konfigurationen exakt geprüft. Kompatibilität nur für diese anlagen- und datensatzgebundene Gleichwertigkeit registriert; Ursprungskennung/Evidence gespeichert. Keine Hashs in Originaljournalen ersetzt, kein Cursor-Sprung. +- **Nachweis der Reparatur:** am 03.10. 12:58 Schweizer Zeit 1'782 statt 1'320 Aufnahmen, jüngste Aufnahme 12:58:09, 25 s alt, vollständiger ehemals blockierter Batch vorhanden, Manager `acknowledged`/Fehlerzahl 0. Quelle `[F]` unten. Keine Garantie über den danach liegenden Zeitraum ableiten. +- Historische Datenqualität verbesserte sich von 0 brauchbaren Fenstern auf 104/8.66 h, nach Nachversand 130/10.82 h; zuletzt Worker 140/11.65 h. Prozentangaben zur zeitlichen Abdeckung sind **keine Messgenauigkeit und keine Prognosegüte**. + +## 10. Installierter Rückmelde- und begrenzter Stellpfad + +Gemeinsamer Code: `NetzfahrplanV4Rueckmeldung.php`, `BatterieNetzfahrplanV4RueckmeldungTrait.php`, `ManagerNetzfahrplanV4EmpfangTrait.php`, `ManagerNetzfahrplanV4TestTrait.php`, `BatterieNetzfahrplanV4TestTrait.php`, `NetzfahrplanV4Regeltest.php`, Manager-/Batterie-Module. + +- Zweifaches Lesen mit Identitäts- und Originalzeitprüfung; momentan Live-Alter maximal 60 s, Quellenversatz maximal 30 s. +- Modus in Lihrenmoos **`virtual_split`**, **`allowEstimatedForTrial=false`**. Auch gültige Rückmeldung oder Vorschau erteilt keine Freigabe. +- Wenn SDL-Auftrag exakt null, physische Batteriesumme statt gefilterter virtueller EV-Anzeige verwenden; unveränderter Nullauftrag ist Zustand, kein Messgeräte-Heartbeat. Eine neue EV-Vorgabe bedeutet noch keine sofortige physische Reaktion. +- Bei SDL ungleich null: zeitliche Abdeckung von Auftragsänderung, Tracking-Toleranz und explizite Schätzkennzeichnung. Gegenläufige EV-/SDL-Flüsse sind nicht eindeutig identifizierbar und bleiben für diesen Test gesperrt. +- Netzziel wird in **Gesamt-Batterieleistung**, nicht zusätzlichen Ladeauftrag übersetzt. Erwartete Änderungen anderer Verbraucher genau einmal berücksichtigen. +- Separate interne Server-Testsitzung, leere Default-Allowlist `NETPLAN_V4_CONTROL_TRIAL_PLANTS`, manuelle Familie, Anlagen-/Asset-/Instanz-/Revisionsbindung, beide lokalen Freigaben `NetzfahrplanV4RegeltestErlaubt`, bewusster lokaler Start, gültiger Plan, Bilanz- und Gerätewatchdog-Nachweis. +- Test max. 30 min / 5 kW je Richtung; Serverauthority maximal 90 s; Batteriebefehl maximal 10 s; Batterie-Softwarewatchdog 1 s, Manager-Testtakt 2 s. Nicht einfach Limits für Produktion hochsetzen. +- Batterie liest vor Ausgabe unabhängig neu und prüft SOC, Reserve, Hysterese, Verfügbarkeit, Sperrzeiten und Netzgrenzen. Unmögliches Netzziel → Testabbruch, keine Erfolgsbehauptung. +- Abbruch widerruft Sitzung vor Nullstellversuch; Stoppfehler bleibt sichtbar. Danach maximal eine **frisch berechnete** normale Zuteilung, keine Rückkehr zu gecachtem Test-/Vor-Testwert. Keine Wiederbelebung nach Neustart oder doppeltem Befehl. +- `shadow_seen` bedeutet Empfang, **nicht ausgeführt**. `canDispatch=false` in Vorschau bedeutet keine Vorschau-Freigabe. Bei einer separat gestarteten Testsitzung wäre `runMode=shadow` alleine trotzdem kein hinreichender Nachweis für null Stellbefehle. +- Softwarewatchdog funktioniert nur bei laufendem Kernel/Kommunikation. Kein Nachweis für Rechnerabsturz, Stromausfall oder unerreichbaren Wechselrichter. Bereits vorhandenes Verhalten prüfen, nicht erfinden und nicht Schutzgates entfernen. + +## 11. Aktueller Blocker: bestätigter Geräteabruf statt falscher Zeitbasis + +Letzter Installationsbericht `[A]`: + +``` +status: corrected_feedback_installed_trial_disabled +feedback.status: unavailable +feedback.reason: feedback_source_skew +batteryW: null +gridW: null +usableForTrial: false +canDispatch: false +``` + +Das ist **kein Installationsfehler**. Die Meldung „Modellanteil: nein“ im Installer ist im Fehlerfall irreführend: fehlendes Feld wird als false dargestellt, obwohl die Berechnung gar nicht erfolgreich war. Kleine UI-/Diagnosekorrektur mitnehmen. + +Letzte Zeituntersuchung `[B]` (237 Aufnahmen / 2 h): 48 Aufnahmen mit Versatz >30 s. Netz median 1 s/max 2 s, GoodWe 1 median 1 s/max 48 s, GoodWe 2 median 1 s/max 46 s, SolarEdge median 12 s/max 59 s; SolarEdge in 223 Aufnahmen unter den ältesten Quellen. Keine der vier Quellen in genau diesem Zeitfenster über 60 s. Beispiel 13:58:38: drei Quellen neu, SolarEdge 31 s alt → Ablehnung. Frühere mehrminütige Lücken trotzdem nicht damit wegargumentieren. + +**Nächste Umsetzung, ohne neue Diagnosekategorie:** + +1. Installierten Gerätemodultyp und unterstützten Leseweg verifizieren. Im Gespräch wurde `ModBus_RequestRead()` als möglicher Ansatz genannt. API-/Erfolgs-/Zeitsemantik und Unterschied `ModBus Address` vs `ModBus Device` anhand offizieller aktueller Dokumentation und vorhandenem Code prüfen; nicht voraussetzen, dass ein boolescher Erfolg alle Register atomar aktualisiert oder einen unabhängigen Gerätezeitpunkt liefert. +2. Erfolgreiche vollständige Antwort mit genau den daraus stammenden Werten, Zuordnung, Status und Bestätigungszeit binden. `VariableUpdated`, Geräteabrufbestätigung und Sammlerzeit getrennt erhalten. +3. Begrenzte Abfragefrequenz, Timeout, Serialisierung, Kommunikationslast und Lesekohärenz berücksichtigen. Keine Netz-I/O in jedem Vorschau-/Regelcallback vervielfachen, keine blockierenden Abfragen unter dem Stellregler-Lock. Existierende erfolgreiche Polls nach Möglichkeit nutzen. +4. Wenn keine echte Bestätigung vorliegt, unsicher/gesperrt bleiben. Keine künstlichen `SetValue`-Herzschläge und keine permissive Freigabe bloss wegen des Pollerintervalls. +5. In bestehenden Batterie-/Managerpfad integrieren; Tests: unveränderte Werte mit erfolgreichem Abruf, fehlgeschlagene/partielle Antwort, Zeitversatz, Race, Wiederholung, Neustart, erneute unabhängige Batterieprüfung und Rückfall. +6. Zusammenhängenden Kandidaten bauen und testen; neue Runtime-Installation nur im bekannten gesicherten Verfahren. Bisherige Installer nicht nochmals ausführen. + +Diese Arbeit wurde am Gesprächsende **noch nicht implementiert oder installiert**. Sie ist nicht durch die vorliegende Dokumentationsübergabe erledigt. + +## 12. Produktionsreife: verbleibende echte Arbeiten + +- Verlässlicher bestätigter Live-Messpfad wie oben; unklares AC/DC/externes SDL nicht durch Labels als verifiziert deklarieren. +- Neue Lastprofile mit tatsächlicher ausreichender Datenbasis/Validierung aufbauen und gezielt zur Schattenrechnung schalten. Aktuelle Einstellungen/Modelle vorher lesen, nicht alte Momentaufnahmen fortschreiben. +- **Dauerbetrieb** mit sauberem Authority-/Zustands-/Neustart-/Fallback-Verhalten implementieren. Bisher nur begrenzter Pilot, keine fertige kontinuierliche Produktionsfreigabe. +- Reale Callback-/Registerstrecke und separat freigegebener Stellversuch; korrekte Vorzeichen, verfügbare Leistungen, Netzziel/Peak-/Exportgrenzen, Abschalten/Verbindungsausfall, lokale Priorität und SDL-Konflikte prüfen. +- Modellautomatik über den deklarierten Frozen-Day-v1-Scope hinaus nur nach Erweiterung/Nachweis bewerben; Grenzen transparent halten. +- Lebenszyklus/Backups/Recovery/Observability, Mehranlagenisolierung und parametrisierte Inbetriebnahme. Einige Installer sind bewusst auf Lihrenmoos gebunden; für andere Anlagen nicht IDs blind übernehmen. +- Git-Authentifizierung/Veröffentlichung lösen; `develop` → geprüfte `beta` → Feldabnahme `main`. Browser-Login, Tailscale oder Portforwarding stellen nicht automatisch Git-Schreibauthentifizierung bereit. Keine Veröffentlichung behaupten, die nicht passiert ist. + +## 13. Wichtige Meilensteine und nicht erneut auszuführende Installer + +| Paket | Status bis Gesprächsende | +|---|---| +| V4-Schattenserver / Native Operation / Tarife | Installiert, Betrieb historisch belegt | +| Passiver Empfänger `/srv/agent/netplan-v4-receiver-stage/install.php` | Installiert, automatische `shadow_seen`-ACKs belegt | +| Rohsammler `/srv/agent/netplan-v4-data-capture-stage` | Separater früherer Sammler; Originaldateien erhalten | +| Leseexport `/srv/agent/netplan-v4-data-review-stage` | Einmaliger Export wegen restriktiver Originalrechte; heute keine erneute allgemeine Exportpflicht | +| Getrennter Beobachter `/srv/agent/netplan-v4-separated-observer-stage` | 23 Quellen, Dateien gezielt lesbar; weiterhin Originalbestand | +| Managerdaten `/srv/agent/netplan-v4-application-build/install.php` | Installiert, Klasse-/Kennungsfehler später behoben | +| `deploy_history_timing.py` | Installiert, neue Historienversion v2 | +| `deploy_unified_release.py --approve-numeric-mapping-compatibility` + `/srv/agent/netplan-v4-unified-release/install.php` | Installiert, Aliasfreigabe exakt registriert, Nachversand aufgeholt | +| `/srv/agent/netplan-v4-feedback-stage/install.php` | **Fertig installiert nach einmaliger Registrierungsschleife**, aktuell Zeitversatzprüfung | + +Fachliche Commitfolge (lokales `develop`; nicht als Remote-/Runtime-Gleichstand missverstehen): +`b5c1fa6` Empfänger, `9394fe3` begrenzter Test, `e0e52d2` Bilanzierung, `76f8b06` Capture, `44362e4` getrennte Beobachtung, `fe3f46b` Datenanwendung, `6d9aba7` Klassenladen, `5890093` Historienzeit, `6c91d6b` Zahlennormalisierung, `8eb9688` Unified RC1, **`9878197` korrigierte Rückmelde-/Testkette**. Einige weitere Commits können dazwischenliegen; `git log` ist massgeblich. + +## 14. Quellenindex und Tests – zuerst Nachweise lesen, nicht neu installieren + +**[A] Letzte erfolgreiche Rückmeldeinstallation (Testhost):** +`/srv/agent/netplan-v4-feedback-stage/INSTALL_RESULT.json` – Ende `2026-10-03T11:59:58Z`. Der frühere `waiting_for_registration`-Bericht ist überholt. Die Sicherung stammt vom ersten Dateidurchlauf, nicht vom zweiten `already_installed`-Durchlauf. + +**[B] Letzte Zeit-/Pipeline-Auswertung (Server):** +`/home/agent/services/qa/feedback-timing-20261003/TIMING_REVIEW.json` – `2026-10-03T12:02:13Z`, 140 brauchbare Fenster/11.646 h, Quelle v2 noch `collecting`. + +**[C] Unified-Serverinstallation:** +`/home/agent/services/netplan-v4-shadow/unified-releases/20261003T105407.036202Z/REPORT.json`; Runtime `unified-rc1`, Mappingkompatibilität registriert, Einstellungen unverändert. + +**[D] Unified-Managerinstallation (Testhost):** +`/srv/agent/netplan-v4-unified-release/INSTALL_RESULT.json`. + +**[E] Historieninstallation:** +`/home/agent/services/netplan-v4-shadow/history-timing-releases/20261003T092245.175100Z/REPORT.json`. + +**[F] Erfolgreiches Aufholen des Datenwegs:** +`/home/agent/services/qa/unified-release-20261003/FLOW-20261003T105834.324945Z.json`. + +**[G] Installiertes und versioniertes Verhalten:** +`docs/netplan-v4-corrected-feedback.md`, `docs/netplan-v4-unified-release.md`, `docs/netplan-v4-history-timing.md`, `docs/netplan-v4-numeric-mapping-fix.md`, `docs/netplan-v4-application.md`, `docs/netplan-v4-controlled-trial.md`, `docs/netplan-v4-accounting.md`. Alte „noch nicht installiert“-Abschnitte dieser Entwicklungsdokumente durch aktuelle Installationsberichte einordnen; nicht mit echten Runtime-Nachweisen verwechseln. + +**[H] Rückmelde-/Testsuite (Testhost):** +`/srv/agent/netplan-v4-feedback-stage/PREPARATION.json`, `TEST_RESULTS.txt`, `INSTALLER_TEST_RESULTS.json`, `ORDINARY_OFFER_TEST.json`, `MANIFEST.json`, `PACKAGE_HASHES.json`, `feedback-config.json`. +125 isolierte Funktionsprüfungen, 4 vollständige Modul-Linkageprüfungen, 10 Installationsszenarien, 13'824 übereinstimmende normale Leistungsangebotsfälle. Keine Aussage, dass dieselbe Zahl Tests im echten Kernel/Wechselrichter gelaufen sei. + +**[I] Unified-Suite (Server):** +`/home/agent/services/qa/unified-release-20261003/PYTHON_TEST_RESULTS.txt`, `NODE_TEST_RESULTS.txt`, `PUBLICATION.json`, `DELIVERY_STATUS.json`. +306 Python- und 16 Portaltests; native separate Tests. Spätere Änderungen brauchen erneute relevante Tests, nicht bloss Übernahme dieser Zahlen. + +**[J] Wiederaufbau und Code:** +EMS `examples/V4CorrectedFeedback/`, `examples/V4UnifiedRelease/`, `services/netplan-v4/commissioning/`. Staging ist nicht dieselbe Quelle wie das laufende Modul. Dateien, Hashs und Abhängigkeiten vor Änderungen vergleichen; Paralleländerungen abbrechen statt überschreiben. + +**[K] Quellen-/Konfigurationsdiagnose (Testhost):** +`/srv/agent/netplan-v4-application-build/inspect_source_update_policy.py`, `inspect_delivery_state.py`, `MAPPING_IDENTITY_DIAGNOSIS.json`, `MAPPING_EQUIVALENCE.json`; `netplan-v4-feedback-stage/review_current_feedback_state.py`. Diese Skripte zuerst lesen; manche erzeugen nur einen Bericht, manche lesen gespeicherte statt live aktuelle Settings. Alte Settings-Snapshots können eine bereits installierte Eigenschaft noch nicht enthalten. + +**Sicherheitsnotiz:** In einem alten Upload-Skript wurden Klartextzugangsdaten bemerkt. Nicht lesen, verbreiten, in Dokumentation/Git übernehmen oder für neue Integration verwenden. Separate autorisierte Rotation/Secret-Auslagerung bleibt Aufgabe; die Werte gehören nicht in diese Übergabe. + +## 15. Übernahmeroutine für einen neuen Agenten + +1. Host-`AGENT_CONTEXT.md`, Repo-`AGENTS.md` und diese Datei vollständig lesen; bei fehlendem Remote-Stand zuerst den Servercheckout prüfen. +2. `git status`/Branch/letzte Commits sowie `[A]`, `[B]`, `[C]` lesen. Zeitstempel nennen; keine alten Zähler als heutige Werte ausgeben. Diese Übernahme selbst ändert keine Anlage. +3. Nur notwendige gezielte Read-only-Prüfungen. Kein weiterer vollständiger Neustart der Analyse oder Wiederholung bereits erledigter Installer. +4. Bei Fortsetzungsauftrag direkt Abschnitt 11 umsetzen und anschliessend echte Restarbeiten aus Abschnitt 12 abarbeiten. Anforderungen, Testfall, Implementierung und Rollout in einem Paket halten. +5. Nach jedem Abschluss diese Übergabe mit Datum, Codecommit, tatsächlichem Installations-/Teststatus und nächstem konkreten Arbeitsschritt ergänzen. Keine gescheiterten Schritte in „fertig“ umdeuten. Bei Widerspruch aktuelle geprüfte Laufzeit und ausdrückliche spätere Nutzerkorrektur nennen. + +**Kurzübernahme:** „Netzfahrplan V4 in ENELIX fortsetzen. Datenkennung/Versand repariert, Unified RC1 und korrigierter Feedback-/Testpfad installiert. Aktuell `feedback_source_skew`; bestätigte Geräte-Lesezeitpunkte in die bestehende lokale Rückmeldung integrieren. EV 161.44 kWh/39 kW. Keine neue Diagnosekategorie, keine alten Installer, kein automatischer Stelltest. Dauerproduktion ist noch nicht fertig; Tests/Abnahme/Veröffentlichung sauber getrennt halten.“ diff --git a/docs/netplan-v4-accounting.md b/docs/netplan-v4-accounting.md new file mode 100644 index 0000000..f921543 --- /dev/null +++ b/docs/netplan-v4-accounting.md @@ -0,0 +1,99 @@ +# V4: getrennte Last-/Batterie-/SDL-Bilanz (Kandidat, keine Stellfreigabe) + +## Status 2026-10-02 + +`NetzfahrplanV4Bilanzierung` ist ein reiner Rechen-/Validierungsbaustein. Noch keine +Einbindung in die laufende Telemetrie, den Trainer oder den Kontrolltest. Er ruft +keine IPS-, Netzwerk-, Archiv- oder Stellfunktionen auf. Ergebnis ist immer +`controlEligible=false`, auch bei geschlossener mathematischer Bilanz. + +34 isolierte PHP-Pruefungen wurden mit der gleichen SHA256-Datei auf iot-symcon01 +bestanden. Das ist kein Nachweis fuer die reale Hardware-/Messgrenzenzuordnung +und kein vollstaendiger PHPUnit- oder Symcon-Kernel-Test. + +## Gefundener Unterschied in Lihrenmoos + +Installierter Manager `Manager/module.php::leseMessleistungen()` verwendet: + + Haus = PV + Netz - Summe der konfigurierten Batterie-Istleistungen + +Konfiguriert ist hier die virtuelle EV-Leistung 52020, nicht die gesamte physische +Batterieleistung. Daher bleibt der externe SDL-Anteil und eine allfaellige +Abweichung der virtuellen Aufteilung im an den Prognosedienst gesendeten Hauswert. +`sendePrognoseTelemetrie()` sendet genau diesen errechneten Wert. Die parallel +existierenden Hausvariablen 46714 und 40713 sind NICHT automatisch diese Quelle. + +Der bisherige Gateway `Bat_EV_SDL_V4` liest die physische Batterieleistung aus +47725, 35724 und 36380 und invertiert diese Vorzeichen. EV und SDL sind daraus +abgeleitete, separat gefilterte virtuelle Konten, keine zwei unabhaengigen Zaehler. +Die Filter koennen alte Teilwerte halten. `UpdateActualPowerSplit` aktualisiert die +Ausgabevariablen trotzdem. Ein neuer Ausgabezeitstempel beweist deshalb nicht die +Frische aller darunter liegenden Originalmessungen. + +## Kandidat fuer die korrekte Bilanz + +Mit einheitlicher Messgrenze und Vorzeichen Batterie+ = Laden, Netz+ = Bezug: + + Verbraucherlast = Netz + PV - gesamte physische Batterieleistung + Grundlast = Verbraucherlast - separat geplante flexible Verbraucher + Zuordnungsrest = physische Batterie - virtuell steuerbare Batterie - externe virtuelle Konten + Externer Netzeffekt = externe virtuelle Konten + Zuordnungsrest + Netz = Grundlast + flexible Last + steuerbare Batterie + externer Netzeffekt - PV + +Der Zuordnungsrest wird nicht erneut als Hausverbrauch trainiert, sondern separat +angezeigt. Eine grosse Abweichung sperrt die Datenqualitaet. Gegenlaeufige virtuelle +Konten (z.B. EV +42 kW, SDL -40 kW, physisch +2 kW) bleiben korrekt getrennt. +Spaetere Ladestationen/Boiler sind ueber `flexible_load` erweiterbar. Solange sie +nicht separat geplant werden, verbleiben sie in der nicht separat steuerbaren Last. + +Keine Messgroesse wird doppelt subtrahiert. Ein negatives Grundlastergebnis wird +nicht heimlich auf null begrenzt. Numerische Nullen sind echte Werte; fehlende, +nicht endliche oder umtypisierte Werte sind keine Nullen. + +## Originalzeit und Datenqualitaet + +Jede Quelle hat explizite Variable/Parent/Ident, Faktor, maximales Alter und optionale +Ursprungsabhaengigkeiten. Alle Quellen werden zweimal gelesen. Wertewechsel waehrend +der Aufnahme werden abgewiesen; Zeitstempel werden nie kuenstlich verjuengt. +Abhaengigkeiten sind zyklenfrei, doppelte Quellen werden verhindert. +Eine virtuelle Summenvariable erbt das aelteste Datum ihrer physischen Eingangsquellen. +Konfigurierbare Alters-/Synchronitaetspruefung ist erforderlich. Diagnoseberechnungen +mit ungeeigneten Daten bleiben `quality_hold`, nicht `base_load`-Trainingsfreigabe. +Die momentanen Beispielgrenzen (60 s Alter, 30 s Versatz) sind konservative +Diagnoseparameter, keine garantierten Geraeteeigenschaften. + +Die konkrete Solar-PV-Herkunft sowie AC/DC-/Wirkungsgradgrenzen aller Hybridgeraete +sind noch zu verifizieren. Eine algebraisch geschlossene Gleichung beweist diese +physische Messgrenze NICHT. Die vorgeschlagene Quellkonfiguration ist nicht freigegeben. + +## Migration in die Prognose (noch offen) + +1. Originalmessquellen und Messgrenze dokumentieren und Qualitaet ueberwachen. +2. Korrigierte Grundlast und externe Kanaele als NEUE versionierte Zeitreihen erfassen. +3. Bestehende Historien nicht ueberschreiben oder als bereinigte Historie umbenennen. + Rekonstruktion nur bei ausreichenden zeitlich passenden Originaldaten. +4. Prognosemodell auf der korrigierten Basis trainieren/validieren. +5. Zukunfts-SDL separat liefern. Unbekannt ist nicht null; eine geschaetzte Annahme + muss explizit bezeichnet werden und darf nicht als veroeffentlichter SDL-Fahrplan gelten. +6. Erst nach diesen Pruefungen eine echte Bilanzierungsreferenz im Trial-Gate verwenden. + `mappingId` aus dieser Klasse ist ein Konfigurationsfingerabdruck, KEIN solcher Nachweis. + +## Gateway-Ausfallverhalten (nicht getestet/freigegeben) + +Quelltextpruefung des installierten Alt-Gateways ergab keinen Frischetest fuer +`Nennleistung_Soll_EV` im Weg `Update -> ApplySetpoints`. Der zwischengespeicherte +EV-Sollwert wird weiter verwendet, solange die Gateway-Schleife laeuft. Der Eingang +wird von `scripts/31800.ips.php` geschrieben; dieser enthaelt ebenfalls keine Lease. +Der vorhandene 30-s-Timeout des EMS-Batterieadapters ist ein Schutz innerhalb von +Symcon, nicht ein unabhaengiger Wechselrichter-Timeout bei Rechnerausfall. + +`State=false` im Alt-Gateway beendet `Update` ohne dort Nullwerte zu schreiben. +Diesen Schalter nicht als nachgewiesenen Hardware-Notstopp verwenden. +Keine Kommunikationsunterbrechung, kein Abschalttest und keine neue Stellwertausgabe +wurden an der realen Anlage durchgefuehrt. Auch aus dem fehlenden Code kann NICHT +gefolgert werden, dass das Geraet selbst keinen Timeout hat. Exakte Typen/Firmware +und das wirksame geraeteseitige Verhalten muessen noch festgestellt werden. + +Wichtig fuer spaetere Anpassung: Ein EV-Befehlsablauf darf nicht pauschal den +externen SDL-Auftrag loeschen. Ein Kernel-interner Timer ersetzt wiederum keinen +Schutz bei komplettem Kernel-/Rechner-/Kommunikationsausfall. diff --git a/docs/netplan-v4-application.md b/docs/netplan-v4-application.md new file mode 100644 index 0000000..452f504 --- /dev/null +++ b/docs/netplan-v4-application.md @@ -0,0 +1,131 @@ +# V4 application data and forecast integration + +## Implemented application path + +`ManagerNetzfahrplanV4DatenTrait` records the configured raw power/SOC/counter sources +inside the existing Manager. This replaces the need for permanent standalone +observation categories once the native delivery path has been accepted. It never +issues device commands. New acquisition properties default disabled. + +The private outbox at `data/enelix-v4/` is append-only and delivery +is acknowledged by dataset and capture time. Unacknowledged data survives network +failures and 429 responses. The native payload budget is below the existing portal +1 MiB request-body limit. Existing authentication and V4 rate limiting are reused. + +Server application source is now versioned under `services/netplan-v4`, rather than +existing only as an untracked server working tree. No live measurements, databases, +credentials, dependency binaries or settings exports are included in this directory. + +The existing worker handles per-plant dataset ingestion, physical five-minute +rollup, model training, model storage and corrected-load substitution into the V4 +optimizer. It continues to consume the existing PV forecasts and price/operation +inputs. The public device route can append measurements but cannot alter source +meaning, register a dataset, change live permissions or arm a controlled trial. + +## Explicit settings and model meaning + +- `forecastSource=legacy` preserves the existing forecast input by default. +- `forecastSource=corrected_profile` plus `measurementDataset` uses the trained + corrected physical load. Missing models are reported; legacy house values are + not substituted silently. +- `trainingCadence=daily|weekly` is now connected to real profile fitting in the + application worker, independently from the optimizer refresh frequency. +- Corrected load profiles are tagged `physical-profile-v1`: robust daily profile, + recent-day profile and weekday/weekend profile paired with PV families 3/13/23. + They are not claimed to be the previous load models unchanged. Economic automatic + selection still requires the separate, not yet finished cost-replay integration. +- Initial models are explicitly bootstrap models. Later candidates record causal + holdout errors; an overlapping validation window cannot certify a promotion. +- Current SDL request persistence is only a labelled shadow scenario. Unknown or + stale SDL is not silently zero, and the scenario is not a guaranteed future + schedule or a robust SDL-aware production control policy. + +## Lihrenmoos package scope + +Effective EV capacity 161.44 kWh and power 39 kW remain unchanged; SDL reserve is +already excluded. The prepared dataset `lihrenmoos-physical-v1` uses a configured +physical estimate with SolarEdge signed terminal power counted once. It requires +at least 24 equivalent usable hours before the first bootstrap profile; the +history window is 28 days. Coverage policy 95% with maximum 10-second unsupported +portion is an explicit modelling assumption. Gaps and source ages remain visible. +It is not an independent electrical metering-boundary proof. + +Server entry point on the service host: + + python3 /home/agent/services/netplan-v4-shadow/commissioning/deploy_application.py \ + --plant e3a08f9e-af12-4695-99bd-8b51c0520021 --apply + +Default without --apply checks source only. The explicit apply command rebuilds +and tests V4 in Python 3.11, tests portal routing, backs up the V4 database and +recreates only V4 and the portal. It configures the dataset but DOES NOT select it +for the existing plan automatically. No forecasts/tariff importer or controllers +are restarted by the server command. A previous-image rollback is prepared. + +Then, inside Lihrenmoos Symcon: + + require '/srv/agent/netplan-v4-application-build/install.php'; + +This is a data-only patch of the currently installed passive manager, with file +backup and library reload. It does NOT install the development control-trial +changes in Manager/Batterie. The first runtime invocation imports at most 48h of +the existing 23-channel observer as a durable backlog, preserving the original +files, then activates acquisition and delivery. If Symcon module registration is +not immediately available, the installer reports that a single repeat is needed. + +The two temporary standalone samplers are not stopped automatically by this +initial package; disable them only after native batch acceptance and history +continuity are confirmed. This package creates no new root diagnostic category. + +## Validation performed before deployment + +219 Python tests passed in the isolated host QA runtime, including actual synthetic +measurement -> profile -> existing optimizer -> stored shadow plan. 16 Node proxy +checks passed. 21 PHP data-path checks and 5 installer scenarios passed with mocked +IPS/HTTP and temporary files on the test host. Full staged PHP syntax passed. +No new container or new manager data integration has yet been run in production. + +Evidence: service `APPLICATION_PIPELINE_TEST_RESULTS.txt`, +`APPLICATION_PORTAL_TEST_RESULTS.txt`, `commissioning/application-source/RELEASE.json`; +test host `netplan-v4-application-build/PREPARATION.json` and `TEST_RESULTS.txt`. + +## Remaining product scope (do not disguise as complete) + +This integrates application data and load forecasting, not completed production +commissioning. Economic replay-based automatic family selection, full corrected +feedback/control integration, long-term outbox/server retention and real live/ +failure acceptance remain open. Existing trial gates are unchanged. No sensor, +accounting or hardware evidence identifier is fabricated by data ingestion. + +## Installer class-path correction (2026-10-03) + +The original commissioning installer loaded `source/libs/NetzfahrplanV4Messaufnahme.php` +while importing observation history. A later Manager callback loads the installed +copy at `modules/Enelix-EMS/libs/NetzfahrplanV4Messaufnahme.php`. PHP's once-only file +inclusion does not deduplicate class declarations across those different files. +The dependent `NetzfahrplanV4Bilanzierung` class is subject to the same collision. + +The tested installer-only fix is preserved in +`examples/V4ApplicationData/application-class-loading.patch` (SHA256 +`6d53d74d4059bb1ae3a446350637a4f4a4a73cc2fd64d2fc2f0967bd23ec7173`). +When building a commissioning package, load both classes only from the installed +module tree, after validating their expected source hashes. Before any require, +check both already-loaded origins without autoload. Reject a foreign staged origin +with a controlled error; do not hide it by wrapping the class declaration in a guard. +Include `installed_capture_bootstrap.php` in the package checksum map. Do not change +the installed-module manifest or bless unrelated source changes. + +The Lihrenmoos staging entrypoint has this fix applied, with backups of its original +installer and checksum map. No running modules, data files, timers, native callbacks, +server components or actuator settings were changed by the repair. Actual completion +still requires the user's fresh Symcon invocation of the corrected installer. + +Validation: the original two-path failure was reproduced with the real PHP classes. +Thirteen additional isolated installer scenarios passed, including callback/reload +loading order, preloaded dependencies, history import, repeat/finished import, +foreign-class refusal, partial-import preservation, source drift, registration wait +and first installation. These are PHP CLI tests with mocked IPS in temporary trees, +not a completed test in the actual Symcon kernel. + +Evidence on the test host: +`/srv/agent/netplan-v4-application-build/class-loading-fix/TEST_RESULTS.json` and +`FIX_RESULT.json`. The test harness and exact candidate source are retained there. diff --git a/docs/netplan-v4-controlled-trial.md b/docs/netplan-v4-controlled-trial.md new file mode 100644 index 0000000..b9b0936 --- /dev/null +++ b/docs/netplan-v4-controlled-trial.md @@ -0,0 +1,96 @@ +# V4: begrenzter Regeltest (Entwicklungskandidat, NICHT in Betrieb) + +Stand 2026-10-02. Diese Erweiterung ist weder Produktionsfreigabe noch ein +Installations-/Startauftrag. Bestehende Schattenplaene und ihre `shadow_seen` +Bestaetigungen bleiben unveraendert. Ein Vorschauwert allein darf niemals +als Stellfreigabe interpretiert werden. + +## Freigabekette + +1. Serverseitige separate Allowlist `NETPLAN_V4_CONTROL_TRIAL_PLANTS` (standardmaessig + leer). Interner authentifizierter `planner/trial/arm`-Aufruf mit expliziter + Bestaetigung, neuer Sitzung, gepruefter Plan-ID/Revision, Batteriezuordnung, + Zeitfenster, Leistungsgrenzen und dokumentierter Geraete-Watchdog-Abnahme. + Die bestehenden oeffentlichen Portal-/Geraeterouten proxien diesen Aufruf NICHT. +2. `NetzfahrplanV4RegeltestErlaubt` lokal im Manager UND im Batteriemodul (jeweils + Standard false). Alte `NetzfahrplanAktiv`-Freigabe muss false bleiben. +3. Zusaetzlicher bewusster lokaler Start einer bestimmten Sitzung. Ein Reload, + Netzwiederkehr oder gespeicherter Haken startet keine Sitzung automatisch. +4. Der Manager prueft den empfangenen Schattenplan und die getrennte + `controlled_trial_authority` gemeinsam. Sie sind gebunden an Anlage, Batterie, + beide Instanz-IDs, Revision, manuelle Modellfamilie und stabilen Planungskontext. +5. Die Batterie nimmt nur einen separaten `controlled_trial_command` mit eigener + Sequenz und kurzer Gueltigkeit an. Normale Managerbefehle erneuern diese + Gueltigkeit nicht. `shadow_seen` bleibt eine Empfangs-, keine Ausfuehrungsmeldung. + +Die Serverantwort bleibt eine Schattenplan-Antwort (`liveEnabled:false`). Eine +getrennte Testfreigabe ist unter `controlledTrial` und +`controlledTrialAuthorized` sichtbar. Der Schattenmodus allein ist bei kuenftig +bewusst gestartetem Test deshalb kein Nachweis fuer 'keine Batteriebewegung'. +Die Controller-Diagnosen benennen eine angenommene Sollvorgabe, nicht bereits +physisch gemessene Umsetzung oder Ersparnis. + +## Bewusst begrenzter Pilotumfang + +- Genau eine eindeutig zugeordnete Batterie je Test; Familie manuell festgelegt. +- 30 bis 1800 Sekunden Sitzung, maximal 5000 W Laden und 5000 W Entladen. +- Server-, Manager- und Batteriegrenze koennen strenger sein; nie grosszuegig + ersetzen oder still auf 39 kW hochsetzen. +- Gesamtsollleistung, NICHT Zusatzleistung zur bereits fliessenden Batterieleistung. +- Andere Verbraucher bleiben im alten Verteiler. Ihre erwartete Leistungsaenderung + wird einmal bei der Batteriekorrektur beruecksichtigt. +- Batterie wird nur einem Stellwertpfad zugeteilt, nicht gleichzeitig dem alten + und neuen Verteiler. Ist der Test ungueltig, stoppt er und es wird eine NEUE + normale Verteilung angefordert; kein alter gespeicherter Sollwert wird restauriert. +- Physische Grenzverletzung, die innerhalb der Pilotleistung nicht behebbar ist, + beendet den Test zugunsten der bestehenden lokalen Regelung. + +## Ausfall- und Zeitverhalten + +- Serverfreigabe maximal 90 Sekunden, lokal regelmaessig frisch abgerufen. + Serverseitiger Widerruf wirkt nicht magisch sofort: Cachefrist beachten. +- Einzelner Batteriebefehl maximal 10 Sekunden; Batterie kontrolliert im laufenden + Kernel mit 1-Sekunden-Timer zusaetzlich zu Messwert-Callbacks. +- Manager liefert kurze Befehle im Test mit zusaetzlichem 2-Sekunden-Takt. +- Monotone Laufzeit und Kalenderzeit werden getrennt geprueft. Eine rueckwaerts + springende Systemuhr darf eine Sitzung nicht verlaengern. +- Abbruch widerruft die Sitzung VOR dem Nullstellversuch. Schreibfehler werden als + `stop_failed`/nicht bestaetigter Stopp sichtbar, nicht als sicherer Zustand. +- Reserve, Hysterese, aktuelle Lade-/Entladeleistung, SOC und Verfuegbarkeit werden + auch im Batteriemodul erneut geprueft. Sicherheitsreduktionen werden nicht durch + die Umschaltsperre verzoegert; Richtungswechsel laufen ueber null. +- Laufzeitsitzungen liegen nur im Buffer, nicht als automatische Wiederanlauf-Freigabe. + Ein persistenter Marker sorgt bei Wiederinitialisierung fuer einen erneuten + Nullstellversuch, falls ein Testwert zuvor ausgegeben worden sein koennte. + +WICHTIGE GRENZE: PHP-Timer und Software-Nullstellversuche funktionieren nur bei +laufendem Kernel und erreichbarem Wechselrichter. Sie sind KEIN physischer +Not-Aus und kein Beweis fuer Abschaltung bei Serverabsturz, Stromausfall oder +unterbrochener Stellwertverbindung. Dafuer ist ein separat nachgewiesener +Befehlsausfall-/Watchdogmechanismus am Geraet bzw. Gateway erforderlich. +Das Feld `NetzfahrplanV4GeraeteWatchdogNachweis` ist eine Referenz auf die gepruefte +Dokumentation/Abnahme, keine automatische Hardwarepruefung und kein Defaultwert. + +## Noch offene Freigabepunkte (nicht mit Platzhaltern umgehen) + +- Der laufende Lihrenmoos-Prognosestrom ist noch `house_total`. Fuer Testfreigabe + muss `base_load` mit nachgewiesener Quelle/SDL-Abgrenzung und + `accountingEvidenceId` aus dem geprueften Adapter vorliegen. Ein Labelwechsel + oder ein erfundener Nachweis reicht nicht; Quelle und Messbilanz pruefen. +- Geraete-/Gateway-Watchdog bei ausbleibenden Stellbefehlen nachweisen. +- Vollstaendiges Manager-/Batterie-Callbackverhalten im IP-Symcon-Kernel pruefen. + Die isolierten Tests verwenden echte neue Traits, aber simulierte IPS-Aufrufe, + Messwerte, Zeit und Register. Reale Modbus-/VGT-Stellreaktion steht aus. +- Zielcontainerpruefung des neuesten Serverstands und Feldabnahme. +- Keine durchgehende unbeaufsichtigte Produktion, kein automatischer Mehranlagenstart. + Modellvergleich, Trainingsanbindung, Mehranlagenabnahme und Betriebs-Release + bleiben weitere Aufgaben. + +## Quelltests + +`php tests/V4ControlTrial/checks.php` nur im separaten CLI-Testprozess ausfuehren, +niemals im laufenden Symcon-Skripteditor. Die Datei verweigert einen realen +Symcon-Kernel. Eine PHPUnit-Huelle bindet sie an die normale Testsuite an. + +Der Server-Test verwendet ausschliesslich synthetische Daten/Temporaerdatenbanken. +Keine Testfunktion liest Zugangsdaten oder schreibt eine reale Stellvariable. diff --git a/docs/netplan-v4-corrected-feedback.md b/docs/netplan-v4-corrected-feedback.md new file mode 100644 index 0000000..527d837 --- /dev/null +++ b/docs/netplan-v4-corrected-feedback.md @@ -0,0 +1,108 @@ +# V4 corrected physical feedback -> local bounded control + +## Scope of this delivery + +The shared physical feedback reader is now connected to the actual Manager preview, +the existing explicit bounded-trial command path, and an independent reread in the +Battery module immediately before hardware output. This is not a new observer. +The installer leaves both local trial permissions disabled and cannot arm a server +trial. Normal operation continues through the existing local controller. + +Lihrenmoos binding: Manager 17004, Battery 44234, virtual asset +`anlage01-virtual-ev`. Effective 161.44 kWh / 39 kW unchanged. Source definitions +come from the existing versioned capture config and current saved topology, not +from new guessed device models. No old histories, sender cursors, virtual energy +accounts, SDL requests, archive policies or inverter polling are changed. + +## Feedback and meaning + +`NetzfahrplanV4Rueckmeldung` checks original value timestamps and object identities +in two read passes. Live max age is bounded to 60s and inter-source skew to 30s. +History interpolation/publication estimates are not accepted as live feedback. +The mapping fingerprint normalizes numbers and ordering, so a JSON property round +trip does not create a new identity. A mapping change cancels an active session. + +Two adapters exist: sum of explicitly assigned physical meters, and a virtual +EV/SDL partition model. For the latter, gateway state and current requests are +read but unchanged zero commands are not mistaken for failed sensor heartbeats. +When SDL request is exactly zero, current physical battery power is used rather +than the filtered virtual EV display. A new EV request does not invent an immediate +physical response or force the measured sign toward the desired sign. + +When SDL is nonzero, source timestamps must cover the latest request change and +physical tracking error must remain within the configured small tolerance. The +partition is explicitly estimated, not separately measured. Opposing EV/SDL flows +cannot be uniquely identified; they remain ineligible for trial control. Unknown +or stale physical sources never fall back to held filtered EV values. + +`allowEstimatedForTrial` is false in the prepared Lihrenmoos config. Configuring +feedback or seeing a valid preview therefore does NOT accept the estimate for a +trial and does NOT satisfy accounting/device-watchdog evidence or other gates. +This adapter does not claim to resolve arbitrary simultaneous SDL activity. + +## Actual control integration + +The same coherent grid/battery snapshot is used in the Manager. Battery command +is a TOTAL power, not a delta added to the existing command. Planned changes in +other consumers are counted once and remain explicitly separate from measured grid. + +A separate explicit server authority, both local consents, accounting evidence, +device-watchdog evidence, a deliberate session start and fresh plan remain required. +The existing trial is limited to 1800s / 5000W per direction; individual command +leases remain <=10s and cannot be renewed by replay or ordinary manager messages. +The local feedback mapping fingerprint and grid caps are bound to the session. + +Immediately before writing, Battery rereads physical feedback and applies current +SOC, reserve, hysteresis, availability and change-lock constraints. Its resulting +command must still meet the bound grid limits; an unreachable target revokes the +trial rather than claiming that the planned grid value was achieved. It cannot +claim physical performance merely because a register call returned successfully. + +Abort revokes the lease and attempts zero, and the Manager wrapper then runs at +most one fresh ordinary allocation, clearing cached pre-trial targets. It does +not recurse forever or revive an old plan. A failed stop stays explicitly failed. +The watchdog still requires a functioning kernel. Independent hardware failure +behavior is NOT certified by software tests. + +Outside a V4 session the existing battery power path is unchanged. The compatible +BatterieRegler library includes the already developed optional reserve charging +limit; with its default (maximal charge) 13,824 old/new ordinary-offer test cases +match exactly. That comparison is not an on-device dynamics test. + +## Installation + +A single prepared Symcon script is the next runtime action: + + require '/srv/agent/netplan-v4-feedback-stage/install.php'; + +It hash-checks nine changed files and all version-matched dependencies, backs up +existing source, installs dependencies before modules and reloads the ENELIX +library. Reload/ApplyChanges may execute existing initialization routines; this +is not promised to be a zero-effect library reload. It configures only the new +feedback mapping on Battery. No server redeployment is required. No shared classes +are included from staging, avoiding the earlier duplicate-class problem. + +A delayed module-registration result requests one repeat. A valid installation +can still report an unavailable feedback sample; that is not reinterpreted as zero. +Any preexisting trial permission, unsaved change, or concurrent source change stops +the installer. Source write failure rolls back its own changed files; reload errors +are reported for review and do not automatically authorize anything. + +## Validation and remaining acceptance + +125 isolated PHP functional checks cover the reader, real receiver, actual trial +traits with a simulated register driver, the extracted actual Manager fallback +wrapper, and the real Battery message builder with optional diagnostic variables absent. +Source quality is stored internally; a last partition estimate is not relabelled as a +measured value just because a trial ends or a diagnostic variable is hidden. Four full module linkage checks and ten isolated installer scenarios also +pass. Full PHP syntax and dependency hashes are checked. Neither these fixtures nor +the ordinary-offer comparison run the real Symcon kernel or an inverter. + +Evidence: test host `/srv/agent/netplan-v4-feedback-stage/PREPARATION.json`, +`TEST_RESULTS.txt`, `INSTALLER_TEST_RESULTS.json`, `ORDINARY_OFFER_TEST.json`. + +Runtime installation, real feedback reception, and a separately authorized field +trial remain outstanding. This finishes the code connection for bounded control; +it is NOT an unrestricted continuous-production controller or a hardware +commissioning certificate. Corrected model selection and actual real-world savings +must be checked against their own data. Existing safety gates are not bypassed. diff --git a/docs/netplan-v4-data-capture.md b/docs/netplan-v4-data-capture.md new file mode 100644 index 0000000..a2237be --- /dev/null +++ b/docs/netplan-v4-data-capture.md @@ -0,0 +1,67 @@ +# Passive V4 data capture - user inventory and raw acquisition + +Status 2026-10-02: source implementation, CLI/mocked tests and staged user installation. +Not part of the live manager, not a control or training release. + +## Lihrenmoos user-reported inventory (not applied to running configuration) + +- GoodWe 1/2: each 50 kW AC, each 156 kWh reported physical battery capacity. +- SolarEdge: 10 kW AC, 10 kWh reported physical battery capacity. +- EV allocation: 30 kW / 160 kWh total, understood as 10 kW EV per inverter. +- PV roof: approximately 20 kWp east and 20 kWp west. Tilt/individual strings unknown. +- Nominal battery sum: 322 kWh. 162 kWh nominal difference to EV allocation is NOT a + verified usable SDL capacity. AC ratings are not verified battery charge/discharge ratings. +- Exact types, firmware, common AC meter boundary and device watchdog remain unverified. +- Current saved manager topology still reports 161.44 kWh / 39 kW. Gateway currently + lists 130/130/8 kWh and 57/57/5 kW. These may use a different usable/nominal basis. + Do not rebase virtual SOC or change current controls based on these notes alone. + +## Why collect before boundary approval + +Record the actual original numerical channels now, with source metadata and timestamps. +The algebraic base-load candidate is useful for diagnosis, but NEVER a certified base-load +training sample until the physical AC/DC boundary and historical recipe are verified. +No dependence on hardware-watchdog approval for passive recording. + +21 channels: PCC power, legacy displayed PCC power, PV for three inverters, three physical +battery powers, EV/SDL virtual powers and SOCs, three physical SOCs, dynamic EV limits, +T1/T2 import counters, EV/SDL requested setpoints (READ ONLY). +A cyclic script reads values already present in Symcon every 30 seconds. It does not issue +additional Modbus polls. Original VariableUpdated/VariableChanged are retained; these are +Symcon timestamps, not independent device-measurement timestamps. Two-pass reads detect +concurrent changes; missing/non-numeric/re-used channels remain explicit quality failures. + +## Storage and interpretation + +Daily append-only raw-YYYYMMDD.jsonl in the separate capture stage data directory, UTC +capture times plus original source timestamps, immutable mapping/inventory snapshots by +SHA256. Raw errors and quality_hold are recorded too; nothing is invented as zero. +The raw observations are a source for later documented 5-minute aggregation, not themselves +aligned 5-minute means. No interpolation, training publication or future SDL assumption. + +A 512 MiB journal quota, 64 MiB daily limit, 256 KiB record cap and 100 MiB free-space +reserve bound recording. Limits fail visibly without deleting old data. File permissions +0640; no upload credentials, request bodies, complete instance configurations or secrets. +A separate status variable and STATUS.json show latest capture and failures. Process restart +retains files; an incomplete final JSONL row is preserved for manual review, not extended. + +## Install / stop + +Staged user entry point: /srv/agent/netplan-v4-data-capture-stage/install.php +Execute in the Symcon PHP editor (not a Linux shell): + + require '/srv/agent/netplan-v4-data-capture-stage/install.php'; + +Installer checks exact source hashes and numeric source identities before creating a NEW +root category ENELIX_V4_PASSIVE_CAPTURE, a status string, collector script and stop script. +IPS_SetScriptTimer enables only this new script at 30 seconds. Repeating installation is +idempotent. Unrelated/edited scripts or object identity collisions are not overwritten. + +Stop using the newly created script 'Nur diese Messaufnahme stoppen'; it disables only +this collection timer, retains collected files, and does not stop the manager or battery. +No module reload, existing archive change, existing timer/poller change, service restart, +forecast selection, battery reservation or actuator command is included. + +Current tests: 36 core/file tests + 19 mocked native lifecycle tests with PHP CLI on the +test host. First real capture and future timer execution require the user activation. +These tests do not certify hardware mappings or a Symcon-kernel run. diff --git a/docs/netplan-v4-history-timing.md b/docs/netplan-v4-history-timing.md new file mode 100644 index 0000000..b96f8d4 --- /dev/null +++ b/docs/netplan-v4-history-timing.md @@ -0,0 +1,90 @@ +# Historical publication timing (2026-10-03) + +## Implemented scope + +`services/netplan-v4/netplan_v4/history_timing.py` implements optional, bounded +retrospective estimates between consecutive identical source values at DIFFERENT +original publication timestamps. It is used by the existing measurement pipeline, +model training and forecast source substitution. It is NOT a real-time estimator, +a measurement certificate, an actuator authority or a repair of device communications. + +No source `maxAgeSeconds` is changed. When a regular hold expires, only its tail +until the next qualifying original publication may be estimated. Strict processing +remains the default. The following forbid the estimate: unequal endpoints, +nonfinite/missing observations, conflicting same-timestamp values, collector gaps, +invalid source intervals or a span beyond the explicit source-specific bound. +There is no extrapolation past the last observation and no fabricated zero. + +Each window reports `publicationEstimatedSeconds`, per-source estimated portions, +uncovered-source seconds and `availableNotBefore`. Training/validation must not use +a confirmation or receipt before it was actually available. Backfilled windows +cannot establish a supposedly causal holdout at a date before their availability. +All physical windows remain configured estimates; even full support is not proof +that the physical waveform stayed constant between the two endpoint readings. + +## Evidence and model assumptions + +Read-only source configuration projection on the test server, snapshot +2026-10-03T07:47:24Z: GoodWe 1 instance 19742 (ModBus Device) has Poller 60000 ms; +GoodWe 2 instance 57658 has Poller 5000 ms; SolarEdge scale instance 48996 +(ModBus Address) has Poller 20000 ms. No polling setting was changed. +Projection program: /srv/agent/netplan-v4-application-build/inspect_source_update_policy.py. + +Official Symcon documentation accessed on 2026-10-03: +https://www.symcon.de/de/service/dokumentation/modulreferenz/geraete/modbus-rtu-tcp/vorlagen/ +It documents value publication for ModBus Device on changes or when the variable is +older than 60 seconds, including ordinary and virtual addresses. VariableUpdated +is therefore not automatically the last successful device poll. The documentation +is not a proof that a particular installed device link was healthy. + +Explicit Lihrenmoos HISTORICAL estimate bounds: +- GoodWe 1 PV/physical storage: equal original publications at most 130 s apart + (60 s publication suppression + configured 60 s poll + 10 s scheduling allowance). +- GoodWe 2 PV/physical storage: at most 75 s (60 + 5 + 10). +- SolarEdge scale: equal scale publications at most 90 s apart. This is a modelling + bound based on the configured 20 s polling and observed sparse scale publication, + not a manufacturer guarantee and not proof of the same internal implementation + as ModBus Device. Raw AC power remains under the original strict limit. +The 10 s margins are explicit scheduling assumptions, NOT measured timing guarantees. +Gaps of many minutes (e.g. the previously observed GoodWe gaps above 600 s) are NOT +bridged. Nonzero changes are never interpolated by this policy. + +## Versioned dataset without restarting collection + +`lihrenmoos-physical-published-v2` references immutable observations in +`lihrenmoos-physical-v1`. No observations are copied, deleted, relabelled or resent. +Registration requires the same installation and identical original mapping, +formula, coverage and training configuration. Cycles/reference chains and device +uploads directly into a derived dataset are rejected. Original ingestion continues +unchanged in the Manager. The 24 usable-hour minimum stays in force. + +Models/windows are stored separately for v1/v2 and record their timing policy. An +existing selected model/family is not automatically switched by this release. +The current SDL request still uses the strict current-state requirements; no +historical bridge grants a future SDL schedule or a live battery permission. + +## Tests and release + +254 isolated Python tests passed (219 previous + 35 new including simulated +installation/recovery). Includes the actual service pipeline from referenced +observations to model and optimizer using synthetic records. No new target-container +or field-history evaluation has been run with this release yet; do not infer an +improved real coverage percentage or production acceptance from unit tests. +Evidence: /home/agent/services/qa/history-timing-20261003/ALL_TEST_RESULTS.txt. + +Prepared command on enelix-services (root required for Docker): + + python3 /home/agent/services/netplan-v4-shadow/commissioning/deploy_history_timing.py \ + --plant e3a08f9e-af12-4695-99bd-8b51c0520021 --apply + +Default without --apply validates staged source only. The apply path backs up source +files, builds and runs the exact target tests, backs up the V4 SQLite database, and +recreates ONLY the V4 service. A failed test restores source without a service +restart; failed post-deploy validation attempts the previous image and preserves +additive data. Concurrently modified files are not silently overwritten. + +No portal/forecast/tariff/Symcon restart, no new sensor requests, no change to +161.44 kWh / 39 kW, SOC accounting, old raw samplers or actuator permissions. +The new dataset is registered but not selected for the active forecast. A root +execution and successful report are still required to install it. Previous release +manifests remain historical records; do not bypass them to reapply an older build. diff --git a/docs/netplan-v4-numeric-mapping-fix.md b/docs/netplan-v4-numeric-mapping-fix.md new file mode 100644 index 0000000..a834702 --- /dev/null +++ b/docs/netplan-v4-numeric-mapping-fix.md @@ -0,0 +1,52 @@ +# Numeric mapping identity defect - 2026-10-03 + +## Confirmed from installed code and read-only evidence + +The pending batch contains 120 observations: zero duplicate timestamps, zero +backwards pairs, but 75 identity failures. The initial 45 observations in this +batch are imported observer records; later native captures use the saved manager +configuration. Evidence: test-host `PENDING_DELIVERY_REVIEW.json`. + +The configured mapping fingerprint is +`517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b`. +Reproducing native configuration loading generates +`3a1a9cf42aae49f55385d2350da74cae780ae12f170e4976849591e48a393994`. +The sole difference in the compared configurations is numeric representation of +`accounting.splitToleranceW`: prepared float 100.0 versus saved integer 100. +Both inventory fingerprints and all measurement definitions match. +`NetzfahrplanV4Bilanzierung::konfigurieren` validated this field using a function +which returns a float, but discarded the normalized return value. Other numeric +factors were already assigned from that validator. + +## Tested source correction + +Assign the returned float to `splitToleranceW`. Equal numeric settings now retain +the original fingerprint through a plain JSON property round trip. Actual numeric +changes still produce distinct fingerprints, and boolean/string values are still +rejected. Input configurations are not mutated. + +42 isolated PHP checks passed on the test host: 34 accounting regressions plus +8 numeric identity regressions. A separate check with the actual prepared capture +configuration reproduces the configured fingerprint after the property round trip. +Evidence: `/srv/agent/netplan-v4-application-build/NUMERIC_MAPPING_TEST_RESULTS.json` +and `NUMERIC_MAPPING_TEST_RESULTS.txt`. Library SHA256: +`ffda2a03dbb226a4405a2ad66704c080a9dc475ce64282ad3f0d7d60034ab199`. + +## Not deployed / recovery still incomplete + +No installed library, original outbox, cursor, server validator, settings, dataset +registration or actuator permission was changed in this turn. Future-capture +normalization alone does not recover already stored native observations. The +server correctly rejects an entire batch containing a mismatched mapping. + +An untested proposal to send distinct immutable dataset versions in separate +batches was removed from the development source and preserved only as a draft at +`/home/agent/services/qa/numeric-mapping-20261003/UNTESTED_SENDER_DRAFT.patch`. +Metadata preparation files are not an installation or permission to ingest into +an unregistered dataset. Tool writes for the complete server recovery package were +blocked; no alternative deployment or identity-check bypass was performed. + +Next recovery must preserve original records and provenance, retain exact +per-dataset identity validation, acknowledge only accepted data and include tests +for the mixed 45/75 batch, retries, partial ACKs and no cursor/data loss. Do not +skip the rejected block, rewrite hashes in raw journals or re-run old installers. diff --git a/docs/netplan-v4-receiver.md b/docs/netplan-v4-receiver.md new file mode 100644 index 0000000..387e383 --- /dev/null +++ b/docs/netplan-v4-receiver.md @@ -0,0 +1,48 @@ +# V4 plan reception and local preview (observation only) + +This module does not execute a battery command. The existing regulator and its +setpoints remain unchanged. `NetzfahrplanAktiv` must remain false. + +The optional `NetzfahrplanV4EmpfangAktiv` property defaults to false. When enabled +with the existing shadow sender, one bounded GET is made before the next native +operation POST, avoiding reading the plan immediately after queuing its replan. +A successful inspection acknowledges `shadow_seen` once per plan/step. This is +not `applied`. An HTTP failure clears the local preview; it does not change the +old control mode or its connection credentials. + +The V4 service must first support `receiverProtocolVersion=1`, installation IDs +in envelope and plan, per-input event references, and the stable controlContext +used at calculation time. Old or cross-plant responses are deliberately refused. +The client checks fresh/lastRun/pending status, exact configuration revision, +model-family registry, known price coverage, contiguous half-open five-minute +intervals, asset mapping and power balance. Numerical feasibility checks are +not evidence of economic performance or permission to control devices. + +A cache watchdog refreshes the diagnostic display every 10 seconds, without any +network requests. Cached response older than 90 seconds, plan older than 900 +seconds, an expired step, invalid local measurement or policy change invalidates +the preview. Missing targets are JSON null and '-' in the display, never zero. +Restart/ApplyChanges clears the cache. Existing control functions are not edited. + +The preview is currently limited to ONE battery. Multiple batteries are still +accepted by the server planner; local allocation is explicitly unsupported in +this stage. Preview feedback uses actual grid and battery power, local capacity, +available power, minimum/maximum SOC, discharge hysteresis, network-charging +permission and import/export/month limits. It exposes any remaining grid-limit +violation instead of pretending the battery can resolve it. It is a momentary +calculation only, NOT the commissioned real-time regulator. PV curtailment and +external SDL/base-load separation must still be validated before any live test. + +Two read-only variables below the manager expose a HTML summary and detailed +JSON. No EnableAction, consumer RequestAction, SendData or actor property writes +are used. The existing form.json and web portal assets are not modified. + +Offline checks: + + php tests/V4Receiver/protocol_checks.php + php tests/V4Receiver/receiver_checks.php + +The PHPUnit wrapper runs these in separate subprocesses, so fake IPS and curl +functions cannot contaminate other module tests. The mocked transport uses no +real credentials or connections. Full Symcon kernel testing and installation +are separate commissioning steps; these unit tests are not field acceptance. diff --git a/docs/netplan-v4-separated-observation.md b/docs/netplan-v4-separated-observation.md new file mode 100644 index 0000000..7647d28 --- /dev/null +++ b/docs/netplan-v4-separated-observation.md @@ -0,0 +1,88 @@ +# Separate physical load / EV-SDL observation (NO dispatch or training permission) + +## Implemented 2026-10-02 + +- `NetzfahrplanV4Messsicht` recalculates independent physical-load, physical-storage, + unfiltered modelled allocation and SolarEdge-origin views from explicitly identified + raw snapshots. Quality failures in held virtual accounts do not veto otherwise + consistent physical inputs. Missing/stale physical inputs still invalidate their + respective view. No automatic relaxation of age/skew limits. +- EV/SDL model: delta = physical - requestedEV - requestedSDL; distribute delta in + proportion to the absolute requested powers. The SDL component is obtained from + physical minus modelled EV to preserve numerical closure. This is an accounting + convention, NOT independently measured EV and SDL and NOT a verified response. +- Command changes more recent than the oldest physical feedback are flagged. Unknown + change timestamps, tracking mismatch, aged/missing origins and idle unallocated + power remain explicit. There is no held-value filter and no forced convergence of + displayed power to the requested value. All proposals keep `canDispatch=false`. +- Actual legacy filter outputs are displayed solely for comparison. They are not + reused as inputs of physical-load or unfiltered partition calculations. No changes + to the legacy filter, physical driver, EV/SDL setpoints, SOC integration or accounts. + +## New read-only origin finding + +Selected event 38921 writes legacy PV variable 20335 using: + + a = 10 ** GetValue(41853) * GetValue(37975) + GetValue(21447) + if (a < 0) a = 0 + SetValue(20335, a) + +Arithmetic statements were inspected without exposing arbitrary scripts/credentials. +Verified saved configuration: variable 37975 parent 35514, register 40083; scale +41853 parent 48996, register 40084; battery 21447 parent 30789. An independently +updated 20335 is therefore not an independent PV reading and cannot certify all +input timestamps. A second constrained arithmetic read confirmed the condition +`if ($a < 0)`; this clamp applies to the legacy calculated PV value, not to the +new signed terminal-power observation. Neither proves the full AC/DC boundary. + +The staged observer additionally records the ORIGINAL mantissa and exponent, not +only their periodically rewritten derived PV result. The alternative candidate +uses the scaled terminal power ONCE for this inverter (removing its derived-PV +and battery terms). GoodWe AC/DC boundaries and full-site physical boundary are +still unverified. This is not a definitive corrected house meter or live forecast. + +## Standalone installation / operational scope + +Stage on test host: `/srv/agent/netplan-v4-separated-observer-stage`. +User invocation inside Symcon only: + + require '/srv/agent/netplan-v4-separated-observer-stage/install.php'; + +Creates only a NEW root category, three textual diagnostic variables, observe and +stop scripts and a NEW 30-second timer. Reads 23 existing numeric variables twice; +no Modbus polling requests, network calls, archive settings, existing module or +control timer updates. Existing 21-channel raw collector continues untouched. + +New daily journals contain the selected numeric raw snapshot and derived views. +A new immutable mapping/inventory reference binds the corrected user EV allocation +161.44 kWh / 39 kW. Original collector metadata and its correction addendum remain +unchanged. No rescaling or resetting of virtual SOC/energy accounts. + +Derived journal and LATEST/STATUS reports are 0644 WITHIN mode-0700 private stage +and data directories, allowing the authorized agent to inspect these selected +new outputs without repeated manual exports. No permissions on original journals, +Symcon directories or unrelated data are changed. Quotas reuse 64 MiB/day, 512 MiB +in total and 100 MiB disk reserve; exceeding them stops recording visibly and does +not delete old data. No network transfer or cloud trainer is activated. + +## Validation and limitations + +40 pure view checks + 32 native-mock checks passed on PHP 8.3 CLI at test host. +Regression: 36 raw-capture + 19 native-capture + 34 accounting checks passed with +new additive `raw_power`/`scale` units. Total 161 isolated checks. Full relevant PHP +source lint passed. Not a full PHPUnit run, Symcon kernel trial or hardware test. + +Tests cover opposite EV/SDL requests, held output, non-unique partition labelling, +negative terminal power, scale/sentinel validity, old dependency timestamps, changed +commands, missing source, no duplicate native objects, unchanged existing signals, +append preservation, source drift, private output restrictions and stop behavior. + +The observer is PREPARED, not started by the assistant. It does not replace the +installed receiver or its existing preview calculation. The preview/driver must +only be switched after measurement-boundary and feedback policy acceptance. Model +training and five-minute reconstruction must consume the new versioned snapshots +separately; there is no training eligibility or zero future-SDL assumption here. + +Device-side command-loss protection and exact model/firmware identification remain +open. Official SolarEdge PDF fetch was unavailable (403/error); no device-specific +register suitability or watchdog behavior was inferred from a third-party source. diff --git a/docs/netplan-v4-unified-release.md b/docs/netplan-v4-unified-release.md new file mode 100644 index 0000000..09bd7e0 --- /dev/null +++ b/docs/netplan-v4-unified-release.md @@ -0,0 +1,77 @@ +# V4 Unified RC1 - actual scope and release boundary + +This release consolidates application data recovery, native delivery status, economic +family comparison, and optional verified archival. It is NOT a production actuation +release. No claim that all original product requirements or hardware acceptance are +complete. Sources are in the existing develop branch, not feature branches. + +## Delivered code + +* Audited recovery for the exact 100.0/100 splitToleranceW serialization defect. + The internal operator endpoint validates the two complete JSON representations, + their exact SHA256, installation, inventory, and every field. Only that numeric + representation may differ. Device-supplied arbitrary aliases are never accepted. + Existing rows/measurements are not rewritten. The original received mapping and + proof reference are recorded with the observation. A mixed 45/75 block and retries + are regression tested. Adoption requires an explicit deployment flag. +* Native normalization prevents recurrence. The sender preserves a separate last + successful receipt even during scheduled/backoff states, reports safe numeric HTTP + status (no response or credential text), and continues trying to deliver a durable + prefix if the current capture cannot be appended. No cursor reset or skipped block. +* Economic comparison is actually connected, not an empty score list. At the start + of a local calendar day the worker freezes comparable plans for families 3/13/23 + with the then-known forecasts, prices and common battery/limit state. Completed + days are evaluated against subsequently acquired physical residual data. Initial + energy, losses, common terminal energy valuation and 15-minute demand charge are + included; monthly maxima are charged once over a comparison period, not per day. + GUI lookback/minimum days/coverage/switch margin are applied by the selector. +* v1 economic scope: one grid-charge-enabled battery, frozen day-ahead plans, with + local grid-target tracking simulated during the day. It is NOT a full receding- + horizon replay. Historical external SDL uses the then-recorded SDL request and is + clearly marked as an estimate, not independently measured delivery. Missing + actuals, unmatched contexts and breaches cannot become zero cost or a successful + comparison. Broader device topologies stay collecting; no fake equivalence. +* Lossless archive implementation and tests are present for acknowledged old native + outbox days and old server observations/plans. Archives are verified before plain + rows/files are retired. Active plans, acknowledged plans and unacknowledged data + are excluded. Archiving is OFF by default; enable only with an explicit storage + policy (native NetzfahrplanV4ArchivAktiv; server NETPLAN_V4_ARCHIVE_ENABLED=1). + Compressed archives are retained: this is not an external backup or infinite disk. + +## Deployment is not data/model/actuator acceptance + +Server entry: commissioning/deploy_unified_release.py. Default checks sources only. +--apply builds and tests a separate candidate context with no network/production +mounts before replacing the V4 service. It preserves the running plant allowlist, +backs up SQLite consistently, and uses a pinned built image. Only the V4 planner JS +asset is updated; unrelated GUI work, portal runtime, tariffs, forecasting, Symcon +and actuators are untouched. Rollback restores previous source/image; additive audit +rows remain. Database schema changes do not overwrite old observations. + +--approve-numeric-mapping-compatibility explicitly authorizes the demonstrated pair +of equivalent configuration representations. Without it, strict original validation +remains and backlog containing the legacy serialization is still rejected. + +Lihrenmoos native stage: /srv/agent/netplan-v4-unified-release/install.php. +This patches three data-only libraries, reloads the existing library, compares all +pre-existing properties and lets the existing data timer resume. No raw replay, +force-skip or actuator callback from the installer. Archiving stays disabled by default. + +## Remaining product work - do not label as complete + +1. The full production dispatcher is not installed. Earlier controlled-trial code + still has bounded pilot grants; continuous production authority and complete + corrected physical/virtual feedback integration are NOT delivered by this RC. +2. The actual host/container/kernel and field acceptance of this release are not + replaced by CLI mocks. Real source validity and independent command-loss behavior + are not asserted by a measurement algebra or an override. +3. The replay limitation above must be accepted as a beta feature, or extended to + continuous rolling reoptimization and additional asset types before advertising + the original full automatic-selection requirement. +4. Native multi-plant commissioning, external backups, retained archive lifecycle, + model quality on field data and publication authentication remain operational + release criteria. The tests do not demonstrate actual cost savings. + +Do not re-run obsolete piecemeal installers after this release, bypass their source +checks, turn the previous grid schedule on, or turn 'productionReady' true manually. +The retained EV capacity is 161.44 kWh / 39 kW; SDL reserve is already outside EV. diff --git a/docs/prognose-bedienintegration.md b/docs/prognose-bedienintegration.md new file mode 100644 index 0000000..54ce155 --- /dev/null +++ b/docs/prognose-bedienintegration.md @@ -0,0 +1,111 @@ +# Prognosebedienung im bestehenden Manager + +Stand: 06.10.2026. Gepruefter Integrationsstand zur von Daniel freigegebenen +Veroeffentlichung auf develop und beta. Kein Deployment und keine neue Stellfreigabe. + +## Umfang + +Der separate Formularbereich `V4 Planempfang / Prognosemanager` entfaellt. +Empfang, Planpruefung, Lizenzportal-Verweis und explizites Ein-/Ausschalten +stehen im bestehenden Bereich `Prognose / Forecast`. + +Der neue Formularaufruf `FormNetzfahrplanSchalten` delegiert an den vorhandenen +`V4ManagerAktivtestSchalten`-Pfad. Er prueft beim Start nochmals gespeicherte +Prognosekonfiguration, Netzfahrplanlizenz, Managerstatus und bestehende lokale +Testfreigaben. Ein Formularaufruf allein erteilt keine Berechtigung. Ausschalten +bleibt auch nach Lizenzverlust moeglich. Der bestehende Stopp beinhaltet eine +frisch berechnete normale Regelung nach Widerruf der V4-Sitzung; er ist kein +anlagenweiter Not-Aus und veraendert keine unabhaengigen SDL-Auftraege. + +Die Anzeige nennt weiterhin Testbetrieb und das bestehende Testende. Es gibt +keine neue Dauerbetriebsfreigabe, keinen Reset der 48-Stunden-Frist und keine +Aenderung an Watchdog-Ausnahmen, Messwertgrenzen oder Geraeteschutzpruefungen. + +## Kompatibilitaet + +- Keine Properties, Attribute, Objekt-IDs, Timer, Datenpfade oder Backend-APIs werden migriert. +- Prognosemodelle, Optimierer, Planempfang, ACKs, Messaufnahme und Versand bleiben unveraendert. +- Die vorhandene Portalpruefung fuer `grid_schedule` bleibt massgeblich. Keine Portal- oder Lizenzbestellung wird geaendert. +- `NetzfahrplanAktiv` bedeutet weiterhin den alten Regler. Bei vorhandener V4-Konfiguration wird dessen ausgeschalteter Schalter ausgeblendet. Ein eingeschalteter Altregler bleibt zum Ausschalten sichtbar. +- Nicht auf V4 eingerichtete Installationen behalten den bisherigen Schalter. Diese Aenderung provisioniert keine neue V4-Installation. +- `PrognoseAktiv` wird nicht zum automatischen V4-Startsignal. Bestehende Aufnahme-/Sender-Properties und die bisherige Laufzeitvariable bleiben erhalten. + +## Dateien + +- `Manager/form.json`: stabiler Anker fuer den bestehenden Prognosebereich. +- `Manager/module.php`: Formularintegration und gepruefter Bedienaufruf. +- `libs/ManagerPrognoseFormular.php`: reine Formularzusammenstellung ohne I/O. +- `libs/ManagerNetzfahrplanV4EmpfangTrait.php`: nur bisherigen Formulargenerator entfernt; Empfangscode unveraendert. +- `tests/PrognoseFormular/`: Formular- und echte RequestAction-Pruefungen mit simulierten IPS-/Stellaufrufen. +- `tests/V4Receiver/receiver_checks.php`: Strukturpruefung an die zusammengefuehrte Oberflaeche angepasst. + +## Tests + +Isoliert im PHP-8.3.6-CLI auf einer Quellkopie, nicht im Symcon-Kernel: + +| Suite | Erfolgreiche Pruefungen | +|---|---:| +| PrognoseFormular/checks.php | 59 | +| PrognoseFormular/manager_checks.php | 26 | +| V4Receiver/receiver_checks.php | 32 | +| V4ControlTrial/checks.php | 96 | +| V4Feedback/receiver_checks.php | 8 | +| V4Feedback/device_read_checks.php | 9 | +| V4ControlTrial/register_checks.php | 1 | +| Gesamt | 231 | + +Zusaetzliche Gesamttests des zusammengefuehrten Stands am 06.10.2026: + +- PHP 8.3.6 / PHPUnit 9.6.36: 357 Tests, 1758 Assertions erfolgreich. +- Syntaxpruefung: alle 150 PHP-Dateien erfolgreich. +- Backend: 310 Python-Tests erfolgreich mit den vorhandenen QA-Abhaengigkeiten. +- Portal: 16 Node-Tests erfolgreich. +- Konfliktmarker- und Patch-Pruefung erfolgreich. + +Die Pruefungen liefen in isolierten Quellkopien. Die PHP-Laufzeit wurde nur im +Agent-Arbeitsordner entpackt, ohne Systeminstallation. Kein Symcon-Kernel-/GUI- +oder Hardwaretest; der vorhandene Docker-Zugang war nicht nutzbar. + +Eine vorbestehende, nicht committe Aenderung an measurement_pipeline.py wurde +bewusst nicht uebernommen: Sie erzeugte fuer jede konfigurierte Messgrenze ein +accountingEvidenceId trotz measurementBoundaryVerified=false und verletzte den +bestehenden Backendtest. Die Datei bleibt im Integrationsstand bytegleich zum +bisherigen versionierten V4-Stand. Die fremde Arbeitskopie und das laufende +Backend wurden nicht veraendert. Eine spaetere Freigabe dieser Messgrenze braucht +einen eigenstaendigen fachlichen Nachweis, keine Anpassung des Tests an das Label. + +## Offener Abschluss + +Der Auftrag ist mit dieser Bedienintegration noch nicht vollstaendig umgesetzt: + +- Die dokumentierte Batterietimerblockade und die fehlende Anlagenabnahme werden hier nicht behoben. +- Ein regulaerer, lizenzierter Dauerbetrieb anstelle des begrenzten Testpfads ist nicht implementiert. +- Die separaten Laufzeitvariablen und Diagnoseordner werden noch nicht entfernt oder ausgeblendet. +- In gespeicherten Settings vom 06.10.2026, 13:06:07 UTC sind nur die Kategorien 21196 (`ENELIX_V4_SEPARATED_OBSERVATION`) und 57590 (`ENELIX_V4_PASSIVE_CAPTURE`) eindeutig als V4-Diagnosebereiche erkannt. Darin liegen weiterhin Beobachtungs-/Aufnahmeskripte. Keine pauschale Loeschung nach Namen. +- Der allgemeine `Testordner` 10249 enthaelt auch Abrechnung, Schnittstellen und weitere nicht zu dieser Integration gehoerende Objekte. Er bleibt unangetastet. +- Vor einem Rollout aktuelle Moduldateien sichern, mit dem getesteten Stand vergleichen und Abhaengigkeiten pruefen. Keine pauschalen Modulupdates, Reloads oder Dienstneustarts aus diesem Dokument ableiten. +- Die Divergenz wurde in einem separaten develop-Checkout auf ubuntu zusammengefuehrt: Server-89-Historie bis ce525a1 und origin/develop bis 5437f3f. Die Originalarbeitsverzeichnisse mit fremden Aenderungen bleiben erhalten. Daniel hat Merge, Commit und Push auf develop und beta ausdruecklich freigegeben; main bleibt unveraendert. + +## Uebergabe und Ruecksetzung + +Die fruehere Einzelpatch-Uebergabe ist durch den zusammengefuehrten Quellstand +ersetzt. Die Ausgangshistorie und die zugehoerigen lokalen V4-Quellen sind unter +/srv/agent/prognose-integration-20261006 gesichert. Der gepruefte Kandidat +enthaelt keine temporaren Live-Hooks, Zugangsdaten oder fremden uncommitteten +Ladestationsaenderungen. Ruecksetzungen nur additiv nach Driftpruefung; keine +fremden Aenderungen verwerfen. Fuer einen spaeteren Runtime-Rollout sind eine +eigene Sicherung, Abhaengigkeitspruefung und Anlagenabnahme erforderlich. + +## Vorschlag PR-Text + +**Titel:** Manager: Prognosebedienung im bestehenden Forecast-Bereich buendeln + +**Aenderung:** Separaten V4-Formularbereich entfernen, bestehende Funktionen +unter Prognose / Forecast anbieten und explizite Bedienaktionen zusaetzlich +gegen Lizenz-, Konfigurations- und Freigabestatus pruefen. + +**Unveraendert:** Prognose-/Optimierungsbackend, Empfangsprotokoll, Anlagenlimits, +befristete Testfreigaben, Datenhistorie und SDL. Keine automatische Aktivierung. + +**Pruefung:** 357 PHPUnit-Tests, 310 Backend-Tests, 16 Portal-Tests sowie die +oben genannten isolierten Bedien-/V4-Checks; kein Hardware- oder Symcon-GUI-Abnahmetest. diff --git a/docs/testing/Netzfahrplan-V4-Bezugszaehler.md b/docs/testing/Netzfahrplan-V4-Bezugszaehler.md new file mode 100644 index 0000000..533aa3a --- /dev/null +++ b/docs/testing/Netzfahrplan-V4-Bezugszaehler.md @@ -0,0 +1,66 @@ +# V4: separate Wirkenergie-Bezugsquelle (Entwicklung, nicht live) + +## Lihrenmoos - Befund und geplante Zuordnung + +User hat die erfolgreiche Aktivierung der T2-Zaehlerarchivierung fuer 26620 +im Archiv 16207 bestaetigt. Der exakte Beginn einer lueckenlosen Messhistorie +ist daraus NICHT nachgewiesen; keine historische Null setzen. + +Die vorbereitete separate V4-Konfiguration fuer diese Anlage lautet: + +```json +[ + {"VariableID":59607,"ElternID":11490,"Ident":"Energy_0","FaktorZuKWh":1.0,"Messgroesse":"WirkenergieBezug"}, + {"VariableID":26620,"ElternID":11490,"Ident":"Energy_1","FaktorZuKWh":1.0,"Messgroesse":"WirkenergieBezug"} +] +``` + +Property: NetzfahrplanV4BezugszaehlerQuellen. Diese Datei setzt KEINE Property. +IDs duerfen nicht fuer andere Anlagen kopiert werden. Zuordnung ist durch +Variablenstruktur, Standardtelegramm und Leistungs-/Energiedifferenz stark +plausibilisiert; Abgleich mit Rohtelegramm/Zaehleranzeige steht noch aus. + +53476, dessen gespeicherte Historie, die alte Manager-Property +NetzbezugEnergieVariableID und alle vorhandenen Spiegelereignisse bleiben +unveraendert. V4 liest seine eigenen Quellen, erzeugt keine Spruenge in alten +Summenzaehlern und kann keine Blindenergie durch einen Faktor korrigieren. + +## Vertrag + +Jede Quelle hat eine ausdrueckliche Messgroesse und Umrechnung nach kWh. +Variable, Elterninstanz und Ident werden bei jedem Lesen geprueft. Ein +fehlender, veralteter, ungueltiger oder zeitlich unpassender Teilwert verwirft +die Summe. Ein aktueller echter Nullwert ist gueltig. Zwei aufeinanderfolgende +Lesevorgaenge reduzieren das Risiko einer gemischten Momentaufnahme; sie sind +kein atomarer Zugriff auf das physische Zaehlertelegramm. + +Die Summe wird unter bezugszaehler in der lokalen V4-Diagnose ausgegeben, mit +den Beobachtungszeitpunkten jedes Teilregisters. VariableUpdated ist ein +Symcon-Aktualisierungszeitpunkt, kein Beweis einer exakten Abrechnungsgrenze. +Die Diagnose hat deshalb billingEvidence=false und historyComplete=false. +Eine falsche Messgroessen-Deklaration kann ohne Rohtelegramm nicht automatisch +erkannt werden; Installationspruefung bleibt erforderlich. + +Ein aus Quell-IDs, Eltern-ID, Idents und Umrechnungsfaktoren gebildeter +stabiler SHA256-Schluessel bindet die spaeteren Messnachweise an genau diese +Quelle. Die Reihenfolge von T1/T2 aendert den Schluessel nicht. Quell-/Faktor- +wechsel dagegen invalidieren alte Nachweise. symcon:53476 wird nicht mehr +als V4-Messnachweis akzeptiert. Der Schluessel beweist Zuordnung, nicht den +Wahrheitsgehalt eines gelieferten Monatspeaks. + +## Noch offen + +Der Erzeuger fuer exakte Viertelstundenenergie und den vollstaendigen +Monatspeak ist weiterhin nicht implementiert. Keine Ableitung eines bereits +bezahlten Monatspeaks aus einer Managergrenze oder unvollstaendiger Historie. +Auch 15 Minuten neuer Aufzeichnung beweisen nicht den bisherigen Monatspeak. +Die vorbereitete native Summe ist noch nicht im laufenden Symcon installiert. +V4 bleibt ohne belastbare Nachweise in awaiting_inputs. + +## Nachweis dieses Schritts + +44 synthetische Offline-Szenarien mit PHP CLI 8.4.23 erfolgreich: +20 Quellen-/Summenpruefungen, 24 native Betriebsdatenkonvertierungen. +Die vom Server exportierten Betriebsdaten-, Trait- und Fixture-Dateien wurden +vor dem Test per SHA256 verglichen. Keine Symcon-Laufzeitpruefung und keine +neue Zielcontainer-/vollstaendige PHPUnit-Pruefung in diesem Schritt. diff --git a/docs/testing/Netzfahrplan-V4-Schatten.md b/docs/testing/Netzfahrplan-V4-Schatten.md new file mode 100644 index 0000000..19a6a4c --- /dev/null +++ b/docs/testing/Netzfahrplan-V4-Schatten.md @@ -0,0 +1,96 @@ +# V4: native Betriebsdaten, ausschliesslich Schattenbetrieb + +Stand 2026-10-01: Der optionale Sender und die Diagnose sind implementiert. +Kein V4-Plan wird damit uebernommen, kein Aktor geschrieben, kein V1-Sollwert +geaendert. Der Sender ist nach Installation standardmaessig AUS. + +## Konfiguration im Manager + +- NetzfahrplanV4SchattenAktiv: bool, Standard false. Eigener 60-s-Timer, + unabhaengig vom bestehenden Prognose-Timer. Fehler betreffen nur den Sendestatus. +- NetzfahrplanV4NetzladenErlaubt: bool, Standard false. Betrifft ausschliesslich + die im Schattenplan angenommene Berechtigung, keine reale Netzladefreigabe. +- NetzfahrplanV4BatterieOptionen: JSON-Objekt je Topologie-ID. Optional + SOCKapazitaet_kWh und MaxSOC_Prozent (sonst 100). Bei unterschiedlicher Nenn- + und Nutzkapazitaet ist SOCKapazitaet_kWh zwingend explizit anzugeben. +- NetzfahrplanV4BezugszaehlerQuellen: JSON-Liste, Standard []. Eigene, explizite + Wirkenergie-Bezugsquellen fuer V4; KEIN Rueckfall auf den Altzaehler. + Details und Lihrenmoos-Kandidat: Netzfahrplan-V4-Bezugszaehler.md. +- NetzfahrplanV4MessnachweisVariableID: JSON-Stringvariable, Standard 0. + Ein vollstaendiger historischer/registerbasierter Messnachweis-Produzent + ist NOCH NICHT integriert. Fehlende Nachweise bleiben fehlend. + +Die Quelle muss der gemeinsame Netzanschluss sein. Topologiebatterien werden +nur einer eindeutigen aktiven Batterieinstanz mit derselben SOC- UND +Leistungsmessquelle zugeordnet. Externe SDL-Batterien werden nicht automatisch +hinzugefuegt. SDL-/Grundlastbereinigung bleibt eine separate, noch offene Aufgabe. +BMS-Grenzen werden mit Topologieleistungen begrenzt. Normale Betriebsreserve +und technisches Minimum werden nicht verwechselt: die hoehere Grenze gilt. + +Der erweiterte Sender uebermittelt Entladesperre und Wiederfreigabe-SOC separat +von der maximalen Entladeleistung. Der neue Python-Rechenkern darf die Sperre +im Zukunftsplan erst nach vorherigem Laden bis zur Wiederfreigabe aufheben. +SOC unterhalb der Betriebsreserve, aber oberhalb des technischen Minimums, +bleibt unveraendert erhalten: Es ist Wiederaufladung statt erfundener Energie +vorgesehen. Unterhalb des technischen Minimums wird kein normaler Plan erzeugt. + +Zusaetzlich wird meterObservation mit frischen Empfangswerten der Netzleistung +und T1/T2-Summe uebermittelt. Dessen laufende Viertelstundenintegration ist eine +explizite Planungsschaetzung, KEIN verifizierter Abrechnungsnachweis. Verwendung +von Schaetzungen muss im V4-Dienst ausdruecklich aktiviert werden. Historische +Peakannahmen werden getrennt von verifizierten Maxima gespeichert. + +Diese Protokollerweiterung benoetigt zuerst den passenden Backendstand. Der +bisherige laufende V4-Container und die Testanlage sind noch nicht aktualisiert. +Offline-Szenarien und Quelltextpruefungen ersetzen keinen Symcon-Laufzeittest. + +## Diagnose ohne Netzwerk oder Stellbefehl + +Im Symcon-Skript nach Installation des neuen Modulstands: + +```php +", + "measuredAt": "2026-10-01T12:05:00Z", + "measuredPeaks": { + "2026-10": {"kw": 18.4, "source": "meter_month_register"} + }, + "quarterPast": { + "start": "2026-10-01T12:00:00Z", + "measuredSeconds": 300, + "importKwh": 0.5 + } +} +``` + +Zulaessige Herkunft: meter_month_register, verified_month_history oder +verified_new_month. Der Nachweis-Produzent muss deren Wahrheit garantieren. +Managergrenzen sind NIE ein gemessener Monatspeak. Der Monatswert muss den +vollstaendigen bisherigen Abrechnungsmonat abdecken. Fuer quarterPast muessen +Zaehlerzeitpunkt, abgedeckte Sekunden und Entscheidungszeitpunkt exakt passen; +veraltete Energie wird nicht hochgerechnet oder als aktuelle Messung ausgegeben. +Die kumulative kWh-Variable allein erfuellt diesen Vertrag noch nicht. + +Die V4-API meldet bei fehlendem Monatspeak/quarterPast weiterhin awaiting_inputs. +Ein HTTP stored bestaetigt nur den Eingang der Telemetrie, nicht einen gueltigen +Fahrplan und schon gar nicht dessen Ausfuehrung. Backend und Portal-Bruecke +muessen separat aktiviert werden. Keine neue Modellwahl oder Aktorfreigabe hier. + +## Tests + +24 offline Konvertierungsszenarien und 20 Quellen-/Summen-Szenarien sowie PHPUnit-Strukturpruefungen. Werte sind +synthetisch; keine reale HTTP-Anbindung oder Symcon-Laufzeit damit behauptet. diff --git a/examples/V4ApplicationData/application-class-loading.patch b/examples/V4ApplicationData/application-class-loading.patch new file mode 100644 index 0000000..5b39d69 --- /dev/null +++ b/examples/V4ApplicationData/application-class-loading.patch @@ -0,0 +1,71 @@ +--- a/install.php ++++ b/install.php +@@ -19,6 +19,8 @@ + } + $before=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $report['files']=v4StageInstall(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); ++ require_once __DIR__.'/installed_capture_bootstrap.php'; ++ $report['classLoading']=v4ApplicationLoadInstalledCapture('/var/lib/symcon/modules/Enelix-EMS',__DIR__.'/MANIFEST.json'); + $controls=[]; + foreach (IPS_GetModuleList() as $mid) if ((IPS_GetModule($mid)['Prefix']??'')==='MC') $controls=array_merge($controls,IPS_GetInstanceListByModuleID($mid)); + if (count($controls)!==1||!function_exists('MC_ReloadModule')) throw new RuntimeException('Module Control nicht eindeutig.'); +@@ -38,7 +40,7 @@ + $marker=$dir.'/observer-import.json'; + if (!is_file($marker)&&!IPS_GetProperty($manager,'NetzfahrplanV4MessdatenAktiv')) { + if (glob($dir.'/raw-*.jsonl')!==[]) throw new RuntimeException('Messdaten ohne Importabschluss vorhanden; nicht doppelt importieren.'); +- require_once __DIR__.'/source/libs/NetzfahrplanV4Messaufnahme.php'; ++ // Reuse the hash-checked installed class; never load the staged copy. + $count=0;$last=0;$cutoff=time()-172800; + $files=glob('/srv/agent/netplan-v4-separated-observer-stage/data/raw-*.jsonl');sort($files,SORT_STRING); + foreach ($files as $file) { +--- /dev/null ++++ b/installed_capture_bootstrap.php +@@ -0,0 +1,48 @@ ++ 'libs/NetzfahrplanV4Bilanzierung.php', ++ 'Belevo\\EnelixEMS\\NetzfahrplanV4Messaufnahme' => 'libs/NetzfahrplanV4Messaufnahme.php', ++ ]; ++ // Check BOTH origins before loading either file, including the transitive dependency. ++ foreach ($classes as $class => $relative) { ++ $path = $target . '/' . $relative; ++ $expected = $manifest['files'][$relative]['after'] ?? null; ++ if (!is_string($expected) || !preg_match('/^[a-f0-9]{64}$/D', $expected) ++ || is_link($path) || !is_file($path) || realpath($path) !== $path ++ || hash_file('sha256', $path) !== $expected) { ++ throw new RuntimeException('Installierte Messbibliothek weicht vom geprueften Paket ab: ' . $relative); ++ } ++ if (class_exists($class, false)) { ++ $origin = (new ReflectionClass($class))->getFileName(); ++ if ($origin !== $path) { ++ throw new RuntimeException('Messbibliothek bereits aus anderem Pfad geladen. Installationsaufruf separat ausfuehren, ohne weitere Sammler-Includes.'); ++ } ++ } ++ } ++ foreach ($classes as $class => $relative) { ++ $path = $target . '/' . $relative; ++ if (!class_exists($class, false)) { ++ require_once $path; ++ } ++ if (!class_exists($class, false) || (new ReflectionClass($class))->getFileName() !== $path) { ++ throw new RuntimeException('Installierte Messbibliothek konnte nicht eindeutig geladen werden.'); ++ } ++ } ++ return ['source' => 'installed_module_only', 'stagedClassesLoaded' => false, ++ 'files' => array_values($classes)]; ++} diff --git a/examples/V4ConfirmedFeedback/MANIFEST.json b/examples/V4ConfirmedFeedback/MANIFEST.json new file mode 100644 index 0000000..0984be8 --- /dev/null +++ b/examples/V4ConfirmedFeedback/MANIFEST.json @@ -0,0 +1,41 @@ +{ + "scope": "confirmed_feedback_trial_disabled", + "managerId": 17004, + "batteryInstanceId": 44234, + "files": { + "libs/NetzfahrplanV4Rueckmeldung.php": { + "before": "a7459a208c4dd84ffe3a166d266dcd00216f3660fe60e5ee61d5042d0b840631", + "after": "0b261f6534c5ca9864018ea4c2caf97e6ce6913d92f13dcc926383b4f401a673" + }, + "libs/NetzfahrplanV4Geraeteabruf.php": { + "before": null, + "after": "2b69a2ed27554139db6c996c906a4afe430918381e6359d23e8bb3369cd9be47" + }, + "libs/BatterieNetzfahrplanV4RueckmeldungTrait.php": { + "before": "24cf0654f1a6bd545d64ce5f115f309b16d8ffca5172ec64fb37c390c6f001a7", + "after": "1de81499da07ad814c98ba778ca728e34817bdfcf29cc7e518f7b5f8fd7733b7" + } + }, + "testSupport": { + "libs/BatterieNetzfahrplanV4TestTrait.php": "80474ff24cf9b2d32dfe481f0ce26bb94584c458b0fa3de232dec3897b64a4b2", + "libs/NetzfahrplanV4Regeltest.php": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249", + "libs/ManagerNetzfahrplanV4TestTrait.php": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18", + "libs/NetzfahrplanV4Planpruefung.php": "24b935d80bad4ce2d9047b6777aaf74a18773236ca91a2844a0907012f4fadf4" + }, + "tests": { + "tests/V4Feedback/checks.php": "b1aacc540ba11adabdd140aaded9e2968d1328f261ca15756e6002db3427330a", + "tests/V4Feedback/device_read_checks.php": "5c5d9e50dd1d0f80cb01a2d95e63018eeec558ba78c07afde208e3c0ef304cc3", + "tests/V4ControlTrial/checks.php": "2f12e94aeb207e6939a645f2f45b5f54e57c18530360ed3b16aa52ff89a1ef39", + "tests/V4ControlTrial/additional_checks.php": "df01119f2b9c0befe1d14ae6b6ddf39c75d3687b669142e9b24b5f9f4821f2c3", + "tests/V4ControlTrial/feedback_checks.php": "e87d41dff8e3ecd414a59b3dd7a234b34183d37acca8cddaf847b82eae92e60c", + "tests/V4Receiver/fixture.php": "0eeb65e63250aff5ca364a82c2b2a71f9f229be2d9d412e3037014979deb793a" + }, + "configurationSha256": "e393c1f01b30febc1d0785233474be43e98d849b152b4b2450b72efa6b73d540", + "dependencies": { + "libs/NetzfahrplanV4Regeltest.php": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249", + "libs/ManagerNetzfahrplanV4TestTrait.php": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18", + "libs/ManagerNetzfahrplanV4EmpfangTrait.php": "936939a3e4a5a5d54fdff68f6fc689a9400e8406aecd5ac09ebc8a6b569c0b66", + "Batterie/module.php": "3d5fcd83e4e7d999d5ac949fe357b492d5661ffa76639d2ca57a3f3934eabdbe", + "Manager/module.php": "cbbb33a8f71dda8cc80a4fc01cd3d6154da8153b5523306b2da723bb44f2dd79" + } +} diff --git a/examples/V4ConfirmedFeedback/README.md b/examples/V4ConfirmedFeedback/README.md new file mode 100644 index 0000000..7c00342 --- /dev/null +++ b/examples/V4ConfirmedFeedback/README.md @@ -0,0 +1,18 @@ +# V4 confirmed feedback for Lihrenmoos + +This incremental package adds synchronous device reads before V4 feedback is evaluated. + +- Grid input 40348 is refreshed through its M-Bus device. +- GoodWe and SolarEdge inputs are refreshed through their ModBus parent instances. +- A successful driver call is required; cached variable timestamps alone are not accepted. +- Daniel Haefliger accepted the existing `virtual_split` model for the bounded field test on 2026-10-04. +- The installer keeps the legacy schedule and both V4 trial permissions disabled. +- A real control trial still requires separate server authorization and evidence of an independent device-side watchdog. + +Build a private staging directory: + +```sh +python3 examples/V4ConfirmedFeedback/build_stage.py /srv/agent/netplan-v4-confirmed-feedback +``` + +Run the staged PHP checks before copying the package to the test plant. Execute `install.php` only from the IP-Symcon script editor on Lihrenmoos. diff --git a/examples/V4ConfirmedFeedback/build_stage.py b/examples/V4ConfirmedFeedback/build_stage.py new file mode 100644 index 0000000..55a129f --- /dev/null +++ b/examples/V4ConfirmedFeedback/build_stage.py @@ -0,0 +1,49 @@ +"""Create the reviewed Lihrenmoos confirmed-feedback package; never installs into Symcon.""" +from pathlib import Path +import hashlib +import json +import sys + +here = Path(__file__).resolve().parent +repo = here.parents[1] +if len(sys.argv) != 2: + raise SystemExit("Usage: python3 build_stage.py NEW_PRIVATE_STAGING_DIRECTORY") +target = Path(sys.argv[1]).resolve() +allowed_roots = [Path("/srv/agent"), Path("/home/agent/services/qa")] +if not any(target != root and target.is_relative_to(root) for root in allowed_roots): + raise SystemExit("Use a private agent staging directory, not live module paths") +if target.exists(): + raise SystemExit("Destination must not exist") + +manifest = json.loads((here / "MANIFEST.json").read_text()) +contents = {} +for group in ("files", "testSupport", "tests"): + for name, record in manifest[group].items(): + expected = record["after"] if group == "files" else record + source = repo / name + if source.is_symlink() or not source.resolve().is_relative_to(repo): + raise SystemExit("Unsafe source: " + name) + data = source.read_bytes() + if hashlib.sha256(data).hexdigest() != expected: + raise SystemExit("Reviewed source changed: " + name) + contents["source/" + name] = data + +for name, expected in manifest["dependencies"].items(): + if hashlib.sha256((repo / name).read_bytes()).hexdigest() != expected: + raise SystemExit("Dependency changed: " + name) + +for name in ("install.php", "file_installer.php", "MANIFEST.json", "feedback-config.json", "README.md"): + contents[name] = (here / name).read_bytes() +if hashlib.sha256(contents["feedback-config.json"]).hexdigest() != manifest["configurationSha256"]: + raise SystemExit("Configuration changed") + +target.mkdir(parents=True, mode=0o700) +for name, data in contents.items(): + path = target / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o640) +(target / "PACKAGE_HASHES.json").write_text( + json.dumps({name: hashlib.sha256(data).hexdigest() for name, data in contents.items()}, indent=2) + "\n" +) +print("Private confirmed-feedback package created:", target, "; both trial permissions remain disabled.") diff --git a/examples/V4ConfirmedFeedback/feedback-config.json b/examples/V4ConfirmedFeedback/feedback-config.json new file mode 100644 index 0000000..73a9b4e --- /dev/null +++ b/examples/V4ConfirmedFeedback/feedback-config.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "assetId": "anlage01-virtual-ev", + "managerId": 17004, + "batteryInstanceId": 44234, + "mode": "virtual_split", + "allowEstimatedForTrial": true, + "maxSkewSeconds": 30, + "maxAgeSeconds": 60, + "idleToleranceW": 50.0, + "trackingToleranceW": 200.0, + "sources": [ + { + "key": "grid", + "variableId": 40348, + "parentId": 11490, + "ident": "Power_8", + "factorToW": 1000, + "role": "grid" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "parentId": 19742, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "parentId": 57658, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "parentId": 30789, + "ident": "Value", + "factorToW": 1, + "role": "physical" + }, + { + "key": "ev_requested", + "variableId": 19651, + "parentId": 58448, + "ident": "Nennleistung_Soll_EV", + "factorToW": 1.0, + "role": "ev_request" + }, + { + "key": "sdl_requested", + "variableId": 38943, + "parentId": 58448, + "ident": "Nennleistung_Soll_SDL", + "factorToW": 1.0, + "role": "sdl_request" + }, + { + "key": "gateway_active", + "role": "gateway_active", + "variableId": 23483, + "parentId": 58448, + "ident": "State", + "factorToW": 1.0 + } + ] +} diff --git a/examples/V4ConfirmedFeedback/file_installer.php b/examples/V4ConfirmedFeedback/file_installer.php new file mode 100644 index 0000000..92b37cd --- /dev/null +++ b/examples/V4ConfirmedFeedback/file_installer.php @@ -0,0 +1,127 @@ + $hash) { + if (!preg_match('~^(libs/[A-Za-z0-9]+\\.php|Batterie/module\\.php|Manager/module\\.php)$~D', $name) + || is_link($target . '/' . $name) + || hash_file('sha256', $target . '/' . $name) !== $hash) { + throw new RuntimeException('Abhaengigkeit wurde parallel geaendert: ' . $name); + } + } + + $before = []; + $source = []; + $already = true; + foreach ($allowed as $name) { + $path = $target . '/' . $name; + $candidate = $stage . '/source/' . $name; + if (is_link($path) || is_link($candidate) || realpath(dirname($path)) !== $target . '/' . dirname($name)) { + throw new RuntimeException('Unerwarteter Dateipfad.'); + } + $source[$name] = (string) file_get_contents($candidate); + if (hash('sha256', $source[$name]) !== $manifest['files'][$name]['after']) { + throw new RuntimeException('Paketpruefsumme geaendert: ' . $name); + } + token_get_all($source[$name], TOKEN_PARSE); + $before[$name] = is_file($path) ? file_get_contents($path) : null; + $hash = $before[$name] === null ? null : hash('sha256', $before[$name]); + if (!in_array($hash, [$manifest['files'][$name]['before'], $manifest['files'][$name]['after']], true)) { + throw new RuntimeException('Paralleler Modulstand; nichts ueberschrieben: ' . $name); + } + if ($hash !== $manifest['files'][$name]['after']) { + $already = false; + } + } + if ($already) { + return ['status' => 'already_installed', 'backup' => null, 'filesChanged' => []]; + } + + $backup = $stage . '/backups/' . gmdate('Ymd\\THis\\Z') . '-' . bin2hex(random_bytes(4)); + if (!mkdir($backup, 0700, true)) { + throw new RuntimeException('Sicherung fehlgeschlagen.'); + } + foreach ($allowed as $name) { + if ($before[$name] !== null) { + $destination = $backup . '/' . $name; + if (!is_dir(dirname($destination))) { + mkdir(dirname($destination), 0700, true); + } + v4ConfirmedWrite($destination, $before[$name], 0600); + } + } + v4ConfirmedWrite( + $backup . '/MANIFEST.json', + json_encode($manifest, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT) . "\n", + 0600 + ); + + $written = []; + try { + foreach ($allowed as $name) { + if ((is_file($target . '/' . $name) ? file_get_contents($target . '/' . $name) : null) !== $before[$name]) { + throw new RuntimeException('Parallele Aenderung waehrend Installation.'); + } + if ($before[$name] === $source[$name]) { + continue; + } + v4ConfirmedWrite($target . '/' . $name, $source[$name]); + $written[] = $name; + } + } catch (Throwable $error) { + foreach (array_reverse($written) as $name) { + $path = $target . '/' . $name; + if (!is_link($path) && is_file($path) + && hash_file('sha256', $path) === $manifest['files'][$name]['after']) { + if ($before[$name] === null) { + unlink($path); + } else { + v4ConfirmedWrite($path, $before[$name]); + } + } + } + throw $error; + } + + return ['status' => 'installed', 'backup' => $backup, 'filesChanged' => $written]; +} diff --git a/examples/V4ConfirmedFeedback/install.php b/examples/V4ConfirmedFeedback/install.php new file mode 100644 index 0000000..4463d31 --- /dev/null +++ b/examples/V4ConfirmedFeedback/install.php @@ -0,0 +1,161 @@ + 'confirmed_feedback_trial_disabled', + 'startedAt' => gmdate('c'), + 'actuatorPermissionGranted' => false, + 'serverUpdateRequired' => true, + 'acceptedModel' => 'virtual_split', +]; + +try { + $hashes = json_decode((string) file_get_contents(__DIR__ . '/PACKAGE_HASHES.json'), true, 32, JSON_THROW_ON_ERROR); + foreach ($hashes as $name => $hash) { + if (str_contains($name, '..') || str_starts_with($name, '/') || is_link(__DIR__ . '/' . $name) + || hash_file('sha256', __DIR__ . '/' . $name) !== $hash) { + throw new RuntimeException('Vorbereitetes Paket veraendert.'); + } + } + foreach ([ + $manager => '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}', + $battery => '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}', + ] as $id => $module) { + if (!IPS_InstanceExists($id) || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== $module) { + throw new RuntimeException('Modulzuordnung ungueltig.'); + } + if (IPS_HasChanges($id)) { + throw new RuntimeException('Offene Aenderungen zuerst speichern oder verwerfen.'); + } + } + + $beforeManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $beforeBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + if (($beforeManager['NetzfahrplanAktiv'] ?? null) !== false + || ($beforeManager['NetzfahrplanV4RegeltestErlaubt'] ?? false) !== false + || ($beforeBattery['NetzfahrplanV4RegeltestErlaubt'] ?? false) !== false) { + throw new RuntimeException('Alte Fahrplanausgabe und beide Regeltestfreigaben muessen AUS bleiben.'); + } + + $configuration = json_decode((string) file_get_contents(__DIR__ . '/feedback-config.json'), true, 32, JSON_THROW_ON_ERROR); + $manifest = json_decode((string) file_get_contents(__DIR__ . '/MANIFEST.json'), true, 32, JSON_THROW_ON_ERROR); + if (($configuration['managerId'] ?? null) !== $manager + || ($configuration['batteryInstanceId'] ?? null) !== $battery + || ($configuration['mode'] ?? null) !== 'virtual_split' + || ($configuration['allowEstimatedForTrial'] ?? null) !== true + || hash_file('sha256', __DIR__ . '/feedback-config.json') !== $manifest['configurationSha256']) { + throw new RuntimeException('Falsche Rueckmeldekonfiguration.'); + } + $desired = json_encode($configuration, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION); + $previousConfiguration = $configuration; + $previousConfiguration['allowEstimatedForTrial'] = false; + $previousDesired = json_encode($previousConfiguration, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION); + $oldMapping = $beforeBattery['NetzfahrplanV4RueckmeldungKonfiguration'] ?? '{}'; + if (!in_array($oldMapping, [$previousDesired, $desired], true)) { + throw new RuntimeException('Abweichende Rueckmeldekonfiguration nicht ueberschrieben.'); + } + + $controls = []; + foreach (IPS_GetModuleList() as $module) { + if ((IPS_GetModule($module)['Prefix'] ?? '') === 'MC') { + $controls = array_merge($controls, IPS_GetInstanceListByModuleID($module)); + } + } + if (count($controls) !== 1 || !function_exists('MC_ReloadModule')) { + throw new RuntimeException('Module Control nicht eindeutig.'); + } + $locked = IPS_SemaphoreEnter('ENELIX.V4.ConfirmedFeedbackInstall', 3000); + if (!$locked) { + throw new RuntimeException('Installation bereits aktiv.'); + } + + $report['files'] = v4ConfirmedFiles(__DIR__, '/var/lib/symcon/modules/Enelix-EMS'); + if (MC_ReloadModule($controls[0], 'Enelix-EMS') === false) { + throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + } + + $afterManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $afterBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + foreach ([[$beforeManager, $afterManager], [$beforeBattery, $afterBattery]] as $pair) { + foreach ($pair[0] as $key => $value) { + if (!array_key_exists($key, $pair[1]) || $pair[1][$key] !== $value) { + throw new RuntimeException('Bestehender Parameter beim Reload veraendert: ' . $key); + } + } + } + + if (!function_exists('ENELIX_GetV4BatterieRueckmeldung')) { + $report['status'] = 'waiting_for_registration'; + echo "Dateien installiert. Modulregistrierung wird abgeschlossen; diesen Aufruf einmal wiederholen. Keine Stellfreigabe.\n"; + } else { + if ($afterBattery['NetzfahrplanV4RueckmeldungKonfiguration'] !== $desired) { + IPS_SetProperty($battery, 'NetzfahrplanV4RueckmeldungKonfiguration', $desired); + $changedMapping = true; + IPS_ApplyChanges($battery); + } + $finalManager = json_decode(IPS_GetConfiguration($manager), true, 128, JSON_THROW_ON_ERROR); + $finalBattery = json_decode(IPS_GetConfiguration($battery), true, 128, JSON_THROW_ON_ERROR); + foreach ([[$beforeManager, $finalManager], [$beforeBattery, $finalBattery]] as $pair) { + foreach ($pair[0] as $key => $value) { + if ($key !== 'NetzfahrplanV4RueckmeldungKonfiguration' + && (!array_key_exists($key, $pair[1]) || $pair[1][$key] !== $value)) { + throw new RuntimeException('Bestehender Parameter unerwartet veraendert: ' . $key); + } + } + } + if (($finalManager['NetzfahrplanAktiv'] ?? null) !== false + || ($finalManager['NetzfahrplanV4RegeltestErlaubt'] ?? null) !== false + || ($finalBattery['NetzfahrplanV4RegeltestErlaubt'] ?? null) !== false) { + throw new RuntimeException('Unerwartete Stellfreigabe nach Installation.'); + } + + $feedback = json_decode(ENELIX_GetV4BatterieRueckmeldung($battery), true, 32, JSON_THROW_ON_ERROR); + $report['feedback'] = array_intersect_key($feedback, array_flip([ + 'status', 'kind', 'method', 'reason', 'reasons', 'estimated', 'batteryW', 'gridW', + 'sourceOldestAt', 'sourceVariableOldestAt', 'confirmedAt', 'checkedAt', 'configHash', + 'deviceReadConfirmed', 'usableForTrial', 'canDispatch', + ])); + $report['status'] = 'confirmed_feedback_installed_trial_disabled'; + $report['originalParametersPreserved'] = true; + echo "OK: Geraeteabfrage bestaetigt die physischen Messwerte; virtual_split ist fuer den begrenzten Regeltest akzeptiert.\n"; + echo 'Rueckmeldung: ' . ($feedback['status'] ?? 'available') + . '; Geraeteabfrage: ' . (($feedback['deviceReadConfirmed'] ?? false) ? 'bestaetigt' : 'nicht bestaetigt') + . '; Modellanteil: ' . (($feedback['estimated'] ?? false) ? 'ja' : 'nein') . ".\n"; + if (isset($feedback['reason'])) { + echo 'Aktueller Hinweis: ' . $feedback['reason'] . "\n"; + } + echo "Beide Regeltestfreigaben AUS. Kein Stellbefehl ohne separat nachgewiesenen Geraete-Watchdog und Serverfreigabe.\n"; + } +} catch (Throwable $error) { + $report['status'] = 'needs_review'; + $report['reason'] = substr($error->getMessage(), 0, 260); + if ($changedMapping && $oldMapping !== null) { + try { + IPS_SetProperty($battery, 'NetzfahrplanV4RueckmeldungKonfiguration', $oldMapping); + IPS_ApplyChanges($battery); + $report['mappingRestored'] = true; + } catch (Throwable $ignored) { + $report['mappingRestored'] = false; + } + } + echo 'FEHLER: ' . $report['reason'] . "\nKeine V4-Stellfreigabe erteilt.\n"; +} finally { + $report['finishedAt'] = gmdate('c'); + v4ConfirmedWrite(__DIR__ . '/INSTALL_RESULT.json', json_encode($report, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT) . "\n", 0644); + if ($locked) { + IPS_SemaphoreLeave('ENELIX.V4.ConfirmedFeedbackInstall'); + } + echo 'CONFIRMED FEEDBACK INSTALL REPORT: ' . __DIR__ . "/INSTALL_RESULT.json\n"; +} diff --git a/examples/V4CorrectedFeedback/MANIFEST.json b/examples/V4CorrectedFeedback/MANIFEST.json new file mode 100644 index 0000000..a9e7c3b --- /dev/null +++ b/examples/V4CorrectedFeedback/MANIFEST.json @@ -0,0 +1,65 @@ +{ + "scope": "corrected_feedback_trial_disabled", + "managerId": 17004, + "batteryInstanceId": 44234, + "files": { + "libs/BatterieNetzfahrplanV4RueckmeldungTrait.php": { + "before": null, + "after": "24cf0654f1a6bd545d64ce5f115f309b16d8ffca5172ec64fb37c390c6f001a7" + }, + "libs/BatterieNetzfahrplanV4TestTrait.php": { + "before": null, + "after": "80474ff24cf9b2d32dfe481f0ce26bb94584c458b0fa3de232dec3897b64a4b2" + }, + "libs/BatterieRegler.php": { + "before": "035988c54d4d0cc3a59d234f1c08a8edbf5fcac42e09a6bcea61d3dd8270771b", + "after": "beecc9c817b722336367e1c574c1a38afc8f3f5346442170d67c101b980adf7b" + }, + "libs/ManagerNetzfahrplanV4EmpfangTrait.php": { + "before": "d083cfa6ae8e8026c134d2b53f34bb6fbe342fb7332a3e7da1b881de33e13e97", + "after": "936939a3e4a5a5d54fdff68f6fc689a9400e8406aecd5ac09ebc8a6b569c0b66" + }, + "libs/ManagerNetzfahrplanV4TestTrait.php": { + "before": null, + "after": "c0d9b6336574dd8acf5f84adb12a32cad3b42ae91c5a31d513d0bc638a560f18" + }, + "libs/NetzfahrplanV4Regeltest.php": { + "before": null, + "after": "bdac3e2e54c50a4c1541da8918ddf3ab2f665bb5cc6696848f546e7b05f5f249" + }, + "libs/NetzfahrplanV4Rueckmeldung.php": { + "before": null, + "after": "a7459a208c4dd84ffe3a166d266dcd00216f3660fe60e5ee61d5042d0b840631" + }, + "Batterie/module.php": { + "before": "34503645f7e21dd1e5bf0a3f754f6592175dd67f9a263bf431f3801209555401", + "after": "3d5fcd83e4e7d999d5ac949fe357b492d5661ffa76639d2ca57a3f3934eabdbe" + }, + "Manager/module.php": { + "before": "83e80883c8f9b817c0af5e239b20bcb019c7cf7f7943ee5bcc6493ac5a743532", + "after": "cbbb33a8f71dda8cc80a4fc01cd3d6154da8153b5523306b2da723bb44f2dd79" + } + }, + "configurationSha256": "f66cfb9ebf5e02259b902f876a4c78756a51b3f2f2c240fd92c0f01161268c56", + "dependencies": { + "libs/NetzfahrplanV4Planpruefung.php": "24b935d80bad4ce2d9047b6777aaf74a18773236ca91a2844a0907012f4fadf4", + "libs/Nachrichtenvertrag.php": "215ad7d4c5240b39b48cb7672f5057afa532aafdb34039862a1a22985a9ee839", + "libs/VerbraucherBasisTrait.php": "10fd9f54283c378104bce11d623d5581558355d26fc2f85bf04b316542874507", + "libs/VerbraucherSchnittstelle.php": "6329d423dca7b551f01fd884204be45be43f92cd6da6e4faedf2c3c9b4654a15", + "libs/ManagerNetzfahrplanV4Trait.php": "e27a3595e4ed8cdb62e2561149ef6a50adb9194e0239eb25c1216aee2716fa88", + "libs/ManagerNetzfahrplanV4DatenTrait.php": "59ae0ae283839ca82c58db9c071dcf28d0ab95e44f7a74c6473aca837ce416c4", + "libs/NetzfahrplanV4Datenarchiv.php": "57ef7a60c273c5270b3e93ff5528d7d294e708725c9edfe81ac7ed819587f983", + "libs/NetzfahrplanV4Messaufnahme.php": "ebe873bb714041e505a4f500b34c9bae3a5a0b93ac459161584ced758507a6e9", + "libs/NetzfahrplanV4Bilanzierung.php": "ffda2a03dbb226a4405a2ad66704c080a9dc475ce64282ad3f0d7d60034ab199", + "libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "libs/ManagerEnergieTrait.php": "2f89b0ad31ee6dd99180a7225173036eb6212b2ea7157e57fee23f0f8b441a51", + "libs/EnergieMessung.php": "16c64e1cbefde81c25f913aaca6546f0d30cc8f2fddecea0768a3760a98e97d2", + "libs/StoerungsSnapshot.php": "c871c1dd58d7fb1ef6c06985de066518688ef080f3cbdadda192d66cedcf49e1", + "libs/Anlagentopologie.php": "1b7e95d103a3048ed38c464a424f2f12821a147b269e149fd6b89bca84878c47", + "libs/Lizenzpruefung.php": "cb93cb49b93ed3ebb2060dba28b5b2dec2f29548d6ccff1afd9c7cb4624ad77e", + "libs/EinspeiseRegler.php": "8b982be5b95275c6dd6611a2bf2b1f89b991ba53cf94e46df0eed3249add65e0", + "libs/ManagerRegler.php": "608a4abebbe45da37a16316a530fa3f5ab21cb880bb341acbdcd3a9aaa93c6ee", + "libs/ManagerSchnittstelle.php": "0aae8c8c85eb252c577fcb0b5a50a12f3247e4a0bb083579122854f6c35878b3" + } +} diff --git a/examples/V4CorrectedFeedback/build_stage.py b/examples/V4CorrectedFeedback/build_stage.py new file mode 100644 index 0000000..e9ab97c --- /dev/null +++ b/examples/V4CorrectedFeedback/build_stage.py @@ -0,0 +1,25 @@ +"""Recreate a reviewed Lihrenmoos package from this commit; never install in Symcon.""" +from pathlib import Path +import hashlib,json,sys +here=Path(__file__).resolve().parent;repo=here.parents[1] +if len(sys.argv)!=2:raise SystemExit('Usage: python3 build_stage.py NEW_PRIVATE_STAGING_DIRECTORY') +target=Path(sys.argv[1]).resolve() +allowed=[Path('/srv/agent'),Path('/home/agent/services/qa')] +if not any(target!=p and target.is_relative_to(p) for p in allowed):raise SystemExit('Use a private agent staging directory, not live module paths') +if target.exists():raise SystemExit('Destination must not exist') +m=json.loads((here/'MANIFEST.json').read_text());contents={} +for n,v in m['files'].items(): + p=repo/n + if p.is_symlink() or not p.resolve().is_relative_to(repo):raise SystemExit('Unsafe source') + data=p.read_bytes() + if hashlib.sha256(data).hexdigest()!=v['after']:raise SystemExit('Reviewed source changed: '+n) + contents['source/'+n]=data +for n,v in m['dependencies'].items(): + if hashlib.sha256((repo/n).read_bytes()).hexdigest()!=v:raise SystemExit('Dependency changed: '+n) +for n in ['install.php','file_installer.php','MANIFEST.json','feedback-config.json']:contents[n]=(here/n).read_bytes() +if hashlib.sha256(contents['feedback-config.json']).hexdigest()!=m['configurationSha256']:raise SystemExit('Configuration changed') +target.mkdir(parents=True,mode=0o700) +for n,data in contents.items(): + p=target/n;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(data);p.chmod(0o640) +(target/'PACKAGE_HASHES.json').write_text(json.dumps({n:hashlib.sha256(v).hexdigest() for n,v in contents.items()},indent=2)+'\n') +print('Private staging package created:',target,'; no running module, settings or control changes.') diff --git a/examples/V4CorrectedFeedback/feedback-config.json b/examples/V4CorrectedFeedback/feedback-config.json new file mode 100644 index 0000000..4b5f04a --- /dev/null +++ b/examples/V4CorrectedFeedback/feedback-config.json @@ -0,0 +1,71 @@ +{ + "version": 1, + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "assetId": "anlage01-virtual-ev", + "managerId": 17004, + "batteryInstanceId": 44234, + "mode": "virtual_split", + "allowEstimatedForTrial": false, + "maxSkewSeconds": 30, + "maxAgeSeconds": 60, + "idleToleranceW": 50.0, + "trackingToleranceW": 200.0, + "sources": [ + { + "key": "grid", + "variableId": 40348, + "parentId": 11490, + "ident": "Power_8", + "factorToW": 1000, + "role": "grid" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "parentId": 19742, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "parentId": 57658, + "ident": "A_6_3_35182", + "factorToW": -1, + "role": "physical" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "parentId": 30789, + "ident": "Value", + "factorToW": 1, + "role": "physical" + }, + { + "key": "ev_requested", + "variableId": 19651, + "parentId": 58448, + "ident": "Nennleistung_Soll_EV", + "factorToW": 1.0, + "role": "ev_request" + }, + { + "key": "sdl_requested", + "variableId": 38943, + "parentId": 58448, + "ident": "Nennleistung_Soll_SDL", + "factorToW": 1.0, + "role": "sdl_request" + }, + { + "key": "gateway_active", + "role": "gateway_active", + "variableId": 23483, + "parentId": 58448, + "ident": "State", + "factorToW": 1.0 + } + ] +} diff --git a/examples/V4CorrectedFeedback/file_installer.php b/examples/V4CorrectedFeedback/file_installer.php new file mode 100644 index 0000000..a5ba371 --- /dev/null +++ b/examples/V4CorrectedFeedback/file_installer.php @@ -0,0 +1,66 @@ +$hash) { + if (!preg_match('~^libs/[A-Za-z0-9]+\.php$~D',$name) || is_link($target.'/'.$name) || hash_file('sha256',$target.'/'.$name)!==$hash) throw new RuntimeException('Abhaengigkeit wurde parallel geaendert: '.$name); + } + $before=[];$source=[];$already=true; + // Explicit dependency order. Both top-level modules are replaced only after all dependencies. + foreach($allowed as$name) { + $path=$target.'/'.$name;$candidate=$stage.'/source/'.$name; + if (is_link($path)||is_link($candidate)||realpath(dirname($path))!==$target.'/'.dirname($name)) throw new RuntimeException('Unerwarteter Dateipfad.'); + $source[$name]=(string)file_get_contents($candidate); + if (hash('sha256',$source[$name])!==$m['files'][$name]['after']) throw new RuntimeException('Paketpruefsumme geaendert: '.$name); + token_get_all($source[$name],TOKEN_PARSE); + $before[$name]=is_file($path)?file_get_contents($path):null; + $hash=$before[$name]===null?null:hash('sha256',$before[$name]); + if (!in_array($hash,[$m['files'][$name]['before'],$m['files'][$name]['after']],true)) throw new RuntimeException('Paralleler Modulstand; nichts ueberschrieben: '.$name); + if($hash!==$m['files'][$name]['after'])$already=false; + } + if($already)return ['status'=>'already_installed','backup'=>null]; + $backup=$stage.'/backups/'.gmdate('Ymd\THis\Z').'-'.bin2hex(random_bytes(4)); + if(!mkdir($backup,0700,true))throw new RuntimeException('Sicherung fehlgeschlagen.'); + foreach($allowed as$name) { + if($before[$name]!==null) { + $dest=$backup.'/'.$name;if(!is_dir(dirname($dest)))mkdir(dirname($dest),0700,true); + v4FeedbackWrite($dest,$before[$name],0600); + } + } + v4FeedbackWrite($backup.'/MANIFEST.json',json_encode($m,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0600); + $written=[]; + try { + foreach($allowed as$name) { + if((is_file($target.'/'.$name)?file_get_contents($target.'/'.$name):null)!==$before[$name])throw new RuntimeException('Parallele Aenderung waehrend Installation.'); + if($before[$name]===$source[$name])continue; + v4FeedbackWrite($target.'/'.$name,$source[$name]);$written[]=$name; + } + } catch(Throwable $e) { + foreach(array_reverse($written)as$name) { + $p=$target.'/'.$name; + if(!is_link($p)&&is_file($p)&&hash_file('sha256',$p)===$m['files'][$name]['after']) { + if($before[$name]===null)unlink($p);else v4FeedbackWrite($p,$before[$name]); + } + } + throw $e; + } + return ['status'=>'installed','backup'=>$backup,'filesChanged'=>$written]; +} diff --git a/examples/V4CorrectedFeedback/install.php b/examples/V4CorrectedFeedback/install.php new file mode 100644 index 0000000..7f01294 --- /dev/null +++ b/examples/V4CorrectedFeedback/install.php @@ -0,0 +1,68 @@ +'corrected_feedback_trial_disabled','startedAt'=>gmdate('c'),'actuatorPermissionGranted'=>false,'serverUpdateRequired'=>false]; +try { + $hashes=json_decode((string)file_get_contents(__DIR__.'/PACKAGE_HASHES.json'),true,32,JSON_THROW_ON_ERROR); + foreach($hashes as$name=>$hash) { + if(str_contains($name,'..')||str_starts_with($name,'/')||is_link(__DIR__.'/'.$name)||hash_file('sha256',__DIR__.'/'.$name)!==$hash)throw new RuntimeException('Vorbereitetes Paket veraendert.'); + } + foreach([$manager=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}',$battery=>'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}']as$id=>$module) { + if(!IPS_InstanceExists($id)||IPS_GetInstance($id)['ModuleInfo']['ModuleID']!==$module)throw new RuntimeException('Modulzuordnung ungueltig.'); + if(IPS_HasChanges($id))throw new RuntimeException('Offene Aenderungen zuerst speichern oder verwerfen.'); + } + $beforeM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $beforeB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + if(($beforeM['NetzfahrplanAktiv']??null)!==false || ($beforeM['NetzfahrplanV4RegeltestErlaubt']??false)!==false || ($beforeB['NetzfahrplanV4RegeltestErlaubt']??false)!==false)throw new RuntimeException('Alte Fahrplanausgabe und beide Regeltestfreigaben muessen AUS bleiben.'); + $c=json_decode((string)file_get_contents(__DIR__.'/feedback-config.json'),true,32,JSON_THROW_ON_ERROR); + $manifest=json_decode((string)file_get_contents(__DIR__.'/MANIFEST.json'),true,32,JSON_THROW_ON_ERROR); + if(($c['managerId']??null)!==$manager || ($c['batteryInstanceId']??null)!==$battery || ($c['allowEstimatedForTrial']??null)!==false + || hash_file('sha256',__DIR__.'/feedback-config.json')!==$manifest['configurationSha256'])throw new RuntimeException('Falsche Rueckmeldekonfiguration.'); + $desired=json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION); + $oldMapping=$beforeB['NetzfahrplanV4RueckmeldungKonfiguration']??'{}'; + if(!in_array($oldMapping,['','{}',$desired],true))throw new RuntimeException('Abweichende Rueckmeldekonfiguration nicht ueberschrieben.'); + $controls=[];foreach(IPS_GetModuleList()as$module)if((IPS_GetModule($module)['Prefix']??'')==='MC')$controls=array_merge($controls,IPS_GetInstanceListByModuleID($module)); + if(count($controls)!==1||!function_exists('MC_ReloadModule'))throw new RuntimeException('Module Control nicht eindeutig.'); + $locked=IPS_SemaphoreEnter('ENELIX.V4.FeedbackInstall',3000);if(!$locked)throw new RuntimeException('Installation bereits aktiv.'); + $report['files']=v4FeedbackFiles(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); + if(MC_ReloadModule($controls[0],'Enelix-EMS')===false)throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + $afterM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $afterB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + foreach([[$beforeM,$afterM],[$beforeB,$afterB]]as$pair)foreach($pair[0]as$key=>$value)if(!array_key_exists($key,$pair[1])||$pair[1][$key]!==$value)throw new RuntimeException('Bestehender Parameter beim Reload veraendert: '.$key); + if(!array_key_exists('NetzfahrplanV4RueckmeldungKonfiguration',$afterB)||!array_key_exists('NetzfahrplanV4RegeltestErlaubt',$afterM) + ||!function_exists('ENELIX_GetV4BatterieRueckmeldung')||!function_exists('ENELIX_GetV4ManagerTestStatus')) { + $report['status']='waiting_for_registration'; + echo "Dateien installiert. Modulregistrierung wird abgeschlossen; diesen Aufruf einmal wiederholen. Keine Stellfreigabe.\n"; + } else { + if($afterM['NetzfahrplanV4RegeltestErlaubt']!==false||$afterB['NetzfahrplanV4RegeltestErlaubt']!==false)throw new RuntimeException('Unerwartete Testfreigabe.'); + if($afterB['NetzfahrplanV4RueckmeldungKonfiguration']!==$desired) { + IPS_SetProperty($battery,'NetzfahrplanV4RueckmeldungKonfiguration',$desired);$changedMapping=true;IPS_ApplyChanges($battery); + } + $finalM=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $finalB=json_decode(IPS_GetConfiguration($battery),true,128,JSON_THROW_ON_ERROR); + foreach([[$beforeM,$finalM],[$beforeB,$finalB]]as$pair)foreach($pair[0]as$key=>$value)if($key!=='NetzfahrplanV4RueckmeldungKonfiguration'&&(!array_key_exists($key,$pair[1])||$pair[1][$key]!==$value))throw new RuntimeException('Bestehender Parameter unerwartet veraendert: '.$key); + if(($finalM['NetzfahrplanAktiv']??null)!==false||($finalM['NetzfahrplanV4RegeltestErlaubt']??null)!==false||($finalB['NetzfahrplanV4RegeltestErlaubt']??null)!==false)throw new RuntimeException('Unerwartete Stellfreigabe nach Installation.'); + $f=json_decode(ENELIX_GetV4BatterieRueckmeldung($battery),true,32,JSON_THROW_ON_ERROR); + $report['feedback']=array_intersect_key($f,array_flip(['status','kind','method','reason','reasons','estimated','batteryW','gridW','sourceOldestAt','checkedAt','configHash','usableForTrial','canDispatch'])); + $report['status']='corrected_feedback_installed_trial_disabled';$report['originalParametersPreserved']=true; + echo "OK: Korrigierte physische Rueckmeldung mit Manager-Vorschau und Batterietreiber verbunden.\n"; + echo 'Rueckmeldung: '.($f['status']??'available').'; Modellanteil: '.(($f['estimated']??false)?'ja':'nein').".\n"; + if(isset($f['reason']))echo 'Aktueller Hinweis: '.$f['reason']."\n"; + echo "Beide Regeltestfreigaben AUS. Keine V4-Ausgabe aktiviert; vorhandene Regelung, Energiekonten und Datensender bleiben bestehen.\n"; + } +} catch(Throwable $e) { + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,260); + if($changedMapping && $oldMapping!==null) { + try{IPS_SetProperty($battery,'NetzfahrplanV4RueckmeldungKonfiguration',$oldMapping);IPS_ApplyChanges($battery);$report['mappingRestored']=true;}catch(Throwable $ignored){$report['mappingRestored']=false;} + } + echo 'FEHLER: '.$report['reason']."\nKeine V4-Stellfreigabe erteilt.\n"; +} finally { + $report['finishedAt']=gmdate('c');v4FeedbackWrite(__DIR__.'/INSTALL_RESULT.json',json_encode($report,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0644); + if($locked)IPS_SemaphoreLeave('ENELIX.V4.FeedbackInstall'); + echo 'FEEDBACK INSTALL REPORT: '.__DIR__."/INSTALL_RESULT.json\n"; +} diff --git a/examples/V4DataCapture/install.php b/examples/V4DataCapture/install.php new file mode 100644 index 0000000..fe2d8b3 --- /dev/null +++ b/examples/V4DataCapture/install.php @@ -0,0 +1,89 @@ + IPS_CreateCategory(), 2 => IPS_CreateVariable(3), 3 => IPS_CreateScript(0) }; + if (!is_int($id) || $id <= 0) throw new RuntimeException('Cannot create capture object'); + $created[] = $id; + if (!IPS_SetParent($id, $parent) || !IPS_SetIdent($id, $ident) || !IPS_SetName($id, $name) || !IPS_SetInfo($id, $marker)) { + throw new RuntimeException('Cannot label capture object'); + } + return $id; +} + +function v4captureScript(int $id, string $content, array $created): void +{ + $current = IPS_GetScriptContent($id); + if ($current === $content) return; + if (!in_array($id, $created, true)) throw new RuntimeException('Existing capture script changed; not overwritten'); + if (!IPS_SetScriptContent($id, $content) || IPS_GetScriptContent($id) !== $content) throw new RuntimeException('Cannot store capture script'); +} + +function v4captureInstall(string $stage): array +{ + if (!function_exists('IPS_GetKernelVersion') || version_compare(IPS_GetKernelVersion(),'8.0','<')) { + throw new RuntimeException('Nur im IP-Symcon-Skripteditor der Testanlage ausfuehren.'); + } + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + if (!IPS_InstanceExists($c['managerId']) + || IPS_GetInstance($c['managerId'])['ModuleInfo']['ModuleID'] !== '{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}') { + throw new RuntimeException('Expected manager not found'); + } + // Verify layout only; stale/missing sensor updates remain diagnostic raw data, not fake zeros. + foreach (array_merge($c['accounting']['sources'], $c['extraSources']) as $s) { + if (!IPS_VariableExists($s['variableId'])) throw new RuntimeException('Expected source missing: ' . $s['key']); + $o = IPS_GetObject($s['variableId']); $v = IPS_GetVariable($s['variableId']); + if ($o['ParentID'] !== $s['parentId'] || $o['ObjectIdent'] !== $s['ident'] || !in_array($v['VariableType'],[1,2],true)) { + throw new RuntimeException('Source identity changed: ' . $s['key']); + } + } + $data = $stage . '/data'; + if (is_link($data) || !is_dir($data) || !is_writable($data)) throw new RuntimeException('Dedicated capture directory not writable'); + $key = 'ENELIX.V4.CaptureInstall.' . $c['installationId']; + if (!IPS_SemaphoreEnter($key, 3000)) throw new RuntimeException('Capture installation busy'); + $created = []; $script = null; $timerSet = false; + $report = ['startedAt' => gmdate('c'), 'scope' => 'passive_raw_capture_only', 'controlEnabled' => false, + 'existingModulesChanged' => false, 'archivesChanged' => false, 'existingHistoriesChanged' => false]; + try { + $category = v4captureEnsure(0,'ENELIX_V4_PASSIVE_CAPTURE',0,'V4 - separate Messaufnahme (keine Steuerung)',$marker,$created); + v4captureEnsure($category,'CaptureStatus',2,'Aufzeichnungsstatus (nur lesen)',$marker,$created); + $script = v4captureEnsure($category,'RawCapture',3,'Messwerte alle 30 Sekunden erfassen',$marker,$created); + $stop = v4captureEnsure($category,'StopCapture',3,'Nur diese Messaufnahme stoppen',$marker,$created); + $include = ' 'capture_installed', 'categoryId' => $category, 'scriptId' => $script, + 'stopScriptId' => $stop, 'intervalSeconds' => 30, 'firstCapture' => $first, + 'futureTimerExecutionVerified' => false]; + echo "OK: Separate Rohdatenaufnahme eingerichtet, alle 30 Sekunden.\n"; + echo 'Erste Aufnahme: ' . $first['capturedAt'] . '; Quellen: ' . $first['sourceCount'] . ".\n"; + echo 'Bilanzqualitaet: ' . $first['accountingQuality'] . " (noch keine Trainings-/Stellfreigabe).\n"; + echo "Bestehende Manager-, Batterie-, Archiv- und Pollingkonfiguration unveraendert.\n"; + } catch (Throwable $e) { + $report['status'] = 'needs_review'; $report['reason'] = substr($e->getMessage(),0,300); + if ($timerSet && $script !== null) IPS_SetScriptTimer($script,0); + // Never delete unrelated objects or stored data during recovery. + echo 'FEHLER: ' . $report['reason'] . "\n"; + } finally { + $report['createdObjects'] = $created; $report['finishedAt'] = gmdate('c'); + try { v4captureAtomicJson($stage . '/INSTALL_RESULT.json', $report); } + finally { IPS_SemaphoreLeave($key); } + echo 'CAPTURE REPORT: ' . $stage . "/INSTALL_RESULT.json\n"; + } + return $report; +} diff --git a/examples/V4DataCapture/runtime.php b/examples/V4DataCapture/runtime.php new file mode 100644 index 0000000..7b0ba2a --- /dev/null +++ b/examples/V4DataCapture/runtime.php @@ -0,0 +1,143 @@ + GetValue($id), 'updated' => (int)$v['VariableUpdated'], 'changed' => (int)$v['VariableChanged'], + 'parentID' => $o['ParentID'], 'ident' => $o['ObjectIdent']]; +} + +function v4captureRun(string $stage, int $category): array +{ + if (!function_exists('IPS_GetKernelVersion')) throw new RuntimeException('Run inside IP-Symcon only'); + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + v4captureAssertObject($category, 0, 'ENELIX_V4_PASSIVE_CAPTURE', 0, $marker); + $statusId = v4captureChild($category, 'CaptureStatus'); + if ($statusId === null) throw new RuntimeException('Capture status missing'); + v4captureAssertObject($statusId, $category, 'CaptureStatus', 2, $marker); + $lock = 'ENELIX.V4.RawCapture.' . $c['installationId']; + if (!IPS_SemaphoreEnter($lock, 0)) return ['status' => 'busy', 'controlEnabled' => false]; + try { + try { + $record = NetzfahrplanV4Messaufnahme::capture($c, 'v4captureRead', static fn(): int => time()); + v4captureKeepSnapshot($stage . '/data/mapping-' . $record['mappingSha256'] . '.json', json_encode($c, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + v4captureKeepSnapshot($stage . '/data/inventory-' . $record['reportedInventorySha256'] . '.json', (string)file_get_contents($stage . '/USER_REPORTED_INVENTORY.json')); + $path = NetzfahrplanV4Messaufnahme::append($stage . '/data', $record); + $status = ['status' => 'recorded', 'capturedAt' => $record['capturedAt'], 'journal' => $path, + 'sourceCount' => count($record['raw']), 'rawIssueCount' => count($record['issues']), + 'accountingQuality' => $record['assessment']['status'] ?? 'unavailable', + 'qualityIssues' => $record['assessment']['issues'] ?? $record['issues'], + 'baseLoadCandidateW' => $record['assessment']['baseLoadCandidateW'] ?? null, + 'sourceSkewSeconds' => $record['assessment']['sourceSkewSeconds'] ?? null, + 'trainingEligible' => false, 'controlEnabled' => false, 'measurementBoundaryVerified' => false]; + v4captureAtomicJson($stage . '/data/STATUS.json', $status); + } catch (Throwable $e) { + $status = ['status' => 'capture_error', 'at' => gmdate('c'), 'reason' => substr($e->getMessage(),0,250), + 'trainingEligible' => false, 'controlEnabled' => false]; + try { v4captureAtomicJson($stage . '/data/STATUS.json', $status); } catch (Throwable $ignored) { } + } + // Only this newly created diagnosis variable, never a meter or controller variable. + SetValue($statusId, json_encode($status, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + return $status; + } finally { IPS_SemaphoreLeave($lock); } +} + +function v4captureStop(string $stage, int $category): void +{ + $c = v4captureConfig($stage); $marker = v4captureMarker($c); + v4captureAssertObject($category, 0, 'ENELIX_V4_PASSIVE_CAPTURE', 0, $marker); + $script = v4captureChild($category, 'RawCapture'); $status = v4captureChild($category, 'CaptureStatus'); + if ($script === null || $status === null) throw new RuntimeException('Capture objects missing'); + v4captureAssertObject($script, $category, 'RawCapture', 3, $marker); + v4captureAssertObject($status, $category, 'CaptureStatus', 2, $marker); + if (!IPS_SetScriptTimer($script, 0)) throw new RuntimeException('Cannot stop capture timer'); + $r = ['status' => 'stopped', 'at' => gmdate('c'), 'oldDataRetained' => true, 'controlEnabled' => false]; + SetValue($status, json_encode($r, JSON_THROW_ON_ERROR)); + v4captureAtomicJson($stage . '/data/STATUS.json', $r); +} diff --git a/examples/V4SeparatedObservation/install.php b/examples/V4SeparatedObservation/install.php new file mode 100644 index 0000000..902d259 --- /dev/null +++ b/examples/V4SeparatedObservation/install.php @@ -0,0 +1,59 @@ +IPS_CreateCategory(),2=>IPS_CreateVariable(3),3=>IPS_CreateScript(0)}; + if(!is_int($id)||$id<=0)throw new RuntimeException('Cannot create observer object');$created[]=$id; + if(!IPS_SetParent($id,$parent)||!IPS_SetIdent($id,$ident)||!IPS_SetName($id,$name)||!IPS_SetInfo($id,$marker))throw new RuntimeException('Cannot label observer object'); + return $id; +} +function v4viewsScript(int $id,string $source,array $created): void +{ + if(IPS_GetScriptContent($id)===$source)return; + if(!in_array($id,$created,true))throw new RuntimeException('Existing observer script differs; not overwritten'); + if(!IPS_SetScriptContent($id,$source)||IPS_GetScriptContent($id)!==$source)throw new RuntimeException('Cannot write observer script'); +} +function v4viewsInstall(string $stage): array +{ + if(!function_exists('IPS_GetKernelVersion')||version_compare(IPS_GetKernelVersion(),'8.0','<'))throw new RuntimeException('Nur im IP-Symcon-Skripteditor ausfuehren.'); + [$c,$p]=v4viewsConfig($stage);$marker=v4viewsMarker($c); + if(!IPS_InstanceExists($c['managerId'])||IPS_GetInstance($c['managerId'])['ModuleInfo']['ModuleID']!=='{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}')throw new RuntimeException('Expected manager missing'); + foreach(array_merge($c['accounting']['sources'],$c['extraSources']) as $s){ + if(!IPS_VariableExists($s['variableId']))throw new RuntimeException('Expected observation source missing'); + $o=IPS_GetObject($s['variableId']);$v=IPS_GetVariable($s['variableId']); + if($o['ParentID']!==$s['parentId']||$o['ObjectIdent']!==$s['ident']||!in_array($v['VariableType'],[1,2],true))throw new RuntimeException('Source identity differs'); + } + if(is_link($stage.'/data')||!is_dir($stage.'/data')||!is_writable($stage.'/data')||(fileperms($stage.'/data')&0077)!==0)throw new RuntimeException('Private data directory required'); + $lock='ENELIX.V4.ViewsInstall.'.$c['installationId'];if(!IPS_SemaphoreEnter($lock,3000))throw new RuntimeException('Observer installation busy'); + $created=[];$script=null;$newTimer=false;$report=['scope'=>'passive_separated_observation','startedAt'=>gmdate('c'), + 'oldCollectorChanged'=>false,'modulesChanged'=>false,'pollingChanged'=>false,'socAccountsChanged'=>false,'trainingEnabled'=>false,'controlEnabled'=>false]; + try{ + $cat=v4viewsEnsure(0,'ENELIX_V4_SEPARATED_OBSERVATION',0,'V4 - getrennte Last und EV-SDL (nur Beobachtung)',$marker,$created); + foreach(['PhysicalLoad'=>'Physische Last - Kandidat, keine Trainingsfreigabe','AllocationModel'=>'EV-SDL-Modell - KEINE separate Messung','ObservationStatus'=>'Beobachtungsstatus'] as $key=>$name)v4viewsEnsure($cat,$key,2,$name,$marker,$created); + $script=v4viewsEnsure($cat,'Observe',3,'Getrennte Messsicht alle 30 Sekunden',$marker,$created); + $stop=v4viewsEnsure($cat,'Stop',3,'Nur diese zusaetzliche Beobachtung stoppen',$marker,$created); + $include=''observer_installed','categoryId'=>$cat,'scriptId'=>$script,'stopId'=>$stop,'firstObservation'=>$first,'futureTimerExecutionVerified'=>false]; + echo "OK: Getrennte Last-/EV-SDL-Beobachtung eingerichtet, alle 30 Sekunden.\n"; + echo 'Erste Aufnahme: '.$first['capturedAt'].'; Quellen: '.$first['sourceCount'].".\n"; + echo 'Physische Last: '.$first['physicalLoadStatus'].'; EV-/SDL-Modell: '.$first['allocationModelStatus'].".\n"; + echo "Keine Trainings-/Stellfreigabe. Bisheriger Sammler, Manager, Gateway, SDL und Polling unveraendert.\n"; + }catch(Throwable $e){ + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,180); + if($newTimer&&$script!==null)IPS_SetScriptTimer($script,0); + echo 'FEHLER: '.$report['reason']."\n"; + }finally{ + $report['createdObjects']=$created;$report['finishedAt']=gmdate('c'); + try{v4viewsJson($stage.'/INSTALL_RESULT.json',$report);}finally{IPS_SemaphoreLeave($lock);} + echo 'OBSERVATION REPORT: '.$stage."/INSTALL_RESULT.json\n"; + } + return $report; +} diff --git a/examples/V4SeparatedObservation/runtime.php b/examples/V4SeparatedObservation/runtime.php new file mode 100644 index 0000000..62171ce --- /dev/null +++ b/examples/V4SeparatedObservation/runtime.php @@ -0,0 +1,89 @@ +GetValue($id),'updated'=>(int)$a['VariableUpdated'],'changed'=>(int)$a['VariableChanged'],'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; +} +function v4viewsJson(string $path,array $data): void +{ + if(is_link($path)||realpath(dirname($path))!==dirname($path)||(fileperms(dirname($path))&0077)!==0)throw new RuntimeException('Private observer output required'); + $tmp=tempnam(dirname($path),'.views-');if($tmp===false)throw new RuntimeException('Cannot create observer report'); + try{$json=json_encode($data,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT|JSON_PRESERVE_ZERO_FRACTION)."\n"; + if(file_put_contents($tmp,$json,LOCK_EX)!==strlen($json)||!chmod($tmp,0644)||!rename($tmp,$path))throw new RuntimeException('Cannot store observer report'); + }finally{if(is_file($tmp))unlink($tmp);} +} +function v4viewsRun(string $stage,int $category): array +{ + if(!function_exists('IPS_GetKernelVersion'))throw new RuntimeException('Run inside Symcon'); + [$c,$policy]=v4viewsConfig($stage);$marker=v4viewsMarker($c); + v4viewsAssert($category,0,'ENELIX_V4_SEPARATED_OBSERVATION',0,$marker); + $ids=[];foreach(['PhysicalLoad','AllocationModel','ObservationStatus'] as $key){ + $id=v4viewsChild($category,$key);if($id===null)throw new RuntimeException('Observer object missing');v4viewsAssert($id,$category,$key,2,$marker);$ids[$key]=$id;} + $lock='ENELIX.V4.SeparatedView.'.$c['installationId'];if(!IPS_SemaphoreEnter($lock,0))return ['status'=>'busy','controlEnabled'=>false]; + try{ + try{ + $raw=\Belevo\EnelixEMS\NetzfahrplanV4Messaufnahme::capture($c,'v4viewsRead',static fn():int=>time()); + $view=\Belevo\EnelixEMS\NetzfahrplanV4Messsicht::calculate($raw,$c,$policy); + $record=['schemaVersion'=>1,'kind'=>'physical_and_allocation_observation','installationId'=>$c['installationId'],'capturedAt'=>$raw['capturedAt'], + 'capture'=>$raw,'views'=>$view,'trainingEligible'=>false,'controlEnabled'=>false]; + $dir=$stage.'/data';if(is_link($dir)||realpath($dir)!==$dir||(fileperms($dir)&0077)!==0)throw new RuntimeException('Private data directory required'); + $path=\Belevo\EnelixEMS\NetzfahrplanV4Messaufnahme::append($dir,$record); + // Derived-only files readable to authorized agent through a mode-0700 stage/data directory. + // Original collector files and permissions are never touched. + if(!chmod($path,0644))throw new RuntimeException('Cannot expose selected derived observation'); + $status=['status'=>'recorded','capturedAt'=>$raw['capturedAt'],'sourceCount'=>count($raw['raw']),'journal'=>$path, + 'physicalLoadStatus'=>$view['physicalLoad']['status'],'allocationModelStatus'=>$view['allocationModel']['status'], + 'solarOriginStatus'=>$view['solarOrigin']['status'],'trainingEligible'=>false,'controlEnabled'=>false]; + v4viewsJson($dir.'/LATEST.json',$record); + SetValue($ids['PhysicalLoad'],json_encode(['captureAt'=>$raw['capturedAt'],'physical'=>$view['physicalLoad'],'solarAcAlternative'=>$view['solarAcAlternative'],'trainingEligible'=>false],JSON_THROW_ON_ERROR)); + SetValue($ids['AllocationModel'],json_encode(['captureAt'=>$raw['capturedAt'],'model'=>$view['allocationModel'],'storage'=>$view['storageFeedback'],'canDispatch'=>false],JSON_THROW_ON_ERROR)); + }catch(Throwable $e){ + $status=['status'=>'observation_error','at'=>gmdate('c'),'errorType'=>get_class($e),'sourceLine'=>$e->getLine(),'trainingEligible'=>false,'controlEnabled'=>false]; + // Never leave a stale numeric proposal in the new diagnosis variables. + SetValue($ids['PhysicalLoad'],json_encode($status,JSON_THROW_ON_ERROR));SetValue($ids['AllocationModel'],json_encode($status,JSON_THROW_ON_ERROR)); + try{v4viewsJson($stage.'/data/LATEST.json',$status);}catch(Throwable $ignored){} + } + SetValue($ids['ObservationStatus'],json_encode($status,JSON_THROW_ON_ERROR));v4viewsJson($stage.'/data/STATUS.json',$status);return $status; + }finally{IPS_SemaphoreLeave($lock);} +} +function v4viewsStop(string $stage,int $category): void +{ + [$c,$p]=v4viewsConfig($stage);$marker=v4viewsMarker($c);v4viewsAssert($category,0,'ENELIX_V4_SEPARATED_OBSERVATION',0,$marker); + $id=v4viewsChild($category,'Observe');if($id===null)throw new RuntimeException('Observer script missing');v4viewsAssert($id,$category,'Observe',3,$marker); + if(!IPS_SetScriptTimer($id,0))throw new RuntimeException('Cannot stop observer'); + $s=['status'=>'stopped','at'=>gmdate('c'),'controlEnabled'=>false,'existingCollectorChanged'=>false]; + foreach(['PhysicalLoad','AllocationModel','ObservationStatus'] as $key){$var=v4viewsChild($category,$key);if($var===null)throw new RuntimeException('Observer status missing');v4viewsAssert($var,$category,$key,2,$marker);SetValue($var,json_encode($s,JSON_THROW_ON_ERROR));} + v4viewsJson($stage.'/data/STATUS.json',$s);v4viewsJson($stage.'/data/LATEST.json',$s); +} diff --git a/examples/V4UnifiedRelease/file_installer.php b/examples/V4UnifiedRelease/file_installer.php new file mode 100644 index 0000000..b19ffe3 --- /dev/null +++ b/examples/V4UnifiedRelease/file_installer.php @@ -0,0 +1,35 @@ +'already_installed']; + foreach($names as$name)if(($old[$name]===null?null:hash('sha256',$old[$name]))!==$m['files'][$name]['before'])throw new RuntimeException('Teilstand zuerst pruefen.'); + $backup=$stage.'/backups/'.gmdate('YmdTHis').'-'.bin2hex(random_bytes(4)); + if(!mkdir($backup.'/libs',0700,true))throw new RuntimeException('Sicherung fehlt.'); + foreach($names as$name)if($old[$name]!==null)v4UnifiedWrite($backup.'/'.$name,$old[$name],0600); + v4UnifiedWrite($backup.'/MANIFEST.json',json_encode($m,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0600); + $written=[]; + try {foreach($names as$name){$p=$target.'/'.$name;if((is_file($p)?file_get_contents($p):null)!==$old[$name])throw new RuntimeException('Parallele Dateiaenderung.');v4UnifiedWrite($p,$new[$name]);$written[]=$name;}} + catch(Throwable $e){foreach(array_reverse($written)as$name){$p=$target.'/'.$name;if(hash_file('sha256',$p)!==$m['files'][$name]['after'])continue;if($old[$name]===null)unlink($p);else v4UnifiedWrite($p,$old[$name]);}throw $e;} + return ['status'=>'installed','backup'=>$backup]; +} diff --git a/examples/V4UnifiedRelease/install.php b/examples/V4UnifiedRelease/install.php new file mode 100644 index 0000000..6062f0c --- /dev/null +++ b/examples/V4UnifiedRelease/install.php @@ -0,0 +1,33 @@ +'unified_native_data_only','startedAt'=>gmdate('c'),'newActuatorPermission'=>false,'cursorReset'=>false,'outboxRewritten'=>false]; +try { + $package=json_decode(file_get_contents(__DIR__.'/PACKAGE_HASHES.json'),true,64,JSON_THROW_ON_ERROR); + foreach($package as$name=>$hash)if(str_contains($name,'..')||str_starts_with($name,'/')||is_link(__DIR__.'/'.$name)||hash_file('sha256',__DIR__.'/'.$name)!==$hash)throw new RuntimeException('Installationspaket wurde geaendert.'); + if(!IPS_InstanceExists($manager)||IPS_GetInstance($manager)['ModuleInfo']['ModuleID']!=='{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}')throw new RuntimeException('Erwarteter Manager fehlt.'); + if(IPS_HasChanges($manager))throw new RuntimeException('Offene Manageraenderungen zuerst speichern oder verwerfen.'); + if(IPS_GetProperty($manager,'NetzfahrplanAktiv')!==false||IPS_GetProperty($manager,'NetzfahrplanV4MessdatenAktiv')!==true)throw new RuntimeException('Erwartet: Datenaufnahme EIN, alter Netzfahrplan AUS.'); + $controls=[];foreach(IPS_GetModuleList()as$mid)if((IPS_GetModule($mid)['Prefix']??'')==='MC')$controls=array_merge($controls,IPS_GetInstanceListByModuleID($mid)); + if(count($controls)!==1||!function_exists('MC_ReloadModule'))throw new RuntimeException('Module Control nicht eindeutig.'); + $lock=IPS_SemaphoreEnter('ENELIX.V4.UnifiedRelease',3000);if(!$lock)throw new RuntimeException('Installation bereits aktiv.'); + $before=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $report['files']=v4UnifiedFiles(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); + // No staged application classes are loaded into this script context. + if(MC_ReloadModule($controls[0],'Enelix-EMS')===false)throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + $after=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + foreach($before as$key=>$value)if(!array_key_exists($key,$after)||$after[$key]!==$value)throw new RuntimeException('Bestehender Parameter unerwartet geaendert: '.$key); + $report['status']='native_data_candidate_installed';$report['parametersUnchanged']=true; + echo "OK: Einheitliche Messdatenkennung und dauerhafte Versandbestaetigung installiert. Archivierung vorbereitet, standardmaessig AUS.\n"; + echo "Der bestehende Datentimer setzt den Versand fort. Sendezeiger und Originalbestand wurden nicht umgeschrieben.\n"; + echo "Keine V4-Stellfreigabe und kein Wechsel der Prognosequelle.\n"; +} catch(Throwable $e) { + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,250); + echo 'FEHLER: '.$report['reason']."\nKeine neue Stellfreigabe.\n"; +} finally { + $report['finishedAt']=gmdate('c');v4UnifiedWrite(__DIR__.'/INSTALL_RESULT.json',json_encode($report,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0644); + if($lock)IPS_SemaphoreLeave('ENELIX.V4.UnifiedRelease'); + echo 'UNIFIED MANAGER REPORT: '.__DIR__."/INSTALL_RESULT.json\n"; +} diff --git a/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php new file mode 100644 index 0000000..eb8484b --- /dev/null +++ b/libs/BatterieNetzfahrplanV4RueckmeldungTrait.php @@ -0,0 +1,130 @@ +RegisterPropertyString('NetzfahrplanV4RueckmeldungKonfiguration', '{}'); + $this->RegisterTimer('NetzfahrplanV4RueckmeldungAktualisieren', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4RueckmeldungAktualisieren', 0);"); + } + + private function konfiguriereV4Rueckmeldung(): void + { + $this->SetBuffer('V4ConfirmedFeedback', ''); + $this->SetBuffer('V4ConfirmedFeedbackConfig', ''); + $this->SetBuffer('V4ConfirmedFeedbackMono', ''); + $configured = !in_array(trim($this->ReadPropertyString('NetzfahrplanV4RueckmeldungKonfiguration')), ['', '{}'], true); + $this->SetTimerInterval('NetzfahrplanV4RueckmeldungAktualisieren', $configured ? 1000 : 0); + } + + /** No transport I/O under the battery control lock. Original sample times never move. */ + private function v4BatterieRueckmeldung(): array + { + $json = $this->ReadPropertyString('NetzfahrplanV4RueckmeldungKonfiguration'); + $sample = json_decode($this->GetBuffer('V4ConfirmedFeedback'), true); + $mono = (float) $this->GetBuffer('V4ConfirmedFeedbackMono'); + $now = time();$age = hrtime(true) / 1e9 - $mono; + if (!is_array($sample) || !is_int($sample['checkedAt'] ?? null) + || $sample['checkedAt'] > $now || $now - $sample['checkedAt'] > 2 + || $age < 0 || $age > 2 + || $this->GetBuffer('V4ConfirmedFeedbackConfig') !== hash('sha256', $json)) { + throw new \RuntimeException('feedback_confirmed_cache_stale_or_invalid'); + } + $config = json_decode($json, true, 32, JSON_THROW_ON_ERROR); + if (($config['batteryInstanceId'] ?? null) !== $this->InstanceID + || !in_array($config['managerId'] ?? 0, $this->zugeordneteManagerIDs(), true)) { + throw new \RuntimeException('feedback_cache_binding_invalid'); + } + foreach ($config['sources'] as $source) { + if (!IPS_VariableExists($source['variableId'])) throw new \RuntimeException('feedback_variable_missing'); + $object = IPS_GetObject($source['variableId']); + if ($object['ParentID'] !== $source['parentId'] || $object['ObjectIdent'] !== $source['ident']) { + throw new \RuntimeException('feedback_cache_source_changed'); + } + } + return $sample; + } + + /** Independent acquisition; never waits behind another read or takes the actuator lock. */ + private function aktualisiereV4Rueckmeldung(): void + { + $json=$this->ReadPropertyString('NetzfahrplanV4RueckmeldungKonfiguration'); + if (strlen($json)>65536) throw new \RuntimeException('feedback_configuration_too_large'); + $c=json_decode($json,true,32,JSON_THROW_ON_ERROR); + if (!is_array($c) || ($c['batteryInstanceId']??null)!==$this->InstanceID) throw new \RuntimeException('feedback_battery_binding_invalid'); + if (!in_array($c['managerId']??0,$this->zugeordneteManagerIDs(),true)) throw new \RuntimeException('feedback_manager_binding_invalid'); + + $lock='ENELIX.V4.Feedback.'.$this->InstanceID; + if (!IPS_SemaphoreEnter($lock,0)) return; + try { + $reader=static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('feedback_variable_missing'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id);$value=GetValue($id); + return ['value'=>$value,'updated'=>$v['VariableUpdated'],'changed'=>$v['VariableChanged'], + 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + }; + $result=null; + for ($attempt=1;$attempt<=3;++$attempt) { + try { + $result=NetzfahrplanV4Geraeteabruf::snapshot( + $c, + static function(int $id):string { + if (!IPS_InstanceExists($id)) throw new \RuntimeException('feedback_parent_instance_missing'); + return (string)(IPS_GetInstance($id)['ModuleInfo']['ModuleID']??''); + }, + static function(int $id,string $module):bool { + if ($module===NetzfahrplanV4Geraeteabruf::MBUS_DEVICE) { + if (!function_exists('MBUS_UpdateValues') && !function_exists(__NAMESPACE__.'\\MBUS_UpdateValues')) throw new \RuntimeException('feedback_mbus_refresh_unavailable'); + return MBUS_UpdateValues($id)===true; + } + if (in_array($module,[NetzfahrplanV4Geraeteabruf::MODBUS_DEVICE,NetzfahrplanV4Geraeteabruf::MODBUS_ADDRESS],true)) { + if (!function_exists('ModBus_RequestRead') && !function_exists(__NAMESPACE__.'\\ModBus_RequestRead')) throw new \RuntimeException('feedback_modbus_refresh_unavailable'); + return ModBus_RequestRead($id)===true; + } + return false; + }, + $reader, + static fn():int=>time(), + static function(int $microseconds):void { usleep($microseconds); } + ); + break; + } catch (\InvalidArgumentException $e) { + if (!str_starts_with($e->getMessage(),'feedback_source_changed_after_confirmed_read:') || $attempt>=3) throw $e; + usleep(150000); + } + } + if (!is_array($result)) throw new \RuntimeException('feedback_snapshot_unavailable'); + $this->SetBuffer('V4ConfirmedFeedback',json_encode($result,JSON_THROW_ON_ERROR)); + $this->SetBuffer('V4ConfirmedFeedbackConfig', hash('sha256', $json)); + $this->SetBuffer('V4ConfirmedFeedbackMono', (string) (hrtime(true) / 1e9)); + } catch (\Throwable $error) { + $this->SetBuffer('V4ConfirmedFeedback', ''); + throw $error; + } finally { + IPS_SemaphoreLeave($lock); + } + } + + /** Read only. A returned current value is NOT actuator permission. */ + public function GetV4BatterieRueckmeldung(): string + { + try { + try { $r=$this->v4BatterieRueckmeldung(); } + catch (\Throwable) { + $this->aktualisiereV4Rueckmeldung(); + $r=$this->v4BatterieRueckmeldung(); + } + } + catch (\Throwable $e) { $r=['kind'=>'local_physical_feedback','status'=>'unavailable','reason'=>substr($e->getMessage(),0,200), + 'batteryW'=>null,'gridW'=>null,'usableForTrial'=>false,'canDispatch'=>false,'deviceReadConfirmed'=>false]; } + return json_encode($r,JSON_THROW_ON_ERROR); + } +} diff --git a/libs/BatterieNetzfahrplanV4TestTrait.php b/libs/BatterieNetzfahrplanV4TestTrait.php new file mode 100644 index 0000000..20a5c5c --- /dev/null +++ b/libs/BatterieNetzfahrplanV4TestTrait.php @@ -0,0 +1,227 @@ +registriereV4Rueckmeldung(); + $this->RegisterPropertyBoolean('NetzfahrplanV4RegeltestErlaubt', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4AktivtestErlaubt', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt', false); + $this->RegisterAttributeString('NetzfahrplanV4LetzteTestsitzung', ''); + $this->RegisterPropertyString('NetzfahrplanV4GeraeteWatchdogNachweis', ''); + $this->RegisterAttributeString('NetzfahrplanV4TestStatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4TestWatchdog', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4TestWatchdog', 0);"); + } + + private function v4BatterieTestSitzung(): array + { + $s = json_decode($this->GetBuffer('V4BatteryTrialSession') ?: '{}', true); + return is_array($s) ? $s : []; + } + + private function v4BatterieTestAktiv(): bool + { + return ($this->v4BatterieTestSitzung()['active'] ?? false) === true; + } + + /** Local test arming by the bound manager; never called by ordinary forecast traffic. */ + public function V4BatterieTestStarten(string $json): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->v4BatterieStartIntern($json); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function v4BatterieStartIntern(string $json): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt')) throw new RuntimeException('Batterietest lokal nicht erlaubt.'); + $s = json_decode($json, true, 32, JSON_THROW_ON_ERROR);$now = time(); + $activeTest = ($s['kind'] ?? null) === 'local_active_test_consent'; + if (!is_array($s) + || !in_array($s['kind'] ?? null, ['local_control_trial_consent', 'local_active_test_consent'], true) + || ($s['batteryInstanceId'] ?? null) !== $this->InstanceID || $this->v4BatterieTestAktiv()) { + throw new RuntimeException('Batterie-Testanmeldung ungueltig oder bereits aktiv.'); + } + NetzfahrplanV4Regeltest::uuid($s['sessionId'] ?? null); + if ($this->ReadAttributeString('NetzfahrplanV4LetzteTestsitzung') === $s['sessionId']) throw new RuntimeException('Testsitzung bereits beendet/verwendet.'); + NetzfahrplanV4Regeltest::uuid($s['installationId'] ?? null); + $manager = NetzfahrplanV4Regeltest::integer($s['managerId'] ?? null, 'Manager', 1, 99999); + $managerPermission = $activeTest ? 'NetzfahrplanV4AktivtestErlaubt' : 'NetzfahrplanV4RegeltestErlaubt'; + if (!in_array($manager, $this->zugeordneteManagerIDs(), true) + || IPS_GetProperty($manager, $managerPermission) !== true) { + throw new RuntimeException('Manager nicht fuer Batterietest freigegeben/zugeordnet.'); + } + $evidence = $this->ReadPropertyString('NetzfahrplanV4GeraeteWatchdogNachweis'); + $watchdogWaived = $activeTest + && $this->ReadPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt') + && ($s['watchdogWaived'] ?? null) === true + && ($s['actuatorWatchdogEvidenceId'] ?? null) === 'explicit_test_plant_watchdog_waiver'; + if (!$watchdogWaived && (strlen($evidence) < 8 || $evidence !== ($s['actuatorWatchdogEvidenceId'] ?? null))) { + throw new RuntimeException('Externer Geraete-Watchdog muss separat nachgewiesen sein.'); + } + $maxDuration = $activeTest + ? NetzfahrplanV4Regeltest::MAX_ACTIVE_TEST_SECONDS + : NetzfahrplanV4Regeltest::MAX_TEST_SECONDS; + $end = NetzfahrplanV4Regeltest::integer($s['expiresAt'] ?? null, 'Testende', $now + 1, $now + $maxDuration); + $s['maxChargeW'] = NetzfahrplanV4Regeltest::number($s['maxChargeW'] ?? null, 'Testladen', 1, 1e9); + $s['maxDischargeW'] = NetzfahrplanV4Regeltest::number($s['maxDischargeW'] ?? null, 'Testentladen', 1, 1e9); + if (!is_string($s['assetId'] ?? null) || $s['assetId'] === '') throw new RuntimeException('Batterie-ID fehlt.'); + if (!$this->GetValue('Aktiv') || $this->ReadPropertyInteger('Batteriemanagement') !== 2) { + throw new RuntimeException('Batterie nicht unter ENELIX-Regelung.'); + } + $feedback = $this->v4BatterieRueckmeldung(); + $now = time(); + if ($now >= $end) throw new RuntimeException('Testsitzung waehrend Geraeteabruf abgelaufen.'); + NetzfahrplanV4Rueckmeldung::requireTrial($feedback, $s, $now); + $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $s['sessionId']); + $s['active'] = true;$s['lastSequence'] = 0;$s['lastWallClockAt'] = $now; + $s['monotonicDeadline'] = hrtime(true) / 1e9 + $end - $now; + $s['commandMonotonicDeadline'] = hrtime(true) / 1e9 + 10; + $this->SetBuffer('V4BatteryTrialCommand', '{}'); + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + // Old ordinary values may not resume after the test without a NEW manager message. + $this->setzeZustand('SollwertGueltig', false); + $this->SetTimerInterval('NetzfahrplanV4TestWatchdog', 1000); + $this->WriteAttributeString('NetzfahrplanV4TestStatus', '{"status":"armed_waiting_command","physicalResponseVerified":false}'); + } + + public function V4BatterieTestBefehl(string $json): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->v4BatterieBefehlIntern($json); } + finally { IPS_SemaphoreLeave($lock); } + } + + private function v4BatterieBefehlIntern(string $json): void + { + $s = $this->v4BatterieTestSitzung();$now = time();$mono = hrtime(true) / 1e9; + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt') + || (($s['kind'] ?? null) === 'local_active_test_consent' + && !$this->ReadPropertyBoolean('NetzfahrplanV4AktivtestErlaubt'))) { + throw new RuntimeException('Batterietest gesperrt.'); + } + $c = json_decode($json, true, 32, JSON_THROW_ON_ERROR); + NetzfahrplanV4Regeltest::pruefeBatteriebefehl($c, $s, $now, $mono); + $s['lastSequence'] = $c['sequence'];$s['lastWallClockAt'] = $now; + $s['commandMonotonicDeadline'] = $mono + min(10, $c['expiresAt'] - $now); + $this->SetBuffer('V4BatteryTrialCommand', json_encode($c, JSON_THROW_ON_ERROR)); + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + $this->aktualisiereIntern(true); + if (!$this->v4BatterieTestAktiv() || $this->ReadAttributeString('Registerfehler') !== '') { + $state = json_decode($this->ReadAttributeString('NetzfahrplanV4TestStatus'), true); + $detail = $this->ReadAttributeString('Registerfehler') ?: ($state['reason'] ?? ''); + throw new RuntimeException('Batterie konnte Testbefehl nicht sicher uebernehmen: ' . substr($detail, 0, 300)); + } + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode([ + 'status' => 'controller_command_accepted', 'sessionId' => $s['sessionId'], + 'sourceShadowPlanId' => $c['sourceShadowPlanId'], 'sequence' => $c['sequence'], + 'commandExpiresAt' => $c['expiresAt'], 'actualRequestedW' => $this->ReadAttributeInteger('LetzterSollwert'), + 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->beendeV4BatterieTest('Befehl abgelehnt: ' . substr($e->getMessage(), 0, 500), true); + throw $e; + } + } + + /** Explicit local abort or ApplyChanges: revoke before any register action. */ + private function beendeV4BatterieTest(string $reason, bool $writeNow): void + { + $oldStatus = json_decode($this->ReadAttributeString('NetzfahrplanV4TestStatus'), true); + $active = $this->v4BatterieTestAktiv() || in_array($oldStatus['status'] ?? '', ['controller_command_accepted', 'armed_waiting_command', 'stop_pending', 'stop_failed'], true); + $this->SetBuffer('V4BatteryTrialSession', '{}');$this->SetBuffer('V4BatteryTrialCommand', '{}'); + $this->SetTimerInterval('NetzfahrplanV4TestWatchdog', 0); + if (!$active) return; + $this->setzeZustand('SollwertGueltig', false);$this->setzeZustand('Sollleistung', 0); + $report = ['status' => 'stop_pending', 'reason' => $reason, 'physicalResponseVerified' => false]; + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + if ($writeNow) { + try { $this->schreibeRegister(0, true);$report['status'] = 'stopped'; } + catch (Throwable $e) { + $report['status'] = 'stop_failed'; + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + throw $e; + } + $this->WriteAttributeString('NetzfahrplanV4TestStatus', json_encode($report, JSON_THROW_ON_ERROR)); + } + } + + public function V4BatterieTestStoppen(): void + { + $lock = 'ENELIX.V4.Battery.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Batterie beschaeftigt.'); + try { $this->beendeV4BatterieTest('Lokaler Testabbruch', true); } + finally { IPS_SemaphoreLeave($lock); } + } + + public function GetV4BatterieTestStatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4TestStatus'); + } + + /** Called inside the existing battery update, immediately before writing registers. */ + private function v4BatterieTestZiel(array $measurements, bool $available, int $now): ?int + { + if (!$this->v4BatterieTestAktiv()) return null; + $s = $this->v4BatterieTestSitzung(); + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt') + || (($s['kind'] ?? null) === 'local_active_test_consent' + && !$this->ReadPropertyBoolean('NetzfahrplanV4AktivtestErlaubt'))) { + throw new RuntimeException('Lokale Testfreigabe entzogen.'); + } + $feedback = $this->v4BatterieRueckmeldung(); + // The confirmed device read may cross a wall-clock second. All freshness, + // watchdog and switching decisions below use the post-read clocks. + $now = time();$mono = hrtime(true) / 1e9; + NetzfahrplanV4Rueckmeldung::requireTrial($feedback, $s, $now); + $c = json_decode($this->GetBuffer('V4BatteryTrialCommand') ?: '{}', true); + if ($now >= $s['expiresAt'] || $mono >= $s['monotonicDeadline'] || $now < $s['lastWallClockAt']) throw new RuntimeException('Testsitzung abgelaufen/Zeitsprung.'); + if (!$c) { + if ($mono >= $s['commandMonotonicDeadline']) throw new RuntimeException('Kein Startbefehl innerhalb Watchdog.'); + return 0; + } + $w = NetzfahrplanV4Regeltest::batterieZiel($c, $s, $measurements, [ + 'available' => $available, 'reserve' => $this->ReadPropertyFloat('ReserveLadezustand'), + 'minimum' => $this->ReadPropertyFloat('MindestLadezustand'), + 'blocked' => (bool) $this->leseZustand('HystereseAktiv'), + 'rechargeLimitW' => $this->ReadPropertyBoolean('NachladenMitMaximalleistung') ? null : $this->ReadPropertyInteger('MaximaleNachladeleistung'), + ], $now, $mono); + // Respect switching lock for increases/reversals; safety reductions and zero are immediate. + $last = $this->ReadAttributeInteger('LetzterSollwert'); + if ($now < $this->ReadAttributeInteger('AenderungFreigabeAb') && $w !== $last) { + if ($last * $w < 0) $w = 0; + elseif (abs($w) > abs($last)) $w = $last; + } + // Final hardware-policy/ramp clipping must not invalidate the binding or grid limits. + if (($c['feedbackConfigHash'] ?? null) !== $feedback['configHash']) throw new RuntimeException('feedback_configuration_changed'); + $expectedGrid = $feedback['gridW'] - $feedback['batteryW'] + $w; + if (($c['importLimitW'] !== null && $expectedGrid > $c['importLimitW'] + 1) + || ($c['exportLimitW'] !== null && -$expectedGrid > $c['exportLimitW'] + 1)) { + throw new RuntimeException('final_battery_clamp_cannot_meet_grid_limit'); + } + $s['lastWallClockAt'] = $now; + $this->SetBuffer('V4BatteryTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + return $w; + } catch (Throwable $e) { + $this->beendeV4BatterieTest($e->getMessage(), false); + return 0; // revoke stale value even if ordinary cached setpoint existed + } + } +} diff --git a/libs/ManagerNetzfahrplanV4AktivtestTrait.php b/libs/ManagerNetzfahrplanV4AktivtestTrait.php new file mode 100644 index 0000000..c6a8e9d --- /dev/null +++ b/libs/ManagerNetzfahrplanV4AktivtestTrait.php @@ -0,0 +1,225 @@ +RegisterPropertyBoolean('NetzfahrplanV4AktivtestErlaubt', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt', false); + $this->RegisterAttributeInteger('NetzfahrplanV4AktivtestBeginn', 0); + $this->RegisterAttributeInteger('NetzfahrplanV4AktivtestEnde', 0); + $this->RegisterVariableBoolean( + 'NetzfahrplanV4Aktivtest', + 'V4 Prognosemanager aktiv (Testbetrieb)', + '~Switch', + 952 + ); + $this->EnableAction('NetzfahrplanV4Aktivtest'); + } + + public function V4ManagerAktivtestSchalten(bool $aktiv): void + { + if (!$aktiv) { + $this->SetValue('NetzfahrplanV4Aktivtest', false); + $this->V4ManagerTestStoppen(); + return; + } + if ($this->GetValue('NetzfahrplanV4Aktivtest') && $this->v4ManagerTestSession()) { + return; + } + + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) { + throw new RuntimeException('Regelung beschaeftigt.'); + } + $armedBattery = null; + try { + $this->v4ManagerTestBedingungen(); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4AktivtestErlaubt') + || !$this->ReadPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt')) { + throw new RuntimeException('Aktivtest oder Watchdog-Verzicht ist lokal nicht freigegeben.'); + } + if ($this->v4ManagerTestSession()) { + throw new RuntimeException('Eine V4-Regelsitzung ist bereits aktiv.'); + } + + $envelope = json_decode( + $this->ReadAttributeString('NetzfahrplanV4Empfang'), + true, + 64, + JSON_THROW_ON_ERROR + ); + if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { + throw new RuntimeException('Kein gueltiger aktueller V4-Plan empfangen.'); + } + [$operation, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 32, JSON_THROW_ON_ERROR); + if (!is_array($assets) || count($assets) !== 1 || !is_string($assets[0]['ID'] ?? null)) { + throw new RuntimeException('Aktivtest braucht genau eine eindeutig konfigurierte Batterie.'); + } + $asset = $assets[0]; + $batteryID = $this->v4AktivtestBatterieinstanz($asset); + + $now = time(); + $start = $this->ReadAttributeInteger('NetzfahrplanV4AktivtestBeginn'); + $until = $this->ReadAttributeInteger('NetzfahrplanV4AktivtestEnde'); + if ($start === 0 && $until === 0) { + $start = $now; + $until = $now + self::V4_AKTIVTEST_DAUER_SEKUNDEN; + $this->WriteAttributeInteger('NetzfahrplanV4AktivtestBeginn', $start); + $this->WriteAttributeInteger('NetzfahrplanV4AktivtestEnde', $until); + } + if ($start <= 0 || $until <= $now + || $until - $start > self::V4_AKTIVTEST_DAUER_SEKUNDEN) { + throw new RuntimeException('Die einmalige 48-Stunden-Aktivtestfreigabe ist abgelaufen.'); + } + + [$maxChargeW, $maxDischargeW] = $this->v4AktivtestLeistungsgrenzen($operation, $asset['ID']); + $session = $this->v4AktivtestUuid(); + $s = [ + 'kind' => 'local_active_test_consent', + 'enabled' => true, + 'legacyEnabled' => false, + 'sessionId' => $session, + 'installationId' => $this->ReadAttributeString('LizenzInstallationID'), + 'managerId' => $this->InstanceID, + 'batteryInstanceId' => $batteryID, + 'assetId' => $asset['ID'], + 'startedAt' => $start, + 'expiresAt' => $until, + 'maxChargeW' => $maxChargeW, + 'maxDischargeW' => $maxDischargeW, + 'watchdogWaived' => true, + 'actuatorWatchdogEvidenceId' => self::V4_AKTIVTEST_WATCHDOG_VERZICHT, + 'feedbackConfigHash' => $operation['localFeedback'][$asset['ID']]['configHash'] ?? '', + 'gridLimits' => NetzfahrplanV4Planpruefung::kontext($operation)['limits'], + 'sequence' => 1, + 'lastWallClockAt' => $now, + 'monotonicDeadline' => hrtime(true) / 1e9 + $until - $now, + ]; + $response = $this->v4AktivtestAntwort($envelope['response'], $s, $now); + NetzfahrplanV4Regeltest::befehl( + $response, + $operation, + $actual, + $s, + $now, + $envelope['receivedAt'] + ); + + $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $session); + $armedBattery = $batteryID; + $this->v4ManagerBatterieAktion($batteryID, 'NetzfahrplanV4TestStart', json_encode($s, JSON_THROW_ON_ERROR)); + $this->SetBuffer('V4ManagerTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + $this->SetTimerInterval('NetzfahrplanV4TestTick', 2000); + $this->SetValue('NetzfahrplanV4Aktivtest', true); + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode([ + 'status' => 'active_test_started', + 'sessionId' => $session, + 'expiresAt' => $until, + 'watchdogWaivedForTestPlant' => true, + 'physicalResponseVerified' => false, + ], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + $this->SetValue('NetzfahrplanV4Aktivtest', false); + $this->SetBuffer('V4ManagerTrialSession', '{}'); + $this->SetTimerInterval('NetzfahrplanV4TestTick', 0); + if ($armedBattery !== null) { + $this->v4ManagerBatterieAktion($armedBattery, 'NetzfahrplanV4TestStop', 0); + } + throw $error; + } finally { + IPS_SemaphoreLeave($lock); + } + } + + private function v4AktivtestLeistungsgrenzen(array $operation, string $assetId): array + { + foreach ($operation['batteries'] ?? [] as $battery) { + if (($battery['id'] ?? null) !== $assetId) { + continue; + } + return [ + NetzfahrplanV4Regeltest::number($battery['maxChargeW'] ?? null, 'Anlagen-Ladegrenze', 1, 1e9), + NetzfahrplanV4Regeltest::number($battery['maxDischargeW'] ?? null, 'Anlagen-Entladegrenze', 1, 1e9), + ]; + } + throw new RuntimeException('Dynamische Leistungsgrenzen der Testbatterie fehlen.'); + } + + private function v4AktivtestBatterieinstanz(array $asset): int + { + $matches = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + if (($asset['SOCVariableID'] ?? null) === IPS_GetProperty($id, 'LadezustandVariableID') + && ($asset['LeistungVariableID'] ?? null) === IPS_GetProperty($id, 'IstleistungVariableID')) { + $matches[] = $id; + } + } + if (count($matches) !== 1) { + throw new RuntimeException('Physische Batteriezuordnung ist nicht eindeutig.'); + } + return $matches[0]; + } + + private function v4AktivtestAntwort(array $response, array $session, int $now): array + { + $plan = $response['plan'] ?? null; + if (!is_array($plan)) { + throw new RuntimeException('V4-Plan fehlt.'); + } + $validUntil = min($now + 90, (int) $session['expiresAt']); + $quality = $plan['inputQuality'] ?? []; + $response['controlledTrial'] = [ + 'version' => 1, + 'kind' => 'controlled_trial_authority', + 'sourcePlanRemainsShadow' => true, + 'installationId' => $session['installationId'], + 'sessionId' => $session['sessionId'], + 'managerId' => $session['managerId'], + 'batteryInstanceId' => $session['batteryInstanceId'], + 'assetId' => $session['assetId'], + 'sourceShadowPlanId' => $plan['planId'] ?? null, + 'revision' => $plan['configRevision'] ?? null, + 'family' => $plan['sourceFamily'] ?? null, + 'issuedAt' => gmdate('c', (int) $session['startedAt']), + 'expiresAt' => gmdate('c', (int) $session['expiresAt']), + 'checkedAt' => gmdate('c', $now), + 'validUntil' => gmdate('c', $validUntil), + 'maxChargeW' => $session['maxChargeW'], + 'maxDischargeW' => $session['maxDischargeW'], + 'accountingEvidenceId' => $quality['accountingEvidenceId'] ?? null, + 'actuatorWatchdogEvidenceId' => self::V4_AKTIVTEST_WATCHDOG_VERZICHT, + 'acceptEstimatedPeak' => true, + 'controlContext' => $plan['controlContext'] ?? null, + ]; + return $response; + } + + private function v4AktivtestUuid(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) + . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } +} diff --git a/libs/ManagerNetzfahrplanV4DatenTrait.php b/libs/ManagerNetzfahrplanV4DatenTrait.php new file mode 100644 index 0000000..98a4a14 --- /dev/null +++ b/libs/ManagerNetzfahrplanV4DatenTrait.php @@ -0,0 +1,209 @@ +RegisterPropertyBoolean('NetzfahrplanV4MessdatenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4ArchivAktiv', false); + $this->RegisterPropertyString('NetzfahrplanV4Messkonfiguration', '{}'); + $this->RegisterPropertyString('NetzfahrplanV4Datensatz', ''); + $this->RegisterAttributeString('NetzfahrplanV4DatenCursor', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4LetzteDatenquittung', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4ArchivTag', ''); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersion', 0); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->RegisterVariableString('NetzfahrplanV4Datenstatus', 'V4 Messdaten und Modellversorgung', '', 199); + $this->RegisterTimer('NetzfahrplanV4DatenErfassen', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4DatenErfassen', true);"); + } + + private function konfiguriereNetzfahrplanV4Daten(): void + { + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 0); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + try { + $this->v4DatenKonfiguration(); + if ($this->ReadAttributeInteger('NetzfahrplanV4DatenVersion')<1) { + // One retry after the data-only upgrade, never a cursor reset or a device write. + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersion', 1); + } + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 30000); + } catch (\Throwable $e) { + $this->v4DatenStatus(['status'=>'configuration_error','errorType'=>get_class($e)]); + } + } + + private function v4DatenKonfiguration(): array + { + $id=$this->ReadAttributeString('LizenzInstallationID'); + $dataset=$this->ReadPropertyString('NetzfahrplanV4Datensatz'); + if (!preg_match('/^[a-zA-Z0-9_-]{1,80}$/D',$dataset) + || !preg_match('/^[0-9a-f-]{36}$/D',$id)) throw new \RuntimeException('Dataset/installation not configured'); + $c=json_decode($this->ReadPropertyString('NetzfahrplanV4Messkonfiguration'),true,64,JSON_THROW_ON_ERROR); + $c=NetzfahrplanV4Messaufnahme::configuration($c); + if ($c['installationId']!==$id || ($c['managerId']??null)!==$this->InstanceID) throw new \RuntimeException('Measurement mapping belongs to another manager'); + return $c; + } + + private function v4DatenVerzeichnis(): string + { + $base=rtrim(IPS_GetKernelDir(),'/').'/data/enelix-v4'; + $directory=$base.'/'.$this->ReadAttributeString('LizenzInstallationID'); + // Fixed application-owned location; no configurable path or arbitrary file access. + foreach ([$base,$directory] as $p) { + if (is_link($p)) throw new \RuntimeException('Measurement directory symlink refused'); + if (!is_dir($p) && !mkdir($p,0700,true) && !is_dir($p)) throw new \RuntimeException('Measurement directory unavailable'); + if ((fileperms($p)&0007)!==0) throw new \RuntimeException('Measurement directory must remain private'); + } + return $directory; + } + + private function v4DatenStatus(array $data): void + { + $receipt=json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteDatenquittung'),true,32,JSON_THROW_ON_ERROR); + $data['lastAcknowledgement']=$receipt; + $this->SetValue('NetzfahrplanV4Datenstatus',json_encode($data+['checkedAt'=>gmdate('c'), + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'controlEnabled'=>false],JSON_THROW_ON_ERROR)); + } + + public function ErfasseNetzfahrplanV4Daten(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + $lock='ENELIX.V4.ApplicationData.'.$this->InstanceID; + if (!IPS_SemaphoreEnter($lock,0)) return; + try { + $c=$this->v4DatenKonfiguration(); + $reader=static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('Missing measurement'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id); + if (!in_array($v['VariableType'],[1,2],true)) throw new \RuntimeException('Numeric source required'); + return ['value'=>GetValue($id),'updated'=>(int)$v['VariableUpdated'],'changed'=>(int)$v['VariableChanged'], + 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + }; + $dir=$this->v4DatenVerzeichnis(); + $archiveResult=null; + if ($this->ReadPropertyBoolean('NetzfahrplanV4ArchivAktiv') && $this->ReadAttributeString('NetzfahrplanV4ArchivTag')!==gmdate('Y-m-d')) { + try { + $archiveCursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $archiveResult=NetzfahrplanV4Datenarchiv::run($dir,$archiveCursor,time()); + if ($archiveResult['archived']<2 && ($archiveResult['status']??null)!=='writer_busy')$this->WriteAttributeString('NetzfahrplanV4ArchivTag',gmdate('Y-m-d')); + } catch (\Throwable $e) { $archiveResult=['status'=>'archive_error','errorType'=>get_class($e),'originalsRetained'=>true]; } + } + try { + $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); + NetzfahrplanV4Messaufnahme::append($dir,$r); + $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), + 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + } catch (\Throwable $e) { + // Storage/capture failure must not prevent retrying the already durable outbox. + $result=['status'=>'capture_error','errorType'=>get_class($e),'dataRetained'=>true]; + } + if ($archiveResult!==null)$result['archive']=$archiveResult; + $last=$this->ReadAttributeInteger('NetzfahrplanV4DatenVersuch'); + $failures=min(5,$this->ReadAttributeInteger('NetzfahrplanV4DatenFehlerzahl')); + // Planner inputs expire after 120 s. Drain an acknowledged backlog faster, but a + // failed HTTP attempt still backs off to 60 s. Once caught up, normal upload is 60 s. + $lastAck=json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteDatenquittung'),true,32,JSON_THROW_ON_ERROR); + $acceptedEpoch=strtotime((string)($lastAck['acceptedThrough']??'')); + $backlog=$acceptedEpoch===false || time()-$acceptedEpoch>120; + $retry=$failures>0?60:($backlog?15:60); + if (time()>=$last+$retry) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch',time()); + try { + $result['delivery']=$this->v4DatenUebertragen($dir); + if (($result['delivery']['status']??null)==='acknowledged') { + $this->WriteAttributeString('NetzfahrplanV4LetzteDatenquittung',json_encode($result['delivery']+['receivedAt'=>gmdate('c')],JSON_THROW_ON_ERROR)); + } + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',0); + } catch (\Throwable $e) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',$failures+1); + $httpCode=(int)$e->getCode(); + $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e), + 'httpStatus'=>$httpCode>=100&&$httpCode<=599?$httpCode:null,'dataRetained'=>true, + 'nextAttemptAt'=>gmdate('c',time()+60)]; + } + } else $result['delivery']=['status'=>'scheduled','nextAttemptAt'=>gmdate('c',$last+$retry)]; + $this->v4DatenStatus($result); + } catch (\Throwable $e) { + // Sensor/configuration exceptions never enter the actuator or module-status path. + $this->v4DatenStatus(['status'=>'capture_error','errorType'=>get_class($e)]); + } finally { IPS_SemaphoreLeave($lock); } + } + + /** Returns only complete records from the append-only prefix; cursor moves after matching ACK. */ + private function v4DatenUebertragen(string $dir): array + { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) throw new \RuntimeException('Forecast licence missing'); + $cursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $day=$cursor['day']??'';$offset=$cursor['offset']??0; + if (!is_string($day) || ($day!==''&&!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day)) || !is_int($offset)||$offset<0) throw new \RuntimeException('Invalid data cursor'); + $files=glob($dir.'/raw-*.jsonl'); + if ($files===false||count($files)>400) throw new \RuntimeException('Measurement journal bounds exceeded'); + sort($files,SORT_STRING); + $lastAck=json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteDatenquittung'),true,32,JSON_THROW_ON_ERROR); + $acceptedEpoch=strtotime((string)($lastAck['acceptedThrough']??'')); + if ($acceptedEpoch===false) $acceptedEpoch=null; + $records=[];$candidate=$cursor;$bytes=0;$lastTime=null;$lastEpoch=null;$skippedDuplicateTimestamps=0;$skippedAcknowledgedTimestamps=0; + foreach ($files as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||($day!==''&&$name<$day)) continue; + if (is_link($file)||!is_file($file)||realpath($file)!==$file) throw new \RuntimeException('Invalid measurement journal'); + $h=fopen($file,'rb');if($h===false)throw new \RuntimeException('Cannot open measurement journal'); + try { + $size=fstat($h)['size'];$position=$name===$day?$offset:0; + if ($position>$size||fseek($h,$position)!==0) throw new \RuntimeException('Measurement journal shortened'); + while (ftell($h)<$size) { + $line=fgets($h,min(262146,$size-ftell($h)+1)); + if ($line===false||!str_ends_with($line,"\n")) break; + if (strlen($line)>262144) throw new \RuntimeException('Oversized measurement'); + if (count($records)>=120||$bytes+strlen($line)>900000) break 2; + $r=json_decode($line,false,64,JSON_THROW_ON_ERROR); + if (!is_object($r)||($r->installationId??null)!==$this->ReadAttributeString('LizenzInstallationID') + ||($r->kind??null)!=='raw_accounting_capture') throw new \RuntimeException('Journal identity invalid'); + $epoch=strtotime((string)($r->capturedAt??'')); + if ($epoch===false) throw new \RuntimeException('Journal capture timestamp invalid'); + $candidate=['day'=>$name,'offset'=>ftell($h)]; + if ($acceptedEpoch!==null && $epoch<=$acceptedEpoch) { $skippedAcknowledgedTimestamps++;continue; } + $bytes+=strlen($line); + if ($lastEpoch!==null && $epoch<$lastEpoch) throw new \RuntimeException('Measurement journal timestamp moved backwards'); + if ($lastEpoch!==null && $epoch===$lastEpoch) { $skippedDuplicateTimestamps++;continue; } + $records[]=$r;$lastTime=$r->capturedAt;$lastEpoch=$epoch; + } + } finally { fclose($h); } + } + if ($records===[]) return ['status'=>'up_to_date']; + $id=$this->ReadAttributeString('LizenzInstallationID');$token=$this->ReadAttributeString('PrognoseInstallationsToken'); + if ($token==='') throw new \RuntimeException('Existing device token unavailable'); + $payload=json_encode(['version'=>1,'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'records'=>$records],JSON_THROW_ON_ERROR); + $h=curl_init('https://license.enelix.ch/api/v1/installations/'.rawurlencode($id).'/prognosis/planner-v4/measurements'); + if ($h===false) throw new \RuntimeException('Cannot prepare data transfer'); + try { + curl_setopt_array($h,[CURLOPT_POST=>true,CURLOPT_RETURNTRANSFER=>true,CURLOPT_FOLLOWLOCATION=>false, + CURLOPT_CONNECTTIMEOUT=>2,CURLOPT_TIMEOUT=>8,CURLOPT_SSL_VERIFYPEER=>true,CURLOPT_SSL_VERIFYHOST=>2, + CURLOPT_HTTPHEADER=>['Content-Type: application/json','Accept: application/json','Authorization: Bearer '.$token], + CURLOPT_POSTFIELDS=>$payload]); + $answer=curl_exec($h);$code=(int)curl_getinfo($h,CURLINFO_HTTP_CODE); + if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged',$code); + $ack=json_decode($answer,true,32,JSON_THROW_ON_ERROR); + if (!in_array($ack['status']??null,['stored','duplicate'],true) + ||($ack['datasetId']??null)!==$this->ReadPropertyString('NetzfahrplanV4Datensatz') + ||($ack['acceptedThrough']??null)!==$lastTime) throw new \RuntimeException('Measurement acknowledgement does not match batch'); + $this->WriteAttributeString('NetzfahrplanV4DatenCursor',json_encode($candidate,JSON_THROW_ON_ERROR)); + return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime, + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'), + 'journalDuplicateTimestampsSkipped'=>$skippedDuplicateTimestamps, + 'journalAcknowledgedTimestampsSkipped'=>$skippedAcknowledgedTimestamps]; + } finally { curl_close($h); } + } +} diff --git a/libs/ManagerNetzfahrplanV4EmpfangTrait.php b/libs/ManagerNetzfahrplanV4EmpfangTrait.php new file mode 100644 index 0000000..d66bc4d --- /dev/null +++ b/libs/ManagerNetzfahrplanV4EmpfangTrait.php @@ -0,0 +1,272 @@ +RegisterPropertyBoolean('NetzfahrplanV4EmpfangAktiv', false); + $this->RegisterAttributeString('NetzfahrplanV4Empfang', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4EmpfangStatus', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4LetzteBestaetigung', '{}'); + $this->RegisterAttributeInteger('NetzfahrplanV4NaechsterAbruf', 0); + $this->RegisterVariableString('NetzfahrplanV4Vorschau', 'V4 Sollwert-Vorschau (KEINE Stellfreigabe)', '~HTMLBox', 950); + $this->RegisterVariableString('NetzfahrplanV4VorschauJSON', 'V4 Vorschau Diagnose (nur lesen)', '', 951); + $this->RegisterTimer('NetzfahrplanV4VorschauAktualisieren', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4VorschauAktualisieren', true);"); + $this->RegisterTimer('NetzfahrplanV4EmpfangAktualisieren', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Empfangen', true);"); + } + + private function stoppeNetzfahrplanV4Vorschau(): void + { + $this->SetTimerInterval('NetzfahrplanV4VorschauAktualisieren', 0); + $this->SetTimerInterval('NetzfahrplanV4EmpfangAktualisieren', 0); + $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); + $this->WriteAttributeString('NetzfahrplanV4LetzteBestaetigung', '{}'); + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', '{"status":"not_received"}'); + $this->WriteAttributeInteger('NetzfahrplanV4NaechsterAbruf', 0); + $this->zeigeNetzfahrplanV4Vorschau(['status' => 'not_received', 'reason' => 'Neu initialisiert; keinen alten Sollwert weiterverwenden.']); + } + + private function konfiguriereNetzfahrplanV4Vorschau(): void + { + $enabled = $this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') + && $this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') + && !$this->ReadPropertyBoolean('NetzfahrplanAktiv'); + $this->SetTimerInterval('NetzfahrplanV4VorschauAktualisieren', $enabled ? 10000 : 0); + $this->SetTimerInterval('NetzfahrplanV4EmpfangAktualisieren', $enabled ? 10000 : 0); + $this->aktualisiereNetzfahrplanV4Vorschau(); + } + + /** Called before the next operation upload, so the preceding completed plan can be seen. */ + private function empfangeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv')) return; + $lock = 'ENELIX.V4.Receive.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 0)) return; + try { + $now = time(); + if ($now < $this->ReadAttributeInteger('NetzfahrplanV4NaechsterAbruf')) return; + $session = method_exists($this, 'v4ManagerTestSession') ? $this->v4ManagerTestSession() : []; + $interval = ($session['kind'] ?? null) === 'local_active_test_consent' ? 45 : 60; + $this->WriteAttributeInteger('NetzfahrplanV4NaechsterAbruf', $now + $interval); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') || $this->ReadPropertyBoolean('NetzfahrplanAktiv')) { + throw new RuntimeException('Schatten-Sender fehlt oder alter Fahrplan ist eingeschaltet.'); + } + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $response = $this->netzfahrplanV4EmpfangHttp('GET', ''); + $receipt = time(); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') || $this->ReadPropertyBoolean('NetzfahrplanAktiv')) { + throw new RuntimeException('Freigabe waehrend des Abrufs geaendert.'); + } + // Validate the candidate before replacing the last accepted envelope. During an + // active test a failed refresh may keep the previous envelope, but its original + // receivedAt is never changed, so invalid traffic can never extend its lifetime. + [$operation, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $checked = NetzfahrplanV4Planpruefung::pruefen($response, + $this->ReadAttributeString('LizenzInstallationID'), $operation, time(), $receipt); + $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); + // Only the known, non-secret protocol envelope is cached. + $keys = ['receiverProtocolVersion', 'installationId', 'checkedAt', 'liveEnabled', 'fresh', + 'pending', 'settings', 'plan', 'families', 'lastRun', 'controlledTrial']; + $safe = array_intersect_key($response, array_flip($keys)); + $this->WriteAttributeString('NetzfahrplanV4Empfang', json_encode( + ['receivedAt' => $receipt, 'response' => $safe], JSON_THROW_ON_ERROR)); + $state = ['status' => 'received_shadow', 'at' => gmdate('c', $receipt), 'acknowledged' => false]; + $ack = ['planId' => $preview['planId'], 'revision' => $preview['revision'], 'step' => $preview['step'], 'status' => 'shadow_seen']; + $last = json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteBestaetigung'), true); + if ($last !== $ack) { + try { + $answer = $this->netzfahrplanV4EmpfangHttp('POST', '/ack', $ack); + if (($answer['status'] ?? null) !== 'recorded') throw new RuntimeException('Empfangsbestaetigung abgelehnt.'); + $this->WriteAttributeString('NetzfahrplanV4LetzteBestaetigung', json_encode($ack, JSON_THROW_ON_ERROR)); + $state['acknowledged'] = true; + } catch (Throwable $error) { + $state['acknowledgementError'] = substr($error->getMessage(), 0, 200); + } + } else { + $state['acknowledged'] = true; + } + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', json_encode($state, JSON_THROW_ON_ERROR)); + $this->aktualisiereNetzfahrplanV4Vorschau(); + } catch (Throwable $error) { + $keepPrevious = false; + $previous = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true); + if (is_array($previous) && is_int($previous['receivedAt'] ?? null)) { + $session = method_exists($this, 'v4ManagerTestSession') ? $this->v4ManagerTestSession() : []; + $keepPrevious = (($session['kind'] ?? null) === 'local_active_test_consent') + && $previous['receivedAt'] <= time() + && time() - $previous['receivedAt'] <= NetzfahrplanV4Planpruefung::MAX_RECEIPT_AGE; + } + $pendingRetry = is_array($response ?? null) + && ($response['installationId'] ?? null) === $this->ReadAttributeString('LizenzInstallationID') + && ($response['pending'] ?? null) !== null; + if ($keepPrevious || $pendingRetry) { + // Also catch a completed plan before the next operation starts another calculation. + $this->WriteAttributeInteger('NetzfahrplanV4NaechsterAbruf', time() + 15); + } + if (!$keepPrevious) { + $this->WriteAttributeString('NetzfahrplanV4Empfang', '{}'); + } + $state = ['status' => 'rejected_or_unavailable', 'at' => gmdate('c'), + 'reason' => substr($error->getMessage(), 0, 300), 'previousEnvelopeRetained' => $keepPrevious]; + $this->WriteAttributeString('NetzfahrplanV4EmpfangStatus', json_encode($state, JSON_THROW_ON_ERROR)); + $this->zeigeNetzfahrplanV4Vorschau($state); + } finally { + IPS_SemaphoreLeave($lock); + } + } + + /** Fixed authenticated read/ack endpoints only; never operation, settings, replan or dispatch. */ + private function netzfahrplanV4EmpfangHttp(string $method, string $suffix, ?array $body = null): array + { + if (!(($method === 'GET' && $suffix === '' && $body === null) + || ($method === 'POST' && $suffix === '/ack' && ($body['status'] ?? null) === 'shadow_seen'))) { + throw new RuntimeException('Unzulaessiger V4-Empfangsaufruf.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $id) + || !preg_match('/^[A-Za-z0-9_-]{43,128}$/D', $token)) { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $handle = curl_init('https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4' . $suffix); + if ($handle === false) throw new RuntimeException('V4-Empfang konnte nicht vorbereitet werden.'); + $response = ''; + try { + $options = [CURLOPT_CUSTOMREQUEST => $method, CURLOPT_FOLLOWLOCATION => false, + CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_PROTOCOLS => CURLPROTO_HTTPS, CURLOPT_HTTPHEADER => ['Authorization: Bearer ' . $token, + 'Accept: application/json', 'Content-Type: application/json', 'Cache-Control: no-cache'], + CURLOPT_WRITEFUNCTION => static function ($curl, string $chunk) use (&$response): int { + if (strlen($response) + strlen($chunk) > NetzfahrplanV4Planpruefung::MAX_RESPONSE_BYTES) return 0; + $response .= $chunk;return strlen($chunk); + }]; + if ($body !== null) $options[CURLOPT_POSTFIELDS] = json_encode($body, JSON_THROW_ON_ERROR); + curl_setopt_array($handle, $options); + $ok = curl_exec($handle);$code = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($ok === false || $code !== 200) throw new RuntimeException('V4-Planempfang HTTP ' . $code . '; kein gueltiger Vorschauwert.'); + return NetzfahrplanV4Planpruefung::dekodieren($response); + } finally { + curl_close($handle); + } + } + + /** Reads independently of the old control loop. Never manufactures fresh timestamps. */ + private function netzfahrplanV4LokaleVorschauwerte(): array + { + $d = json_decode($this->GetNetzfahrplanV4Diagnose(), true, 64, JSON_THROW_ON_ERROR); + if (!is_array($d['operation'] ?? null)) throw new RuntimeException('Lokale Betriebsdaten ungueltig; Vorschau gesperrt.'); + $topology = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 64, JSON_THROW_ON_ERROR); + $actual = [];$now = time(); + foreach ($topology as $asset) { + $controllers=[]; + foreach ($this->aktiveVerbraucherIDs() as $controller) { + if (IPS_InstanceExists($controller) + && IPS_GetInstance($controller)['ModuleInfo']['ModuleID']==='{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}' + && IPS_GetProperty($controller,'IstleistungVariableID')===($asset['LeistungVariableID']??null) + && IPS_GetProperty($controller,'LadezustandVariableID')===($asset['SOCVariableID']??null)) $controllers[]=$controller; + } + if (count($controllers)>1) throw new RuntimeException('feedback_controller_ambiguous'); + if (count($controllers)===1 && function_exists('ENELIX_GetV4BatterieRueckmeldung')) { + $controller=$controllers[0];$configured=IPS_GetProperty($controller,'NetzfahrplanV4RueckmeldungKonfiguration'); + if (is_string($configured) && !in_array(trim($configured),['','{}'],true)) { + if (count($topology)!==1) throw new RuntimeException('feedback_multi_asset_adapter_required'); + $f=json_decode(ENELIX_GetV4BatterieRueckmeldung($controller),true,32,JSON_THROW_ON_ERROR); + $feedbackNow=time(); + if (($f['status']??null)==='unavailable') { + $reason=is_string($f['reason']??null)?$f['reason']:'unknown'; + throw new RuntimeException('feedback_unavailable:'.substr($reason,0,180)); + } + if (($f['installationId']??null)!==$this->ReadAttributeString('LizenzInstallationID') + || ($f['assetId']??null)!==$asset['ID'] || ($f['managerId']??null)!==$this->InstanceID + || ($f['batteryInstanceId']??null)!==$controller || !is_numeric($f['batteryW']??null) + || !is_numeric($f['gridW']??null) || !is_int($f['checkedAt']??null) + || $feedbackNow-$f['checkedAt']>2 || $f['checkedAt']>$feedbackNow) { + throw new RuntimeException('feedback_invalid_no_legacy_fallback'); + } + $actual[$asset['ID']]=$f['batteryW'];$d['operation']['gridW']=$f['gridW']; + $d['operation']['localFeedback'][$asset['ID']]=$f; + continue; + } + } + // Legacy value remains display-only until an explicit physical feedback mapping exists. + $id = (int) ($asset['LeistungVariableID'] ?? 0); + if (!IPS_VariableExists($id)) throw new RuntimeException('Batterie-Istleistung fehlt.'); + $before = IPS_GetVariable($id);$v = GetValue($id);$after = IPS_GetVariable($id); + $factor = $asset['Leistungsfaktor'] ?? null; + if ((!is_int($v) && !is_float($v)) || !is_finite((float) $v) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) || $factor == 0 + || $before['VariableUpdated'] !== $after['VariableUpdated'] + || $after['VariableUpdated'] > $now || $now - $after['VariableUpdated'] > $this->ReadPropertyInteger('MesswertMaxAlter')) { + throw new RuntimeException('Batterie-Istleistung fehlt, ist veraltet oder inkonsistent.'); + } + $actual[$asset['ID']] = (float) $v * $factor; + } + return [$d['operation'], $actual]; + } + + public function GetNetzfahrplanV4Vorschau(): string + { + try { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') || !$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + throw new RuntimeException('V4-Empfang ausgeschaltet.'); + } + if ($this->ReadPropertyBoolean('NetzfahrplanAktiv')) throw new RuntimeException('Alter Fahrplan ist aktiv; V4-Vorschau gesperrt.'); + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { + throw new RuntimeException('Noch kein gueltiger V4-Plan empfangen.'); + } + [$operation, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $checked = NetzfahrplanV4Planpruefung::pruefen($envelope['response'], + $this->ReadAttributeString('LizenzInstallationID'), $operation, time(), $envelope['receivedAt']); + $out = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, time()); + $out['localFeedback'] = $operation['localFeedback'] ?? []; + $out['feedbackMeaning'] = isset($operation['localFeedback']) ? 'fresh_physical_or_explicit_partition_model' : 'legacy_display_only_not_for_trial'; + $out['reception'] = json_decode($this->ReadAttributeString('NetzfahrplanV4EmpfangStatus'), true); + } catch (Throwable $error) { + $out = ['status' => 'unavailable', 'reason' => substr($error->getMessage(), 0, 300), + 'reception' => json_decode($this->ReadAttributeString('NetzfahrplanV4EmpfangStatus'), true)]; + } + return json_encode(array_merge(['gridTargetW' => null, 'plannedBatteryW' => null, 'previewBatteryW' => null], + $out, ['canDispatch' => false, 'liveEnabled' => false]), JSON_THROW_ON_ERROR); + } + + private function aktualisiereNetzfahrplanV4Vorschau(): void + { + $this->zeigeNetzfahrplanV4Vorschau(json_decode($this->GetNetzfahrplanV4Vorschau(), true, 64, JSON_THROW_ON_ERROR)); + } + + private function zeigeNetzfahrplanV4Vorschau(array $value): void + { + $value = array_merge(['gridTargetW' => null, 'plannedBatteryW' => null, 'previewBatteryW' => null], $value, + ['canDispatch' => false, 'liveEnabled' => false]); + $escape = static fn($v): string => htmlspecialchars((string) $v, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8'); + $html = '

V4: NUR VORSCHAU - KEINE STELLFREIGABE

'; + foreach (['status' => 'Status', 'planId' => 'Plan-ID', 'family' => 'Modellfamilie', 'step' => 'Zeitschritt (UTC)', + 'stepValidUntil' => 'Gueltig bis', 'gridTargetW' => 'Geplantes Netzziel W', 'plannedBatteryW' => 'Geplante Batterie W', + 'previewBatteryW' => 'Lokal berechnete Batterie W (nicht ausgegeben)', 'reason' => 'Ablehnungsgrund'] as $key => $label) { + $item = $value[$key] ?? '-';if (is_float($item)) $item = number_format($item, 1, '.', "'"); + $html .= ''; + } + $html .= '
' . $escape($label) . '' . $escape($item) . '

Batterie + = Laden, - = Entladen. Netz + = Bezug, - = Einspeisung.

'; + foreach ($value['reasons'] ?? [] as $reason) $html .= '

' . $escape($reason) . '

'; + $html .= '

Empfangsbestaetigung ist keine Ausfuehrungsbestaetigung.

'; + $this->SetValue('NetzfahrplanV4VorschauJSON', json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + $this->SetValue('NetzfahrplanV4Vorschau', $html); + } +} diff --git a/libs/ManagerNetzfahrplanV4TestTrait.php b/libs/ManagerNetzfahrplanV4TestTrait.php new file mode 100644 index 0000000..9391d7c --- /dev/null +++ b/libs/ManagerNetzfahrplanV4TestTrait.php @@ -0,0 +1,223 @@ +RegisterPropertyBoolean('NetzfahrplanV4RegeltestErlaubt', false); + $this->RegisterAttributeString('NetzfahrplanV4LetzteTestsitzung', ''); + $this->RegisterAttributeString('NetzfahrplanV4RegeltestStatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4TestTick', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'Regeln', true);"); + } + + private function v4ManagerTestSession(): array + { + $value = json_decode($this->GetBuffer('V4ManagerTrialSession') ?: '{}', true); + return is_array($value) ? $value : []; + } + + private function v4ManagerTestBedingungen(): void + { + if (IPS_GetInstance($this->InstanceID)['InstanceStatus'] !== 102 + || !$this->ReadPropertyBoolean('NetzfahrplanV4RegeltestErlaubt') + || !$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv') + || !$this->ReadPropertyBoolean('NetzfahrplanV4EmpfangAktiv') + || $this->ReadPropertyBoolean('NetzfahrplanAktiv') || !$this->GetValue('Aktiv') + || !$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Keine lokale Testberechtigung oder konkurrierender Fahrplan.'); + } + } + + /** Explicit kernel-local user action only. No network request can call this via the portal. */ + public function V4ManagerTestStarten(string $json): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Regelung beschaeftigt.'); + $armedBattery = null; + try { + $this->v4ManagerTestBedingungen(); + if ($this->v4ManagerTestSession()) throw new RuntimeException('Testsitzung bereits aktiv; nicht verlaengern.'); + $request = json_decode($json, true, 16, JSON_THROW_ON_ERROR); + if (!is_array($request) || ($request['confirmation'] ?? null) !== 'START_BOUNDED_V4_CONTROL_TRIAL') { + throw new RuntimeException('Ausdrueckliche lokale Testbestaetigung erforderlich.'); + } + $session = NetzfahrplanV4Regeltest::uuid($request['sessionId'] ?? null); + if ($this->ReadAttributeString('NetzfahrplanV4LetzteTestsitzung') === $session) throw new RuntimeException('Testsitzung bereits benutzt; neue Freigabe erforderlich.'); + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + $a = $envelope['response']['controlledTrial'] ?? null; + if (!is_array($a) || ($a['sessionId'] ?? null) !== $session) throw new RuntimeException('Keine passende separate Serverfreigabe empfangen.'); + [$op, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + $batteryID = (int) ($a['batteryInstanceId'] ?? 0); + if (!in_array($batteryID, $this->aktiveVerbraucherIDs(), true) + || !IPS_InstanceExists($batteryID) + || IPS_GetInstance($batteryID)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + throw new RuntimeException('Testbatterie ist keine aktive zugeordnete Batterieinstanz.'); + } + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 32, JSON_THROW_ON_ERROR); + if (count($assets) !== 1 || $assets[0]['ID'] !== ($a['assetId'] ?? null) + || $assets[0]['SOCVariableID'] !== IPS_GetProperty($batteryID, 'LadezustandVariableID') + || $assets[0]['LeistungVariableID'] !== IPS_GetProperty($batteryID, 'IstleistungVariableID')) { + throw new RuntimeException('Physische Batteriezuordnung ist nicht eindeutig.'); + } + $now = time();$until = min(NetzfahrplanV4Planpruefung::zeit($a['expiresAt']), + $now + NetzfahrplanV4Regeltest::integer($request['durationSeconds'] ?? null, 'Lokale Testdauer', 30, 1800)); + $s = ['kind' => 'local_control_trial_consent', 'enabled' => true, 'legacyEnabled' => false, + 'sessionId' => $session, 'installationId' => $this->ReadAttributeString('LizenzInstallationID'), + 'managerId' => $this->InstanceID, 'batteryInstanceId' => $batteryID, 'assetId' => $a['assetId'], + 'expiresAt' => $until, 'maxChargeW' => NetzfahrplanV4Regeltest::number($request['maxChargeW'] ?? null, 'Testladegrenze', 1, 1e9), + 'maxDischargeW' => NetzfahrplanV4Regeltest::number($request['maxDischargeW'] ?? null, 'Testentladegrenze', 1, 1e9), + 'actuatorWatchdogEvidenceId' => IPS_GetProperty($batteryID, 'NetzfahrplanV4GeraeteWatchdogNachweis'), + 'feedbackConfigHash' => $op['localFeedback'][$a['assetId']]['configHash'] ?? '', + 'gridLimits' => NetzfahrplanV4Planpruefung::kontext($op)['limits'], + 'sequence' => 1, 'lastWallClockAt' => $now, + 'monotonicDeadline' => hrtime(true) / 1e9 + $until - $now]; + // Validate everything BEFORE arming either module. This rejects all normal shadow replies. + NetzfahrplanV4Regeltest::befehl($envelope['response'], $op, $actual, $s, $now, $envelope['receivedAt']); + $this->WriteAttributeString('NetzfahrplanV4LetzteTestsitzung', $session); + $armedBattery = $batteryID; + $this->v4ManagerBatterieAktion($batteryID, 'NetzfahrplanV4TestStart', json_encode($s, JSON_THROW_ON_ERROR)); + $this->SetBuffer('V4ManagerTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + $this->SetTimerInterval('NetzfahrplanV4TestTick', 2000); + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode([ + 'status' => 'armed_local_trial', 'sessionId' => $session, 'expiresAt' => $until, + 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->SetBuffer('V4ManagerTrialSession', '{}');$this->SetTimerInterval('NetzfahrplanV4TestTick', 0); + if ($armedBattery !== null) $this->v4ManagerBatterieAktion($armedBattery, 'NetzfahrplanV4TestStop', 0); + throw $e; + } finally { IPS_SemaphoreLeave($lock); } + } + + /** Revocation precedes I/O; if the device call fails its own ten-second lease still expires. */ + private function v4ManagerTestAbbrechen(string $reason): void + { + $s = $this->v4ManagerTestSession(); + $this->SetBuffer('V4ManagerTrialSession', '{}');$this->SetTimerInterval('NetzfahrplanV4TestTick', 0); + $this->SetValue('NetzfahrplanV4Aktivtest', false); + if (!$s) return; + $result = ['status' => 'stopped_fallback_pending', 'reason' => $reason, + 'sessionId' => $s['sessionId'], 'physicalResponseVerified' => false]; + // Reissue the next ordinary target even if numerically equal to a previously cached value. + $this->WriteAttributeString('LetzteSollwerte', '{}'); + try { + $this->v4ManagerBatterieAktion($s['batteryInstanceId'], 'NetzfahrplanV4TestStop', 0); + $result['controllerStopAccepted'] = true; + } catch (Throwable $e) { + $result['controllerStopAccepted'] = false;$result['reason'] .= '; Geraete-Abbruch fehlgeschlagen, Watchdog pruefen.'; + } + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode($result, JSON_THROW_ON_ERROR)); + } + + public function V4ManagerTestStoppen(): void + { + $lock = 'ENELIX.V4.Control.' . $this->InstanceID; + if (!IPS_SemaphoreEnter($lock, 1000)) throw new RuntimeException('Regelung beschaeftigt; Stop erneut anfordern.'); + try { $this->v4ManagerTestAbbrechen('Lokaler Testabbruch'); } + finally { IPS_SemaphoreLeave($lock); } + $this->regeln(true); // fresh ordinary command; never restore a cached pre-trial target + } + + public function GetV4ManagerTestStatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4RegeltestStatus'); + } + + private function v4ManagerBatterieAktion(int $id, string $ident, $value): void + { + if (IPS_RequestAction($id, $ident, $value) !== true) { + throw new RuntimeException('Batterieaktion nicht bestaetigt: ' . $ident); + } + } + + /** Called under the existing regulator's serialization, after allocation but before writes. */ + private function v4ManagerTestUebergabe(array &$allocation, array $consumers, bool $synchronized): void + { + $s = $this->v4ManagerTestSession(); + if (!$s) return; + try { + $this->v4ManagerTestBedingungen();$now = time(); + $activeTest = ($s['kind'] ?? null) === 'local_active_test_consent'; + // The ordinary commissioning trial still requires a fully synchronized consumer set. + // In the explicitly approved 48h active test, V4 is the sole actuator owner: stale or + // unrelated consumers receive no new Manager target and their physical effect is seen + // through the verified grid feedback on the next V4 cycle. + if ((!$activeTest && !$synchronized) || $now >= $s['expiresAt'] || hrtime(true) / 1e9 >= $s['monotonicDeadline']) { + throw new RuntimeException($activeTest ? 'Testende erreicht.' : 'Testende oder Verbraucher nicht synchronisiert.'); + } + [$op, $actual] = $this->netzfahrplanV4LokaleVorschauwerte(); + // Confirmed device reads can cross a wall-clock second. Validate their checkedAt + // against a post-read clock, never against the timestamp captured before I/O. + $now = time(); + if ($now >= $s['expiresAt'] || hrtime(true) / 1e9 >= $s['monotonicDeadline']) { + throw new RuntimeException('Testende erreicht.'); + } + $envelope = json_decode($this->ReadAttributeString('NetzfahrplanV4Empfang'), true, 64, JSON_THROW_ON_ERROR); + // Other consumers stay under the existing controller. Compensate their predicted + // change once; no double-counting of the battery's already flowing actual power. + // The explicitly approved active test owns its bound battery directly. It must not + // depend on the legacy allocation list as long as the confirmed V4 feedback still + // proves the same battery binding and is usable for the trial. + $otherDelta = 0.0;$found = false; + if ($activeTest) { + $feedback = $op['localFeedback'][$s['assetId']] ?? null; + $found = is_array($feedback) + && ($feedback['batteryInstanceId'] ?? null) === $s['batteryInstanceId'] + && ($feedback['usableForTrial'] ?? false) === true + && ($feedback['deviceReadConfirmed'] ?? false) === true; + } + foreach ($consumers as $consumer) { + $id = $consumer['InstanzID']; + if ($id === $s['batteryInstanceId']) { $found = true;continue; } + if (!array_key_exists($id, $allocation['Sollwerte'])) continue; + if ($activeTest) { + unset($allocation['Sollwerte'][$id]); + continue; + } + $d = $consumer['Daten']; + if (($d['Leistungsquelle'] ?? null) !== Nachrichtenvertrag::LEISTUNGSQUELLE_GEMESSEN) throw new RuntimeException('Andere Verbraucherleistung nicht gemessen.'); + $otherDelta += NetzfahrplanV4Regeltest::number($allocation['Sollwerte'][$id], 'Anderer Sollwert', -1e9, 1e9) + - NetzfahrplanV4Regeltest::number($d['Istleistung_W'] ?? null, 'Andere Istleistung', -1e9, 1e9); + } + if (!$found) throw new RuntimeException('Testbatterie nicht mehr in aktiver Zuordnung.'); + $op['gridW'] += $otherDelta; + $op['localPredictedOtherDeltaW']=$otherDelta; + if (!is_array($envelope['response'] ?? null) || !is_int($envelope['receivedAt'] ?? null)) { + throw new RuntimeException('Kein gueltiger empfangener Testkontext.'); + } + $response = $envelope['response']; + if (($s['kind'] ?? null) === 'local_active_test_consent') { + if (!$this->GetValue('NetzfahrplanV4Aktivtest') + || !$this->ReadPropertyBoolean('NetzfahrplanV4AktivtestErlaubt') + || !$this->ReadPropertyBoolean('NetzfahrplanV4WatchdogVerzichtErlaubt')) { + throw new RuntimeException('Aktivtestfreigabe entzogen.'); + } + $response = $this->v4AktivtestAntwort($response, $s, $now); + } + $command = NetzfahrplanV4Regeltest::befehl($response, $op, $actual, $s, $now, $envelope['receivedAt']); + // A controller-side refusal stops the session; never downgrade it to an ordinary target. + $this->v4ManagerBatterieAktion($s['batteryInstanceId'], 'NetzfahrplanV4TestCommand', json_encode($command, JSON_THROW_ON_ERROR)); + $s['sequence']++;$s['lastWallClockAt'] = $now; + $this->SetBuffer('V4ManagerTrialSession', json_encode($s, JSON_THROW_ON_ERROR)); + unset($allocation['Sollwerte'][$s['batteryInstanceId']]); // one owner of this actuator + $this->WriteAttributeString('NetzfahrplanV4RegeltestStatus', json_encode([ + 'status' => 'controller_command_sent', 'sessionId' => $s['sessionId'], + 'sourceShadowPlanId' => $command['sourceShadowPlanId'], 'command' => $command, + 'otherConsumerPredictedDeltaW' => $otherDelta, 'otherConsumersFrozen' => $activeTest, + 'physicalResponseVerified' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $e) { + $this->v4ManagerTestAbbrechen($e->getMessage()); + // Stop writes zero and next NEW ordinary allocation runs on this same pass. + } + } +} diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php new file mode 100644 index 0000000..af6c281 --- /dev/null +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -0,0 +1,191 @@ +registriereNetzfahrplanV4Empfang(); + $this->registriereNetzfahrplanV4Daten(); + $this->RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); + $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); + $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]'); + $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4Senden', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); + } + + public function GetNetzfahrplanV4Diagnose(): string + { + try { + $manager = []; + foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', + 'Lastspitzenmodus'] as $key) { + $manager[$key] = $this->ReadPropertyInteger($key); + } + foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { + $manager[$key] = $this->ReadPropertyFloat($key); + } + $manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified'; + $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); + $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); + $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); + $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $manager['V4BezugszaehlerQuellen'] = json_decode( + $this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR + ); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); + $cache = $this->leseJsonAttribut('VerbraucherCache'); + $controllers = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + $c = ['InstanzID' => $id]; + foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + $c[$key] = IPS_GetProperty($id, $key); + } + $entry = $cache[(string) $id] ?? []; + $c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0; + $c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false; + if (function_exists('ENELIX_GetV4BatterieRueckmeldung')) { + $configured = IPS_GetProperty($id, 'NetzfahrplanV4RueckmeldungKonfiguration'); + if (is_string($configured) && !in_array(trim($configured), ['', '{}'], true)) { + try { + $feedback = json_decode(ENELIX_GetV4BatterieRueckmeldung($id), true, 32, JSON_THROW_ON_ERROR); + $checkedAt = $feedback['checkedAt'] ?? null; + if (($feedback['usableForTrial'] ?? false) === true + && ($feedback['deviceReadConfirmed'] ?? false) === true + && is_int($checkedAt) && $checkedAt <= time() + && time() - $checkedAt <= max(2, (int) ($c['MesswertMaxAlter'] ?? 30))) { + $c['EmpfangenAm'] = $checkedAt; + $c['Verfuegbar'] = true; + } + } catch (Throwable) { + // Keep the ordinary cache state; V4 remains fail-safe if direct feedback is unavailable. + } + } + } + foreach (($entry['Daten']['Zustand'] ?? []) as $state) { + if (($state['Kennung'] ?? '') === 'HystereseAktiv') { + $c['HystereseAktiv'] = $state['Wert']; + } + } + $controllers[] = $c; + } + $evidence = []; + $evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID'); + if ($evidenceID > 0) { + if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) { + throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.'); + } + $evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($evidence)) { + throw new RuntimeException('Ungueltiger Messnachweis.'); + } + } + $read = static function (int $id): array { + if (!IPS_VariableExists($id)) { + throw new RuntimeException('Messvariable ' . $id . ' fehlt.'); + } + $before = IPS_GetVariable($id); + $value = GetValue($id); + $after = IPS_GetVariable($id); + if ($before['VariableUpdated'] !== $after['VariableUpdated']) { + throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); + } + $object = IPS_GetObject($id); + return ['value' => $value, 'updated' => (int) $after['VariableUpdated'], + 'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']]; + }; + return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, + $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } catch (Throwable $error) { + return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR); + } + } + + public function GetNetzfahrplanV4Sendestatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4Sendestatus'); + } + + private function sendeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + return; + } + // Read the preceding completed plan BEFORE sending a new operation/replan trigger. + $this->empfangeNetzfahrplanV4(); + try { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $snapshot = $this->GetNetzfahrplanV4Diagnose(); + $data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR); + if (!isset($data['operation'])) { + throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.'); + } + // Decoding as objects preserves empty JSON dictionaries in the API contract. + $payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR); + if (!isset($payload->operation)) { + throw new RuntimeException('Keine konsistenten Betriebsdaten.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation'; + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.'); + } + try { + curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, + CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR), + CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]); + $answer = curl_exec($handle); + $status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($answer === false || $status !== 200) { + throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.'); + } + if (strlen($answer) > 65536) { + throw new RuntimeException('V4-Antwort ist zu gross.'); + } + $ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR); + if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) { + throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.'); + } + } finally { + curl_close($handle); + } + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt, + 'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + // Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands. + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } + } +} diff --git a/libs/ManagerPrognoseFormular.php b/libs/ManagerPrognoseFormular.php new file mode 100644 index 0000000..fe36a24 --- /dev/null +++ b/libs/ManagerPrognoseFormular.php @@ -0,0 +1,107 @@ + $zustand['jetzt']); + } + + public static function statustext(array $zustand): string + { + $text = ($zustand['aktivtestAktiv'] ?? false) + ? 'Netzfahrplan: befristeter Testbetrieb eingeschaltet.' + : 'Netzfahrplan: Steuerung ausgeschaltet.'; + if ($zustand['legacyAktiv'] ?? false) { + $text = 'Bisheriger Netzfahrplan eingeschaltet; neuer Netzfahrplan gesperrt.'; + } + if (!($zustand['lizenziert'] ?? false)) { + $text .= ' Netzfahrplanlizenz fehlt.'; + } + $ende = $zustand['ende'] ?? 0; + if (is_int($ende) && $ende > 0) { + $text .= ' Bestehendes Testende: ' . gmdate('d.m.Y H:i:s', $ende) . ' UTC.'; + if ($ende <= ($zustand['jetzt'] ?? 0)) { + $text .= ' Testfreigabe abgelaufen.'; + } + } + return $text . ' Kein freigegebener Dauerbetrieb. Bestehende Laufzeit-, Messwert- und Geraeteschutzpruefungen bleiben wirksam.'; + } + + public static function integrieren(array $formular, array $zustand): array + { + $gefunden = false; + $v4Konfiguriert = ($zustand['senderAktiv'] ?? false) + || ($zustand['empfangAktiv'] ?? false) || ($zustand['messdatenAktiv'] ?? false); + foreach ($formular['elements'] as &$element) { + if (($element['name'] ?? null) !== 'PrognoseForecast') { + continue; + } + if ($gefunden) { + throw new LogicException('Prognosebereich ist nicht eindeutig.'); + } + $gefunden = true; + foreach ($element['items'] as &$feld) { + if (($feld['name'] ?? null) === 'NetzfahrplanAktiv' && $v4Konfiguriert) { + // An active legacy controller remains visible so it can still be disabled. + $feld['visible'] = (bool) ($zustand['legacyAktiv'] ?? false); + $feld['enabled'] = (bool) ($zustand['legacyAktiv'] ?? false); + $feld['caption'] = 'Bisheriger Netzfahrplan (konkurrierende Steuerung)'; + } + } + unset($feld); + $element['items'][] = [ + 'type' => 'Button', 'name' => 'PrognoseLizenzVerwalten', + 'caption' => 'Prognose und Netzfahrplan im Lizenzportal verwalten', 'link' => true, + 'onClick' => "echo 'https://license.enelix.ch';", + ]; + $element['items'][] = [ + 'type' => 'CheckBox', 'name' => 'NetzfahrplanV4EmpfangAktiv', + 'caption' => 'Aktuellen Netzfahrplan empfangen und anzeigen', + 'enabled' => (bool) (($zustand['lizenziert'] ?? false) || ($zustand['empfangAktiv'] ?? false)), + ]; + $element['items'][] = [ + 'type' => 'Label', 'name' => 'PrognoseNetzfahrplanStatus', + 'caption' => self::statustext($zustand), + ]; + $element['items'][] = [ + 'type' => 'Button', 'name' => 'PrognoseNetzfahrplanStart', + 'caption' => 'Netzfahrplan einschalten (Testbetrieb)', + 'enabled' => self::startErlaubt($zustand), + 'onClick' => "IPS_RequestAction(\$id, 'FormNetzfahrplanSchalten', true);", + ]; + $element['items'][] = [ + 'type' => 'Button', 'name' => 'PrognoseNetzfahrplanStop', + 'caption' => 'Netzfahrplan ausschalten', 'enabled' => true, + 'onClick' => "IPS_RequestAction(\$id, 'FormNetzfahrplanSchalten', false);", + ]; + $element['items'][] = [ + 'type' => 'Button', 'name' => 'PrognoseNetzfahrplanPruefen', + 'caption' => 'Plan jetzt pruefen', + 'onClick' => "IPS_RequestAction(\$id, 'NetzfahrplanV4Empfangen', true);", + ]; + } + unset($element); + if (!$gefunden) { + throw new LogicException('Prognosebereich fehlt; Formular nicht erweitert.'); + } + return $formular; + } +} diff --git a/libs/NetzfahrplanV4Betriebsdaten.php b/libs/NetzfahrplanV4Betriebsdaten.php new file mode 100644 index 0000000..35495da --- /dev/null +++ b/libs/NetzfahrplanV4Betriebsdaten.php @@ -0,0 +1,219 @@ + $max) { + throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.'); + } + return (float) $value; + } + + private static function timestamp($value): int + { + if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) { + throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.'); + } + return (new DateTimeImmutable($value))->getTimestamp(); + } + + /** $read returns ['value' => int|float, 'updated' => int], without changing a device. */ + private static function measurement(callable $read, int $id, int $now, int $age, string $label): array + { + if ($id <= 0) { + throw new InvalidArgumentException($label . ': Messquelle fehlt.'); + } + $m = $read($id); + if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now + || $now - $m['updated'] > $age) { + throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.'); + } + return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']]; + } + + /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ + private static function evidence(array $input, int $now, string $meterID, array &$warnings): array + { + $peaks = []; + $past = null; + if ($input === []) { + $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; + return [(object) [], null]; + } + if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) { + throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); + } + $measured = self::timestamp($input['measuredAt'] ?? null); + if ($measured > $now || $now - $measured > 120) { + throw new InvalidArgumentException('Messnachweis ist nicht aktuell.'); + } + $month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m'); + foreach (($input['measuredPeaks'] ?? []) as $key => $p) { + if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month + || !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) { + throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.'); + } + $peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']]; + } + if (!array_key_exists($month, $peaks)) { + $warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.'; + } + $q = intdiv($now, 900) * 900; + if (isset($input['quarterPast'])) { + $p = $input['quarterPast']; + // Stale quarter evidence remains absent, not extrapolated to the decision time. + if ($measured === $now && self::timestamp($p['start'] ?? null) === $q + && ($p['measuredSeconds'] ?? null) === $now - $q) { + $past = ['start' => gmdate('c', $q), + 'measuredSeconds' => $now - $q, + 'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)]; + } else { + $warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.'; + } + } + if ($now !== $q && $past === null) { + $warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.'; + } + return [(object) $peaks, $past]; + } + + /** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */ + public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array + { + $warnings = []; + $meter = NetzfahrplanV4Bezugszaehler::lesen( + $manager['V4BezugszaehlerQuellen'] ?? [], $read, $now + ); + $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, + (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); + $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); + $mode = $manager['Lastspitzenmodus'] ?? 0; + $import = null; + $months = []; + if ($mode === 1) { + $import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0); + } elseif ($mode === 2) { + foreach (($manager['Monatsgrenzen'] ?? []) as $row) { + $m = $row['MonatIndex'] ?? null; + if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) { + throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.'); + } + $months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0); + } + if (count($months) !== 12) { + throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.'); + } + } elseif ($mode !== 0) { + throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.'); + } + $export = !empty($manager['EinspeisebegrenzungAktiv']) + ? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null; + $result = []; + $seen = []; + $used = []; + foreach ($assets as $asset) { + $id = $asset['ID'] ?? ''; + if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) { + throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.'); + } + $seen[$id] = true; + $socID = (int) ($asset['SOCVariableID'] ?? 0); + $matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool { + return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID; + })); + if (count($matches) !== 1) { + throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.'); + } + $c = $matches[0]; + $controllerID = $c['InstanzID']; + if (isset($used[$controllerID])) { + throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.'); + } + $used[$controllerID] = true; + if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) { + throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.'); + } + $opts = $manager['BatterieOptionen'][$id] ?? []; + $nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001); + $usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal); + if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) { + throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.'); + } + $capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal); + $age = (int) ($c['MesswertMaxAlter'] ?? 30); + $soc = self::measurement($read, $socID, $now, $age, $id . ' SOC'); + $charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden'); + $discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen'); + $min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100), + self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); + $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); + $socValue = self::number($soc['value'], 'SOC', 0, 100); + $physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min); + if ($socValue < $physicalMin || $socValue > $max) { + throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); + } + if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { + throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.'); + } + $maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0)); + $maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0)); + if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) { + $maxCharge = $maxDischarge = 0.0; + $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; + } + $rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100)); + $blocked = !empty($c['HystereseAktiv']) || $socValue <= $min; + if ($blocked) { + $warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.'; + } + if ($socValue < $min) { + $warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.'; + } + $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, + 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, + 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true, + 'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true, + 'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm]; + } + if ($result === []) { + throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); + } + [$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings); + $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, + 'meteringBoundary' => 'common_pcc', 'batteries' => $result, + 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], + 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + // A fresh acquisition sample is an operational estimate. It does NOT + // claim a calibrated billing-period boundary or full historical coverage. + $policy = [ + 'adapterVersion' => 'v4-native-estimated-meter-1', + 'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified', + 'mode' => $mode, 'importW' => $import, 'exportW' => $export, + 'months' => $months, + 'batteryPolicy' => array_map(static function (array $b): array { + return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent'])); + }, $result), + ]; + $payload['meterObservation'] = [ + 'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now), + 'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'], + 'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)), + ]; + $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; + } +} diff --git a/libs/NetzfahrplanV4Bezugszaehler.php b/libs/NetzfahrplanV4Bezugszaehler.php new file mode 100644 index 0000000..dda9b72 --- /dev/null +++ b/libs/NetzfahrplanV4Bezugszaehler.php @@ -0,0 +1,108 @@ + 8 + || array_keys($sources) !== range(0, count($sources) - 1)) { + throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.'); + } + $result = []; + $parent = null; + foreach ($sources as $source) { + if (!is_array($source) || count($source) !== 5 + || ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') { + throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.'); + } + $id = $source['VariableID'] ?? null; + $pid = $source['ElternID'] ?? null; + $ident = $source['Ident'] ?? null; + $factor = $source['FaktorZuKWh'] ?? null; + if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0 + || !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) + || $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) { + throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.'); + } + if ($parent !== null && $parent !== $pid) { + throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.'); + } + $parent = $pid; + foreach ($result as $existing) { + if ($existing['Ident'] === $ident) { + throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.'); + } + } + $result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident, + 'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug']; + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + public static function identitaet(array $sources): string + { + // Source/factor changes invalidate old measurement evidence; names do not matter. + return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources), + JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + } + + private static function probe(array $source, callable $read, int $now, int $maxAge): array + { + $m = $read($source['VariableID']); + if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID'] + || ($m['ident'] ?? null) !== $source['Ident']) { + throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.'); + } + $value = $m['value'] ?? null; + $updated = $m['updated'] ?? null; + if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0 + || !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) { + throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.'); + } + $kwh = (float) $value * $source['FaktorZuKWh']; + if (!is_finite($kwh) || $kwh > 1.0e12) { + throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.'); + } + return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value, + 'totalKwh' => $kwh, 'updatedAt' => $updated]; + } + + /** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */ + public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array + { + if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) { + throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.'); + } + $sources = self::quellen($sources); + $samples = []; + foreach ($sources as $source) { + $samples[] = self::probe($source, $read, $now, $maxAge); + } + // Reread all channels to reject concurrent updates, including changes in the same second. + foreach ($sources as $i => $source) { + if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) { + throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.'); + } + } + $times = array_column($samples, 'updatedAt'); + if (max($times) - min($times) > $maxSkew) { + throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.'); + } + return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh', + 'totalKwh' => array_sum(array_column($samples, 'totalKwh')), + 'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)), + 'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples, + 'billingEvidence' => false, 'historyComplete' => false]; + } +} diff --git a/libs/NetzfahrplanV4Bilanzierung.php b/libs/NetzfahrplanV4Bilanzierung.php new file mode 100644 index 0000000..a468dc4 --- /dev/null +++ b/libs/NetzfahrplanV4Bilanzierung.php @@ -0,0 +1,121 @@ + $max) { + throw new InvalidArgumentException($name . ': ungueltiger Zahlenwert.'); + } + return (float) $v; + } + + private static function listArray(array $a): bool + { + return $a === [] || array_keys($a) === range(0, count($a)-1); + } + + public static function konfigurieren(array $c): array + { + $fields = ['version','sources','maxSkewSeconds','splitToleranceW']; + if (count($c) !== count($fields) || array_diff($fields,array_keys($c)) || $c['version'] !== 1 + || !is_array($c['sources']) || !self::listArray($c['sources']) || count($c['sources']) < 3 || count($c['sources']) > 80 + || !is_int($c['maxSkewSeconds']) || $c['maxSkewSeconds'] < 0 || $c['maxSkewSeconds'] > 60) { + throw new InvalidArgumentException('Bilanzkonfiguration unvollstaendig.'); + } + // Preserve a stable numeric representation across JSON property round-trips. + $c['splitToleranceW'] = self::number($c['splitToleranceW'], 'Zuordnungstoleranz W', 0, 500); + $seen=[];$keys=[];$roles=[]; + foreach ($c['sources'] as &$s) { + if (!is_array($s) || count($s)!==8 || array_diff(['key','role','variableId','parentId','ident','factorToW','maxAgeSeconds','dependsOn'],array_keys($s))) { + throw new InvalidArgumentException('Explizite Quellenzuordnung erforderlich.'); + } + if (!is_string($s['key']) || !preg_match('/^[A-Za-z][A-Za-z0-9_-]{0,63}$/D',$s['key']) || isset($keys[$s['key']]) + || !in_array($s['role'], ['grid','pv','physical_storage','controlled_virtual','external_virtual','flexible_load','reference'],true) + || !is_int($s['variableId']) || $s['variableId']<=0 || isset($seen[$s['variableId']]) + || !is_int($s['parentId']) || $s['parentId']<=0 || !is_string($s['ident']) || strlen($s['ident'])>100 + || !is_int($s['maxAgeSeconds']) || $s['maxAgeSeconds']<1 || $s['maxAgeSeconds']>300) { + throw new InvalidArgumentException('Doppelte oder ungueltige Bilanzquelle.'); + } + $s['factorToW']=self::number($s['factorToW'],'Einheiten-/Vorzeichenfaktor',-1e6,1e6); + if ($s['factorToW']==0 || !is_array($s['dependsOn']) || !self::listArray($s['dependsOn']) || count($s['dependsOn'])>40) { + throw new InvalidArgumentException('Abhaengigkeiten oder Faktor fehlen.'); + } + foreach ($s['dependsOn'] as $dep) if (!is_string($dep)) throw new InvalidArgumentException('Quellenschluessel erforderlich.'); + if (count(array_unique($s['dependsOn']))!==count($s['dependsOn'])) throw new InvalidArgumentException('Doppelte Abhaengigkeit.'); + sort($s['dependsOn']);$keys[$s['key']]=true;$seen[$s['variableId']]=true;$roles[$s['role']]=($roles[$s['role']]??0)+1; + } + unset($s); + if (($roles['grid']??0)!==1 || !isset($roles['pv'],$roles['physical_storage'],$roles['controlled_virtual'])) { + throw new InvalidArgumentException('Netz, PV, physische und steuerbare virtuelle Batterie muessen explizit zugeordnet sein.'); + } + $graph=[];foreach ($c['sources'] as $s) $graph[$s['key']]=$s['dependsOn']; + $visit=function($key,$stack=[]) use (&$visit,$graph): void { + if (!array_key_exists($key,$graph) || isset($stack[$key])) throw new InvalidArgumentException('Fehlende oder zyklische Ursprungsquelle.'); + $stack[$key]=true;foreach ($graph[$key] as $dep) $visit($dep,$stack); + }; + foreach (array_keys($graph) as $key) $visit($key); + usort($c['sources'],static fn($a,$b)=>strcmp($a['key'],$b['key']));return $c; + } + + /** Reader: {value:number, updated:int, parentID:int, ident:string}. */ + public static function aufnehmen(array $configuration, callable $read, int $now): array + { + if ($now<=0) throw new InvalidArgumentException('Ungueltige Aufnahmezeit.'); + $c=self::konfigurieren($configuration);$samples=[];$sources=[];$issues=[]; + $probe=static function(array $s) use ($read): array { + $v=$read($s['variableId']); + if (!is_array($v) || ($v['parentID']??null)!==$s['parentId'] || ($v['ident']??null)!==$s['ident'] + || !is_int($v['updated']??null) || $v['updated']<=0) throw new InvalidArgumentException('Quelle/Originalzeitpunkt ungueltig: '.$s['key']); + return ['raw'=>self::number($v['value']??null,$s['key']), 'updated'=>$v['updated']]; + }; + foreach ($c['sources'] as $s) {$sources[$s['key']]=$s;$samples[$s['key']]=$probe($s);} + foreach ($c['sources'] as $s) if ($samples[$s['key']]!==$probe($s)) throw new InvalidArgumentException('Aufnahme waehrend des Lesens geaendert: '.$s['key']); + $origin=function($key) use (&$origin,$sources,$samples): int { + $stamp=$samples[$key]['updated'];foreach ($sources[$key]['dependsOn'] as $dep) $stamp=min($stamp,$origin($dep));return $stamp; + }; + $totals=array_fill_keys(['grid','pv','physical_storage','controlled_virtual','external_virtual','flexible_load','reference'],0.0); + $out=[];$times=[]; + foreach ($sources as $key=>$s) { + $t=$origin($key);$w=$samples[$key]['raw']*$s['factorToW']; + if (!is_finite($w) || abs($w)>1e9) throw new InvalidArgumentException('Leistung ausserhalb des Bereichs.'); + if ($samples[$key]['updated']>$now || $t>$now) $issues[]='future_source:'.$key; + if ($now-$t>$s['maxAgeSeconds']) $issues[]='stale_origin:'.$key; + if (in_array($s['role'],['pv','flexible_load'],true) && $w<0) $issues[]='negative_unsigned_source:'.$key; + $totals[$s['role']]+=$w; + if ($s['role']!=='reference') $times[]=$t; + $out[$key]=['variableId'=>$s['variableId'],'role'=>$s['role'],'powerW'=>$w,'sourceUpdatedAt'=>$samples[$key]['updated'],'oldestOriginAt'=>$t]; + } + $skew=max($times)-min($times);if ($skew>$c['maxSkewSeconds']) $issues[]='source_time_skew'; + $load=$totals['grid']+$totals['pv']-$totals['physical_storage']; + $base=$load-$totals['flexible_load']; + // This channel preserves unallocated dynamics instead of hiding them in house load. + $residual=$totals['physical_storage']-$totals['controlled_virtual']-$totals['external_virtual']; + $external=$totals['external_virtual']+$residual; + if ($base<0) $issues[]='negative_base_load'; + if (abs($residual)>$c['splitToleranceW']) $issues[]='virtual_physical_allocation_mismatch'; + $reconstructed=$base+$totals['flexible_load']+$totals['controlled_virtual']+$external-$totals['pv']; + $identity=hash('sha256',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + return ['version'=>1,'status'=>$issues===[]?'consistent_candidate':'quality_hold', + 'capturedAt'=>gmdate('c',$now),'sourceObservedFrom'=>gmdate('c',min($times)),'sourceObservedUntil'=>gmdate('c',max($times)), + 'sourceSkewSeconds'=>$skew,'mappingId'=>'accounting-map:'.$identity, + 'baseLoadCandidateW'=>$base,'consumerLoadCandidateW'=>$load,'pvW'=>$totals['pv'],'gridW'=>$totals['grid'], + 'physicalStorageW'=>$totals['physical_storage'],'controlledVirtualW'=>$totals['controlled_virtual'], + 'flexibleLoadW'=>$totals['flexible_load'],'reportedExternalW'=>$totals['external_virtual'], + 'allocationResidualW'=>$residual,'externalEffectW'=>$external,'reconstructedGridW'=>$reconstructed, + 'arithmeticBalanceErrorW'=>$reconstructed-$totals['grid'],'issues'=>$issues,'sources'=>$out, + 'controlEligible'=>false,'meteringBoundaryVerified'=>false,'forecastHistoryMigrated'=>false, + 'note'=>'Algebraic closure is not independent meter validation. No relabelling of old forecasts, no future SDL assumption.']; + } +} diff --git a/libs/NetzfahrplanV4Datenarchiv.php b/libs/NetzfahrplanV4Datenarchiv.php new file mode 100644 index 0000000..3211530 --- /dev/null +++ b/libs/NetzfahrplanV4Datenarchiv.php @@ -0,0 +1,78 @@ +0]; + if (!is_string($day)||!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day) + ||!is_int($cursor['offset']??null)||$cursor['offset']<0 + ||is_link($directory)||realpath($directory)!==$directory) { + throw new \RuntimeException('Invalid archive cursor/directory'); + } + $cutoff='raw-'.gmdate('Ymd',$now-7*86400).'.jsonl'; + $archive=$directory.'/archive'; + if (is_link($archive)) throw new \RuntimeException('Archive symlink refused'); + if (!is_dir($archive)&&!mkdir($archive,0700)) throw new \RuntimeException('Archive directory unavailable'); + if ((fileperms($archive)&0007)!==0) throw new \RuntimeException('Archive must remain private'); + $lockPath=$directory.'/.writer.lock'; + if (is_link($lockPath)) throw new \RuntimeException('Writer lock symlink refused'); + $lock=fopen($lockPath,'c+b'); + if ($lock===false) throw new \RuntimeException('Archive lock unavailable'); + $done=[]; + try { + if (!flock($lock,LOCK_EX|LOCK_NB)) return ['archived'=>0,'status'=>'writer_busy']; + foreach (glob($directory.'/raw-*.jsonl')?:[] as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||$name>=$day||$name>=$cutoff) continue; + if (is_link($file)||realpath($file)!==$file||!is_file($file)) throw new \RuntimeException('Unexpected journal'); + $before=stat($file); + if (!$before||$before['size']>67108864) throw new \RuntimeException('Archive input bounds'); + $digest=hash_file('sha256',$file); + if (!is_string($digest)) throw new \RuntimeException('Cannot hash journal'); + $target=$archive.'/'.$name.'.'.$digest.'.gz'; + if (is_link($target)) throw new \RuntimeException('Archive target symlink'); + if (!is_file($target)) { + $tmp=tempnam($archive,'.pending-'); + if ($tmp===false) throw new \RuntimeException('Cannot create archive'); + try { + chmod($tmp,0600);$in=fopen($file,'rb');$gz=gzopen($tmp,'wb6'); + if ($in===false||$gz===false) {if(is_resource($in))fclose($in);if(is_resource($gz))gzclose($gz);throw new \RuntimeException('Cannot open archive streams');} + try { + while (!feof($in)) { + $block=fread($in,65536); + if ($block===false) throw new \RuntimeException('Archive read failed'); + $offset=0; + while($offset$before['size'])throw new \RuntimeException('Oversized decompressed archive');hash_update($hash,$block);}} + finally {gzclose($gz);} + clearstatcache(true,$file);$after=stat($file); + if ($bytes!==$before['size']||hash_final($hash)!==$digest||!$after + ||$after['size']!==$before['size']||$after['ino']!==$before['ino']||$after['mtime']!==$before['mtime'] + ||hash_file('sha256',$file)!==$digest) throw new \RuntimeException('Archive mismatch; original retained'); + $directoryHandle=fopen($archive,'r'); + if ($directoryHandle===false) throw new \RuntimeException('Cannot sync archive directory'); + try {if(!fsync($directoryHandle))throw new \RuntimeException('Archive directory sync failed');} finally {fclose($directoryHandle);} + if (!unlink($file)) throw new \RuntimeException('Verified original could not be retired'); + $done[]=$name; + if (count($done)>=2) break; + } + } finally {flock($lock,LOCK_UN);fclose($lock);} + return ['archived'=>count($done),'compressedOriginalsRetained'=>true,'unacknowledgedDaysTouched'=>false]; + } +} diff --git a/libs/NetzfahrplanV4Geraeteabruf.php b/libs/NetzfahrplanV4Geraeteabruf.php new file mode 100644 index 0000000..5e33a01 --- /dev/null +++ b/libs/NetzfahrplanV4Geraeteabruf.php @@ -0,0 +1,115 @@ + $sources) { + $module = $moduleId($parentId); + if (!in_array($module, [self::MBUS_DEVICE, self::MODBUS_DEVICE, self::MODBUS_ADDRESS], true)) { + throw new InvalidArgumentException('feedback_refresh_module_unsupported:' . $sources[0]['key']); + } + $confirmed = false; + for ($attempt = 1; $attempt <= 4; ++$attempt) { + if ($refresh($parentId, $module) === true) { + $confirmed = true; + break; + } + if ($attempt < 4 && $pause !== null) { + $pause(200000); + } + } + if (!$confirmed) { + throw new InvalidArgumentException('feedback_device_read_failed:' . $sources[0]['key']); + } + $confirmedAt = $clock(); + if (!is_int($confirmedAt) || $confirmedAt < 1) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + foreach ($sources as $source) { + $record = $reader($source['variableId']); + if (!is_array($record)) { + throw new InvalidArgumentException('feedback_reader_invalid:' . $source['key']); + } + $record['confirmedAt'] = $confirmedAt; + $record['confirmedModuleId'] = $module; + $snapshot[$source['variableId']] = $record; + } + } + + foreach ($config['sources'] as $source) { + if (!isset($snapshot[$source['variableId']])) { + $record = $reader($source['variableId']); + if (!is_array($record)) { + throw new InvalidArgumentException('feedback_reader_invalid:' . $source['key']); + } + $snapshot[$source['variableId']] = $record; + } + } + + foreach ($snapshot as $variableId => $record) { + $again = $reader($variableId); + // The variable may legitimately receive a newer process value immediately after + // the explicit device read. Keep the confirmed snapshot, but require the source + // identity to remain stable so a reconfigured/reparented variable is never accepted. + $identityKeys = ['parentID', 'ident']; + if (!is_array($again) || array_intersect_key($again, array_flip($identityKeys)) + !== array_intersect_key($record, array_flip($identityKeys))) { + throw new InvalidArgumentException( + 'feedback_source_changed_after_confirmed_read:' . $sourcesById[$variableId]['key'] + ); + } + } + + $checkedAt = $clock(); + if (!is_int($checkedAt) || $checkedAt < 1) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + foreach ($snapshot as $record) { + if (isset($record['confirmedAt']) && $record['confirmedAt'] > $checkedAt) { + throw new InvalidArgumentException('feedback_confirmation_time_invalid'); + } + } + + return NetzfahrplanV4Rueckmeldung::read( + $config, + static fn(int $variableId): array => $snapshot[$variableId], + $checkedAt + ); + } +} diff --git a/libs/NetzfahrplanV4Messaufnahme.php b/libs/NetzfahrplanV4Messaufnahme.php new file mode 100644 index 0000000..19af299 --- /dev/null +++ b/libs/NetzfahrplanV4Messaufnahme.php @@ -0,0 +1,165 @@ + 60 + || !is_string($c['reportedInventorySha256'] ?? null) + || !preg_match('/^[0-9a-f]{64}$/D', $c['reportedInventorySha256'])) { + throw new InvalidArgumentException('Invalid capture configuration'); + } + $c['accounting'] = NetzfahrplanV4Bilanzierung::konfigurieren($c['accounting'] ?? []); + $keys = []; $ids = []; + foreach ($c['accounting']['sources'] as $s) { $keys[$s['key']] = true; $ids[$s['variableId']] = true; } + foreach ($c['extraSources'] as $s) { + if (count($s) !== 5 || !is_string($s['key'] ?? null) + || !preg_match('/^[A-Za-z][A-Za-z0-9_-]{0,63}$/D', $s['key']) || isset($keys[$s['key']]) + || !is_int($s['variableId'] ?? null) || $s['variableId'] < 1 || isset($ids[$s['variableId']]) + || !is_int($s['parentId'] ?? null) || $s['parentId'] < 1 + || !is_string($s['ident'] ?? null) || strlen($s['ident']) > 100 + || !in_array($s['unit'] ?? null, ['W','kWh','percent','raw_power','scale'], true)) { + throw new InvalidArgumentException('Invalid or duplicate raw source'); + } + $keys[$s['key']] = true; $ids[$s['variableId']] = true; + } + usort($c['extraSources'], static fn(array $a, array $b): int => strcmp($a['key'], $b['key'])); + return $c; + } + + private static function readSource(array $s, callable $read): array + { + try { + $r = $read($s['variableId']); + if (!is_array($r)) throw new RuntimeException('Missing reading'); + } catch (Throwable $e) { + return ['value' => null, 'sourceUpdatedAt' => null, 'sourceChangedAt' => null, 'issues' => ['read_failed']]; + } + $issues = []; + $v = $r['value'] ?? null; + if ((!is_int($v) && !is_float($v)) || !is_finite((float) $v) || abs((float)$v) > 1.0e12) { + $v = null; $issues[] = 'invalid_numeric_value'; + } + $at = $r['updated'] ?? null; + if (!is_int($at) || $at < 1) { $at = null; $issues[] = 'missing_original_timestamp'; } + $changed = $r['changed'] ?? null; + if (!is_int($changed) || $changed < 0) $changed = null; + if (($r['parentID'] ?? null) !== $s['parentId'] || ($r['ident'] ?? null) !== $s['ident']) { + // Do not record a numeric value from an object re-used for a different purpose. + $v = null; $issues[] = 'source_identity_mismatch'; + } + return ['value' => $v, 'sourceUpdatedAt' => $at, 'sourceChangedAt' => $changed, 'issues' => $issues]; + } + + public static function capture(array $config, callable $read, callable $clock): array + { + $c = self::configuration($config); + $start = $clock(); + if (!is_int($start) || $start <= 0) throw new InvalidArgumentException('Integer UTC epoch required'); + $sources = array_merge($c['accounting']['sources'], $c['extraSources']); + $first = []; $second = []; $raw = []; $issues = []; + foreach ($sources as $s) $first[$s['key']] = self::readSource($s, $read); + foreach ($sources as $s) $second[$s['key']] = self::readSource($s, $read); + $end = $clock(); + if (!is_int($end) || $end < $start) throw new RuntimeException('Acquisition clock moved backwards'); + foreach ($sources as $s) { + $key = $s['key']; $r = $first[$key]; + if ($r !== $second[$key]) { + $r['issues'][] = 'changed_during_capture'; + $r['secondValue'] = $second[$key]['value']; + $r['secondUpdatedAt'] = $second[$key]['sourceUpdatedAt']; + } + if ($r['sourceUpdatedAt'] !== null && $r['sourceUpdatedAt'] > $end) $r['issues'][] = 'future_source'; + $r['sourceAgeSeconds'] = $r['sourceUpdatedAt'] === null ? null : $end - $r['sourceUpdatedAt']; + $raw[$key] = ['variableId' => $s['variableId']] + $r; + foreach ($r['issues'] as $issue) $issues[] = $key . ':' . $issue; + } + $assessment = null; $accountingUsable = true; $samples = []; + foreach ($c['accounting']['sources'] as $s) { + $r = $raw[$s['key']]; + if ($r['issues'] !== []) $accountingUsable = false; + $samples[$s['variableId']] = ['value' => $r['value'], 'updated' => $r['sourceUpdatedAt'], 'parentID' => $s['parentId'], 'ident' => $s['ident']]; + } + if ($accountingUsable) { + try { $assessment = NetzfahrplanV4Bilanzierung::aufnehmen($c['accounting'], static fn(int $id): array => $samples[$id], $end); } + catch (Throwable $e) { $issues[] = 'accounting_input_invalid'; } + } else { $issues[] = 'accounting_snapshot_unusable'; } + if ($end - $start > 5) $issues[] = 'long_acquisition'; + return ['schemaVersion' => 1, 'kind' => 'raw_accounting_capture', 'installationId' => $c['installationId'], + 'captureStartedAt' => gmdate('c', $start), 'capturedAt' => gmdate('c', $end), + 'mappingSha256' => hash('sha256', json_encode($c, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)), + 'reportedInventorySha256' => $c['reportedInventorySha256'], + 'timestampMeaning' => 'Symcon VariableUpdated; not proof of an independent device timestamp', + 'raw' => $raw, 'issues' => $issues, 'assessment' => $assessment, + 'trainingEligible' => false, 'controlEligible' => false, 'measurementBoundaryVerified' => false, + 'futureSdl' => 'unknown_no_assumption', 'historicalDataRewritten' => false]; + } + + /** Append-only daily JSONL with exclusive lock, bounds and crash-tail detection. + * No retention deletion; quota stops capture visibly, never deletes old histories. + */ + public static function append(string $directory, array $record, int $quota = 536870912): string + { + if (!is_dir($directory) || is_link($directory) || realpath($directory) !== $directory || $quota < 1) { + throw new RuntimeException('Invalid dedicated data directory'); + } + $json = json_encode($record, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION) . "\n"; + if (strlen($json) > 262144) throw new RuntimeException('Capture exceeds record limit'); + $epoch = strtotime($record['capturedAt'] ?? ''); + if ($epoch === false) throw new RuntimeException('Capture timestamp missing'); + $file = $directory . '/raw-' . gmdate('Ymd', $epoch) . '.jsonl'; + $lockPath = $directory . '/.writer.lock'; + if (is_link($lockPath) || is_link($file)) throw new RuntimeException('Symbolic link refused'); + $lock = fopen($lockPath, 'c+b'); + if ($lock === false) throw new RuntimeException('Capture lock unavailable'); + try { + if (!flock($lock, LOCK_EX | LOCK_NB)) throw new RuntimeException('Capture writer busy'); + clearstatcache(); $used = 0; $files = glob($directory . '/raw-*.jsonl'); + if ($files === false || count($files) > 400) throw new RuntimeException('Data directory bounds exceeded'); + foreach ($files as $p) { + if (is_link($p)) throw new RuntimeException('Symbolic journal refused'); + $size = filesize($p); if ($size === false) throw new RuntimeException('Cannot size journal'); + $used += $size; + } + if ($used + strlen($json) > $quota) throw new RuntimeException('Capture quota reached; no data deleted'); + $free = disk_free_space($directory); + if ($free === false || $free < 104857600 + strlen($json)) throw new RuntimeException('Insufficient free disk; capture paused'); + $h = fopen($file, 'c+b'); + if ($h === false) throw new RuntimeException('Cannot open capture journal'); + try { + fseek($h, 0, SEEK_END); $old = ftell($h); + if ($old === false || $old + strlen($json) > 67108864) throw new RuntimeException('Daily capture bounds exceeded'); + if ($old > 0) { + fseek($h, -1, SEEK_END); + if (fread($h, 1) !== "\n") throw new RuntimeException('Incomplete journal tail; preserve for review'); + } + fseek($h, 0, SEEK_END); $n = 0; + try { + while ($n < strlen($json)) { + $written = fwrite($h, substr($json, $n)); + if ($written === false || $written === 0) throw new RuntimeException('Capture write incomplete'); + $n += $written; + } + if (!fflush($h)) throw new RuntimeException('Capture flush failed'); + } catch (Throwable $e) { ftruncate($h, $old); fflush($h); throw $e; } + if (!chmod($file, 0640)) throw new RuntimeException('Cannot restrict journal permissions'); + } finally { fclose($h); } + return $file; + } finally { flock($lock, LOCK_UN); fclose($lock); } + } +} diff --git a/libs/NetzfahrplanV4Messsicht.php b/libs/NetzfahrplanV4Messsicht.php new file mode 100644 index 0000000..b390b19 --- /dev/null +++ b/libs/NetzfahrplanV4Messsicht.php @@ -0,0 +1,162 @@ + 300 + || !self::numeric($p['trackingToleranceW']) || $p['trackingToleranceW'] < 0 || $p['trackingToleranceW'] > 1000) { + throw new InvalidArgumentException('Invalid measurement-view policy'); + } + $extra = array_column($c['extraSources'], null, 'key'); + foreach (['evRequestKey','sdlRequestKey'] as $k) { + if (!is_string($p[$k]) || !isset($extra[$p[$k]]) || $extra[$p[$k]]['unit'] !== 'W') { + throw new InvalidArgumentException('Explicit separate request channels required'); + } + } + if ($p['evRequestKey'] === $p['sdlRequestKey']) throw new InvalidArgumentException('Duplicate request channel'); + if ($p['solarReference'] !== null) { + $s=$p['solarReference']; $keys=['pvKey','batteryKey','rawKey','scaleKey']; + if (!is_array($s) || count($s)!==4 || array_diff($keys,array_keys($s))) throw new InvalidArgumentException('Invalid solar-origin policy'); + foreach ($keys as $k) if (!is_string($s[$k])) throw new InvalidArgumentException('Invalid solar-origin key'); + $primary=array_column($c['accounting']['sources'],null,'key'); + if (($primary[$s['pvKey']]['role']??null)!=='pv' || ($primary[$s['batteryKey']]['role']??null)!=='physical_storage' + || ($extra[$s['rawKey']]['unit']??null)!=='raw_power' || ($extra[$s['scaleKey']]['unit']??null)!=='scale') { + throw new InvalidArgumentException('Solar references must be explicit original sources'); + } + } + return $p; + } + + public static function calculate(array $r, array $config, array $policy): array + { + $c=NetzfahrplanV4Messaufnahme::configuration($config);$p=self::policy($policy,$c); + $mapping=hash('sha256',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + if (($r['kind']??null)!=='raw_accounting_capture' || ($r['schemaVersion']??null)!==1 + || ($r['installationId']??null)!==$c['installationId'] || ($r['mappingSha256']??null)!==$mapping + || ($r['reportedInventorySha256']??null)!==$c['reportedInventorySha256']) throw new InvalidArgumentException('Wrong capture identity'); + foreach (['capturedAt','captureStartedAt'] as $k) { + if (!is_string($r[$k]??null) || !preg_match('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$/D',$r[$k])) throw new InvalidArgumentException('Explicit UTC capture time required'); + $date=new \DateTimeImmutable($r[$k]);$err=\DateTimeImmutable::getLastErrors(); + if ($err!==false && ($err['warning_count']||$err['error_count'])) throw new InvalidArgumentException('Invalid capture time'); + $time[$k]=$date->getTimestamp(); + } + $now=$time['capturedAt'];$duration=$now-$time['captureStartedAt']; + if ($duration<0) throw new InvalidArgumentException('Reversed acquisition clock'); + $sources=array_column(array_merge($c['accounting']['sources'],$c['extraSources']),null,'key'); + $v=[]; + foreach ($sources as $key=>$s) { + $raw=$r['raw'][$key]??[]; $issues=[]; $value=$raw['value']??null; $updated=$raw['sourceUpdatedAt']??null; + if (($raw['variableId']??null)!==$s['variableId']) {$issues[]='identity';$value=null;} + if (!self::numeric($value)) {$issues[]='not_numeric';$value=null;} + if (!is_int($updated) || $updated<=0) {$issues[]='timestamp';$updated=null;} + if (($raw['issues']??null)!==[]) $issues[]='capture_quality'; // never copy arbitrary source text + if ($updated!==null && ($updated>$now || $now-$updated>($s['maxAgeSeconds']??$p['extraMaxAgeSeconds']))) $issues[]='source_age'; + $scaled=$value===null?null:(float)$value*($s['factorToW']??1.0); + if ($scaled!==null && (!is_finite($scaled)||abs($scaled)>1e12)) {$scaled=null;$issues[]='converted_value';} + $changed=$raw['sourceChangedAt']??null; + if (!is_int($changed)||$changed<0||$updated===null||$changed>$updated) $changed=null; + $v[$key]=['value'=>$scaled,'updated'=>$updated,'changed'=>$changed,'issues'=>$issues]; + } + $closure=function(array $keys)use(&$closure,$sources):array { + $out=[];foreach($keys as $key){if(!isset($sources[$key]))throw new InvalidArgumentException('Missing origin');$out[]=$key; + foreach($closure($sources[$key]['dependsOn']??[]) as $dep)$out[]=$dep;} + return array_values(array_unique($out)); + }; + $quality=function(array $keys)use($closure,$v,$c,$duration):array { + $keys=$closure($keys);$issues=[];$ts=[]; + foreach($keys as $key){foreach($v[$key]['issues'] as $i)$issues[]=$key.':'.$i;if($v[$key]['updated']!==null)$ts[]=$v[$key]['updated'];} + $skew=$ts===[]?null:max($ts)-min($ts); + if($duration>5)$issues[]='long_acquisition'; + if($skew!==null && $skew>$c['accounting']['maxSkewSeconds'])$issues[]='source_time_skew'; + return ['status'=>$issues===[]?'consistent_candidate':'quality_hold','issues'=>$issues,'oldestSourceUpdatedAt'=>$ts===[]?null:min($ts),'sourceSkewSeconds'=>$skew]; + }; + $byRole=[];foreach($c['accounting']['sources'] as $s)$byRole[$s['role']][]=$s['key']; + $physical=$byRole['physical_storage'];$pv=$byRole['pv'];$grid=$byRole['grid'];$flex=$byRole['flexible_load']??[]; + $sum=function(array $keys)use($v):?float { $total=0.0;foreach($keys as $key){if($v[$key]['value']===null)return null;$total+=$v[$key]['value'];}return $total;}; + $storage=$quality($physical);$storage['physicalStorageCandidateW']=$sum($physical); + $solar=['status'=>'not_configured','acTerminalCandidateW'=>null,'reconstructedPvCandidateW'=>null,'derivedPvDifferenceW'=>null,'issues'=>['origin_not_configured']]; + $loadKeys=array_merge($grid,$pv,$physical,$flex);$sr=$p['solarReference']; + if($sr!==null){ + $origin=[$sr['rawKey'],$sr['scaleKey'],$sr['batteryKey']];$loadKeys=array_merge($loadKeys,$origin); + $solar=$quality($origin);$raw=$v[$sr['rawKey']]['value'];$sf=$v[$sr['scaleKey']]['value']; + $solar+=['acTerminalCandidateW'=>null,'reconstructedPvCandidateW'=>null,'derivedPvDifferenceW'=>null]; + // int16 invalid sentinel excluded, including scale sentinel; no unsigned guessing. + if($raw===null||$sf===null||floor($raw)!==$raw||$raw<=-32768||$raw>32767||floor($sf)!==$sf||$sf < -6||$sf>6){ + $solar['status']='quality_hold';$solar['issues'][]='invalid_raw_or_scale'; + }else{ + $ac=$raw*(10**(int)$sf); + if(abs($ac)>1e9){$solar['status']='quality_hold';$solar['issues'][]='scaled_power_range';} + else{$solar['acTerminalCandidateW']=$ac;$b=$v[$sr['batteryKey']]['value']; + if($b!==null)$solar['reconstructedPvCandidateW']=$ac+$b; + $reported=$v[$sr['pvKey']]['value']; + if($reported!==null&&$solar['reconstructedPvCandidateW']!==null)$solar['derivedPvDifferenceW']=$reported-max(0.0,$solar['reconstructedPvCandidateW']);} + } + } + $load=$quality($loadKeys);$net=$sum($grid);$pvSum=$sum($pv);$phys=$sum($physical);$fl=$sum($flex); + $candidate=($net===null||$pvSum===null||$phys===null||$fl===null)?null:$net+$pvSum-$phys-$fl; + if($candidate!==null&&$candidate<0){$load['status']='quality_hold';$load['issues'][]='negative_load';} + foreach(array_merge($pv,$flex) as $key)if($v[$key]['value']!==null&&$v[$key]['value']<0){$load['status']='quality_hold';$load['issues'][]='negative_unsigned:'.$key;} + if($sr!==null&&$solar['status']!=='consistent_candidate'){$load['status']='quality_hold';$load['issues'][]='solar_origin_unverified_value';} + $load+=['loadCandidateW'=>$candidate,'gridW'=>$net,'pvCandidateW'=>$pvSum,'physicalStorageCandidateW'=>$phys,'flexibleLoadW'=>$fl, + 'virtualAllocationUsed'=>false,'measurementBoundaryVerified'=>false]; + // AC terminal alternative eliminates SolarEdge battery/PV cancellation and zero-clamp artefact. + $alternative=null;$altQuality=null; + if($sr!==null){ + $altKeys=array_values(array_diff(array_merge($grid,$pv,$physical,$flex),[$sr['pvKey'],$sr['batteryKey']])); + $altKeys=array_merge($altKeys,[$sr['rawKey'],$sr['scaleKey']]);$altQuality=$quality($altKeys); + $pvRest=$sum(array_values(array_diff($pv,[$sr['pvKey']])));$bRest=$sum(array_values(array_diff($physical,[$sr['batteryKey']]))); + if($net!==null&&$pvRest!==null&&$bRest!==null&&$fl!==null&&$solar['acTerminalCandidateW']!==null){ + $alternative=$net+$pvRest-$bRest+$solar['acTerminalCandidateW']-$fl; + if($alternative<0){$altQuality['status']='quality_hold';$altQuality['issues'][]='negative_load';} + }else{$altQuality['status']='quality_hold';$altQuality['issues'][]='solar_ac_unavailable';} + foreach(array_merge(array_values(array_diff($pv,[$sr['pvKey']])),$flex) as $key)if($v[$key]['value']!==null&&$v[$key]['value']<0){$altQuality['status']='quality_hold';$altQuality['issues'][]='negative_unsigned:'.$key;} + $altQuality+=['loadCandidateW'=>$alternative,'measurementBoundaryVerified'=>false,'solarBatterySubtractedAgain'=>false]; + } + $requestKeys=[$p['evRequestKey'],$p['sdlRequestKey']];$alloc=$quality(array_merge($physical,$requestKeys)); + $ev=$v[$p['evRequestKey']]['value'];$sdl=$v[$p['sdlRequestKey']]['value'];$evModel=null;$sdlModel=null;$unallocated=$phys;$delta=null; + foreach($requestKeys as $key){ + if($v[$key]['changed']===null){$alloc['issues'][]='request_change_time_unknown:'.$key;$alloc['status']='quality_hold';} + elseif($storage['oldestSourceUpdatedAt']!==null && $v[$key]['changed']>$storage['oldestSourceUpdatedAt']){$alloc['issues'][]='request_newer_than_feedback:'.$key;$alloc['status']='quality_hold';} + } + if($phys!==null&&$ev!==null&&$sdl!==null){ + $abs=abs($ev)+abs($sdl);$delta=$phys-$ev-$sdl; + if($abs>=1.0){$evModel=$ev+$delta*abs($ev)/$abs;$sdlModel=$phys-$evModel;$unallocated=0.0; + if(abs($delta)>$p['trackingToleranceW']){$alloc['issues'][]='physical_request_tracking_error';$alloc['status']='quality_hold';} + }else{$alloc['issues'][]='idle_power_unallocated';$alloc['status']='quality_hold';} + } + $reportedEv=$sum($byRole['controlled_virtual']);$reportedSdl=$sum($byRole['external_virtual']??[]); + $reportedResidual=($phys===null||$reportedEv===null||$reportedSdl===null)?null:$phys-$reportedEv-$reportedSdl; + $alloc+=['physicalStorageCandidateW'=>$phys,'requestedEvW'=>$ev,'requestedSdlW'=>$sdl,'unfilteredModelEvW'=>$evModel, + 'unfilteredModelSdlW'=>$sdlModel,'unallocatedPhysicalW'=>$unallocated,'physicalRequestErrorW'=>$delta, + 'reportedEvW'=>$reportedEv,'reportedSdlW'=>$reportedSdl,'reportedAllocationResidualW'=>$reportedResidual, + 'reportedVirtualQuality'=>$quality(array_merge($byRole['controlled_virtual'],$byRole['external_virtual']??[])), + 'underlyingTimesMeaning'=>'Symcon VariableUpdated; not independently measured EV/SDL channel times', + 'reportedAllocationWithinTolerance'=>$reportedResidual!==null&&abs($reportedResidual)<=$c['accounting']['splitToleranceW'], + 'separatelyMeasured'=>false,'method'=>'request_weighted_residual_unfiltered_not_unique_measurement']; + return ['schemaVersion'=>1,'kind'=>'separated_measurement_views','installationId'=>$c['installationId'],'capturedAt'=>$r['capturedAt'], + 'mappingSha256'=>$mapping,'policySha256'=>hash('sha256',json_encode($p,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)), + 'physicalLoad'=>$load,'storageFeedback'=>$storage,'allocationModel'=>$alloc,'solarOrigin'=>$solar,'solarAcAlternative'=>$altQuality, + 'trainingEligible'=>false,'canDispatch'=>false,'controlEnabled'=>false,'socAccountsChanged'=>false,'futureSdl'=>'unknown_no_assumption', + 'note'=>'Observation only. Source times are Symcon update times. Algebraic partition is not separate EV/SDL measurement. GoodWe and combined AC/DC boundaries remain unverified.']; + } +} diff --git a/libs/NetzfahrplanV4Planpruefung.php b/libs/NetzfahrplanV4Planpruefung.php new file mode 100644 index 0000000..3138201 --- /dev/null +++ b/libs/NetzfahrplanV4Planpruefung.php @@ -0,0 +1,310 @@ +getTimestamp(); + } + + private static function zahl($value, string $name, float $low = -1.0e9, float $high = 1.0e9): float + { + if ((!is_float($value) && !is_int($value)) || !is_finite((float) $value) || $value < $low || $value > $high) { + throw new InvalidArgumentException($name . ': endlicher Zahlenwert erforderlich.'); + } + return (float) $value; + } + + private static function uuid($value): string + { + if (!is_string($value) || !preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $value)) { + throw new InvalidArgumentException('Ungueltige UUID.'); + } + return $value; + } + + public static function dekodieren(string $raw): array + { + if (strlen($raw) > self::MAX_RESPONSE_BYTES) { + throw new InvalidArgumentException('V4-Antwort zu gross.'); + } + $result = json_decode($raw, true, 64, JSON_THROW_ON_ERROR); + if (!is_array($result) || self::liste($result)) { + throw new InvalidArgumentException('V4-Antwort ist kein Objekt.'); + } + return $result; + } + + /** Stable local planning policy. Volatile SOC/availability are checked separately. */ + public static function kontext(array $operation): array + { + $assets = []; + foreach ($operation['batteries'] ?? [] as $b) { + if (!is_array($b) || !is_string($b['id'] ?? null) || isset($assets[$b['id']])) { + throw new InvalidArgumentException('Batteriezuordnung ungueltig.'); + } + $assets[$b['id']] = [ + 'capacityKwh' => self::zahl($b['capacityKwh'] ?? null, 'Kapazitaet', .001), + 'minSocPercent' => self::zahl($b['minSocPercent'] ?? null, 'Reserve', 0, 100), + 'maxSocPercent' => self::zahl($b['maxSocPercent'] ?? null, 'Maximal-SOC', 0, 100), + 'physicalMinSocPercent' => self::zahl($b['physicalMinSocPercent'] ?? 0.0, 'Minimum', 0, 100), + 'rearmSocPercent' => self::zahl($b['rearmSocPercent'] ?? $b['minSocPercent'], 'Hysterese', 0, 100), + 'gridCharging' => $b['gridCharging'] ?? null, + ]; + if (!is_bool($assets[$b['id']]['gridCharging'])) { + throw new InvalidArgumentException('Netzladefreigabe fehlt.'); + } + } + if (!$assets || count($assets) > 20 || !is_array($operation['limits'] ?? null)) { + throw new InvalidArgumentException('Batterien oder lokale Grenzen fehlen.'); + } + ksort($assets); + $limits = $operation['limits']; + $expected = ['importW', 'exportW', 'managerMonthLimitsW']; + if (count($limits) !== 3 || array_diff($expected, array_keys($limits)) || !is_array($limits['managerMonthLimitsW'])) { + throw new InvalidArgumentException('Unvollstaendige lokale Grenzen.'); + } + foreach (['importW', 'exportW'] as $key) { + if ($limits[$key] !== null) $limits[$key] = self::zahl($limits[$key], 'Grenze', 0); + } + foreach ($limits['managerMonthLimitsW'] as $month => $limit) { + if ((string) (int) $month !== (string) $month || (int) $month < 1 || (int) $month > 12) { + throw new InvalidArgumentException('Ungueltige Monatsgrenze.'); + } + $limits['managerMonthLimitsW'][$month] = self::zahl($limit, 'Monatsgrenze', 0); + } + ksort($limits['managerMonthLimitsW']); + return ['batteries' => $assets, 'limits' => $limits]; + } + + /** Authenticated endpoint must echo installation identity; old protocol is refused. */ + public static function pruefen(array $response, string $installation, array $operation, int $now, int $receivedAt): array + { + self::uuid($installation); + if (($response['receiverProtocolVersion'] ?? null) !== 1 + || ($response['installationId'] ?? null) !== $installation + || ($response['liveEnabled'] ?? null) !== false) { + throw new InvalidArgumentException('V4-Protokoll/Anlagenzuordnung oder Schattenmodus ungueltig.'); + } + if ($receivedAt > $now || $now - $receivedAt > self::MAX_RECEIPT_AGE) { + throw new InvalidArgumentException('Empfang veraltet; Vorschau verworfen.'); + } + $serverAt = self::zeit($response['checkedAt'] ?? null); + if ($serverAt > $now + 5 || $now - $serverAt > self::MAX_RECEIPT_AGE) { + throw new InvalidArgumentException('Serverantwort nicht aktuell.'); + } + if (($response['fresh'] ?? null) !== true || ($response['pending'] ?? null) !== null) { + throw new InvalidArgumentException('Server hat noch keinen aktuellen freigegebenen Schattenplan.'); + } + $settings = $response['settings'] ?? []; + $p = $response['plan'] ?? []; + if (!is_array($p) || !is_int($settings['revision'] ?? null) || $settings['revision'] < 0 + || ($p['schemaVersion'] ?? null) !== 2 || ($p['installationId'] ?? null) !== $installation + || ($p['configRevision'] ?? null) !== $settings['revision'] + || ($settings['runMode'] ?? null) !== 'shadow' + || ($p['runMode'] ?? null) !== 'shadow' || ($p['liveEnabled'] ?? null) !== false + || ($p['executable'] ?? null) !== true + || !in_array($p['status'] ?? '', ['optimal', 'feasible_time_limit'], true)) { + throw new InvalidArgumentException('Planrevision, Status oder Schattenmodus ungueltig.'); + } + self::uuid($p['planId'] ?? null); + if (($response['lastRun']['detail']['planId'] ?? null) !== $p['planId'] + || !in_array($response['lastRun']['status'] ?? '', ['optimal', 'feasible_time_limit'], true)) { + throw new InvalidArgumentException('Letzte Rechnung passt nicht zum empfangenen Plan.'); + } + $families = array_column($response['families'] ?? [], 'key'); + if (!is_string($p['sourceFamily'] ?? null) || !in_array($p['sourceFamily'], $families, true) + || (($settings['family'] ?? null) !== 'auto' && $settings['family'] !== $p['sourceFamily'])) { + throw new InvalidArgumentException('Modellfamilie unbekannt oder nicht ausgewaehlt.'); + } + $generated = self::zeit($p['generatedAt'] ?? null); + $start = self::zeit($p['validFrom'] ?? null); + $until = self::zeit($p['validUntil'] ?? null); + if ($generated > $now || $now - $generated > self::MAX_PLAN_AGE || $start > $now || $until <= $now + || $start !== $generated || $until - $start > 48 * 3600 + || self::zeit($p['pricesKnownUntil'] ?? null) < $until) { + throw new InvalidArgumentException('Plan abgelaufen, zukuenftig oder ohne bekannten Preishorizont.'); + } + $opAt = self::zeit($operation['observedAt'] ?? null); + if ($opAt > $now || $now - $opAt > 60) { + throw new InvalidArgumentException('Lokale Betriebsdaten veraltet.'); + } + $context = self::kontext($operation); + $pc = $p['controlContext'] ?? null; + if (!is_array($pc) || count($pc) !== 2 || !is_array($pc['batteries'] ?? null) || !is_array($pc['limits'] ?? null)) { + throw new InvalidArgumentException('Planungskontext fehlt.'); + } + $policyAssets = []; + foreach ($pc['batteries'] as $id => $policy) { + if (!is_array($policy) || count($policy) !== 6) { + throw new InvalidArgumentException('Unbekannter Batteriekontext.'); + } + $policyAssets[] = ['id' => $id] + $policy; + } + // Normalize explicitly typed numeric values; strings/bools are not numbers. + $plannedContext = self::kontext(['batteries' => $policyAssets, 'limits' => $pc['limits']]); + if ($plannedContext != $context) { + throw new InvalidArgumentException('Lokale Batterie-/Grenzkonfiguration passt nicht zum Plan.'); + } + if (!is_array($p['inputRefs'] ?? null) || !is_string($p['inputRefs']['operation'] ?? null) + || $p['inputRefs']['operation'] === '') { + throw new InvalidArgumentException('Plan ohne nachvollziehbare Eingangsdaten.'); + } + self::zahl($settings['roundtripEfficiency'] ?? null, 'Wirkungsgrad', .01, 1); + $points = $p['points'] ?? null; + if (!is_array($points) || !self::liste($points) || count($points) < 1 || count($points) > 576) { + throw new InvalidArgumentException('Ungueltige Zeitschrittliste.'); + } + $current = null; + $last = $start; + $ids = array_keys($context['batteries']); + foreach ($points as $point) { + $a = self::zeit($point['time'] ?? null); + $b = self::zeit($point['validUntil'] ?? null); + if ($a !== $last || $b <= $a || $b - $a > 300 || $b % 300 !== 0) { + throw new InvalidArgumentException('Luecke/Ueberlappung oder falsches Fuenfminutenraster.'); + } + $last = $b; + $grid = self::zahl($point['gridTargetW'] ?? null, 'Netzziel'); + $bat = self::zahl($point['batteryTargetW'] ?? null, 'Batterieziel'); + $base = self::zahl($point['baselineGridW'] ?? null, 'Basisnetz'); + $curtail = self::zahl($point['pvCurtailmentW'] ?? null, 'PV-Abregelung', 0); + if (abs($base + $bat + $curtail - $grid) > 1.0) { + throw new InvalidArgumentException('Leistungsbilanz des Plans verletzt.'); + } + if (!in_array($point['intent'] ?? '', ['hold', 'gridCharge', 'pvCharge', 'discharge', 'export'], true)) { + throw new InvalidArgumentException('Unbekannte Planabsicht.'); + } + $targets = $point['assetTargetsW'] ?? null; + $socs = $point['socEndPercent'] ?? null; + if (!is_array($targets) || !is_array($socs)) { + throw new InvalidArgumentException('Batterieziele fehlen.'); + } + $keys = array_keys($targets);sort($keys);$socKeys = array_keys($socs);sort($socKeys); + if ($keys !== $ids || $socKeys !== $ids) { + throw new InvalidArgumentException('Plan enthaelt falsche Batteriezuordnung.'); + } + $sum = 0.0; + foreach ($targets as $id => $w) { + $sum += self::zahl($w, 'Einzelziel'); + self::zahl($socs[$id], 'Plan-SOC', 0, 100); + } + if (abs($sum - $bat) > 1.0) { + throw new InvalidArgumentException('Summe der Batterieziele passt nicht.'); + } + foreach (['importLimitW', 'exportLimitW'] as $limit) { + if (!array_key_exists($limit, $point)) { + throw new InvalidArgumentException('Plangrenze fehlt.'); + } + if ($point[$limit] !== null) { + $v = self::zahl($point[$limit], 'Plangrenze', 0); + if (($limit === 'importLimitW' ? $grid : -$grid) > $v + 1) { + throw new InvalidArgumentException('Netzgrenze im Plan verletzt.'); + } + } + } + if ($a <= $now && $now < $b) { + $current = $point; + } + } + if ($last !== $until || $current === null) { + throw new InvalidArgumentException('Kein derzeit gueltiger Planabschnitt.'); + } + return ['plan' => $p, 'point' => $current, 'efficiency' => (float) $settings['roundtripEfficiency']]; + } + + /** Proposed single-battery correction only. Never use this result as a dispatch permit. */ + public static function vorschau(array $checked, array $operation, array $actualBatteryW, int $now): array + { + $point = $checked['point'];$p = $checked['plan']; + if (count($operation['batteries']) !== 1) { + throw new InvalidArgumentException('Lokale Vorschau fuer mehrere Batterien noch nicht freigegeben.'); + } + $b = $operation['batteries'][0];$id = $b['id']; + $age = $now - self::zeit($b['measuredAt'] ?? null); + if ($age < 0 || $age > 60) { + throw new InvalidArgumentException('Batteriemessung veraltet.'); + } + $grid = self::zahl($operation['gridW'], 'Netzleistung'); + $battery = self::zahl($actualBatteryW[$id] ?? null, 'Batterie-Istleistung'); + $soc = self::zahl($b['socPercent'], 'Aktueller SOC', $b['physicalMinSocPercent'] ?? 0, $b['maxSocPercent']); + $remainingHours = (self::zeit($point['validUntil']) - $now) / 3600; + if ($remainingHours <= 0) { + throw new InvalidArgumentException('Planabschnitt abgelaufen.'); + } + $eta = sqrt($checked['efficiency']);$capacity = self::zahl($b['capacityKwh'], 'Kapazitaet', .001); + $charge = min(self::zahl($b['maxChargeW'], 'Ladeleistung', 0), + max(0., $b['maxSocPercent'] - $soc) * $capacity / 100 / $eta / $remainingHours * 1000); + $discharge = min(self::zahl($b['maxDischargeW'], 'Entladeleistung', 0), + max(0., $soc - $b['minSocPercent']) * $capacity / 100 * $eta / $remainingHours * 1000); + $reasons = []; + if (($b['dischargeBlocked'] ?? false) === true || $soc <= $b['minSocPercent']) { + $discharge = 0.0;$reasons[] = 'Lokale Entladesperre/Reserve'; + } + $residual = $grid - $battery; + if ($b['gridCharging'] !== true) { + $charge = min($charge, max(0., -$residual));$reasons[] = 'Netzladen lokal gesperrt'; + } + $target = (float) $point['gridTargetW'];$limits = $operation['limits']; + $import = $limits['importW'];$export = $limits['exportW']; + $month = (int) (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('n'); + if (isset($limits['managerMonthLimitsW'][(string) $month])) { + $monthly = self::zahl($limits['managerMonthLimitsW'][(string) $month], 'Monatsgrenze', 0); + $import = $import === null ? $monthly : min($import, $monthly); + } + if ($import !== null) $target = min($target, self::zahl($import, 'Bezugsgrenze', 0)); + if ($export !== null) $target = max($target, -self::zahl($export, 'Einspeisegrenze', 0)); + $raw = $battery + $target - $grid; + $wanted = $raw; + if (abs($target) > 1.0) { + $wanted = match ($point['intent']) { + 'gridCharge' => max(0., $wanted), + 'pvCharge' => max(0., min(max(0., -$residual), $wanted)), + 'discharge', 'export' => min(0., $wanted), + 'hold' => 0., + }; + } + // Hard connection limits override economic intent, never BMS/availability. + if ($import !== null) $wanted = min($wanted, $import - $residual); + if ($export !== null) $wanted = max($wanted, -$export - $residual); + $clipped = min($charge, max(-$discharge, $wanted)); + if (abs($clipped - $raw) > 1) $reasons[] = 'Leistung, SOC, Planabsicht oder Netzgrenze begrenzt'; + $expectedGrid = $residual + $clipped; + $violation = max(0., $import === null ? 0. : $expectedGrid - $import, $export === null ? 0. : -$export - $expectedGrid); + if ($violation > 1) $reasons[] = 'Netzgrenze durch Batterie allein nicht erreichbar; lokale Schutzregelung erforderlich'; + return ['status' => 'preview_only', 'liveEnabled' => false, 'canDispatch' => false, + 'installationId' => $p['installationId'], 'planId' => $p['planId'], 'revision' => $p['configRevision'], + 'family' => $p['sourceFamily'], 'checkedAt' => gmdate('c', $now), 'step' => $point['time'], + 'stepValidUntil' => $point['validUntil'], 'gridTargetW' => (float) $point['gridTargetW'], + 'plannedBatteryW' => (float) $point['batteryTargetW'], 'previewBatteryW' => $clipped, + 'actualGridW' => $grid, 'actualBatteryW' => $battery, 'expectedGridW' => $expectedGrid, + 'unmetGridLimitW' => $violation, 'reasons' => $reasons, 'inputWarnings' => $p['warnings'] ?? [], + 'validityNote' => 'Nur Momentvorschau; keine Stellfreigabe und keine nachgewiesene Netzreaktion.']; + } +} diff --git a/libs/NetzfahrplanV4Regeltest.php b/libs/NetzfahrplanV4Regeltest.php new file mode 100644 index 0000000..99b5610 --- /dev/null +++ b/libs/NetzfahrplanV4Regeltest.php @@ -0,0 +1,207 @@ + $max) { + throw new InvalidArgumentException($name . ': ungueltiger Wert.'); + } + return (float) $value; + } + + public static function uuid($value): string + { + if (!is_string($value) || !preg_match('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/D', $value)) { + throw new InvalidArgumentException('Eindeutige Testkennung erforderlich.'); + } + return $value; + } + + public static function integer($value, string $name, int $min, int $max): int + { + if (!is_int($value) || $value < $min || $value > $max) { + throw new InvalidArgumentException($name . ': Ganzzahl erforderlich.'); + } + return $value; + } + + /** Validated server authorization plus independent LOCAL consent and physical mapping. */ + public static function befehl(array $response, array $operation, array $actual, array $local, int $now, int $receivedAt): array + { + if (($local['enabled'] ?? null) !== true || ($local['legacyEnabled'] ?? null) !== false) { + throw new InvalidArgumentException('Keine lokale Testfreigabe oder konkurrierender Fahrplan.'); + } + $installation = self::uuid($local['installationId'] ?? null); + $session = self::uuid($local['sessionId'] ?? null); + $manager = self::integer($local['managerId'] ?? null, 'Manager', 1, 99999); + $controller = self::integer($local['batteryInstanceId'] ?? null, 'Batterieinstanz', 1, 99999); + $activeTest = ($local['kind'] ?? null) === 'local_active_test_consent'; + $maxSeconds = $activeTest ? self::MAX_ACTIVE_TEST_SECONDS : self::MAX_TEST_SECONDS; + $end = self::integer($local['expiresAt'] ?? null, 'Lokales Testende', $now + 1, $now + $maxSeconds); + if (($local['lastWallClockAt'] ?? $now) > $now) throw new InvalidArgumentException('Systemzeit rueckwaerts; Test abbrechen.'); + $checked = NetzfahrplanV4Planpruefung::pruefen($response, $installation, $operation, $now, $receivedAt); + $plan = $checked['plan'];$a = $response['controlledTrial'] ?? null; + if (!is_array($a) || ($a['version'] ?? null) !== 1 || ($a['kind'] ?? null) !== 'controlled_trial_authority' + || ($a['sourcePlanRemainsShadow'] ?? null) !== true || ($a['installationId'] ?? null) !== $installation + || ($a['sessionId'] ?? null) !== $session || ($a['managerId'] ?? null) !== $manager + || ($a['batteryInstanceId'] ?? null) !== $controller || ($a['sourceShadowPlanId'] ?? null) !== $plan['planId'] + || ($a['revision'] ?? null) !== $plan['configRevision'] || ($a['family'] ?? null) !== $plan['sourceFamily']) { + throw new InvalidArgumentException('Keine passende separate Server-Testfreigabe.'); + } + $issued = NetzfahrplanV4Planpruefung::zeit($a['issuedAt'] ?? null); + $expires = NetzfahrplanV4Planpruefung::zeit($a['expiresAt'] ?? null); + $authorityAt = NetzfahrplanV4Planpruefung::zeit($a['checkedAt'] ?? null); + $authorityUntil = NetzfahrplanV4Planpruefung::zeit($a['validUntil'] ?? null); + if ($issued > $now || $expires <= $now || $expires - $issued > $maxSeconds + || $end > $expires || $authorityAt > $now + 2 || $authorityUntil <= $now + || $authorityUntil - $authorityAt > 90 || $authorityUntil > $expires) { + throw new InvalidArgumentException('Testfreigabe abgelaufen oder ungueltig.'); + } + $quality = $plan['inputQuality'] ?? []; + $evidence = $quality['accountingEvidenceId'] ?? null; + $watchdogBound = is_string($a['actuatorWatchdogEvidenceId'] ?? null) + && strlen($a['actuatorWatchdogEvidenceId']) >= 8 + && $a['actuatorWatchdogEvidenceId'] === ($local['actuatorWatchdogEvidenceId'] ?? null); + $watchdogWaived = $activeTest && ($local['watchdogWaived'] ?? null) === true + && ($a['actuatorWatchdogEvidenceId'] ?? null) === 'explicit_test_plant_watchdog_waiver'; + if (($quality['loadBasis'] ?? null) !== 'base_load' || !is_string($evidence) || strlen($evidence) < 8 + || $evidence !== ($a['accountingEvidenceId'] ?? null) + || (!$watchdogBound && !$watchdogWaived)) { + throw new InvalidArgumentException('Anlagenbilanz oder Geraete-Watchdog nicht nachgewiesen.'); + } + if (($plan['peakCostIsEstimate'] ?? true) && ($a['acceptEstimatedPeak'] ?? null) !== true) { + throw new InvalidArgumentException('Geschaetzte Peakbasis nicht fuer Test akzeptiert.'); + } + if (count($operation['batteries']) !== 1 || $operation['batteries'][0]['id'] !== ($a['assetId'] ?? null) + || ($local['assetId'] ?? null) !== $a['assetId']) { + throw new InvalidArgumentException('Test braucht eindeutige Batteriezuordnung.'); + } + if (($a['controlContext'] ?? null) != $plan['controlContext']) { + throw new InvalidArgumentException('Testkonfiguration stimmt nicht mehr.'); + } + $feedback = $operation['localFeedback'][$a['assetId']] ?? []; + NetzfahrplanV4Rueckmeldung::requireTrial($feedback, $local, $now); + if (abs(self::number($actual[$a['assetId']] ?? null,'Batterie Ist',-1e9,1e9) - self::number($feedback['batteryW'],'Rueckmeldung',-1e9,1e9)) > 0.000001) throw new InvalidArgumentException('feedback_power_not_from_verified_snapshot'); + $otherDelta=self::number($operation['localPredictedOtherDeltaW']??0.0,'Andere Verbraucherabweichung',-1e9,1e9); + if (abs($operation['gridW']-$feedback['gridW']-$otherDelta)>0.001) throw new InvalidArgumentException('feedback_grid_not_from_same_snapshot'); + $preview = NetzfahrplanV4Planpruefung::vorschau($checked, $operation, $actual, $now); + $battery = $operation['batteries'][0]; + $charge = min(self::number($a['maxChargeW'] ?? null, 'Server-Ladegrenze', 1, 1e9), + self::number($local['maxChargeW'] ?? null, 'Lokale Ladegrenze', 1, 1e9), + self::number($battery['maxChargeW'] ?? null, 'Anlagen-Ladegrenze', 1, 1e9)); + $discharge = min(self::number($a['maxDischargeW'] ?? null, 'Server-Entladegrenze', 1, 1e9), + self::number($local['maxDischargeW'] ?? null, 'Lokale Entladegrenze', 1, 1e9), + self::number($battery['maxDischargeW'] ?? null, 'Anlagen-Entladegrenze', 1, 1e9)); + $watts = (int) (max(-$discharge, min($charge, $preview['previewBatteryW']))); // truncate towards zero + $residual = $preview['actualGridW'] - $preview['actualBatteryW']; + $grid = $residual + $watts; + $p = $checked['point']; + if (($p['importLimitW'] !== null && $grid > $p['importLimitW'] + 1) + || ($p['exportLimitW'] !== null && -$grid > $p['exportLimitW'] + 1)) { + throw new InvalidArgumentException('Testleistungsgrenze reicht fuer Netzgrenze nicht; lokale Regelung hat Vorrang.'); + } + return ['kind' => 'controlled_trial_command', 'version' => 1, + 'sessionId' => $session, 'installationId' => $installation, 'assetId' => $a['assetId'], + 'managerId' => $manager, 'batteryInstanceId' => $controller, + 'sequence' => self::integer($local['sequence'] ?? null, 'Befehlsfolge', 1, PHP_INT_MAX), + 'issuedAt' => $now, 'expiresAt' => min($now + self::COMMAND_TTL_SECONDS, $authorityUntil, $end, + NetzfahrplanV4Planpruefung::zeit($p['validUntil'])), + 'watts' => $watts, 'maxChargeW' => (int) $charge, 'maxDischargeW' => (int) $discharge, + 'sourceShadowPlanId' => $plan['planId'], 'sourceStep' => $p['time'], + 'revision' => $plan['configRevision'], 'expectedGridW' => $grid, + 'feedbackConfigHash' => $feedback['configHash'], 'feedbackEstimated' => $feedback['estimated'], + 'feedbackSourceOldestAt' => $feedback['sourceOldestAt'], + 'importLimitW' => $p['importLimitW'], 'exportLimitW' => $p['exportLimitW'], + 'executionMeaning' => $activeTest + ? 'explicit_time_boxed_test_plant_operation' + : 'separate_local_control_trial_not_shadow_plan_execution']; + } + + /** Independently checked by the battery before any physical register call. */ + public static function pruefeBatteriebefehl(array $command, array $session, int $now, float $monotonicNow): void + { + if (($command['kind'] ?? null) !== 'controlled_trial_command' || ($command['version'] ?? null) !== 1) { + throw new InvalidArgumentException('Vorschau ist kein Stellbefehl.'); + } + foreach (['sessionId', 'installationId', 'assetId', 'managerId', 'batteryInstanceId'] as $key) { + if (!array_key_exists($key, $session) || ($command[$key] ?? null) !== $session[$key]) { + throw new InvalidArgumentException('Befehl gehoert nicht zur lokal angemeldeten Testsitzung.'); + } + } + if (($session['active'] ?? null) !== true || $now >= ($session['expiresAt'] ?? 0) + || $monotonicNow >= ($session['monotonicDeadline'] ?? 0) + || $now < ($session['lastWallClockAt'] ?? $now)) { + throw new InvalidArgumentException('Lokale Testsitzung abgelaufen/abgebrochen.'); + } + $sent = self::integer($command['issuedAt'] ?? null, 'Befehlszeit', $now - self::COMMAND_TTL_SECONDS, $now); + $end = self::integer($command['expiresAt'] ?? null, 'Befehlsende', $now + 1, $sent + self::COMMAND_TTL_SECONDS); + if ($end > $session['expiresAt']) throw new InvalidArgumentException('Befehl verlaengert Test.'); + $seq = self::integer($command['sequence'] ?? null, 'Befehlsfolge', 1, PHP_INT_MAX); + if ($seq <= ($session['lastSequence'] ?? 0)) throw new InvalidArgumentException('Alter/doppelter Befehl verworfen.'); + $sessionCharge = self::number($session['maxChargeW'] ?? null, 'Sitzungs-Ladegrenze', 1, 1e9); + $sessionDischarge = self::number($session['maxDischargeW'] ?? null, 'Sitzungs-Entladegrenze', 1, 1e9); + $w = self::integer($command['watts'] ?? null, 'Sollwert W', -(int) $sessionDischarge, (int) $sessionCharge); + $c = self::number($command['maxChargeW'] ?? null, 'Befehl-Ladegrenze', 1, $sessionCharge); + $d = self::number($command['maxDischargeW'] ?? null, 'Befehl-Entladegrenze', 1, $sessionDischarge); + if ($w > $c || $w < -$d) { + throw new InvalidArgumentException('Befehl ueberschreitet lokale Testgrenze.'); + } + if (!is_string($command['feedbackConfigHash'] ?? null) || !preg_match('/^[a-f0-9]{64}$/D', $command['feedbackConfigHash']) + || $command['feedbackConfigHash'] !== ($session['feedbackConfigHash'] ?? null)) throw new InvalidArgumentException('feedback_command_binding_invalid'); + foreach (['importLimitW','exportLimitW'] as $limit) { + if (!array_key_exists($limit,$command)) throw new InvalidArgumentException('feedback_command_limits_missing'); + if ($command[$limit] !== null) self::number($command[$limit],$limit,0,1e9); + } + $l=$session['gridLimits'] ?? null; + if (!is_array($l) || !array_key_exists('importW',$l) || !array_key_exists('exportW',$l) || !is_array($l['managerMonthLimitsW']??null)) throw new InvalidArgumentException('session_grid_limits_missing'); + $month=(int)(new \DateTimeImmutable('@'.$now))->setTimezone(new \DateTimeZone('Europe/Zurich'))->format('n'); + $import=$l['importW'];$monthly=$l['managerMonthLimitsW'][$month]??null; + if ($monthly!==null) $import=$import===null?$monthly:min($import,$monthly); + foreach (['importLimitW'=>$import,'exportLimitW'=>$l['exportW']] as $key=>$limit) { + if ($limit!==null) { + self::number($limit,$key,0,1e9); + if ($command[$key]===null || $command[$key]>$limit) throw new InvalidArgumentException('command_relaxes_local_grid_limit'); + } + } + self::uuid($command['sourceShadowPlanId'] ?? null); + } + + /** Re-evaluated on every battery update; no reliance on the manager's earlier clipping. */ + public static function batterieZiel(array $command, array $session, array $m, array $policy, int $now, float $monotonicNow): int + { + if ($now >= $command['expiresAt'] || $now < $session['lastWallClockAt'] + || $monotonicNow >= $session['commandMonotonicDeadline'] || $monotonicNow >= $session['monotonicDeadline']) { + throw new InvalidArgumentException('Befehls-Watchdog abgelaufen.'); + } + if (($m['Gueltig'] ?? null) !== true || ($policy['available'] ?? null) !== true) { + throw new InvalidArgumentException('Batterie nicht verfuegbar oder Messwerte ungueltig.'); + } + $soc = self::number($m['Ladezustand'] ?? null, 'SOC', 0, 100); + $reserve = self::number($policy['reserve'] ?? null, 'Reserve', 0, 100); + $minimum = self::number($policy['minimum'] ?? null, 'Minimum', 0, $reserve); + if ($soc < $minimum) throw new InvalidArgumentException('SOC unter technischer Untergrenze.'); + $charge = min($session['maxChargeW'], $command['maxChargeW'], self::number($m['MaxLaden'] ?? null, 'Max Laden', 0, 1e9)); + $discharge = min($session['maxDischargeW'], $command['maxDischargeW'], self::number($m['MaxEntladen'] ?? null, 'Max Entladen', 0, 1e9)); + if ($soc >= 99) $charge = 0; // existing battery hardware policy is more conservative than planner max 100 + if ($soc <= $reserve || ($policy['blocked'] ?? true)) $discharge = 0; + if (($soc <= $reserve || ($policy['blocked'] ?? true)) && $policy['rechargeLimitW'] !== null) { + $charge = min($charge, self::number($policy['rechargeLimitW'], 'Nachladegrenze', 0, 1e9)); + } + return (int) max(-$discharge, min($charge, $command['watts'])); + } +} diff --git a/libs/NetzfahrplanV4Rueckmeldung.php b/libs/NetzfahrplanV4Rueckmeldung.php new file mode 100644 index 0000000..df89be3 --- /dev/null +++ b/libs/NetzfahrplanV4Rueckmeldung.php @@ -0,0 +1,147 @@ +$hi) { + throw new InvalidArgumentException('feedback_numeric_invalid'); + } + return (float)$v; + } + + private static function ordered(array $v): array + { + if (!array_is_list($v)) ksort($v, SORT_STRING); + foreach ($v as &$x) if (is_array($x)) $x=self::ordered($x); + unset($x);return $v; + } + + public static function configuration(array $c): array + { + $keys=['version','installationId','assetId','managerId','batteryInstanceId','mode','allowEstimatedForTrial', + 'maxSkewSeconds','maxAgeSeconds','idleToleranceW','trackingToleranceW','sources']; + if (count($c)!==count($keys) || array_diff($keys,array_keys($c)) || $c['version']!==1 + || !is_string($c['installationId']) || !preg_match('/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/D',$c['installationId']) + || !is_string($c['assetId']) || $c['assetId']==='' || strlen($c['assetId'])>100 + || !in_array($c['mode'],['physical_sum','virtual_split'],true) || !is_bool($c['allowEstimatedForTrial'])) { + throw new InvalidArgumentException('feedback_configuration_invalid'); + } + foreach (['managerId','batteryInstanceId'] as $k) if (!is_int($c[$k]) || $c[$k]<1 || $c[$k]>99999) throw new InvalidArgumentException('feedback_binding_invalid'); + if ($c['managerId']===$c['batteryInstanceId']) throw new InvalidArgumentException('feedback_binding_invalid'); + foreach (['maxSkewSeconds'=>30,'maxAgeSeconds'=>60] as $k=>$max) { + if (!is_int($c[$k]) || $c[$k]<1 || $c[$k]>$max) throw new InvalidArgumentException('feedback_freshness_invalid'); + } + $c['idleToleranceW']=self::number($c['idleToleranceW'],0,100); + $c['trackingToleranceW']=self::number($c['trackingToleranceW'],0,500); + if (!is_array($c['sources']) || !array_is_list($c['sources']) || count($c['sources'])<2 || count($c['sources'])>40) throw new InvalidArgumentException('feedback_sources_invalid'); + $ids=[];$names=[];$roles=[]; + foreach ($c['sources'] as &$s) { + $fields=['key','role','variableId','parentId','ident','factorToW']; + if (!is_array($s) || count($s)!==count($fields) || array_diff($fields,array_keys($s)) + || !is_string($s['key']) || !preg_match('/^[a-zA-Z][a-zA-Z0-9_-]{0,63}$/D',$s['key']) + || isset($names[$s['key']]) || !in_array($s['role'],['grid','physical','ev_request','sdl_request','gateway_active'],true) + || !is_int($s['variableId']) || $s['variableId']<1 || $s['variableId']>99999 || isset($ids[$s['variableId']]) + || !is_int($s['parentId']) || $s['parentId']<1 || !is_string($s['ident']) || strlen($s['ident'])>100) throw new InvalidArgumentException('feedback_source_identity_invalid'); + $s['factorToW']=self::number($s['factorToW'],-1e6,1e6); + if ($s['factorToW']==0 || ($s['role']==='gateway_active' && $s['factorToW']!==1.0)) throw new InvalidArgumentException('feedback_factor_invalid'); + $ids[$s['variableId']]=true;$names[$s['key']]=true;$roles[$s['role']]=($roles[$s['role']]??0)+1; + } + unset($s); + if (($roles['grid']??0)!==1 || ($roles['physical']??0)<1) throw new InvalidArgumentException('feedback_measurements_required'); + foreach (['ev_request','sdl_request','gateway_active'] as $role) { + if (($roles[$role]??0)!==($c['mode']==='virtual_split'?1:0)) throw new InvalidArgumentException('feedback_partition_sources_required'); + } + usort($c['sources'],static fn($a,$b)=>strcmp($a['key'],$b['key'])); + return self::ordered($c); + } + + public static function fingerprint(array $c): string + { + return hash('sha256',json_encode(self::configuration($c),JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + } + + /** Reader supplies value, original updated/changed, parentID, ident. Never trusts collector timestamp alone. */ + public static function read(array $config, callable $reader, int $now): array + { + $c=self::configuration($config); + if ($now<1) throw new InvalidArgumentException('feedback_time_invalid'); + $first=[];$values=[];$times=[];$physicalTimes=[];$variableTimes=[];$changed=[]; + foreach ($c['sources'] as $s) { + $r=$reader($s['variableId']); + if (!is_array($r) || ($r['parentID']??null)!==$s['parentId'] || ($r['ident']??null)!==$s['ident'] + || !is_int($r['updated']??null) || $r['updated']<1 || $r['updated']>$now + || !is_int($r['changed']??null) || $r['changed']<0 || $r['changed']>$r['updated'] + || (in_array($s['role'],['grid','physical'],true) + && (!is_int($r['confirmedAt']??null) || $r['confirmedAt']<1 || $r['confirmedAt']>$now + || $r['updated']>$r['confirmedAt']))) throw new InvalidArgumentException('feedback_origin_invalid:'.$s['key']); + $first[$s['key']]=array_intersect_key($r,array_flip(['value','updated','changed','parentID','ident','confirmedAt'])); + if ($s['role']==='gateway_active') { + if (($r['value']??null)!==true) throw new InvalidArgumentException('feedback_gateway_inactive'); + } else { + $w=self::number($r['value']??null,-1e9,1e9)*$s['factorToW']; + $values[$s['role']]=($values[$s['role']]??0)+self::number($w,-1e9,1e9); + } + // VariableUpdated is retained as provenance. Freshness comes only from a successful device response. + if (in_array($s['role'],['grid','physical'],true)) { + if ($now-$r['confirmedAt']>$c['maxAgeSeconds']) throw new InvalidArgumentException('feedback_stale_physical:'.$s['key']); + $times[]=$r['confirmedAt'];$variableTimes[]=$r['updated']; + if ($s['role']==='physical') $physicalTimes[]=$r['confirmedAt']; + } else $changed[$s['role']]=$r['changed']; + } + foreach ($c['sources'] as $s) { + $second=$reader($s['variableId']); + if (!is_array($second) || array_intersect_key($second,array_flip(['value','updated','changed','parentID','ident','confirmedAt']))!==$first[$s['key']]) { + throw new InvalidArgumentException('feedback_changed_during_read:'.$s['key']); + } + } + if (max($times)-min($times)>$c['maxSkewSeconds']) throw new InvalidArgumentException('feedback_source_skew'); + $physical=(float)$values['physical'];$power=$physical;$external=0.0;$estimated=$c['mode']==='virtual_split';$reasons=[];$tracking=null; + if ($estimated) { + if ($changed['gateway_active']>min($physicalTimes) || $changed['sdl_request']>min($physicalTimes)) throw new InvalidArgumentException('feedback_external_transition_not_observed'); + $ev=(float)$values['ev_request'];$sdl=(float)$values['sdl_request'];$tracking=$physical-$ev-$sdl; + if ($sdl===0.0) { + // No external commanded contribution. Physical response may lag a NEW EV command. + $power=$physical; + if ($ev===0.0 && abs($physical)>$c['idleToleranceW']) throw new InvalidArgumentException('feedback_unassigned_idle_power'); + } else { + if ($changed['ev_request']>min($physicalTimes)) throw new InvalidArgumentException('feedback_ev_transition_not_observed'); + if (abs($tracking)>$c['trackingToleranceW']) throw new InvalidArgumentException('feedback_partition_not_tracking'); + $power=$ev+$tracking*abs($ev)/(abs($ev)+abs($sdl));$external=$physical-$power; + if ($ev*$sdl<0) $reasons[]='opposing_virtual_flows_not_identifiable'; + } + if (!$c['allowEstimatedForTrial']) $reasons[]='modelled_feedback_not_accepted_for_trial'; + } + return ['version'=>1,'kind'=>'local_physical_feedback','installationId'=>$c['installationId'],'assetId'=>$c['assetId'], + 'managerId'=>$c['managerId'],'batteryInstanceId'=>$c['batteryInstanceId'],'checkedAt'=>$now, + 'sourceOldestAt'=>min($times),'sourceNewestAt'=>max($times),'physicalOldestAt'=>min($physicalTimes), + 'sourceSkewSeconds'=>max($times)-min($times),'sourceVariableOldestAt'=>min($variableTimes), + 'sourceVariableNewestAt'=>max($variableTimes),'deviceReadConfirmed'=>true,'maxAgeSeconds'=>$c['maxAgeSeconds'], + 'configHash'=>self::fingerprint($c),'gridW'=>(float)$values['grid'],'physicalStorageW'=>$physical, + 'batteryW'=>$power,'externalModelW'=>$external,'trackingErrorW'=>$tracking, + 'estimated'=>$estimated,'separatelyMeasured'=>!$estimated,'usableForTrial'=>$reasons===[], + 'reasons'=>$reasons,'method'=>$estimated?'physical_request_partition_unfiltered':'physical_sum', + 'canDispatch'=>false,'gatewayWatchdogVerified'=>false,'heldFilterUsed'=>false]; + } + + public static function requireTrial(array $f, array $binding, int $now): void + { + foreach (['installationId','assetId','managerId','batteryInstanceId'] as $key) { + if (!array_key_exists($key,$binding) || ($f[$key]??null)!==$binding[$key]) throw new InvalidArgumentException('feedback_wrong_binding'); + } + if (($f['kind']??null)!=='local_physical_feedback' || ($f['version']??null)!==1 || ($f['usableForTrial']??null)!==true + || ($f['deviceReadConfirmed']??null)!==true + || !is_int($f['checkedAt']??null) || $f['checkedAt']>$now || $now-$f['checkedAt']>2 + || !is_int($f['sourceOldestAt']??null) || !is_int($f['maxAgeSeconds']??null) + || $f['maxAgeSeconds']<1 || $f['maxAgeSeconds']>60 || $f['sourceOldestAt']>$now || $now-$f['sourceOldestAt']>$f['maxAgeSeconds'] + || !is_string($f['configHash']??null) || !preg_match('/^[a-f0-9]{64}$/D',$f['configHash'])) throw new InvalidArgumentException('feedback_not_usable_for_trial'); + if (isset($binding['feedbackConfigHash']) && $binding['feedbackConfigHash']!==$f['configHash']) throw new InvalidArgumentException('feedback_configuration_changed'); + self::number($f['batteryW']??null,-1e9,1e9);self::number($f['gridW']??null,-1e9,1e9); + } +} diff --git a/services/netplan-v4/.dockerignore b/services/netplan-v4/.dockerignore new file mode 100644 index 0000000..7264271 --- /dev/null +++ b/services/netplan-v4/.dockerignore @@ -0,0 +1,5 @@ +data/ +.git/ +__pycache__/ +**/__pycache__/ +*.log diff --git a/services/netplan-v4/.gitignore b/services/netplan-v4/.gitignore new file mode 100644 index 0000000..eb476e8 --- /dev/null +++ b/services/netplan-v4/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +data/ +*-reports/ +application-releases/ +.env +*.env diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile new file mode 100644 index 0000000..33e5701 --- /dev/null +++ b/services/netplan-v4/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY netplan_v4 ./netplan_v4 +COPY tests ./tests +COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile . +COPY integrations ./integrations +COPY gui ./gui +COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ +COPY acceptance ./acceptance +COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py +COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py +# Host sources can be 0600/0700. COPY makes them root-owned. +# Normalize only packaged application code; never change host secrets or sockets. +RUN find /app -type d -exec chmod 0755 {} + \ + && find /app -type f -exec chmod 0644 {} + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results +USER 1000:1000 +# Fail the build early if the actual unprivileged runtime cannot read the code. +RUN python /app/runtime_preflight.py +CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"] diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json new file mode 100644 index 0000000..9cbe60b --- /dev/null +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -0,0 +1,106 @@ +{ + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "commissioning/deploy_unified_release.py": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/economic_replay.py": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "netplan_v4/mapping_identity.py": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d", + "netplan_v4/measurement_pipeline.py": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/retention.py": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f", + "netplan_v4/store.py": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_economic_replay.py": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_mapping_recovery.py": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68", + "tests/test_retention.py": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51", + "tests/test_unified_release.py": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "scope": "application_source_only", + "productionReady": false +} diff --git a/services/netplan-v4/acceptance/Dockerfile.php b/services/netplan-v4/acceptance/Dockerfile.php new file mode 100644 index 0000000..d7aa137 --- /dev/null +++ b/services/netplan-v4/acceptance/Dockerfile.php @@ -0,0 +1,7 @@ +FROM php:8.3-cli +WORKDIR /check +COPY php-src/ ./ +RUN find /check -type d -exec chmod 0755 {} + \ + && find /check -type f -exec chmod 0644 {} + +USER 1000:1000 +CMD ["php", "/check/check.php"] diff --git a/services/netplan-v4/acceptance/check_forecast.py b/services/netplan-v4/acceptance/check_forecast.py new file mode 100644 index 0000000..f42cabb --- /dev/null +++ b/services/netplan-v4/acceptance/check_forecast.py @@ -0,0 +1,36 @@ +"""Offline candidate-forecast tests: no telemetry, model loading or publication. +Run in an unprivileged, networkless test container with no live data volumes. +""" +from pathlib import Path +import ast +import hashlib +import json +import sys +import unittest + + +def main(): + root = Path('/app/forecast') + sys.path.insert(0, str(root)) + manifest = json.loads((root / 'SOURCE_MANIFEST.json').read_text()) + for relative, expected in manifest.items(): + p = root / relative + if Path(relative).is_absolute() or '..' in Path(relative).parts or not p.resolve().is_relative_to(root): + raise ValueError('Unsafe manifest path') + raw = p.read_bytes() + if hashlib.sha256(raw).hexdigest() != expected: + raise ValueError('Test image source checksum mismatch') + if p.suffix == '.py': + ast.parse(raw, filename=str(p)) + import pandas, numpy, scipy, sklearn + versions = {'python': sys.version.split()[0], 'pandas': pandas.__version__, + 'numpy': numpy.__version__, 'scipy': scipy.__version__, 'sklearn': sklearn.__version__} + print('Candidate forecast runtime:', json.dumps(versions), flush=True) + suite = unittest.defaultTestLoader.discover(str(root / 'tests')) + result = unittest.TextTestRunner(verbosity=2).run(suite) + print('Forecast tests only; no live data, no publication, no training job.', flush=True) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..f35881c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json @@ -0,0 +1,25 @@ +{ + "main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" +} diff --git a/services/netplan-v4/acceptance/forecast-src/main.py b/services/netplan-v4/acceptance/forecast-src/main.py new file mode 100644 index 0000000..02748d7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/main.py @@ -0,0 +1,908 @@ +from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts +from model_isolation import collect_predictions +import os +import sqlite3 +import json +import time +import datetime +import traceback +import warnings +import subprocess +import sys +import threading + +import numpy as np +import pandas as pd +import pytz +from influxdb_client import InfluxDBClient, Point, WritePrecision +from influxdb_client.client.write_api import SYNCHRONOUS +from soc_diagnostics import battery_soc_points +from telemetry_quality import require_recent_telemetry, sanitize_measured_frame +try: + from influxdb_client.client.warnings import MissingPivotFunction + warnings.simplefilter("ignore", MissingPivotFunction) +except Exception: + pass + +import methods.var_1 as v1 +import methods.var_2 as v2 +import methods.var_3 as v3 +import methods.var_10 as v10 +import methods.var_11 as v11 +import methods.var_13 as v13 +import methods.var_21 as v21 +import methods.var_22 as v22 +import methods.var_23 as v23 + +INFLUX_URL = os.getenv("INFLUX_URL", "http://influxdb:8086") +INFLUX_TOKEN = os.environ["INFLUX_TOKEN"] +INFLUX_ORG = os.getenv("INFLUX_ORG", "belevo") +INFLUX_BUCKET = os.getenv("INFLUX_BUCKET", "energy_data") +SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "/app/data/users.db") +HISTORY_START = os.getenv("FORECAST_HISTORY_START", "1970-01-01T00:00:00Z") +QUALITY_LOOKBACK_DAYS = int(os.getenv("FORECAST_QUALITY_LOOKBACK_DAYS", "14")) +LOCAL_TZ = pytz.timezone(os.getenv("TZ", "Europe/Zurich")) +INFLUX_TIMEOUT_MS = int(os.getenv("FORECAST_INFLUX_TIMEOUT_MS", "120000")) +FORECAST_TIMEOUT_SECONDS = int(os.getenv("FORECAST_RUN_TIMEOUT_SECONDS", "900")) +FORECAST_HORIZON_HOURS = max(24, min(72, int(os.getenv("FORECAST_HORIZON_HOURS", "48")))) +TRAINING_TIMEOUT_SECONDS = int(os.getenv("FORECAST_TRAINING_TIMEOUT_SECONDS", "3600")) +FORECAST_STALE_SECONDS = int(os.getenv("FORECAST_STALE_SECONDS", "5400")) +WATCHDOG_INTERVAL_SECONDS = int(os.getenv("FORECAST_WATCHDOG_INTERVAL_SECONDS", "300")) +TRAINING_HOUR = int(os.getenv("FORECAST_TRAINING_HOUR", "2")) +LAST_SUCCESS_PATH = os.getenv("FORECAST_LAST_SUCCESS_PATH", "/tmp/forecast_engine_last_success.json") +LAST_TRAINING_PATH = os.getenv("FORECAST_LAST_TRAINING_PATH", "/app/data/forecast_training_status.json") +last_trained_day = None +_FORECAST_PROCESS_LOCK = threading.Lock() +_TRAINING_THREAD = None + +MODEL_MODULES = {1: v1, 2: v2, 3: v3, 10: v10, 11: v11, 13: v13, 21: v21, 22: v22, 23: v23} +QUALITY_TARGETS = {1: "PV", 10: "PV", 21: "PV", 2: "Hausverbrauch", 11: "Hausverbrauch", 22: "Hausverbrauch"} + + +def active(config, n): + return bool(int(config.get(f"prog_var_{n}", config.get(f"var_{n}", 0)) or 0)) + + +def get_configs(): + conn = sqlite3.connect(SQLITE_DB_PATH) + conn.row_factory = sqlite3.Row + columns = {row["name"] for row in conn.execute("PRAGMA table_info(anlagen_meta)")} + if "batt_grid_charging_enabled" not in columns: + conn.execute( + "ALTER TABLE anlagen_meta ADD COLUMN batt_grid_charging_enabled INTEGER NOT NULL DEFAULT 0" + ) + conn.commit() + rows = conn.execute("SELECT * FROM anlagen_meta").fetchall() + conn.close() + configs = [] + for r in rows: + d = dict(r) + try: + d["daecher"] = json.loads(d.get("daecher") or "[]") + except Exception: + d["daecher"] = [] + for key, default in [ + ("ac_leistung", 10.0), + ("batt_capacity_kwh", 0.0), + ("batt_power_kw", 0.0), + ("tarif_bezug_fest", 0.30), + ("tarif_einspeisung_fest", 0.10), + ("tarif_peak_fest", 5.0), + ]: + raw_value = d.get(key) + d[key] = float(default if raw_value is None or raw_value == "" else raw_value) + configs.append(d) + return configs + + +def _time_literal(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _query_df(query): + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + try: + df = client.query_api().query_data_frame(org=INFLUX_ORG, query=query) + finally: + client.close() + if isinstance(df, list): + df = pd.concat(df, ignore_index=True) if df else pd.DataFrame() + if df is None or df.empty or "_time" not in df.columns: + return pd.DataFrame() + df["_time"] = pd.to_datetime(df["_time"], utc=True).dt.tz_localize(None) + return df + + +def _pivot_frame(df, fields): + if df.empty: + return pd.DataFrame() + out = df.set_index("_time") + keep = [c for c in fields if c in out.columns] + out = out[keep] if keep else pd.DataFrame(index=out.index) + out = out.apply(pd.to_numeric, errors="coerce") + out = out[~out.index.duplicated(keep="last")] + return out.sort_index().resample("5min").mean(numeric_only=True) + + +def _tariff_frame(df, config=None): + if df is None or df.empty: + return pd.DataFrame() + df = df.copy() + if "_time" in df.columns: + df["_time"] = pd.to_datetime(df["_time"], errors="coerce") + df = df.dropna(subset=["_time"]).set_index("_time") + if df.empty: + return pd.DataFrame() + + price = df["price_chf_kwh"] if "price_chf_kwh" in df.columns else df.get("_value") + if price is None: + return pd.DataFrame() + price = pd.to_numeric(price, errors="coerce") + model = df.get("tariff_model", pd.Series("", index=df.index)).astype(str).str.lower() + typ = df.get("type", pd.Series("", index=df.index)).astype(str).str.lower() + tariff_name = df.get("tariff_name", pd.Series("", index=df.index)).astype(str).str.lower() + provider = df.get("provider", pd.Series("", index=df.index)).astype(str).str.lower() + key = (model + " " + tariff_name + " " + provider).str.lower() + + rows = pd.DataFrame({"price": price, "key": key, "type": typ}, index=df.index) + rows = rows[pd.notna(rows["price"])].sort_index() + if rows.empty: + return pd.DataFrame() + + cfg = config or {} + import_choice = str(cfg.get("tarif_bezug", "") or "").lower() + export_choice = str(cfg.get("tarif_einspeisung", "") or "").lower() + + out = pd.DataFrame(index=rows.index.unique().sort_values()) + + import_base = ( + rows["type"].str.contains("consumption|import|bezug", regex=True, na=False) + | rows["key"].str.contains("dynamic|dynamisch|home|business", regex=True, na=False) + ) + if "business" in import_choice: + import_mask = import_base & rows["key"].str.contains("business|gewerbe|commercial", regex=True, na=False) + elif "home" in import_choice or "privat" in import_choice: + import_mask = import_base & rows["key"].str.contains("home|privat|private", regex=True, na=False) + elif "dynam" in import_choice: + import_mask = import_base + else: + import_mask = pd.Series(False, index=rows.index) + if not import_mask.any() and "dynam" in import_choice: + import_mask = import_base + + export_base = ( + rows["type"].str.contains("feed|einspeis|export", regex=True, na=False) + | rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|feed", regex=True, na=False) + ) + if "referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice: + export_mask = export_base & rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|belevo", regex=True, na=False) + else: + export_mask = rows["key"].str.contains("standard_feedin|ckw statisch", regex=True, na=False) & export_base + if not export_mask.any() and ("referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice): + export_mask = export_base + + if import_mask.any(): + out["import_price"] = rows.loc[import_mask, "price"].groupby(level=0).last() + if export_mask.any(): + out["export_price"] = rows.loc[export_mask, "price"].groupby(level=0).last() + + if out.empty: + return out + return out.sort_index().resample("5min").mean().ffill().bfill() + +def fetch_influx_frames(config, training): + aid = config["anlagen_id"] + start = HISTORY_START if training else "-14d" + future_stop = _time_literal(datetime.datetime.utcnow() + datetime.timedelta(hours=FORECAST_HORIZON_HOURS)) + + q_tel = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "PV" or r["_field"] == "Hausverbrauch" or r["_field"] == "Netzleistung" or r["_field"] == "SOC") + |> filter(fn: (r) => not exists r["data_type"] or (r["data_type"] != "forecast" and r["data_type"] != "forecast_snapshot")) + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false, timeSrc: "_start") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_tel = _pivot_frame(_query_df(q_tel), ["PV", "Hausverbrauch", "Netzleistung", "SOC"]) + + q_wea = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "weather_forecast") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_wea = _pivot_frame(_query_df(q_wea), ["temp_c", "temperature", "cloud", "cloud_cover", "precip_mm", "wind_kph", "chance_of_snow"]) + if "temperature" in df_wea.columns and "temp_c" not in df_wea.columns: + df_wea["temp_c"] = df_wea["temperature"] + if "cloud_cover" in df_wea.columns and "cloud" not in df_wea.columns: + df_wea["cloud"] = df_wea["cloud_cover"] + if "cloud" in df_wea.columns and "cloud_cover" not in df_wea.columns: + df_wea["cloud_cover"] = df_wea["cloud"] + + q_tar = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -7d, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "tariffs") + |> filter(fn: (r) => r["_field"] == "price_chf_kwh") +''' + df_tar = _tariff_frame(_query_df(q_tar), config) + return df_tel, df_wea, df_tar + + +def _consistent_tail(df): + if df.empty or "PV" not in df.columns or "Hausverbrauch" not in df.columns: + return df + probe = df[["PV", "Hausverbrauch"]].copy() + filled = probe.interpolate(limit=3, limit_direction="both") + valid = filled.notna().all(axis=1) + if not valid.any(): + return df.iloc[0:0] + run = 0 + last_break = -1 + for i, ok in enumerate(valid.to_numpy()): + if ok: + run = 0 + else: + run += 1 + if run >= 4: + last_break = i + if last_break >= 0: + after = np.where(valid.iloc[last_break + 1:].to_numpy())[0] + if len(after): + return df.iloc[last_break + 1 + after[0]:] + return df.iloc[0:0] + return df.loc[valid[valid].index[0]:] + + + +def _longest_consistent_segment(df, column): + if df.empty or column not in df.columns: + return pd.DataFrame() + series = pd.to_numeric(df[column], errors="coerce") + valid = series.notna().to_numpy() + if not valid.any(): + return pd.DataFrame() + best_start = best_end = None + start = 0 + gap = 0 + for i, ok in enumerate(valid): + if ok: + gap = 0 + else: + gap += 1 + if gap >= 4: + end = i - gap + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + start = i + 1 + gap = 0 + end = len(valid) - 1 + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + if best_start is None: + return pd.DataFrame() + segment = df.iloc[best_start:best_end + 1].copy() + first = pd.to_numeric(segment[column], errors="coerce").first_valid_index() + last = pd.to_numeric(segment[column], errors="coerce").last_valid_index() + if first is None or last is None: + return pd.DataFrame() + return segment.loc[first:last] + + +def _add_time_features(df): + hour = df.index.hour + df.index.minute / 60.0 + doy = df.index.dayofyear + df["hour_float"] = hour + df["hour_sin"] = np.sin(2 * np.pi * hour / 24.0) + df["hour_cos"] = np.cos(2 * np.pi * hour / 24.0) + df["sin_year"] = np.sin(2 * np.pi * doy / 365.25) + df["cos_year"] = np.cos(2 * np.pi * doy / 365.25) + df["weekday"] = df.index.weekday + df["is_weekday"] = (df.index.weekday < 5).astype(int) + return df + + +def _fill_defaults(df, history): + defaults = { + "PV": np.nan, + "Hausverbrauch": np.nan, + "SOC": np.nan, + "Netzleistung": np.nan, + "temp_c": 15.0, + "cloud": 20.0, + "cloud_cover": 20.0, + "precip_mm": 0.0, + "wind_kph": 0.0, + "chance_of_snow": 0.0, + "import_price": np.nan, + "export_price": np.nan, + } + for col, default in defaults.items(): + if col not in df.columns: + df[col] = default + df[col] = pd.to_numeric(df[col], errors="coerce") + # Measurements are never interpolated/filled here. Outages are not zero load, + # zero PV, a fresh SOC, or a measured grid peak. Weather defaults are separate. + df = sanitize_measured_frame(df) + for col, default in defaults.items(): + if col not in ["PV", "Hausverbrauch", "SOC", "Netzleistung"]: + df[col] = df[col].ffill().bfill() + if np.isfinite(default): + df[col] = df[col].fillna(default) + if "cloud_cover" in df.columns: + df["cloud"] = df["cloud"].fillna(df["cloud_cover"]) + df["import_price"] = df["import_price"].fillna(0.30) + df["export_price"] = df["export_price"].fillna(0.10) + return _add_time_features(df) + + +def build_data_object(config, training=False): + now = datetime.datetime.utcnow().replace(second=0, microsecond=0) + now = now - datetime.timedelta(minutes=now.minute % 5) + df_tel, df_wea, df_tar = fetch_influx_frames(config, training) + + frames = [f for f in [df_tel, df_wea, df_tar] if not f.empty] + combined = frames[0] if frames else pd.DataFrame() + for f in frames[1:]: + combined = combined.join(f, how="outer") + combined = combined.sort_index() + + full_hist_raw = combined.loc[:now - datetime.timedelta(minutes=5)] if not combined.empty else pd.DataFrame() + # A trailing telemetry gap must not discard all earlier valid observations. + hist_raw = full_hist_raw.copy() + # Freshness is determined from measurements, not the outer-joined weather grid. + recent_raw = sanitize_measured_frame(df_tel.loc[:now - datetime.timedelta(minutes=5)].copy()) if not df_tel.empty else pd.DataFrame() + start_hist = hist_raw.index.min().floor("5min") if training and not hist_raw.empty else now - datetime.timedelta(days=14) + + idx_hist = pd.date_range(start=start_hist, end=now - datetime.timedelta(minutes=5), freq="5min") + df_hist = pd.DataFrame(index=idx_hist).join(hist_raw, how="left") + df_hist = _fill_defaults(df_hist, history=True) + + df_load_training = pd.DataFrame() + if training and "Hausverbrauch" in full_hist_raw.columns: + load_segment = _longest_consistent_segment(full_hist_raw, "Hausverbrauch") + if not load_segment.empty: + idx_load = pd.date_range( + start=load_segment.index.min().floor("5min"), + end=load_segment.index.max().floor("5min"), + freq="5min", + ) + df_load_training = pd.DataFrame(index=idx_load).join(load_segment, how="left") + df_load_training = _fill_defaults(df_load_training, history=True) + + df_pv_training = pd.DataFrame() + if training and "PV" in full_hist_raw.columns: + pv_first = full_hist_raw["PV"].first_valid_index() + if pv_first is not None: + idx_pv = pd.date_range( + start=pv_first.floor("5min"), + end=now - datetime.timedelta(minutes=5), + freq="5min", + ) + df_pv_training = pd.DataFrame(index=idx_pv).join(full_hist_raw, how="left") + df_pv_training = _fill_defaults(df_pv_training, history=True) + + idx_fut = pd.date_range(start=now, periods=FORECAST_HORIZON_HOURS * 12, freq="5min") + fut_raw = combined.reindex(combined.index.union(idx_fut)).sort_index() if not combined.empty else pd.DataFrame(index=idx_fut) + df_fut = pd.DataFrame(index=idx_fut).join(fut_raw, how="left") + df_fut = _fill_defaults(df_fut, history=False) + + month_hist = df_hist[(df_hist.index.year == now.year) & (df_hist.index.month == now.month)] + current_peak_kw = 0.0 + if not month_hist.empty and "Netzleistung" in month_hist.columns: + measured_grid = pd.to_numeric(month_hist["Netzleistung"], errors="coerce").dropna() + measured_peak = measured_grid.resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not measured_peak.empty: + current_peak_kw = max(0.0, float(measured_peak.max()) / 1000.0) + if current_peak_kw <= 0.0 and not month_hist.empty and {"Hausverbrauch", "PV"}.issubset(month_hist.columns): + fallback_residual = (month_hist["Hausverbrauch"] - month_hist["PV"]).resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not fallback_residual.empty: + current_peak_kw = max(0.0, float(fallback_residual.max()) / 1000.0) + + min_soc = float( + config.get("batt_min_soc", config.get("batt_min_soc_percent", 0.0)) or 0.0 + ) + current_soc = max(0.0, min(100.0, min_soc)) + current_soc_source = "safe_minimum" + current_soc_age_minutes = None + if "SOC" in full_hist_raw.columns: + soc_values = pd.to_numeric(full_hist_raw["SOC"], errors="coerce").dropna() + if not soc_values.empty: + latest_soc_time = pd.Timestamp(soc_values.index[-1]) + current_soc_age_minutes = max( + 0.0, + (pd.Timestamp(now) - latest_soc_time).total_seconds() / 60.0, + ) + try: + max_soc_age_minutes = max( + 5.0, + float(config.get("batt_soc_max_age_minutes", 30.0) or 30.0), + ) + except (TypeError, ValueError): + max_soc_age_minutes = 30.0 + if current_soc_age_minutes <= max_soc_age_minutes: + current_soc = float(soc_values.iloc[-1]) + current_soc_source = "telemetry" + current_soc = max(0.0, min(100.0, current_soc)) + + return { + "config": config, + "metadata": config, + "now": now, + "df_hist": df_hist, + "df_recent_raw": recent_raw, + "df_load_training": df_load_training, + "df_pv_training": df_pv_training, + "df_fut": df_fut, + "current_soc_perc": current_soc, + "current_soc_source": current_soc_source, + "current_soc_age_minutes": current_soc_age_minutes, + "current_month_peak_kw": current_peak_kw, + } + + +def run_training(): + for config in get_configs(): + if not any(active(config, n) for n in MODEL_MODULES): + continue + print(f"Training Anlage {config['anlagen_id']}...") + data_obj = build_data_object(config, training=True) + for n, module in MODEL_MODULES.items(): + if active(config, n) and hasattr(module, "train"): + try: + target = QUALITY_TARGETS.get(n) + if target: + require_recent_telemetry(data_obj, [target]) + print(f" var_{n}: {module.train(data_obj)}") + except Exception: + print(f" var_{n}: Training fehlgeschlagen") + traceback.print_exc() + + +def _forecast_point(aid, field, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast") + .field(field, float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _snapshot_point(aid, field, run_hour, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast_snapshot") + .tag("run_hour", run_hour) + .field(f"{field}_run_{run_hour}", float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _quality_query(aid, forecast_field, target_field): + stop = _time_literal(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) + return f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -{QUALITY_LOOKBACK_DAYS}d, stop: {stop}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "{target_field}" or r["_field"] == "{forecast_field}") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> group() + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + + +def _forecast_quality(aid, variant, target_field): + forecast_field = f"prog_var_{variant}" + df = _query_df(_quality_query(aid, forecast_field, target_field)) + if df.empty or forecast_field not in df.columns or target_field not in df.columns: + return None + pair = df[[target_field, forecast_field]].apply(pd.to_numeric, errors="coerce").dropna() + if len(pair) < 3: + return None + actual = pair[target_field].to_numpy(dtype=float) + pred = pair[forecast_field].to_numpy(dtype=float) + ss_res = float(np.sum((actual - pred) ** 2)) + ss_tot = float(np.sum((actual - np.mean(actual)) ** 2)) + r2 = None if ss_tot <= 0 else 1.0 - (ss_res / ss_tot) + mae = float(np.mean(np.abs(actual - pred))) + rmse = float(np.sqrt(np.mean((actual - pred) ** 2))) + return {"r2": r2, "mae": mae, "rmse": rmse, "samples": int(len(pair))} + + +def _quality_point(aid, variant, target, metrics): + p = ( + Point("forecast_metrics") + .tag("anlagen_id", aid) + .tag("forecast", f"prog_var_{variant}") + .tag("target", target) + .field("samples", int(metrics["samples"])) + .field("mae", float(metrics["mae"])) + .field("rmse", float(metrics["rmse"])) + .time(datetime.datetime.utcnow(), WritePrecision.S) + ) + if metrics["r2"] is not None: + p.field("r2", float(metrics["r2"])) + return p + + +def write_quality_metrics(write_api, aid, active_variants): + points = [] + + for variant, target in QUALITY_TARGETS.items(): + if variant not in active_variants: + continue + try: + metrics = _forecast_quality(aid, variant, target) + if metrics: + points.append(_quality_point(aid, variant, target, metrics)) + r2_text = "nan" if metrics["r2"] is None else f"{metrics['r2']:.3f}" + print(f"R2 Anlage {aid} prog_var_{variant}: {r2_text} / samples={metrics['samples']}") + except Exception: + print(f"R2 Anlage {aid} prog_var_{variant}: Berechnung fehlgeschlagen") + traceback.print_exc() + if points: + try: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + except Exception: + print(f"R2 Anlage {aid}: Schreiben der Qualitaetswerte fehlgeschlagen") + traceback.print_exc() + + +def run_forecast(only_anlagen_id=None, manual=False): + configs = get_configs() + if only_anlagen_id: + configs = [c for c in configs if c.get("anlagen_id") == only_anlagen_id] + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + write_api = client.write_api(write_options=SYNCHRONOUS) + completed = [] + failures = [] + try: + for config in configs: + aid = config["anlagen_id"] + try: + data_obj = build_data_object(config, training=False) + targets = [] + if any(active(config, n) for n in (1, 3, 10, 13, 21, 23)): + targets.append("PV") + if any(active(config, n) for n in (2, 3, 11, 13, 22, 23)): + targets.append("Hausverbrauch") + if float(config.get("batt_capacity_kwh", 0.0) or 0.0) > 0 and any(active(config, n) for n in (3, 13, 23)): + targets.append("SOC") + # Fail BEFORE model prediction, snapshot publication or V1/V4 plan writes. + data_obj["telemetry_quality"] = require_recent_telemetry(data_obj, targets) + soc_age = data_obj.get("current_soc_age_minutes") + soc_age_text = "keine Messung" if soc_age is None else f"{soc_age:.1f} min" + print( + f"Forecast Anlage {aid}: Batterie-SOC {data_obj['current_soc_perc']:.1f}% " + f"({data_obj['current_soc_source']}, Alter {soc_age_text})." + ) + forecasts, model_errors = collect_predictions( + data_obj, config, + {1: v1, 2: v2, 10: v10, 11: v11, 21: v21, 22: v22}, active, + ) + data_obj['forecast_model_status'] = model_errors + for variant, detail in model_errors.items(): + print(f"Forecast Anlage {aid} prog_var_{variant}: unavailable ({detail['errorType']}); keine Nullwerte eingesetzt.") + p_1, p_2, p_10, p_11, p_21, p_22 = ( + forecasts[n] for n in (1, 2, 10, 11, 21, 22) + ) + + # ENELIX_V4_SHADOW_BRIDGE + _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)]) + + p_3 = v3.predict(data_obj, p_1, p_2) if active(config, 3) and p_1 and p_2 else {} + p_13 = v13.predict(data_obj, p_10, p_11) if active(config, 13) and p_10 and p_11 else {} + p_23 = v23.predict(data_obj, p_21, p_22) if active(config, 23) and p_21 and p_22 else {} + + forecast_sets = [ + (1, p_1), (2, p_2), (3, p_3), + (10, p_10), (11, p_11), (13, p_13), + (21, p_21), (22, p_22), (23, p_23), + ] + run_hour = str(int(datetime.datetime.now(LOCAL_TZ).strftime("%H"))) + points = [] + for variant_id, pv_src, load_src, grid_src in [ + (3, p_1, p_2, p_3), + (13, p_10, p_11, p_13), + (23, p_21, p_22, p_23), + ]: + if grid_src and pv_src and load_src: + points.extend(battery_soc_points(data_obj, variant_id, pv_src, load_src, grid_src)) + active_variants = set() + battery_plans = data_obj.get("battery_plans", {}) + for t in data_obj["df_fut"].index: + for n, values in forecast_sets: + if t in values: + field = f"prog_var_{n}" + value = float(values[t]) + points.append(_forecast_point(aid, field, t, value)) + points.append(_snapshot_point(aid, field, run_hour, t, value)) + if n in battery_plans and t in battery_plans[n].get("battery", {}): + points.append(_forecast_point( + aid, + f"prog_var_{n}_battery", + t, + float(battery_plans[n]["battery"][t]), + )) + active_variants.add(n) + if points: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + print(f"Forecast Anlage {aid}: {len(points)} Punkte geschrieben inkl. run_{run_hour}.") + else: + print(f"Forecast Anlage {aid}: keine aktiven Prognosen oder keine Daten.") + write_quality_metrics(write_api, aid, active_variants) + completed.append(aid) + except Exception as exc: + failures.append({"anlagen_id": aid, "error": f"{type(exc).__name__}: {exc}"}) + print(f"Forecast Anlage {aid}: Lauf fehlgeschlagen.") + traceback.print_exc() + finally: + client.close() + if failures: + raise RuntimeError(f"Forecast-Fehler: {failures}") + return {"completed": completed, "manual": bool(manual)} + + +def _write_json_atomic(path, payload): + directory = os.path.dirname(path) or "." + os.makedirs(directory, exist_ok=True) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + with open(tmp_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +def _read_json(path): + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + return {} + + +def _last_success_age_seconds(): + stamp = _read_json(LAST_SUCCESS_PATH).get("timestamp") + if not stamp: + return None + try: + then = datetime.datetime.fromisoformat(str(stamp).replace("Z", "+00:00")) + now = datetime.datetime.now(datetime.timezone.utc) + return max(0.0, (now - then).total_seconds()) + except Exception: + return None + + +def _child_environment(training=False): + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + if training: + max_threads = str(max(1, int(env.get("FORECAST_TRAINING_MAX_THREADS", "1")))) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "LOKY_MAX_CPU_COUNT", + ): + env[name] = max_threads + return env + + +def _run_child(mode, timeout_seconds, only_anlagen_id=None): + command = [sys.executable, "-u", os.path.abspath(__file__), mode] + if only_anlagen_id: + command.append(str(only_anlagen_id)) + label = "Training" if mode == "--train-once" else "Forecast" + print(f"{label}-Kindprozess startet (Timeout {timeout_seconds}s).") + process = subprocess.Popen(command, env=_child_environment(training=mode == "--train-once")) + try: + return_code = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + print(f"{label}-Kindprozess hat das Zeitlimit erreicht und wird beendet.") + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=15) + return 124 + if return_code != 0: + print(f"{label}-Kindprozess beendet mit Status {return_code}.") + return return_code + + +def run_forecast_isolated(only_anlagen_id=None, source="scheduler"): + if not _FORECAST_PROCESS_LOCK.acquire(blocking=False): + print(f"Forecast-Aufruf ({source}) uebersprungen: bereits ein Lauf aktiv.") + return {"status": "busy", "source": source} + try: + return_code = _run_child("--forecast-once", FORECAST_TIMEOUT_SECONDS, only_anlagen_id) + return { + "status": "ok" if return_code == 0 else "error", + "source": source, + "return_code": return_code, + } + finally: + _FORECAST_PROCESS_LOCK.release() + + +def _training_worker(day_text): + global _TRAINING_THREAD + started = datetime.datetime.now(datetime.timezone.utc).isoformat() + _write_json_atomic(LAST_TRAINING_PATH, { + "day": day_text, + "status": "running", + "started_at": started, + }) + return_code = 1 + error = None + try: + return_code = _run_child("--train-once", TRAINING_TIMEOUT_SECONDS) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + finally: + status = "ok" if return_code == 0 else ("timeout" if return_code == 124 else "error") + payload = { + "day": day_text, + "status": status, + "started_at": started, + "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "return_code": return_code, + } + if error: + payload["error"] = error + _write_json_atomic(LAST_TRAINING_PATH, payload) + print(f"Nachttraining beendet: {status}.") + _TRAINING_THREAD = None + + +def start_training_async(day): + global _TRAINING_THREAD + if _TRAINING_THREAD is not None and _TRAINING_THREAD.is_alive(): + print("Nachttraining laeuft bereits.") + return False + day_text = day.isoformat() + _TRAINING_THREAD = threading.Thread(target=_training_worker, args=(day_text,), daemon=True) + _TRAINING_THREAD.start() + return True + + +def _watchdog_loop(): + while True: + time.sleep(max(60, WATCHDOG_INTERVAL_SECONDS)) + try: + age = _last_success_age_seconds() + if age is None or age > FORECAST_STALE_SECONDS: + age_text = "unbekannt" if age is None else f"{age / 60.0:.1f} Minuten" + print(f"Forecast-Watchdog: letzter erfolgreicher Lauf {age_text}; neuer Lauf wird gestartet.") + run_forecast_isolated(source="watchdog") + except Exception: + print("Forecast-Watchdog: Pruefung fehlgeschlagen.") + traceback.print_exc() + + +# BEGIN EMS RESIMULATE HTTP SERVER +_RESIMULATE_SERVER_STARTED = False + + +def _start_resimulate_server(): + global _RESIMULATE_SERVER_STARTED + if _RESIMULATE_SERVER_STARTED: + return + _RESIMULATE_SERVER_STARTED = True + + import json as _json + import os as _os + import threading as _threading + import traceback as _traceback + import urllib.parse as _urlparse + from http.server import BaseHTTPRequestHandler as _BaseHTTPRequestHandler, ThreadingHTTPServer as _ThreadingHTTPServer + + class _Handler(_BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def _send(self, code, body): + raw = _json.dumps(body).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def _handle(self): + try: + parsed = _urlparse.urlparse(self.path) + if parsed.path == "/health": + age = _last_success_age_seconds() + healthy = age is not None and age <= FORECAST_STALE_SECONDS + self._send(200 if healthy else 503, { + "status": "ok" if healthy else "stale", + "last_success_age_seconds": age, + "training": _read_json(LAST_TRAINING_PATH), + }) + return + if parsed.path not in ("/run_now", "/resimulate"): + self._send(404, {"detail": "not found"}) + return + params = _urlparse.parse_qs(parsed.query) + anlagen_id = (params.get("anlagen_id") or [None])[0] + result = run_forecast_isolated(only_anlagen_id=anlagen_id, source="http") + result["anlagen_id"] = anlagen_id + code = 200 if result["status"] == "ok" else (409 if result["status"] == "busy" else 500) + self._send(code, result) + except Exception as exc: + _traceback.print_exc() + self._send(500, {"detail": str(exc)}) + + port = int(_os.getenv("FORECAST_ENGINE_HTTP_PORT", "9000")) + server = _ThreadingHTTPServer(("0.0.0.0", port), _Handler) + thread = _threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + print(f"Forecast Resimulate HTTP Server startet auf Port {port}.") +# END EMS RESIMULATE HTTP SERVER + +def main(): + _start_resimulate_server() + global last_trained_day + print("Forecast Engine startet.") + threading.Thread(target=_watchdog_loop, daemon=True).start() + run_forecast_isolated(source="startup") + while True: + try: + now_local = datetime.datetime.now(LOCAL_TZ) + if now_local.hour == TRAINING_HOUR and last_trained_day != now_local.date(): + if start_training_async(now_local.date()): + last_trained_day = now_local.date() + next_run = (now_local + datetime.timedelta(hours=1)).replace(minute=0, second=0, microsecond=0) + time.sleep(max(60.0, (next_run - now_local).total_seconds())) + run_forecast_isolated(source="scheduler") + except Exception: + traceback.print_exc() + time.sleep(60) + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "--forecast-once": + selected_anlage = sys.argv[2] if len(sys.argv) >= 3 else None + run_forecast(only_anlagen_id=selected_anlage, manual=True) + _write_json_atomic(LAST_SUCCESS_PATH, { + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "anlagen_id": selected_anlage, + "pid": os.getpid(), + }) + elif len(sys.argv) >= 2 and sys.argv[1] == "--train-once": + run_training() + else: + main() diff --git a/services/netplan-v4/acceptance/forecast-src/methods/__init__.py b/services/netplan-v4/acceptance/forecast-src/methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py new file mode 100644 index 0000000..8615394 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py @@ -0,0 +1,240 @@ +import numpy as np +import pandas as pd +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import lil_matrix + +DT_H = 5.0 / 60.0 + + +def train_artifact(kind): + return {"trained": True, "type": "battery_48h_cost_milp_v3", "source": kind} + + +def _cfg_float(config, key, default): + try: + value = config.get(key, default) + return float(default if value is None or value == "" else value) + except Exception: + return float(default) + + +def _cfg_bool(config, key, default=False): + value = config.get(key, default) + if value is None or value == "": + return bool(default) + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "ja", "on"} + + +def _use_dynamic(config, key): + value = str(config.get(key, "") or "").strip().lower() + return any(token in value for token in ("dynam", "marktpreis", "referenzmarktpreis", "market")) + + +def _price(data_obj, timestamp, column, fallback, dynamic_enabled): + if not dynamic_enabled: + return fallback + frame = data_obj["df_fut"] + if column in frame.columns and timestamp in frame.index: + try: + value = float(frame.at[timestamp, column]) + if np.isfinite(value): + return value + except Exception: + pass + return fallback + + +def _battery_meta(config, data_obj): + cap_kwh = _cfg_float(config, "batt_capacity_kwh", 0.0) + max_power_w = _cfg_float(config, "batt_power_kw", 0.0) * 1000.0 + min_soc = _cfg_float(config, "batt_min_soc", _cfg_float(config, "batt_min_soc_percent", 0.0)) + max_soc = _cfg_float(config, "batt_max_soc", _cfg_float(config, "batt_max_soc_percent", 100.0)) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None: + start_soc_value = min_soc + try: + start_soc = float(start_soc_value) + except (TypeError, ValueError): + start_soc = min_soc + min_soc = max(0.0, min(100.0, min_soc)) + max_soc = max(min_soc, min(100.0, max_soc)) + start_soc = max(min_soc, min(max_soc, start_soc)) + charge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_discharge_efficiency", 0.95))) + return cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff + + +def _quarter_groups(index): + groups = {} + for position, timestamp in enumerate(index): + quarter = timestamp.floor("15min") if hasattr(timestamp, "floor") else position // 3 + groups.setdefault(quarter, []).append(position) + return list(groups.values()) + + +def _fallback_plan(index, residual_w): + grid = {timestamp: float(value) for timestamp, value in zip(index, residual_w)} + battery = {timestamp: 0.0 for timestamp in index} + return {"grid": grid, "battery": battery, "solver": "fallback"} + + +def optimize_battery_plan(data_obj, pv_dict, load_dict): + config = data_obj["config"] + cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff = _battery_meta(config, data_obj) + index = list(data_obj["df_fut"].index) + if not index: + return {"grid": {}, "battery": {}, "solver": "empty"} + + load_w = np.array([max(0.0, float(load_dict.get(t, 0.0))) for t in index]) + pv_w = np.array([max(0.0, float(pv_dict.get(t, 0.0))) for t in index]) + residual_w = load_w - pv_w + if cap_kwh <= 0.0 or max_power_w <= 0.0: + return _fallback_plan(index, residual_w) + + import_fixed = _cfg_float(config, "tarif_bezug_fest", 0.30) + export_fixed = _cfg_float(config, "tarif_einspeisung_fest", 0.10) + import_dynamic = _use_dynamic(config, "tarif_bezug") + export_dynamic = _use_dynamic(config, "tarif_einspeisung") + import_price = np.array([ + _price(data_obj, t, "import_price", import_fixed, import_dynamic) for t in index + ]) + export_price = np.array([ + _price(data_obj, t, "export_price", export_fixed, export_dynamic) for t in index + ]) + + n = len(index) + imp, exp, charge, discharge, curtail, soc = 0, n, 2 * n, 3 * n, 4 * n, 5 * n + peak = 6 * n + 1 + battery_mode = peak + 1 + grid_mode = battery_mode + n + variable_count = grid_mode + n + + max_import_w = max(float(load_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_import_limit = _cfg_float(config, "grid_import_limit_w", 0.0) + if configured_import_limit > 0.0: + max_import_w = min(max_import_w, configured_import_limit) + max_export_w = max(float(pv_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_export_limit = _cfg_float(config, "grid_export_limit_w", 0.0) + if configured_export_limit > 0.0: + max_export_w = min(max_export_w, configured_export_limit) + + lower = np.zeros(variable_count) + upper = np.full(variable_count, np.inf) + upper[imp:imp + n] = max_import_w + upper[exp:exp + n] = max_export_w + upper[charge:charge + n] = max_power_w + if not _cfg_bool(config, "batt_grid_charging_enabled", False): + upper[charge:charge + n] = np.minimum(max_power_w, np.maximum(0.0, pv_w - load_w)) + upper[discharge:discharge + n] = max_power_w + upper[curtail:curtail + n] = pv_w + reserve_soc = max( + min_soc, + min(100.0, _cfg_float(config, "batt_economic_reserve_soc_percent", 10.0)), + ) + economic_min_soc = max(min_soc, min(start_soc, reserve_soc)) + min_kwh = cap_kwh * economic_min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + lower[soc:soc + n + 1] = min_kwh + upper[soc:soc + n + 1] = max_kwh + current_peak_kw = max(0.0, float(data_obj.get("current_month_peak_kw", 0.0) or 0.0)) + lower[peak] = current_peak_kw + upper[peak] = max(current_peak_kw, max_import_w / 1000.0) + upper[battery_mode:battery_mode + n] = 1.0 + upper[grid_mode:grid_mode + n] = 1.0 + + objective = np.zeros(variable_count) + objective[imp:imp + n] = import_price * DT_H / 1000.0 + objective[exp:exp + n] = -export_price * DT_H / 1000.0 + degradation = max(0.0, _cfg_float(config, "batt_degradation_chf_kwh", 0.03)) + objective[charge:charge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[discharge:discharge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[curtail:curtail + n] = 1e-9 * DT_H / 1000.0 + objective[peak] = max(0.0, _cfg_float(config, "tarif_peak_fest", 0.0)) + terminal_value = _cfg_float(config, "batt_terminal_value_chf_kwh", np.median(import_price)) + objective[soc + n] = -max(0.0, terminal_value) * discharge_eff + + equality_rows = 2 * n + 1 + equality = lil_matrix((equality_rows, variable_count), dtype=float) + equality_rhs = np.zeros(equality_rows) + for i in range(n): + equality[i, imp + i] = 1.0 + equality[i, exp + i] = -1.0 + equality[i, charge + i] = -1.0 + equality[i, discharge + i] = 1.0 + equality[i, curtail + i] = -1.0 + equality_rhs[i] = residual_w[i] + + row = n + i + equality[row, soc + i] = -1.0 + equality[row, soc + i + 1] = 1.0 + equality[row, charge + i] = -charge_eff * DT_H / 1000.0 + equality[row, discharge + i] = DT_H / (1000.0 * discharge_eff) + equality[2 * n, soc] = 1.0 + equality_rhs[2 * n] = cap_kwh * start_soc / 100.0 + + quarter_groups = _quarter_groups(pd.Index(index)) + inequality_rows = 4 * n + len(quarter_groups) + inequality = lil_matrix((inequality_rows, variable_count), dtype=float) + inequality_upper = np.zeros(inequality_rows) + row = 0 + for i in range(n): + inequality[row, charge + i] = 1.0 + inequality[row, battery_mode + i] = -max_power_w + row += 1 + inequality[row, discharge + i] = 1.0 + inequality[row, battery_mode + i] = max_power_w + inequality_upper[row] = max_power_w + row += 1 + inequality[row, imp + i] = 1.0 + inequality[row, grid_mode + i] = -max_import_w + row += 1 + inequality[row, exp + i] = 1.0 + inequality[row, grid_mode + i] = max_export_w + inequality_upper[row] = max_export_w + row += 1 + for group in quarter_groups: + for i in group: + inequality[row, imp + i] = 1.0 / (len(group) * 1000.0) + inequality[row, peak] = -1.0 + row += 1 + + integrality = np.zeros(variable_count, dtype=int) + integrality[battery_mode:battery_mode + n] = 1 + integrality[grid_mode:grid_mode + n] = 1 + constraints = [ + LinearConstraint(equality.tocsr(), equality_rhs, equality_rhs), + LinearConstraint(inequality.tocsr(), -np.inf, inequality_upper), + ] + result = milp( + objective, + integrality=integrality, + bounds=Bounds(lower, upper), + constraints=constraints, + options={"time_limit": max(5.0, _cfg_float(config, "batt_optimizer_timeout_seconds", 30.0))}, + ) + if not result.success or result.x is None: + return _fallback_plan(index, residual_w) + + grid_values = result.x[imp:imp + n] - result.x[exp:exp + n] + battery_values = result.x[charge:charge + n] - result.x[discharge:discharge + n] + threshold_w = max(25.0, max_power_w * 0.005) + grid_values[np.abs(grid_values) < threshold_w] = 0.0 + battery_values[np.abs(battery_values) < threshold_w] = 0.0 + return { + "grid": {t: float(v) for t, v in zip(index, grid_values)}, + "battery": {t: float(v) for t, v in zip(index, battery_values)}, + "solver": "scipy-milp", + "objective_chf": float(result.fun), + "planned_peak_kw": float(result.x[peak]), + "start_soc_percent": float(start_soc), + "economic_min_soc_percent": float(economic_min_soc), + } + + +def optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=None): + plan = optimize_battery_plan(data_obj, pv_dict, load_dict) + if forecast_id is not None: + data_obj.setdefault("battery_plans", {})[int(forecast_id)] = plan + return plan["grid"] diff --git a/services/netplan-v4/acceptance/forecast-src/methods/common.py b/services/netplan-v4/acceptance/forecast-src/methods/common.py new file mode 100644 index 0000000..a0ab5e9 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/common.py @@ -0,0 +1,32 @@ +import os +import joblib + +MODEL_DIR = "/app/data/models" + + +def model_path(aid, forecast_id): + os.makedirs(MODEL_DIR, exist_ok=True) + return os.path.join(MODEL_DIR, f"forecast_var_{forecast_id}_{aid}.pkl") + + +def save_model(aid, forecast_id, artifact): + path = model_path(aid, forecast_id) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + joblib.dump(artifact, tmp_path) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + return path + + +def load_model(aid, forecast_id): + path = model_path(aid, forecast_id) + if not os.path.exists(path): + return None + try: + return joblib.load(path) + except Exception as exc: + print(f"Modell {path} konnte nicht geladen werden: {exc}") + return None diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_1.py b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py new file mode 100644 index 0000000..98cbe44 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py @@ -0,0 +1,66 @@ +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model +from shared_utils import calc_pure_math_pv, roof_features + +FORECAST_ID = 1 +FEATURES = [ + "math_pv", + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "sin_year", + "cos_year", + "pv_kwp_total", + "roof_azimuth_sin", + "roof_azimuth_cos", + "roof_tilt_avg", + "roof_south_factor", +] + + +def _features(frame, config): + out = frame.copy() + out["math_pv"] = [calc_pure_math_pv(config, t) for t in out.index] + rf = roof_features(config) + for k, v in rf.items(): + out[k] = v + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out[FEATURES].astype(float) + + +def train(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + df = data_obj.get("df_pv_training", data_obj["df_hist"]).copy() + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + X = _features(df, config) + y = pd.to_numeric(df["PV"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + model = RandomForestRegressor(n_estimators=400, max_depth=18, min_samples_leaf=2, random_state=42, n_jobs=-1) + model.fit(X, y) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES}) + return {"trained": True, "samples": int(len(X)), "path": path} + + +def predict(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + X = _features(data_obj["df_fut"], config) + if artifact and "model" in artifact: + values = artifact["model"].predict(X) + else: + values = X["math_pv"].to_numpy() + ac_limit = float(config.get("ac_leistung", 10.0) or 10.0) * 1000.0 + return {t: max(0.0, min(float(v), ac_limit)) for t, v in zip(data_obj["df_fut"].index, values)} diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_10.py b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py new file mode 100644 index 0000000..02ea37d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 10, {"type": "repeat_pv_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'PV') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_11.py b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py new file mode 100644 index 0000000..7e1dc3c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 11, {"type": "repeat_load_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'Hausverbrauch') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_13.py b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py new file mode 100644 index 0000000..9775eeb --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 13 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_10_11"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=13) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_2.py b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py new file mode 100644 index 0000000..fbb3d59 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py @@ -0,0 +1,169 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor + +from methods.common import load_model, save_model +from telemetry_quality import profile_source_value + +FORECAST_ID = 2 +FEATURES = [ + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "weekday", + "is_weekday", + "load_24h_ago", + "load_7d_ago", + "energy_24h_rolling", + "load_3d_same_time_mean", + "load_7d_same_time_mean", +] + + +def _history_features(df): + out = df.copy() + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_24h_rolling"] = (out["Hausverbrauch"] * 5 / 60 / 1000).shift(1).rolling(288).sum() + same_time_lags = [out["Hausverbrauch"].shift(288 * d) for d in range(1, 8)] + out["load_3d_same_time_mean"] = pd.concat(same_time_lags[:3], axis=1).mean(axis=1) + out["load_7d_same_time_mean"] = pd.concat(same_time_lags, axis=1).mean(axis=1) + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = data_obj.get("df_load_training", data_obj["df_hist"]).copy() + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = _history_features(df) + X = df[FEATURES].astype(float) + y = pd.to_numeric(df["Hausverbrauch"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + validation_day = X.index.max().normalize() + validation_mask = X.index >= validation_day + if int(validation_mask.sum()) < 144: + validation_day -= datetime.timedelta(days=1) + validation_mask = (X.index >= validation_day) & (X.index < validation_day + datetime.timedelta(days=1)) + score = None + if int((~validation_mask).sum()) >= 288 and int(validation_mask.sum()) >= 96: + eval_model = HistGradientBoostingRegressor( + max_iter=900, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + eval_model.fit(X.loc[~validation_mask], y.loc[~validation_mask]) + score = float(eval_model.score(X.loc[validation_mask], y.loc[validation_mask])) + + model = HistGradientBoostingRegressor( + max_iter=1200, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + model.fit(X, y) + path = save_model( + aid, + FORECAST_ID, + {"model": model, "features": FEATURES, "r2_last_day": score}, + ) + return { + "trained": True, + "samples": int(len(X)), + "r2_last_day": score, + "path": path, + } + + +def _history_value(history, ts): + if history is None or history.empty or ts not in history.index or "Hausverbrauch" not in history.columns: + return None + value = history.at[ts, "Hausverbrauch"] + return float(value) if pd.notna(value) and np.isfinite(value) and value >= 0 else None + + +def _same_time_values(history, t, days): + values = [] + for day in range(1, days + 1): + value = _history_value(history, t - datetime.timedelta(days=day)) + if value is not None and np.isfinite(value): + values.append(max(0.0, value)) + return values + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + + frames = [ + frame + for frame in ( + data_obj.get("df_load_training"), + data_obj.get("df_hist"), + data_obj.get("df_recent_raw"), + ) + if frame is not None and not frame.empty and "Hausverbrauch" in frame.columns + ] + history = pd.concat(frames).sort_index() if frames else pd.DataFrame(columns=["Hausverbrauch"]) + history = history[~history.index.duplicated(keep="last")] + fut = data_obj["df_fut"] + res = {} + + for t in fut.index: + same_values = _same_time_values(history, t, 7) + load_24 = _history_value(history, t - datetime.timedelta(days=1)) + load_7d = _history_value(history, t - datetime.timedelta(days=7)) + if load_24 is None: + load_24 = load_7d if load_7d is not None else profile_source_value(history, t, "Hausverbrauch") + if load_7d is None: + load_7d = load_24 + same_median = float(np.median(same_values)) if same_values else load_7d + profile = max(0.0, (0.50 * load_24) + (0.30 * load_7d) + (0.20 * same_median)) + + window = history.loc[ + t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), + "Hausverbrauch", + ] + energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + same_3 = same_values[:3] + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "cloud"]), + float(fut.at[t, "hour_sin"]), + float(fut.at[t, "hour_cos"]), + int(fut.at[t, "weekday"]), + int(fut.at[t, "is_weekday"]), + load_24, + load_7d, + energy, + float(np.mean(same_3)) if same_3 else profile, + float(np.mean(same_values)) if same_values else profile, + ]], columns=FEATURES) + + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + model_pred = min(max(model_pred, profile * 0.25), max(500.0, profile * 3.0)) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + + res[t] = pred + history.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_21.py b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py new file mode 100644 index 0000000..1c84981 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py @@ -0,0 +1,62 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model + +FORECAST_ID = 21 +FEATURES = ["temp_c", "hour_cos", "pv_24h_ago"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + if "PV" in out.columns: + out["pv_24h_ago"] = out["PV"].shift(288) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_pv_training", data_obj["df_hist"]).copy()) + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + df = df.dropna(subset=FEATURES + ["PV"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["PV"]) + score = float(eval_model.score(test_df[FEATURES], test_df["PV"])) + print(f"[var_21] R2 PV letzter kompletter Tag: {score:.3f}") + + model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + model.fit(df[FEATURES], df["PV"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + hist = data_obj["df_hist"] + fut = data_obj["df_fut"].copy() + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + pv_24 = float(hist.at[t_24, "PV"]) if t_24 in hist.index else 0.0 + row = pd.DataFrame([[float(fut.at[t, "temp_c"]), float(fut.at[t, "hour_cos"]), pv_24]], columns=FEATURES) + pred = float(model.predict(row)[0]) if model is not None else pv_24 + res[t] = max(0.0, pred) + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_22.py b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py new file mode 100644 index 0000000..3f1d6d8 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py @@ -0,0 +1,121 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.inspection import permutation_importance + +from methods.common import load_model, save_model + +FORECAST_ID = 22 +FEATURES = ["temp_c", "hour_cos", "load_24h_ago", "load_7d_ago", "energy_24h_rolling", "weekday"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + out["weekday"] = out.index.weekday + if "Hausverbrauch" in out.columns: + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_5m_kwh"] = out["Hausverbrauch"] * (5 / 60) / 1000 + out["energy_24h_rolling"] = out["energy_5m_kwh"].shift(1).rolling(window=288).sum() + out = out.drop(columns=["energy_5m_kwh"]) + return out + + +def _history_value(history, recent, reference, ts): + for frame in (history, recent, reference): + if frame is not None and not frame.empty and ts in frame.index and "Hausverbrauch" in frame.columns: + value = frame.at[ts, "Hausverbrauch"] + if pd.notna(value): + return float(value) + return None + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_load_training", data_obj["df_hist"]).copy()) + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = df.dropna(subset=FEATURES + ["Hausverbrauch"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + importance = {} + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["Hausverbrauch"]) + score = float(eval_model.score(test_df[FEATURES], test_df["Hausverbrauch"])) + print(f"[var_22] R2 Hausverbrauch letzter kompletter Tag: {score:.3f}") + try: + perm = permutation_importance(eval_model, test_df[FEATURES], test_df["Hausverbrauch"], n_repeats=10, random_state=42) + order = perm.importances_mean.argsort()[::-1] + importance = {FEATURES[i]: float(perm.importances_mean[i]) for i in order} + print("[var_22] Feature-Wichtigkeit Hausverbrauch:") + for name, val in importance.items(): + print(f" {name}: {val:.4f}") + except Exception as exc: + print(f"[var_22] permutation_importance nicht berechnet: {exc}") + + model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + model.fit(df[FEATURES], df["Hausverbrauch"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score, "importance": importance}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + hist = data_obj["df_hist"].copy() + recent = data_obj.get("df_recent_raw", pd.DataFrame()) + reference = data_obj.get("df_load_training", pd.DataFrame()) + fut = data_obj["df_fut"] + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + t_7d = t - datetime.timedelta(days=7) + load_7d = _history_value(hist, recent, reference, t_7d) + load_24 = _history_value(hist, recent, reference, t_24) + if load_24 is None: + load_24 = load_7d if load_7d is not None else 0.0 + if load_7d is None: + load_7d = load_24 + ref_end = t - datetime.timedelta(days=7) + ref_start = ref_end - datetime.timedelta(days=1) + if not reference.empty and "Hausverbrauch" in reference.columns: + window = reference.loc[ref_start:ref_end - datetime.timedelta(minutes=5), "Hausverbrauch"] + else: + window = pd.Series(dtype=float) + if window.empty and "Hausverbrauch" in recent.columns: + window = recent.loc[t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), "Hausverbrauch"] + roll_energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "hour_cos"]), + load_24, + load_7d, + roll_energy, + int(fut.at[t, "weekday"]), + ]], columns=FEATURES) + profile = max(0.0, (0.65 * load_24) + (0.35 * load_7d)) + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + res[t] = pred + hist.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_23.py b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py new file mode 100644 index 0000000..3cafe6d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 23 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_21_22"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=23) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_3.py b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py new file mode 100644 index 0000000..db46d6f --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 3 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_1_2"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=3) diff --git a/services/netplan-v4/acceptance/forecast-src/model_isolation.py b/services/netplan-v4/acceptance/forecast-src/model_isolation.py new file mode 100644 index 0000000..e64d3d1 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/model_isolation.py @@ -0,0 +1,45 @@ +"""Forecast family fault isolation; no IO, fabricated samples or model switching.""" +from collections.abc import Mapping +from math import isfinite +from numbers import Real + + +def collect_predictions(data, config, models, enabled): + """Return complete finite PV/load series independently for each requested model. + + A model with insufficient history must not prevent other valid families being + published. It is omitted, not filled with zeros or replaced by another model. + Global raw-telemetry checks still run BEFORE this helper in run_forecast. + """ + expected = tuple(data['df_fut'].index) + if not expected or len(set(expected)) != len(expected): + raise ValueError('Nonempty unique future interval index required') + expected_keys = set(expected) + predictions, errors = {}, {} + requested = 0 + for variant, model in models.items(): + predictions[variant] = {} + if not enabled(config, variant): + continue + requested += 1 + try: + values = model.predict(data) + if not isinstance(values, Mapping) or set(values) != expected_keys: + raise ValueError('Missing, extra or non-matching forecast timestamps') + cleaned = {} + for at in expected: + value = values[at] + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError('Non-numeric forecast power') + value = float(value) + if not isfinite(value) or value < 0: + raise ValueError('Invalid nonnegative forecast power') + cleaned[at] = value + predictions[variant] = cleaned + except Exception as error: + # Expected data failures and unexpected model failures are visible, + # but model exception text may contain filesystem paths. Do not leak it. + errors[variant] = {'status': 'unavailable', 'errorType': type(error).__name__} + if requested and all(not p for p in predictions.values()): + raise ValueError('All requested PV/load models failed; no forecasts published') + return predictions, errors diff --git a/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/acceptance/forecast-src/requirements.txt b/services/netplan-v4/acceptance/forecast-src/requirements.txt new file mode 100644 index 0000000..69cb44c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/requirements.txt @@ -0,0 +1,11 @@ +schedule +pandas +numpy +influxdb-client +pytz +holidays +pvlib +geopy +scikit-learn +scipy +joblib diff --git a/services/netplan-v4/acceptance/forecast-src/shared_utils.py b/services/netplan-v4/acceptance/forecast-src/shared_utils.py new file mode 100644 index 0000000..fb40dce --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/shared_utils.py @@ -0,0 +1,93 @@ +import math + + +def as_float(value, default=0.0): + try: + if value is None: + return default + return float(value) + except Exception: + return default + + +def roof_kwp(roof): + return as_float(roof.get("kwp", roof.get("leistung", roof.get("pv_kwp", 0.0))), 0.0) + + +def roof_azimuth_deg(roof): + return as_float(roof.get("azimut", roof.get("azimuth", roof.get("ausrichtung", 180.0))), 180.0) + + +def roof_tilt_deg(roof): + return as_float(roof.get("neigung", roof.get("tilt", 30.0)), 30.0) + + +def roof_features(config): + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "azimut": 180.0, "neigung": 30.0}] + total_kwp = sum(roof_kwp(r) for r in roofs) + if total_kwp <= 0: + total_kwp = as_float(config.get("ac_leistung", 0.0), 0.0) + weighted_az = 0.0 + weighted_tilt = 0.0 + south_factor = 0.0 + for roof in roofs: + w = roof_kwp(roof) / total_kwp if total_kwp > 0 else 0.0 + az = roof_azimuth_deg(roof) + tilt = roof_tilt_deg(roof) + weighted_az += w * az + weighted_tilt += w * tilt + south_factor += w * max(0.0, math.cos(math.radians(az - 180.0))) + return { + "pv_kwp_total": total_kwp, + "roof_azimuth_sin": math.sin(math.radians(weighted_az)), + "roof_azimuth_cos": math.cos(math.radians(weighted_az)), + "roof_tilt_avg": weighted_tilt, + "roof_south_factor": south_factor, + } + + +def calc_pure_math_pv(config, dt): + ac_limit = as_float(config.get("ac_leistung", 10.0), 10.0) * 1000.0 + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "neigung": 30.0, "azimut": 180.0}] + + day = dt.timetuple().tm_yday + hour = dt.hour + dt.minute / 60.0 + lat = math.radians(as_float(config.get("latitude", 47.0), 47.0)) + decl = math.radians(23.45 * math.sin(math.radians(360.0 * (day - 81) / 365.0))) + hour_angle = math.radians(15.0 * (hour - 12.0)) + + sin_alt = math.sin(lat) * math.sin(decl) + math.cos(lat) * math.cos(decl) * math.cos(hour_angle) + sun_alt = math.asin(max(-1.0, min(1.0, sin_alt))) + if sun_alt <= 0.0: + return 0.0 + + sun_az = math.atan2( + math.sin(hour_angle), + math.cos(hour_angle) * math.sin(lat) - math.tan(decl) * math.cos(lat), + ) + + total = 0.0 + for roof in roofs: + kwp = roof_kwp(roof) + if kwp <= 0: + continue + tilt = math.radians(roof_tilt_deg(roof)) + az = math.radians(roof_azimuth_deg(roof)) + cos_inc = math.sin(sun_alt) * math.cos(tilt) + math.cos(sun_alt) * math.sin(tilt) * math.cos(sun_az - az) + if cos_inc > 0: + total += kwp * 1000.0 * cos_inc + + return max(0.0, min(total, ac_limit)) + + +def calc_pure_math_load(dt): + hour = dt.hour + dt.minute / 60.0 + base = 650.0 + morning = 220.0 * math.exp(-((hour - 7.0) ** 2) / 5.0) + evening = 380.0 * math.exp(-((hour - 19.0) ** 2) / 8.0) + weekend = 1.12 if dt.weekday() >= 5 else 1.0 + return max(0.0, (base + morning + evening) * weekend) diff --git a/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py new file mode 100644 index 0000000..f2bf2e7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py @@ -0,0 +1,66 @@ +from influxdb_client import Point, WritePrecision + +DT_H = 5.0 / 60.0 + + +def _f(config, key, default): + try: + return float(config.get(key, default) or default) + except Exception: + return float(default) + + +def battery_soc_points(data_obj, forecast_var, pv_dict, load_dict, grid_dict): + config = data_obj["config"] + aid = str(config["anlagen_id"]) + cap_kwh = _f(config, "batt_capacity_kwh", 0.0) + if cap_kwh <= 0 or not grid_dict: + return [] + + min_soc = _f(config, "batt_min_soc", _f(config, "batt_min_soc_percent", 0.0)) + max_soc = _f(config, "batt_max_soc", _f(config, "batt_max_soc_percent", 100.0)) + charge_eff = max(0.01, min(1.0, _f(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _f(config, "batt_discharge_efficiency", 0.95))) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None or start_soc_value == "": + start_soc_value = config.get("batt_soc_percent", 50.0) + start_soc = float(start_soc_value) + start_soc = max(min_soc, min(max_soc, start_soc)) + + min_kwh = cap_kwh * min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + soc_kwh = max(min_kwh, min(max_kwh, cap_kwh * start_soc / 100.0)) + + planned_battery = ( + data_obj.get("battery_plans", {}) + .get(int(forecast_var), {}) + .get("battery", {}) + ) + points = [] + for t in data_obj["df_fut"].index: + if t not in grid_dict: + continue + if t in planned_battery: + battery_target_w = float(planned_battery[t]) # positiv = laden + else: + residual_w = float(load_dict.get(t, 0.0)) - float(pv_dict.get(t, 0.0)) + grid_w = float(grid_dict.get(t, 0.0)) + battery_target_w = grid_w - residual_w + + if battery_target_w > 0: + soc_kwh += battery_target_w * DT_H / 1000.0 * charge_eff + elif battery_target_w < 0: + soc_kwh += battery_target_w * DT_H / 1000.0 / discharge_eff + + soc_kwh = max(min_kwh, min(max_kwh, soc_kwh)) + soc_percent = max(0.0, min(100.0, soc_kwh / cap_kwh * 100.0)) + + points.append( + Point("forecast_diagnostics") + .tag("anlagen_id", aid) + .tag("data_type", "battery_soc_simulation") + .tag("forecast_var", f"prog_var_{forecast_var}") + .field("soc_percent", float(soc_percent)) + .time(t.to_pydatetime() if hasattr(t, "to_pydatetime") else t, WritePrecision.S) + ) + return points diff --git a/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py new file mode 100644 index 0000000..bf316d2 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py @@ -0,0 +1,106 @@ +"""Measured telemetry integrity and repeat-profile helpers. +No IO, no fabricated measurements and no fixed household-load fallback. +Forecast freshness is checked against raw telemetry, never against filled features. +""" +from __future__ import annotations +import datetime +import math +import numpy as np +import pandas as pd + +MEASURED_COLUMNS = ('PV', 'Hausverbrauch', 'Netzleistung', 'SOC') + +class TelemetryUnavailable(ValueError): + """No publishable forecast can be derived from the supplied observations.""" + + +def sanitize_measured_frame(frame): + out = frame.copy() + for column in MEASURED_COLUMNS: + if column not in out: + continue + raw = out[column] + numeric = pd.to_numeric(raw, errors='coerce').astype(float) + bad = ~np.isfinite(numeric) | raw.map(lambda x: isinstance(x, (bool, np.bool_))) + if column in ('PV', 'Hausverbrauch', 'SOC'): + bad |= numeric < 0 + if column == 'SOC': + bad |= numeric > 100 + out[column] = numeric.mask(bad) + return out + + +def _naive_utc(value): + stamp = pd.Timestamp(value) + if stamp.tzinfo is not None: + stamp = stamp.tz_convert('UTC').tz_localize(None) + return stamp + + +def require_recent_telemetry(data_obj, fields=('PV','Hausverbrauch'), max_age_minutes=30.0): + """Refuse missing/stale inputs without changing real zeros or raw samples.""" + if isinstance(max_age_minutes, bool) or not math.isfinite(max_age_minutes) or max_age_minutes <= 0: + raise ValueError('Positive telemetry age limit required') + now = _naive_utc(data_obj['now']) + raw = data_obj.get('df_recent_raw') + raw = sanitize_measured_frame(raw) if raw is not None else pd.DataFrame() + if not raw.empty: + raw.index = pd.DatetimeIndex([_naive_utc(t) for t in raw.index]) + raw = raw.loc[raw.index < now].sort_index() + report = {} + errors = [] + for field in fields: + if field not in MEASURED_COLUMNS: + raise ValueError('Unknown telemetry target') + values = raw[field].dropna() if field in raw else pd.Series(dtype=float) + if values.empty: + errors.append(field + ': keine gemessenen Werte') + continue + stamp = values.index[-1] + age = (now-stamp).total_seconds()/60.0 + report[field] = {'lastObservedInterval': stamp.isoformat()+'Z', 'ageMinutes': age, + 'observedIntervals': int(len(values)), 'lastValue': float(values.iloc[-1])} + if age > max_age_minutes: + errors.append(field + ': Messdaten veraltet (' + format(age,'.1f') + ' min)') + if errors: + raise TelemetryUnavailable('; '.join(errors) + '. Keine neuen Prognosen/Fahrplaene veroeffentlicht.') + return report + + +def _finite_nonnegative(value): + if isinstance(value, (bool, np.bool_)): + return None + try: + value = float(value) + except (ValueError, TypeError): + return None + return value if math.isfinite(value) and value >= 0.0 else None + + +def profile_source_value(history, at, column, predictions=None): + """Repeat yesterday; prefer same weekday if yesterday is missing, then older days. + Explicitly generated first-day values may be repeated on the second forecast day. + Missing historical values are not zeros. No future measurement is read. + """ + if column not in ('PV','Hausverbrauch'): + raise ValueError('Unsupported repeat-profile target') + predictions = {} if predictions is None else predictions + at = pd.Timestamp(at) + for day in (1, 7, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14): + source = at - datetime.timedelta(days=day) + if source in predictions: + value = _finite_nonnegative(predictions[source]) + if value is not None: + return value + if history is not None and column in history and source in history.index: + value = _finite_nonnegative(history.at[source,column]) + if value is not None: + return value + raise TelemetryUnavailable(column + ': kein gemessener Tagesprofilwert fuer ' + str(at)) + + +def repeat_daily_profile(history, future_index, column): + result = {} + for at in future_index: + result[at] = profile_source_value(history, at, column, result) + return result diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py new file mode 100644 index 0000000..e533d76 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py @@ -0,0 +1,133 @@ +import unittest + +import pandas as pd + +from methods.battery_optimizer import DT_H, optimize_battery_plan + + +class BatteryCostOptimizerTest(unittest.TestCase): + def plan(self, load, pv, import_prices=None, export_prices=None, **overrides): + index = pd.date_range("2026-09-29T00:00:00", periods=len(load), freq="5min") + config = { + "batt_capacity_kwh": 2.0, + "batt_power_kw": 1.0, + "batt_min_soc": 0.0, + "batt_max_soc": 100.0, + "batt_charge_efficiency": 1.0, + "batt_discharge_efficiency": 1.0, + "batt_soc_percent": 0.0, + "batt_grid_charging_enabled": False, + "batt_degradation_chf_kwh": 0.0, + "tarif_bezug": "dynamic", + "tarif_einspeisung": "dynamic", + "tarif_peak_fest": 0.0, + } + config.update(overrides) + data = { + "config": config, + "df_fut": pd.DataFrame({ + "import_price": import_prices or [0.30] * len(index), + "export_price": export_prices or [0.10] * len(index), + }, index=index), + "current_soc_perc": config["batt_soc_percent"], + "current_month_peak_kw": overrides.get("current_month_peak_kw", 0.0), + } + return index, optimize_battery_plan(data, dict(zip(index, pv)), dict(zip(index, load))) + + def test_grid_charging_is_opt_in(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + ) + self.assertTrue(all(plan["battery"][t] <= 1e-6 for t in index)) + + def test_cheap_grid_energy_is_shifted_to_expensive_period(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + batt_grid_charging_enabled=True, + ) + self.assertGreater(plan["battery"][index[0]], 0.0) + self.assertLess(plan["battery"][index[-1]], 0.0) + self.assertGreater(plan["grid"][index[0]], 500.0) + self.assertAlmostEqual(plan["grid"][index[-1]], 0.0, places=5) + + def test_high_feed_in_value_prefers_export(self): + index, plan = self.plan( + [0.0, 1000.0], + [1000.0, 0.0], + import_prices=[0.20, 0.20], + export_prices=[0.60, 0.60], + ) + self.assertAlmostEqual(plan["grid"][index[0]], -1000.0, places=5) + self.assertAlmostEqual(plan["grid"][index[1]], 1000.0, places=5) + + def test_peak_tariff_prevents_grid_charge_above_existing_peak(self): + index, plan = self.plan( + [1000.0] * 6, + [0.0] * 6, + import_prices=[0.05] * 3 + [0.50] * 3, + batt_grid_charging_enabled=True, + tarif_peak_fest=20.0, + current_month_peak_kw=1.0, + ) + self.assertLessEqual(plan["planned_peak_kw"], 1.0 + 1e-7) + self.assertTrue(all(plan["grid"][t] <= 1000.0 + 1e-5 for t in index)) + + def test_nearly_empty_battery_is_not_discharged_further_at_low_value(self): + index, plan = self.plan( + [1000.0] * 4, + [0.0] * 4, + import_prices=[0.01] * 4, + export_prices=[0.0] * 4, + batt_soc_percent=5.0, + batt_economic_reserve_soc_percent=10.0, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(all(plan["battery"][timestamp] >= -1e-6 for timestamp in index)) + self.assertTrue(all(abs(plan["grid"][timestamp] - 1000.0) <= 1e-5 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 5.0) + + def test_profitable_export_discharge_remains_allowed_above_reserve(self): + index, plan = self.plan( + [100.0, 100.0], + [1000.0, 1000.0], + import_prices=[0.20, 0.20], + export_prices=[0.80, 0.80], + batt_soc_percent=100.0, + batt_economic_reserve_soc_percent=10.0, + batt_degradation_chf_kwh=0.03, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(any(plan["battery"][timestamp] < -1e-6 for timestamp in index)) + self.assertTrue(all(plan["grid"][timestamp] <= -899.0 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 10.0) + + def test_power_and_soc_limits_hold(self): + load = [0.0] * 12 + [1000.0] * 12 + pv = [1000.0] * 12 + [0.0] * 12 + index, plan = self.plan( + load, + pv, + batt_capacity_kwh=1.0, + batt_power_kw=0.5, + batt_min_soc=20.0, + batt_max_soc=80.0, + batt_soc_percent=20.0, + ) + soc = 0.2 + for timestamp in index: + target = plan["battery"][timestamp] + self.assertLessEqual(abs(target), 500.0 + 1e-6) + if target >= 0.0: + soc += target * DT_H / 1000.0 + else: + soc += target * DT_H / 1000.0 + self.assertGreaterEqual(soc, 0.2 - 1e-8) + self.assertLessEqual(soc, 0.8 + 1e-8) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py new file mode 100644 index 0000000..42767ef --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py @@ -0,0 +1,66 @@ +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from methods.var_2 import predict +from methods.var_11 import predict as predict_repeat + + +class LoadProfileForecastTest(unittest.TestCase): + def test_profile_keeps_daily_shape_across_48_hours(self): + history_index = pd.date_range("2026-09-21", periods=8 * 288, freq="5min") + phase = 2 * np.pi * ( + history_index.hour.to_numpy() + history_index.minute.to_numpy() / 60.0 + ) / 24.0 + history = pd.DataFrame( + {"Hausverbrauch": 1800.0 + 900.0 * np.cos(phase - np.pi)}, + index=history_index, + ) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=2 * 288, freq="5min") + future_phase = 2 * np.pi * ( + future_index.hour.to_numpy() + future_index.minute.to_numpy() / 60.0 + ) / 24.0 + future = pd.DataFrame( + { + "temp_c": 15.0, + "cloud": 20.0, + "hour_sin": np.sin(future_phase), + "hour_cos": np.cos(future_phase), + "weekday": future_index.weekday, + "is_weekday": (future_index.weekday < 5).astype(int), + }, + index=future_index, + ) + data = { + "config": {"anlagen_id": "test"}, + "df_hist": history, + "df_load_training": history, + "df_recent_raw": history.iloc[-2 * 288 :], + "df_fut": future, + } + + with patch("methods.var_2.load_model", return_value=None): + values = np.array(list(predict(data).values())) + + self.assertGreater(float(values.max() - values.min()), 1200.0) + np.testing.assert_allclose(values[:288], values[288:], rtol=0.0, atol=1e-6) + + def test_repeat_profile_remains_available_on_second_day(self): + history_index = pd.date_range("2026-09-29", periods=288, freq="5min") + daily_values = np.arange(288, dtype=float) + 1000.0 + history = pd.DataFrame({"Hausverbrauch": daily_values}, index=history_index) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=576, freq="5min") + + values = np.array(list(predict_repeat({ + "df_hist": history, + "df_fut": pd.DataFrame(index=future_index), + }).values())) + + np.testing.assert_allclose(values[:288], daily_values) + np.testing.assert_allclose(values[288:], daily_values) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py new file mode 100644 index 0000000..dd6220c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py @@ -0,0 +1,71 @@ +import ast +import contextlib +import datetime +import io +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock +from model_isolation import collect_predictions + + +class ModelIsolationTest(unittest.TestCase): + def setUp(self): + self.index = [datetime.datetime(2026,10,2,6,0) + datetime.timedelta(minutes=5*i) for i in range(3)] + self.data = {'df_fut': SimpleNamespace(index=self.index)} + self.good = {t: 1000.0 for t in self.index} + def model(self, values=None): + return SimpleNamespace(predict=Mock(return_value=self.good if values is None else values)) + def call(self, models, enabled=lambda c,n:True): + return collect_predictions(self.data, {}, models, enabled) + def test_one_failed_model_does_not_remove_other_families(self): + bad=self.model();bad.predict.side_effect=ValueError('private path not logged') + forecasts,errors=self.call({1:self.model(),10:bad,21:self.model()}) + self.assertEqual(forecasts[1],self.good);self.assertEqual(forecasts[21],self.good) + self.assertEqual(forecasts[10],{});self.assertEqual(errors[10]['errorType'],'ValueError') + self.assertNotIn('private',str(errors)) + def test_missing_timestamp_does_not_get_filled(self): + forecasts,errors=self.call({1:self.model(),10:self.model({self.index[0]:20.0})}) + self.assertFalse(forecasts[10]);self.assertIn(10,errors) + def test_nonfinite_negative_and_boolean_rejected(self): + for value in (float('nan'),float('inf'),-1.0,True,'2'): + with self.subTest(value=value): + result,errors=self.call({1:self.model(),10:self.model({t:value for t in self.index})}) + self.assertFalse(result[10]);self.assertIn(10,errors) + def test_explicit_zero_forecast_is_not_imputed(self): + result,errors=self.call({10:self.model({t:0.0 for t in self.index})}) + self.assertEqual(errors,{});self.assertEqual(sum(result[10].values()),0.) + def test_all_requested_models_fail_closed(self): + with self.assertRaisesRegex(ValueError,'All requested'): + self.call({10:self.model({})}) + def test_disabled_models_not_called(self): + model=self.model();result,errors=self.call({10:model},lambda c,n:False) + model.predict.assert_not_called();self.assertEqual(result,{10:{}});self.assertEqual(errors,{}) + def test_input_time_order_preserved(self): + result,_=self.call({1:self.model(dict(reversed(list(self.good.items()))))}) + self.assertEqual(list(result[1]),self.index) + def test_run_forecast_publishes_valid_pairs_after_repeat_failure(self): + root=Path(__file__).resolve().parents[1] + tree=ast.parse((root/'main.py').read_text()) + fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run_forecast') + modules={n:self.model() for n in (1,2,3,10,11,13,21,22,23)} + modules[10].predict.side_effect=ValueError('profile gap') + client=Mock();published=Mock() + data={**self.data,'current_soc_perc':20.,'current_soc_source':'telemetry'} + ns={'datetime':datetime,'LOCAL_TZ':datetime.timezone.utc,'traceback':Mock(), + 'get_configs':lambda:[{'anlagen_id':'test','batt_capacity_kwh':10.}], + 'InfluxDBClient':Mock(return_value=client),'SYNCHRONOUS':object(), + 'INFLUX_URL':'offline','INFLUX_TOKEN':'synthetic','INFLUX_ORG':'offline', + 'INFLUX_BUCKET':'offline','INFLUX_TIMEOUT_MS':1, + 'build_data_object':lambda *a,**k:data,'active':lambda c,n:True, + 'require_recent_telemetry':lambda *a,**k:{},'collect_predictions':collect_predictions, + '_v4_publish_forecasts':published,'battery_soc_points':lambda *a:[], + '_forecast_point':lambda *a:a,'_snapshot_point':lambda *a:a, + 'write_quality_metrics':Mock(),**{'v'+str(n):m for n,m in modules.items()}} + exec(compile(ast.Module(body=[fn],type_ignores=[]),'source-run-forecast','exec'),ns) + with contextlib.redirect_stdout(io.StringIO()):r=ns['run_forecast']() + self.assertEqual(r['completed'],['test']) + families=published.call_args.args[1] + self.assertTrue(families[0][1] and families[0][2]);self.assertFalse(families[1][1]) + self.assertTrue(families[2][1] and families[2][2]);modules[13].predict.assert_not_called() + client.write_api.return_value.write.assert_called_once() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py new file mode 100644 index 0000000..693c530 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py @@ -0,0 +1,213 @@ +"""Offline regressions against the exact reviewed orchestration source. +Only selected function definitions are compiled; main is NOT imported, and there +is no database, network, model-file, prediction-publication or device access. +""" +import ast +import contextlib +import datetime +import io +from pathlib import Path +import unittest +from unittest.mock import Mock, patch + +import numpy as np +import pandas as pd +from telemetry_quality import (TelemetryUnavailable, require_recent_telemetry, + sanitize_measured_frame, repeat_daily_profile, profile_source_value) + +ROOT = Path(__file__).resolve().parents[1] +AT = pd.Timestamp('2026-10-01T20:00:00') + + +def functions(*names, **extra): + tree = ast.parse((ROOT/'main.py').read_text()) + nodes = [n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names] + if {n.name for n in nodes} != set(names): + raise AssertionError('Reviewed function missing') + ns = {'pd':pd,'np':np,'datetime':datetime, + 'sanitize_measured_frame':sanitize_measured_frame, + 'require_recent_telemetry':require_recent_telemetry} + ns.update(extra) + exec(compile(ast.Module(body=nodes,type_ignores=[]),str(ROOT/'main.py'),'exec'),ns) + return ns + + +def raw_frame(start=None, periods=288): + index=pd.date_range(start if start is not None else AT-pd.Timedelta(days=1),periods=periods,freq='5min') + return pd.DataFrame({'PV':200.,'Hausverbrauch':3200.,'SOC':25.,'Netzleistung':3000.},index=index) + + +class MeasuredTelemetryTest(unittest.TestCase): + def test_empty_history_is_not_zero_consumption(self): + ns=functions('_fill_defaults','_add_time_features') + frame=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=3,freq='5min')),True) + self.assertTrue(frame[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + self.assertEqual(frame['temp_c'].tolist(),[15.]*3) + + def test_recorded_zero_is_preserved(self): + frame=raw_frame(periods=3);frame[:]=0. + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(out['Hausverbrauch'].tolist(),[0.]*3) + self.assertEqual(out['PV'].tolist(),[0.]*3) + + def test_internal_and_trailing_measurement_gaps_remain_missing(self): + frame=raw_frame(periods=7) + frame.loc[frame.index[[0,2,3,6]],['PV','Hausverbrauch','SOC','Netzleistung']]=np.nan + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(int(out['Hausverbrauch'].isna().sum()),4) + self.assertEqual(int(out['SOC'].isna().sum()),4) + self.assertEqual(int(out['Netzleistung'].isna().sum()),4) + + def test_bad_values_not_real_measurements(self): + frame=pd.DataFrame({'Hausverbrauch':[np.inf,-1.,True,0.,250.], 'Netzleistung':[-100.,np.nan,0.,1.,2.], 'SOC':[101.,-1.,np.inf,0.,100.]}) + out=sanitize_measured_frame(frame) + self.assertTrue(out['Hausverbrauch'].iloc[:3].isna().all()) + self.assertEqual(out['Hausverbrauch'].iloc[3],0.) + self.assertEqual(out['Netzleistung'].iloc[0],-100.) + self.assertTrue(out['SOC'].iloc[:3].isna().all()) + + def test_recent_recorded_zero_passes(self): + frame=raw_frame();frame[['PV','Hausverbrauch']]=0. + report=require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_missing_raw_cannot_be_hidden_by_filled_feature_grid(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':pd.DataFrame(),'df_hist':raw_frame()}) + + def test_stale_values_fail(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':raw_frame(AT-pd.Timedelta(days=2))}) + + def test_freshness_is_checked_per_field(self): + frame=raw_frame();frame.loc[frame.index[-12:],'Hausverbrauch']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'Hausverbrauch'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_future_measurements_do_not_rescue_freshness(self): + frame=raw_frame(AT,periods=3) + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_aware_timestamps_normalized_to_utc(self): + frame=raw_frame();frame.index=frame.index.tz_localize('UTC').tz_convert('Europe/Zurich') + report=require_recent_telemetry({'now':AT.tz_localize('UTC'),'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_soc_staleness_cannot_be_hidden(self): + frame=raw_frame();frame.loc[frame.index[-24:],'SOC']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'SOC'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame},['SOC']) + + def test_only_exogenous_future_defaults_are_filled(self): + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=6,freq='5min')),False) + self.assertTrue(out[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + + def test_input_unchanged(self): + frame=raw_frame();before=frame.copy(deep=True) + sanitize_measured_frame(frame);require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + pd.testing.assert_frame_equal(frame,before) + + +class HistoryOrchestrationTest(unittest.TestCase): + def data(self,telemetry): + class FrozenDateTime(datetime.datetime): + @classmethod + def utcnow(cls):return AT.to_pydatetime() + import types + dates=types.SimpleNamespace(datetime=FrozenDateTime,timedelta=datetime.timedelta) + weather=pd.DataFrame({'temp_c':15.,'cloud':20.},index=pd.date_range(AT-pd.Timedelta(days=14),AT+pd.Timedelta(days=2),freq='5min')) + fetch=Mock(return_value=(telemetry,weather,pd.DataFrame())) + ns=functions('build_data_object','_fill_defaults','_add_time_features','_longest_consistent_segment',datetime=dates,fetch_influx_frames=fetch,FORECAST_HORIZON_HOURS=48) + return ns['build_data_object']({'anlagen_id':'offline'},False) + + def test_weather_tail_does_not_erase_historical_load(self): + telemetry=raw_frame(AT-pd.Timedelta(days=2),periods=288) + data=self.data(telemetry) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw'].index.max(),telemetry.index.max()) + self.assertTrue(data['df_hist']['Hausverbrauch'].iloc[-24:].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_no_measurements_preserves_all_missing(self): + data=self.data(pd.DataFrame()) + self.assertTrue(data['df_hist']['Hausverbrauch'].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_reconnected_short_tail_does_not_delete_past_profile(self): + old=raw_frame(AT-pd.Timedelta(days=2),periods=288) + new=raw_frame(AT-pd.Timedelta(minutes=10),periods=2) + data=self.data(pd.concat([old,new])) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),290) + self.assertEqual(require_recent_telemetry(data)['Hausverbrauch']['ageMinutes'],5.) + + def test_stale_forecast_never_predicts_or_publishes(self): + import types + models={n:types.SimpleNamespace(predict=Mock()) for n in (1,2,3,10,11,13,21,22,23)} + client=Mock();write=Mock();client.write_api.return_value=write + publish=Mock();trace=Mock() + ns=functions('run_forecast',get_configs=lambda:[{'anlagen_id':'offline','batt_capacity_kwh':10}], + InfluxDBClient=Mock(return_value=client),SYNCHRONOUS=object(), + INFLUX_URL='offline',INFLUX_TOKEN='synthetic',INFLUX_ORG='offline',INFLUX_BUCKET='offline',INFLUX_TIMEOUT_MS=1, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},active=lambda *args:True, + _v4_publish_forecasts=publish,traceback=trace,**{'v'+str(k):v for k,v in models.items()}) + with contextlib.redirect_stdout(io.StringIO()),self.assertRaises(RuntimeError):ns['run_forecast']() + for model in models.values():model.predict.assert_not_called() + publish.assert_not_called();write.write.assert_not_called();client.close.assert_called_once() + + def test_stale_training_does_not_overwrite_model(self): + import types + train=Mock();mod=types.SimpleNamespace(train=train) + ns=functions('run_training',get_configs=lambda:[{'anlagen_id':'offline'}], + MODEL_MODULES={2:mod},QUALITY_TARGETS={2:'Hausverbrauch'},active=lambda *a:True, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},traceback=Mock()) + with contextlib.redirect_stdout(io.StringIO()):ns['run_training']() + train.assert_not_called() + + def test_queries_exclude_forecasts_and_align_interval_starts(self): + source=(ROOT/'main.py').read_text();ns=functions('fetch_influx_frames', + HISTORY_START='1970-01-01T00:00:00Z',FORECAST_HORIZON_HOURS=48,INFLUX_BUCKET='offline', + _query_df=Mock(return_value=pd.DataFrame()),_pivot_frame=lambda df,fields:df, + _tariff_frame=lambda df,cfg:df,_time_literal=lambda t:t.isoformat()) + ns['fetch_influx_frames']({'anlagen_id':'offline'},False) + query=ns['_query_df'].call_args_list[0].args[0] + self.assertIn('timeSrc: "_start"',query) + self.assertIn('!= "forecast_snapshot"',query) + self.assertIn('!= "forecast"',query) + + +class RepeatProfileIntegrityTest(unittest.TestCase): + def test_pv_repeats_on_second_day(self): + history=raw_frame();history['PV']=np.maximum(0.,np.sin(np.arange(288)*2*np.pi/288))*12000. + idx=pd.date_range(AT,periods=576,freq='5min') + actual=np.array(list(repeat_daily_profile(history,idx,'PV').values())) + np.testing.assert_allclose(actual[:288],history['PV']) + np.testing.assert_allclose(actual[288:],history['PV']) + + def test_missing_yesterday_uses_older_finite_day(self): + t=AT + history=pd.DataFrame({'Hausverbrauch':[3500.,np.nan]},index=[t-pd.Timedelta(days=7),t-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,t,'Hausverbrauch'),3500.) + + def test_missing_profile_is_not_zero(self): + history=pd.DataFrame({'Hausverbrauch':[np.nan]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):repeat_daily_profile(history,[AT],'Hausverbrauch') + + def test_recorded_profile_zero_is_valid(self): + history=pd.DataFrame({'PV':[0.]},index=[AT-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,AT,'PV'),0.) + + def test_negative_profile_cannot_be_silently_clamped(self): + history=pd.DataFrame({'Hausverbrauch':[-100.]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + def test_future_value_is_never_used_as_history(self): + history=pd.DataFrame({'Hausverbrauch':[3300.]},index=[AT+pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..c667975 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json @@ -0,0 +1,9 @@ +{ + "libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" +} diff --git a/services/netplan-v4/acceptance/php-src/check.php b/services/netplan-v4/acceptance/php-src/check.php new file mode 100644 index 0000000..feb59a8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/check.php @@ -0,0 +1,45 @@ + $expected) { + $path = __DIR__ . '/' . $relative; + if (!is_file($path) || hash_file('sha256', $path) !== $expected) { + throw new RuntimeException('Source manifest mismatch: ' . $relative); + } + $command = escapeshellarg(PHP_BINARY) . ' -l ' . escapeshellarg($path); + $lines = []; + exec($command, $lines, $code); + if ($code !== 0) { + throw new RuntimeException('PHP syntax failure: ' . $relative); + } + } + require __DIR__ . '/libs/NetzfahrplanV4Bezugszaehler.php'; + require __DIR__ . '/libs/NetzfahrplanV4Betriebsdaten.php'; + require __DIR__ . '/libs/ManagerNetzfahrplanV4Trait.php'; + require __DIR__ . '/tests/fixtures/NativeV4Scenarios.php'; + require __DIR__ . '/tests/fixtures/V4BezugszaehlerScenarios.php'; + $native = \Belevo\EnelixEMS\Tests\NativeV4Scenarios::run(); + $meters = \Belevo\EnelixEMS\Tests\V4BezugszaehlerScenarios::run(); + if (count($native) !== 24 || count($meters) !== 20) { + throw new RuntimeException('Expected 24 native and 20 meter scenarios.'); + } + foreach (array_merge($native, $meters) as $name) { + echo 'PASS ', $name, "\n"; + } + echo 'PHP ', PHP_VERSION, ': 44 offline conversion scenarios passed.', "\n"; + echo "NOT a full EMS suite and NOT an IP-Symcon runtime/physical test.\n"; +} catch (Throwable $error) { + fwrite(STDERR, 'FAIL: ' . $error->getMessage() . "\n"); + exit(1); +} diff --git a/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php new file mode 100644 index 0000000..c6e7412 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php @@ -0,0 +1,164 @@ +RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); + $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); + $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]'); + $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4Senden', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); + } + + public function GetNetzfahrplanV4Diagnose(): string + { + try { + $manager = []; + foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', + 'Lastspitzenmodus'] as $key) { + $manager[$key] = $this->ReadPropertyInteger($key); + } + foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { + $manager[$key] = $this->ReadPropertyFloat($key); + } + $manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified'; + $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); + $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); + $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); + $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $manager['V4BezugszaehlerQuellen'] = json_decode( + $this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR + ); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); + $cache = $this->leseJsonAttribut('VerbraucherCache'); + $controllers = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + $c = ['InstanzID' => $id]; + foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + $c[$key] = IPS_GetProperty($id, $key); + } + $entry = $cache[(string) $id] ?? []; + $c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0; + $c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false; + foreach (($entry['Daten']['Zustand'] ?? []) as $state) { + if (($state['Kennung'] ?? '') === 'HystereseAktiv') { + $c['HystereseAktiv'] = $state['Wert']; + } + } + $controllers[] = $c; + } + $evidence = []; + $evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID'); + if ($evidenceID > 0) { + if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) { + throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.'); + } + $evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($evidence)) { + throw new RuntimeException('Ungueltiger Messnachweis.'); + } + } + $read = static function (int $id): array { + if (!IPS_VariableExists($id)) { + throw new RuntimeException('Messvariable ' . $id . ' fehlt.'); + } + $before = IPS_GetVariable($id); + $value = GetValue($id); + $after = IPS_GetVariable($id); + if ($before['VariableUpdated'] !== $after['VariableUpdated']) { + throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); + } + $object = IPS_GetObject($id); + return ['value' => $value, 'updated' => (int) $after['VariableUpdated'], + 'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']]; + }; + return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, + $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } catch (Throwable $error) { + return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR); + } + } + + public function GetNetzfahrplanV4Sendestatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4Sendestatus'); + } + + private function sendeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + return; + } + try { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $snapshot = $this->GetNetzfahrplanV4Diagnose(); + $data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR); + if (!isset($data['operation'])) { + throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.'); + } + // Decoding as objects preserves empty JSON dictionaries in the API contract. + $payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR); + if (!isset($payload->operation)) { + throw new RuntimeException('Keine konsistenten Betriebsdaten.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation'; + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.'); + } + try { + curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, + CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR), + CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]); + $answer = curl_exec($handle); + $status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($answer === false || $status !== 200) { + throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.'); + } + if (strlen($answer) > 65536) { + throw new RuntimeException('V4-Antwort ist zu gross.'); + } + $ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR); + if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) { + throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.'); + } + } finally { + curl_close($handle); + } + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt, + 'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + // Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands. + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php new file mode 100644 index 0000000..35495da --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php @@ -0,0 +1,219 @@ + $max) { + throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.'); + } + return (float) $value; + } + + private static function timestamp($value): int + { + if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) { + throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.'); + } + return (new DateTimeImmutable($value))->getTimestamp(); + } + + /** $read returns ['value' => int|float, 'updated' => int], without changing a device. */ + private static function measurement(callable $read, int $id, int $now, int $age, string $label): array + { + if ($id <= 0) { + throw new InvalidArgumentException($label . ': Messquelle fehlt.'); + } + $m = $read($id); + if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now + || $now - $m['updated'] > $age) { + throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.'); + } + return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']]; + } + + /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ + private static function evidence(array $input, int $now, string $meterID, array &$warnings): array + { + $peaks = []; + $past = null; + if ($input === []) { + $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; + return [(object) [], null]; + } + if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) { + throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); + } + $measured = self::timestamp($input['measuredAt'] ?? null); + if ($measured > $now || $now - $measured > 120) { + throw new InvalidArgumentException('Messnachweis ist nicht aktuell.'); + } + $month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m'); + foreach (($input['measuredPeaks'] ?? []) as $key => $p) { + if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month + || !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) { + throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.'); + } + $peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']]; + } + if (!array_key_exists($month, $peaks)) { + $warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.'; + } + $q = intdiv($now, 900) * 900; + if (isset($input['quarterPast'])) { + $p = $input['quarterPast']; + // Stale quarter evidence remains absent, not extrapolated to the decision time. + if ($measured === $now && self::timestamp($p['start'] ?? null) === $q + && ($p['measuredSeconds'] ?? null) === $now - $q) { + $past = ['start' => gmdate('c', $q), + 'measuredSeconds' => $now - $q, + 'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)]; + } else { + $warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.'; + } + } + if ($now !== $q && $past === null) { + $warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.'; + } + return [(object) $peaks, $past]; + } + + /** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */ + public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array + { + $warnings = []; + $meter = NetzfahrplanV4Bezugszaehler::lesen( + $manager['V4BezugszaehlerQuellen'] ?? [], $read, $now + ); + $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, + (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); + $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); + $mode = $manager['Lastspitzenmodus'] ?? 0; + $import = null; + $months = []; + if ($mode === 1) { + $import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0); + } elseif ($mode === 2) { + foreach (($manager['Monatsgrenzen'] ?? []) as $row) { + $m = $row['MonatIndex'] ?? null; + if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) { + throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.'); + } + $months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0); + } + if (count($months) !== 12) { + throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.'); + } + } elseif ($mode !== 0) { + throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.'); + } + $export = !empty($manager['EinspeisebegrenzungAktiv']) + ? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null; + $result = []; + $seen = []; + $used = []; + foreach ($assets as $asset) { + $id = $asset['ID'] ?? ''; + if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) { + throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.'); + } + $seen[$id] = true; + $socID = (int) ($asset['SOCVariableID'] ?? 0); + $matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool { + return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID; + })); + if (count($matches) !== 1) { + throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.'); + } + $c = $matches[0]; + $controllerID = $c['InstanzID']; + if (isset($used[$controllerID])) { + throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.'); + } + $used[$controllerID] = true; + if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) { + throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.'); + } + $opts = $manager['BatterieOptionen'][$id] ?? []; + $nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001); + $usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal); + if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) { + throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.'); + } + $capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal); + $age = (int) ($c['MesswertMaxAlter'] ?? 30); + $soc = self::measurement($read, $socID, $now, $age, $id . ' SOC'); + $charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden'); + $discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen'); + $min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100), + self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); + $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); + $socValue = self::number($soc['value'], 'SOC', 0, 100); + $physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min); + if ($socValue < $physicalMin || $socValue > $max) { + throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); + } + if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { + throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.'); + } + $maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0)); + $maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0)); + if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) { + $maxCharge = $maxDischarge = 0.0; + $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; + } + $rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100)); + $blocked = !empty($c['HystereseAktiv']) || $socValue <= $min; + if ($blocked) { + $warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.'; + } + if ($socValue < $min) { + $warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.'; + } + $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, + 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, + 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true, + 'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true, + 'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm]; + } + if ($result === []) { + throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); + } + [$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings); + $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, + 'meteringBoundary' => 'common_pcc', 'batteries' => $result, + 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], + 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + // A fresh acquisition sample is an operational estimate. It does NOT + // claim a calibrated billing-period boundary or full historical coverage. + $policy = [ + 'adapterVersion' => 'v4-native-estimated-meter-1', + 'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified', + 'mode' => $mode, 'importW' => $import, 'exportW' => $export, + 'months' => $months, + 'batteryPolicy' => array_map(static function (array $b): array { + return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent'])); + }, $result), + ]; + $payload['meterObservation'] = [ + 'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now), + 'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'], + 'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)), + ]; + $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php new file mode 100644 index 0000000..dda9b72 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php @@ -0,0 +1,108 @@ + 8 + || array_keys($sources) !== range(0, count($sources) - 1)) { + throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.'); + } + $result = []; + $parent = null; + foreach ($sources as $source) { + if (!is_array($source) || count($source) !== 5 + || ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') { + throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.'); + } + $id = $source['VariableID'] ?? null; + $pid = $source['ElternID'] ?? null; + $ident = $source['Ident'] ?? null; + $factor = $source['FaktorZuKWh'] ?? null; + if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0 + || !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) + || $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) { + throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.'); + } + if ($parent !== null && $parent !== $pid) { + throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.'); + } + $parent = $pid; + foreach ($result as $existing) { + if ($existing['Ident'] === $ident) { + throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.'); + } + } + $result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident, + 'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug']; + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + public static function identitaet(array $sources): string + { + // Source/factor changes invalidate old measurement evidence; names do not matter. + return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources), + JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + } + + private static function probe(array $source, callable $read, int $now, int $maxAge): array + { + $m = $read($source['VariableID']); + if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID'] + || ($m['ident'] ?? null) !== $source['Ident']) { + throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.'); + } + $value = $m['value'] ?? null; + $updated = $m['updated'] ?? null; + if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0 + || !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) { + throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.'); + } + $kwh = (float) $value * $source['FaktorZuKWh']; + if (!is_finite($kwh) || $kwh > 1.0e12) { + throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.'); + } + return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value, + 'totalKwh' => $kwh, 'updatedAt' => $updated]; + } + + /** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */ + public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array + { + if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) { + throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.'); + } + $sources = self::quellen($sources); + $samples = []; + foreach ($sources as $source) { + $samples[] = self::probe($source, $read, $now, $maxAge); + } + // Reread all channels to reject concurrent updates, including changes in the same second. + foreach ($sources as $i => $source) { + if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) { + throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.'); + } + } + $times = array_column($samples, 'updatedAt'); + if (max($times) - min($times) > $maxSkew) { + throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.'); + } + return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh', + 'totalKwh' => array_sum(array_column($samples, 'totalKwh')), + 'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)), + 'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples, + 'billingEvidence' => false, 'historyComplete' => false]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php new file mode 100644 index 0000000..adb80c8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -0,0 +1,29 @@ +regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) { + self::assertStringNotContainsString($forbidden, $source); + } + self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source); + self::assertStringContainsString('/prognosis/planner-v4/operation', $source); + self::assertStringNotContainsString('/prognosis/schedule', $source); + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php new file mode 100644 index 0000000..41a2992 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php @@ -0,0 +1,29 @@ + 1, 'NetzbezugEnergieVariableID' => 9, + 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, + 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $m['V4BezugszaehlerQuellen'] = [ + ['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, + 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; + $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, + 'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5, + 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, + 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; + $read = static function (int $id) use ($now): array { + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id], + 'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other']; + }; + $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; + $fail = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid-input rejection'); + }; + $cases = []; + $build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array { + return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now); + }; + $cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void { + $x = $build()['operation']; $b = $x['batteries'][0]; + $check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0 + && $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0); + }; + $cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void { + $x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null); + }; + $cases['empty_maps_are_objects'] = static function () use ($build, $check): void { + $x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW)); + }; + $cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void { + $m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0); + }; + $cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void { + $m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === null && $x['limits']['exportW'] === null); + }; + $cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void { + $m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = []; + for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; } + $x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0); + array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m)); + }; + $cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void { + $a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a)); + $m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]]; + $check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44); + }; + $cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void { + $a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a)); + }; + $cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void { + $c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c)); + }; + $cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void { + $c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c)); + }; + $cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void { + $r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { + $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); + }; + $cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void { + $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; + $b = $build(null, null, null, $r)['operation']['batteries'][0]; + $check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true); + }; + $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { + $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; + $check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0); + }; + $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { + $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); + $check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1); + }; + $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { + $check($build()['operation']['batteries'][0]['gridCharging'] === false); + $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); + }; + $ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now), + 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], + 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; + $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { + $x = $build(null, null, null, null, $ev); + $check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5); + }; + $cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void { + $ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev); + $check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0); + }; + $cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void { + $ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { + $check($build()['operation']['eventId'] === $build()['operation']['eventId']); + }; + $cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void { + unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m)); + }; + $cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void { + $v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203 + && !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php new file mode 100644 index 0000000..7e08052 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php @@ -0,0 +1,113 @@ + 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $samples = [ + 59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'], + 26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'], + ]; + $check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } }; + $reject = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid source/sample rejection'); + }; + $snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array { + $mm = $mm ?? $samples; + return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now); + }; + $cases = []; + $cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void { + $v = $snapshot(); + $check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']); + $check(!isset($v['measuredPeaks']) && !isset($v['quarterPast'])); + }; + $cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void { + $samples[26620]['value'] = 27.25; + $check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9); + }; + $cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void { + $samples[59607]['value'] = 0; + $check($snapshot(null, $samples)['totalKwh'] === 0.0); + }; + $cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void { + $check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources))); + }; + $cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001; + $check($old !== Meter::identitaet($sources)); + }; + $cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345; + $check($old !== Meter::identitaet($sources)); + }; + $cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); }; + $cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources)); + }; + $cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources)); + }; + $cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources)); + }; + $cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void { + unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources)); + }; + $cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void { + unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void { + foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) { + $samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples)); + } + }; + $cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void { + $calls = 0; + $read = static function (int $id) use ($samples, &$calls): array { + $v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v; + }; + $reject(static fn() => Meter::lesen($sources, $read, $now)); + }; + $cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void { + $samples[59607]['updated'] -= 1; + $v = $snapshot(null, $samples); + $check($v['sourceObservationFrom'] === gmdate('c', $now - 1)); + $check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now)); + $check(!$v['billingEvidence']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/read_native_forecasts.py b/services/netplan-v4/acceptance/read_native_forecasts.py new file mode 100644 index 0000000..abae9e8 --- /dev/null +++ b/services/netplan-v4/acceptance/read_native_forecasts.py @@ -0,0 +1,100 @@ +"""Read-only probe executed INSIDE the existing forecast-engine container. + +No get_configs() (it may migrate SQL), no training/prediction, no forecast writing, +no manual /run_now request, and no user credentials in output. Reads one plant's +stored raw 5-minute forecast values and the input frames used by the engine. +""" +import contextlib +from datetime import datetime, timezone +import hashlib +import io +import json +import math +import os +from pathlib import Path +import sqlite3 +from urllib.parse import quote +from uuid import UUID + +class Discard(io.TextIOBase): + def write(self,text):return len(text) + +def clean_time(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def run(): + aid=str(UUID(os.environ['ENELIX_ACCEPTANCE_PLANT'])) + os.environ['FORECAST_INFLUX_TIMEOUT_MS']='20000' + # These assignments affect only this diagnostic process, not the service. + with contextlib.redirect_stdout(Discard()),contextlib.redirect_stderr(Discard()): + import main + import numpy as np + import pandas as pd + path=Path(main.SQLITE_DB_PATH) + if not path.is_file():raise RuntimeError('Existing configuration database missing') + con=sqlite3.connect('file:'+quote(str(path))+'?mode=ro',uri=True) + con.row_factory=sqlite3.Row + try:row=con.execute('SELECT * FROM anlagen_meta WHERE anlagen_id=?',(aid,)).fetchone() + finally:con.close() + if row is None:raise RuntimeError('Installation not found') + cfg=dict(row) + cfg['daecher']=json.loads(cfg.get('daecher') or '[]') + for key,default in [('ac_leistung',10.),('batt_capacity_kwh',0.),('batt_power_kw',0.),('tarif_bezug_fest',.3),('tarif_einspeisung_fest',.1),('tarif_peak_fest',5.)]: + value=cfg.get(key);cfg[key]=float(default if value is None or value=='' else value) + data=main.build_data_object(cfg,training=False) + frames={} + for key in ('df_hist','df_recent_raw','df_load_training','df_fut'): + frame=data.get(key) + if frame is None:frames[key]={'available':False};continue + item={'rows':len(frame),'from':clean_time(frame.index.min()) if len(frame) else None,'until':clean_time(frame.index.max()) if len(frame) else None,'columns':{}} + for col in ('Hausverbrauch','PV','Netzleistung','SOC'): + if col not in frame.columns:continue + values=pd.to_numeric(frame[col],errors='coerce');valid=values[np.isfinite(values)] + item['columns'][col]={'finite':len(valid),'zeros':int((valid==0).sum()),'median':float(valid.median()) if len(valid) else None,'maximum':float(valid.max()) if len(valid) else None,'lastFiniteAt':clean_time(valid.index[-1]) if len(valid) else None,'recentValues':[{'time':clean_time(t),'value':float(v)} for t,v in valid.tail(12).items()]} + frames[key]=item + start=datetime.now(timezone.utc).replace(second=0,microsecond=0) + from datetime import timedelta + end=start+timedelta(hours=48) + fields=('prog_var_1','prog_var_2','prog_var_10','prog_var_11','prog_var_21','prog_var_22') + field_filter=' or '.join('r["_field"] == '+json.dumps(f) for f in fields) + query='''from(bucket: %s) + |> range(start: %s, stop: %s) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == %s) + |> filter(fn: (r) => r["data_type"] == "forecast") + |> filter(fn: (r) => %s) + |> keep(columns: ["_time", "_field", "_value"]) + ''' % (json.dumps(main.INFLUX_BUCKET),start.isoformat(),end.isoformat(),json.dumps(aid),field_filter) + client=main.InfluxDBClient(url=main.INFLUX_URL,token=main.INFLUX_TOKEN,org=main.INFLUX_ORG,timeout=20000) + series={field:[] for field in fields} + try: + tables=client.query_api().query(org=main.INFLUX_ORG,query=query) + for table in tables: + for record in table.records: + value=record.get_value();field=record.get_field() + if field in series and isinstance(value,(int,float)) and math.isfinite(value): + series[field].append({'time':clean_time(record.get_time()),'value':float(value)}) + finally:client.close() + for field in series:series[field].sort(key=lambda p:p['time']) + versions={} + for relative in ('main.py','methods/var_1.py','methods/var_2.py','methods/var_10.py','methods/var_11.py','methods/var_21.py','methods/var_22.py'): + source=Path('/app')/relative + if source.is_file():versions[relative]=hashlib.sha256(source.read_bytes()).hexdigest() + summary={} + for field,points in series.items(): + values=[p['value'] for p in points] + summary[field]={'points':len(values),'allZero':bool(values) and max(abs(v) for v in values)==0,'maximumW':max(values) if values else None,'from':points[0]['time'] if points else None,'until':points[-1]['time'] if points else None} + native=True + for points in series.values(): + stamps=[datetime.fromisoformat(p['time']).timestamp() for p in points] + if not stamps or any(t%300 for t in stamps) or any(b-a!=300 for a,b in zip(stamps,stamps[1:])):native=False + return {'status':'read_only_acquired','installationId':aid,'observedAt':datetime.now(timezone.utc).isoformat(),'queryResolution':'raw_no_chart_resampling','nativeFiveMinuteForecast':native,'generationTimeVerified':False,'inputFrames':frames,'forecastSummary':summary,'forecastSeries':series,'runningSourceHashes':versions,'modelWrite':False,'liveControlChanged':False} + +try: + result=run() +except Exception as error: + # Never echo exception details containing SQL payloads, credentials or URLs. + result={'status':'read_only_probe_failed','errorType':type(error).__name__,'liveControlChanged':False} +print(json.dumps(result,allow_nan=False)) diff --git a/services/netplan-v4/approved_previous_assets.json b/services/netplan-v4/approved_previous_assets.json new file mode 100644 index 0000000..3c59814 --- /dev/null +++ b/services/netplan-v4/approved_previous_assets.json @@ -0,0 +1,5 @@ +{ + "license/public/netplan-v4.js": [ + "e859e91a52dd1d6c96f411d16caed17375f6925bf867d09992ab32ce93bff5d7" + ] +} diff --git a/services/netplan-v4/commissioning/APPLICATION_STATUS.md b/services/netplan-v4/commissioning/APPLICATION_STATUS.md new file mode 100644 index 0000000..b716bd1 --- /dev/null +++ b/services/netplan-v4/commissioning/APPLICATION_STATUS.md @@ -0,0 +1,80 @@ +# Integrated application data/model path - 2026-10-02 + +User request: finish application implementation, stop asking for repeated hardware +confirmations and adding standalone diagnostic samplers. No actuator permission is +implied by that development request. Confirmed EV allocation remains 161.44 kWh / +39 kW each direction, with SDL reserve already excluded. + +## Implemented in the existing application + +- Native ManagerNetzfahrplanV4DatenTrait: 30-second acquisition of configured numeric + sources; durable private app-owned outbox; original timestamps, no new Modbus + polls; acknowledged per-plant batches with retries/backoff. Cursor advances only + after exact dataset/acceptedThrough acknowledgement. Batches bounded below the + existing portal's 1 MiB body limit. No credentials in data payload or reports. +- Existing device proxy now forwards /planner-v4/measurements through the existing + authentication, plant binding, licence check and separate V4 rate limiter. + Device route cannot modify the dataset mapping or issue a trial grant. +- Existing V4 database stores immutable versioned source mappings and observations. + Same-time conflicts reject the whole batch; repeated identical data is idempotent. + No writes to portal/users/legacy measurement databases. +- Existing V4 worker aggregates physical load using original source times. Virtual + EV/SDL filter outputs are not load inputs. Solar terminal variant counts signed + terminal power once. Small unobserved parts remain quantified, never zero-filled. + This configured formula remains an estimate, not independent AC/DC certification. +- Application profile training actually runs daily/weekly. Models are immutable, + bootstrap/holdout status explicit. Incumbent replacement requires a causal + holdout when available. Existing data and model versions are not relabelled. +- Three corrected load variants (daily robust profile, last-day profile, + weekday/weekend profile) are paired with the existing PV families 3/13/23. + They are tagged physical-profile-v1; they are NOT claimed to be the old trained + load algorithms under a new data name. Automatic future family registry work + and economic replay are still distinct from this load-profile implementation. +- Selecting corrected_profile in the existing GUI/setting now actually changes + the load entering the existing optimizer. Missing trained data cannot silently + fall back to the contaminated legacy household series. +- A current SDL request is an explicitly labelled PERSISTENCE SCENARIO for the + shadow economic calculation, not a published future schedule. Missing/stale SDL + is not zero. This is not a robust/full SDL production-dispatch policy. +- GUI now exposes dataset/source selection, records, usable equivalent hours, + model version, validation and real training cadence. Existing normal portal + app.js/index files are not changed. + +## Deployment package + +Server: commissioning/deploy_application.py --plant --apply +Rebuild/test V4 with Python 3.11, portal routing tests, consistent SQLite backup, +replace ONLY V4 and portal, configure the new per-plant dataset. No automatic +forecast-source switch and no control grant. Atomic file-bind replacement requires +portal recreation, not a simple restart. Additive DB rollback retains new data. +Default invocation without --apply validates source manifests only. + +Test host: /srv/agent/netplan-v4-application-build/install.php inside Symcon. +Installs a DATA-ONLY patch of the currently installed passive manager (not the +unreleased controller-trial candidate), reloads EMS, configures acquisition and +imports up to 48h of the existing 23-source observer as a delivery backlog. +Original observer and old raw journals remain intact. Existing separate observers +are NOT automatically stopped by this initial installer; consolidation should +follow confirmed native batch acceptance. No new root diagnostic category. +An asynchronous module registration may require one repeat, handled explicitly. + +Dataset lihrenmoos-physical-v1: minimum coverage 95%, longest unsupported portion +10s, at least 24 equivalent usable hours, 28-day profile history. The 95% rule is +an explicit modelling policy, not a statement that missing energy was measured. +Insufficient data remains collecting; no promised time to good forecasts. +All original acquisition thresholds remain unchanged. + +## Tests and honest limits + +Host tests use existing isolated QA dependencies; native tests mock IPS/HTTP and +use temporary files. Real container and kernel tests occur on the user-run deploy +commands, not during preparation. No services/modules/settings/devices were modified +by this development session. + +Still NOT a complete production commissioning: the corrected profile-to-local +feedback/dispatch integration, full automatic cost replay, independent long-term +outbox/server retention and staged live/failure acceptance remain open. The trial +code from previous commits remains gated and is not included in the data-only +native installation. No accountingEvidenceId/device watchdog proof is fabricated. +This release is an integrated application build, NOT a claim that the full original +multi-plant production scope is done. No main/beta release is authorized by tests alone. diff --git a/services/netplan-v4/commissioning/activate_corrected_forecast.py b/services/netplan-v4/commissioning/activate_corrected_forecast.py new file mode 100644 index 0000000..f3b251a --- /dev/null +++ b/services/netplan-v4/commissioning/activate_corrected_forecast.py @@ -0,0 +1,256 @@ +"""Commission the corrected Lihrenmoos forecast in shadow mode. + +The command is a dry run unless --apply is supplied. It never enables live control, +changes Symcon actuator flags, or reads/prints service credentials. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sqlite3 +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +SERVICE_ROOT = Path(__file__).resolve().parents[1] +if str(SERVICE_ROOT) not in sys.path: + sys.path.insert(0, str(SERVICE_ROOT)) +RUNTIME_ROOT = Path("/home/agent/services/netplan-v4-shadow") +PLANT = "e3a08f9e-af12-4695-99bd-8b51c0520021" +DATASET = "lihrenmoos-physical-published-v2" +SOURCE_FILES = ( + "netplan_v4/store.py", + "netplan_v4/measurement_pipeline.py", + "netplan_v4/service.py", +) + + +def digest(path: Path) -> str: + if path.is_symlink() or not path.is_file(): + raise ValueError("Source file missing or is a symlink") + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify_runtime_source(runtime_root: Path) -> None: + for name in SOURCE_FILES: + if digest(SERVICE_ROOT / name) != digest(runtime_root / name): + raise ValueError("Runtime source differs from reviewed repository: " + name) + + +def read_state(db: Path, plant: str, dataset: str, now: int) -> dict: + stamp = db.stat() + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + con.row_factory = sqlite3.Row + try: + from netplan_v4.store import DEFAULT_SETTINGS + + row = con.execute( + "SELECT revision,value FROM planner_settings WHERE plant=?", (plant,) + ).fetchone() + settings = {**DEFAULT_SETTINGS, **(json.loads(row["value"]) if row else {})} + settings["revision"] = row["revision"] if row else 0 + ds = con.execute( + "SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?", + (plant, dataset), + ).fetchone() + if not ds: + raise ValueError("Corrected dataset is not registered") + config = json.loads(ds["config"]) + state = con.execute( + "SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?", + (plant, dataset), + ).fetchone() + model = con.execute( + """SELECT m.model_id,m.trained_at,m.trained_through,m.value + FROM planner_model_current c JOIN planner_load_models m + ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id + WHERE c.plant=? AND c.dataset=?""", + (plant, dataset), + ).fetchone() + observation_dataset = config.get("sourceDatasetId", dataset) + observation = con.execute( + """SELECT COUNT(*) records,MAX(captured_at) last_capture, + MAX(received_at) last_received + FROM planner_observations WHERE plant=? AND dataset=?""", + (plant, observation_dataset), + ).fetchone() + plan = con.execute( + """SELECT p.value FROM planner_current c JOIN planner_plans p + ON p.plan_id=c.plan_id WHERE c.plant=? AND c.mode='shadow'""", + (plant,), + ).fetchone() + detail = json.loads(state["detail"]) if state else {} + model_value = json.loads(model["value"]) if model else None + return { + "settings": settings, + "dataset": dataset, + "observationDataset": observation_dataset, + "pipelineStatus": state["status"] if state else None, + "pipelineDetail": detail, + "records": observation["records"], + "lastCapture": observation["last_capture"], + "lastReceived": observation["last_received"], + "activeModel": { + "modelId": model["model_id"], + "trainedAt": model["trained_at"], + "trainedThrough": model["trained_through"], + "validation": model_value.get("validation"), + } if model else None, + "currentPlan": json.loads(plan["value"]) if plan else None, + "checkedAt": now, + "databaseMtimeNs": stamp.st_mtime_ns, + } + finally: + con.close() + + +def require_ready(state: dict, now: int) -> None: + if state["settings"]["runMode"] != "shadow": + raise ValueError("Only shadow mode may be commissioned") + if state["pipelineStatus"] != "model_ready" or not state["activeModel"]: + raise ValueError("Corrected profile model is not ready") + if state["records"] < 1 or not isinstance(state["lastCapture"], int): + raise ValueError("Corrected observations are missing") + if not 0 <= now - state["lastCapture"] <= 600: + raise ValueError("Corrected observations are not current") + if not 0 <= now - state["lastReceived"] <= 600: + raise ValueError("Corrected delivery is not current") + detail = state["pipelineDetail"] + if detail.get("modelId") != state["activeModel"]["modelId"]: + raise ValueError("Pipeline and active model disagree") + if detail.get("usableEquivalentHours", 0) < detail.get("requiredEquivalentHours", 24): + raise ValueError("Minimum usable equivalent hours not reached") + + +def backup_database(source: Path, destination: Path) -> None: + if destination.exists() or source.is_symlink(): + raise ValueError("Backup destination must be new and source must be regular") + with sqlite3.connect(f"file:{source}?mode=ro", uri=True) as src, sqlite3.connect(destination) as dst: + src.backup(dst) + if dst.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise ValueError("Database backup integrity check failed") + destination.chmod(0o600) + + +def plan_uses_dataset(db: Path, plant: str, dataset: str, revision: int) -> dict | None: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + row = con.execute( + """SELECT p.value FROM planner_current c JOIN planner_plans p + ON p.plan_id=c.plan_id WHERE c.plant=? AND c.mode='shadow'""", + (plant,), + ).fetchone() + if not row: + return None + plan = json.loads(row[0]) + pipeline = plan.get("inputQuality", {}).get("dataPipeline", {}) + if plan.get("configRevision") == revision and pipeline.get("datasetId") == dataset: + return plan + return None + finally: + con.close() + + +def activate(db: Path, plant: str, dataset: str, now: datetime) -> dict: + from netplan_v4.store import PlannerStore + + store = PlannerStore(str(db)) + try: + before = store.settings(plant) + if before["runMode"] != "shadow": + raise ValueError("Runtime left shadow mode") + if before["forecastSource"] == "corrected_profile" and before["measurementDataset"] == dataset: + store.request(plant, "manual", now) + return {"changed": False, "revision": before["revision"]} + result = store.save_settings( + plant, + {"forecastSource": "corrected_profile", "measurementDataset": dataset}, + before["revision"], + now, + ) + return {"changed": True, "revision": result["revision"]} + finally: + store.close() + + +def commission(runtime_root: Path, plant: str, dataset: str, apply: bool, wait_seconds: int = 90) -> dict: + plant = str(UUID(plant)) + if runtime_root.resolve() != RUNTIME_ROOT: + raise ValueError("Unexpected runtime root") + verify_runtime_source(runtime_root) + db = runtime_root / "data/netplan-v4.sqlite" + now = datetime.now(timezone.utc).replace(microsecond=0) + before = read_state(db, plant, dataset, int(now.timestamp())) + require_ready(before, int(now.timestamp())) + report = { + "scope": "corrected_forecast_shadow_commissioning", + "startedAt": now.isoformat(), + "installationId": plant, + "datasetId": dataset, + "liveEnabled": False, + "actuatorPermissionGranted": False, + "before": { + "settings": before["settings"], + "pipelineStatus": before["pipelineStatus"], + "records": before["records"], + "lastCapture": before["lastCapture"], + "activeModel": before["activeModel"], + "usableEquivalentHours": before["pipelineDetail"].get("usableEquivalentHours"), + }, + } + if not apply: + report["status"] = "ready_no_changes" + return report + + folder = runtime_root / "corrected-forecast-releases" / now.strftime("%Y%m%dT%H%M%SZ") + folder.mkdir(parents=True, mode=0o700) + backup_database(db, folder / "before.sqlite") + activated = activate(db, plant, dataset, now) + report["activation"] = activated + deadline = time.monotonic() + wait_seconds + plan = None + while time.monotonic() < deadline: + plan = plan_uses_dataset(db, plant, dataset, activated["revision"]) + if plan: + break + time.sleep(2) + after = read_state(db, plant, dataset, int(datetime.now(timezone.utc).timestamp())) + settings = after["settings"] + if settings["runMode"] != "shadow" or settings["forecastSource"] != "corrected_profile" or settings["measurementDataset"] != dataset: + raise ValueError("Corrected shadow settings were not retained") + report["after"] = { + "settings": settings, + "pipelineStatus": after["pipelineStatus"], + "activeModel": after["activeModel"], + "planId": plan.get("planId") if plan else None, + "planStatus": plan.get("status") if plan else None, + "sourceFamily": plan.get("sourceFamily") if plan else None, + "dataPipeline": plan.get("inputQuality", {}).get("dataPipeline") if plan else None, + } + report["status"] = "corrected_forecast_running_shadow" if plan else "corrected_forecast_activated_plan_pending" + report["finishedAt"] = datetime.now(timezone.utc).isoformat() + output = folder / "REPORT.json" + output.write_text(json.dumps(report, indent=2) + "\n") + output.chmod(0o640) + report["reportPath"] = str(output) + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plant", default=PLANT) + parser.add_argument("--dataset", default=DATASET) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--wait-seconds", type=int, default=90) + args = parser.parse_args() + if not 1 <= args.wait_seconds <= 300: + raise SystemExit("wait-seconds must be 1..300") + try: + result = commission(RUNTIME_ROOT, args.plant, args.dataset, args.apply, args.wait_seconds) + except Exception as exc: + raise SystemExit("Stopped: " + type(exc).__name__ + ". No actuator permission was changed.") from exc + print(json.dumps(result, indent=2)) diff --git a/services/netplan-v4/commissioning/application-source/RELEASE.json b/services/netplan-v4/commissioning/application-source/RELEASE.json new file mode 100644 index 0000000..0395956 --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/RELEASE.json @@ -0,0 +1,100 @@ +{ + "scope": "integrated_measurement_application", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "portalBefore": { + "netplan-v4-bridge.mjs": "f58c652b04cd2ac20f2bec11e1161c44b72c26267b4dfb42096a1fad36a3c9c2", + "public/netplan-v4.js": "6cd1d887bb08c73a815c92eb684a0992e8cb092f748770cc94debcfd9705a628" + }, + "createdAt": "2026-10-02T21:00:35.231591+00:00", + "controlEnabled": false +} diff --git a/services/netplan-v4/commissioning/application-source/server-dataset.json b/services/netplan-v4/commissioning/application-source/server-dataset.json new file mode 100644 index 0000000..7fd1190 --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/server-dataset.json @@ -0,0 +1 @@ +{"datasetId":"lihrenmoos-physical-v1","mappingSha256":"517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b","inventorySha256":"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc","sources":[{"key":"grid","variableId":40348,"factorToW":1000,"maxAgeSeconds":60,"role":"grid"},{"key":"pv_goodwe1","variableId":48459,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_goodwe2","variableId":53802,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_solaredge","variableId":20335,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"physical_goodwe1","variableId":47725,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_goodwe2","variableId":35724,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_solaredge","variableId":21447,"factorToW":1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"ev_account","variableId":52020,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"sdl_account","variableId":25085,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"grid_display","variableId":49301,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe1","variableId":27361,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe2","variableId":23109,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_solaredge","variableId":51938,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_ev","variableId":32871,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_sdl","variableId":23879,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_charge","variableId":50230,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_discharge","variableId":43899,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t1","variableId":59607,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t2","variableId":26620,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_requested","variableId":19651,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"sdl_requested","variableId":38943,"role":"sdl_request","factorToW":1,"maxAgeSeconds":120},{"key":"solar_ac_value","variableId":37975,"role":"solar_raw","factorToW":1,"maxAgeSeconds":60},{"key":"solar_ac_scale","variableId":41853,"role":"solar_scale","factorToW":1,"maxAgeSeconds":60}],"formula":"solar_terminal_v1","solarReference":{"pvKey":"pv_solaredge","batteryKey":"physical_solaredge","rawKey":"solar_ac_value","scaleKey":"solar_ac_scale"},"minimumCoverage":0.95,"maximumGapSeconds":10,"minimumTrainingHours":24,"historyDays":28} diff --git a/services/netplan-v4/commissioning/deploy_application.py b/services/netplan-v4/commissioning/deploy_application.py new file mode 100644 index 0000000..4e27762 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_application.py @@ -0,0 +1,131 @@ +"""Deploy the integrated measurement/training application, NOT actuator permission. + +Only V4 and the portal are recreated. Existing legacy forecasts, tariffs, Symcon and +battery dispatch are unchanged. Source-only checks are the default; --apply is explicit. +""" +from pathlib import Path +from datetime import datetime, timezone +from uuid import UUID +import argparse,hashlib,json,os,sqlite3,subprocess,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PORTAL=ROOT.parent/'license' +MANIFEST=Path(__file__).with_name('application-source')/'RELEASE.json' +DATASET=Path(__file__).with_name('application-source')/'server-dataset.json' +PORTAL_FILES={'integrations/netplan-v4-bridge.mjs':'netplan-v4-bridge.mjs','gui/netplan-v4.js':'public/netplan-v4.js'} + +def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest() + +def verify(): + m=json.loads(MANIFEST.read_text()) + if m.get('scope')!='integrated_measurement_application' or not m.get('sourceHashes'): + raise ValueError('Unexpected application release manifest') + for n,h in m['sourceHashes'].items(): + p=ROOT/n + if Path(n).is_absolute() or '..' in Path(n).parts or p.is_symlink() or not p.is_file() or digest(p)!=h: + raise ValueError('Source drift: '+n) + for n,old in m['portalBefore'].items(): + if n not in PORTAL_FILES.values():raise ValueError('Unexpected portal target') + p=PORTAL/n + if p.is_symlink() or not p.is_file():raise ValueError('Portal target changed') + source=next(s for s,t in PORTAL_FILES.items() if t==n) + if digest(p) not in (old,m['sourceHashes'][source]):raise ValueError('Concurrent portal change; not overwritten') + return m + +def atomic(path,data,mode=0o644): + if path.is_symlink():raise ValueError('Symlink refused') + fd,name=tempfile.mkstemp(prefix='.v4-app-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.chmod(name,mode);os.replace(name,path) + finally: + if os.path.exists(name):os.unlink(name) + +def backup_database(source,destination): + if not source.is_file() or source.is_symlink() or destination.exists():raise ValueError('Explicit existing database and new backup path required') + with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as a,sqlite3.connect(destination) as b: + a.backup(b) + if b.execute('PRAGMA integrity_check').fetchone()[0]!='ok':raise ValueError('Backup failed') + destination.chmod(0o600) + +BOOTSTRAP='''import json,os,urllib.request,sys +p=json.load(sys.stdin) +base='http://127.0.0.1:9100/internal/v2/prognosis/'+p['plant']+'/planner' +h={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +req=urllib.request.Request(base+'/datasets/'+p['dataset']['datasetId'],data=json.dumps(p['dataset']).encode(),headers=h,method='PUT') +receipt=json.load(urllib.request.urlopen(req,timeout=10)) +state=json.load(urllib.request.urlopen(urllib.request.Request(base,headers=h),timeout=10)) +assert state['liveEnabled'] is False +print(json.dumps({'dataset':receipt,'existingForecastSource':state['settings']['forecastSource'],'liveEnabled':False})) +''' + +def run(plant,apply=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Use the prepared installation-specific mapping') + if not apply: + print('APPLICATION SOURCE CHECK PASSED:',len(m['sourceHashes']),'files. No deployment.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker permissions') + folder=ROOT/'application-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ') + folder.mkdir(parents=True,mode=0o700) + env={**os.environ,'NETPLAN_V4_PLANTS':plant} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')] + portal=['docker','compose','-f',str(PORTAL/'compose.yaml'),'-f',str(ROOT/'compose.portal-bridge.yaml')] + result={'scope':'integrated_measurement_application','startedAt':datetime.now(timezone.utc).isoformat(), + 'liveEnabled':False,'productionCommissioned':False,'steps':[],'sourceHashes':m['sourceHashes']} + changed={};old_image=None;v4_replaced=False;portal_replaced=False + def cmd(args,timeout=180,capture=False,input=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=input, + stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + try: + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip() + result['previousV4Image']=old_image + cmd(compose+['build','netplan-v4'],timeout=900) + cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240) + cmd(['node','--test',str(ROOT/'tests/portal.test.mjs')]) + cmd(['node','--check',str(ROOT/'gui/netplan-v4.js')]) + result['steps'].append('target_python_and_portal_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite') + result['steps'].append('consistent_database_backup') + for source,target in PORTAL_FILES.items(): + p=PORTAL/target;old=p.read_bytes();new=(ROOT/source).read_bytes() + if old==new:continue + if hashlib.sha256(old).hexdigest()!=m['portalBefore'][target]:raise ValueError('Concurrent portal change') + dest=folder/'portal-before'/target;dest.parent.mkdir(parents=True,exist_ok=True);dest.write_bytes(old) + atomic(p,new);changed[target]=(old,new) + v4_replaced=True;cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4']) + # Recreate rather than restart: atomic replacement of a file bind mount needs a new mount. + portal_replaced=True;cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + payload=json.dumps({'plant':plant,'dataset':json.loads(DATASET.read_text())}) + receipt=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,input=payload) + result['application']=json.loads(receipt.stdout) + result['status']='application_deployed_no_actuator_permission' + result['steps'].append('configured_dataset_and_health_verified') + except Exception as e: + result['status']='needs_review';result['errorType']=type(e).__name__ + restored=[] + for target,(old,new) in changed.items(): + p=PORTAL/target + if p.read_bytes()==new:atomic(p,old);restored.append(target) + result['restoredPortalFiles']=restored + try: + if v4_replaced and old_image: + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + cmd(compose+['-f',str(rollback),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + if portal_replaced:cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + result['rollback']='previous_runtime_restored_additive_data_retained' + except Exception:result['rollback']='manual_review_required' + raise + finally: + result['finishedAt']=datetime.now(timezone.utc).isoformat() + report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n') + uid=ROOT.stat().st_uid;gid=ROOT.stat().st_gid + os.chown(folder,uid,gid);os.chown(folder.parent,uid,gid);os.chown(report,uid,gid);report.chmod(0o640) + print('APPLICATION RELEASE REPORT:',report) + print('V4 data/model application and portal updated. Install manager data integration separately. No V4 actuation enabled.') + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True,type=lambda v:str(UUID(v)));p.add_argument('--apply',action='store_true');a=p.parse_args() + try:run(a.plant,a.apply) + except Exception as e:raise SystemExit('Stopped: '+type(e).__name__+'. See the application release report.') diff --git a/services/netplan-v4/commissioning/deploy_history_timing.py b/services/netplan-v4/commissioning/deploy_history_timing.py new file mode 100644 index 0000000..c4c5914 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_history_timing.py @@ -0,0 +1,146 @@ +"""Install the tested historical-publication correction in the V4 service only. + +Default is source validation. --apply requires root. No Symcon changes, polling, +portal restart, old-data rewrite, forecast-source switch or actuator permission. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import hashlib +import json +import os +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = Path(__file__).with_name('history-timing-source') +TARGETS = ('Dockerfile', 'netplan_v4/history_timing.py', 'netplan_v4/measurement_pipeline.py', + 'tests/test_history_timing.py', 'tests/test_history_timing_deployment.py') + + +def digest(path): + if path.is_symlink(): + raise ValueError('Source symlink refused') + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +def verify(): + manifest = json.loads((PACKAGE/'RELEASE.json').read_text()) + if manifest.get('scope') != 'historical_publication_only' or tuple(manifest.get('files', {})) != TARGETS: + raise ValueError('Unexpected timing release scope') + for name, versions in manifest['files'].items(): + if digest(PACKAGE/'source'/name) != versions['after']: + raise ValueError('Timing candidate changed: '+name) + if digest(ROOT/name) not in (versions['before'], versions['after']): + raise ValueError('Concurrent application change: '+name) + if digest(PACKAGE/'dataset.json') != manifest['datasetSha256']: + raise ValueError('Dataset candidate changed') + if digest(ROOT/'commissioning/deploy_application.py') != manifest['existingDeploymentHelperSha256']: + raise ValueError('Existing deployment helper changed') + if digest(Path(__file__).resolve()) != manifest['installerSha256']: + raise ValueError('Timing installer changed') + return manifest + + +BOOTSTRAP = r'''import json,os,sys,time,urllib.request +payload=json.load(sys.stdin) +base='http://127.0.0.1:9100/internal/v2/prognosis/'+payload['plant']+'/planner' +headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +def get(): + return json.load(urllib.request.urlopen(urllib.request.Request(base,headers=headers),timeout=10)) +before=get() +assert before['dataPipeline'].get('historyTimingVersion')==1 +assert before['liveEnabled'] is False +req=urllib.request.Request(base+'/datasets/'+payload['dataset']['datasetId'], + data=json.dumps(payload['dataset']).encode(),headers=headers,method='PUT') +receipt=json.load(urllib.request.urlopen(req,timeout=10)) +after=get() +assert before['settings']==after['settings'] and after['liveEnabled'] is False +view=next(d for d in after['dataPipeline']['datasets'] if d['datasetId']==payload['dataset']['datasetId']) +print(json.dumps({'dataset':receipt,'originalObservationDataset':view['observationDatasetId'], + 'availableRecords':view['records'],'pipelineStatus':view['status'], + 'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'], + 'liveEnabled':False})) +''' + + +def run(plant, apply=False): + manifest = verify() + if plant != manifest['installationId']: + raise ValueError('Use the reviewed installation') + if not apply: + print('HISTORY TIMING SOURCE CHECK PASSED; no runtime changes.') + return + if os.geteuid() != 0: + raise ValueError('Run as root; do not change Docker permissions') + # Reuse the reviewed atomic-write and consistent-backup implementation. + from deploy_application import atomic, backup_database + folder = ROOT/'history-timing-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ') + folder.mkdir(parents=True, mode=0o700) + result={'scope':'historical_publication_only','startedAt':datetime.now(timezone.utc).isoformat(), + 'steps':[],'liveEnabled':False,'symconChanged':False,'pollingChanged':False, + 'sourceFreshnessLimitsChanged':False,'forecastSelectionChanged':False} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')] + env={**os.environ,'NETPLAN_V4_PLANTS':plant} + def cmd(args, timeout=180, capture=False, data=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data, + stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + changed={}; previous_image=None; replaced=False + try: + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1: raise ValueError('Expected one running V4 container') + previous_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip() + result['previousImage']=previous_image + for name, versions in manifest['files'].items(): + target=ROOT/name; content=(PACKAGE/'source'/name).read_bytes() + before=target.read_bytes() if target.is_file() else None + if digest(target)==versions['after']: continue + if digest(target)!=versions['before']: raise ValueError('Concurrent source change') + backup=folder/'source-before'/name; backup.parent.mkdir(parents=True,exist_ok=True) + if before is not None: backup.write_bytes(before) + atomic(target,content); changed[name]=(before,content) + verify() + cmd(compose+['build','netplan-v4'],timeout=900) + cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240) + result['steps'].append('target_tests_passed'); verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite') + result['steps'].append('consistent_database_backup') + rollback=folder/'rollback.yaml' + rollback.write_text('services:\n netplan-v4:\n image: '+previous_image+'\n') + replaced=True + cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4']) + payload=json.dumps({'plant':plant,'dataset':json.loads((PACKAGE/'dataset.json').read_text())}) + response=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=payload) + result['application']=json.loads(response.stdout) + result['status']='historical_timing_installed_no_actuation' + result['steps'].append('versioned_dataset_registered_original_observations_retained') + except Exception as exc: + result['status']='needs_review';result['errorType']=type(exc).__name__ + conflicts=[] + for name,(before,after) in changed.items(): + path=ROOT/name + if digest(path)!=hashlib.sha256(after).hexdigest(): conflicts.append(name);continue + if before is None:path.unlink() + else:atomic(path,before) + result['concurrentFilesNotOverwritten']=conflicts + if replaced and previous_image: + try: + cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + result['rollback']='previous_image_restored_additive_dataset_retained' + except Exception: result['rollback']='manual_review_required' + raise + finally: + result['finishedAt']=datetime.now(timezone.utc).isoformat() + report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n') + uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid + for path in (folder.parent,folder,report):os.chown(path,uid,gid) + report.chmod(0o640) + print('HISTORY TIMING REPORT:',report) + print('Only V4 updated. Original data, manager, polling, forecast selection and actuator permissions unchanged.') + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True);parser.add_argument('--apply',action='store_true') + args=parser.parse_args() + try:run(args.plant,args.apply) + except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. See HISTORY TIMING REPORT; do not enable actuators.') diff --git a/services/netplan-v4/commissioning/deploy_unified_release.py b/services/netplan-v4/commissioning/deploy_unified_release.py new file mode 100644 index 0000000..edf2f93 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_unified_release.py @@ -0,0 +1,124 @@ +"""Install the reviewed V4 release candidate as one transaction-like deployment. + +Tests run in an isolated candidate image BEFORE replacing any running service. +No device control permission, forecast-source switch, telemetry rewrite or cursor +jump. Exactly proven numeric representations are registered for backlog recovery. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse,hashlib,json,os,subprocess,sys,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PACKAGE=ROOT/'commissioning/unified-release' +GUI=ROOT.parent/'license/public/netplan-v4.js' + + +def digest(p): + if p.is_symlink():raise ValueError('Symlink refused') + return hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None + + +def verify(): + m=json.loads((PACKAGE/'RELEASE.json').read_text()) + if m.get('scope')!='unified_v4_candidate' or not m.get('files'):raise ValueError('Unexpected release') + if digest(Path(__file__).resolve())!=m['installerSha256']:raise ValueError('Installer changed') + for n,v in m['files'].items(): + if Path(n).is_absolute() or '..' in Path(n).parts:raise ValueError('Unsafe release path') + if digest(PACKAGE/'context'/n)!=v['after']:raise ValueError('Candidate drift: '+n) + if digest(ROOT/n) not in (v['before'],v['after']):raise ValueError('Concurrent runtime source edit: '+n) + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent GUI edit') + if digest(PACKAGE/'mapping-proof.json')!=m['proofSha256']:raise ValueError('Compatibility proof changed') + if digest(ROOT/'commissioning/deploy_application.py')!=m['backupHelperSha256']:raise ValueError('Backup helper changed') + return m + + +BOOTSTRAP=r'''import json,os,sys,urllib.request +x=json.load(sys.stdin);base='http://127.0.0.1:9100' +health=json.load(urllib.request.urlopen(base+'/health',timeout=5)) +assert health.get('applicationRelease')=='unified-rc1' and health.get('liveEnabled') is False +headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +url=base+'/internal/v2/prognosis/'+x['plant']+'/planner' +def get():return json.load(urllib.request.urlopen(urllib.request.Request(url,headers=headers),timeout=20)) +before=get() +receipt={'status':'not_registered_requires_explicit_approval'} +if x['approveMapping']: + request=urllib.request.Request(url+'/datasets/'+x['dataset']+'/mapping-compatibility',data=json.dumps(x['proof']).encode(),headers=headers,method='PUT') + receipt=json.load(urllib.request.urlopen(request,timeout=15)) +after=get();assert before['settings']==after['settings'] and after['liveEnabled'] is False +print(json.dumps({'health':health,'mappingRecovery':receipt,'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'],'economicReplayConnected':after['economicComparison']['connected'],'dataSets':[{'datasetId':d['datasetId'],'records':d['records'],'lastCapture':d['lastCapture'],'lastReceived':d.get('lastReceived'),'status':d['status']} for d in after['dataPipeline']['datasets']]})) +''' + + +def run(plant,apply=False,approve_mapping=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Wrong prepared installation') + if not apply: + print('UNIFIED RELEASE SOURCE CHECK PASSED; no installation.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker socket permissions') + from deploy_application import atomic,backup_database + folder=ROOT/'unified-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ');folder.mkdir(parents=True,mode=0o700) + report={'scope':'unified_v4_candidate','releaseId':m['releaseId'],'startedAt':datetime.now(timezone.utc).isoformat(),'liveEnabled':False,'productionCommissioned':False,'steps':[]} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')];env=dict(os.environ) + def cmd(args,timeout=180,capture=False,data=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data,stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + changed={};old_gui=None;new_gui=None;replaced=False;old_image=None + try: + # Read only this non-secret allowlist; never print or persist the complete environment. + env['NETPLAN_V4_PLANTS']=plant + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + allowed=cmd(['docker','exec',ids[0],'python','-c',"import os;print(os.getenv('NETPLAN_V4_PLANTS',''))"],capture=True).stdout.strip() + if plant not in allowed.split(','):raise ValueError('Installation not in running allowlist') + env['NETPLAN_V4_PLANTS']=allowed + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip();report['previousImage']=old_image + tag='enelix-netplan-v4:rc-'+m['releaseId'][:16] + cmd(['docker','build','-t',tag,str(PACKAGE/'context')],timeout=900) + candidate=cmd(['docker','image','inspect','--format','{{.Id}}',tag],capture=True).stdout.strip() + cmd(['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--tmpfs','/tmp:rw,nosuid,nodev,noexec,size=256m','--entrypoint','python',candidate,'/app/run_tests.py'],timeout=600) + cmd(['node','--test',str(PACKAGE/'context/tests/portal.test.mjs')]);cmd(['node','--check',str(PACKAGE/'context/gui/netplan-v4.js')]) + report['steps'].append('isolated_candidate_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite');report['steps'].append('consistent_backup') + for name,v in m['files'].items(): + target=ROOT/name;after=(PACKAGE/'context'/name).read_bytes();before=target.read_bytes() if target.is_file() else None + if before==after:continue + if digest(target)!=v['before']:raise ValueError('Source changed during tests') + backup=folder/'source-before'/name;backup.parent.mkdir(parents=True,exist_ok=True) + if before is not None:backup.write_bytes(before) + target.parent.mkdir(parents=True,exist_ok=True);atomic(target,after);changed[name]=(before,after) + override=folder/'candidate.yaml';override.write_text('services:\n netplan-v4:\n image: '+candidate+'\n') + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + replaced=True;cmd(compose+['-f',str(override),'up','-d','--no-deps','--no-build','--wait','netplan-v4'],timeout=180) + data=json.dumps({'plant':plant,'dataset':'lihrenmoos-physical-v1','proof':json.loads((PACKAGE/'mapping-proof.json').read_text()),'approveMapping':approve_mapping}) + result=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=data) + report['application']=json.loads(result.stdout) + old_gui=GUI.read_bytes();new_gui=(PACKAGE/'context/gui/netplan-v4.js').read_bytes() + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent portal UI edit') + (folder/'gui-before.js').write_bytes(old_gui);atomic(GUI,new_gui) + report['steps'].append('audited_backlog_compatibility_and_application_verified');report['candidateImage']=candidate + report['status']='unified_candidate_installed_no_actuation' + except Exception as exc: + report['status']='needs_review';report['errorType']=type(exc).__name__ + if old_gui is not None and GUI.read_bytes()==new_gui:atomic(GUI,old_gui) + for name,(before,after) in reversed(list(changed.items())): + target=ROOT/name + if target.read_bytes()!=after:continue + if before is None:target.unlink() + else:atomic(target,before) + if replaced and old_image: + try: + cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + report['rollback']='previous_image_restored_additive_audit_tables_retained' + except Exception:report['rollback']='manual_review_required' + raise + finally: + report['finishedAt']=datetime.now(timezone.utc).isoformat();p=folder/'REPORT.json';p.write_text(json.dumps(report,indent=2)+'\n') + uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid + for x in (folder.parent,folder,p):os.chown(x,uid,gid) + p.chmod(0o640);print('UNIFIED RELEASE REPORT:',p) + print('V4 candidate and planner UI installed. Mapping equivalence is activated only with explicit approval. No data/cursor rewrite and no device-control grant.') + + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True);p.add_argument('--apply',action='store_true');p.add_argument('--approve-numeric-mapping-compatibility',action='store_true',help='Explicitly approve only the audited 100 versus 100.0 mapping equivalence; never rewrite original records');a=p.parse_args() + try:run(a.plant,a.apply,a.approve_numeric_mapping_compatibility) + except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. Review UNIFIED RELEASE REPORT; no actuator permission enabled.') diff --git a/services/netplan-v4/commissioning/history-timing-source/RELEASE.json b/services/netplan-v4/commissioning/history-timing-source/RELEASE.json new file mode 100644 index 0000000..65f5a18 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/RELEASE.json @@ -0,0 +1,29 @@ +{ + "scope": "historical_publication_only", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "files": { + "Dockerfile": { + "before": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "after": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90" + }, + "netplan_v4/history_timing.py": { + "before": null, + "after": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f" + }, + "netplan_v4/measurement_pipeline.py": { + "before": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "after": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d" + }, + "tests/test_history_timing.py": { + "before": null, + "after": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2" + }, + "tests/test_history_timing_deployment.py": { + "before": null, + "after": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e" + } + }, + "datasetSha256": "671216e1cc04148f4f553103077bcb4500bf0120cb74780703bfcf26cd1121d3", + "installerSha256": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "existingDeploymentHelperSha256": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e" +} diff --git a/services/netplan-v4/commissioning/history-timing-source/dataset.json b/services/netplan-v4/commissioning/history-timing-source/dataset.json new file mode 100644 index 0000000..bf153a3 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/dataset.json @@ -0,0 +1,206 @@ +{ + "datasetId": "lihrenmoos-physical-published-v2", + "mappingSha256": "517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b", + "inventorySha256": "0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc", + "sources": [ + { + "key": "grid", + "variableId": 40348, + "factorToW": 1000, + "maxAgeSeconds": 60, + "role": "grid" + }, + { + "key": "pv_goodwe1", + "variableId": 48459, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "pv_goodwe2", + "variableId": 53802, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "pv_solaredge", + "variableId": 20335, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "pv" + }, + { + "key": "physical_goodwe1", + "variableId": 47725, + "factorToW": -1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "physical_goodwe2", + "variableId": 35724, + "factorToW": -1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "physical_solaredge", + "variableId": 21447, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "physical_storage" + }, + { + "key": "ev_account", + "variableId": 52020, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "reference" + }, + { + "key": "sdl_account", + "variableId": 25085, + "factorToW": 1, + "maxAgeSeconds": 60, + "role": "reference" + }, + { + "key": "grid_display", + "variableId": 49301, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_goodwe1", + "variableId": 27361, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_goodwe2", + "variableId": 23109, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_solaredge", + "variableId": 51938, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_ev", + "variableId": 32871, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "soc_sdl", + "variableId": 23879, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_available_charge", + "variableId": 50230, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_available_discharge", + "variableId": 43899, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "energy_t1", + "variableId": 59607, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "energy_t2", + "variableId": 26620, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "ev_requested", + "variableId": 19651, + "role": "reference", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "sdl_requested", + "variableId": 38943, + "role": "sdl_request", + "factorToW": 1, + "maxAgeSeconds": 120 + }, + { + "key": "solar_ac_value", + "variableId": 37975, + "role": "solar_raw", + "factorToW": 1, + "maxAgeSeconds": 60 + }, + { + "key": "solar_ac_scale", + "variableId": 41853, + "role": "solar_scale", + "factorToW": 1, + "maxAgeSeconds": 60 + } + ], + "formula": "solar_terminal_v1", + "solarReference": { + "pvKey": "pv_solaredge", + "batteryKey": "physical_solaredge", + "rawKey": "solar_ac_value", + "scaleKey": "solar_ac_scale" + }, + "minimumCoverage": 0.95, + "maximumGapSeconds": 10, + "minimumTrainingHours": 24, + "historyDays": 28, + "sourceDatasetId": "lihrenmoos-physical-v1", + "historyTimingPolicy": { + "version": 1, + "method": "equal_endpoint_v1", + "sources": { + "pv_goodwe1": { + "maxSpanSeconds": 130, + "evidenceId": "symcon-modbus-publish60-poll60-jitter10" + }, + "physical_goodwe1": { + "maxSpanSeconds": 130, + "evidenceId": "symcon-modbus-publish60-poll60-jitter10" + }, + "pv_goodwe2": { + "maxSpanSeconds": 75, + "evidenceId": "symcon-modbus-publish60-poll5-jitter10" + }, + "physical_goodwe2": { + "maxSpanSeconds": 75, + "evidenceId": "symcon-modbus-publish60-poll5-jitter10" + }, + "solar_ac_scale": { + "maxSpanSeconds": 90, + "evidenceId": "configured-scale-poll20-bounded-equal-value-estimate" + } + } + } +} diff --git a/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile b/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile new file mode 100644 index 0000000..ceff456 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY netplan_v4 ./netplan_v4 +COPY tests ./tests +COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile . +COPY integrations ./integrations +COPY gui ./gui +COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ +COPY acceptance ./acceptance +COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py +# Host sources can be 0600/0700. COPY makes them root-owned. +# Normalize only packaged application code; never change host secrets or sockets. +RUN find /app -type d -exec chmod 0755 {} + \ + && find /app -type f -exec chmod 0644 {} + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results +USER 1000:1000 +# Fail the build early if the actual unprivileged runtime cannot read the code. +RUN python /app/runtime_preflight.py +CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"] diff --git a/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py new file mode 100644 index 0000000..6205d84 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/history_timing.py @@ -0,0 +1,58 @@ +"""Explicit, retrospective equal-endpoint estimates for sparse state publication. + +Never changes a source timestamp, real-time freshness limit, meter evidence or +actuator lease. A repeated value after a bounded gap supports a MODEL estimate, +not proof that the physical signal was constant between the observations. +""" +from math import isfinite +import re + + +def validate_policy(policy, sources): + if policy is None: + return None + if (not isinstance(policy, dict) or set(policy) != {'version', 'method', 'sources'} + or type(policy['version']) is not int or policy['version'] != 1 + or policy['method'] != 'equal_endpoint_v1' + or not isinstance(policy['sources'], dict) or not policy['sources']): + raise ValueError('Explicit historical timing policy required') + defined = {s['key']: s for s in sources} + for key, rule in policy['sources'].items(): + if key not in defined or defined[key]['role'] not in ('pv', 'physical_storage', 'solar_scale'): + raise ValueError('Timing estimates limited to explicit power/scale sources') + if not isinstance(rule, dict) or set(rule) != {'maxSpanSeconds', 'evidenceId'}: + raise ValueError('Timing bound and evidence reference required') + if type(rule['maxSpanSeconds']) is not int or not defined[key]['maxAgeSeconds'] <= rule['maxSpanSeconds'] <= 180: + raise ValueError('Historical endpoint span outside reviewed bounds') + if not isinstance(rule['evidenceId'], str) or not re.fullmatch(r'[A-Za-z0-9_-]{8,100}', rule['evidenceId']): + raise ValueError('Invalid publication evidence reference') + return policy + + +def endpoint_bridges(series, first_observed, blocks, capture_gaps, policy, source_specs): + """Index strictly consecutive, equal original timestamps; never invent endpoints. + +Only the stale tail of a normal hold is filled. A missing/invalid capture or a +source conflict forbids bridging. The right endpoint must have been observed; +its availability is retained by the caller for causal training and replay. +""" + result = {key: {} for key in series} + if policy is None: + return result + for key, rule in policy['sources'].items(): + if key not in series: + continue # e.g. an unused DC channel in an AC-terminal formula + values = series[key] + times = sorted(values) + for left, right in zip(times, times[1:]): + value, next_value = values[left], values[right] + if (value is None or next_value is None or type(value) not in (int, float) + or not isfinite(value) or value != next_value): + continue + if not source_specs[key]['maxAgeSeconds'] < right-left <= rule['maxSpanSeconds']: + continue + if any(start < right and end > left for start, end in (*blocks[key], *capture_gaps)): + continue + result[key][left] = {'end': right, 'availableAt': first_observed[key][right], + 'spanSeconds': right-left} + return result diff --git a/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py new file mode 100644 index 0000000..0579467 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/netplan_v4/measurement_pipeline.py @@ -0,0 +1,500 @@ +"""Application data path: versioned numeric observations -> physical load -> trained profiles. + +Lives in the existing planner service/database; no separate diagnostic service. +The device may append only to an operator-configured dataset. Original observations, +model revisions and prediction vintages are preserved. Output is never an actuator grant. +""" +from __future__ import annotations +from bisect import bisect_right +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from math import isfinite +from statistics import median +from zoneinfo import ZoneInfo +import json +from .history_timing import validate_policy, endpoint_bridges + +UTC = timezone.utc +LOCAL = ZoneInfo('Europe/Zurich') +FAMILIES = ('3', '13', '23') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond: + raise ValueError('Explicit whole-second UTC timestamp required') + return int(t.timestamp()) + + +def iso(t): + return datetime.fromtimestamp(t, UTC).isoformat() + + +def numeric(value, bound=1e12): + return type(value) in (int, float) and isfinite(value) and abs(value) <= bound + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_data_sets( + plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_observations( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,captured_at)); + CREATE TABLE IF NOT EXISTS planner_load_windows( + plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL, + available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,start)); + CREATE TABLE IF NOT EXISTS planner_load_models( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,model_id)); + CREATE TABLE IF NOT EXISTS planner_model_current( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_pipeline_state( + plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL, + status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_prediction_vintages( + plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL, + target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL, + load_w REAL NOT NULL, pv_w REAL NOT NULL, + PRIMARY KEY(plant,dataset,issued_at,target,family)); + CREATE INDEX IF NOT EXISTS planner_observation_window + ON planner_observations(plant,dataset,captured_at); + CREATE INDEX IF NOT EXISTS planner_prediction_target + ON planner_prediction_vintages(plant,dataset,target); + ''') + + +def validate_config(c): + fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', + 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', + 'minimumTrainingHours', 'historyDays'} + optional = {'sourceDatasetId', 'historyTimingPolicy'} + if not isinstance(c, dict) or not fields <= set(c) or set(c)-fields-optional: + raise ValueError('Explicit dataset configuration required') + name = c['datasetId'] + if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): + raise ValueError('Invalid dataset ID') + for field in ('mappingSha256', 'inventorySha256'): + h = c[field] + if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h): + raise ValueError('Explicit mapping/inventory fingerprint required') + if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'): + raise ValueError('Unknown physical formula') + if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1: + raise ValueError('Coverage must be .90..1; recorded gaps remain visible') + for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)): + if type(c[field]) is not int or not lo <= c[field] <= hi: + raise ValueError('Invalid '+field) + sources = c['sources'] + if not isinstance(sources, list) or not 3 <= len(sources) <= 80: + raise ValueError('Source list required') + seen, ids = set(), set() + roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'} + for s in sources: + if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}: + raise ValueError('Explicit source definition required') + k = s['key'] + if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids: + raise ValueError('Duplicate/invalid source') + if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0: + raise ValueError('Source role/factor invalid') + if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300: + raise ValueError('Source lifetime invalid') + seen.add(k); ids.add(s['variableId']) + if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources): + raise ValueError('Grid and PV measurement sources required') + sr = c['solarReference'] + if c['formula'] == 'solar_terminal_v1': + if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}: + raise ValueError('Solar terminal sources required') + bykey = {s['key']: s['role'] for s in sources} + if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))): + raise ValueError('Solar origin roles mismatch') + elif sr is not None: + raise ValueError('No unused solar mapping allowed') + validate_policy(c.get('historyTimingPolicy'), sources) + source = c.get('sourceDatasetId') + if source is not None and (not isinstance(source, str) or not 1 <= len(source) <= 80 or source == name or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in source)): + raise ValueError('Invalid source dataset reference') + canonical(c) + return c + + +def register_dataset(con, plant, c, now): + """Operator endpoint only; device append endpoint cannot change units or limits.""" + validate_config(c) + if c.get('sourceDatasetId'): + origin = configuration(con, plant, c['sourceDatasetId']) + if origin.get('sourceDatasetId'): + raise ValueError('Dataset reference chains are not allowed') + comparable = lambda v: {k:x for k,x in v.items() if k not in ('datasetId','sourceDatasetId','historyTimingPolicy')} + if canonical(comparable(origin)) != canonical(comparable(c)): + raise ValueError('Referenced observations must retain identical measurement meaning') + value = canonical(c) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone() + if old and old[0] != value: + raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning') + con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False} + + +def configuration(con, plant, dataset): + row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone() + if row is None: + raise ValueError('Dataset not configured for this installation') + return json.loads(row[0]) + + +def project(record, c, plant, now): + if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: + raise ValueError('Wrong capture identity') + if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + raise ValueError('Wrong capture mapping or inventory') + t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) + if start > t or t > now+30 or t < now-90*86400: + raise ValueError('Capture timestamp outside permitted range') + if not isinstance(record.get('raw'), dict): + raise ValueError('Numeric raw observations required') + out = {}; issues = [] + for s in c['sources']: + r = record['raw'].get(s['key'], {}) + if not isinstance(r, dict): + r = {} + v, at = r.get('value'), r.get('sourceUpdatedAt') + good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == [] + if not good: + v = at = None + issues.append(s['key']) + # Unknown/free-text fields, credentials, client quality claims never persisted. + out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)} + return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues} + + +def ingest_batch(con, plant, payload, now): + if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: + raise ValueError('Measurement batch version/fields invalid') + c = configuration(con,plant,payload['datasetId']) + if c.get('sourceDatasetId'): + raise ValueError('Derived dataset is read-only; append to the original measurement dataset') + records = payload['records'] + if not isinstance(records,list) or not 1 <= len(records) <= 120: + raise ValueError('Batch requires 1..120 captures') + rows = [project(r,c,plant,now) for r in records] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): + raise ValueError('Batch must be in increasing capture order') + stored = duplicate = 0 + con.execute('BEGIN IMMEDIATE') + try: + for r in rows: + value = canonical(r); digest = sha256(value.encode()).hexdigest() + old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() + if old: + if old[0] != digest: + raise ValueError('Conflicting immutable observation') + duplicate += 1 + else: + con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + con.commit() + except Exception: + con.rollback(); raise + return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate, + 'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False} + + +def physical_value(values, c): + """Same explicit sign convention as configured acquisition. No virtual power in load.""" + total = defaultdict(float) + sr = c['solarReference'] + for s in c['sources']: + if s['role'] not in ('grid','pv','physical_storage','flexible_load'): + continue + if sr and s['key'] in (sr['pvKey'],sr['batteryKey']): + continue + val = values[s['key']]*s['factorToW'] + if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0): + raise ValueError('Invalid physical power') + total[s['role']] += val + solar = 0. + if sr: + raw, sf = values[sr['rawKey']], values[sr['scaleKey']] + if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6: + raise ValueError('Invalid solar power/scaling sentinel') + solar = raw*10**int(sf) + load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar + if not numeric(load,1e9) or load < 0: + raise ValueError('Negative/nonfinite physical load') + return load + + +def reconstruct(records, c): + """Bounded retrospective estimation, never a real-time feedback signal. + +Missing observations split support. Source timestamps are not refreshed. Small +uncovered portions remain quantified and are never filled with zero. +""" + if len(records) < 2: + return [] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])): + raise ValueError('Capture sequence not ordered') + sr = c['solarReference'] + primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')} + if sr: + primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) + for s in c['sources']: + if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) + first,last = records[0]['capturedAt'],records[-1]['capturedAt'] + series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + first_observed = {k:{} for k in primary} + pending = {k:None for k in primary}; high = {k:0 for k in primary} + edges = {first,last} + for a,b in zip(records,records[1:]): + if b['capturedAt']-a['capturedAt'] > 45: + gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1]) + for r in records: + at = r['capturedAt'] + for k in primary: + v = r['raw'].get(k,{}) + t = v.get('sourceUpdatedAt') + if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5: + if pending[k] is None: pending[k] = at + continue + high[k] = max(high[k],t) + if pending[k] is not None: + blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None + if t in series[k] and series[k][t] != v['value']: + series[k][t] = None + else: + series[k].setdefault(t,v['value']) + first_observed[k].setdefault(t, max(at, r.get('_receivedAt', at))) + for k,s in primary.items(): + if pending[k] is not None: + blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) + for t in series[k]: edges.update((t,t+s['maxAgeSeconds'])) + edges.update(range(first//300*300+300,last,300)) + edges = sorted(x for x in edges if first <= x <= last) + knots = {k:sorted(v) for k,v in series.items()} + bridges = endpoint_bridges(series, first_observed, blocks, gaps, timing_policy, primary) + bins = {} + for a,b in zip(edges,edges[1:]): + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0, + 'publicationEstimatedSeconds':0,'publicationEstimatedBySource':{},'knownAt':0,'missingSourceSeconds':{}}) + vals = {}; usable = not any(x <= a < y for x,y in gaps) + extended = []; unavailable = []; known_at = 0 + for k,s in primary.items(): + pos = bisect_right(knots[k],a)-1 + t = knots[k][pos] if pos >= 0 else None + bridge = bridges[k].get(t) + expired = t is None or a >= t+s['maxAgeSeconds'] + supported_tail = bool(bridge and a < bridge['end']) + if t is None or (expired and not supported_tail) or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False; unavailable.append(k) + else: + vals[k] = series[k][t] + known_at = max(known_at, first_observed[k][t]) + if expired: + extended.append(k); known_at = max(known_at, bridge['availableAt']) + load = None + if usable: + try: load = physical_value(vals,c) + except ValueError: usable = False + if usable: + item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + item['knownAt'] = max(item['knownAt'], known_at) + if extended: item['publicationEstimatedSeconds'] += b-a + for key in extended: item['publicationEstimatedBySource'][key] = item['publicationEstimatedBySource'].get(key,0)+b-a + else: + item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + for key in unavailable: item['missingSourceSeconds'][key] = item['missingSourceSeconds'].get(key,0)+b-a + out=[] + for t,item in sorted(bins.items()): + # Partial beginning/end bins remain diagnostic and cannot train. + complete_extent = first <= t and last >= t+300 + coverage = item['seconds']/300 + eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds'] + out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], + 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, + 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, + 'meterBoundaryVerified':False,'method':c['formula'], + 'historyTimingMethod':(timing_policy or {}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':item['publicationEstimatedSeconds'], + 'publicationEstimatedBySourceSeconds':item['publicationEstimatedBySource'], + 'missingSourceSeconds':item['missingSourceSeconds'], + 'availableNotBefore':max(t+300,item['knownAt'])}) + return out + + +def slot(t): + local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL) + return local.hour*12+local.minute//5 + + +def build_profiles(rows): + samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)} + anchor=max(r['start'] for r in rows) + for r in rows: + i=slot(r['start']); v=r['loadW']; samples[i].append(v) + if anchor-r['start'] < 86400: recent[i].append(v) + weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v) + overall = median([r['loadW'] for r in rows]) + def profile(values): + # Missing calendar slots are a model estimate, not invented historical measurements. + result=[] + for i in range(288): + local=values.get(i,[]) + if not local: + local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])] + result.append(float(median(local)) if local else float(overall)) + return result + return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)}, + 'slotCoverage':len(samples)/288} + + +def predict(model, family, t): + p=model['profiles'][family] + if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday'] + return p[slot(t)] + + +def advance(con, plant, dataset, settings, now): + """Called by the existing worker; bounded data/model update once per five-minute tick.""" + c=configuration(con,plant,dataset); tick=now//300 + old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() + if old and old[0]==tick: return + observation_dataset=c.get('sourceDatasetId',dataset) + fetched=con.execute('SELECT value,received_at FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,observation_dataset,now-172800-300,now,now)).fetchall() + records=[{**json.loads(r[0]),'_receivedAt':r[1]} for r in fetched] + windows=reconstruct(records,c) + with con: + for w in windows: + if w['start']+300 > now-30 or w.get('availableNotBefore',0)>now: continue + con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', + (plant,dataset,w['start'],now,w['coverage'],canonical(w))) + rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] + good=[r for r in rows if r['profileUsable']] + active=current_model(con,plant,dataset,now) + cadence=86400 if settings['trainingCadence']=='daily' else 604800 + detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], + 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True, + 'observationDatasetId':observation_dataset, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':sum(r.get('publicationEstimatedSeconds',0) for r in good), + 'originalFreshnessLimitsChanged':False, + 'lastCapture':iso(records[-1]['capturedAt']) if records else None} + state='collecting' + if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: + state='model_ready' if active else 'training' + attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0] + if attempted is None or now-attempted>=cadence: + profiles=build_profiles(good) + candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), + 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], + 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'timingPolicySha256':sha256(canonical(c.get('historyTimingPolicy')).encode()).hexdigest(), + 'observationDatasetId':observation_dataset, + 'measurementBoundaryVerified':False} + # Causal held-out validation: build validation profiles without the final day. + split=good[-1]['start']-86400 + train=[r for r in good if r['start']+300<=split and r.get('availableNotBefore',r['start']+300)<=split]; test=[r for r in good if r['start']>=split] + if len(train)>=288 and len(test)>=240: + val={'profiles':build_profiles(train)} + errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} + candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors} + # Initial model is labelled bootstrap, never a production measurement proof. + # Existing model can be replaced only with held-out evidence and no aggregate regression. + promote=active is None + if active and candidate['validation']['status']=='causal_holdout': + past_model_eligible=active['trainedThrough']<=split and active['trainedAt']<=split + if past_model_eligible: + incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + promote=challenger<=incumbent + candidate['validation']['incumbentCompared']=True + else: + candidate['validation']['status']='holdout_overlaps_active_training' + ident=sha256(canonical(candidate).encode()).hexdigest() + with con: + con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate))) + if promote: + con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident)) + detail['candidateModelId']=ident;detail['candidatePromoted']=promote + state='model_ready' if promote or active else 'candidate_pending' + active=current_model(con,plant,dataset,now) + if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']}) + with con: + con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail', + (plant,dataset,tick,state,canonical(detail))) + + +def current_model(con,plant,dataset,at): + row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone() + return {**json.loads(row[1]),'modelId':row[0]} if row else None + + +def apply_load_forecast(con,plant,dataset,forecast,decision): + model=current_model(con,plant,dataset,decision) + if not model: + raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') + c=configuration(con,plant,dataset) + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,c.get('sourceDatasetId',dataset),decision,decision)).fetchone() + if not last: raise ValueError('No recent corrected observation') + last=json.loads(last[0]) + if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') + sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] + if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') + s=sdl_sources[0];r=last['raw'][s['key']] + if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']: + raise ValueError('No current external SDL request for the persistence scenario') + sdl=r['value']*s['factorToW'] + result=json.loads(canonical(forecast)); result['families']={} + result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + for family,old in forecast['families'].items(): + if family not in FAMILIES: continue + points=[] + for p in old['points']: + t=epoch(p['time']) + points.append({**p,'loadW':predict(model,family,t),'externalW':sdl}) + result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']), + 'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'], + 'loadMethodVersion':model['methodVersion'],'loadVariant':family, + 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), + 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, + 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation'], + 'historyTimingMethod':model.get('historyTimingMethod','strict_expiry'), + 'observationDatasetId':model.get('observationDatasetId',dataset)}} + return result + + +def pipeline_status(con,plant): + out=[] + for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): + ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() + out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], + 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], + 'observationDatasetId':c.get('sourceDatasetId',ds), + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry')}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False,'historyTimingVersion':1} diff --git a/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py new file mode 100644 index 0000000..b05b046 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing.py @@ -0,0 +1,204 @@ +"""Synthetic publication gaps plus full source-reference/application integration.""" +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.history_timing import validate_policy +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once +from test_measurement_pipeline import config, record, NOW, AID +from test_v4 import inputs, TOKEN + + +def timed_config(**kw): + return config(historyTimingPolicy={'version':1,'method':'equal_endpoint_v1', + 'sources':{'pv':{'maxSpanSeconds':130,'evidenceId':'synthetic-publication-policy'}}}, **kw) + + +def projected(span=360, period=120, c=None): + c = c or timed_config() + result=[] + for delta in range(0,span+1,30): + r=record(NOW+delta,c) + r['raw']['pv']['sourceUpdatedAt']=NOW+(delta//period)*period + result.append(m.project(r,c,AID,NOW+span)) + return result + + +class HistoricalTimingTest(unittest.TestCase): + def test_strict_policy_unchanged(self): + rows=projected(); result=m.reconstruct(rows,config()) + self.assertEqual(result[0]['coveredSeconds'],180) + self.assertEqual(result[0]['publicationEstimatedSeconds'],0) + + def test_equal_endpoint_bounded_tail_estimate(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertEqual(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],120) + self.assertEqual(w['publicationEstimatedBySourceSeconds'],{'pv':120}) + self.assertEqual(w['loadW'],6000) + self.assertFalse(w['fullPhysicalIntervalMeasured']) + self.assertFalse(w['meterBoundaryVerified']) + + def test_no_extension_at_open_end(self): + rows=projected(span=300) + for r in rows: r['raw']['pv']['sourceUpdatedAt']=NOW + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['coveredSeconds'],60) + + def test_equal_zero_requires_new_original_timestamp(self): + rows=projected() + for r in rows:r['raw']['pv']['value']=0 + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],300) + for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],60) + + def test_changed_endpoints_not_interpolated(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=r['raw']['pv']['sourceUpdatedAt']-NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_no_floating_tolerance(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=1e-9*(r['raw']['pv']['sourceUpdatedAt']-NOW) + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_long_gap_not_recovered_by_equal_zero(self): + rows=projected(span=720,period=660) + for r in rows:r['raw']['pv']['value']=0 + out=m.reconstruct(rows,timed_config()) + self.assertEqual(sum(w['publicationEstimatedSeconds'] for w in out),0) + self.assertEqual(out[0]['coveredSeconds'],60) + + def test_collector_gap_not_bridged(self): + rows=projected(); rows=[r for r in rows if r['capturedAt']!=NOW+90] + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_source_blocks_equality_inference(self): + rows=projected();rows[3]['raw']['pv']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + self.assertLess(w['coverage'],1) + + def test_conflicting_same_timestamp_blocks_equality(self): + rows=projected();rows[1]['raw']['pv']['value']+=10 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_other_physical_input_still_vetoes(self): + rows=projected();rows[3]['raw']['battery']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coverage'],1) + + def test_future_receipt_not_available_early(self): + rows=projected() + for r in rows:r['_receivedAt']=NOW+900 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['availableNotBefore'],NOW+900) + + def test_unrelated_virtual_failure_does_not_veto(self): + rows=projected() + for r in rows:r['raw']['sdl']['valid']=False + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coverage'],1) + + def test_no_input_mutation_or_source_age_change(self): + rows=projected();c=timed_config();before=copy.deepcopy((rows,c)) + m.reconstruct(rows,c) + self.assertEqual((rows,c),before) + self.assertEqual(c['sources'][1]['maxAgeSeconds'],60) + + def test_policy_requires_trusted_explicit_bounds(self): + for change in ({'version':True},{'method':'always_fill'},{'sources':{'pv':{'maxSpanSeconds':600,'evidenceId':'test-policy'}}}, + {'sources':{'grid':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}, + {'sources':{'sdl':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}): + c=timed_config();c['historyTimingPolicy'].update(change) + with self.subTest(change=change),self.assertRaises(ValueError):m.validate_config(c) + + def test_profile_values_stay_estimates(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertTrue(w['estimated']) + self.assertNotIn('controlEnabled',w) + self.assertNotIn('accountingEvidenceId',w) + + +class HistoryReferenceIntegrationTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.tmp.name)/'test.sqlite')) + self.c=config();self.new=timed_config(datasetId='physical-v2',sourceDatasetId='physical-v1') + m.register_dataset(self.store.con,AID,self.c,NOW) + def tearDown(self):self.store.close();self.tmp.cleanup() + def register(self):m.register_dataset(self.store.con,AID,self.new,NOW) + def fill(self): + raw=[] + for t in range(NOW-7200,NOW+1,30): + r=record(t);r['raw']['pv']['sourceUpdatedAt']=t-(t-(NOW-7200))%120;raw.append(r) + for i in range(0,len(raw),120):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':raw[i:i+120]},NOW) + return len(raw) + def test_reference_shares_original_immutable_observations(self): + n=self.fill();self.register() + self.assertEqual(m.pipeline_status(self.store.con,AID)['datasets'][1]['records'],n) + self.assertEqual(self.store.con.execute('SELECT count(*) FROM planner_observations').fetchone()[0],n) + self.assertEqual(m.configuration(self.store.con,AID,'physical-v1'),self.c) + def test_no_append_into_derived_dataset(self): + self.register() + with self.assertRaises(ValueError):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v2','records':[record(NOW)]},NOW) + def test_source_mapping_cannot_change_in_reference(self): + for key,value in [('mappingSha256','c'*64),('formula','solar_terminal_v1'),('minimumCoverage',.9)]: + c=copy.deepcopy(self.new);c[key]=value + with self.subTest(key=key),self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_no_reference_to_other_plant(self): + with self.assertRaises(ValueError):m.register_dataset(self.store.con,'00000000-0000-4000-8000-000000000099',self.new,NOW) + def test_no_cycles_or_reference_chains(self): + self.register(); c={**self.new,'datasetId':'physical-v3','sourceDatasetId':'physical-v2'} + with self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_training_model_is_versioned_and_policy_annotated(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v1',settings,NOW+60) + m.advance(self.store.con,AID,'physical-v2',settings,NOW+60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v1',NOW+60)) + model=m.current_model(self.store.con,AID,'physical-v2',NOW+60) + self.assertIsNotNone(model);self.assertEqual(model['historyTimingMethod'],'equal_endpoint_v1') + self.assertFalse(model['measurementBoundaryVerified']) + state=m.pipeline_status(self.store.con,AID)['datasets'][1]['detail'] + self.assertGreater(state['publicationEstimatedSeconds'],0) + self.assertFalse(state['originalFreshnessLimitsChanged']) + def test_no_future_receipt_leaks_into_model(self): + self.fill();self.register() + m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW-60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v2',NOW-60)) + def test_existing_sdl_freshness_not_relaxed_by_history(self): + self.fill();self.register();m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW+30) + op,fc,tar=inputs(datetime.fromtimestamp(NOW+180,timezone.utc)) + with self.assertRaises(ValueError):m.apply_load_forecast(self.store.con,AID,'physical-v2',fc,NOW+180) + def test_full_application_plan_uses_reference_model(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v2',settings,NOW) + op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc)) + for kind,val in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.store,AID,kind,val,datetime.fromtimestamp(NOW,timezone.utc)) + self.store.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v2'},0,datetime.fromtimestamp(NOW,timezone.utc)) + out=run_once(self.store,datetime.fromtimestamp(NOW,timezone.utc)) + self.assertTrue(out['executable'],out);self.assertFalse(out['liveEnabled']) + self.assertEqual(out['inputQuality']['dataPipeline']['historyTimingMethod'],'equal_endpoint_v1') + def test_backfilled_or_late_confirmed_training_is_not_causal_holdout(self): + # Completed synthetic windows learned only NOW cannot validate a model at a past split. + with self.store.con: + for t in range(NOW-3*86400,NOW,300): + window={'start':t,'profileUsable':True,'coverage':1.,'loadW':5000.,'availableNotBefore':NOW} + self.store.con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?)',(AID,'physical-v1',t,NOW,1.,m.canonical(window))) + m.advance(self.store.con,AID,'physical-v1',self.store.settings(AID),NOW) + model=m.current_model(self.store.con,AID,'physical-v1',NOW) + self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout') + + def test_v1_raw_ingest_works_after_reference_added(self): + self.register() + r=m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':[record(NOW)]},NOW) + self.assertEqual(r['stored'],1) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py new file mode 100644 index 0000000..2acf194 --- /dev/null +++ b/services/netplan-v4/commissioning/history-timing-source/source/tests/test_history_timing_deployment.py @@ -0,0 +1,94 @@ +"""Installer ordering/recovery tests; subprocesses, privileges and DB backup are simulated.""" +import hashlib +import importlib.util +import json +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPT=Path(__file__).resolve().parents[1]/'commissioning/deploy_history_timing.py' +spec=importlib.util.spec_from_file_location('history_deploy_tests',SCRIPT) +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class TimingDeploymentTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name) + self.package=self.root/'commissioning/history-timing-source';self.package.mkdir(parents=True) + files={} + for index,name in enumerate(d.TARGETS): + original=(b'old:'+name.encode()) if index%2==0 else None + target=self.root/name;target.parent.mkdir(parents=True,exist_ok=True) + if original is not None:target.write_bytes(original) + candidate=self.package/'source'/name;candidate.parent.mkdir(parents=True,exist_ok=True);candidate.write_bytes(b'new:'+name.encode()) + files[name]={'before':hashlib.sha256(original).hexdigest() if original else None,'after':d.digest(candidate)} + (self.package/'dataset.json').write_text('{}') + (self.root/'commissioning/deploy_application.py').write_text('synthetic helper') + self.manifest={'scope':'historical_publication_only','installationId':'plant','files':files, + 'existingDeploymentHelperSha256':d.digest(self.root/'commissioning/deploy_application.py'), + 'datasetSha256':d.digest(self.package/'dataset.json'),'installerSha256':d.digest(SCRIPT)} + (self.package/'RELEASE.json').write_text(json.dumps(self.manifest)) + self.context=patch.multiple(d,ROOT=self.root,PACKAGE=self.package);self.context.start() + self.calls=[] + def tearDown(self):self.context.stop();self.tmp.cleanup() + def fake_run(self,args,**kwargs): + self.calls.append(args) + if 'inspect' in args:out='sha256:test-old-image' + elif 'ps' in args:out='test-container' + elif 'exec' in args:out=json.dumps({'settingsUnchanged':True,'liveEnabled':False}) + else:out='' + return subprocess.CompletedProcess(args,0,out,'') + def execute(self,fail_at=None): + def run(args,**kwargs): + out=self.fake_run(args,**kwargs) + if fail_at and fail_at in args:raise subprocess.CalledProcessError(1,args) + return out + def atomic(path,data,*args,**kwargs):path.write_bytes(data) + def backup(source,dest):self.calls.append(['test-backup']);dest.write_text('synthetic backup') + helpers=types.SimpleNamespace(atomic=atomic,backup_database=backup) + with patch.object(d.os,'geteuid',return_value=0),patch.object(d.os,'chown'),patch.object(d.subprocess,'run',side_effect=run),patch.dict(sys.modules,{'deploy_application':helpers}): + d.run('plant',True) + def report(self):return json.loads(next(self.root.glob('history-timing-releases/*/REPORT.json')).read_text()) + def test_default_checks_only(self): + d.run('plant');self.assertEqual(self.calls,[]) + self.assertFalse((self.root/'history-timing-releases').exists()) + def test_order_tests_backup_then_only_v4_replacement(self): + self.execute() + test=next(i for i,c in enumerate(self.calls) if '/app/run_tests.py' in c) + backup=next(i for i,c in enumerate(self.calls) if c==['test-backup']) + up=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(test,backup);self.assertLess(backup,up) + self.assertTrue(all('license-portal' not in c and 'forecast-engine' not in c for c in self.calls)) + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_failed_tests_restore_host_source_without_restart(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('/app/run_tests.py') + self.assertFalse(any('up' in c for c in self.calls)) + for name,v in self.manifest['files'].items():self.assertEqual(d.digest(self.root/name),v['before']) + def test_failed_registration_rolls_image_back(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('exec') + self.assertIn('previous_image_restored',self.report()['rollback']) + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_source_drift_refused_before_actions(self): + (self.root/d.TARGETS[0]).write_text('parallel change') + with self.assertRaises(ValueError):d.verify() + self.assertEqual(self.calls,[]) + def test_source_already_installed_idempotent(self): + for name in d.TARGETS:(self.root/name).write_bytes((self.package/'source'/name).read_bytes()) + self.assertEqual(d.verify()['scope'],'historical_publication_only') + self.execute() + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_no_production_selection_or_permission_in_bootstrap(self): + self.assertNotIn("'/settings'",d.BOOTSTRAP) + self.assertNotIn('/trial/arm',d.BOOTSTRAP) + self.assertIn("before['settings']==after['settings']",d.BOOTSTRAP) + self.assertIn("historyTimingVersion",d.BOOTSTRAP) + def test_target_image_contains_installer_test_dependency(self): + dockerfile=(SCRIPT.parents[1]/'Dockerfile').read_text() + self.assertIn('COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py',dockerfile) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/commissioning/unified-release/.gitignore b/services/netplan-v4/commissioning/unified-release/.gitignore new file mode 100644 index 0000000..2b3f20c --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/.gitignore @@ -0,0 +1 @@ +context/ diff --git a/services/netplan-v4/commissioning/unified-release/RELEASE.json b/services/netplan-v4/commissioning/unified-release/RELEASE.json new file mode 100644 index 0000000..611d7e0 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/RELEASE.json @@ -0,0 +1,414 @@ +{ + "scope": "unified_v4_candidate", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "files": { + ".dockerignore": { + "before": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "after": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790" + }, + "Dockerfile": { + "before": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", + "after": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33" + }, + "acceptance/Dockerfile.php": { + "before": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "after": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062" + }, + "acceptance/check_forecast.py": { + "before": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "after": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5" + }, + "acceptance/forecast-src/SOURCE_MANIFEST.json": { + "before": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "after": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5" + }, + "acceptance/forecast-src/main.py": { + "before": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "after": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c" + }, + "acceptance/forecast-src/methods/__init__.py": { + "before": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "after": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "acceptance/forecast-src/methods/battery_optimizer.py": { + "before": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "after": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6" + }, + "acceptance/forecast-src/methods/common.py": { + "before": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "after": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8" + }, + "acceptance/forecast-src/methods/var_1.py": { + "before": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "after": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4" + }, + "acceptance/forecast-src/methods/var_10.py": { + "before": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "after": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7" + }, + "acceptance/forecast-src/methods/var_11.py": { + "before": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "after": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc" + }, + "acceptance/forecast-src/methods/var_13.py": { + "before": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "after": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9" + }, + "acceptance/forecast-src/methods/var_2.py": { + "before": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "after": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a" + }, + "acceptance/forecast-src/methods/var_21.py": { + "before": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "after": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15" + }, + "acceptance/forecast-src/methods/var_22.py": { + "before": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "after": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6" + }, + "acceptance/forecast-src/methods/var_23.py": { + "before": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "after": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678" + }, + "acceptance/forecast-src/methods/var_3.py": { + "before": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "after": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984" + }, + "acceptance/forecast-src/model_isolation.py": { + "before": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "after": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5" + }, + "acceptance/forecast-src/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "acceptance/forecast-src/requirements.txt": { + "before": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "after": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083" + }, + "acceptance/forecast-src/shared_utils.py": { + "before": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "after": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57" + }, + "acceptance/forecast-src/soc_diagnostics.py": { + "before": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "after": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a" + }, + "acceptance/forecast-src/telemetry_quality.py": { + "before": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "after": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810" + }, + "acceptance/forecast-src/tests/test_battery_optimizer.py": { + "before": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "after": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8" + }, + "acceptance/forecast-src/tests/test_load_forecast.py": { + "before": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "after": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c" + }, + "acceptance/forecast-src/tests/test_model_isolation.py": { + "before": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "after": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30" + }, + "acceptance/forecast-src/tests/test_telemetry_integrity.py": { + "before": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "after": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" + }, + "acceptance/php-src/SOURCE_MANIFEST.json": { + "before": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "after": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50" + }, + "acceptance/php-src/check.php": { + "before": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "after": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473" + }, + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": { + "before": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "after": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7" + }, + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": { + "before": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "after": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9" + }, + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": { + "before": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "after": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a" + }, + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": { + "before": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "after": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" + }, + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": { + "before": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "after": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c" + }, + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": { + "before": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "after": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6" + }, + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": { + "before": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "after": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2" + }, + "acceptance/read_native_forecasts.py": { + "before": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "after": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f" + }, + "approved_previous_assets.json": { + "before": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "after": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0" + }, + "commissioning/APPLICATION_STATUS.md": { + "before": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "after": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc" + }, + "commissioning/application-source/server-dataset.json": { + "before": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "after": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d" + }, + "commissioning/deploy_application.py": { + "before": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "after": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e" + }, + "commissioning/deploy_history_timing.py": { + "before": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "after": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb" + }, + "commissioning/deploy_unified_release.py": { + "before": null, + "after": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e" + }, + "compose.portal-bridge.yaml": { + "before": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "after": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197" + }, + "compose.yaml": { + "before": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "after": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607" + }, + "deploy_integrated_shadow.py": { + "before": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "after": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898" + }, + "forecast_acceptance.py": { + "before": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "after": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9" + }, + "gui/netplan-v4.css": { + "before": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "after": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd" + }, + "gui/netplan-v4.html": { + "before": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "after": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755" + }, + "gui/netplan-v4.js": { + "before": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "after": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9" + }, + "install_hooks.py": { + "before": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "after": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3" + }, + "integrations/NetzfahrplanV4.php": { + "before": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "after": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3" + }, + "integrations/netplan-v4-bridge.mjs": { + "before": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "after": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7" + }, + "integrations/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "netplan_v4/__init__.py": { + "before": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "after": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48" + }, + "netplan_v4/battery_model.py": { + "before": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "after": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332" + }, + "netplan_v4/controlled_trial.py": { + "before": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "after": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4" + }, + "netplan_v4/domain.py": { + "before": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "after": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b" + }, + "netplan_v4/economic_replay.py": { + "before": null, + "after": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302" + }, + "netplan_v4/forecast_quality.py": { + "before": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "after": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840" + }, + "netplan_v4/history_timing.py": { + "before": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "after": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f" + }, + "netplan_v4/mapping_identity.py": { + "before": null, + "after": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d" + }, + "netplan_v4/measurement_pipeline.py": { + "before": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", + "after": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f" + }, + "netplan_v4/meter_runtime.py": { + "before": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "after": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547" + }, + "netplan_v4/metering.py": { + "before": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "after": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987" + }, + "netplan_v4/optimizer.py": { + "before": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "after": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063" + }, + "netplan_v4/peak_policy.py": { + "before": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "after": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576" + }, + "netplan_v4/receiver_contract.py": { + "before": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "after": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c" + }, + "netplan_v4/retention.py": { + "before": null, + "after": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3" + }, + "netplan_v4/selection.py": { + "before": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "after": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad" + }, + "netplan_v4/service.py": { + "before": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "after": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f" + }, + "netplan_v4/store.py": { + "before": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "after": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2" + }, + "release_preflight.py": { + "before": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "after": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5" + }, + "requirements.txt": { + "before": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "after": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464" + }, + "run_tests.py": { + "before": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "after": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4" + }, + "runtime_preflight.py": { + "before": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "after": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1" + }, + "tests/manager_protocol.php": { + "before": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "after": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93" + }, + "tests/portal.test.mjs": { + "before": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "after": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a" + }, + "tests/test_application_deployment.py": { + "before": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "after": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5" + }, + "tests/test_battery_recovery.py": { + "before": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "after": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3" + }, + "tests/test_container_access.py": { + "before": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "after": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9" + }, + "tests/test_controlled_trial.py": { + "before": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "after": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64" + }, + "tests/test_controlled_trial_pipeline.py": { + "before": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "after": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d" + }, + "tests/test_delivery.py": { + "before": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "after": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09" + }, + "tests/test_economic_replay.py": { + "before": null, + "after": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3" + }, + "tests/test_forecast_acceptance.py": { + "before": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "after": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76" + }, + "tests/test_forecast_quality.py": { + "before": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "after": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82" + }, + "tests/test_history_timing.py": { + "before": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "after": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2" + }, + "tests/test_history_timing_deployment.py": { + "before": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", + "after": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e" + }, + "tests/test_integrated_deployment.py": { + "before": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "after": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883" + }, + "tests/test_mapping_recovery.py": { + "before": null, + "after": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e" + }, + "tests/test_measurement_pipeline.py": { + "before": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "after": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4" + }, + "tests/test_metering.py": { + "before": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "after": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d" + }, + "tests/test_peak_release.py": { + "before": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "after": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d" + }, + "tests/test_receiver_contract.py": { + "before": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "after": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e" + }, + "tests/test_release_preflight.py": { + "before": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "after": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68" + }, + "tests/test_retention.py": { + "before": null, + "after": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51" + }, + "tests/test_unified_release.py": { + "before": null, + "after": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236" + }, + "tests/test_v4.py": { + "before": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", + "after": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + } + }, + "installerSha256": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", + "guiBefore": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "proofSha256": "11cb827ab65611bbb06bf8a181d0f44a16202ae957d82f56e0a18a916325128d", + "backupHelperSha256": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "autoArchiveDefault": false, + "mappingRegistrationRequiresExplicitApproval": true, + "productionReady": false, + "releaseId": "aeff5f094f6c8fbd7dc962d2d091d364a14c45e10fc7bd2f4847dd6dba329a22" +} diff --git a/services/netplan-v4/commissioning/unified-release/build_context.py b/services/netplan-v4/commissioning/unified-release/build_context.py new file mode 100644 index 0000000..ae11ab4 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/build_context.py @@ -0,0 +1,28 @@ +"""Reproduce the frozen candidate context from the versioned application sources. + +No Docker/runtime/secret reads. Existing output is verified, never silently replaced. +""" +from pathlib import Path +import hashlib,json,shutil + +PACKAGE=Path(__file__).resolve().parent +ROOT=PACKAGE.parents[1] + +def build(): + manifest=json.loads((PACKAGE/'RELEASE.json').read_text()) + context=PACKAGE/'context' + for name,v in manifest['files'].items(): + if Path(name).is_absolute() or '..' in Path(name).parts:raise ValueError('Unsafe source path') + source=ROOT/name;target=context/name + if source.is_symlink() or not source.is_file() or hashlib.sha256(source.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Checked-out source does not match release: '+name) + if target.is_symlink():raise ValueError('Target symlink refused') + if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Existing context differs: '+name) + for name in manifest['files']: + target=context/name + if target.exists():continue + target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(ROOT/name,target);target.chmod(0o644) + print('Frozen build context verified:',len(manifest['files']),'source files; no installation.') + +if __name__=='__main__':build() diff --git a/services/netplan-v4/commissioning/unified-release/mapping-proof.json b/services/netplan-v4/commissioning/unified-release/mapping-proof.json new file mode 100644 index 0000000..382cdca --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/mapping-proof.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "canonicalJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100.0},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}", + "legacyJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}" +} diff --git a/services/netplan-v4/compose.portal-bridge.yaml b/services/netplan-v4/compose.portal-bridge.yaml new file mode 100644 index 0000000..0bf8355 --- /dev/null +++ b/services/netplan-v4/compose.portal-bridge.yaml @@ -0,0 +1,6 @@ +services: + license-portal: + environment: + NETPLAN_V4_URL: http://netplan-v4:9100 + volumes: + - /home/agent/services/license/netplan-v4-bridge.mjs:/app/netplan-v4-bridge.mjs:ro diff --git a/services/netplan-v4/compose.yaml b/services/netplan-v4/compose.yaml new file mode 100644 index 0000000..84c7075 --- /dev/null +++ b/services/netplan-v4/compose.yaml @@ -0,0 +1,37 @@ +name: enelix-netplan-v4-shadow +services: + netplan-v4: + build: . + restart: unless-stopped + init: true + user: "1000:1000" + read_only: true + env_file: + - /home/agent/services/prognosis-integration.env + environment: + NETPLAN_V4_DB: /data/netplan-v4.sqlite + NETPLAN_V4_PLANTS: ${NETPLAN_V4_PLANTS:?Explicit installation ID required} + NETPLAN_V4_TEST_REPORT_DIR: /tmp/test-results + volumes: + - ./data:/data + tmpfs: + - /tmp:size=64m,mode=1777,noexec,nosuid + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + networks: + prognosis_integration: + aliases: + - netplan-v4 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9100/health', timeout=3).read()"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + # No public port, no Docker socket, no access to legacy databases. +networks: + prognosis_integration: + external: true + name: enelix-platform-internal diff --git a/services/netplan-v4/deploy_integrated_shadow.py b/services/netplan-v4/deploy_integrated_shadow.py new file mode 100644 index 0000000..6e4d614 --- /dev/null +++ b/services/netplan-v4/deploy_integrated_shadow.py @@ -0,0 +1,175 @@ +"""Install the reviewed server-side SHADOW integration; no Symcon/device writes. + +Default: verify source and print scope only. --apply requires root. Build and test +all changed code before recreating services. Save old image/config references and +an integrity-checked online backup of the V4 DB. Does not enable native telemetry +or dispatch: operation ingress may still be awaiting installation on Symcon. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from uuid import UUID + +ROOT = Path(__file__).resolve().parent +SERVICES = ROOT.parent +PROJECT = SERVICES / 'prognosis-manager-enelix2' +PORTAL = SERVICES / 'license' +PREFLIGHT = ROOT / 'acceptance-reports/20261002T043021Z/REPORT.json' +MANIFEST = ROOT / 'INTEGRATED_SHADOW_SOURCE.json' + + +def verify_sources(): + report = json.loads(PREFLIGHT.read_text()) + if report.get('preflightChecksPassed') is not True: + raise ValueError('Expected successful earlier runtime preflight is missing') + expected = json.loads(MANIFEST.read_text())['files'] + for relative, digest in expected.items(): + path = SERVICES / relative + if not path.resolve().is_relative_to(SERVICES) or path.is_symlink() or not path.is_file(): + raise ValueError('Invalid release source path: ' + relative) + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError('Source changed since preparation: ' + relative) + from forecast_acceptance import verify_forecast_bundle + verify_forecast_bundle(ROOT/'acceptance/forecast-src', PROJECT/'forecast_engine') + from release_preflight import verify_native_bundle + verify_native_bundle() + return expected + + +def compose_groups(): + return { + 'v4': (ROOT, [ROOT/'compose.yaml'], ['netplan-v4']), + 'forecast': (PROJECT, [PROJECT/'compose.yaml', ROOT/'compose.forecast-bridge.yaml'], ['api','forecast-engine','tariff-importer']), + 'portal': (PORTAL, [PORTAL/'compose.yaml', ROOT/'compose.portal-bridge.yaml'], ['license-portal']), + } + + +def compose_args(files): + result=['docker','compose'] + for path in files:result.extend(['-f',str(path)]) + return result + + +def backup_database(source, destination): + if not source.exists():return {'exists':False} + src=sqlite3.connect(source.resolve().as_uri()+'?mode=ro',uri=True,timeout=15) + dst=sqlite3.connect(destination,timeout=15) + try: + src.backup(dst) + if dst.execute('PRAGMA integrity_check').fetchone()[0]!='ok': + raise ValueError('V4 database backup failed integrity check') + finally:dst.close();src.close() + os.chmod(destination,0o640) + if os.geteuid()==0:os.chown(destination,1000,1000) + return {'exists':True,'path':str(destination),'sha256':hashlib.sha256(destination.read_bytes()).hexdigest(),'integrity':'ok'} + + +def save_json(path, value): + path.write_text(json.dumps(value,indent=2,allow_nan=False)+'\n') + os.chmod(path,0o640) + if os.geteuid()==0:os.chown(path,1000,1000) + + +def run(plant): + sources=verify_sources() + env=dict(os.environ,NETPLAN_V4_PLANTS=plant) + stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + folder=ROOT/'deployment-reports'/stamp + folder.mkdir(parents=True,exist_ok=False) + if os.geteuid()==0: + os.chown(folder.parent,1000,1000);os.chown(folder,1000,1000) + report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant, + 'scope':'server_shadow_only','liveEnabled':False,'productionReady':False, + 'sourceHashes':sources,'steps':[],'before':{},'existingServiceRecreated':False} + def command(args,cwd=ROOT,capture=False,stdin=None,timeout=900): + result=subprocess.run(args,cwd=cwd,env=env,input=stdin,text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None,timeout=timeout,check=False) + if result.returncode: + raise RuntimeError('Step failed (exit '+str(result.returncode)+'): '+args[0]) + return result.stdout if capture else '' + def step(name,args,cwd=ROOT,timeout=900): + print('\n=== '+name+' ===',flush=True) + command(args,cwd,timeout=timeout) + report['steps'].append(name);save_json(folder/'DEPLOYMENT.json',report) + try: + # Capture only selected nonsecret Docker fields, never the environment. + for group,(cwd,files,services) in compose_groups().items(): + oldfiles=[files[0]] + items=[] + for service in services: + cid=command(compose_args(oldfiles)+['ps','-q',service],cwd,True).strip() + if not cid or '\n' in cid:raise ValueError('Expected exactly one existing container: '+service) + image=command(['docker','inspect','--format','{{.Image}}',cid],cwd,True).strip() + configs=command(['docker','inspect','--format','{{index .Config.Labels "com.docker.compose.project.config_files"}}',cid],cwd,True).strip() + items.append({'service':service,'containerId':cid,'imageId':image,'previousComposeFiles':configs}) + report['before'][group]=items + save_json(folder/'DEPLOYMENT.json',report) + step('review optional source installer',[sys.executable,str(ROOT/'install_hooks.py')]) + # All new forecasting tests run offline in a disposable Python 3.11 image. + step('build forecast acceptance image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')]) + step('forecast candidate offline tests',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local']) + step('V4 and bridge target tests',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only']) + verify_sources() + from install_hooks import plan,apply + receipt=apply(plan()) + report['sourceInstallReceipt']=str(receipt) if receipt else None + step('portal syntax',['node','--check',str(PORTAL/'server.mjs')]) + # Build every production-shaped candidate before changing a running container. + cwd,files,svcs=compose_groups()['forecast'] + step('build server-side images',compose_args(files)+['build']+svcs,cwd) + report['v4DatabaseBackup']=backup_database(ROOT/'data/netplan-v4.sqlite',folder/'netplan-v4-before.sqlite') + report['existingServiceRecreated']=True + save_json(folder/'DEPLOYMENT.json',report) + for group,(cwd,files,svcs) in compose_groups().items(): + step('start shadow integration '+group,compose_args(files)+['up','-d','--no-deps','--no-build','--wait','--wait-timeout','180']+svcs,cwd,timeout=300) + cwd,files,_=compose_groups()['v4'] + # Check shadow guard and report incoming types without disclosing payloads. + probe='''import json,os,sqlite3,urllib.request +from datetime import datetime,timezone +plant=os.environ['NETPLAN_V4_PLANTS'] +health=json.loads(urllib.request.urlopen('http://127.0.0.1:9100/health',timeout=5).read()) +assert health.get('mode')=='shadow' and health.get('liveEnabled') is False +con=sqlite3.connect('file:/data/netplan-v4.sqlite?mode=ro',uri=True) +rows=con.execute('SELECT kind,count(*) FROM planner_inputs WHERE plant=? GROUP BY kind',(plant,)).fetchall() +status=con.execute('SELECT status FROM planner_run_status WHERE plant=?',(plant,)).fetchone() +con.close() +print(json.dumps({'checkedAt':datetime.now(timezone.utc).isoformat(),'health':health,'inputEventsByKind':dict(rows),'lastRunStatus':status[0] if status else None,'nativeSenderInstalledByThisCommand':False})) +''' + report['readiness']=json.loads(command(compose_args(files)+['exec','-T','netplan-v4','python','-B','-'],cwd,True,probe,60)) + report['status']='server_shadow_installed' + report['remaining']='Native operation sender, plant-level initialization and real shadow-plan acceptance still required; no live dispatch installed' + except Exception as error: + report['status']='failed';report['errorType']=type(error).__name__;report['error']=str(error)[:500] + report['remaining']='Inspect completed steps and before-image references. No automatic database restore or device changes.' + raise + finally: + save_json(folder/'DEPLOYMENT.json',report) + print('\nDEPLOYMENT REPORT: '+str(folder/'DEPLOYMENT.json'),flush=True) + print('SERVER SHADOW ONLY. Symcon, device permissions and actuators were not modified.',flush=True) + print(json.dumps(report['readiness'],indent=2)) + print('Native operating-state input may still be missing; this is NOT production commissioning.') + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda p:str(UUID(p))) + parser.add_argument('--apply',action='store_true') + args=parser.parse_args() + if args.plant!='e3a08f9e-af12-4695-99bd-8b51c0520021': + raise SystemExit('This first integration rollout is allowlisted for Lihrenmoos only.') + if not args.apply: + files=verify_sources() + print('CHECK ONLY:',len(files),'reviewed source files. No running service changed.') + for group,(_,_,svcs) in compose_groups().items():print(group,', '.join(svcs)) + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not change Docker socket access.') + try:run(args.plant) + except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + raise SystemExit('Deployment stopped: '+str(error)) diff --git a/services/netplan-v4/forecast_acceptance.py b/services/netplan-v4/forecast_acceptance.py new file mode 100644 index 0000000..686765f --- /dev/null +++ b/services/netplan-v4/forecast_acceptance.py @@ -0,0 +1,38 @@ +"""Verify the separately staged forecast repair before target-runtime testing. +Only explicit Python source and requirements are copied. No models, credentials, +live databases or telemetry are needed by this offline test image. +""" +from pathlib import Path +import hashlib +import json + + +def source_paths(root): + root = Path(root) + paths = [root / 'requirements.txt'] + list(root.glob('*.py')) + for folder in ('methods', 'tests'): + paths.extend((root / folder).glob('*.py')) + if not (root / 'telemetry_quality.py').is_file(): + raise ValueError('Forecast telemetry repair missing') + for path in paths: + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError('External forecast source is not allowed') + return sorted(paths) + + +def verify_forecast_bundle(bundle, source): + bundle, source = Path(bundle), Path(source) + manifest_path = bundle / 'SOURCE_MANIFEST.json' + manifest = json.loads(manifest_path.read_text()) + expected_paths = {str(p.relative_to(source)) for p in source_paths(source)} + if not isinstance(manifest, dict) or not manifest or set(manifest) != expected_paths: + raise ValueError('Forecast source set changed; rebuild reviewed test bundle') + if {str(p.relative_to(bundle)) for p in source_paths(bundle)} != expected_paths: + raise ValueError('Staged forecast source set does not match development source') + for relative, expected in manifest.items(): + if Path(relative).is_absolute() or '..' in Path(relative).parts: + raise ValueError('Unsafe forecast manifest path') + for path in (source / relative, bundle / relative): + if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise ValueError('Forecast source changed after staging; review before test') + return dict(manifest) diff --git a/services/netplan-v4/gui/netplan-v4.css b/services/netplan-v4/gui/netplan-v4.css new file mode 100644 index 0000000..2460732 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.css @@ -0,0 +1 @@ +body{font:16px system-ui,sans-serif;background:#eef4f4;color:#243c42}main{max-width:1100px;margin:2rem auto;padding:1.5rem;background:white}a{color:#006b83}label{display:block;margin:.8rem 0}select,button{font:inherit;padding:.6rem;margin:.3rem;border:1px solid #bacaca;border-radius:.25rem}button{background:#006b83;color:white;cursor:pointer}.notice{background:#e9f7fa;padding:1rem;border-left:4px solid #006b83}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#eef4f4;padding:1rem}svg{width:100%;height:auto}.curve{fill:none;stroke-width:2}.baseline{stroke:#7d888c}.optimized{stroke:#006b83}.axis{stroke:#b7c8cc}button:disabled{opacity:.5} diff --git a/services/netplan-v4/gui/netplan-v4.html b/services/netplan-v4/gui/netplan-v4.html new file mode 100644 index 0000000..1a96991 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.html @@ -0,0 +1 @@ +ENELIX Netzfahrplan V4
Zurueck zum Portal

Netzfahrplan V4

Schattenbetrieb. Diese Ansicht steuert keine Verbraucher. Die bestehende Regelung bleibt unveraendert.

diff --git a/services/netplan-v4/gui/netplan-v4.js b/services/netplan-v4/gui/netplan-v4.js new file mode 100644 index 0000000..7e76572 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.js @@ -0,0 +1,113 @@ +const message = document.getElementById('message'); +const panel = document.getElementById('panel'); +const plants = document.getElementById('plants'); +let session, generation = 0; +const element = (tag, text) => { const e = document.createElement(tag); if (text !== undefined) e.textContent = text; return e; }; +const time = value => value ? new Date(value).toLocaleString('de-CH', { timeZone: 'Europe/Zurich' }) : 'nicht vorhanden'; +const num = (v, digits = 2) => typeof v === 'number' && Number.isFinite(v) ? v.toLocaleString('de-CH', { maximumFractionDigits: digits }) : 'unbekannt'; + +async function request(url, method = 'GET', body) { + const headers = {}; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + if (method !== 'GET') headers['X-CSRF-Token'] = session.csrfToken; + const response = await fetch(url, { method, headers, credentials: 'same-origin', body: body === undefined ? undefined : JSON.stringify(body) }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || data.detail || 'Anfrage fehlgeschlagen'); + return data; +} +function select(label, choices, selected) { + const wrap = element('label', label + ' '), field = element('select'); + for (const [value, caption] of choices) { const option = element('option', caption); option.value = value; field.append(option); } + field.value = selected; wrap.append(field); return { wrap, field }; +} +function table(title, rows) { + panel.append(element('h3', title)); const grid = element('table'); + for (const [name, value] of rows) { const row = element('tr'); row.append(element('th', name), element('td', String(value))); grid.append(row); } + panel.append(grid); +} +function curve(title, points, series) { + panel.append(element('h3', title)); + if (!points.length) { panel.append(element('p', 'Keine Daten.')); return; } + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + svg.setAttribute('viewBox', '0 0 1000 210'); svg.setAttribute('role', 'img'); svg.setAttribute('aria-label', title); + const values = points.flatMap(p => series.map(s => s.value(p))).filter(v => typeof v === 'number' && Number.isFinite(v)); + if (!values.length) { panel.append(element('p', 'Keine gueltigen Werte.')); return; } + const low = Math.min(0, ...values), high = Math.max(1, ...values); + const start = Date.parse(points[0].time), end = Date.parse(points[points.length - 1].validUntil || points[points.length - 1].time); + for (const [i, source] of series.entries()) { + let section = []; + const flush = () => { if (!section.length) return; const line = document.createElementNS(svg.namespaceURI, 'polyline'); line.setAttribute('class', 'curve ' + (i ? 'optimized' : 'baseline')); line.setAttribute('points', section.join(' ')); svg.append(line); section = []; }; + for (const p of points) { + const value = source.value(p); + if (typeof value !== 'number' || !Number.isFinite(value)) { flush(); continue; } + const x = 10 + 980 * (Date.parse(p.time) - start) / Math.max(1, end - start); + const y = 190 - (value - low) / (high - low) * 180; section.push(`${x},${y}`); + } + flush(); + } + panel.append(svg, element('p', `${num(low)} bis ${num(high)}. ${series.map((s, i) => (i ? 'Blau: ' : 'Grau: ') + s.label).join('; ')}. ${time(points[0].time)} – ${time(points[points.length - 1].validUntil)}.`)); +} +async function render() { + const ticket = ++generation, id = plants.value; if (!id) return; + const base = `/api/plants/${encodeURIComponent(id)}/prognosis/planner-v4`; + message.textContent = 'Lade Planungsstand ...'; + try { + const state = await request(base); if (ticket !== generation) return; panel.replaceChildren(); + panel.append(element('p', 'SCHATTENBETRIEB: Die angezeigten Vorgaben steuern keine Geraete.')); + const form = element('form'); + const family = select('Modellfamilie', [['auto', 'Automatisch – historischer Kostenvergleich'], ...state.families.map(f => [f.key, f.label])], state.settings.family); + const measurement = select('Messdatenbasis', [['verified_only', 'Nur verifizierte Peak-/Viertelstundenwerte'], ['allow_estimates', 'Gekennzeichnete Schaetzungen fuer die Planung zulassen']], state.settings.measurementPolicy || 'verified_only'); + const outlook = select('Peakbewertung', [['full_incremental', 'Voller zusaetzlicher Monatstarif'], ['empirical_if_available', 'Restmonats-Szenarien bei ausreichender Historie']], state.settings.peakOutlookPolicy || 'full_incremental'); + const dataSource = select('Verbrauchsprognose', [['legacy', 'Bisherige Datenquelle'], ['corrected_profile', 'Bereinigte physische Lastprofile']], state.settings.forecastSource || 'legacy'); + const datasets = state.dataPipeline?.datasets || []; + const dataset = select('Messdatensatz', [['', 'Datensatz auswaehlen'], ...datasets.map(d => [d.datasetId, d.datasetId + ' (' + d.status + ')'])], state.settings.measurementDataset || ''); + const cadence = select('Nachtraining', [['daily', 'Taeglich'], ['weekly', 'Woechentlich']], state.settings.trainingCadence); + const relianceWrap = element('label', 'Gewicht der Restmonatsprognose (0 bis 100 %) '), reliance = element('input'); + reliance.type = 'number'; reliance.min = '0'; reliance.max = '100'; reliance.step = '1'; reliance.value = String(100 * (state.settings.peakOutlookReliance ?? .5)); relianceWrap.append(reliance); + const save = element('button', 'Speichern und neu berechnen'), replan = element('button', 'Neu berechnen'), refresh = element('button', 'Status aktualisieren'); + save.type = 'submit'; replan.type = refresh.type = 'button'; + form.append(family.wrap, dataSource.wrap, dataset.wrap, measurement.wrap, outlook.wrap, relianceWrap, cadence.wrap, save, replan, refresh); panel.append(form); + let revision = state.settings.revision; + form.addEventListener('submit', async event => { + event.preventDefault(); save.disabled = true; + try { + const r = await request(base + '/settings', 'PUT', { expectedRevision: revision, changes: { family: family.field.value, forecastSource: dataSource.field.value, measurementDataset: dataset.field.value, measurementPolicy: measurement.field.value, peakOutlookPolicy: outlook.field.value, peakOutlookReliance: Number(reliance.value) / 100, trainingCadence: cadence.field.value } }); + revision = r.revision; message.textContent = `Revision ${revision} gespeichert. Neuberechnung angefordert, noch nicht abgeschlossen.`; + } catch (error) { message.textContent = error.message; } finally { save.disabled = false; } + }); + replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } }); + refresh.addEventListener('click', render); + panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.')); + panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik erstellt vergleichbare Kosten-Replays aus eingefrorenen Tagesplaenen und spaeter eingegangenen Messwerten. Ohne ausreichende Vergleichstage bleibt die bisherige Familie aktiv.')); + for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Letzter Serverempfang', time(d.lastReceived)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + if (state.economicComparison?.connected) { + table('Wirtschaftlicher Modellvergleich', [['Gespeicherte Tagesvergleiche', state.economicComparison.cohorts], ['Abgeschlossene Vergleiche', state.economicComparison.completedComparisons.length], ['Methode', 'Eingefrorener Tagesplan mit simulierter Netzregelung'], ['Grenze dieses Vergleichs', 'Eine netzladefaehige Batterie; keine vollstaendige Nachbildung laufender Neuoptimierungen'], ['Bewertung', 'Energiekosten + zusaetzlicher Peak + Verluste + gemeinsame Restenergiebewertung']]); + panel.append(element('p', 'Der SDL-Anteil der historischen Bilanz ist eine gekennzeichnete Schaetzung aus dem damaligen Auftrag. Die Vergleichskosten sind keine nachgewiesenen Einsparungen auf einer Stromrechnung.')); + for (const item of state.economicComparison.completedComparisons) table('Vergleich ' + item.day, Object.entries(item.results).map(([family, result]) => ['Familie ' + family, num(result.costChf) + ' CHF; Datenabdeckung ' + num(100 * result.coverage, 1) + ' %'])); + } + if (state.maintenance) table('Datenaufbewahrung', [['Archivierung', state.maintenance.status], ['Originale', 'Nach verifizierter Kompression weiterhin wiederherstellbar; externe Sicherung separat betreiben']]); + panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.')); + if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.')); + for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]); + const plan = state.plan; + if (plan) { + if (!state.fresh) panel.append(element('p', 'ACHTUNG: Plan veraltet oder Neuberechnung ausstehend.')); + table('Plan und Kosten', [['Plan-ID', plan.planId], ['Modellfamilie', plan.sourceFamily], ['Revision', plan.configRevision], ['Prognose bis', time(plan.forecastUntil)], ['Bekannte Preise bis', time(plan.pricesKnownUntil)], ['Energiekosten', num(plan.energyCostChf) + ' CHF'], ['Zusaetzliche Peakkosten zum vollen Tarif', num(plan.additionalPeakCostChf) + ' CHF'], ['Peakbewertung unter Restmonats-Szenarien', num(plan.planningPeakCostChf) + ' CHF'], ['Planung mit geschaetzter Peakbasis', plan.peakCostIsEstimate ? 'Ja – kein Abrechnungsnachweis' : 'Nein'], ['V4 steuert die Anlage', 'Nein']]); + for (const warning of plan.warnings || []) panel.append(element('p', warning)); + for (const [month, basis] of Object.entries(plan.peakBasis || {})) table(`Peak ${month}`, [['Ausgangsbasis', num(basis.kw) + ' kW (' + basis.quality + ')'], ['Geplant', num(plan.plannedPeaksKw?.[month]) + ' kW'], ['Bewertung', plan.peakOutlook]]); + curve('Netzleistung (W)', plan.points, [{ label: 'ohne Optimierung', value: p => p.baselineGridW }, { label: 'optimiert', value: p => p.gridTargetW }]); + curve('Batterieleistung (W; positiv = Laden)', plan.points, [{ label: 'Batterieplan', value: p => p.batteryTargetW }]); + for (const asset of Object.keys(plan.points[0]?.socEndPercent || {})) curve(`SOC ${asset} (%)`, plan.points, [{ label: 'geplanter SOC', value: p => p.socEndPercent?.[asset] }]); + curve('Kumulierte Zahlungen inklusive vollem Peakpreis (CHF)', plan.points, [{ label: 'ohne Optimierung', value: p => p.cumulativeBaselineCashCostChf }, { label: 'optimiert', value: p => p.cumulativeCashCostChf }]); + panel.append(element('p', 'Kostenverlaeufe sind Prognosen. Restwert der Batterie und Restmonatsbewertung sind keine bereits erzielten Erloese.')); + } + message.textContent = ''; + } catch (error) { if (ticket === generation) { panel.replaceChildren(); message.textContent = error.message; } } +} +try { + session = await request('/api/session'); if (!session.authenticated) throw new Error('Bitte zuerst im Portal anmelden.'); + const data = await request('/api/plants'); + for (const plant of data.plants || []) { if (!plant.installation_id) continue; const option = element('option', plant.name); option.value = plant.id; plants.append(option); } + if (!plants.options.length) throw new Error('Keine verknuepfte Anlage vorhanden.'); + plants.addEventListener('change', render); await render(); +} catch (error) { message.textContent = error.message; } diff --git a/services/netplan-v4/install_hooks.py b/services/netplan-v4/install_hooks.py new file mode 100644 index 0000000..b6e8c76 --- /dev/null +++ b/services/netplan-v4/install_hooks.py @@ -0,0 +1,131 @@ +"""Count-checked optional bridge installation. Default CHECK ONLY, no service action. +Only dedicated V4 GUI files are added. app.js/index.html/styles.css stay untouched. +Every changed legacy source is backed up with hashes. Concurrent changes abort. +""" +from pathlib import Path +from datetime import datetime,timezone +import argparse,ast,hashlib,json,os,tempfile +ROOT=Path(__file__).resolve().parent +SERVICES=Path('/home/agent/services') + +def once(text,old,new): + if text.count(old)!=1:raise ValueError('Source anchor changed/ambiguous; review before updating') + return text.replace(old,new,1) + +def patch_forecast(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts\n'+text + anchor=' p_3 = v3.predict(data_obj, p_1, p_2)' + text=once(text,anchor,' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)])\n\n'+anchor) + text=once(text,' d[key] = float(d.get(key) or default)',' raw_value = d.get(key)\n d[key] = float(default if raw_value is None or raw_value == "" else raw_value)') + ast.parse(text);return text + +def patch_api(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_tariffs as _v4_publish_tariffs\n'+text + anchor=' _portal_store_configuration(anlagen_id, configuration)\n' + text=once(text,anchor,anchor+' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_tariffs(anlagen_id, configuration)\n') + ast.parse(text);return text + +def patch_tariff(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_ckw as _v4_publish_ckw\n'+text + anchor=' return candidate_points\n' + text=once(text,anchor," # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_ckw(source['label'], rows, data.get('publication_timestamp') if isinstance(data, dict) else None, tariff_type)\n"+anchor) + for name in ('row','integrated'): + old=f'{name}.get("value") or {name}.get("price") or {name}.get("amount")' + if old in text:text=once(text,old,f'next(({name}[key] for key in ("value", "price", "amount") if {name}.get(key) is not None), None)') + ast.parse(text);return text + +def patch_portal(text): + if '// ENELIX_V4_SHADOW_BRIDGE' in text:return text + anchor='const server = createServer(async (req, res) => {' + extra="""// ENELIX_V4_SHADOW_BRIDGE +// No dependency or changed route while NETPLAN_V4_URL is absent. +const plannerV4Bridge = process.env.NETPLAN_V4_URL + ? (await import('./netplan-v4-bridge.mjs')).createPlannerV4Bridge({ + configuredPrognosisPlant, roleAllowed, bodyJson, json, deviceActivation, + checkDeviceRate, ownedLicenseState, serviceToken: prognosisServiceToken, + upstreamUrl: process.env.NETPLAN_V4_URL + }) + : null; + +""" + text=once(text,anchor,extra+anchor) + anchor=' if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {' + return once(text,anchor,' if (plannerV4Bridge && await plannerV4Bridge(req, res, url)) return;\n'+anchor) + +def plan(root=SERVICES, approved_assets=None): + root=Path(root).resolve();project=root/'prognosis-manager-enelix2';result={} + if approved_assets is None: + manifest=ROOT/'approved_previous_assets.json' + approved_assets=json.loads(manifest.read_text()) if manifest.is_file() else {} + for path,patch in [(project/'api/main.py',patch_api),(project/'forecast_engine/main.py',patch_forecast),(project/'tariff_importer/main.py',patch_tariff),(root/'license/server.mjs',patch_portal)]: + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes();new=patch(old.decode()).encode() + if old!=new:result[path]=(old,new) + files={project/service/'netplan_v4_publisher.py':ROOT/'integrations/netplan_v4_publisher.py' for service in ('api','forecast_engine','tariff_importer')} + files[root/'license/netplan-v4-bridge.mjs']=ROOT/'integrations/netplan-v4-bridge.mjs' + for name in ('netplan-v4.html','netplan-v4.js','netplan-v4.css'):files[root/'license/public'/name]=ROOT/'gui'/name + for path,source in files.items(): + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes() if path.exists() else None;new=source.read_bytes() + if old is not None and old!=new: + approved=approved_assets.get(str(path.relative_to(root)),[]) + if hashlib.sha256(old).hexdigest() not in approved: + raise ValueError('Existing V4 file differs from the reviewed previous release; explicit merge required: '+str(path)) + if old!=new:result[path]=(old,new) + return result + +def atomic(path,data,mode=0o644): + path.parent.mkdir(parents=True,exist_ok=True);fd,tmp=tempfile.mkstemp(prefix=path.name+'.v4-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as out:out.write(data);out.flush();os.fsync(out.fileno()) + os.chmod(tmp,mode);os.replace(tmp,path) + finally: + if os.path.exists(tmp):os.unlink(tmp) + +def apply(entries,root=SERVICES): + root=Path(root).resolve() + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent change: no files modified') + if not entries:return None + backup=root/'change-backups'/'netplan-v4'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + backup.mkdir(parents=True);receipt={'root':str(root),'files':[]};written=[] + try: + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent source change during install') + rel=path.relative_to(root);mode=path.stat().st_mode&0o777 if path.exists() else 0o644 + if old is not None: + original=backup/'originals'/rel;original.parent.mkdir(parents=True,exist_ok=True);original.write_bytes(old) + atomic(path,new,mode);written.append(path) + receipt['files'].append({'path':str(rel),'before':hashlib.sha256(old).hexdigest() if old is not None else None,'after':hashlib.sha256(new).hexdigest(),'mode':mode}) + atomic(backup/'receipt.json',json.dumps(receipt,indent=2).encode());return backup/'receipt.json' + except Exception: + for path in reversed(written): + old,new=entries[path] + if path.read_bytes()==new: + if old is None:path.unlink() + else:atomic(path,old) + raise + +def rollback(receipt_path): + path=Path(receipt_path).resolve();data=json.loads(path.read_text());root=Path(data['root']).resolve() + for item in data['files']: + target=(root/item['path']).resolve() + if not target.is_relative_to(root) or hashlib.sha256(target.read_bytes()).hexdigest()!=item['after']:raise ValueError('Later change detected; automatic rollback refused') + for item in reversed(data['files']): + target=root/item['path'] + if item['before'] is None:target.unlink() + else: + original=(path.parent/'originals'/item['path']).read_bytes() + if hashlib.sha256(original).hexdigest()!=item['before']:raise ValueError('Backup checksum mismatch') + atomic(target,original,item['mode']) + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--apply',action='store_true');parser.add_argument('--rollback');args=parser.parse_args() + if args.rollback:rollback(args.rollback);print('Source rollback completed. No service restart.') + else: + entries=plan() + for path in entries:print(path) + print('Receipt:',apply(entries)) if args.apply else print('CHECK ONLY:',len(entries),'files. No source/database/service changed.') diff --git a/services/netplan-v4/integrations/NetzfahrplanV4.php b/services/netplan-v4/integrations/NetzfahrplanV4.php new file mode 100644 index 0000000..5ffe258 --- /dev/null +++ b/services/netplan-v4/integrations/NetzfahrplanV4.php @@ -0,0 +1,71 @@ +getTimestamp(); + } + private static function number(mixed $value): float + { + if ((!is_float($value) && !is_int($value)) || !is_finite((float)$value)) { + throw new InvalidArgumentException('Finite numeric value required'); + } + return (float)$value; + } + public static function currentPoint(array $plan, int $now, int $revision, bool $shadowInspection=false): ?array + { + if (($plan['schemaVersion']??null)!==2 || ($plan['configRevision']??null)!==$revision + || ($plan['executable']??false)!==true || !is_array($plan['points']??null) + || !in_array($plan['status']??'', ['optimal','feasible_time_limit'], true) + || !is_string($plan['planId']??null) || $plan['planId']==='') return null; + if ($shadowInspection ? ($plan['runMode']??'')!=='shadow' + : (($plan['runMode']??'')!=='live' || ($plan['liveEnabled']??false)!==true)) return null; + try { + $age=$now-self::stamp($plan['generatedAt']); + if ($age < -30 || $age>900 || self::stamp($plan['validFrom'])>$now || self::stamp($plan['validUntil'])<=$now) return null; + $selected=null; $previousEnd=null; + foreach ($plan['points'] as $p) { + $start=self::stamp($p['time']); $end=self::stamp($p['validUntil']); + self::number($p['gridTargetW']); + if ($end<=$start || $end-$start>300 || ($previousEnd!==null && $previousEnd!==$start)) return null; + $previousEnd=$end; + if ($start<=$now && $now<$end) $selected=$p; + } + return $selected; + } catch (\Throwable $e) { return null; } + } + public static function correction(array $point, float $grid, float $battery, float $chargeAvailable, float $dischargeAvailable, ?float $hardImportLimit=null): array + { + foreach ([$grid,$battery,$chargeAvailable,$dischargeAvailable] as $v) self::number($v); + if ($chargeAvailable<0 || $dischargeAvailable<0) throw new InvalidArgumentException('Invalid available power'); + $target=self::number($point['gridTargetW']??null); + if ($hardImportLimit!==null) { + if (self::number($hardImportLimit)<0) throw new InvalidArgumentException('Negative hard limit'); + $target=min($target,$hardImportLimit); + } + $raw=$battery+$target-$grid; + if (abs($target)>1) { + $raw=match ($point['intent']??'') { + 'gridCharge'=>max(0.,$raw), + 'pvCharge'=>max(0.,min($battery-$grid,$raw)), + 'discharge','export'=>min(0.,$raw), + 'hold'=>0., + default=>throw new InvalidArgumentException('Unknown plan intent'), + }; + } + if ($hardImportLimit!==null) $raw=min($raw,$battery+$hardImportLimit-$grid); + $wanted=min($chargeAvailable,max(-$dischargeAvailable,$raw)); + $expectedGrid=$grid+$wanted-$battery; + return ['batteryTargetW'=>$wanted,'gridTargetW'=>$target,'limited'=>abs($wanted-$raw)>1, + 'expectedGridW'=>$expectedGrid,'trackingErrorW'=>$expectedGrid-$target]; + } +} diff --git a/services/netplan-v4/integrations/netplan-v4-bridge.mjs b/services/netplan-v4/integrations/netplan-v4-bridge.mjs new file mode 100644 index 0000000..1e8786e --- /dev/null +++ b/services/netplan-v4/integrations/netplan-v4-bridge.mjs @@ -0,0 +1,39 @@ +/** Scoped opt-in proxy. Never intercept the existing V1 schedule. */ +export function createPlannerV4Bridge({configuredPrognosisPlant,roleAllowed,bodyJson,json, + deviceActivation,checkDeviceRate,ownedLicenseState,serviceToken, + upstreamUrl='http://netplan-v4:9100',fetchImpl=fetch}){ + const base=new URL(upstreamUrl); + if(!['http:','https:'].includes(base.protocol)||base.username||base.password)throw new Error('Invalid V4 URL'); + async function forward(req,res,installation,suffix,method){ + if(!serviceToken){json(res,503,{error:'V4 ist nicht angebunden.'});return;} + const url=new URL(`/internal/v2/prognosis/${encodeURIComponent(installation)}/planner${suffix}`,base); + let response; + const payload=method==='GET'?undefined:await bodyJson(req); + try{response=await fetchImpl(url,{method,headers:{'X-Enelix-Service-Token':serviceToken,'Content-Type':'application/json'},body:payload===undefined?undefined:JSON.stringify(payload),signal:AbortSignal.timeout(10000),redirect:'error'});} + catch{json(res,503,{error:'V4 nicht erreichbar. Bestehende Regelung bleibt unveraendert.'});return;} + const data=await response.json().catch(()=>null); + if(!response.ok){const code=[400,401,403,404,409,413].includes(response.status)?response.status:503;json(res,code,{error:code===409?'Versionskonflikt: Ansicht neu laden.':'V4-Anfrage abgelehnt oder Dienst nicht verfuegbar.'});return;} + if(!data||typeof data!=='object'){json(res,503,{error:'Ungueltige V4-Antwort.'});return;} + json(res,200,data); + } + return async function(req,res,url){ + const c=url.pathname.match(/^\/api\/plants\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(settings|replan))?$/i); + if(c){ + const action=c[2]||'',method={'':'GET',settings:'PUT',replan:'POST'}[action]; + if(req.method!==method){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + const access=configuredPrognosisPlant(req,res,c[1],method!=='GET');if(!access)return true; + if(Number(access.license.quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + if(method!=='GET'&&!roleAllowed(access.session,['owner','admin','operator'])){json(res,403,{error:'Keine Aenderungsberechtigung.'});return true;} + await forward(req,res,access.plant.installation_id,action?`/${action}`:'',method);return true; + } + const d=url.pathname.match(/^\/api\/v1\/installations\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(operation|ack|measurements))?$/i); + if(d){ + const action=d[2]||'';if(req.method!==(action?'POST':'GET')){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + if(!checkDeviceRate(req,d[1],'planner-v4')){json(res,429,{error:'Zu viele Anfragen.'});return true;} + const activation=deviceActivation(req,res,d[1]);if(!activation)return true; + if(Number(ownedLicenseState(activation.plant_id).quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + await forward(req,res,d[1],action==='operation'?'/inputs/operation':action==='measurements'?'/measurements':action?'/ack':'',req.method);return true; + } + return false; + }; +} diff --git a/services/netplan-v4/integrations/netplan_v4_publisher.py b/services/netplan-v4/integrations/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/integrations/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/netplan_v4/__init__.py b/services/netplan-v4/netplan_v4/__init__.py new file mode 100644 index 0000000..21e0e8d --- /dev/null +++ b/services/netplan-v4/netplan_v4/__init__.py @@ -0,0 +1 @@ +"""ENELIX V4 shadow planner. No live actuator interface.""" diff --git a/services/netplan-v4/netplan_v4/battery_model.py b/services/netplan-v4/netplan_v4/battery_model.py new file mode 100644 index 0000000..0c7813e --- /dev/null +++ b/services/netplan-v4/netplan_v4/battery_model.py @@ -0,0 +1,44 @@ +"""SOC-aware battery model, including reserve recovery and hysteresis rearming.""" +from math import sqrt + + +class BatteryModel: + @staticmethod + def build(model,battery,steps,direction): + cap=battery.capacity_kwh;eta=sqrt(battery.roundtrip_efficiency) + reserve=cap*battery.min_soc_percent/100 + initial=cap*battery.soc_percent/100 + lower=min(initial,reserve) if battery.recovery_allowed else reserve + upper=cap*battery.max_soc_percent/100 + energies=[model.variable(lower,upper) for _ in range(len(steps)+1)] + model.constraint({energies[0]:1},initial,initial) + terminal=battery.terminal_soc_min_percent + if terminal is None:terminal=max(battery.soc_percent,battery.min_soc_percent) + model.constraint({energies[-1]:1},cap*terminal/100) + model.cost[energies[-1]]=-battery.terminal_value_chf_kwh + rearm=cap*(battery.rearm_soc_percent if battery.rearm_soc_percent is not None else battery.min_soc_percent)/100 + enabled=[model.variable(0,1,integer=True) for _ in steps] + # Only a numerical threshold, not an undisclosed extra operating reserve. + epsilon=min(1e-5,max(0.,upper-reserve)/1000) + initially_enabled=not battery.discharge_blocked and initial>=reserve+max(epsilon,1e-8) + model.constraint({enabled[0]:1},int(initially_enabled),int(initially_enabled)) + big=max(upper-lower+epsilon,1.) + charge,discharge=[],[] + for i,step in enumerate(steps): + dt=step.seconds/3600;cmax=battery.max_charge_w/1000;dmax=battery.max_discharge_w/1000 + c=model.variable(0,cmax,battery.throughput_chf_kwh*dt) + d=model.variable(0,dmax,battery.throughput_chf_kwh*dt) + charge.append(c);discharge.append(d) + model.constraint({c:1,direction[i]:-cmax},upper=0) + model.constraint({d:1,direction[i]:dmax},upper=dmax) + model.constraint({d:1,enabled[i]:-dmax},upper=0) + model.constraint({energies[i]:1,enabled[i]:-big},lower=reserve+epsilon-big) + # Once enabled, discharge may consume only energy above reserve. + model.constraint({energies[i+1]:1,enabled[i]:-big},lower=reserve-big) + if i: + # A disabled battery can rearm only AFTER prior charging has + # actually reached the configured hysteresis threshold. + model.constraint({energies[i]:1,enabled[i]:-big,enabled[i-1]:big},lower=rearm-big) + model.constraint({energies[i+1]:1,energies[i]:-1,c:-eta*dt,d:dt/eta},0,0) + return {'charge':charge,'discharge':discharge,'energy':energies, + 'terminal_min_percent':terminal,'discharge_enabled':enabled} diff --git a/services/netplan-v4/netplan_v4/controlled_trial.py b/services/netplan-v4/netplan_v4/controlled_trial.py new file mode 100644 index 0000000..cf9afcb --- /dev/null +++ b/services/netplan-v4/netplan_v4/controlled_trial.py @@ -0,0 +1,162 @@ +"""Explicit, time-limited commissioning authority; NEVER enables shadow execution. + +Only a reviewed internal operator call can arm a trial, and only for a separate +allowlist (empty by default). The manager and battery must additionally consent +locally. Existing shadow plans/acknowledgements keep their original meaning. +""" +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from uuid import UUID +import json + +MAX_SECONDS = 1800 +MAX_POWER_W = 5000 +AUTHORITY_TTL_SECONDS = 90 + + +def schema(con): + con.execute('''CREATE TABLE IF NOT EXISTS planner_controlled_trials( + plant TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, + value TEXT NOT NULL, revoked_at TEXT)''') + + +def timestamp(v): + if not isinstance(v, str): + raise ValueError('Explicit timestamp required') + t = datetime.fromisoformat(v.replace('Z', '+00:00')) + if t.tzinfo is None: + raise ValueError('Timezone required') + return t.astimezone(timezone.utc) + + +def uuid(v): + if not isinstance(v, str) or str(UUID(v)) != v: + raise ValueError('Canonical UUID required') + return v + + +def power(v): + if type(v) not in (int, float) or not 0 < v <= MAX_POWER_W: + raise ValueError('Pilot limit must be explicit and at most 5000 W') + return float(v) + + +def accounting(plan): + # A human checkbox alone must not relabel aggregate house consumption. + quality = plan.get('inputQuality', {}) + if quality.get('loadBasis') != 'base_load': + raise ValueError('Verified base-load/SDL adapter required before a control trial') + evidence = quality.get('accountingEvidenceId') + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Plan lacks traceable base-load accounting evidence') + return evidence + + +def eligibility(view, plant): + if view.get('installationId') != plant or view.get('liveEnabled') is not False: + raise ValueError('Wrong plant or service mode') + p = view.get('plan') + if view.get('fresh') is not True or not isinstance(p, dict): + raise ValueError('Fresh independently validated planning input required') + if (p.get('installationId') != plant or p.get('runMode') != 'shadow' + or p.get('liveEnabled') is not False or p.get('executable') is not True): + raise ValueError('Wrong source plan identity or mode') + settings = view.get('settings', {}) + if settings.get('family') == 'auto' or p.get('sourceFamily') != settings.get('family'): + raise ValueError('First controlled trial requires a fixed model family') + if p.get('configRevision') != settings.get('revision'): + raise ValueError('Configuration revision changed') + if len(p.get('controlContext', {}).get('batteries', {})) != 1: + raise ValueError('First controlled trial supports exactly one battery') + accounting(p) + return p + + +def arm(store, plant, request, view, now, allowed_plants): + """Caller is authenticated internal operator; not exposed via device proxy.""" + if plant not in allowed_plants: + raise ValueError('Controlled trial disabled by server allowlist') + fields = {'sessionId', 'expectedPlanId', 'expectedRevision', 'durationSeconds', + 'maxChargeW', 'maxDischargeW', 'assetId', 'managerId', 'batteryInstanceId', + 'acceptEstimatedPeak', 'actuatorWatchdogEvidenceId', 'confirmation'} + if set(request) != fields or request['confirmation'] != 'ARM_BOUNDED_CONTROL_TRIAL': + raise ValueError('Explicit reviewed commissioning request required') + session = uuid(request['sessionId']); p = eligibility(view, plant) + if type(request['expectedRevision']) is not int or request['expectedRevision'] < 0: + raise ValueError('Expected revision must be an integer') + if request['expectedPlanId'] != p['planId'] or request['expectedRevision'] != p['configRevision']: + raise ValueError('Source plan/revision changed; review again') + seconds = request['durationSeconds'] + if type(seconds) is not int or not 30 <= seconds <= MAX_SECONDS: + raise ValueError('Trial duration must be 30..1800 seconds') + for k in ('managerId', 'batteryInstanceId'): + if type(request[k]) is not int or not 1 <= request[k] <= 99999: + raise ValueError('Explicit local instance binding required') + if request['managerId'] == request['batteryInstanceId']: + raise ValueError('Manager and battery instance must differ') + if request['assetId'] not in p['controlContext']['batteries']: + raise ValueError('Wrong trial battery') + if type(request['acceptEstimatedPeak']) is not bool: + raise ValueError('Explicit estimated-peak policy required') + if p.get('peakCostIsEstimate') and not request['acceptEstimatedPeak']: + raise ValueError('Estimated peak has not been accepted for this trial') + evidence = request['actuatorWatchdogEvidenceId'] + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Device-side command-loss watchdog proof required') + value = {'kind': 'controlled_trial_grant', 'version': 1, 'sessionId': session, + 'installationId': plant, 'issuedAt': now.isoformat(), + 'expiresAt': (now + timedelta(seconds=seconds)).isoformat(), + 'revision': p['configRevision'], 'family': p['sourceFamily'], + 'assetId': request['assetId'], 'managerId': request['managerId'], + 'batteryInstanceId': request['batteryInstanceId'], + 'maxChargeW': power(request['maxChargeW']), + 'maxDischargeW': power(request['maxDischargeW']), + 'acceptEstimatedPeak': request['acceptEstimatedPeak'], + 'accountingEvidenceId': accounting(p), + 'actuatorWatchdogEvidenceId': evidence, + 'controlContext': deepcopy(p['controlContext'])} + with store.con: + existing = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if existing and existing[1] is None and timestamp(json.loads(existing[0])['expiresAt']) > now: + raise ValueError('Existing trial must first end; implicit extension forbidden') + # Reusing an expired/revoked session ID must never resurrect it. + used = store.con.execute("SELECT 1 FROM planner_audit WHERE plant=? AND kind='trial_arm' AND detail=?", (plant, session)).fetchone() + if used: + raise ValueError('Session ID already used') + store.con.execute('INSERT INTO planner_controlled_trials VALUES(?,?,?,NULL) ON CONFLICT(plant) DO UPDATE SET session_id=excluded.session_id,value=excluded.value,revoked_at=NULL', + (plant, session, json.dumps(value, sort_keys=True, allow_nan=False))) + store.con.execute("INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,'trial_arm',?)", (plant, now.isoformat(), session)) + return value + + +def revoke(store, plant, session, now): + uuid(session) + with store.con: + store.con.execute('UPDATE planner_controlled_trials SET revoked_at=? WHERE plant=? AND session_id=?', (now.isoformat(), plant, session)) + return {'status': 'revoked', 'sessionId': session, 'remoteRevocationMaxSeconds': AUTHORITY_TTL_SECONDS} + + +def authority(store, plant, view, now, allowed_plants): + """Separate authorization envelope, not a mutation of the shadow plan.""" + if plant not in allowed_plants: + return None + row = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if not row or row[1] is not None: + return None + grant = json.loads(row[0]) + try: + p = eligibility(view, plant) + if not timestamp(grant['issuedAt']) <= now < timestamp(grant['expiresAt']): + return None + if (p['configRevision'] != grant['revision'] or p['sourceFamily'] != grant['family'] + or p['controlContext'] != grant['controlContext'] + or accounting(p) != grant['accountingEvidenceId'] + or p.get('peakCostIsEstimate') and not grant['acceptEstimatedPeak']): + return None + except (ValueError, KeyError, TypeError): + return None + until = min(timestamp(grant['expiresAt']), timestamp(p['validUntil']), + now + timedelta(seconds=AUTHORITY_TTL_SECONDS)) + return {**grant, 'kind': 'controlled_trial_authority', + 'sourceShadowPlanId': p['planId'], 'checkedAt': now.isoformat(), + 'validUntil': until.isoformat(), 'sourcePlanRemainsShadow': True} diff --git a/services/netplan-v4/netplan_v4/domain.py b/services/netplan-v4/netplan_v4/domain.py new file mode 100644 index 0000000..140acb8 --- /dev/null +++ b/services/netplan-v4/netplan_v4/domain.py @@ -0,0 +1,170 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from math import isfinite +from typing import Mapping +from zoneinfo import ZoneInfo + +UTC = timezone.utc +ZURICH = ZoneInfo('Europe/Zurich') + +def utc(value): + if isinstance(value, str): + value = datetime.fromisoformat(value.replace('Z', '+00:00')) + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError('Timezone required') + return value.astimezone(UTC) + +def number(value, name, minimum=None, maximum=None): + if isinstance(value, bool) or not isinstance(value, (float, int)): + raise ValueError(f'{name}: finite number required') + value = float(value) + if not isfinite(value) or minimum is not None and value < minimum or maximum is not None and value > maximum: + raise ValueError(f'{name}: outside permitted range') + return value + +def quarter_start(value): + value = utc(value) + return value.replace(minute=value.minute // 15 * 15, second=0, microsecond=0) + +def month_key(value): + return utc(value).astimezone(ZURICH).strftime('%Y-%m') + +@dataclass(frozen=True) +class Family: + key: str + pv: str + load: str + schedule: str + label: str + +class FamilyRegistry: + def __init__(self, families=()): + self._families = {} + for family in families: + self.register(family) + def register(self, family): + if family.key in self._families or any(f.schedule == family.schedule for f in self._families.values()): + raise ValueError('Duplicate family or schedule identifier') + self._families[family.key] = family + def get(self, key): + if key not in self._families: + raise ValueError(f'Unknown model family: {key}') + return self._families[key] + def entries(self): + return tuple(self._families.values()) + +def default_registry(): + return FamilyRegistry([ + Family('3','prog_var_1','prog_var_2','prog_var_3','Variante 1 (1 / 2 / 3)'), + Family('13','prog_var_10','prog_var_11','prog_var_13','Variante 2 (10 / 11 / 13)'), + Family('23','prog_var_21','prog_var_22','prog_var_23','Variante 3 (21 / 22 / 23)'), + ]) + +@dataclass(frozen=True) +class Price: + chf_kwh: float + published_at: datetime | None = None + mode: str = 'static' + def known_at(self, at): + number(self.chf_kwh, 'energy price') + if self.mode not in ('static','dynamic'): + raise ValueError('Explicit static/dynamic price mode required') + return self.mode == 'static' or self.published_at is not None and utc(self.published_at) <= utc(at) + +@dataclass(frozen=True) +class Step: + start: datetime + base_load_w: float + pv_w: float + import_price: Price | None + export_price: Price | None + external_w: float = 0.0 + seconds: int = 300 + @property + def end(self): + return utc(self.start) + timedelta(seconds=self.seconds) + @property + def residual_w(self): + return self.base_load_w + self.external_w - self.pv_w + +def split_base_load(measured_house_w, flexible_w, external_w=0.0, external_already_removed=False): + measured = number(measured_house_w, 'measured_house_w') + flex = sum(number(v, 'flexible measurement') for v in flexible_w) + external = number(external_w, 'external measurement') + base = measured - flex - (0.0 if external_already_removed else external) + if base < -1.0: + raise ValueError('Negative base load: inconsistent measurement boundary or double subtraction') + return max(0.0, base) + +def priced_prefix(steps, at): + result = [] + for step in steps: + if not step.import_price or not step.export_price: + break + if not step.import_price.known_at(at) or not step.export_price.known_at(at): + break + result.append(step) + while result and result[-1].end != quarter_start(result[-1].end): + result.pop() + return result + +@dataclass(frozen=True) +class Battery: + asset_id: str + capacity_kwh: float + soc_percent: float + min_soc_percent: float + max_soc_percent: float + max_charge_w: float + max_discharge_w: float + measured_at: datetime + roundtrip_efficiency: float = 0.90 + grid_charging: bool = False + throughput_chf_kwh: float = 0.0 + terminal_soc_min_percent: float | None = None + terminal_value_chf_kwh: float = 0.0 + recovery_allowed: bool = False + physical_min_soc_percent: float = 0.0 + discharge_blocked: bool = False + rearm_soc_percent: float | None = None + def validate(self, at): + if not self.asset_id: + raise ValueError('Battery asset_id required') + number(self.capacity_kwh, 'capacity_kwh', 0.001) + lo = number(self.min_soc_percent, 'min_soc_percent', 0, 100) + hi = number(self.max_soc_percent, 'max_soc_percent', lo, 100) + physical=number(self.physical_min_soc_percent,'physical minimum SOC',0,lo) + if type(self.recovery_allowed) is not bool or type(self.discharge_blocked) is not bool: + raise ValueError('Explicit battery recovery and hysteresis flags required') + number(self.soc_percent, 'SOC', physical if self.recovery_allowed else lo, hi) + if self.rearm_soc_percent is not None:number(self.rearm_soc_percent,'hysteresis rearm SOC',lo,hi) + number(self.max_charge_w, 'max_charge_w', 0) + number(self.max_discharge_w, 'max_discharge_w', 0) + number(self.roundtrip_efficiency, 'roundtrip_efficiency', 0.01, 1) + number(self.throughput_chf_kwh, 'throughput_chf_kwh', 0) + number(self.terminal_value_chf_kwh, 'terminal_value_chf_kwh', 0) + if self.terminal_soc_min_percent is not None: + number(self.terminal_soc_min_percent, 'terminal_soc_min_percent', lo, hi) + age = (utc(at) - utc(self.measured_at)).total_seconds() + if age < -30 or age > 1800: + raise ValueError('SOC is stale or from the future') + +@dataclass(frozen=True) +class QuarterPast: + import_kwh: float + measured_seconds: int + +@dataclass(frozen=True) +class Limits: + export_w: float | None = None + import_w: float | None = None + manager_month_limits_w: Mapping[int,float] = field(default_factory=dict) + def import_limit(self, timestamp): + values = [] + if self.import_w is not None: + values.append(number(self.import_w, 'import limit', 0)) + m = utc(timestamp).astimezone(ZURICH).month + if m in self.manager_month_limits_w: + values.append(number(self.manager_month_limits_w[m], 'monthly manager limit', 0)) + return min(values) if values else None diff --git a/services/netplan-v4/netplan_v4/economic_replay.py b/services/netplan-v4/netplan_v4/economic_replay.py new file mode 100644 index 0000000..239647f --- /dev/null +++ b/services/netplan-v4/netplan_v4/economic_replay.py @@ -0,0 +1,207 @@ +"""Causal daily economic family comparison using frozen plans and later physical data. + +This is a labelled *simulation*, not savings on an invoice. A cohort is captured +near local midnight, uses only prices/forecasts then available, and all families +share initial storage, physical outcomes, tariff and constraints. No device calls. +Supported v1: one grid-charge-enabled battery; other topologies remain explicit. +""" +from dataclasses import asdict, replace +from datetime import datetime, timedelta +from hashlib import sha256 +from math import sqrt +import json +from .domain import ZURICH, utc, month_key, quarter_start +from .selection import ReplayScore +from . import measurement_pipeline as m + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_economic_cohorts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, issued_at INTEGER NOT NULL, + ends_at INTEGER NOT NULL, policy_id TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_results( + plant TEXT NOT NULL, local_day TEXT NOT NULL, evaluated_at INTEGER NOT NULL, + policy_id TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_attempts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, checked_at INTEGER NOT NULL, + reason TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + ''') + + +def _policy(data, cfg): + batteries=[] + for b in data['batteries']: + v=asdict(b) + for k in ('soc_percent','measured_at','discharge_blocked'):v.pop(k) + batteries.append(v) + return sha256(m.canonical({'battery':batteries,'limits':asdict(data['limits']), + 'mapping':cfg['mappingSha256'],'formula':cfg['formula'],'dataset':cfg['datasetId'], + 'method':'frozen_day_ahead_grid_tracking_v1','external':'sdl_request_estimate','peakTariffs':data['peak_prices']}).encode()).hexdigest() + + +def capture(store, plant, now, assemble, optimizer): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile': return + local=utc(now).astimezone(ZURICH); day=local.date().isoformat() + # Freeze only at the beginning of a local day. Never reconstruct an old forecast from hindsight. + if local.hour!=0 or local.minute>=15:return + if store.con.execute('SELECT 1 FROM planner_economic_cohorts WHERE plant=? AND local_day=?',(plant,day)).fetchone():return + attempted=store.con.execute('SELECT checked_at FROM planner_economic_attempts WHERE plant=? AND local_day=?',(plant,day)).fetchone() + if attempted and now.timestamp()-attempted[0]<300:return + reason='awaiting_comparable_snapshot' + try: + cfg=m.configuration(store.con,plant,settings['measurementDataset']);plans={}; common=None;policy=None + end=datetime.combine(local.date()+timedelta(days=1),datetime.min.time(),tzinfo=ZURICH).astimezone(utc(now).tzinfo) + for family in store.registry.entries(): + data,_,quality=assemble(store,plant,family.key,now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + if len(data['batteries'])!=1 or not data['batteries'][0].grid_charging: + raise ValueError('unsupported_replay_topology') + if data['steps'][-1].end=rearm-1e-9:blocked=False + discharge=0. if blocked else min(b['max_discharge_w'],max(0.,(energy-low)*eta/dt*1000)) + target=p['gridTargetW'];cap=limits['import_w'];monthly=limits['manager_month_limits_w'].get(str(start.astimezone(ZURICH).month),limits['manager_month_limits_w'].get(start.astimezone(ZURICH).month)) + if monthly is not None:cap=monthly if cap is None else min(cap,monthly) + if cap is not None:target=min(target,cap) + if limits['export_w'] is not None:target=max(target,-limits['export_w']) + battery=min(charge,max(-discharge,target-residual));grid=residual+battery + energy+=battery/1000*dt*(eta if battery>=0 else 1/eta) + if energyhigh+1e-6:raise ValueError('Replay SOC invariant') + if energy<=low+1e-9:blocked=True + if cap is not None and grid>cap+1.:breaches+=1 + if limits['export_w'] is not None and grid < -limits['export_w']-1.:breaches+=1 + money+=(max(grid,0)*p['importPriceChfKwh']-max(-grid,0)*p['exportPriceChfKwh'])/1000*dt + wear+=abs(battery)/1000*dt*b['throughput_chf_kwh'] + q=quarter_start(start).isoformat();v=quarters.setdefault(q,[0.,0]);v[0]+=max(grid,0)/1000*dt;v[1]+=seconds + if end==quarter_start(start)+timedelta(minutes=15): + if v[1]!=900:raise ValueError('Incomplete simulated billing quarter') + month=month_key(start);peaks[month]=max(peaks[month],v[0]/.25);quarter_max[month]=max(quarter_max.get(month,0.),v[0]/.25) + additional=sum(max(0,v-ctx['peaks'][month])*ctx['peakTariffs'][month] for month,v in peaks.items()) + # Identical terminal valuation for all families, known at cohort creation; separate from cash. + last=plan['points'][-1];buy=max(0.,last['importPriceChfKwh']);sell=max(0.,min(buy,last['exportPriceChfKwh'])) + terminal=max(initial-energy,0)/eta*buy-max(energy-initial,0)*eta*sell + return {'cashCostChf':money+additional,'energyCostChf':money,'peakCostChf':additional,'throughputCostChf':wear, + 'terminalAdjustmentChf':terminal,'costChf':money+additional+wear+terminal, + 'initialEnergyKwh':initial,'finalEnergyKwh':energy,'coverage':min(coverage), + 'quarterMaximaKw':quarter_max,'initialPeaksKw':ctx['peaks'],'peakTariffs':ctx['peakTariffs'], + 'constraintBreaches':breaches,'terminalNormalized':True, + 'terminalMethod':'common_known_end_price_inventory_valuation_not_physical_restoration'} + + +def advance(store,plant,now): + timestamp=int(now.timestamp()) + rows=store.con.execute('SELECT c.* FROM planner_economic_cohorts c LEFT JOIN planner_economic_results r USING(plant,local_day) WHERE c.plant=? AND c.ends_at<=? AND c.ends_at>=? AND r.local_day IS NULL AND NOT EXISTS(SELECT 1 FROM planner_economic_attempts a WHERE a.plant=c.plant AND a.local_day=c.local_day AND a.checked_at>?) ORDER BY c.issued_at LIMIT 1',(plant,timestamp-120,timestamp-90*86400,timestamp-300)).fetchall() + for row in rows: + with store.con:store.con.execute('INSERT INTO planner_economic_attempts VALUES(?,?,?,?) ON CONFLICT(plant,local_day) DO UPDATE SET checked_at=excluded.checked_at,reason=excluded.reason',(plant,row['local_day'],timestamp,'evaluating_actuals')) + try: + cohort=json.loads(row['value']);cfg=m.configuration(store.con,plant,cohort['datasetId']);source=cfg.get('sourceDatasetId',cfg['datasetId']);start=m.epoch(cohort['start']);end=m.epoch(cohort['end']) + observed=store.con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',(plant,source,start-600,end+600,timestamp)) + windows=actuals([json.loads(r[0]) for r in observed],cfg);outcomes={w['start']:w for w in windows} + results={f:simulate(cohort,f,outcomes) for f in cohort['plans']} + value={'status':'evaluated','results':results,'start':cohort['start'],'end':cohort['end'], + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','method':cohort['method'], + 'actualCashSavings':False,'controlEnabled':False} + with store.con:store.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(plant,row['local_day'],timestamp,row['policy_id'],m.canonical(value))) + except (ValueError,KeyError,TypeError): + # Retain unscored cohort. Lack of actual data must never become a zero cost. + pass + + +def scores(store,plant,now): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile':return [] + since=int((utc(now)-timedelta(days=settings['autoLookbackDays'])).timestamp()) + recent=store.con.execute('SELECT policy_id,value FROM planner_economic_cohorts WHERE plant=? ORDER BY issued_at DESC LIMIT 1',(plant,)).fetchone() + if not recent or json.loads(recent[1])['datasetId']!=settings.get('measurementDataset'):return [] + rows=store.con.execute('SELECT c.issued_at,c.ends_at,r.evaluated_at,r.value FROM planner_economic_results r JOIN planner_economic_cohorts c USING(plant,local_day) WHERE r.plant=? AND r.policy_id=? AND c.issued_at>=? AND r.evaluated_at<=? ORDER BY c.issued_at',(plant,recent[0],since,int(now.timestamp()))).fetchall() + if not rows:return [] + keys=[f.key for f in store.registry.entries()];valid=[] + for r in rows: + v=json.loads(r['value']);out=v['results'] + if set(out)!=set(keys) or any(out[k]['constraintBreaches'] for k in keys):continue + valid.append((r,v)) + if not valid:return [] + result=[] + for k in keys: + # Monthly demand cost is paid ONCE for the maximum, not once per replay day. + total=sum(v['results'][k]['energyCostChf']+v['results'][k]['throughputCostChf']+v['results'][k]['terminalAdjustmentChf'] for _,v in valid) + bases={}; maxima={}; rates={} + for _,v in valid: + d=v['results'][k] + for month,peak in d['quarterMaximaKw'].items(): + bases.setdefault(month,d['initialPeaksKw'][month]) + maxima[month]=max(maxima.get(month,0.),peak);rates[month]=d['peakTariffs'][month] + total+=sum(max(0.,maxima[month]-bases[month])*rates[month] for month in maxima) + cover=min(v['results'][k]['coverage'] for _,v in valid) + first=utc(valid[0][1]['start']);end=utc(valid[-1][1]['end']);available=datetime.fromtimestamp(max(r['evaluated_at'] for r,_ in valid),utc(now).tzinfo) + result.append(ReplayScore(k,recent[0],first,end,first,available,total,cover,len(valid))) + return result + + +def status(store,plant): + rows=store.con.execute('SELECT local_day,value FROM planner_economic_results WHERE plant=? ORDER BY local_day DESC LIMIT 14',(plant,)) + evaluated=[{'day':r[0],**json.loads(r[1])} for r in rows] + count=store.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts WHERE plant=?',(plant,)).fetchone()[0] + return {'connected':True,'method':'frozen_day_ahead_grid_tracking_v1','cohorts':count,'completedComparisons':evaluated, + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','isBillingEvidence':False, + 'limitations':['one_grid_charging_battery','frozen_daily_plan_not_receding_horizon_field_replay']} diff --git a/services/netplan-v4/netplan_v4/forecast_quality.py b/services/netplan-v4/netplan_v4/forecast_quality.py new file mode 100644 index 0000000..4a83446 --- /dev/null +++ b/services/netplan-v4/netplan_v4/forecast_quality.py @@ -0,0 +1,21 @@ +"""Minimal validity gates, not a claim of forecast accuracy or model ranking.""" +from math import isfinite + + +def assess_family(family, *, minimum_steps=3): + points=family.get('points',[]) + if len(points) left for start, end in (*blocks[key], *capture_gaps)): + continue + result[key][left] = {'end': right, 'availableAt': first_observed[key][right], + 'spanSeconds': right-left} + return result diff --git a/services/netplan-v4/netplan_v4/mapping_identity.py b/services/netplan-v4/netplan_v4/mapping_identity.py new file mode 100644 index 0000000..255f5f1 --- /dev/null +++ b/services/netplan-v4/netplan_v4/mapping_identity.py @@ -0,0 +1,100 @@ +"""Audited numeric-representation compatibility, not relaxed device identity checks. + +Only the internal operator API can register equivalence. Raw journals are never +rewritten, and the original received fingerprint is retained with each receipt. +""" +from hashlib import sha256 +import json +import math +import re + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_mapping_compatibility( + plant TEXT NOT NULL, dataset TEXT NOT NULL, alias TEXT NOT NULL, + canonical TEXT NOT NULL, inventory TEXT NOT NULL, evidence TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset,alias)); + CREATE TABLE IF NOT EXISTS planner_observation_origins( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_mapping TEXT NOT NULL, inventory TEXT NOT NULL, + evidence_id TEXT, received_at INTEGER NOT NULL, + PRIMARY KEY(plant,dataset,captured_at,received_mapping)); + ''') + + +def _decode(text): + if not isinstance(text, str) or len(text.encode()) > 131072: + raise ValueError('Bounded configuration evidence required') + def pairs(items): + d = {} + for k, v in items: + if k in d: raise ValueError('Duplicate configuration key') + d[k] = v + return d + def constant(_): raise ValueError('Nonfinite configuration') + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + + +def _same(a, b): + if type(a) is not type(b): return False + if isinstance(a, dict): + return list(a) == list(b) and all(_same(a[k], b[k]) for k in a) + if isinstance(a, list): return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b)) + return a == b + + +def validate_evidence(plant, config, payload): + if not isinstance(payload, dict) or set(payload) != {'version', 'canonicalJson', 'legacyJson'} or type(payload['version']) is not int or payload['version'] != 1: + raise ValueError('Explicit versioned representation evidence required') + ca, le = payload['canonicalJson'], payload['legacyJson'] + a, b = _decode(ca), _decode(le) + canonical_hash, alias = sha256(ca.encode()).hexdigest(), sha256(le.encode()).hexdigest() + if canonical_hash != config['mappingSha256'] or alias == canonical_hash: + raise ValueError('Evidence does not match configured mapping') + for c in (a, b): + if not isinstance(c, dict) or c.get('installationId') != plant or c.get('reportedInventorySha256') != config['inventorySha256']: + raise ValueError('Evidence belongs to another installation or inventory') + x = a.get('accounting', {}).get('splitToleranceW') + y = b.get('accounting', {}).get('splitToleranceW') + if type(x) is not float or type(y) is not int or not math.isfinite(x) or x != y or not 0 <= x <= 500: + raise ValueError('Only demonstrated float/integer tolerance representation is compatible') + b['accounting']['splitToleranceW'] = float(y) + if not _same(a, b): + raise ValueError('Other configuration differences are not representation compatibility') + # Proof includes the exact hashed JSON strings; no arbitrary labels as evidence. + evidence = json.dumps(payload, sort_keys=True, separators=(',', ':'), allow_nan=False) + return {'alias': alias, 'canonical': canonical_hash, 'inventory': config['inventorySha256'], + 'evidenceId': sha256(evidence.encode()).hexdigest(), 'evidence': evidence} + + +def register(con, plant, config, payload, now): + if config.get('sourceDatasetId'): raise ValueError('Register compatibility on original dataset only') + v = validate_evidence(plant, config, payload) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT canonical,inventory,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND alias=?', + (plant, config['datasetId'], v['alias'])).fetchone() + expected = (v['canonical'], v['inventory'], v['evidence']) + if old and tuple(old) != expected: raise ValueError('Immutable mapping compatibility conflict') + con.execute('INSERT OR IGNORE INTO planner_mapping_compatibility VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], v['alias'], *expected, now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status': 'registered', 'datasetId': config['datasetId'], 'evidenceId': v['evidenceId'], + 'compatibleMapping': v['alias'], 'canonicalMapping': v['canonical'], 'controlEnabled': False} + + +def approved(con, plant, config): + rows = con.execute('SELECT alias,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND canonical=? AND inventory=?', + (plant, config['datasetId'], config['mappingSha256'], config['inventorySha256'])) + return {r['alias']: sha256(r['evidence'].encode()).hexdigest() for r in rows} + + +def save_origin(con, plant, config, record, captured_at, received_at, aliases): + source = record['mappingSha256'] + if source != config['mappingSha256'] and source not in aliases: + raise ValueError('Unknown mapping; no receipt written') + con.execute('INSERT OR IGNORE INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], captured_at, source, config['inventorySha256'], aliases.get(source), received_at)) diff --git a/services/netplan-v4/netplan_v4/measurement_pipeline.py b/services/netplan-v4/netplan_v4/measurement_pipeline.py new file mode 100644 index 0000000..e5af4da --- /dev/null +++ b/services/netplan-v4/netplan_v4/measurement_pipeline.py @@ -0,0 +1,523 @@ +"""Application data path: versioned numeric observations -> physical load -> trained profiles. + +Lives in the existing planner service/database; no separate diagnostic service. +The device may append only to an operator-configured dataset. Original observations, +model revisions and prediction vintages are preserved. Output is never an actuator grant. +""" +from __future__ import annotations +from bisect import bisect_right +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from math import ceil, isfinite +from statistics import median +from zoneinfo import ZoneInfo +import json +from .history_timing import validate_policy, endpoint_bridges +from . import mapping_identity + +UTC = timezone.utc +LOCAL = ZoneInfo('Europe/Zurich') +FAMILIES = ('3', '13', '23') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond: + raise ValueError('Explicit whole-second UTC timestamp required') + return int(t.timestamp()) + + +def availability_epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0: + raise ValueError('Explicit UTC timestamp required') + # Never claim an input existed before its sub-second observation completed. + return int(ceil(t.timestamp())) + + +def iso(t): + return datetime.fromtimestamp(t, UTC).isoformat() + + +def numeric(value, bound=1e12): + return type(value) in (int, float) and isfinite(value) and abs(value) <= bound + + +def schema(con): + mapping_identity.schema(con) + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_data_sets( + plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_observations( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,captured_at)); + CREATE TABLE IF NOT EXISTS planner_load_windows( + plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL, + available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,start)); + CREATE TABLE IF NOT EXISTS planner_load_models( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,model_id)); + CREATE TABLE IF NOT EXISTS planner_model_current( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_pipeline_state( + plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL, + status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_prediction_vintages( + plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL, + target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL, + load_w REAL NOT NULL, pv_w REAL NOT NULL, + PRIMARY KEY(plant,dataset,issued_at,target,family)); + CREATE INDEX IF NOT EXISTS planner_observation_window + ON planner_observations(plant,dataset,captured_at); + CREATE INDEX IF NOT EXISTS planner_prediction_target + ON planner_prediction_vintages(plant,dataset,target); + ''') + + +def validate_config(c): + fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', + 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', + 'minimumTrainingHours', 'historyDays'} + optional = {'sourceDatasetId', 'historyTimingPolicy'} + if not isinstance(c, dict) or not fields <= set(c) or set(c)-fields-optional: + raise ValueError('Explicit dataset configuration required') + name = c['datasetId'] + if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): + raise ValueError('Invalid dataset ID') + for field in ('mappingSha256', 'inventorySha256'): + h = c[field] + if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h): + raise ValueError('Explicit mapping/inventory fingerprint required') + if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'): + raise ValueError('Unknown physical formula') + if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1: + raise ValueError('Coverage must be .90..1; recorded gaps remain visible') + for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)): + if type(c[field]) is not int or not lo <= c[field] <= hi: + raise ValueError('Invalid '+field) + sources = c['sources'] + if not isinstance(sources, list) or not 3 <= len(sources) <= 80: + raise ValueError('Source list required') + seen, ids = set(), set() + roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'} + for s in sources: + if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}: + raise ValueError('Explicit source definition required') + k = s['key'] + if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids: + raise ValueError('Duplicate/invalid source') + if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0: + raise ValueError('Source role/factor invalid') + if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300: + raise ValueError('Source lifetime invalid') + seen.add(k); ids.add(s['variableId']) + if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources): + raise ValueError('Grid and PV measurement sources required') + sr = c['solarReference'] + if c['formula'] == 'solar_terminal_v1': + if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}: + raise ValueError('Solar terminal sources required') + bykey = {s['key']: s['role'] for s in sources} + if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))): + raise ValueError('Solar origin roles mismatch') + elif sr is not None: + raise ValueError('No unused solar mapping allowed') + validate_policy(c.get('historyTimingPolicy'), sources) + source = c.get('sourceDatasetId') + if source is not None and (not isinstance(source, str) or not 1 <= len(source) <= 80 or source == name or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in source)): + raise ValueError('Invalid source dataset reference') + canonical(c) + return c + + +def register_dataset(con, plant, c, now): + """Operator endpoint only; device append endpoint cannot change units or limits.""" + validate_config(c) + if c.get('sourceDatasetId'): + origin = configuration(con, plant, c['sourceDatasetId']) + if origin.get('sourceDatasetId'): + raise ValueError('Dataset reference chains are not allowed') + comparable = lambda v: {k:x for k,x in v.items() if k not in ('datasetId','sourceDatasetId','historyTimingPolicy')} + if canonical(comparable(origin)) != canonical(comparable(c)): + raise ValueError('Referenced observations must retain identical measurement meaning') + value = canonical(c) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone() + if old and old[0] != value: + raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning') + con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False} + + +def configuration(con, plant, dataset): + row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone() + if row is None: + raise ValueError('Dataset not configured for this installation') + return json.loads(row[0]) + + +def project(record, c, plant, now, approved_mappings=()): + if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: + raise ValueError('Wrong capture identity') + incoming = record.get('mappingSha256') + if not isinstance(incoming, str) or (incoming != c['mappingSha256'] and incoming not in approved_mappings) or record.get('reportedInventorySha256') != c['inventorySha256']: + raise ValueError('Wrong capture mapping or inventory') + t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) + if start > t or t > now+30 or t < now-90*86400: + raise ValueError('Capture timestamp outside permitted range') + if not isinstance(record.get('raw'), dict): + raise ValueError('Numeric raw observations required') + out = {}; issues = [] + for s in c['sources']: + r = record['raw'].get(s['key'], {}) + if not isinstance(r, dict): + r = {} + v, at = r.get('value'), r.get('sourceUpdatedAt') + good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == [] + if not good: + v = at = None + issues.append(s['key']) + # Unknown/free-text fields, credentials, client quality claims never persisted. + out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)} + return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues} + + +def ingest_batch(con, plant, payload, now): + if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: + raise ValueError('Measurement batch version/fields invalid') + c = configuration(con,plant,payload['datasetId']) + if c.get('sourceDatasetId'): + raise ValueError('Derived dataset is read-only; append to the original measurement dataset') + records = payload['records'] + if not isinstance(records,list) or not 1 <= len(records) <= 120: + raise ValueError('Batch requires 1..120 captures') + aliases = mapping_identity.approved(con, plant, c) + rows = [project(r,c,plant,now,aliases) for r in records] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): + raise ValueError('Batch must be in increasing capture order') + stored = duplicate = 0 + con.execute('BEGIN IMMEDIATE') + try: + for original, r in zip(records, rows): + value = canonical(r); digest = sha256(value.encode()).hexdigest() + old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() + if old: + if old[0] != digest: + raise ValueError('Conflicting immutable observation') + duplicate += 1 + else: + con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + mapping_identity.save_origin(con, plant, c, original, r['capturedAt'], now, aliases) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate, + 'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False} + + +def physical_value(values, c): + """Same explicit sign convention as configured acquisition. No virtual power in load.""" + total = defaultdict(float) + sr = c['solarReference'] + for s in c['sources']: + if s['role'] not in ('grid','pv','physical_storage','flexible_load'): + continue + if sr and s['key'] in (sr['pvKey'],sr['batteryKey']): + continue + val = values[s['key']]*s['factorToW'] + if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0): + raise ValueError('Invalid physical power') + total[s['role']] += val + solar = 0. + if sr: + raw, sf = values[sr['rawKey']], values[sr['scaleKey']] + if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6: + raise ValueError('Invalid solar power/scaling sentinel') + solar = raw*10**int(sf) + load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar + if not numeric(load,1e9) or load < 0: + raise ValueError('Negative/nonfinite physical load') + return load + + +def reconstruct(records, c, *, projection=None): + """Bounded retrospective estimation, never a real-time feedback signal. + +Missing observations split support. Source timestamps are not refreshed. Small +uncovered portions remain quantified and are never filled with zero. +""" + if len(records) < 2: + return [] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])): + raise ValueError('Capture sequence not ordered') + sr = c['solarReference'] + primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')} + if sr: + primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) + for s in c['sources']: + if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) + if projection is not None: + configured={s['key']:s for s in c['sources']} + if set(projection)!= {'keys','calculate'} or not callable(projection['calculate']) or not projection['keys'] or not set(projection['keys']) <= set(configured): + raise ValueError('Invalid internal projection') + primary={k:configured[k] for k in projection['keys']} + first,last = records[0]['capturedAt'],records[-1]['capturedAt'] + series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + first_observed = {k:{} for k in primary} + pending = {k:None for k in primary}; high = {k:0 for k in primary} + edges = {first,last} + for a,b in zip(records,records[1:]): + if b['capturedAt']-a['capturedAt'] > 45: + gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1]) + for r in records: + at = r['capturedAt'] + for k in primary: + v = r['raw'].get(k,{}) + t = v.get('sourceUpdatedAt') + if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5: + if pending[k] is None: pending[k] = at + continue + high[k] = max(high[k],t) + if pending[k] is not None: + blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None + if t in series[k] and series[k][t] != v['value']: + series[k][t] = None + else: + series[k].setdefault(t,v['value']) + first_observed[k].setdefault(t, max(at, r.get('_receivedAt', at))) + for k,s in primary.items(): + if pending[k] is not None: + blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) + for t in series[k]: edges.update((t,t+s['maxAgeSeconds'])) + edges.update(range(first//300*300+300,last,300)) + edges = sorted(x for x in edges if first <= x <= last) + knots = {k:sorted(v) for k,v in series.items()} + bridges = endpoint_bridges(series, first_observed, blocks, gaps, timing_policy, primary) + bins = {} + for a,b in zip(edges,edges[1:]): + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0, + 'publicationEstimatedSeconds':0,'publicationEstimatedBySource':{},'knownAt':0,'missingSourceSeconds':{}}) + vals = {}; usable = not any(x <= a < y for x,y in gaps) + extended = []; unavailable = []; known_at = 0 + for k,s in primary.items(): + pos = bisect_right(knots[k],a)-1 + t = knots[k][pos] if pos >= 0 else None + bridge = bridges[k].get(t) + expired = t is None or a >= t+s['maxAgeSeconds'] + supported_tail = bool(bridge and a < bridge['end']) + if t is None or (expired and not supported_tail) or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False; unavailable.append(k) + else: + vals[k] = series[k][t] + known_at = max(known_at, first_observed[k][t]) + if expired: + extended.append(k); known_at = max(known_at, bridge['availableAt']) + load = None + if usable: + try: + load = physical_value(vals,c) if projection is None else projection['calculate'](vals,c) + if not numeric(load,1e9):raise ValueError('Invalid historical projection') + except ValueError: usable = False + if usable: + item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + item['knownAt'] = max(item['knownAt'], known_at) + if extended: item['publicationEstimatedSeconds'] += b-a + for key in extended: item['publicationEstimatedBySource'][key] = item['publicationEstimatedBySource'].get(key,0)+b-a + else: + item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + for key in unavailable: item['missingSourceSeconds'][key] = item['missingSourceSeconds'].get(key,0)+b-a + out=[] + for t,item in sorted(bins.items()): + # Partial beginning/end bins remain diagnostic and cannot train. + complete_extent = first <= t and last >= t+300 + coverage = item['seconds']/300 + eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds'] + out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], + 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, + 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, + 'meterBoundaryVerified':False,'method':c['formula'], + 'historyTimingMethod':(timing_policy or {}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':item['publicationEstimatedSeconds'], + 'publicationEstimatedBySourceSeconds':item['publicationEstimatedBySource'], + 'missingSourceSeconds':item['missingSourceSeconds'], + 'availableNotBefore':max(t+300,item['knownAt'])}) + return out + + +def slot(t): + local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL) + return local.hour*12+local.minute//5 + + +def build_profiles(rows): + samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)} + anchor=max(r['start'] for r in rows) + for r in rows: + i=slot(r['start']); v=r['loadW']; samples[i].append(v) + if anchor-r['start'] < 86400: recent[i].append(v) + weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v) + overall = median([r['loadW'] for r in rows]) + def profile(values): + # Missing calendar slots are a model estimate, not invented historical measurements. + result=[] + for i in range(288): + local=values.get(i,[]) + if not local: + local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])] + result.append(float(median(local)) if local else float(overall)) + return result + return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)}, + 'slotCoverage':len(samples)/288} + + +def predict(model, family, t): + p=model['profiles'][family] + if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday'] + return p[slot(t)] + + +def advance(con, plant, dataset, settings, now): + """Called by the existing worker; bounded data/model update once per five-minute tick.""" + c=configuration(con,plant,dataset); tick=now//300 + old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() + if old and old[0]==tick: return + observation_dataset=c.get('sourceDatasetId',dataset) + fetched=con.execute('SELECT value,received_at FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,observation_dataset,now-172800-300,now,now)).fetchall() + records=[{**json.loads(r[0]),'_receivedAt':r[1]} for r in fetched] + windows=reconstruct(records,c) + with con: + for w in windows: + if w['start']+300 > now-30 or w.get('availableNotBefore',0)>now: continue + con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', + (plant,dataset,w['start'],now,w['coverage'],canonical(w))) + rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] + good=[r for r in rows if r['profileUsable']] + active=current_model(con,plant,dataset,now) + cadence=86400 if settings['trainingCadence']=='daily' else 604800 + detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], + 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True, + 'observationDatasetId':observation_dataset, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'publicationEstimatedSeconds':sum(r.get('publicationEstimatedSeconds',0) for r in good), + 'originalFreshnessLimitsChanged':False, + 'lastCapture':iso(records[-1]['capturedAt']) if records else None} + state='collecting' + if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: + state='model_ready' if active else 'training' + attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0] + if attempted is None or now-attempted>=cadence: + profiles=build_profiles(good) + candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), + 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], + 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry'), + 'timingPolicySha256':sha256(canonical(c.get('historyTimingPolicy')).encode()).hexdigest(), + 'observationDatasetId':observation_dataset, + 'measurementBoundaryVerified':False} + # Causal held-out validation: build validation profiles without the final day. + split=good[-1]['start']-86400 + train=[r for r in good if r['start']+300<=split and r.get('availableNotBefore',r['start']+300)<=split]; test=[r for r in good if r['start']>=split] + if len(train)>=288 and len(test)>=240: + val={'profiles':build_profiles(train)} + errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} + candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors} + # Initial model is labelled bootstrap, never a production measurement proof. + # Existing model can be replaced only with held-out evidence and no aggregate regression. + promote=active is None + if active and candidate['validation']['status']=='causal_holdout': + past_model_eligible=active['trainedThrough']<=split and active['trainedAt']<=split + if past_model_eligible: + incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + promote=challenger<=incumbent + candidate['validation']['incumbentCompared']=True + else: + candidate['validation']['status']='holdout_overlaps_active_training' + ident=sha256(canonical(candidate).encode()).hexdigest() + with con: + con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate))) + if promote: + con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident)) + detail['candidateModelId']=ident;detail['candidatePromoted']=promote + state='model_ready' if promote or active else 'candidate_pending' + active=current_model(con,plant,dataset,now) + if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']}) + with con: + con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail', + (plant,dataset,tick,state,canonical(detail))) + + +def current_model(con,plant,dataset,at): + row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone() + return {**json.loads(row[1]),'modelId':row[0]} if row else None + + +def apply_load_forecast(con,plant,dataset,forecast,decision): + model=current_model(con,plant,dataset,decision) + if not model: + raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') + c=configuration(con,plant,dataset) + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,c.get('sourceDatasetId',dataset),decision,decision)).fetchone() + if not last: raise ValueError('No recent corrected observation') + last=json.loads(last[0]) + if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') + sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] + if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') + s=sdl_sources[0];r=last['raw'][s['key']] + if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']: + raise ValueError('No current external SDL request for the persistence scenario') + sdl=r['value']*s['factorToW'] + result=json.loads(canonical(forecast)); result['families']={} + result['observedAt']=iso(max(availability_epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + for family,old in forecast['families'].items(): + if family not in FAMILIES: continue + points=[] + for p in old['points']: + t=epoch(p['time']) + points.append({**p,'loadW':predict(model,family,t),'externalW':sdl}) + result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']), + 'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'], + 'loadMethodVersion':model['methodVersion'],'loadVariant':family, + 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), + 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, + 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation'], + 'historyTimingMethod':model.get('historyTimingMethod','strict_expiry'), + 'observationDatasetId':model.get('observationDatasetId',dataset)}} + return result + + +def pipeline_status(con,plant): + out=[] + for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): + ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at),MAX(received_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() + out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], + 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'lastReceived':iso(count[3]) if count[3] else None, + 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], + 'observationDatasetId':c.get('sourceDatasetId',ds), + 'historyTimingMethod':c.get('historyTimingPolicy',{}).get('method','strict_expiry')}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False,'historyTimingVersion':1} diff --git a/services/netplan-v4/netplan_v4/meter_runtime.py b/services/netplan-v4/netplan_v4/meter_runtime.py new file mode 100644 index 0000000..db92eda --- /dev/null +++ b/services/netplan-v4/netplan_v4/meter_runtime.py @@ -0,0 +1,145 @@ +"""Persistent, explicitly estimated operational demand tracking. + +No inference from a configured cap. Samples are device-reception observations; +last-value integration is a labelled control estimate, not settlement metering. +No reset, stale sample or long communication gap is bridged silently. +""" +from datetime import datetime,timedelta,timezone +import json +from .domain import number,utc,month_key,quarter_start,ZURICH +from .peak_policy import basis_record + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_peak_assumptions( + plant TEXT NOT NULL,month TEXT NOT NULL,kw REAL NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_runtime_samples( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,at INTEGER NOT NULL, + power_w REAL NOT NULL,total_kwh REAL,policy_id TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,at)); + CREATE TABLE IF NOT EXISTS planner_runtime_quarters( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,start INTEGER NOT NULL, + import_kwh REAL NOT NULL,policy_id TEXT NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,start)); + CREATE INDEX IF NOT EXISTS idx_runtime_samples_time ON planner_runtime_samples(plant,at); + ''') + + +def validate_observation(obs, observed_at): + if not isinstance(obs,dict) or set(obs)!={'meterId','sampleAt','powerW','totalImportKwh','controlPolicyId'}: + raise ValueError('Explicit meter observation schema required') + import re + if not isinstance(obs['meterId'],str) or not re.fullmatch(r'symcon-active-import:[0-9a-f]{64}',obs['meterId']): + raise ValueError('Active import identity required') + if not isinstance(obs['controlPolicyId'],str) or not 1<=len(obs['controlPolicyId'])<=160: + raise ValueError('Control policy identity required') + t=utc(obs['sampleAt']) + if t.microsecond or not 0<=(utc(observed_at)-t).total_seconds()<=60: + raise ValueError('Fresh whole-second acquisition timestamp required') + number(obs['powerW'],'meter power',-1e9,1e9) + number(obs['totalImportKwh'],'active import total',0,1e12) + + +def integrate_power(samples,start,end,max_gap=120): + """samples sorted tuples (epoch,power,total,policy). Return None on gaps/reset. + + A source can keep the same value while receiving fresh telemetry. The caller + records all acquisitions, not only changes. A quarter is never labelled exact. + """ + if end<=start:return None + rows=sorted(samples,key=lambda r:r[0]) + if len(rows)<2:return None + covered=energy=0.;policies=set();largest_gap=0 + previous=None + for row in rows: + if previous is not None: + ta,pa,ea,pola=previous;tb,pb,eb,polb=row + if tb<=ta:return None + left=max(start,ta);right=min(end,tb) + if right>left: + if tb-ta>max_gap or pola!=polb:return None + if ea is not None and eb is not None and eb=validated['kw']:return old + con.execute('INSERT INTO planner_peak_assumptions VALUES(?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET kw=excluded.kw,value=excluded.value', + (plant,month,validated['kw'],json.dumps(validated,sort_keys=True,allow_nan=False))) + return validated + + +def assumptions(con,plant): + return {r[0]:json.loads(r[1]) for r in con.execute('SELECT month,value FROM planner_peak_assumptions WHERE plant=?',(plant,))} + + +def observe(con,plant,obs,at): + validate_observation(obs,at) + mid=obs['meterId'];stamp=int(utc(obs['sampleAt']).timestamp()) + existing=con.execute('SELECT power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at=?',(plant,mid,stamp)).fetchone() + values=(float(obs['powerW']),float(obs['totalImportKwh']),obs['controlPolicyId']) + if existing and tuple(existing)!=values:raise ValueError('Conflicting meter acquisition at same time') + con.execute('INSERT OR IGNORE INTO planner_runtime_samples VALUES(?,?,?,?,?,?)',(plant,mid,stamp,*values)) + # Only a just-completed quarter is finalised; late acquisition never promotes + # a historical gap to complete data without all supporting samples. + current=stamp//900*900 + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,mid,current-900-120,stamp))] + report=integrate_power(rows,current-900,current) + if report: + quarter=current-900 + previous=con.execute('SELECT import_kwh FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start=?',(plant,mid,quarter)).fetchone() + if previous is None: + con.execute('INSERT INTO planner_runtime_quarters VALUES(?,?,?,?,?,?)',(plant,mid,quarter,report['importKwh'],report['controlPolicyId'],json.dumps(report))) + m=month_key(datetime.fromtimestamp(quarter,timezone.utc)) + save_assumption(con,plant,m,{'kw':report['importKwh']/.25,'quality':'estimated','source':'sampled_power_estimate', + 'observedAt':utc(at).isoformat(),'meterId':mid,'notes':'Maximum of available sampled quarters; earlier month may be incomplete'},at) + # Samples are retained for reproducibility; production retention job required. + return report + + +def current_quarter(con,plant,obs,decision): + q=quarter_start(decision);start=int(q.timestamp());end=int(utc(decision).timestamp()) + if start==end:return None + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,obs['meterId'],start-120,end))] + report=integrate_power(rows,start,end) + if not report:return None + return {'start':q.isoformat(),'measuredSeconds':end-start,'importKwh':report['importKwh'], + 'quality':'estimated','source':'sampled_power_estimate','coverage':1.0,'notes':'Acquisition power estimate, not an exact billing counter boundary'} + + +def daily_peaks(con,plant,meter_id,now): + rows=con.execute('SELECT start,import_kwh,policy_id FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start>=? ORDER BY start', + (plant,meter_id,int((utc(now)-timedelta(days=91)).timestamp()))) + grouped={} + for start,energy,policy in rows: + t=datetime.fromtimestamp(start,timezone.utc).astimezone(ZURICH) + grouped.setdefault((t.strftime('%Y-%m-%d'),policy),[]).append((start,energy)) + result=[] + for (day,policy),items in grouped.items(): + begin=datetime.strptime(day,'%Y-%m-%d').replace(tzinfo=ZURICH);end=begin+timedelta(days=1) + if utc(end)>utc(now):continue + expected=set(range(int(begin.timestamp()),int(end.timestamp()),900)) + complete={t for t,e in items}==expected + result.append({'day':day,'peakKw':max(e/.25 for t,e in items),'controlPolicyId':policy,'complete':complete, + 'observedAt':utc(end).isoformat(),'quality':'estimated'}) + return result diff --git a/services/netplan-v4/netplan_v4/metering.py b/services/netplan-v4/netplan_v4/metering.py new file mode 100644 index 0000000..ecca015 --- /dev/null +++ b/services/netplan-v4/netplan_v4/metering.py @@ -0,0 +1,245 @@ +"""Auditable import-energy evidence. Pure calculations; no device/database writes. + +Raw change archives are NOT proof of uninterrupted meter communication. Boundaries +without an exact counter record yield intervals, never silently interpolated +billing facts. Only a separately verified source and chronology can be promoted +into the existing strict planner evidence contract. +""" +from __future__ import annotations +from bisect import bisect_left, bisect_right +from dataclasses import dataclass +from datetime import datetime, timedelta +import hashlib +import json +from math import isfinite +from .domain import utc, month_key, quarter_start, ZURICH + + +@dataclass(frozen=True) +class Reading: + at: datetime + kwh: float + + +class CounterSeries: + def __init__(self, source_id: str, rows, *, max_bracket_seconds=180, + quantity='active_import', unit='kWh', timestamp_verified=False, + identity_verified=False, chronology_verified=False): + if quantity != 'active_import' or unit != 'kWh': + raise ValueError('Active import energy in kWh required') + if not isinstance(source_id, str) or not source_id: + raise ValueError('Nonempty stable source identity required') + if type(max_bracket_seconds) is not int or max_bracket_seconds < 1: + raise ValueError('Positive bracket age required') + self.source_id = source_id + self.max_gap = max_bracket_seconds + self.verified = all(x is True for x in + (timestamp_verified, identity_verified, chronology_verified)) + values = {} + for row in rows: + at = utc(row.at) + value = row.kwh + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value < 0: + raise ValueError('Invalid cumulative active import value') + if at in values and abs(values[at] - value) > 1e-9: + raise ValueError('Conflicting duplicate timestamp') + values[at] = float(value) + self.times = sorted(values) + self.values = [values[t] for t in self.times] + self.resets = [self.times[i] for i in range(1,len(self.times)) + if self.values[i] < self.values[i-1] - 1e-9] + + def boundary(self, at): + at = utc(at) + pos = bisect_left(self.times, at) + if pos < len(self.times) and self.times[pos] == at: + value = self.values[pos] + return {'lowerKwh':value, 'upperKwh':value, 'exactRecord':True, + 'before':at.isoformat(), 'after':at.isoformat()} + if pos == 0 or pos == len(self.times): + return None + before, after = self.times[pos-1], self.times[pos] + if (after-before).total_seconds() > self.max_gap: + return None + if self.values[pos] < self.values[pos-1] - 1e-9: + return None + return {'lowerKwh':self.values[pos-1], 'upperKwh':self.values[pos], + 'exactRecord':False, 'before':before.isoformat(), 'after':after.isoformat()} + + def energy(self, start, end): + start, end = utc(start), utc(end) + if end <= start: + raise ValueError('Positive interval required') + left, right = self.boundary(start), self.boundary(end) + if not left or not right: + return None + support_start = utc(left['before']) + support_end = utc(right['after']) + if any(support_start < reset <= support_end for reset in self.resets): + return None + lower = max(0., right['lowerKwh'] - left['upperKwh']) + upper = right['upperKwh'] - left['lowerKwh'] + if upper < lower - 1e-9: + return None + exact = bool(left['exactRecord'] and right['exactRecord']) + return {'lowerKwh':lower, 'upperKwh':max(lower, upper), + 'exactRecords':exact, 'billingEvidence':exact and self.verified} + + +def native_meter_id(sources): + """Same ordered JSON and SHA256 as NetzfahrplanV4Bezugszaehler::identitaet.""" + import re + if not isinstance(sources,list) or not 1<=len(sources)<=8: + raise ValueError('Explicit native import sources required') + normalized=[];ids=set();idents=set();parents=set() + for source in sources: + if set(source)!={'VariableID','ElternID','Ident','FaktorZuKWh','Messgroesse'}: + raise ValueError('Unknown native source fields') + sid,pid,ident,factor=(source[k] for k in ('VariableID','ElternID','Ident','FaktorZuKWh')) + if (type(sid) is not int or sid<=0 or type(pid) is not int or pid<=0 + or sid in ids or ident in idents or not isinstance(ident,str) + or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}',ident) + or isinstance(factor,bool) or not isinstance(factor,(float,int)) + or not isfinite(factor) or not 0=0 else '')+str(power) + text=re.sub(r'"FaktorZuKWh":([0-9]+(?:\.[0-9]+)?)e([+-]?[0-9]+)',exponent,text) + return 'symcon-active-import:'+hashlib.sha256(text.encode()).hexdigest() + + +class MeterEvidence: + def __init__(self, series, *, native_sources=None): + self.series = list(series) + if not self.series or len({s.source_id for s in self.series}) != len(self.series): + raise ValueError('Unique, nonempty set of counter sources required') + self.native_bound = native_sources is not None + if self.native_bound: + self.meter_id=native_meter_id(native_sources) + if {s.source_id for s in self.series}!={'symcon:'+str(s['VariableID']) for s in native_sources}: + raise ValueError('Evidence stream does not match native source set') + else: + signature = json.dumps(sorted(s.source_id for s in self.series),separators=(',',':')) + self.meter_id = 'audit-only:' + hashlib.sha256(signature.encode()).hexdigest() + + def interval(self, start, end): + start, end = utc(start), utc(end) + parts = [s.energy(start,end) for s in self.series] + missing = [s.source_id for s,p in zip(self.series,parts) if p is None] + if missing: + return {'status':'missing', 'start':start.isoformat(),'end':end.isoformat(), + 'missingSources':missing, 'billingEvidence':False} + lo = sum(p['lowerKwh'] for p in parts) + hi = sum(p['upperKwh'] for p in parts) + seconds = (end-start).total_seconds() + exact = all(p['exactRecords'] for p in parts) + return {'status':'exact_records' if exact else 'bounded_records', + 'start':start.isoformat(),'end':end.isoformat(),'lowerKwh':lo,'upperKwh':hi, + 'lowerAverageKw':lo*3600/seconds,'upperAverageKw':hi*3600/seconds, + 'billingEvidence':all(p['billingEvidence'] for p in parts)} + + def month(self, at): + at = utc(at) + local = at.astimezone(ZURICH) + begin = utc(local.replace(day=1,hour=0,minute=0,second=0,microsecond=0)) + stop = quarter_start(at) + count = int((stop-begin).total_seconds())//900 + covered = exact = verified = 0 + lo = hi = 0. + gaps = [] + quarters = [] + for n in range(count): + start = begin + timedelta(minutes=15*n) + result = self.interval(start,start+timedelta(minutes=15)) + quarters.append(result) + if result['status'] == 'missing': + if len(gaps)<8:gaps.append(start.isoformat()) + continue + covered += 1 + exact += result['status']=='exact_records' + verified += result['billingEvidence'] + lo = max(lo,result['lowerAverageKw']) + hi = max(hi,result['upperAverageKw']) + complete = count>0 and covered==count + certified = count>0 and verified==count + return {'meterId':self.meter_id,'month':month_key(at),'completedQuarters':count, + 'coveredQuarters':covered,'exactRecordQuarters':exact,'verifiedQuarters':verified, + 'historyComplete':complete,'billingEvidence':certified, + 'observedPeakLowerKw':lo if covered else None, + 'observedPeakUpperKw':hi if covered else None, + 'monthPeakUpperKw':hi if complete else None, + 'verifiedMonthPeakKw':hi if certified else None, + 'firstMissingQuarters':gaps,'quarters':quarters} + + def strict_contract(self, at): + """Fail closed; do not promote archive gaps, estimated boundaries or a cap.""" + at = utc(at) + if not self.native_bound: + raise ValueError('Explicit native counter identity required') + result = self.month(at) + if not result['billingEvidence']: + raise ValueError('Full verified month history missing') + q = quarter_start(at) + past = None + if at != q: + partial = self.interval(q,at) + if not partial['billingEvidence']: + raise ValueError('Verified elapsed-quarter energy missing') + past = {'start':q.isoformat(),'measuredSeconds':int((at-q).total_seconds()), + 'importKwh':partial['upperKwh']} + return {'version':1,'meterId':self.meter_id,'measuredAt':at.isoformat(), + 'measuredPeaks':{month_key(at):{'kw':result['verifiedMonthPeakKw'], + 'source':'verified_month_history'}}, + 'quarterPast':past} + + +def audit_capture(payload): + """Read-only report from the dedicated Symcon export, never planner ingestion.""" + if payload.get('schemaVersion') != 1 or payload.get('kind') != 'v4_meter_capture': + raise ValueError('Unknown capture contract') + now = utc(payload['capturedAt']) + channels = payload['channels'] + series = [] + summaries = [] + for variable in ('59607','26620'): + channel = channels[variable] + rows = channel.get('history',[]) + readings = [Reading(utc(datetime.fromtimestamp(r['TimeStamp'],now.tzinfo)),r['Value']) for r in rows] + # An explicitly captured current value is usable from its capture, not + # from an earlier last-change timestamp, and never before logging began. + current = channel.get('snapshot') + if current and channel.get('snapshotConsistent'): + readings.append(Reading(utc(current['capturedAt']),current['value'])) + s = CounterSeries('symcon:'+variable, readings, timestamp_verified=False, + identity_verified=False, chronology_verified=False) + series.append(s) + summaries.append({'variableId':int(variable),'ident':channel['ident'], + 'logging':channel.get('logging'), 'rows':len(rows), + 'queryComplete':channel.get('queryComplete',False), + 'firstRecord':s.times[0].isoformat() if s.times else None, + 'lastRecord':s.times[-1].isoformat() if s.times else None, + 'counterDecreases':[t.isoformat() for t in s.resets]}) + meter = MeterEvidence(series) + month = meter.month(now) + current = None + q = quarter_start(now) + if now>q:current=meter.interval(q,now) + blockers=['Meter register identity/unit and acquisition timestamps still require verification.', + 'Change-only archive does not prove uninterrupted meter acquisition.'] + if any(not s['queryComplete'] for s in summaries):blockers.append('Archive export incomplete; do not infer complete history.') + if not month['historyComplete']:blockers.append('At least one completed billing quarter lacks bounded readings for every tariff.') + return {'capturedAt':now.isoformat(),'status':'audit_only','sourceSummary':summaries, + 'month':{k:v for k,v in month.items() if k!='quarters'}, + 'recentQuarters':month['quarters'][-8:], 'currentQuarter':current, + 'billingEvidence':False, 'liveEnabled':False,'blockers':blockers} diff --git a/services/netplan-v4/netplan_v4/optimizer.py b/services/netplan-v4/netplan_v4/optimizer.py new file mode 100644 index 0000000..8dabed5 --- /dev/null +++ b/services/netplan-v4/netplan_v4/optimizer.py @@ -0,0 +1,199 @@ +from __future__ import annotations +from math import sqrt +from uuid import uuid4 +import numpy as np +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import coo_matrix +from .domain import Battery, Limits, Step, month_key, number, quarter_start, utc +from .peak_policy import basis_record, RestMonthOutlook + +class Model: + def __init__(self): + self.lower,self.upper,self.cost,self.integer=[],[],[],[] + self.rows,self.row_lo,self.row_hi=[],[],[] + def variable(self, lower=0., upper=np.inf, cost=0., integer=False): + index=len(self.lower) + self.lower.append(lower);self.upper.append(upper);self.cost.append(cost);self.integer.append(int(integer)) + return index + def constraint(self, coefficients, lower=-np.inf, upper=np.inf): + self.rows.append(coefficients);self.row_lo.append(lower);self.row_hi.append(upper) + def matrices(self): + rr,cc,vv=[],[],[] + for row,coefficients in enumerate(self.rows): + for col,value in coefficients.items(): + rr.append(row);cc.append(col);vv.append(value) + matrix=coo_matrix((vv,(rr,cc)),shape=(len(self.rows),len(self.lower))).tocsr() + return matrix,np.array(self.row_lo),np.array(self.row_hi) + +from .battery_model import BatteryModel + +class AssetRegistry: + """Reviewed asset classes only; future EV/thermal models add their own constraints.""" + def __init__(self):self.models={Battery:BatteryModel} + def register(self,asset_type,implementation): + if asset_type in self.models:raise ValueError('Asset model already registered') + self.models[asset_type]=implementation + def build(self,model,asset,steps,direction): + if type(asset) not in self.models:raise ValueError('Unsupported asset model') + return self.models[type(asset)].build(model,asset,steps,direction) + +def _error(reason,status='invalid_inputs'): + return {'schemaVersion':2,'status':status,'executable':False,'points':[],'reason':str(reason)} + +def _validate(steps,assets,at,past,observed_peaks,peak_prices,limits): + if not steps or len(steps)>576:raise ValueError('Need 1..576 steps') + for i,step in enumerate(steps): + start=utc(step.start) + if type(step.seconds) is not int or not 1<=step.seconds<=300:raise ValueError('Interval duration must be 1..300 seconds') + if start.microsecond or step.end.second or step.end.microsecond or step.end.minute%5:raise ValueError('Intervals must end on a 5-minute boundary') + if i and (step.seconds!=300 or start.second or start.minute%5):raise ValueError('Only first interval may be shortened') + if i and start!=steps[i-1].end:raise ValueError('Missing/duplicated/overlapping interval') + number(step.base_load_w,'base load',0);number(step.pv_w,'PV',0);number(step.external_w,'external flow') + if not step.import_price or not step.export_price or not step.import_price.known_at(at) or not step.export_price.known_at(at):raise ValueError('Unknown or unpublished prices') + limits.import_limit(start) + if utc(at)!=utc(steps[0].start):raise ValueError('First interval must start at decision time') + if steps[-1].end!=quarter_start(steps[-1].end):raise ValueError('End horizon on complete billing quarter') + if limits.export_w is not None:number(limits.export_w,'export limit',0) + ids=set() + for asset in assets: + asset.validate(at) + if asset.asset_id in ids:raise ValueError('Duplicate asset_id') + ids.add(asset.asset_id) + q=quarter_start(steps[0].start);elapsed=int((utc(steps[0].start)-q).total_seconds()) + if any(key!=q for key in past):raise ValueError('Only elapsed energy of first quarter permitted') + if not elapsed and q in past and (past[q].import_kwh!=0 or past[q].measured_seconds!=0):raise ValueError('No elapsed energy at quarter boundary') + if elapsed: + if q not in past or past[q].measured_seconds!=elapsed:raise ValueError('Actual elapsed quarter import energy missing') + number(past[q].import_kwh,'quarter import energy',0) + for month in {month_key(s.start) for s in steps}: + if month not in observed_peaks or month not in peak_prices:raise ValueError(f'Measured peak state or tariff missing for {month}') + number(observed_peaks[month],'measured peak',0);number(peak_prices[month],'peak tariff',0) + +def optimize(steps,batteries,*,at,limits=None,observed_peaks=None,peak_prices=None,quarter_history=None,config_revision=1,family='3',timeout_seconds=30.,asset_registry=None,peak_context=None,peak_outlooks=None): + """Pure MILP. Peak state is measured, never a configured cap. No device calls.""" + limits=limits or Limits();observed_peaks=observed_peaks or {};peak_prices=peak_prices or {} + past={utc(k):v for k,v in (quarter_history or {}).items()} + try: + _validate(steps,batteries,at,past,observed_peaks,peak_prices,limits) + number(timeout_seconds,'solver timeout',.01,600) + contexts={} + for month in {month_key(s.start) for s in steps}: + if peak_context is None: + contexts[month]={'kw':observed_peaks[month],'quality':'verified','source':'legacy_verified_contract','observedAt':utc(at).isoformat()} + else: + contexts[month]=basis_record(peak_context[month],month,at,allow_estimates=True) + if abs(contexts[month]['kw']-observed_peaks[month])>1e-8:raise ValueError('Peak context differs from numerical basis') + outlooks=peak_outlooks or {} + if set(outlooks)-set(contexts):raise ValueError('Outlook for month outside horizon') + for month,outlook in outlooks.items(): + if not isinstance(outlook,RestMonthOutlook) or outlook.month!=month:raise ValueError('Invalid peak outlook') + outlook.validate(at,steps[-1].end) + except (ValueError,TypeError,KeyError,AttributeError) as exc:return _error(exc) + model=Model();direction=[model.variable(0,1,integer=True) for _ in steps];registry=asset_registry or AssetRegistry() + try:handles={b.asset_id:registry.build(model,b,steps,direction) for b in batteries} + except ValueError as exc:return _error(exc) + total_charge=sum(b.max_charge_w for b in batteries)/1000 + total_discharge=sum(b.max_discharge_w for b in batteries)/1000 + imp,exp,curtail=[],[],[];quarters={} + for i,step in enumerate(steps): + residual=step.residual_w/1000;dt=step.seconds/3600 + imax=max(0.,(step.base_load_w+step.external_w)/1000)+total_charge + emax=max(0.,-residual)+total_discharge;limit=limits.import_limit(step.start) + if limit is not None:imax=min(imax,limit/1000) + if limits.export_w is not None:emax=min(emax,limits.export_w/1000) + pi=model.variable(0,imax,step.import_price.chf_kwh*dt) + pe=model.variable(0,emax,-step.export_price.chf_kwh*dt) + pc=model.variable(0,step.pv_w/1000) + imp.append(pi);exp.append(pe);curtail.append(pc) + gm=model.variable(0,1,integer=True) + model.constraint({pi:1,gm:-imax},upper=0);model.constraint({pe:1,gm:emax},upper=emax) + balance={pi:1,pe:-1,pc:-1};pv_only={} + for b in batteries: + h=handles[b.asset_id];balance[h['charge'][i]]=-1;balance[h['discharge'][i]]=1 + if not b.grid_charging:pv_only[h['charge'][i]]=1 + model.constraint(balance,residual,residual) + if pv_only: + model.constraint(pv_only,upper=max(0.,-residual)) + no_grid_max=sum(b.max_charge_w for b in batteries if not b.grid_charging)/1000 + model.constraint({**pv_only,gm:no_grid_max},upper=no_grid_max) + quarters.setdefault(quarter_start(step.start),[]).append(i) + months=sorted({month_key(s.start) for s in steps}) + peaks={m:model.variable(observed_peaks[m]) for m in months} + for m in months: + if m in outlooks:outlooks[m].add_to_model(model,peaks[m],observed_peaks[m],peak_prices[m]) + else:model.cost[peaks[m]]=peak_prices[m] + for quarter,positions in quarters.items(): + coefficients={imp[i]:steps[i].seconds/900 for i in positions};coefficients[peaks[month_key(quarter)]]=-1 + used=past[quarter].import_kwh if quarter in past else 0. + model.constraint(coefficients,upper=-used/.25) + matrix,row_lo,row_hi=model.matrices() + try: + result=milp(np.array(model.cost),integrality=np.array(model.integer),bounds=Bounds(model.lower,model.upper),constraints=LinearConstraint(matrix,row_lo,row_hi),options={'time_limit':float(timeout_seconds),'mip_rel_gap':1e-4}) + except Exception as exc:return _error(f'Solver exception: {type(exc).__name__}','solver_error') + x=result.x + if result.status not in (0,1) or x is None:return _error(result.message,'no_feasible_plan') + if not np.isfinite(x).all():return _error('Non-finite solver result','validation_failed') + ax=matrix@x;tol=1e-6 + if (np.any(xnp.array(model.upper)+tol) or np.any(axrow_hi+tol) or any(abs(x[i]-round(x[i]))>tol for i,flag in enumerate(model.integer) if flag)): + return _error('Solver incumbent violates constraints','validation_failed') + running_peaks=dict(observed_peaks);baseline_peaks=dict(observed_peaks);peak_deltas={};baseline_peak_deltas={} + for quarter,positions in quarters.items(): + used=past[quarter].import_kwh if quarter in past else 0.;m=month_key(quarter) + demand=(used+sum(x[imp[i]]*steps[i].seconds/3600 for i in positions))/.25 + baseline=(used+sum(max(0.,steps[i].residual_w)*steps[i].seconds/3600000 for i in positions))/.25 + before=running_peaks[m];running_peaks[m]=max(before,demand);peak_deltas[positions[-1]]=(running_peaks[m]-before)*peak_prices[m] + before=baseline_peaks[m];baseline_peaks[m]=max(before,baseline);baseline_peak_deltas[positions[-1]]=(baseline_peaks[m]-before)*peak_prices[m] + points=[];cumulative_energy=cumulative_cash=cumulative_baseline_energy=cumulative_baseline_cash=cumulative_throughput=0. + for i,step in enumerate(steps): + dt=step.seconds/3600;p_import=max(0.,float(x[imp[i]]));p_export=max(0.,float(x[exp[i]])) + energy_cost=(p_import*step.import_price.chf_kwh-p_export*step.export_price.chf_kwh)*dt + baseline_import=max(0.,step.residual_w)/1000;baseline_export=max(0.,-step.residual_w)/1000 + if limits.export_w is not None:baseline_export=min(baseline_export,limits.export_w/1000) + baseline_cost=(baseline_import*step.import_price.chf_kwh-baseline_export*step.export_price.chf_kwh)*dt + targets,soc_end={},{};throughput_cost=0. + for b in batteries: + h=handles[b.asset_id];targets[b.asset_id]=float((x[h['charge'][i]]-x[h['discharge'][i]])*1000) + soc_end[b.asset_id]=float(100*x[h['energy'][i+1]]/b.capacity_kwh) + throughput_cost+=float((x[h['charge'][i]]+x[h['discharge'][i]])*dt*b.throughput_chf_kwh) + cumulative_energy+=energy_cost;cumulative_cash+=energy_cost+peak_deltas.get(i,0.) + cumulative_baseline_energy+=baseline_cost;cumulative_baseline_cash+=baseline_cost+baseline_peak_deltas.get(i,0.) + cumulative_throughput+=throughput_cost;battery_w=sum(targets.values()) + if battery_w>1:intent='gridCharge' if p_import>.001 else 'pvCharge' + elif battery_w< -1:intent='export' if p_export>.001 else 'discharge' + else:intent='hold' + points.append({'time':utc(step.start).isoformat(),'validUntil':step.end.isoformat(), + 'baselineGridW':float(step.residual_w),'gridTargetW':(p_import-p_export)*1000, + 'batteryTargetW':battery_w,'assetTargetsW':targets,'socEndPercent':soc_end, + 'pvCurtailmentW':float(x[curtail[i]]*1000),'intent':intent, + 'importLimitW':limits.import_limit(step.start),'exportLimitW':limits.export_w, + 'importPriceChfKwh':step.import_price.chf_kwh,'exportPriceChfKwh':step.export_price.chf_kwh, + 'energyCostChf':energy_cost,'additionalPeakCostChf':peak_deltas.get(i,0.), + 'throughputCostChf':throughput_cost,'cumulativeEnergyCostChf':cumulative_energy, + 'cumulativeCashCostChf':cumulative_cash,'cumulativeBaselineEnergyCostChf':cumulative_baseline_energy, + 'cumulativeBaselineCashCostChf':cumulative_baseline_cash,'baselineAdditionalPeakCostChf':baseline_peak_deltas.get(i,0.)}) + end_value=sum(float(x[handles[b.asset_id]['energy'][-1]])*b.terminal_value_chf_kwh for b in batteries) + def planning_cost(chosen): + return sum(outlooks[m].incremental_cost(observed_peaks[m],chosen[m],peak_prices[m]) if m in outlooks + else max(0.,chosen[m]-observed_peaks[m])*peak_prices[m] for m in months) + planning_peak=planning_cost(running_peaks) + full_peak=cumulative_cash-cumulative_energy + estimated=any(c['quality']=='estimated' for c in contexts.values()) + return {'schemaVersion':2,'planId':str(uuid4()),'configRevision':config_revision,'sourceFamily':family, + 'status':'optimal' if result.status==0 else 'feasible_time_limit','executable':True, + 'generatedAt':utc(at).isoformat(),'validFrom':points[0]['time'],'validUntil':points[-1]['validUntil'], + 'intervalMinutes':5,'points':points,'solverGap':float(result.mip_gap) if getattr(result,'mip_gap',None) is not None else None, + 'measuredPeaksKw':{m:observed_peaks[m] for m in months if contexts[m]['quality']=='verified'}, + 'peakBasisKw':{m:observed_peaks[m] for m in months},'peakBasis':contexts, + 'peakCostIsEstimate':estimated,'planningPeakCostChf':planning_peak, + 'restMonthAdjustmentChf':planning_peak-full_peak, + 'peakScenarioOutlook':{m:o.as_dict() for m,o in outlooks.items()}, + 'plannedPeaksKw':{m:running_peaks[m] for m in months}, + 'additionalPeakCostChf':cumulative_cash-cumulative_energy,'energyCostChf':cumulative_energy,'cashCostChf':cumulative_cash, + 'baselineEnergyCostChf':cumulative_baseline_energy,'baselineCashCostChf':cumulative_baseline_cash, + 'baselineAdditionalPeakCostChf':sum(baseline_peak_deltas.values()),'baselinePeaksKw':baseline_peaks, + 'throughputCostChf':cumulative_throughput,'terminalValueChf':end_value, + 'objectiveChf':cumulative_energy+planning_peak+cumulative_throughput-end_value, + 'baselinePlanningPeakCostChf':planning_cost(baseline_peaks), + 'cashCostMeaning':'Projected horizon energy plus full incremental monthly tariff, relative to the labelled peak basis; not an invoice', + 'terminalMinSocPercent':{b.asset_id:handles[b.asset_id]['terminal_min_percent'] for b in batteries}, + 'peakOutlook':'rest_month_scenarios' if outlooks else 'full_incremental_tariff'} diff --git a/services/netplan-v4/netplan_v4/peak_policy.py b/services/netplan-v4/netplan_v4/peak_policy.py new file mode 100644 index 0000000..30dad71 --- /dev/null +++ b/services/netplan-v4/netplan_v4/peak_policy.py @@ -0,0 +1,173 @@ +"""Economic peak policy. Planning assumptions NEVER become metering facts. + +Monthly marginal cost is convex. A rest-month scenario is a peak expected after +this horizon under a comparable control policy, not a configured limit, free +allowance, or a promise of savings. With insufficient evidence use full tariff. +""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime, timedelta +from math import isclose +from .domain import number, utc, month_key, ZURICH + +VERIFIED_SOURCES = frozenset({'meter_month_register','verified_month_history','verified_new_month'}) +ESTIMATE_SOURCES = frozenset({'power_history_estimate','sampled_power_estimate','counter_interval_estimate','operator_estimate','new_month'}) + + +def valid_month(value): + if not isinstance(value,str) or len(value)!=7: + raise ValueError('Calendar month YYYY-MM required') + if datetime.strptime(value,'%Y-%m').strftime('%Y-%m') != value: + raise ValueError('Invalid calendar month') + return value + + +def basis_record(value, month, at, allow_estimates=False): + """Strict provenance; missing stays missing. Caller decides explicit opt-in.""" + valid_month(month) + if not isinstance(value,dict) or set(value)-{'kw','quality','source','observedAt','notes','coverage','meterId'}: + raise ValueError('Invalid peak-basis schema') + kw=number(value.get('kw'),'peak basis kW',0,1e6) + quality=value.get('quality') + source=value.get('source') + if quality=='verified': + if source not in VERIFIED_SOURCES:raise ValueError('Not a verified peak source') + elif quality=='estimated': + if not allow_estimates or source not in ESTIMATE_SOURCES: + raise ValueError('Estimated planning basis not permitted or source invalid') + elif quality=='new_month': + if source!='new_month' or kw!=0 or month<=month_key(at): + raise ValueError('Zero future-month state is not a past measured peak') + else:raise ValueError('Peak quality must be explicit') + stamp=utc(value.get('observedAt')) + if stamp>utc(at):raise ValueError('Peak evidence from the future') + if quality!='new_month' and month>month_key(at): + raise ValueError('Future peak is an outlook, not historical evidence') + notes=value.get('notes','') + if not isinstance(notes,str) or len(notes)>500:raise ValueError('Invalid peak notes') + coverage=value.get('coverage') + if coverage is not None:number(coverage,'peak coverage',0,1) + mid=value.get('meterId') + if mid is not None and (not isinstance(mid,str) or not 1<=len(mid)<=160):raise ValueError('Invalid meter identity') + return {**value,'kw':kw,'quality':quality,'source':source,'observedAt':stamp.isoformat()} + + +@dataclass(frozen=True) +class PeakScenario: + future_peak_kw: float + probability: float + + +@dataclass(frozen=True) +class RestMonthOutlook: + month: str + scenarios: tuple[PeakScenario, ...] + issued_at: datetime + future_from: datetime + history_until: datetime + control_policy_id: str + method: str + evidence_id: str + reliance: float = 0.5 + + def validate(self, at, horizon_end): + valid_month(self.month) + at=utc(at);end=utc(horizon_end) + if utc(self.issued_at)>at or utc(self.history_until)>utc(self.issued_at): + raise ValueError('Rest-month outlook contains future information') + if at-utc(self.issued_at)>timedelta(days=2):raise ValueError('Stale rest-month outlook') + if utc(self.future_from)=end:return None + good={} + for r in daily_records: + if r.get('controlPolicyId')!=control_policy_id or r.get('complete') is not True:continue + if r.get('quality') not in ('verified','estimated'):continue + day=datetime.strptime(r['day'],'%Y-%m-%d').replace(tzinfo=ZURICH) + finished=utc(day+timedelta(days=1)) + observed=utc(r['observedAt']) + if finished>utc(at) or observed>utc(at) or observedtimedelta(days=90):continue + kw=number(r['peakKw'],'historical daily peak',0,1e6) + if r['day'] in good and good[r['day']]!=kw:raise ValueError('Conflicting daily peak evidence') + good[r['day']]=kw + if len(good)len(longest):longest=segment + segment=[] + segment.append(day) + if len(segment)>len(longest):longest=segment + if len(longest)67108864:raise ValueError('Archive batch exceeds memory budget') + ident=hashlib.sha256(raw).hexdigest();name=kind+'-'+ident+'.jsonl.gz';path=directory/name + if path.is_symlink():raise ValueError('Archive target symlink refused') + if not path.exists(): + fd,tmp=tempfile.mkstemp(prefix='.archive-',dir=directory) + try: + os.fchmod(fd,0o600) + with os.fdopen(fd,'wb') as out: + with gzip.GzipFile(fileobj=out,mode='wb',mtime=0) as zipped:zipped.write(raw) + out.flush();os.fsync(out.fileno()) + os.replace(tmp,path) + dfd=os.open(directory,os.O_RDONLY) + try:os.fsync(dfd) + finally:os.close(dfd) + finally: + if os.path.exists(tmp):os.unlink(tmp) + # Read back exact bytes before removing any database row. + with gzip.open(path,'rb') as f:verified=f.read(len(raw)+1) + if verified!=raw:raise ValueError('Archive verification failed; original rows retained') + for r in rows: + if kind=='observations': + key=(r['plant'],r['dataset'],r['captured_at']) + con.execute('DELETE FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',key) + con.execute('DELETE FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=? AND fingerprint=?',(*key,r['fingerprint'])) + else:con.execute('DELETE FROM planner_plans WHERE plan_id=?',(r['plan_id'],)) + con.execute('INSERT OR IGNORE INTO planner_archives VALUES(?,?,?,?,?,?)',(ident,now,name,ident,len(rows),kind)) + con.commit() + return {'archived':len(rows),'kind':kind,'file':name,'sha256':ident,'sourceRecoverable':True} + except Exception: + con.rollback();raise + + +def maintain(store,now): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return + con=store.con;stamp=int(now.timestamp());old=con.execute("SELECT checked_at FROM planner_maintenance WHERE name='archive'").fetchone() + if old and stamp-old[0]<3600:return + db=con.execute('PRAGMA database_list').fetchone()[2] + if not db:return # In-memory test/ephemeral databases have no archival location. + try: + directory=Path(db).resolve().parent/'archives' + results=[archive_batch(con,directory,stamp,kind) for kind in ('plans','observations')] + if any(r['archived']>=100 for r in results):stamp-=3300 + state='ok';detail={'results':results,'archivesRetained':True,'rawRetentionDays':120,'ordinaryPlanRetentionDays':7} + except (OSError,ValueError,sqlite3.Error) as exc: + state='archive_error';detail={'errorType':type(exc).__name__,'unverifiedDataNotDeleted':True} + with con:con.execute('INSERT INTO planner_maintenance VALUES(?,?,?,?) ON CONFLICT(name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,detail=excluded.detail', + ('archive',stamp,state,json.dumps(detail,separators=(',',':')))) + + +def status(con): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return {'status':'disabled_requires_operator_opt_in'} + r=con.execute("SELECT checked_at,status,detail FROM planner_maintenance WHERE name='archive'").fetchone() + # Public per-plant state must not expose archival filenames/rows of other tenants. + return {'checkedAtEpoch':r[0],'status':r[1],'rawRetentionDays':120,'ordinaryPlanRetentionDays':7,'archivesRetained':True} if r else {'status':'not_yet_run'} diff --git a/services/netplan-v4/netplan_v4/selection.py b/services/netplan-v4/netplan_v4/selection.py new file mode 100644 index 0000000..9da38d3 --- /dev/null +++ b/services/netplan-v4/netplan_v4/selection.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass +from datetime import timedelta +from .domain import default_registry,number,utc,ZURICH + +@dataclass(frozen=True) +class ReplayScore: + family:str + comparison_key:str + first_decision:object + last_decision:object + forecast_issued_at:object + actual_available_at:object + cost_chf:float + coverage:float + days:int + constraint_breaches:int=0 + terminal_normalized:bool=True + def valid(self): + number(self.cost_chf,'replay cost');number(self.coverage,'coverage',0,1) + return (utc(self.forecast_issued_at)<=utc(self.first_decision) and utc(self.actual_available_at)>=utc(self.last_decision) and self.terminal_normalized and self.constraint_breaches==0) + +def choose_family(setting,current,scores,*,registry=None,lookback_days=14,minimum_days=7,minimum_coverage=.9,margin_chf=1.,now): + registry=registry or default_registry() + if setting!='auto': + registry.get(setting) + return {'family':setting,'mode':'configured','reason':'Explicit installation setting'} + registry.get(current);number(margin_chf,'margin',0);valid=[] + for score in scores: + registry.get(score.family) + if (score.valid() and score.coverage>=minimum_coverage and score.days>=minimum_days + and utc(score.first_decision)>=utc(now)-timedelta(days=lookback_days) + and utc(score.last_decision)<=utc(now) and utc(score.actual_available_at)<=utc(now)): + valid.append(score) + groups={} + for score in valid: + key=(score.comparison_key,utc(score.first_decision),utc(score.last_decision),score.days) + groups.setdefault(key,{})[score.family]=score + complete=[g for g in groups.values() if len(g)==len(registry.entries())] + if not complete:return {'family':current,'mode':'collecting','reason':'Insufficient comparable out-of-sample replay evidence'} + group=max(complete,key=lambda g:utc(next(iter(g.values())).last_decision)) + best=min(group,key=lambda f:(group[f].cost_chf,f));improvement=group[current].cost_chf-group[best].cost_chf + chosen=best if improvement>margin_chf else current + return {'family':chosen,'mode':'economic_replay','improvementChf':improvement,'reason':'Matched historical cost replay; switching margin applied','costByFamilyChf':{k:v.cost_chf for k,v in group.items()}} + +def training_due(last_trained_at,now,cadence='daily'): + if cadence not in ('daily','weekly'):raise ValueError('Unknown training cadence') + if last_trained_at is None:return True + now,last=utc(now).astimezone(ZURICH),utc(last_trained_at).astimezone(ZURICH) + return (now.date()-last.date()).days >= (1 if cadence=='daily' else 7) + +def promote_candidate(*,active_cost,candidate_cost,valid_coverage,no_data_leakage,constraints_passed): + number(active_cost,'active cost');number(candidate_cost,'candidate cost') + return bool(valid_coverage and no_data_leakage and constraints_passed and candidate_cost20 or len({b.asset_id for b in result})!=len(result):raise ValueError('Duplicate/too many batteries') + return result + +def validate(kind,value,now,registry): + if kind not in KINDS or type(value.get('version')) is not int or value['version']!=1:raise ValueError('Unsupported event version/kind') + identifier=value['eventId'] + if not isinstance(identifier,str) or not 1<=len(identifier)<=160:raise ValueError('Event ID required') + observed=utc(value['observedAt']) + if observed>now+timedelta(seconds=30):raise ValueError('Observation from future') + fields={'version','eventId','observedAt'} + if kind=='forecast': + fields|={'families','modelVersions'} + if not value['families']:raise ValueError('No forecast families') + for key,family in value['families'].items(): + registry.get(key) + if family['loadBasis'] not in ('base_load','house_total'):raise ValueError('Explicit metering basis required') + evidence=family.get('accountingEvidenceId') + if evidence is not None and (family['loadBasis']!='base_load' or not isinstance(evidence,str) or not 8<=len(evidence)<=160):raise ValueError('Explicit base-load accounting evidence required') + if family.get('trainedUntil') and utc(family['trainedUntil'])>observed:raise ValueError('Training leakage') + if not 1<=len(family['points'])<=576:raise ValueError('Need 1..576 forecast intervals') + previous=None + for p in family['points']: + t=utc(p['time']) + if t.second or t.microsecond or t.minute%5:raise ValueError('Forecast interval alignment') + if previous and t-previous!=timedelta(minutes=5):raise ValueError('Forecast gap/overlap') + previous=t;number(p['pvW'],'PV',0,1e9);number(p['loadW'],'load',0,1e9);number(p.get('externalW',0.),'external',-1e9,1e9) + elif kind=='operation': + fields|={'gridW','meteringBoundary','batteries','limits','measuredPeaks','quarterPast','planningPeaks','quarterEstimate','meterObservation'} + if value['meteringBoundary']!='common_pcc':raise ValueError('Common metering boundary required') + number(value['gridW'],'grid W',-1e9,1e9) + for b in batteries(value['batteries']):b.validate(observed) + limits=value['limits'] + if set(limits)!={'importW','exportW','managerMonthLimitsW'}:raise ValueError('Explicit limits required; null unlimited, zero zero') + for k in ('importW','exportW'): + if limits[k] is not None:number(limits[k],k,0,1e9) + for k,v in limits['managerMonthLimitsW'].items(): + if str(int(k))!=k or not 1<=int(k)<=12:raise ValueError('Invalid manager month') + number(v,'manager limit',0,1e9) + for m,p in value.get('measuredPeaks',{}).items(): + datetime.strptime(m,'%Y-%m');number(p['kw'],'peak kW',0) + if m>month_key(observed) or p['source'] not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Measured peak source invalid; cap is not paid peak') + past=value.get('quarterPast') + if past: + q=quarter_start(observed) + if utc(past['start'])!=q or type(past['measuredSeconds']) is not int or past['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Quarter measurement timestamp mismatch') + number(past['importKwh'],'quarter energy',0) + for m,p in value.get('planningPeaks',{}).items(): + record=basis_record(p,m,observed,allow_estimates=True) + if record['quality']!='estimated':raise ValueError('planningPeaks contains estimates only') + estimate=value.get('quarterEstimate') + if estimate: + if set(estimate)-{'start','measuredSeconds','importKwh','quality','source','coverage','notes'}:raise ValueError('Unknown quarter estimate field') + if estimate.get('quality')!='estimated' or estimate.get('source') not in ('power_history_estimate','sampled_power_estimate','counter_interval_estimate'):raise ValueError('Explicit quarter estimate provenance required') + q=quarter_start(observed) + if utc(estimate['start'])!=q or type(estimate['measuredSeconds']) is not int or estimate['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Estimated quarter timing mismatch') + number(estimate['importKwh'],'estimated quarter energy',0) + if number(estimate.get('coverage'),'quarter estimate coverage',0,1)<1.:raise ValueError('Missing current-quarter coverage') + if value.get('meterObservation') is not None:meter_runtime.validate_observation(value['meterObservation'],observed) + elif kind=='planning_basis': + fields|={'peaks'} + if not isinstance(value.get('peaks'),dict) or not value['peaks']:raise ValueError('Explicit peak estimates required') + for m,p in value['peaks'].items(): + if basis_record(p,m,observed,allow_estimates=True)['quality']!='estimated':raise ValueError('Planning basis is not a metering import') + elif kind=='peak_outlook': + fields|={'outlooks'} + for m,v in value['outlooks'].items(): + outlook=RestMonthOutlook.from_dict(v) + if m!=outlook.month:raise ValueError('Outlook month mismatch') + outlook.validate(observed,observed) + elif kind=='tariffs': + fields|={'import','export','peakChfKwMonth'} + for side in ('import','export'): + p=value[side] + if p['mode'] not in ('static','dynamic') or not p['tariffId']:raise ValueError('Explicit price mode/id required') + if p['mode']=='static':number(p['staticChfKwh'],'static price') + peaks=value['peakChfKwMonth'] + if isinstance(peaks,dict): + for m,v in peaks.items():datetime.strptime(m,'%Y-%m');number(v,'peak tariff',0) + else:number(peaks,'peak tariff',0) + else: + fields|={'periods'} + if len(value['periods'])>3000:raise ValueError('Too many price intervals') + for p in value['periods']: + if p['unit'] not in ('CHF/kWh','CHF_kWh','Rp/kWh','CHF/MWh') or p['side'] not in ('import','export'):raise ValueError('Explicit price unit/direction required') + if p['sourceKind'] not in ('published_interval','estimate','carried_forward'):raise ValueError('Explicit price provenance required') + number(p['value'],'price') + if utc(p['end'])<=utc(p['start']) or utc(p['observedAt'])>observed:raise ValueError('Invalid price interval or observation') + if p.get('publishedAt') and utc(p['publishedAt'])>utc(p['observedAt']):raise ValueError('Price not published when observed') + if set(value)-fields:raise ValueError('Unknown fields: extra device data/credentials must not be submitted') + canonical(value) + +def ingest(store,plant,kind,value,now): + validate(kind,value,now,store.registry);data=canonical(value);con=store.con;con.execute('BEGIN IMMEDIATE') + try: + old=con.execute('SELECT value FROM planner_inputs WHERE plant=? AND kind=? AND event_id=?',(plant,kind,value['eventId'])).fetchone() + if old: + if old[0]!=data:raise ValueError('Immutable event conflict') + con.commit();return {'status':'duplicate','queued':False} + current=latest(store,plant,kind) + if current and utc(current['observedAt'])==utc(value['observedAt']): + left,right=dict(current),dict(value);left.pop('eventId');right.pop('eventId') + if canonical(left)!=canonical(right):raise ValueError('Conflicting simultaneous observations') + con.execute('INSERT INTO planner_inputs VALUES(?,?,?,?,?)',(plant,kind,value['eventId'],utc(value['observedAt']).isoformat(timespec='microseconds'),data)) + newer=not current or utc(current['observedAt'])<=utc(value['observedAt']) + if newer: + if kind=='operation': + known=store.peaks(plant) + for m,p in value.get('measuredPeaks',{}).items(): + if m in known and p['kw']=end and utc(p['observedAt'])<=now): + factor={'CHF/kWh':1.,'CHF_kWh':1.,'Rp/kWh':.01,'CHF/MWh':.001}[p['unit']] + candidates.append((utc(p['observedAt']),p['value']*factor)) + if not candidates:return None + last=max(t for t,v in candidates);values={v for t,v in candidates if t==last} + if len(values)!=1:raise ValueError('Conflicting published price intervals') + return Price(values.pop(),last,'dynamic') + +class AwaitingInput(ValueError):pass + +def assemble(store,plant,family,now): + values={k:latest(store,plant,k) for k in ('operation','forecast','tariffs')} + for k,v in values.items(): + if not v:raise AwaitingInput('Missing '+k+' input') + op,forecast,tariffs=(values[k] for k in ('operation','forecast','tariffs')) + decision=utc(op['observedAt']).replace(microsecond=0) + settings=store.settings(plant);allow_estimates=settings['measurementPolicy']=='allow_estimates' + input_quality={'quarter':'verified','warnings':[]} + if not 0<=(now-decision).total_seconds()<=120:raise AwaitingInput('Fresh manager observation required (120s maximum)') + if (now-utc(forecast['observedAt'])).total_seconds()>5400:raise AwaitingInput('Forecast older than 90 minutes') + if utc(forecast['observedAt'])>decision or utc(tariffs['observedAt'])>decision:raise AwaitingInput('New data awaiting fresh manager observation') + if settings['forecastSource']=='corrected_profile': + try:forecast=measurement_pipeline.apply_load_forecast(store.con,plant,settings['measurementDataset'],forecast,int(decision.timestamp())) + except ValueError as exc:raise AwaitingInput(str(exc)) from exc + if family not in forecast['families']:raise AwaitingInput('Chosen family is unavailable; no silent switch') + source=forecast['families'][family];steps=[] + future_source={**source,'points':[p for p in source['points'] if utc(p['time'])+timedelta(minutes=5)>decision]} + assessment=assess_family(future_source) + if not assessment['valid']:raise AwaitingInput(assessment['reason']) + input_quality['forecastAssessment']=assessment + if source.get('dataPipeline'): + input_quality['dataPipeline']=source['dataPipeline'] + input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule') + for p in source['points']: + start=utc(p['time']);end=start+timedelta(minutes=5) + if end<=decision:continue + start=max(start,decision) + external=p.get('externalW',0.) if source['loadBasis']=='base_load' else 0. + steps.append(Step(start,p['loadW'],p['pvW'],price_at(store,plant,'import',tariffs['import'],start,end,decision),price_at(store,plant,'export',tariffs['export'],start,end,decision),external,int((end-start).total_seconds()))) + if not steps or steps[0].start!=decision:raise AwaitingInput('Forecast has no current interval') + full_end=steps[-1].end;steps=priced_prefix(steps,decision) + if not steps:raise AwaitingInput('No complete published-price billing quarter') + q=quarter_start(decision);elapsed=int((decision-q).total_seconds());past={} + if elapsed: + p=op.get('quarterPast') + if not p and allow_estimates: + p=op.get('quarterEstimate') + if not p and op.get('meterObservation'):p=meter_runtime.current_quarter(store.con,plant,op['meterObservation'],decision) + if p: + input_quality['quarter']='estimated' + input_quality['warnings'].append('Current quarter uses an explicitly estimated energy value, not a billing measurement') + if not p or utc(p['start'])!=q or p['measuredSeconds']!=elapsed:raise AwaitingInput('Current-quarter energy missing (measured or explicitly permitted estimate)') + past[q]=QuarterPast(p['importKwh'],elapsed) + peaks=store.peaks(plant);months={month_key(s.start) for s in steps};contexts={} + estimates=meter_runtime.assumptions(store.con,plant) + for m in months: + if m>month_key(decision): + peaks[m]=0.;contexts[m]={'kw':0.,'quality':'new_month','source':'new_month','observedAt':decision.isoformat()} + elif m in peaks: + r=store.con.execute('SELECT source,updated_at FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone() + known_at=op['observedAt'] if m in op.get('measuredPeaks',{}) else r['updated_at'] + contexts[m]={'kw':peaks[m],'quality':'verified','source':r['source'],'observedAt':known_at} + # A later acquired larger quarter may raise an older verified baseline, + # but the resulting combined planning basis must then say estimated. + if allow_estimates and m in estimates and estimates[m]['kw']>peaks[m] and utc(estimates[m]['observedAt'])>utc(known_at): + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True) + peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('A newer sampled quarter increased the earlier verified peak baseline; current planning maximum is estimated') + elif allow_estimates and m in estimates: + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True);peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('Monthly peak '+m+' is a planning estimate, not an authoritative billing maximum') + else:raise AwaitingInput('Peak basis missing; supply a verified maximum or explicitly permit a labelled estimate') + peak_prices=tariffs['peakChfKwMonth'] + if not isinstance(peak_prices,dict):peak_prices={m:peak_prices for m in months} + lim=op['limits'];limits=Limits(lim['exportW'],lim['importW'],{int(k):v for k,v in lim['managerMonthLimitsW'].items()}) + outlooks={};horizon_end=steps[-1].end + if settings['peakOutlookPolicy']=='empirical_if_available': + explicit=latest(store,plant,'peak_outlook') + for m in months: + if explicit and m in explicit['outlooks']: + candidate=RestMonthOutlook.from_dict(explicit['outlooks'][m]) + try: + candidate.validate(decision,horizon_end) + policy=op.get('meterObservation',{}).get('controlPolicyId') + if not policy or candidate.control_policy_id!=policy:raise ValueError('Different or unknown control policy') + except ValueError:input_quality['warnings'].append('Stale, overlapping or incomparable rest-month outlook ignored; full incremental tariff used') + else:outlooks[m]=replace(candidate,reliance=min(candidate.reliance,settings['peakOutlookReliance'])) + elif op.get('meterObservation'): + observation=op['meterObservation'] + candidate=empirical_rest_month(meter_runtime.daily_peaks(store.con,plant,observation['meterId'],decision), + month=m,at=decision,horizon_end=horizon_end,control_policy_id=observation['controlPolicyId'], + reliance=settings['peakOutlookReliance']) + if candidate is not None:outlooks[m]=candidate + if not outlooks:input_quality['warnings'].append('Insufficient comparable rest-month history; full incremental peak tariff used, no arbitrary free peak allowance') + data={'steps':steps,'batteries':batteries(op['batteries']),'limits':limits,'observed_peaks':peaks,'peak_prices':peak_prices, + 'quarter_history':past,'at':decision,'peak_context':contexts,'peak_outlooks':outlooks} + if source['loadBasis']=='base_load' and source.get('accountingEvidenceId'): + input_quality['accountingEvidenceId']=source['accountingEvidenceId'] + return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)} + +def run_once(store,now): + retention.maintain(store,now) + stamp=int(now.timestamp())//300 + plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')] + for plant in plants: + config=store.settings(plant) + economic_replay.advance(store,plant,now) + datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))] + for dataset in datasets: + try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp())) + except ValueError as exc:logging.getLogger(__name__).warning('Data pipeline unavailable for configured dataset: %s',type(exc).__name__) + with store.con: + old=store.con.execute('SELECT tick FROM planner_ticks WHERE plant=?',(plant,)).fetchone() + if not old or old[0]!=stamp: + store._request(plant,store.settings(plant)['revision'],'five_minute_tick',now) + store.con.execute('INSERT INTO planner_ticks VALUES(?,?) ON CONFLICT(plant) DO UPDATE SET tick=excluded.tick',(plant,stamp)) + claim=store.claim(now) + if not claim:return {'status':'idle'} + plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]} + try: + settings=store.settings(plant);previous=store.current(plant) + current=previous['sourceFamily'] if previous else store.registry.entries()[0].key + selection=choose_family(settings['family'],current,economic_replay.scores(store,plant,now),registry=store.registry,now=now, + lookback_days=settings['autoLookbackDays'],minimum_days=settings['autoMinimumDays'], + minimum_coverage=settings['autoMinimumCoverage'],margin_chf=settings['autoSwitchMarginChf']) + data,full_end,quality=assemble(store,plant,selection['family'],now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + result=optimize(**data,config_revision=settings['revision'],family=selection['family']) + if result['executable']: + result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])}) + store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token']) + economic_replay.capture(store,plant,now,assemble,optimize) + return result + except (ValueError,TypeError,KeyError) as exc: + result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]} + return result + finally: + detail={k:v for k,v in result.items() if k in ('status','reason','planId','configRevision','sourceFamily')} + with store.con:store.con.execute('INSERT INTO planner_run_status VALUES(?,?,?,?) ON CONFLICT(plant) DO UPDATE SET updated_at=excluded.updated_at,status=excluded.status,detail=excluded.detail',(plant,now.isoformat(),result['status'],canonical(detail))) + store.finish(claim) + +def status(store,plant,now): + plan=store.current(plant);settings=store.settings(plant) + row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone() + pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone() + ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone() + fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit')) + return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)} + +def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()): + allowed={str(UUID(p)) for p in plants} + trial_allowed={str(UUID(p)) for p in controlled_trial_plants} + if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist') + if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required') + path=Path(db_path).resolve() + if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required') + path.parent.mkdir(parents=True,exist_ok=True);stop=Event() + def factory():return PlannerStore(str(path)) + def loop(): + while not stop.is_set(): + s=factory() + try:run_once(s,datetime.now(timezone.utc).replace(microsecond=0)) + except Exception as exc:logging.getLogger(__name__).error('V4 worker error: %s',type(exc).__name__) + finally:s.close() + stop.wait(1.) + @asynccontextmanager + async def lifespan(app): + thread=Thread(target=loop,name='v4-shadow',daemon=True) + if start_worker:thread.start() + yield + stop.set() + if start_worker:thread.join(35) + app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan) + app.state.store_factory=factory + def authorize(plant,token): + if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized') + try:plant=str(UUID(plant)) + except ValueError:raise HTTPException(400,'Invalid installation ID') + if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial') + return factory() + @app.get('/health') + def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1,'applicationRelease':'unified-rc1','mappingCompatibilityVersion':1,'economicReplayVersion':1,'archiveVersion':1} + @app.get('/internal/v2/prognosis/{plant}/planner') + def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc);view=status(s,plant,now) + view['controlledTrial']=controlled_trial.authority(s,plant,view,now,trial_allowed) + view['controlledTrialAuthorized']=view['controlledTrial'] is not None + return view + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/arm') + def arm_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + return controlled_trial.arm(s,plant,payload,status(s,plant,now),now,trial_allowed) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409,str(exc)[:300]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/revoke') + def revoke_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if set(payload)!={'sessionId'}:raise ValueError('Session ID only') + return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}/mapping-compatibility') + def mapping_compatibility(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + from .mapping_identity import register + cfg=measurement_pipeline.configuration(s.con,plant,dataset) + return register(s.con,plant,cfg,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError,AttributeError): + raise HTTPException(400, 'Mapping compatibility proof rejected') + finally:s.close() + + @app.put('/internal/v2/prognosis/{plant}/planner/settings') + def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return s.save_settings(plant,payload['changes'],payload['expectedRevision'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409 if 'Revision conflict' in str(exc) else 400,str(exc)) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/replan') + def replan(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.request(plant,'manual',datetime.now(timezone.utc));return {'status':'queued','liveEnabled':False} + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/inputs/{kind}') + def input_event(plant:str,kind:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return ingest(s,plant,kind,payload,datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}') + def configure_dataset(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if payload.get('datasetId')!=dataset:raise ValueError('Dataset path/payload mismatch') + return measurement_pipeline.register_dataset(s.con,plant,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/measurements') + def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp())) + # Existing five-minute worker handles rollup/training; no per-record optimizer flood. + return result + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/ack') + def ack(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.acknowledge(plant,payload['planId'],payload['revision'],datetime.now(timezone.utc),payload.get('step'),payload.get('status','shadow_seen'));return {'status':'recorded'} + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + from starlette.responses import JSONResponse + class BodyLimit: + def __init__(self,app):self.app=app + async def __call__(self,scope,receive,send): + if scope['type']!='http' or scope['method'] not in ('POST','PUT'):return await self.app(scope,receive,send) + chunks=[];total=0 + while True: + msg=await receive() + if msg['type']=='http.disconnect':return + total+=len(msg.get('body',b'')) + if total>2000000:return await JSONResponse({'detail':'Request too large'},status_code=413)(scope,receive,send) + chunks.append(msg) + if not msg.get('more_body',False):break + async def replay():return chunks.pop(0) if chunks else await receive() + return await self.app(scope,replay,send) + app.add_middleware(BodyLimit) + return app + +def from_environment(): + return create_app(os.environ.get('NETPLAN_V4_DB','/data/netplan-v4.sqlite'),os.environ.get('PROGNOSIS_SERVICE_TOKEN',''),[p.strip() for p in os.environ.get('NETPLAN_V4_PLANTS','').split(',') if p.strip()], controlled_trial_plants=[p.strip() for p in os.environ.get('NETPLAN_V4_CONTROL_TRIAL_PLANTS','').split(',') if p.strip()]) diff --git a/services/netplan-v4/netplan_v4/store.py b/services/netplan-v4/netplan_v4/store.py new file mode 100644 index 0000000..236e376 --- /dev/null +++ b/services/netplan-v4/netplan_v4/store.py @@ -0,0 +1,149 @@ +from __future__ import annotations +import json +import sqlite3 +from datetime import datetime,timedelta +from uuid import uuid4 +from .domain import default_registry,month_key,number,quarter_start,utc +from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline + +def canonical(value): + return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False) + +DEFAULT_SETTINGS={'family':'3','autoLookbackDays':14,'autoMinimumDays':7,'autoMinimumCoverage':.9,'autoSwitchMarginChf':1.,'tariffPolicy':'published_only','trainingCadence':'daily','trainingPromotion':'validated_only','runMode':'shadow','roundtripEfficiency':.90,'measurementPolicy':'verified_only','peakOutlookPolicy':'empirical_if_available','peakOutlookReliance':.5,'forecastSource':'legacy','measurementDataset':''} + +class PlannerStore: + """Own SQLite file, no mutation of legacy application databases.""" + def __init__(self,path,registry=None): + self.registry=registry or default_registry();self.con=sqlite3.connect(path,timeout=10) + self.con.row_factory=sqlite3.Row;self.con.execute('PRAGMA foreign_keys=ON') + self.con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_settings(plant TEXT PRIMARY KEY,revision INTEGER NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_audit(id INTEGER PRIMARY KEY,plant TEXT NOT NULL,at TEXT NOT NULL,kind TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_work(plant TEXT PRIMARY KEY,sequence INTEGER NOT NULL,revision INTEGER NOT NULL,reasons TEXT NOT NULL,requested_at TEXT NOT NULL,lease_until TEXT,lease_token TEXT); + CREATE TABLE IF NOT EXISTS planner_measurements(plant TEXT NOT NULL,start TEXT NOT NULL,import_kwh REAL NOT NULL,PRIMARY KEY(plant,start)); + CREATE TABLE IF NOT EXISTS planner_month_peaks(plant TEXT NOT NULL,month TEXT NOT NULL,peak_kw REAL NOT NULL,source TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_snapshots(id TEXT PRIMARY KEY,plant TEXT NOT NULL,issued_at TEXT NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_plans(plan_id TEXT PRIMARY KEY,plant TEXT NOT NULL,revision INTEGER NOT NULL,mode TEXT NOT NULL,value TEXT NOT NULL,created_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_current(plant TEXT NOT NULL,mode TEXT NOT NULL,plan_id TEXT NOT NULL REFERENCES planner_plans(plan_id),PRIMARY KEY(plant,mode)); + CREATE TABLE IF NOT EXISTS planner_ack(plant TEXT PRIMARY KEY,plan_id TEXT NOT NULL,revision INTEGER NOT NULL,received_at TEXT NOT NULL,applied_step TEXT,status TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_inputs(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,observed_at TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(plant,kind,event_id)); + CREATE TABLE IF NOT EXISTS planner_input_current(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,PRIMARY KEY(plant,kind)); + CREATE TABLE IF NOT EXISTS planner_run_status(plant TEXT PRIMARY KEY,updated_at TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_ticks(plant TEXT PRIMARY KEY,tick INTEGER NOT NULL); + ''') + meter_runtime.schema(self.con) + controlled_trial.schema(self.con) + economic_replay.schema(self.con) + retention.schema(self.con) + measurement_pipeline.schema(self.con) + def close(self):self.con.close() + def settings(self,plant): + row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone() + return {'revision':row['revision'],**DEFAULT_SETTINGS,**json.loads(row['value'])} if row else {'revision':0,**DEFAULT_SETTINGS} + def _validate_settings(self,value): + if set(value)!=set(DEFAULT_SETTINGS):raise ValueError('Unknown or missing setting') + if value['family']!='auto':self.registry.get(value['family']) + for key in ('autoLookbackDays','autoMinimumDays'): + if type(value[key]) is not int:raise ValueError('Days must be integers') + number(value['autoLookbackDays'],'lookback',7,90);number(value['autoMinimumDays'],'minimum days',1,value['autoLookbackDays']) + number(value['autoMinimumCoverage'],'coverage',.5,1);number(value['autoSwitchMarginChf'],'margin',0) + if value['tariffPolicy']!='published_only':raise ValueError('Only published-price policy implemented') + if value['trainingCadence'] not in ('daily','weekly') or value['trainingPromotion']!='validated_only':raise ValueError('Training must use validated promotion') + if value['runMode']!='shadow':raise ValueError('Shadow-only: live release requires separate validation') + number(value['roundtripEfficiency'],'roundtrip efficiency',.01,1) + if value['measurementPolicy'] not in ('verified_only','allow_estimates'):raise ValueError('Invalid measurement policy') + if value['peakOutlookPolicy'] not in ('full_incremental','empirical_if_available'):raise ValueError('Invalid peak outlook policy') + number(value['peakOutlookReliance'],'peak outlook reliance',0,1) + if value['forecastSource'] not in ('legacy','corrected_profile'):raise ValueError('Unknown forecast source') + if not isinstance(value['measurementDataset'],str) or len(value['measurementDataset'])>80:raise ValueError('Invalid measurement dataset') + if value['forecastSource']=='corrected_profile' and not value['measurementDataset']:raise ValueError('Corrected forecast requires an explicit dataset') + def _request(self,plant,revision,reason,now): + row=self.con.execute('SELECT * FROM planner_work WHERE plant=?',(plant,)).fetchone() + reasons=set(json.loads(row['reasons'])) if row else set();reasons.add(reason) + seq=row['sequence']+1 if row else 1 + self.con.execute('''INSERT INTO planner_work(plant,sequence,revision,reasons,requested_at) VALUES(?,?,?,?,?) + ON CONFLICT(plant) DO UPDATE SET sequence=excluded.sequence,revision=excluded.revision,reasons=excluded.reasons,requested_at=excluded.requested_at''',(plant,seq,revision,canonical(sorted(reasons)),utc(now).isoformat())) + def save_settings(self,plant,changes,expected_revision,now): + if type(expected_revision) is not int or expected_revision<0:raise ValueError('Invalid expected revision') + self.con.execute('BEGIN IMMEDIATE') + try: + current=self.settings(plant) + if current.pop('revision')!=expected_revision:raise ValueError('Revision conflict; reload before saving') + current.update(changes);self._validate_settings(current);revision=expected_revision+1 + self.con.execute('''INSERT INTO planner_settings VALUES(?,?,?) ON CONFLICT(plant) DO UPDATE SET revision=excluded.revision,value=excluded.value''',(plant,revision,canonical(current))) + self._request(plant,revision,'configuration_changed',now) + self.con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',(plant,utc(now).isoformat(),'settings',canonical({'revision':revision,'changes':changes}))) + self.con.commit();return {'revision':revision,**current} + except Exception:self.con.rollback();raise + def request(self,plant,reason,now): + if reason not in ('prices_changed','telemetry_changed','five_minute_tick','manual','model_promoted','forecast_changed','operation_changed','tariffs_changed'):raise ValueError('Unknown trigger') + with self.con:self._request(plant,self.settings(plant)['revision'],reason,now) + def claim(self,now,lease_seconds=120): + at=utc(now);self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT * FROM planner_work WHERE lease_until IS NULL OR lease_until < ? ORDER BY requested_at LIMIT 1',(at.isoformat(),)).fetchone() + if not row:self.con.commit();return None + token=str(uuid4());self.con.execute('UPDATE planner_work SET lease_until=?,lease_token=? WHERE plant=?',((at+timedelta(seconds=lease_seconds)).isoformat(),token,row['plant'])) + self.con.commit();return {**dict(row),'lease_token':token} + except Exception:self.con.rollback();raise + def finish(self,claim): + self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(claim['plant'],)).fetchone() + if not row or row['lease_token']!=claim['lease_token']:self.con.commit();return False + if row['sequence']==claim['sequence']:self.con.execute('DELETE FROM planner_work WHERE plant=?',(claim['plant'],)) + else:self.con.execute('UPDATE planner_work SET lease_until=NULL,lease_token=NULL WHERE plant=?',(claim['plant'],)) + self.con.commit();return True + except Exception:self.con.rollback();raise + def initialize_peak(self,plant,month,peak_kw,source,now): + number(peak_kw,'authoritative measured peak',0) + if source not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Configured cap is NOT measured peak') + datetime.strptime(month,'%Y-%m') + if month>month_key(now):raise ValueError('Future month cannot have a measured peak') + with self.con: + old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,month)).fetchone() + if old and peak_kwreceived_at:raise ValueError('Completed aligned intervals required') + with self.con: + old=self.con.execute('SELECT import_kwh FROM planner_measurements WHERE plant=? AND start=?',(plant,start.isoformat())).fetchone() + if old and abs(old[0]-import_kwh)>1e-9:raise ValueError('Conflicting metering fact') + self.con.execute('INSERT OR IGNORE INTO planner_measurements VALUES(?,?,?)',(plant,start.isoformat(),import_kwh)) + q=quarter_start(start);rows=self.con.execute('SELECT start,import_kwh FROM planner_measurements WHERE plant=? AND start>=? AND start8000000:raise RuntimeError('Probe execution failed or oversized') + probe=json.loads(result.stdout) + save(reports/'native-inputs.json',probe) + report['nativeInputStatus']=probe.get('status') + report['forecastSummary']=probe.get('forecastSummary',{}) + report['dataBlockers']=[] + for key,value in report['forecastSummary'].items(): + if key in ('prog_var_2','prog_var_11','prog_var_22') and (not value.get('points') or value.get('allZero')): + report['dataBlockers'].append(key+': missing or unconfirmed all-zero load forecast') + recent=probe.get('inputFrames',{}).get('df_recent_raw',{}).get('columns',{}).get('Hausverbrauch',{}).get('lastFiniteAt') + if not recent or (datetime.now(timezone.utc)-datetime.fromisoformat(recent)).total_seconds()>1800: + report['dataBlockers'].append('Recent raw load telemetry missing or older than 30 minutes') + if probe.get('status')!='read_only_acquired' or report['dataBlockers']:success=False + except (ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + report['nativeInputStatus']='failed';report['nativeInputErrorType']=type(error).__name__;success=False + try: + verify_native_bundle() + report['sourceStableDuringTests']=(source_manifest()==report['sourceHashes'] and verify_forecast_bundle(ROOT/'acceptance/forecast-src',PROJECT/'forecast_engine')==forecast_hashes) + except (OSError,ValueError): + report['sourceStableDuringTests']=False + if not report['sourceStableDuringTests']: + report.setdefault('dataBlockers',[]).append('Source changed during checks; results cannot certify current candidate') + success=False + report['preflightChecksPassed']=success + save(reports/'REPORT.json',report) + save(ROOT/'acceptance-reports'/'LATEST.json',{'report':str(reports/'REPORT.json')}) + print('\nPRECHECK '+('PASSED' if success else 'NEEDS REVIEW')+'; NOT a production release.') + print('REPORT: '+str(reports/'REPORT.json')) + for key,value in report.get('forecastSummary',{}).items():print(key,json.dumps(value)) + for blocker in report.get('dataBlockers',[]):print('DATA BLOCKER:',blocker) + print('Existing containers, Symcon settings, timers and actuators unchanged.') + return 0 if success else 1 + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda v:str(UUID(v))) + parser.add_argument('--show-commands',action='store_true') + args=parser.parse_args() + if args.show_commands: + for name,command,cwd,timeout in command_list(args.plant):print(name,json.dumps(command)) + print('Native input probe: read-only docker compose exec forecast-engine python with reviewed stdin script') + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not alter Docker socket permissions.') + try:raise SystemExit(run(args.plant)) + except (OSError,ValueError) as error:raise SystemExit('Preflight stopped safely: '+str(error)) diff --git a/services/netplan-v4/requirements.txt b/services/netplan-v4/requirements.txt new file mode 100644 index 0000000..595f8e7 --- /dev/null +++ b/services/netplan-v4/requirements.txt @@ -0,0 +1,5 @@ +numpy>=2.2,<3 +scipy>=1.15,<1.18 +fastapi>=0.115,<0.129 +httpx>=0.27,<0.29 +uvicorn>=0.30,<0.41 diff --git a/services/netplan-v4/run_tests.py b/services/netplan-v4/run_tests.py new file mode 100644 index 0000000..4876825 --- /dev/null +++ b/services/netplan-v4/run_tests.py @@ -0,0 +1,21 @@ +"""Run against the exact server working tree; no live data or credentials used.""" +from pathlib import Path +import hashlib,json,os,sys,unittest +ROOT=Path(__file__).resolve().parent +DEPS=Path('/home/agent/services/qa/netplan-v4-candidate-20261001/python-dependencies') +if DEPS.exists():sys.path.insert(0,str(DEPS)) +sys.path.insert(0,str(ROOT)) +if __name__=='__main__': + import numpy,scipy,fastapi,httpx + versions={'python':sys.version.split()[0],**{m.__name__:m.__version__ for m in (numpy,scipy,fastapi,httpx)}} + print('Runtime',json.dumps(versions),flush=True) + reports=Path(os.getenv('NETPLAN_V4_TEST_REPORT_DIR',str(ROOT))) + reports.mkdir(parents=True,exist_ok=True) + suite=unittest.defaultTestLoader.discover(str(ROOT/'tests')) + with (reports/'TEST_RESULTS.txt').open('w') as output: + output.write('Actual server/target tests; synthetic data only.\n'+json.dumps(versions)+'\n\n') + result=unittest.TextTestRunner(stream=output,verbosity=2).run(suite) + print((reports/'TEST_RESULTS.txt').read_text()) + hashes={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(ROOT.rglob('*.py')) if '__pycache__' not in p.parts} + (reports/'TESTED_SOURCE_HASHES.json').write_text(json.dumps(hashes,indent=2)+'\n') + raise SystemExit(0 if result.wasSuccessful() else 1) diff --git a/services/netplan-v4/runtime_preflight.py b/services/netplan-v4/runtime_preflight.py new file mode 100644 index 0000000..8991424 --- /dev/null +++ b/services/netplan-v4/runtime_preflight.py @@ -0,0 +1,46 @@ +"""Check packaged code readability and writable temporary test output as non-root.""" +import os +from pathlib import Path +import tempfile + +SOURCE_DIRS = ("netplan_v4", "tests", "integrations", "gui") +SOURCE_FILES = ("run_tests.py", "install_hooks.py", "runtime_preflight.py", "requirements.txt") + + +def verify_access(root, reports): + root, reports = Path(root).resolve(), Path(reports).resolve() + if reports == root or root in reports.parents: + raise ValueError("Test reports must be outside the packaged application tree") + if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK): + raise PermissionError(f"Application directory is not accessible: {root}") + paths = [root / name for name in SOURCE_FILES] + for name in SOURCE_DIRS: + directory = root / name + if not directory.is_dir(): + raise FileNotFoundError(f"Missing packaged directory: {directory}") + def on_error(error): + raise error + for parent, directories, files in os.walk(directory, onerror=on_error): + base = Path(parent) + if not os.access(base, os.R_OK | os.X_OK): + raise PermissionError(f"Packaged directory is not accessible: {base}") + if any((base / child).is_symlink() for child in directories + files): + raise ValueError(f"Unexpected symlink in packaged source: {base}") + paths.extend(base / name for name in files) + for path in paths: + with path.open("rb") as handle: + handle.read(1) + reports.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryFile(dir=reports) as probe: + probe.write(b"permission check") + probe.flush() + return len(paths) + + +if __name__ == "__main__": + if os.geteuid() == 0: + raise SystemExit("Runtime preflight must run as the unprivileged container user") + reports = Path(os.getenv("NETPLAN_V4_TEST_REPORT_DIR", "/tmp/test-results")) + count = verify_access(Path(__file__).resolve().parent, reports) + import numpy, scipy, fastapi, httpx + print(f"Non-root runtime check OK: uid={os.geteuid()}, readable_files={count}, reports={reports}") diff --git a/services/netplan-v4/tests/manager_protocol.php b/services/netplan-v4/tests/manager_protocol.php new file mode 100644 index 0000000..9a56099 --- /dev/null +++ b/services/netplan-v4/tests/manager_protocol.php @@ -0,0 +1,24 @@ +'2026-10-01T12:00:00Z','validUntil'=>'2026-10-01T12:05:00Z','gridTargetW'=>0.,'intent'=>'pvCharge']; +$plan=['schemaVersion'=>2,'planId'=>'test','configRevision'=>7,'status'=>'optimal','executable'=>true, + 'runMode'=>'shadow','liveEnabled'=>false,'generatedAt'=>$point['time'],'validFrom'=>$point['time'],'validUntil'=>'2026-10-01T12:10:00Z', + 'points'=>[$point,['time'=>'2026-10-01T12:05:00Z','validUntil'=>'2026-10-01T12:10:00Z','gridTargetW'=>9000.,'intent'=>'gridCharge']]]; +$now=strtotime('2026-10-01T12:03:00Z'); +check(NetzfahrplanV4::currentPoint($plan,$now,7)===null,'Shadow never live'); +check(NetzfahrplanV4::currentPoint($plan,$now,7,true)['gridTargetW']===0.,'Current interval, not nearest future point'); +check(NetzfahrplanV4::currentPoint($plan,$now,8,true)===null,'Revision mismatch'); +check(NetzfahrplanV4::currentPoint($plan,$now+3600,7,true)===null,'Expired plan'); +$bad=$plan;$bad['points'][1]['time']=$point['time'];check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Overlapping intervals'); +$bad=$plan;$bad['points'][0]['gridTargetW']=NAN;check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'NaN rejected'); +$bad=$plan;$bad['generatedAt']='2026-10-01T12:00:00';check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Timezone required'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,39000.,0.,15000.);check($r['batteryTargetW']===5332.,'Surplus charges');check($r['gridTargetW']===0.,'Peak cap never replaces zero target');check($r['expectedGridW']===0.,'Expected grid balanced'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,3000.,0.);check($r['batteryTargetW']===3000.&&$r['limited'],'Charging availability enforced'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],1000.,0.,39000.,0.);check($r['batteryTargetW']===0.,'No inadvertent grid charge from load forecast'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],7000.,0.,39000.,1000.,5000.);check($r['batteryTargetW']===-1000.&&$r['limited'],'Safety override still respects available discharge'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>-40000.,'intent'=>'pvCharge'],-10000.,0.,39000.,10000.);check($r['batteryTargetW']===0.,'No unintended discharge in PV charge intent');check($r['trackingErrorW']===30000.,'Intent saturation reports unmet target'); +echo "Netzfahrplan V4: {$count} assertions passed. No IPS or device calls.\n"; diff --git a/services/netplan-v4/tests/portal.test.mjs b/services/netplan-v4/tests/portal.test.mjs new file mode 100644 index 0000000..05bb0fa --- /dev/null +++ b/services/netplan-v4/tests/portal.test.mjs @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createPlannerV4Bridge} from '../integrations/netplan-v4-bridge.mjs'; +const PLANT='00000000-0000-4000-8000-000000000001',INSTALL='e3a08f9e-af12-4695-99bd-8b51c0520021'; +function fixture(options={}){ + const sent=[],auth=[],upstream=[]; + const bridge=createPlannerV4Bridge({ + configuredPrognosisPlant:(req,res,id,csrf)=>{auth.push({id,csrf});return options.denied?null:{plant:{installation_id:INSTALL},license:{quantities:{grid_schedule:options.unlicensed?0:1}},session:{}};}, + roleAllowed:()=>!options.viewer,bodyJson:async()=>({expectedRevision:0,changes:{family:'23'}}), + json:(res,status,payload)=>sent.push({status,payload}),deviceActivation:()=>options.deviceDenied?null:{plant_id:PLANT}, + checkDeviceRate:()=>!options.ratelimited,ownedLicenseState:()=>({quantities:{grid_schedule:1}}),serviceToken:'synthetic-test-only', + fetchImpl:async(url,args)=>{upstream.push({url:String(url),args});if(options.down)throw new Error('secret-host-detail');return {ok:!options.conflict,status:options.conflict?409:200,json:async()=>options.conflict?{detail:'internal-field'}:{liveEnabled:false,plan:{planId:'p1'}}};} + });return {bridge,sent,auth,upstream}; +} +const url=suffix=>new URL(`https://portal.test/api/plants/${PLANT}/prognosis/planner-v4${suffix}`); +test('read scoped by customer plant, upstream uses installation id',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},url('')),true);assert.equal(f.auth[0].csrf,false);assert.match(f.upstream[0].url,new RegExp(INSTALL));assert.equal(f.sent[0].status,200);}); +test('save requires existing CSRF checks',async()=>{const f=fixture();await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.auth[0].csrf,true);assert.equal(f.upstream[0].args.method,'PUT');}); +test('viewer cannot mutate',async()=>{const f=fixture({viewer:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,403);assert.equal(f.upstream.length,0);}); +test('unowned plant blocked',async()=>{const f=fixture({denied:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.upstream.length,0);}); +test('license required',async()=>{const f=fixture({unlicensed:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,403);}); +test('revision conflict preserved without internal error disclosure',async()=>{const f=fixture({conflict:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,409);assert.doesNotMatch(JSON.stringify(f.sent),/internal-field/);}); +test('service failure neither exposes details nor changes live plan',async()=>{const f=fixture({down:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,503);assert.doesNotMatch(JSON.stringify(f.sent),/secret-host-detail|synthetic-test-only/);}); +test('V1 live schedule is NOT intercepted',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/schedule`)),false);assert.equal(f.upstream.length,0);}); +test('device telemetry mapped only to operation ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/operation`));assert.match(f.upstream[0].url,/\/inputs\/operation$/);}); +test('unauthenticated device blocked',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.upstream.length,0);}); +test('device rate limit reused',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); +test('unexpected method rejected',async()=>{const f=fixture();await f.bridge({method:'DELETE'},{},url(''));assert.equal(f.sent[0].status,405);assert.equal(f.upstream.length,0);}); + +test('measurement batches use authenticated dedicated data ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.match(f.upstream[0].url,/\/planner\/measurements$/);assert.equal(f.upstream[0].args.method,'POST');}); +test('unauthenticated measurement upload cannot reach storage',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.upstream.length,0);}); +test('device cannot change dataset mapping through public proxy',async()=>{const f=fixture();assert.equal(await f.bridge({method:'PUT'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/datasets/physical-v1`)),false);assert.equal(f.upstream.length,0);}); +test('measurement upload keeps device rate protection',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); diff --git a/services/netplan-v4/tests/test_application_deployment.py b/services/netplan-v4/tests/test_application_deployment.py new file mode 100644 index 0000000..44b2d8e --- /dev/null +++ b/services/netplan-v4/tests/test_application_deployment.py @@ -0,0 +1,39 @@ +import importlib.util +from pathlib import Path +import sqlite3,tempfile,unittest +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('app_deploy',ROOT/'commissioning/deploy_application.py');d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) +class ApplicationDeploymentTest(unittest.TestCase): + def test_backup_consistent_and_original_not_changed(self): + with tempfile.TemporaryDirectory() as tmp: + src=Path(tmp)/'source.sqlite';out=Path(tmp)/'backup.sqlite' + with sqlite3.connect(src) as c:c.execute('CREATE TABLE sample(value)');c.execute('INSERT INTO sample VALUES(3)') + before=src.read_bytes();d.backup_database(src,out) + with sqlite3.connect(out) as c:self.assertEqual(c.execute('SELECT value FROM sample').fetchone()[0],3) + self.assertEqual(before,src.read_bytes()) + def test_backup_cannot_invent_missing_database(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError):d.backup_database(Path(tmp)/'missing.sqlite',Path(tmp)/'new.sqlite') + def test_atomic_refuses_symlink(self): + with tempfile.TemporaryDirectory() as tmp: + a=Path(tmp)/'a';b=Path(tmp)/'b';a.write_text('original');b.symlink_to(a) + with self.assertRaises(ValueError):d.atomic(b,b'changed') + self.assertEqual(a.read_text(),'original') + def test_only_two_reviewed_portal_files(self): + self.assertEqual(set(d.PORTAL_FILES.values()),{'netplan-v4-bridge.mjs','public/netplan-v4.js'}) + def test_build_tests_backup_before_runtime_replacement(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertLess(s.index("'target_python_and_portal_tests_passed'"),s.index('v4_replaced=True')) + self.assertLess(s.index("'consistent_database_backup'"),s.index('v4_replaced=True')) + def test_does_not_change_optimizer_or_device_permissions(self): + self.assertNotIn('/settings',d.BOOTSTRAP) + self.assertNotIn('trial/arm',d.BOOTSTRAP) + self.assertNotIn('IPS_',d.BOOTSTRAP) + self.assertIn("state['liveEnabled'] is False",d.BOOTSTRAP) + def test_image_contains_application_installer_imported_by_tests(self): + self.assertIn('COPY commissioning/deploy_application.py ./commissioning/deploy_application.py',(ROOT/'Dockerfile').read_text()) + def test_file_bind_mount_recreated_not_only_restarted(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertIn("'--force-recreate','--wait','license-portal'",s) + self.assertIn('additive_data_retained',s) +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_battery_recovery.py b/services/netplan-v4/tests/test_battery_recovery.py new file mode 100644 index 0000000..e82974c --- /dev/null +++ b/services/netplan-v4/tests/test_battery_recovery.py @@ -0,0 +1,43 @@ +from dataclasses import replace +import unittest +from netplan_v4.optimizer import optimize +from netplan_v4.domain import Limits +from test_v4 import AT,seq,batt + +class BatteryRecoveryTest(unittest.TestCase): + def plan(self,b,steps=None): + steps=steps or seq([0.]*3+[1000.]*3,pv=[1000.]*3+[0.]*3,sell=[0.]*6) + return optimize(steps,[b],at=AT,observed_peaks={'2026-10':1.},peak_prices={'2026-10':0.}) + def test_reserve_recovery_preserves_real_initial_energy(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + energy=.05 + for x in p['points']: + target=x['batteryTargetW'];energy+=target/1000/12*(.9**.5 if target>=0 else 1/(.9**.5)) + self.assertAlmostEqual(100*energy,x['socEndPercent']['b'],places=5) + self.assertGreaterEqual(energy,.05-1e-6) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],15.-1e-5) + def test_below_hardware_min_not_fabricated(self): + b=batt(soc_percent=1.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True) + self.assertFalse(self.plan(b)['executable']) + def test_disabled_can_rearm_after_charging(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + self.assertAlmostEqual(min(0.,p['points'][0]['batteryTargetW']),0.) + self.assertTrue(any(x['batteryTargetW']<-1. for x in p['points'][3:])) + first=next(i for i,x in enumerate(p['points']) if x['batteryTargetW']<-1.) + self.assertGreaterEqual(p['points'][first-1]['socEndPercent']['b'],17.-1e-5) + def test_cannot_discharge_below_rearm_while_blocked(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False) + p=self.plan(b,seq([1000.]*6));self.assertTrue(p['executable']) + self.assertTrue(all(x['batteryTargetW']>=-1e-5 for x in p['points'])) + def test_unavailable_power_does_not_get_invented(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,max_charge_w=0.,max_discharge_w=0.) + p=self.plan(b);self.assertFalse(p['executable']);self.assertEqual(p['status'],'no_feasible_plan') + def test_invalid_hysteresis_range_rejected(self): + b=batt(rearm_soc_percent=95.) + self.assertFalse(self.plan(b)['executable']) + def test_no_recovery_permission_keeps_strict_behavior(self): + self.assertFalse(self.plan(batt(soc_percent=5.))['executable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_container_access.py b/services/netplan-v4/tests/test_container_access.py new file mode 100644 index 0000000..8a24ebc --- /dev/null +++ b/services/netplan-v4/tests/test_container_access.py @@ -0,0 +1,60 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import unittest +from runtime_preflight import SOURCE_DIRS, SOURCE_FILES, verify_access + + +class ContainerAccessTest(unittest.TestCase): + def fixture(self, tmp): + root = Path(tmp) / "app" + root.mkdir() + for name in SOURCE_DIRS: + (root / name).mkdir() + (root / name / "sample.txt").write_text("readable") + for name in SOURCE_FILES: + (root / name).write_text("readable") + return root, Path(tmp) / "reports" + + def test_code_readable_and_reports_outside_readonly_tree(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + self.assertEqual(verify_access(root, reports), 8) + self.assertEqual(list(reports.iterdir()), []) + + def test_missing_runner_is_reported(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + (root / "run_tests.py").unlink() + with self.assertRaises(FileNotFoundError): + verify_access(root, reports) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_unreadable_source_fails(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "netplan_v4" / "sample.txt" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o600) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_untraversable_directory_is_not_silently_skipped(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "tests" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o700) + + def test_reports_cannot_target_application_tree(self): + with TemporaryDirectory() as tmp: + root, _ = self.fixture(tmp) + with self.assertRaises(ValueError): + verify_access(root, root / "test-results") diff --git a/services/netplan-v4/tests/test_controlled_trial.py b/services/netplan-v4/tests/test_controlled_trial.py new file mode 100644 index 0000000..ddb1f2a --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial.py @@ -0,0 +1,120 @@ +import copy +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app +from netplan_v4 import controlled_trial as t + +AT=datetime(2026,10,2,12,tzinfo=timezone.utc) +PLANT='00000000-0000-4000-8000-000000000001' +SESSION='00000000-0000-4000-8000-000000000002' +PLAN='00000000-0000-4000-8000-000000000003' +TOKEN='synthetic-only-control-test-token' + +def fixture(): + context={'batteries':{'b':{'capacityKwh':10.}},'limits':{'importW':10000.,'exportW':None,'managerMonthLimitsW':{}}} + plan={'planId':PLAN,'installationId':PLANT,'sourceFamily':'3','configRevision':1, + 'runMode':'shadow','liveEnabled':False,'executable':True,'validUntil':(AT+timedelta(hours=1)).isoformat(), + 'controlContext':context,'inputQuality':{'loadBasis':'base_load','accountingEvidenceId':'synthetic-meter-boundary-v1'}, + 'peakCostIsEstimate':True} + view={'installationId':PLANT,'liveEnabled':False,'fresh':True,'plan':plan,'settings':{'family':'3','revision':1}} + request={'sessionId':SESSION,'expectedPlanId':PLAN,'expectedRevision':1,'durationSeconds':300, + 'maxChargeW':5000.,'maxDischargeW':5000.,'assetId':'b','managerId':11111,'batteryInstanceId':22222, + 'acceptEstimatedPeak':True,'actuatorWatchdogEvidenceId':'synthetic-hardware-watchdog', + 'confirmation':'ARM_BOUNDED_CONTROL_TRIAL'} + return view,request + +class ControlledTrialTest(unittest.TestCase): + def setUp(self): + self.s=PlannerStore(':memory:');self.v,self.r=fixture() + def tearDown(self): self.s.close() + def arm(self,**kw): + return t.arm(self.s,PLANT,kw.get('request',self.r),kw.get('view',self.v),kw.get('now',AT),kw.get('allowed',{PLANT})) + def test_default_has_no_authority(self): + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,set())) + with self.assertRaises(ValueError):self.arm(allowed=set()) + def test_shadow_plan_never_becomes_live(self): + before=copy.deepcopy(self.v);self.arm() + a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(self.v,before);self.assertEqual(a['kind'],'controlled_trial_authority') + self.assertTrue(a['sourcePlanRemainsShadow']);self.assertFalse(self.v['plan']['liveEnabled']) + def test_short_authority_lease(self): + self.arm();a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(t.timestamp(a['validUntil'])-AT,timedelta(seconds=90)) + def test_session_expiry(self): + self.arm();self.assertIsNone(t.authority(self.s,PLANT,self.v,AT+timedelta(seconds=300),{PLANT})) + def test_revoke(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT);self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_expired_or_revoked_session_not_resurrected(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT) + with self.assertRaises(ValueError):self.arm() + def test_active_session_cannot_be_extended(self): + self.arm() + with self.assertRaises(ValueError):self.arm(now=AT+timedelta(seconds=30)) + def test_other_plant(self): + self.arm();self.assertIsNone(t.authority(self.s,SESSION,self.v,AT,{SESSION})) + def test_revocation_mismatched_id_leaves_real_session(self): + self.arm();t.revoke(self.s,PLANT,PLAN,AT);self.assertIsNotNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_stale_source_plan_cannot_arm(self): + self.v['fresh']=False + with self.assertRaises(ValueError):self.arm() + def test_stale_source_removes_authority(self): + self.arm();self.v['fresh']=False;self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_revision_bool_not_an_integer(self): + self.r['expectedRevision']=True + with self.assertRaises(ValueError):self.arm() + def test_wrong_revision(self): + self.r['expectedRevision']=2 + with self.assertRaises(ValueError):self.arm() + def test_revision_change_invalidates_authority(self): + self.arm();self.v['settings']['revision']=2;self.v['plan']['configRevision']=2 + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_policy_change_invalidates_authority(self): + self.arm();self.v['plan']['controlContext']['limits']['importW']=12000. + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_newer_plan_same_policy_supported(self): + self.arm();self.v['plan']['planId']=SESSION + self.assertEqual(t.authority(self.s,PLANT,self.v,AT,{PLANT})['sourceShadowPlanId'],SESSION) + def test_auto_family_not_enabled_for_first_trial(self): + self.v['settings']['family']='auto' + with self.assertRaises(ValueError):self.arm() + def test_aggregate_house_refused(self): + self.v['plan']['inputQuality']['loadBasis']='house_total' + with self.assertRaises(ValueError):self.arm() + def test_no_accounting_evidence_no_trial(self): + del self.v['plan']['inputQuality']['accountingEvidenceId'] + with self.assertRaises(ValueError):self.arm() + def test_no_watchdog_evidence_no_trial(self): + self.r['actuatorWatchdogEvidenceId']='' + with self.assertRaises(ValueError):self.arm() + def test_no_implicit_estimate_acceptance(self): + self.r['acceptEstimatedPeak']=False + with self.assertRaises(ValueError):self.arm() + def test_power_cap_and_nonfinite(self): + for v in (5001, float('nan'),float('inf'),True,0,'5000'): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'maxChargeW':v}) + def test_duration_bound(self): + for v in (0,1801,True,30.5): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'durationSeconds':v}) + def test_cannot_replace_asset(self): + self.r['assetId']='other' + with self.assertRaises(ValueError):self.arm() + def test_two_assets_not_supported(self): + self.v['plan']['controlContext']['batteries']['other']={} + with self.assertRaises(ValueError):self.arm() + def test_operator_endpoint_requires_auth(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False) + with TestClient(app) as c: + self.assertEqual(c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',json=self.r).status_code,401) + r=c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',headers={'X-Enelix-Service-Token':TOKEN},json=self.r) + self.assertEqual(r.status_code,409);self.assertIn('allowlist',r.text) + self.assertIsNone(c.get(f'/internal/v2/prognosis/{PLANT}/planner',headers={'X-Enelix-Service-Token':TOKEN}).json()['controlledTrial']) + def test_trial_allowlist_cannot_expand_installation_access(self): + with tempfile.TemporaryDirectory() as td,self.assertRaises(ValueError): + create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[SESSION]) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_controlled_trial_pipeline.py b/services/netplan-v4/tests/test_controlled_trial_pipeline.py new file mode 100644 index 0000000..894242e --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial_pipeline.py @@ -0,0 +1,44 @@ +import copy +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch +from fastapi.testclient import TestClient +from netplan_v4.service import create_app, ingest, run_once +from test_v4 import inputs, AT +from test_controlled_trial import fixture, PLANT, SESSION, TOKEN + +class ControlledTrialPipelineTest(unittest.TestCase): + def test_real_planner_to_internal_trial_api_and_revoke(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'trial.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[PLANT]) + store=app.state.store_factory() + op,fc,tar=inputs() + for f in fc['families'].values():f['accountingEvidenceId']='synthetic-accounting-review' + for k,v in [('operation',op),('forecast',fc),('tariffs',tar)]:ingest(store,PLANT,k,v,AT) + plan=run_once(store,AT);self.assertEqual(plan['status'],'optimal');store.close() + headers={'X-Enelix-Service-Token':TOKEN};prefix=f'/internal/v2/prognosis/{PLANT}/planner' + with patch('netplan_v4.service.datetime',wraps=datetime) as clock,TestClient(app) as c: + clock.now.return_value=AT + view=c.get(prefix,headers=headers).json() + self.assertIsNone(view['controlledTrial']);self.assertTrue(view['fresh']) + _,request=fixture();request.update(expectedPlanId=plan['planId'],expectedRevision=0,acceptEstimatedPeak=False) + grant=c.post(prefix+'/trial/arm',headers=headers,json=request) + self.assertEqual(grant.status_code,200,grant.text) + view=c.get(prefix,headers=headers).json() + self.assertEqual(view['controlledTrial']['sourceShadowPlanId'],plan['planId']) + self.assertFalse(view['liveEnabled']);self.assertFalse(view['plan']['liveEnabled']) + self.assertEqual(view['plan']['runMode'],'shadow') + self.assertEqual(c.post(prefix+'/ack',headers=headers,json={'planId':plan['planId'],'revision':0,'status':'applied'}).status_code,400) + self.assertEqual(c.post(prefix+'/trial/revoke',headers=headers,json={'sessionId':SESSION}).status_code,200) + self.assertIsNone(c.get(prefix,headers=headers).json()['controlledTrial']) + self.assertEqual(c.post(prefix+'/trial/arm',headers=headers,json=request).status_code,409) + def test_accounting_marker_cannot_label_aggregate_load_verified(self): + from netplan_v4.store import PlannerStore + s=PlannerStore(':memory:');op,fc,tar=inputs() + f=fc['families']['3'];f['loadBasis']='house_total';f['accountingEvidenceId']='synthetic-incorrect-label' + with self.assertRaises(ValueError):ingest(s,PLANT,'forecast',fc,AT) + s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_corrected_forecast_commissioning.py b/services/netplan-v4/tests/test_corrected_forecast_commissioning.py new file mode 100644 index 0000000..dda7d69 --- /dev/null +++ b/services/netplan-v4/tests/test_corrected_forecast_commissioning.py @@ -0,0 +1,116 @@ +import importlib.util +import json +import sqlite3 +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "activate_corrected_forecast", ROOT / "commissioning/activate_corrected_forecast.py" +) +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + +from netplan_v4 import measurement_pipeline +from netplan_v4.store import PlannerStore + +PLANT = module.PLANT +DATASET = module.DATASET +NOW = 1791099600 + + +class CorrectedForecastCommissioningTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db = Path(self.tmp.name) / "planner.sqlite" + self.store = PlannerStore(str(self.db)) + config = { + "datasetId": DATASET, + "mappingSha256": "a" * 64, + "inventorySha256": "b" * 64, + "formula": "physical_sum_v1", + "solarReference": None, + "minimumCoverage": 0.95, + "maximumGapSeconds": 10, + "minimumTrainingHours": 24, + "historyDays": 14, + "sources": [ + {"key": "grid", "variableId": 1, "role": "grid", "factorToW": 1, "maxAgeSeconds": 60}, + {"key": "pv", "variableId": 2, "role": "pv", "factorToW": 1, "maxAgeSeconds": 60}, + {"key": "battery", "variableId": 3, "role": "physical_storage", "factorToW": 1, "maxAgeSeconds": 60}, + ], + } + measurement_pipeline.register_dataset(self.store.con, PLANT, config, NOW - 100) + model = { + "version": 1, + "datasetId": DATASET, + "methodVersion": "profile_v1", + "trainedAt": NOW - 60, + "trainedThrough": NOW - 300, + "profiles": {}, + "validation": {"status": "bootstrap_insufficient_holdout"}, + } + model_id = "c" * 64 + with self.store.con: + self.store.con.execute( + "INSERT INTO planner_load_models VALUES(?,?,?,?,?,?)", + (PLANT, DATASET, model_id, NOW - 60, NOW - 300, json.dumps(model)), + ) + self.store.con.execute( + "INSERT INTO planner_model_current VALUES(?,?,?)", (PLANT, DATASET, model_id) + ) + self.store.con.execute( + "INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?)", + ( + PLANT, + DATASET, + 1, + "model_ready", + json.dumps( + { + "modelId": model_id, + "usableEquivalentHours": 24.1, + "requiredEquivalentHours": 24, + } + ), + ), + ) + self.store.con.execute( + "INSERT INTO planner_observations VALUES(?,?,?,?,?,?)", + (PLANT, DATASET, NOW - 30, NOW - 20, "d" * 64, "{}"), + ) + self.store.close() + + def tearDown(self): + self.tmp.cleanup() + + def test_ready_state_can_activate_only_corrected_shadow_settings(self): + state = module.read_state(self.db, PLANT, DATASET, NOW) + module.require_ready(state, NOW) + result = module.activate( + self.db, PLANT, DATASET, datetime.fromtimestamp(NOW, timezone.utc) + ) + self.assertTrue(result["changed"]) + store = PlannerStore(str(self.db)) + settings = store.settings(PLANT) + self.assertEqual(settings["forecastSource"], "corrected_profile") + self.assertEqual(settings["measurementDataset"], DATASET) + self.assertEqual(settings["runMode"], "shadow") + store.close() + + def test_stale_observations_refused(self): + state = module.read_state(self.db, PLANT, DATASET, NOW + 700) + with self.assertRaises(ValueError): + module.require_ready(state, NOW + 700) + + def test_backup_is_consistent(self): + backup = Path(self.tmp.name) / "backup.sqlite" + module.backup_database(self.db, backup) + with sqlite3.connect(backup) as con: + self.assertEqual(con.execute("PRAGMA integrity_check").fetchone()[0], "ok") + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/tests/test_delivery.py b/services/netplan-v4/tests/test_delivery.py new file mode 100644 index 0000000..3cbe890 --- /dev/null +++ b/services/netplan-v4/tests/test_delivery.py @@ -0,0 +1,82 @@ +import importlib.util +from pathlib import Path +from tempfile import TemporaryDirectory +from datetime import datetime,timedelta,timezone +import unittest +ROOT=Path(__file__).resolve().parents[1] +def module(name,path): + spec=importlib.util.spec_from_file_location(name,path);value=importlib.util.module_from_spec(spec);spec.loader.exec_module(value);return value +publisher=module('publisher',ROOT/'integrations/netplan_v4_publisher.py') +hooks=module('hooks',ROOT/'install_hooks.py') +AT=datetime(2026,10,1,12,0,tzinfo=timezone.utc) + +class PublisherTest(unittest.TestCase): + def test_tariff_null_preserved_and_mode_explicit(self): + cfg={'tarif_bezug_modus':'statisch','tarif_bezug':'custom','tarif_bezug_fest':0.,'tarif_einspeisung_modus':'dynamisch','tarif_einspeisung':'dynamic','tarif_peak_fest':0.} + p=publisher.tariff_payload(cfg,AT);self.assertEqual(p['import']['staticChfKwh'],0.);self.assertEqual(p['peakChfKwMonth'],0.) + del cfg['tarif_bezug_modus'] + with self.assertRaises(ValueError):publisher.tariff_payload(cfg,AT) + def test_native_ckw_unit_zero_and_delivery_range(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'CHF_kWh','value':0.}]}] + p=publisher.ckw_payload('CKW Dynamisch Home',rows,AT-timedelta(hours=1),AT) + self.assertEqual(p['periods'][0]['value'],0.);self.assertEqual(p['periods'][0]['unit'],'CHF/kWh');self.assertEqual(p['periods'][0]['sourceKind'],'published_interval') + del rows[0]['end_timestamp'] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + def test_unknown_unit_and_future_publication_rejected(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'unknown','value':100.}]}] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + rows[0]['integrated'][0]['unit']='CHF_kWh' + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,AT+timedelta(minutes=1),AT) + def test_forecast_not_relabelled_as_base_load(self): + p=publisher.forecast_payload([(23,{AT:5000.},{AT:2000.})],AT) + self.assertEqual(p['families']['23']['loadBasis'],'house_total');self.assertIsNone(p['families']['23']['trainedUntil']) + +class InstallerTest(unittest.TestCase): + def fixtures(self,root): + files={'prognosis-manager-enelix2/api/main.py':'def f():\n _portal_store_configuration(anlagen_id, configuration)\n', + 'prognosis-manager-enelix2/forecast_engine/main.py':'def f():\n for a in []:\n for key, default in []:\n d[key] = float(d.get(key) or default)\n for c in []:\n p_3 = v3.predict(data_obj, p_1, p_2)\n', + 'prognosis-manager-enelix2/tariff_importer/main.py':'def f():\n for a in []:\n for b in []:\n for c in []:\n if True:\n return candidate_points\n', + 'license/server.mjs':'const server = createServer(async (req, res) => {\n if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {status:"ok"});\n});', + 'license/public/app.js':'// Parallel GUI work. Must remain byte-identical.'} + for name,data in files.items():p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(data) + return files + def test_install_idempotent_and_verified_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);originals=self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root) + self.assertFalse(hooks.plan(root));self.assertEqual((root/'license/public/app.js').read_text(),originals['license/public/app.js']) + hooks.rollback(receipt) + for name,data in originals.items():self.assertEqual((root/name).read_text(),data) + self.assertFalse((root/'license/public/netplan-v4.html').exists()) + def test_concurrent_changes_never_overwritten(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);entries=hooks.plan(root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Concurrent') + with self.assertRaises(ValueError):hooks.apply(entries,root) + self.assertTrue(p.read_text().endswith('// Concurrent')) + def test_later_changes_block_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Later') + with self.assertRaises(ValueError):hooks.rollback(receipt) + self.assertTrue(p.read_text().endswith('// Later')) + def test_changed_or_ambiguous_anchor_fails_closed(self): + with self.assertRaises(ValueError):hooks.once('x x','x','y') + with self.assertRaises(ValueError):hooks.patch_api('def f():pass') + +class HorizonValidationTest(unittest.TestCase): + def test_synthetic_48h_energy_soc_limits(self): + import math + from netplan_v4.domain import Battery,Step,Price,Limits + from netplan_v4.optimizer import optimize + steps=[] + for i in range(576): + solar=max(0.,math.sin(math.pi*((i%288)/12-6)/12))*10000. + steps.append(Step(AT+timedelta(minutes=5*i),3000.,solar,Price(.15 if i%288<72 else .35),Price(.1))) + b=Battery('synthetic',40.,20.,10.,90.,10000.,8000.,AT,grid_charging=True) + p=optimize(steps,[b],at=AT,limits=Limits(import_w=12000.,export_w=10000.),observed_peaks={'2026-10':8.},peak_prices={'2026-10':5.}) + self.assertTrue(p['executable']);self.assertEqual(len(p['points']),576) + for s,x in zip(steps,p['points']): + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=3) + self.assertLessEqual(x['gridTargetW'],12000.001);self.assertGreaterEqual(x['gridTargetW'],-10000.001) + self.assertTrue(9.999<=x['socEndPercent']['synthetic']<=90.001) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['synthetic'],19.999) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_economic_replay.py b/services/netplan-v4/tests/test_economic_replay.py new file mode 100644 index 0000000..3b5dc43 --- /dev/null +++ b/services/netplan-v4/tests/test_economic_replay.py @@ -0,0 +1,144 @@ +import copy +import json +import tempfile +import unittest +from dataclasses import asdict +from datetime import datetime,timedelta,timezone +from pathlib import Path +from netplan_v4 import economic_replay as e, measurement_pipeline as m +from netplan_v4.domain import Battery,Limits,Step,Price +from netplan_v4.store import PlannerStore +from netplan_v4.selection import choose_family +from test_measurement_pipeline import config,record +from test_v4 import AID + +AT=datetime(2026,10,2,22,tzinfo=timezone.utc) +END=AT+timedelta(days=1) +TS=int(AT.timestamp()) + + +def fixture(): + battery=Battery('b',20.,50.,10.,90.,5000.,5000.,AT,grid_charging=True) + steps=[Step(AT+timedelta(minutes=5*i),1000.,0.,Price(.1 if i<144 else .4),Price(.03)) for i in range(288)] + data={'steps':steps,'batteries':[battery],'limits':Limits(import_w=10000.),'observed_peaks':{'2026-10':5.},'peak_prices':{'2026-10':5.},'quarter_history':{},'at':AT} + def optimizer(**kw): + points=[] + family=kw['family'] + for i,s in enumerate(kw['steps']): + target=1000. if family in ('3','13') else (1600. if i<144 else 500.) + points.append({'time':s.start.isoformat(),'validUntil':s.end.isoformat(),'gridTargetW':target, + 'importPriceChfKwh':s.import_price.chf_kwh,'exportPriceChfKwh':s.export_price.chf_kwh}) + return {'executable':True,'points':points} + def assemble(*args):return copy.deepcopy(data),END,{} + return data,assemble,optimizer + + +class EconomicReplayTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'));self.c=config() + m.register_dataset(self.s.con,AID,self.c,TS) + self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':self.c['datasetId'],'autoMinimumDays':1},0,AT) + self.data,self.assemble,self.optimizer=fixture() + def tearDown(self):self.s.close();self.tmp.cleanup() + def capture(self):e.capture(self.s,AID,AT,self.assemble,self.optimizer) + def cohort(self):return json.loads(self.s.con.execute('SELECT value FROM planner_economic_cohorts').fetchone()[0]) + def populate(self): + now=int(END.timestamp())+180 + rows=[record(t,self.c) for t in range(TS-60,now+1,30)] + for i in range(0,len(rows),120):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows[i:i+120]},now) + def outcomes(self,value=1000.):return {TS+300*i:{'loadW':value,'coverage':1.,'profileUsable':True} for i in range(288)} + def test_snapshot_frozen_once(self): + self.capture();first=self.cohort();self.capture();self.assertEqual(first,self.cohort()) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_no_retroactive_midday_snapshot(self): + e.capture(self.s,AID,AT+timedelta(hours=12),self.assemble,self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_silent_legacy_replay(self): + self.s.save_settings(AID,{'forecastSource':'legacy'},1,AT);self.capture() + self.assertEqual(e.scores(self.s,AID,END),[]) + def test_all_families_same_initial_energy(self): + self.capture();c=self.cohort();values=[e.simulate(c,f,self.outcomes()) for f in c['plans']] + self.assertEqual(len({v['initialEnergyKwh'] for v in values}),1) + for v in values:self.assertGreaterEqual(v['finalEnergyKwh'],2.-1e-6);self.assertLessEqual(v['finalEnergyKwh'],18.00001) + def test_profitable_time_shift_has_lower_normalized_cost(self): + self.capture();c=self.cohort();a=e.simulate(c,'3',self.outcomes());b=e.simulate(c,'23',self.outcomes()) + self.assertLess(b['costChf'],a['costChf']);self.assertTrue(b['terminalNormalized']) + def test_end_energy_not_free_savings(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=0.;p['importPriceChfKwh']=.4 + d=e.simulate(c,'3',self.outcomes());self.assertGreater(d['terminalAdjustmentChf'],0) + def test_missing_actual_not_zero(self): + self.capture();out=self.outcomes();out.pop(TS+300) + with self.assertRaises(ValueError):e.simulate(self.cohort(),'3',out) + def test_export_and_import_limits_not_ignored(self): + self.capture();c=self.cohort();c['context']['limits']['import_w']=500. + result=e.simulate(c,'3',self.outcomes(20000.));self.assertGreater(result['constraintBreaches'],0) + def test_peak_increment_not_total_billed_twice(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=6000. + result=e.simulate(c,'3',self.outcomes(6000.));self.assertAlmostEqual(result['peakCostChf'],5.) + def test_no_actual_results_before_day_end(self): + self.capture();self.populate();e.advance(self.s,AID,AT+timedelta(hours=12)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + def test_real_pipeline_from_frozen_plans_to_actuals_to_selection(self): + self.capture();self.populate();now=END+timedelta(minutes=4) + e.advance(self.s,AID,now);scores=e.scores(self.s,AID,now) + self.assertEqual(len(scores),3);self.assertEqual(scores[0].days,1) + chosen=choose_family('auto','3',scores,minimum_days=1,margin_chf=0,now=now) + self.assertEqual(chosen['family'],'23');self.assertEqual(chosen['mode'],'economic_replay') + self.assertTrue(e.status(self.s,AID)['connected']);self.assertFalse(e.status(self.s,AID)['isBillingEvidence']) + def test_evaluation_idempotent(self): + self.capture();self.populate();now=END+timedelta(minutes=4);e.advance(self.s,AID,now);e.advance(self.s,AID,now) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],1) + def test_missing_outcomes_preserve_cohort(self): + self.capture();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_changed_tariffs_between_families_reject_cohort(self): + def changed(**kwargs): + v=self.optimizer(**kwargs) + if kwargs['family']=='23':v['points'][0]['importPriceChfKwh']+=.5 + return v + e.capture(self.s,AID,AT,self.assemble,changed) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_grid_charging_unsupported_is_explicit(self): + self.data['batteries'][0]=__import__('dataclasses').replace(self.data['batteries'][0],grid_charging=False) + e.capture(self.s,AID,AT,lambda *args:(self.data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_short_price_horizon_no_fake_extension(self): + data=copy.deepcopy(self.data);data['steps']=data['steps'][:12] + e.capture(self.s,AID,AT,lambda *args:(data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_other_plant_no_results(self): + self.capture();self.populate();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(e.scores(self.s,'30509683-4569-49e4-848f-4905e4cc813a',END+timedelta(minutes=4)),[]) + def test_residual_projection_does_not_change_load_profile(self): + rows=[m.project(record(t,self.c),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1000.) + self.assertEqual(m.reconstruct(rows,self.c)[0]['loadW'],6000.) + def test_monthly_peak_not_charged_again_every_comparison_day(self): + self.capture();c=self.cohort() + for family in c['plans']: + for point in c['plans'][family]['points']:point['gridTargetW']=6000. + result={f:e.simulate(c,f,self.outcomes(6000.)) for f in c['plans']} + self.assertEqual(result['3']['peakCostChf'],5.) + policy=self.s.con.execute('SELECT policy_id FROM planner_economic_cohorts').fetchone()[0] + with self.s.con: + for n in (0,1): + start=AT+timedelta(days=n);end=END+timedelta(days=n);day=start.astimezone(__import__('zoneinfo').ZoneInfo('Europe/Zurich')).date().isoformat() + if n: + self.s.con.execute('INSERT INTO planner_economic_cohorts VALUES(?,?,?,?,?,?)',(AID,day,int(start.timestamp()),int(end.timestamp()),policy,json.dumps(c))) + v={'results':result,'start':start.isoformat(),'end':end.isoformat()} + self.s.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(AID,day,int(end.timestamp())+120,policy,json.dumps(v))) + scores=e.scores(self.s,AID,END+timedelta(days=1,minutes=4)) + value=next(x.cost_chf for x in scores if x.family=='3') + expected=2*(result['3']['energyCostChf']+result['3']['throughputCostChf']+result['3']['terminalAdjustmentChf'])+5. + self.assertAlmostEqual(value,expected) + + def test_external_sdl_not_assumed_zero(self): + rows=[m.project(record(t,self.c,sdl=250.),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1250.) + for r in rows:r['raw']['sdl']['valid']=False + self.assertFalse(e.actuals(rows,self.c)[0]['profileUsable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_acceptance.py b/services/netplan-v4/tests/test_forecast_acceptance.py new file mode 100644 index 0000000..eb89b2e --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_acceptance.py @@ -0,0 +1,80 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from forecast_acceptance import source_paths, verify_forecast_bundle + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('forecast_release_test', ROOT / 'release_preflight.py') +preflight = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preflight) + + +class ForecastAcceptanceTest(unittest.TestCase): + def fixture(self, root): + src, bundle = root / 'source', root / 'bundle' + for path in (src, bundle): + path.mkdir() + for name, value in {'main.py': 'x=1\n', 'telemetry_quality.py': 'x=2\n', 'requirements.txt': 'pandas\n'}.items(): + (path / name).write_text(value) + manifest = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in source_paths(src)} + (bundle / 'SOURCE_MANIFEST.json').write_text(json.dumps(manifest)) + return src, bundle + + def test_exact_source_is_verified(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_modified_development_source_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (src / 'main.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_extra_staged_python_file_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'surprise.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_symlink_is_rejected(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'main.py').unlink() + (bundle / 'main.py').symlink_to(src / 'main.py') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_models_secrets_and_databases_not_staged(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + for name in ('.env', 'model.pkl', 'users.db'): + (src / name).write_text('not a source file') + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_forecast_checks_have_no_network_or_live_mounts(self): + args = next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='forecast_candidate_python311') + self.assertEqual(args[args.index('--network')+1], 'none') + self.assertEqual(args[args.index('--user')+1], '1000:1000') + self.assertIn('--read-only', args) + for forbidden in ('--volume', '-v', '--env-file', '--privileged', 'up', 'restart'): + self.assertNotIn(forbidden, args) + + def test_runner_does_not_import_application_main(self): + source = (ROOT / 'acceptance/check_forecast.py').read_text() + tree = ast.parse(source) + imports = {a.name for node in ast.walk(tree) if isinstance(node, ast.Import) for a in node.names} + self.assertNotIn('main', imports) + self.assertNotIn('run_forecast(', source) + self.assertNotIn('get_configs(', source) + + +if __name__ == '__main__': + unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_quality.py b/services/netplan-v4/tests/test_forecast_quality.py new file mode 100644 index 0000000..0a81f63 --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_quality.py @@ -0,0 +1,31 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from netplan_v4.forecast_quality import assess_family +from netplan_v4.service import ingest,run_once +from netplan_v4.store import PlannerStore +from test_v4 import inputs,AT,AID + +class ForecastQualityTest(unittest.TestCase): + def test_zero_load_not_free_energy_assumption(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_confirmed_pv_only_plant_is_supported(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)],'zeroLoadConfirmed':True} + self.assertTrue(assess_family(f)['valid']) + def test_boolean_and_nan_are_not_forecast_powers(self): + for value in (float('nan'),True,-1.,None): + f={'points':[{'loadW':value,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_selected_invalid_family_not_silently_replaced(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));op,fc,tar=inputs() + for p in fc['families']['3']['points']:p['loadW']=0. + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(s,AID,k,v,AT) + plan=run_once(s,AT) + self.assertEqual(plan['status'],'awaiting_inputs');self.assertIn('all-zero',plan['reason']);self.assertIsNone(s.current(AID));s.close() + def test_nonzero_family_remains_usable(self): + f={'points':[{'loadW':50.,'pvW':0.} for _ in range(3)]} + self.assertTrue(assess_family(f)['valid']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_history_timing.py b/services/netplan-v4/tests/test_history_timing.py new file mode 100644 index 0000000..b05b046 --- /dev/null +++ b/services/netplan-v4/tests/test_history_timing.py @@ -0,0 +1,204 @@ +"""Synthetic publication gaps plus full source-reference/application integration.""" +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.history_timing import validate_policy +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once +from test_measurement_pipeline import config, record, NOW, AID +from test_v4 import inputs, TOKEN + + +def timed_config(**kw): + return config(historyTimingPolicy={'version':1,'method':'equal_endpoint_v1', + 'sources':{'pv':{'maxSpanSeconds':130,'evidenceId':'synthetic-publication-policy'}}}, **kw) + + +def projected(span=360, period=120, c=None): + c = c or timed_config() + result=[] + for delta in range(0,span+1,30): + r=record(NOW+delta,c) + r['raw']['pv']['sourceUpdatedAt']=NOW+(delta//period)*period + result.append(m.project(r,c,AID,NOW+span)) + return result + + +class HistoricalTimingTest(unittest.TestCase): + def test_strict_policy_unchanged(self): + rows=projected(); result=m.reconstruct(rows,config()) + self.assertEqual(result[0]['coveredSeconds'],180) + self.assertEqual(result[0]['publicationEstimatedSeconds'],0) + + def test_equal_endpoint_bounded_tail_estimate(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertEqual(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],120) + self.assertEqual(w['publicationEstimatedBySourceSeconds'],{'pv':120}) + self.assertEqual(w['loadW'],6000) + self.assertFalse(w['fullPhysicalIntervalMeasured']) + self.assertFalse(w['meterBoundaryVerified']) + + def test_no_extension_at_open_end(self): + rows=projected(span=300) + for r in rows: r['raw']['pv']['sourceUpdatedAt']=NOW + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['coveredSeconds'],60) + + def test_equal_zero_requires_new_original_timestamp(self): + rows=projected() + for r in rows:r['raw']['pv']['value']=0 + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],300) + for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coveredSeconds'],60) + + def test_changed_endpoints_not_interpolated(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=r['raw']['pv']['sourceUpdatedAt']-NOW + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_no_floating_tolerance(self): + rows=projected() + for r in rows:r['raw']['pv']['value']+=1e-9*(r['raw']['pv']['sourceUpdatedAt']-NOW) + self.assertEqual(m.reconstruct(rows,timed_config())[0]['publicationEstimatedSeconds'],0) + + def test_long_gap_not_recovered_by_equal_zero(self): + rows=projected(span=720,period=660) + for r in rows:r['raw']['pv']['value']=0 + out=m.reconstruct(rows,timed_config()) + self.assertEqual(sum(w['publicationEstimatedSeconds'] for w in out),0) + self.assertEqual(out[0]['coveredSeconds'],60) + + def test_collector_gap_not_bridged(self): + rows=projected(); rows=[r for r in rows if r['capturedAt']!=NOW+90] + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coveredSeconds'],300) + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_source_blocks_equality_inference(self): + rows=projected();rows[3]['raw']['pv']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + self.assertLess(w['coverage'],1) + + def test_conflicting_same_timestamp_blocks_equality(self): + rows=projected();rows[1]['raw']['pv']['value']+=10 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['publicationEstimatedSeconds'],60) + + def test_invalid_other_physical_input_still_vetoes(self): + rows=projected();rows[3]['raw']['battery']['valid']=False + w=m.reconstruct(rows,timed_config())[0] + self.assertLess(w['coverage'],1) + + def test_future_receipt_not_available_early(self): + rows=projected() + for r in rows:r['_receivedAt']=NOW+900 + w=m.reconstruct(rows,timed_config())[0] + self.assertEqual(w['availableNotBefore'],NOW+900) + + def test_unrelated_virtual_failure_does_not_veto(self): + rows=projected() + for r in rows:r['raw']['sdl']['valid']=False + self.assertEqual(m.reconstruct(rows,timed_config())[0]['coverage'],1) + + def test_no_input_mutation_or_source_age_change(self): + rows=projected();c=timed_config();before=copy.deepcopy((rows,c)) + m.reconstruct(rows,c) + self.assertEqual((rows,c),before) + self.assertEqual(c['sources'][1]['maxAgeSeconds'],60) + + def test_policy_requires_trusted_explicit_bounds(self): + for change in ({'version':True},{'method':'always_fill'},{'sources':{'pv':{'maxSpanSeconds':600,'evidenceId':'test-policy'}}}, + {'sources':{'grid':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}, + {'sources':{'sdl':{'maxSpanSeconds':90,'evidenceId':'test-policy'}}}): + c=timed_config();c['historyTimingPolicy'].update(change) + with self.subTest(change=change),self.assertRaises(ValueError):m.validate_config(c) + + def test_profile_values_stay_estimates(self): + w=m.reconstruct(projected(),timed_config())[0] + self.assertTrue(w['estimated']) + self.assertNotIn('controlEnabled',w) + self.assertNotIn('accountingEvidenceId',w) + + +class HistoryReferenceIntegrationTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.tmp.name)/'test.sqlite')) + self.c=config();self.new=timed_config(datasetId='physical-v2',sourceDatasetId='physical-v1') + m.register_dataset(self.store.con,AID,self.c,NOW) + def tearDown(self):self.store.close();self.tmp.cleanup() + def register(self):m.register_dataset(self.store.con,AID,self.new,NOW) + def fill(self): + raw=[] + for t in range(NOW-7200,NOW+1,30): + r=record(t);r['raw']['pv']['sourceUpdatedAt']=t-(t-(NOW-7200))%120;raw.append(r) + for i in range(0,len(raw),120):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':raw[i:i+120]},NOW) + return len(raw) + def test_reference_shares_original_immutable_observations(self): + n=self.fill();self.register() + self.assertEqual(m.pipeline_status(self.store.con,AID)['datasets'][1]['records'],n) + self.assertEqual(self.store.con.execute('SELECT count(*) FROM planner_observations').fetchone()[0],n) + self.assertEqual(m.configuration(self.store.con,AID,'physical-v1'),self.c) + def test_no_append_into_derived_dataset(self): + self.register() + with self.assertRaises(ValueError):m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v2','records':[record(NOW)]},NOW) + def test_source_mapping_cannot_change_in_reference(self): + for key,value in [('mappingSha256','c'*64),('formula','solar_terminal_v1'),('minimumCoverage',.9)]: + c=copy.deepcopy(self.new);c[key]=value + with self.subTest(key=key),self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_no_reference_to_other_plant(self): + with self.assertRaises(ValueError):m.register_dataset(self.store.con,'00000000-0000-4000-8000-000000000099',self.new,NOW) + def test_no_cycles_or_reference_chains(self): + self.register(); c={**self.new,'datasetId':'physical-v3','sourceDatasetId':'physical-v2'} + with self.assertRaises(ValueError):m.register_dataset(self.store.con,AID,c,NOW) + def test_training_model_is_versioned_and_policy_annotated(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v1',settings,NOW+60) + m.advance(self.store.con,AID,'physical-v2',settings,NOW+60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v1',NOW+60)) + model=m.current_model(self.store.con,AID,'physical-v2',NOW+60) + self.assertIsNotNone(model);self.assertEqual(model['historyTimingMethod'],'equal_endpoint_v1') + self.assertFalse(model['measurementBoundaryVerified']) + state=m.pipeline_status(self.store.con,AID)['datasets'][1]['detail'] + self.assertGreater(state['publicationEstimatedSeconds'],0) + self.assertFalse(state['originalFreshnessLimitsChanged']) + def test_no_future_receipt_leaks_into_model(self): + self.fill();self.register() + m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW-60) + self.assertIsNone(m.current_model(self.store.con,AID,'physical-v2',NOW-60)) + def test_existing_sdl_freshness_not_relaxed_by_history(self): + self.fill();self.register();m.advance(self.store.con,AID,'physical-v2',self.store.settings(AID),NOW+30) + op,fc,tar=inputs(datetime.fromtimestamp(NOW+180,timezone.utc)) + with self.assertRaises(ValueError):m.apply_load_forecast(self.store.con,AID,'physical-v2',fc,NOW+180) + def test_full_application_plan_uses_reference_model(self): + self.fill();self.register();settings=self.store.settings(AID) + m.advance(self.store.con,AID,'physical-v2',settings,NOW) + op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc)) + for kind,val in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.store,AID,kind,val,datetime.fromtimestamp(NOW,timezone.utc)) + self.store.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v2'},0,datetime.fromtimestamp(NOW,timezone.utc)) + out=run_once(self.store,datetime.fromtimestamp(NOW,timezone.utc)) + self.assertTrue(out['executable'],out);self.assertFalse(out['liveEnabled']) + self.assertEqual(out['inputQuality']['dataPipeline']['historyTimingMethod'],'equal_endpoint_v1') + def test_backfilled_or_late_confirmed_training_is_not_causal_holdout(self): + # Completed synthetic windows learned only NOW cannot validate a model at a past split. + with self.store.con: + for t in range(NOW-3*86400,NOW,300): + window={'start':t,'profileUsable':True,'coverage':1.,'loadW':5000.,'availableNotBefore':NOW} + self.store.con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?)',(AID,'physical-v1',t,NOW,1.,m.canonical(window))) + m.advance(self.store.con,AID,'physical-v1',self.store.settings(AID),NOW) + model=m.current_model(self.store.con,AID,'physical-v1',NOW) + self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout') + + def test_v1_raw_ingest_works_after_reference_added(self): + self.register() + r=m.ingest_batch(self.store.con,AID,{'version':1,'datasetId':'physical-v1','records':[record(NOW)]},NOW) + self.assertEqual(r['stored'],1) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_history_timing_deployment.py b/services/netplan-v4/tests/test_history_timing_deployment.py new file mode 100644 index 0000000..2acf194 --- /dev/null +++ b/services/netplan-v4/tests/test_history_timing_deployment.py @@ -0,0 +1,94 @@ +"""Installer ordering/recovery tests; subprocesses, privileges and DB backup are simulated.""" +import hashlib +import importlib.util +import json +import subprocess +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +SCRIPT=Path(__file__).resolve().parents[1]/'commissioning/deploy_history_timing.py' +spec=importlib.util.spec_from_file_location('history_deploy_tests',SCRIPT) +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class TimingDeploymentTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name) + self.package=self.root/'commissioning/history-timing-source';self.package.mkdir(parents=True) + files={} + for index,name in enumerate(d.TARGETS): + original=(b'old:'+name.encode()) if index%2==0 else None + target=self.root/name;target.parent.mkdir(parents=True,exist_ok=True) + if original is not None:target.write_bytes(original) + candidate=self.package/'source'/name;candidate.parent.mkdir(parents=True,exist_ok=True);candidate.write_bytes(b'new:'+name.encode()) + files[name]={'before':hashlib.sha256(original).hexdigest() if original else None,'after':d.digest(candidate)} + (self.package/'dataset.json').write_text('{}') + (self.root/'commissioning/deploy_application.py').write_text('synthetic helper') + self.manifest={'scope':'historical_publication_only','installationId':'plant','files':files, + 'existingDeploymentHelperSha256':d.digest(self.root/'commissioning/deploy_application.py'), + 'datasetSha256':d.digest(self.package/'dataset.json'),'installerSha256':d.digest(SCRIPT)} + (self.package/'RELEASE.json').write_text(json.dumps(self.manifest)) + self.context=patch.multiple(d,ROOT=self.root,PACKAGE=self.package);self.context.start() + self.calls=[] + def tearDown(self):self.context.stop();self.tmp.cleanup() + def fake_run(self,args,**kwargs): + self.calls.append(args) + if 'inspect' in args:out='sha256:test-old-image' + elif 'ps' in args:out='test-container' + elif 'exec' in args:out=json.dumps({'settingsUnchanged':True,'liveEnabled':False}) + else:out='' + return subprocess.CompletedProcess(args,0,out,'') + def execute(self,fail_at=None): + def run(args,**kwargs): + out=self.fake_run(args,**kwargs) + if fail_at and fail_at in args:raise subprocess.CalledProcessError(1,args) + return out + def atomic(path,data,*args,**kwargs):path.write_bytes(data) + def backup(source,dest):self.calls.append(['test-backup']);dest.write_text('synthetic backup') + helpers=types.SimpleNamespace(atomic=atomic,backup_database=backup) + with patch.object(d.os,'geteuid',return_value=0),patch.object(d.os,'chown'),patch.object(d.subprocess,'run',side_effect=run),patch.dict(sys.modules,{'deploy_application':helpers}): + d.run('plant',True) + def report(self):return json.loads(next(self.root.glob('history-timing-releases/*/REPORT.json')).read_text()) + def test_default_checks_only(self): + d.run('plant');self.assertEqual(self.calls,[]) + self.assertFalse((self.root/'history-timing-releases').exists()) + def test_order_tests_backup_then_only_v4_replacement(self): + self.execute() + test=next(i for i,c in enumerate(self.calls) if '/app/run_tests.py' in c) + backup=next(i for i,c in enumerate(self.calls) if c==['test-backup']) + up=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(test,backup);self.assertLess(backup,up) + self.assertTrue(all('license-portal' not in c and 'forecast-engine' not in c for c in self.calls)) + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_failed_tests_restore_host_source_without_restart(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('/app/run_tests.py') + self.assertFalse(any('up' in c for c in self.calls)) + for name,v in self.manifest['files'].items():self.assertEqual(d.digest(self.root/name),v['before']) + def test_failed_registration_rolls_image_back(self): + with self.assertRaises(subprocess.CalledProcessError):self.execute('exec') + self.assertIn('previous_image_restored',self.report()['rollback']) + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_source_drift_refused_before_actions(self): + (self.root/d.TARGETS[0]).write_text('parallel change') + with self.assertRaises(ValueError):d.verify() + self.assertEqual(self.calls,[]) + def test_source_already_installed_idempotent(self): + for name in d.TARGETS:(self.root/name).write_bytes((self.package/'source'/name).read_bytes()) + self.assertEqual(d.verify()['scope'],'historical_publication_only') + self.execute() + self.assertEqual(self.report()['status'],'historical_timing_installed_no_actuation') + def test_no_production_selection_or_permission_in_bootstrap(self): + self.assertNotIn("'/settings'",d.BOOTSTRAP) + self.assertNotIn('/trial/arm',d.BOOTSTRAP) + self.assertIn("before['settings']==after['settings']",d.BOOTSTRAP) + self.assertIn("historyTimingVersion",d.BOOTSTRAP) + def test_target_image_contains_installer_test_dependency(self): + dockerfile=(SCRIPT.parents[1]/'Dockerfile').read_text() + self.assertIn('COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py',dockerfile) + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_integrated_deployment.py b/services/netplan-v4/tests/test_integrated_deployment.py new file mode 100644 index 0000000..d9ed2d7 --- /dev/null +++ b/services/netplan-v4/tests/test_integrated_deployment.py @@ -0,0 +1,71 @@ +import ast +import hashlib +import importlib.util +import io +import contextlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('integrated_deployment_test',ROOT/'deploy_integrated_shadow.py') +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class IntegratedDeploymentTest(unittest.TestCase): + def test_exact_server_scope_and_no_symcon(self): + scope=d.compose_groups() + self.assertEqual(set(scope),{'v4','forecast','portal'}) + self.assertEqual(scope['forecast'][2],['api','forecast-engine','tariff-importer']) + self.assertNotIn('license-admin',scope['portal'][2]) + self.assertNotIn('/var/lib/symcon',(ROOT/'deploy_integrated_shadow.py').read_text()) + def test_sqlite_backup_is_consistent_and_source_unchanged(self): + with tempfile.TemporaryDirectory() as temp: + source=Path(temp)/'before.sqlite';dest=Path(temp)/'backup.sqlite' + con=sqlite3.connect(source);con.execute('CREATE TABLE test(value INTEGER)') + con.execute('INSERT INTO test VALUES (37)');con.commit();con.close() + before=source.read_bytes() + with patch.object(d.os,'geteuid',return_value=1000):result=d.backup_database(source,dest) + self.assertEqual(result['integrity'],'ok');self.assertEqual(before,source.read_bytes()) + check=sqlite3.connect(dest);self.assertEqual(check.execute('SELECT value FROM test').fetchone()[0],37);check.close() + self.assertEqual(result['sha256'],hashlib.sha256(dest.read_bytes()).hexdigest()) + def test_missing_database_not_fabricated(self): + with tempfile.TemporaryDirectory() as temp: + dst=Path(temp)/'backup' + self.assertEqual(d.backup_database(Path(temp)/'missing',dst),{'exists':False}) + self.assertFalse(dst.exists()) + def test_scope_uses_no_environment_dump_or_control_activation(self): + text=(ROOT/'deploy_integrated_shadow.py').read_text() + self.assertNotIn('.Config.Env',text) + self.assertNotIn('NetzfahrplanAktiv',text) + self.assertNotIn('IPS_RequestAction',text) + self.assertNotIn('PROGNOSIS_SERVICE_TOKEN',text) + ast.parse(text) + def test_build_and_tests_precede_all_service_recreation(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp)/'netplan';root.mkdir();project=Path(temp)/'forecast';project.mkdir() + portal=Path(temp)/'portal';portal.mkdir();(root/'acceptance').mkdir() + commands=[] + def fake(args,**kwargs): + commands.append(list(args));out='' + if 'ps' in args:out='0123456789abcdef\n' + elif 'inspect' in args: + out='sha256:'+('a'*64) if '{{.Image}}' in args else '/some/compose.yaml' + elif 'exec' in args: + out=json.dumps({'health':{'mode':'shadow','liveEnabled':False},'inputEventsByKind':{},'lastRunStatus':None}) + return SimpleNamespace(returncode=0,stdout=out) + with patch.object(d,'ROOT',root),patch.object(d,'PROJECT',project),patch.object(d,'PORTAL',portal),patch.object(d,'verify_sources',return_value={}),patch.object(d.os,'geteuid',return_value=1000),patch.object(d.subprocess,'run',side_effect=fake),patch('install_hooks.plan',return_value={}),patch('install_hooks.apply',return_value=None),contextlib.redirect_stdout(io.StringIO()): + d.run('e3a08f9e-af12-4695-99bd-8b51c0520021') + ups=[i for i,c in enumerate(commands) if 'up' in c] + self.assertEqual(len(ups),3) + test_at=next(i for i,c in enumerate(commands) if '--test-only' in c) + build_at=max(i for i,c in enumerate(commands) if 'build' in c) + self.assertLess(test_at,min(ups));self.assertLess(build_at,min(ups)) + self.assertTrue(all('--no-build' in commands[i] and '--no-deps' in commands[i] for i in ups)) + report=json.loads(next((root/'deployment-reports').glob('*/DEPLOYMENT.json')).read_text()) + self.assertEqual(report['status'],'server_shadow_installed') + self.assertFalse(report['liveEnabled']);self.assertFalse(report['productionReady']) diff --git a/services/netplan-v4/tests/test_mapping_recovery.py b/services/netplan-v4/tests/test_mapping_recovery.py new file mode 100644 index 0000000..1532535 --- /dev/null +++ b/services/netplan-v4/tests/test_mapping_recovery.py @@ -0,0 +1,102 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path +from hashlib import sha256 +from netplan_v4.store import PlannerStore +from netplan_v4 import mapping_identity as mi, measurement_pipeline as m +from netplan_v4.service import create_app +from fastapi.testclient import TestClient +from test_measurement_pipeline import config, record, NOW +from test_v4 import AID, TOKEN + + +def proof(c): + a = {'version':1, 'installationId':AID, 'reportedInventorySha256':c['inventorySha256'], + 'accounting':{'splitToleranceW':100.0,'sources':copy.deepcopy(c['sources'])}} + b=copy.deepcopy(a); b['accounting']['splitToleranceW']=100 + p={'version':1, 'canonicalJson':json.dumps(a,separators=(',',':')), 'legacyJson':json.dumps(b,separators=(',',':'))} + c['mappingSha256']=sha256(p['canonicalJson'].encode()).hexdigest() + return p + + +class MappingRecoveryTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(); self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.c=config(); self.proof=proof(self.c); self.alias=sha256(self.proof['legacyJson'].encode()).hexdigest() + m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self): self.s.close(); self.tmp.cleanup() + def register(self): return mi.register(self.s.con,AID,self.c,self.proof,NOW) + def batch(self,rows): return m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows},NOW) + def r(self,t,legacy=False): + r=record(t,self.c) + if legacy:r['mappingSha256']=self.alias + return r + def test_original_mixed_batch_defect_and_atomic_recovery(self): + rows=[self.r(NOW-3600+i*30,i>=45) for i in range(120)]; before=copy.deepcopy(rows) + with self.assertRaises(ValueError):self.batch(rows) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],0) + self.register(); result=self.batch(rows) + self.assertEqual(result['stored'],120); self.assertEqual(rows,before) + self.assertEqual(result['acceptedThrough'],rows[-1]['capturedAt']) + origins=self.s.con.execute('SELECT received_mapping,COUNT(*) FROM planner_observation_origins GROUP BY received_mapping').fetchall() + self.assertEqual(dict(origins)[self.alias],75) + def test_retry_does_not_duplicate(self): + self.register(); self.batch([self.r(NOW,True)]) + self.assertEqual(self.batch([self.r(NOW,True)])['duplicates'],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1) + def test_canonical_retry_preserves_both_origins(self): + self.register(); self.batch([self.r(NOW,True)]); self.batch([self.r(NOW)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],2) + def test_unknown_alias_rolls_back_entire_block(self): + self.register(); bad=self.r(NOW,True);bad['mappingSha256']='f'*64 + with self.assertRaises(ValueError): self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],0) + def test_conflicting_measurement_keeps_existing_and_does_not_ack_prefix(self): + self.register(); self.batch([self.r(NOW)]) + bad=self.r(NOW,True);bad['raw']['grid']['value']+=500 + with self.assertRaises(ValueError):self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_proof_wrong_canonical_hash_rejected(self): + p=copy.deepcopy(self.proof);p['canonicalJson']+=' ' + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_numeric_change_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":101,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_boolean_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":true,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_other_field_change_rejected(self): + p=copy.deepcopy(self.proof);b=json.loads(p['legacyJson']);b['accounting']['sources'][0]['factorToW']=2;p['legacyJson']=json.dumps(b) + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_inventory_remains_strict(self): + self.register();r=self.r(NOW,True);r['reportedInventorySha256']='d'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_other_plant_does_not_inherit_proof(self): + self.register();other='30509683-4569-49e4-848f-4905e4cc813a';m.register_dataset(self.s.con,other,self.c,NOW) + r=self.r(NOW,True);r['installationId']=other + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,other,{'version':1,'datasetId':self.c['datasetId'],'records':[r]},NOW) + def test_other_dataset_does_not_inherit_proof(self): + self.register();c=copy.deepcopy(self.c);c['datasetId']='unrelated';m.register_dataset(self.s.con,AID,c,NOW) + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':'unrelated','records':[self.r(NOW,True)]},NOW) + def test_direct_project_still_strict(self): + self.register() + with self.assertRaises(ValueError):m.project(self.r(NOW,True),self.c,AID,NOW) + def test_registration_idempotent_and_persistent(self): + a=self.register();b=self.register();self.assertEqual(a,b) + self.s.close();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.assertEqual(self.batch([self.r(NOW,True)])['stored'],1) + def test_malformed_proof_no_registration(self): + for x in ({}, {'version':True,'canonicalJson':'{}','legacyJson':'{}'}): + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,x,NOW) + def test_internal_api_authentication(self): + app=create_app(str(Path(self.tmp.name)/'db.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + url=f'/internal/v2/prognosis/{AID}/planner/datasets/{self.c["datasetId"]}/mapping-compatibility' + self.assertEqual(client.put(url,json=self.proof).status_code,401) + response=client.put(url,json=self.proof,headers={'X-Enelix-Service-Token':TOKEN}) + self.assertEqual(response.status_code,200,response.text);self.assertFalse(response.json()['controlEnabled']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_measurement_pipeline.py b/services/netplan-v4/tests/test_measurement_pipeline.py new file mode 100644 index 0000000..85b4f38 --- /dev/null +++ b/services/netplan-v4/tests/test_measurement_pipeline.py @@ -0,0 +1,167 @@ +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once, status +from test_v4 import inputs, AID, TOKEN + +NOW=int(datetime(2026,10,2,12,tzinfo=timezone.utc).timestamp()) + +def config(**changes): + c={'datasetId':'physical-v1','mappingSha256':'a'*64,'inventorySha256':'b'*64, + 'formula':'physical_sum_v1','solarReference':None,'minimumCoverage':.95, + 'maximumGapSeconds':10,'minimumTrainingHours':1,'historyDays':14,'sources':[]} + for i,(k,role) in enumerate((('grid','grid'),('pv','pv'),('battery','physical_storage'),('sdl','sdl_request'))): + c['sources'].append({'key':k,'role':role,'variableId':101+i,'factorToW':1,'maxAgeSeconds':60}) + c.update(changes);return c + +def record(t, c=None, **values): + c=c or config(); v={'grid':3000.,'pv':5000.,'battery':2000.,'sdl':0.,**values} + return {'schemaVersion':1,'kind':'raw_accounting_capture','installationId':AID, + 'capturedAt':m.iso(t),'captureStartedAt':m.iso(t),'mappingSha256':c['mappingSha256'], + 'reportedInventorySha256':c['inventorySha256'],'raw':{s['key']:{'variableId':s['variableId'],'value':v.get(s['key'],0), + 'sourceUpdatedAt':t,'issues':[]} for s in c['sources']}} + +class MeasurementPipelineTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.path=str(Path(self.tmp.name)/'pipeline.sqlite') + self.s=PlannerStore(self.path);self.c=config();m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self):self.s.close();self.tmp.cleanup() + def batch(self,rows,now=NOW,plant=AID):return m.ingest_batch(self.s.con,plant,{'version':1,'datasetId':'physical-v1','records':rows},now) + def history(self,hours=2): + rows=[record(t) for t in range(NOW-hours*3600,NOW+1,30)] + for i in range(0,len(rows),120):self.batch(rows[i:i+120]) + def test_append_ack_idempotence(self): + r=record(NOW);self.assertEqual(self.batch([r])['stored'],1) + a=self.batch([r]);self.assertEqual(a['duplicates'],1);self.assertEqual(a['acceptedThrough'],m.iso(NOW)) + def test_conflict_rolls_back_whole_batch(self): + self.batch([record(NOW)]) + with self.assertRaises(ValueError):self.batch([record(NOW-30),record(NOW,grid=9000)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_registration_immutable(self): + self.assertEqual(m.register_dataset(self.s.con,AID,self.c,NOW)['status'],'configured') + with self.assertRaises(ValueError):m.register_dataset(self.s.con,AID,{**self.c,'minimumCoverage':1.},NOW) + def test_plants_isolated(self): + with self.assertRaises(ValueError):self.batch([record(NOW)],plant='30509683-4569-49e4-848f-4905e4cc813a') + def test_mapping_is_not_device_supplied(self): + r=record(NOW);r['mappingSha256']='c'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_no_unknown_text_persisted(self): + r=record(NOW);r['unknown']='do-not-store';r['raw']['grid']['value']='do-not-store';r['assessment']={'status':'verified'} + self.batch([r]);data=self.s.con.execute('SELECT value FROM planner_observations').fetchone()[0] + self.assertNotIn('do-not-store',data);self.assertIn('null',data);self.assertNotIn('verified',data) + def test_missing_zero_not_invented(self): + r=record(NOW);r['raw'].pop('grid');p=m.project(r,self.c,AID,NOW) + self.assertIsNone(p['raw']['grid']['value']);self.assertFalse(p['raw']['grid']['valid']) + def test_boolean_not_numeric(self): + p=m.project(record(NOW,grid=True),self.c,AID,NOW);self.assertFalse(p['raw']['grid']['valid']) + def test_batch_order_required(self): + with self.assertRaises(ValueError):self.batch([record(NOW),record(NOW-30)]) + def test_batch_size_limit(self): + with self.assertRaises(ValueError):self.batch([record(NOW)]*121) + def test_no_naive_time(self): + r=record(NOW);r['capturedAt']='2026-10-02T12:00:00' + with self.assertRaises(ValueError):self.batch([r]) + def test_measurement_time_remains_whole_second(self): + r=record(NOW);r['capturedAt']='2026-10-02T12:00:00.000001+00:00' + with self.assertRaises(ValueError):self.batch([r]) + def test_future_source_not_rejuvenated(self): + r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1 + self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid']) + def test_profile_uses_physical_not_virtual(self): + r=record(NOW);r['raw']['virtual_ev']={'value':-90000} + self.assertEqual(m.physical_value({'grid':3000,'pv':5000,'battery':2000},self.c),6000) + def test_zero_numeric_valid(self): + p=m.project(record(NOW,grid=0,pv=0,battery=0),self.c,AID,NOW);self.assertTrue(p['raw']['grid']['valid']) + def projected(self,rows):return [m.project(r,self.c,AID,NOW) for r in rows] + def test_rollup_time_weighted(self): + rows=[record(t,grid=3000 if t=a-1e-10 for a,b in zip(slopes,slopes[1:])));self.assertAlmostEqual(slopes[-1],5.) + self.assertAlmostEqual(o.incremental_cost(1.,.8,5.),0.) + def test_hard_cap_beats_outlook(self): + p=self.solve(limits=Limits(import_w=1000.),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(max(x['gridTargetW'] for x in p['points']),1000.001) + def test_no_fixed_cap_is_supported(self): + p=self.solve(limits=Limits(),peak_outlooks={'2026-10':outlook()}) + self.assertTrue(p['executable']);self.assertTrue(all(x['importLimitW'] is None for x in p['points'])) + def test_monthly_cap_beats_outlook(self): + p=self.solve(limits=Limits(manager_month_limits_w={10:1000.}),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_estimated_basis_not_in_measured_map(self): + p=self.solve(peak_context={'2026-10':estimate(1.)}) + self.assertEqual(p['measuredPeaksKw'],{});self.assertEqual(p['peakBasisKw']['2026-10'],1.) + self.assertTrue(p['peakCostIsEstimate']) + def test_context_cannot_lie_about_numerical_basis(self): + self.assertFalse(self.solve(peak_context={'2026-10':estimate(3.)})['executable']) + def test_no_free_cap_source(self): + with self.assertRaises(ValueError):basis_record(estimate(source='manager_cap'),'2026-10',AT,True) + def test_overlap_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),future_from=AT)})['executable']) + def test_future_observation_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),issued_at=AT+timedelta(seconds=1))})['executable']) + def test_wrong_month_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-11':outlook()})['executable']) + def test_invalid_probabilities_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':outlook(scenarios=((5.,.8),))})['executable']) + def test_no_history_no_outlook_not_fake_allowance(self): + self.assertIsNone(empirical_rest_month([],month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p')) + def test_empirical_requires_comparable_complete_history(self): + records=[] + for n in range(20): + d=AT.astimezone(ZURICH).replace(hour=0,minute=0)-timedelta(days=n+1) + records.append({'day':d.strftime('%Y-%m-%d'),'peakKw':3.+n%4,'observedAt':(d+timedelta(days=1)).isoformat(),'quality':'estimated','complete':True,'controlPolicyId':'p'}) + args=dict(month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p') + o=empirical_rest_month(records,**args);self.assertIsNotNone(o);o.validate(AT,args['horizon_end']) + for r in records:r['controlPolicyId']='different' + self.assertIsNone(empirical_rest_month(records,**args)) + + +class EstimatedBasisServiceTest(unittest.TestCase): + def setUp(self):self.tmp=TemporaryDirectory();self.path=str(Path(self.tmp.name)/'planner.sqlite');self.s=PlannerStore(self.path) + def tearDown(self):self.s.close();self.tmp.cleanup() + def load(self,when=AT,quarter=False): + op,fc,tar=inputs(when);op['measuredPeaks']={};op['planningPeaks']={'2026-10':estimate(at=when)} + if quarter: + op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.2,'measuredSeconds':int((when-AT).total_seconds()),'quality':'estimated','source':'power_history_estimate','coverage':1.} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,when) + return op + def allow(self):self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},self.s.settings(AID)['revision'],AT) + def test_opt_in_required(self): + self.load();self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + self.allow();p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertTrue(p['peakCostIsEstimate']) + self.assertEqual(self.s.peaks(AID),{}) + def test_no_peak_not_invented_even_when_opted_in(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.allow() + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + def test_historical_seed_independent_from_live_operation(self): + self.allow();op,fc,tar=inputs();op['measuredPeaks']={} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + ingest(self.s,AID,'planning_basis',{'version':1,'eventId':'seed','observedAt':AT.isoformat(),'peaks':{'2026-10':estimate()}},AT) + p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertEqual(p['measuredPeaksKw'],{}) + def test_estimates_persist_but_plans_do_not_increase_them(self): + self.load();self.allow();p=run_once(self.s,AT) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['kw'],5.) + self.s.close();self.s=PlannerStore(self.path) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['quality'],'estimated') + def test_verified_source_has_precedence_even_if_lower_than_estimate(self): + self.load();self.allow();self.s.initialize_peak(AID,'2026-10',4.,'verified_month_history',AT) + p=run_once(self.s,AT);self.assertEqual(p['measuredPeaksKw'],{'2026-10':4.});self.assertFalse(p['peakCostIsEstimate']) + def test_quarter_estimate_never_disguised(self): + self.load(AT+timedelta(minutes=5),True);self.allow();p=run_once(self.s,AT+timedelta(minutes=5)) + self.assertTrue(p['executable']);self.assertEqual(p['inputQuality']['quarter'],'estimated') + def test_incomplete_quarter_refused(self): + op,_,_=inputs();op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.1,'measuredSeconds':0,'quality':'estimated','source':'power_history_estimate','coverage':.8} + with self.assertRaises(ValueError):ingest(self.s,AID,'operation',op,AT) + def test_new_settings_load_old_rows(self): + old=self.s.settings(AID);old.pop('revision');old.pop('measurementPolicy');old.pop('peakOutlookPolicy');old.pop('peakOutlookReliance') + with self.s.con:self.s.con.execute('INSERT INTO planner_settings VALUES(?,?,?)',(AID,1,json.dumps(old))) + self.assertEqual(self.s.settings(AID)['measurementPolicy'],'verified_only') + self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},1,AT) + def test_http_explicit_estimate_optin_and_no_live(self): + app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as c: + url=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':0,'changes':{'measurementPolicy':'allow_estimates'}}).status_code,200) + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + + +class RuntimeMeterTest(unittest.TestCase): + def test_exact_samples_still_estimates(self): + r=integrate_power([(0,1000.,0.,'p'),(60,1000.,.02,'p'),(120,1000.,.04,'p')],0,120) + self.assertAlmostEqual(r['importKwh'],1/30);self.assertFalse(r['billingEvidence']) + def test_gap_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,0.,'p'),(180,1000.,.1,'p')],0,180)) + def test_reset_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,1.,'p'),(60,1000.,0.,'p')],0,60)) + def test_control_policy_change_not_comparable(self):self.assertIsNone(integrate_power([(0,1000.,0.,'a'),(60,1000.,1.,'b')],0,60)) + def test_only_completed_quarters_update_peak(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));mid='symcon-active-import:'+'a'*64 + for i in range(16): + at=AT+timedelta(minutes=i) + obs={'meterId':mid,'sampleAt':at.isoformat(),'powerW':2000.,'totalImportKwh':10+i/30,'controlPolicyId':'p'} + with s.con:observe(s.con,AID,obs,at) + if i<15:self.assertEqual(assumptions(s.con,AID),{}) + self.assertAlmostEqual(assumptions(s.con,AID)['2026-10']['kw'],2.) + self.assertEqual(s.peaks(AID),{});s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_receiver_contract.py b/services/netplan-v4/tests/test_receiver_contract.py new file mode 100644 index 0000000..1377ae8 --- /dev/null +++ b/services/netplan-v4/tests/test_receiver_contract.py @@ -0,0 +1,45 @@ +import copy +from datetime import timedelta +import tempfile +from pathlib import Path +import unittest +from netplan_v4.receiver_contract import control_context, provenance +from netplan_v4.store import PlannerStore +from netplan_v4.service import ingest, run_once, status +from test_v4 import inputs, AT, AID + +class ReceiverContractTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'qa.sqlite')) + op,fc,tar=inputs() + for kind,value in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,kind,value,AT) + def tearDown(self):self.s.close();self.tmp.cleanup() + def test_published_plan_and_response_bound_to_same_plant(self): + p=run_once(self.s,AT);r=status(self.s,AID,AT) + self.assertEqual(p['installationId'],AID);self.assertEqual(r['installationId'],AID) + self.assertEqual(r['receiverProtocolVersion'],1);self.assertTrue(r['fresh']) + self.assertFalse(r['liveEnabled']);self.assertFalse(p['liveEnabled']) + self.assertEqual(p['inputRefs'],{'operation':'op1','forecast':'fc1','tariffs':'tar1'}) + def test_control_context_tracks_stable_policy_not_volatile_soc(self): + op=inputs()[0];a=control_context(op);op['batteries'][0]['socPercent']=80.;op['gridW']=1000. + self.assertEqual(a,control_context(op));op['batteries'][0]['minSocPercent']=15. + self.assertNotEqual(a,control_context(op)) + def test_context_is_snapshot_not_reference_to_future_changes(self): + op,fc,tar=inputs();p=provenance({'operation':op,'forecast':fc,'tariffs':tar}) + op['limits']['importW']=100.;self.assertEqual(p['controlContext']['limits']['importW'],10000.) + def test_no_plan_for_another_installation(self): + run_once(self.s,AT);r=status(self.s,'00000000-0000-4000-8000-000000000002',AT) + self.assertIsNone(r['plan']);self.assertFalse(r['fresh']) + def test_shadow_ack_is_not_execution(self): + p=run_once(self.s,AT) + self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,step=p['points'][0]['time'],status='shadow_seen') + self.assertEqual(status(self.s,AID,AT)['acknowledgement']['status'],'shadow_seen') + with self.assertRaises(ValueError):self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,status='applied') + def test_new_configuration_invalidates_fresh_flag(self): + run_once(self.s,AT);self.s.save_settings(AID,{'family':'23'},0,AT+timedelta(seconds=1)) + self.assertFalse(status(self.s,AID,AT+timedelta(seconds=2))['fresh']) + def test_rearm_default_matches_php_contract(self): + op=inputs()[0];op['batteries'][0]['rearmSocPercent']=None + self.assertEqual(control_context(op)['batteries']['b']['rearmSocPercent'],10.) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_release_preflight.py b/services/netplan-v4/tests/test_release_preflight.py new file mode 100644 index 0000000..d4942a9 --- /dev/null +++ b/services/netplan-v4/tests/test_release_preflight.py @@ -0,0 +1,51 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +import unittest + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('release_preflight_test',ROOT/'release_preflight.py') +preflight=importlib.util.module_from_spec(spec);spec.loader.exec_module(preflight) + +class ReleasePreflightTest(unittest.TestCase): + def test_preflight_never_restarts_existing_services(self): + for name,args,cwd,timeout in preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021'): + self.assertNotIn('up',args);self.assertNotIn('restart',args);self.assertNotIn('--with-bridges',args) + self.assertGreater(timeout,0) + first=preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021')[0][1] + self.assertIn('--test-only',first) + def test_php_runs_without_network_or_writeable_root(self): + args=next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='php83_offline') + self.assertIn('--read-only',args);self.assertEqual(args[args.index('--network')+1],'none') + self.assertEqual(args[args.index('--user')+1],'1000:1000') + def test_native_bundle_requires_same_development_source(self): + with TemporaryDirectory() as d: + root=Path(d)/'root';repo=Path(d)/'repo';out=root/'acceptance/php-src' + out.mkdir(parents=True);repo.mkdir() + source=b'=-1e-5 for x in p['points'][:3])) + def test_roundtrip_90_percent(self): + p=solve(seq([0.]*3+[5000.]*3,pv=[5000.]*3+[0.]*3,buy=[.3]*6,sell=[0.]*6),[batt(soc_percent=10.,grid_charging=False,max_discharge_w=5000.)]) + c=sum(max(0,x['batteryTargetW']) for x in p['points']);d=sum(max(0,-x['batteryTargetW']) for x in p['points']);self.assertAlmostEqual(d/c,.9,places=5) + def test_nulleinspeisung(self): + p=solve(seq([0.]*3,pv=[10000.]*3),limits=Limits(export_w=0.));self.assertTrue(p['executable']);self.assertTrue(all(abs(x['gridTargetW'])<1e-4 for x in p['points'])) + def test_nulltariffs(self): + p=solve(seq([1000.]*3,buy=[0.]*3,sell=[0.]*3));self.assertEqual(p['cashCostChf'],0.) + def test_negative_prices(self): + p=solve(seq([1000.]*3,buy=[-.1]*3));self.assertLess(p['energyCostChf'],0.) + def test_asymmetric_limits_and_balance(self): + steps=seq([0.]*3+[3000.]*3,pv=[5000.]*3+[0.]*3) + p=solve(steps,[batt(max_charge_w=700.,max_discharge_w=300.)]);self.assertTrue(p['executable']) + for s,x in zip(steps,p['points']): + self.assertLessEqual(x['batteryTargetW'],700.001);self.assertGreaterEqual(x['batteryTargetW'],-300.001) + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=4) + self.assertTrue(9.999<=x['socEndPercent']['b']<=90.001) + def test_stale_soc_rejected(self):self.assertFalse(solve(seq([0.]*3),[batt(measured_at=AT-timedelta(hours=1))])['executable']) + def test_soc_not_fabricated(self):self.assertFalse(solve(seq([0.]*3),[batt(soc_percent=5.)])['executable']) + def test_peak_increment_only_and_baseline(self): + p=solve(seq([2000.]*3),observed_peaks={'2026-10':1.5},peak_prices={'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],2.5);self.assertAlmostEqual(p['baselineCashCostChf'],2.65) + def test_peak_blocks_unprofitable_charging(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.)],observed_peaks={'2026-10':1.},peak_prices={'2026-10':20.}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_month_switch(self): + at=datetime(2026,9,30,21,45,tzinfo=timezone.utc) + p=solve(seq([2000.]*6,at=at),observed_peaks={'2026-09':2.,'2026-10':.5},peak_prices={'2026-09':5.,'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],7.5) + def test_manager_cap_is_not_measured_peak(self): + p=solve(seq([1000.]*3),limits=Limits(manager_month_limits_w={10:1500.}),observed_peaks={'2026-10':0.},peak_prices={'2026-10':5.}) + self.assertEqual(p['measuredPeaksKw']['2026-10'],0.);self.assertAlmostEqual(p['additionalPeakCostChf'],5.) + def test_infeasible_not_fake_residual(self): + p=solve(seq([10000.]*3),limits=Limits(import_w=1000.));self.assertFalse(p['executable']);self.assertEqual(p['points'],[]) + def test_elapsed_quarter_required(self): + s=seq([1000.],at=AT+timedelta(minutes=10));self.assertFalse(solve(s)['executable']) + p=solve(s,quarter_history={AT:QuarterPast(.5,600)},peak_prices={'2026-10':5.},observed_peaks={'2026-10':0.}) + self.assertAlmostEqual(p['plannedPeaksKw']['2026-10'],7/3) + def test_partial_first_step(self): + s=seq([1000.]*3);s[0]=replace(s[0],start=AT+timedelta(seconds=10),seconds=290) + p=solve(s,quarter_history={AT:QuarterPast(.02,10)});self.assertTrue(p['executable']);self.assertEqual(p['validFrom'],s[0].start.isoformat()) + def test_terminal_guard(self): + p=solve(seq([0.]*3,sell=[10.]*3),[batt(soc_percent=50.,grid_charging=False)]);self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],49.999) + def test_grid_charging_opt_in(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.,grid_charging=False)]);self.assertTrue(all(x['batteryTargetW']<=1e-5 for x in p['points'])) + +class StoreAndSelectionTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def test_settings_revision_and_shadow_only(self): + s=self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(s['revision'],1) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'family':'3'},0,AT) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'runMode':'live'},1,AT) + def test_request_during_calculation_survives(self): + self.store.request(AID,'manual',AT);claim=self.store.claim(AT);self.store.request(AID,'prices_changed',AT) + p=solve(seq([0.]*3),config_revision=0) + with self.assertRaises(ValueError):self.store.publish_shadow(AID,p,0,AT,claim['sequence'],claim['lease_token']) + self.store.finish(claim);self.assertIsNotNone(self.store.claim(AT)) + def test_month_peak_persistent_and_monotonic(self): + self.store.initialize_peak(AID,'2026-10',20.,'meter_month_register',AT) + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',15.,'meter_month_register',AT) + self.store.close();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'));self.assertEqual(self.store.peaks(AID)['2026-10'],20.) + def test_cap_is_not_valid_peak_source(self): + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',20.,'configured_limit',AT) + def test_meter_peak_requires_complete_quarter(self): + self.store.initialize_peak(AID,'2026-10',0.,'verified_new_month',AT) + for i in range(3):r=self.store.record_import_interval(AID,AT+timedelta(minutes=5*i),.5,AT+timedelta(hours=1)) + self.assertEqual(r['quarterPeakKw'],6.);self.assertEqual(self.store.peaks(AID)['2026-10'],6.) + def test_shadow_ack_cannot_claim_applied(self): + p=solve(seq([0.]*3),config_revision=0);self.store.publish_shadow(AID,p,0,AT) + with self.assertRaises(ValueError):self.store.acknowledge(AID,p['planId'],0,AT,status='applied') + self.store.acknowledge(AID,p['planId'],0,AT,status='shadow_seen') + def test_family_registry_extensible(self): + r=default_registry();r.register(Family('42','p42','l42','g42','Future'));self.assertEqual(r.get('42').pv,'p42') + def test_sdl_removed_once(self): + self.assertEqual(split_base_load(15000,[3000],2000),10000);self.assertEqual(split_base_load(13000,[3000],2000,True),10000) + with self.assertRaises(ValueError):split_base_load(100,[1000],0) + def test_auto_uses_comparable_cost_and_margin(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT,c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['family'],'23');self.assertEqual(choose_family('auto','3',scores,now=AT,margin_chf=6.)['family'],'3') + def test_auto_no_future_outcomes(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT+timedelta(hours=1),c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['mode'],'collecting') + def test_training_schedule_and_validation(self): + self.assertTrue(training_due(AT-timedelta(days=1),AT));self.assertFalse(training_due(AT-timedelta(days=1),AT,'weekly')) + self.assertFalse(promote_candidate(active_cost=10,candidate_cost=1,valid_coverage=True,no_data_leakage=False,constraints_passed=True)) + +class ServiceIntegrationTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def load(self,values=None,now=AT): + values=values or inputs() + for kind,value in zip(('operation','forecast','tariffs'),values):ingest(self.store,AID,kind,value,now) + def test_pipeline_shadow_plan(self): + self.load();p=run_once(self.store,AT);self.assertTrue(p['executable']);self.assertEqual(p['runMode'],'shadow');self.assertTrue(status(self.store,AID,AT)['fresh']);self.assertEqual(self.store.current(AID)['planId'],p['planId']) + def test_exact_model_selection(self): + self.load();self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(run_once(self.store,AT)['sourceFamily'],'23') + def test_one_tick_not_every_poll(self): + self.load();run_once(self.store,AT);self.assertEqual(run_once(self.store,AT)['status'],'idle') + def test_stale_operation_visible(self): + self.load();p=run_once(self.store,AT+timedelta(minutes=5));self.assertEqual(p['status'],'awaiting_inputs');self.assertIsNone(self.store.current(AID)) + def test_missing_peak_not_invented(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_static_zero_roundtrip(self): + op,fc,tar=inputs();tar['import']['staticChfKwh']=0.;tar['export']['staticChfKwh']=0.;tar['peakChfKwMonth']=0.;self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['cashCostChf'],0.) + def test_immutable_id_conflict(self): + op,fc,tar=inputs();self.load((op,fc,tar));self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'duplicate');op['gridW']=123. + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_older_input_never_overwrites_current(self): + op,fc,tar=inputs();self.load((op,fc,tar));op['observedAt']=(AT-timedelta(seconds=1)).isoformat();op['eventId']='old' + self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'archived_older');self.assertEqual(latest(self.store,AID,'operation')['eventId'],'op1') + def test_unpublished_dynamic_tail_not_executed(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_published_negative_price_and_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(minutes=15)).isoformat(),'value':-2.,'unit':'Rp/kWh','observedAt':AT.isoformat(),'sourceKind':'published_interval'}]} + ingest(self.store,AID,'prices',price,AT);p=run_once(self.store,AT);self.assertEqual(len(p['points']),3);self.assertEqual(p['points'][0]['importPriceChfKwh'],-.02) + def test_carried_price_does_not_extend_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(hours=48)).isoformat(),'value':.1,'unit':'CHF/kWh','observedAt':AT.isoformat(),'sourceKind':'carried_forward'}]} + ingest(self.store,AID,'prices',price,AT);self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_aggregate_load_warning(self): + op,fc,tar=inputs();fc['families']['3']['loadBasis']='house_total';self.load((op,fc,tar));self.assertTrue(run_once(self.store,AT)['warnings']) + def test_extra_credentials_not_stored(self): + op,fc,tar=inputs();op['password']='do-not-store' + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_http_auth_allowlist_and_revision_conflict(self): + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + path=f'/internal/v2/prognosis/{AID}/planner';headers={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(client.get(path).status_code,401);self.assertEqual(client.get(path,headers=headers).status_code,200) + other='30509683-4569-49e4-848f-4905e4cc813a';self.assertEqual(client.get(path.replace(AID,other),headers=headers).status_code,403) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'23'}}).status_code,200) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'3'}}).status_code,409) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + def test_body_limit_and_no_legacy_db(self): + with self.assertRaises(ValueError):create_app(str(Path(self.temp.name)/'users.db'),TOKEN,[AID]) + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + r=client.post(f'/internal/v2/prognosis/{AID}/planner/inputs/forecast',json={'x':'a'*2000001});self.assertEqual(r.status_code,413) + +if __name__=='__main__':unittest.main() diff --git a/tests/BatterieModulstrukturTest.php b/tests/BatterieModulstrukturTest.php index da8e329..fe9b0f6 100644 --- a/tests/BatterieModulstrukturTest.php +++ b/tests/BatterieModulstrukturTest.php @@ -10,6 +10,24 @@ final class BatterieModulstrukturTest extends TestCase { private const MODULPFAD = __DIR__ . '/../Batterie/'; + public function testV4StoppUndSollwertVerwerfenBleibenGemeinsamErhalten(): void + { + $modul = (string) file_get_contents(self::MODULPFAD . 'module.php'); + self::assertMatchesRegularExpression( + "/case 'SichererZustand':\\s*\\\$this->V4BatterieTestStoppen\\(\\);" + . "\\s*\\\$this->verwerfeSollwert\\(\\);\\s*\\\$this->schreibeRegister\\(0, true\\);/", + $modul + ); + self::assertSame(1, preg_match( + '/private function ManagerdatenEmpfangenIntern\\(array \\$daten\\): void(.*?)private function aktualisiere\\(/s', + $modul, + $empfang + )); + self::assertStringContainsString('$this->verwerfeSollwert();', $empfang[1]); + self::assertStringContainsString('$this->aktualisiereIntern(false);', $empfang[1]); + self::assertStringNotContainsString('$this->aktualisiere(', $empfang[1]); + } + public function testModulIstVollstaendigUndMetadatenSindGueltig(): void { foreach (['module.json', 'form.json', 'module.php', 'README.md'] as $datei) { @@ -127,4 +145,17 @@ final class BatterieModulstrukturTest extends TestCase substr_count($manager, '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') ); } + + public function testBatterieformularHatKeinenSeparatenV4Bereich(): void + { + $modul = (string) file_get_contents(self::MODULPFAD . 'module.php'); + $formular = (string) file_get_contents(self::MODULPFAD . 'form.json'); + self::assertStringNotContainsString('V4 physische Rueckmeldung', $modul); + self::assertStringNotContainsString('NetzfahrplanV4RueckmeldungKonfiguration', $formular); + self::assertStringNotContainsString("'name'=>'NetzfahrplanV4RueckmeldungKonfiguration'", $modul); + + $rueckmeldung = (string) file_get_contents(__DIR__ . '/../libs/BatterieNetzfahrplanV4RueckmeldungTrait.php'); + self::assertStringContainsString("RegisterPropertyString('NetzfahrplanV4RueckmeldungKonfiguration'", $rueckmeldung); + self::assertStringContainsString('$this->konfiguriereV4Rueckmeldung();', $modul); + } } diff --git a/tests/ManagerModulstrukturTest.php b/tests/ManagerModulstrukturTest.php index 5835a8f..45ef997 100644 --- a/tests/ManagerModulstrukturTest.php +++ b/tests/ManagerModulstrukturTest.php @@ -154,9 +154,14 @@ final class ManagerModulstrukturTest extends TestCase "WriteAttributeString('LizenzInstallationID'", $treffer[0] ); - self::assertStringContainsString( - "parent::ApplyChanges();\n\n \$this->initialisiereLizenzInstallationID();", - $inhalt + self::assertSame(1, preg_match( + '/public function ApplyChanges\(\): void(.*?)public function /s', + $inhalt, + $applyChanges + )); + self::assertMatchesRegularExpression( + '/parent::ApplyChanges\(\);.*?\$this->initialisiereLizenzInstallationID\(\);/s', + $applyChanges[1] ); self::assertSame(1, substr_count( $inhalt, diff --git a/tests/NetzfahrplanV4BetriebsdatenTest.php b/tests/NetzfahrplanV4BetriebsdatenTest.php new file mode 100644 index 0000000..adb80c8 --- /dev/null +++ b/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -0,0 +1,29 @@ +regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) { + self::assertStringNotContainsString($forbidden, $source); + } + self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source); + self::assertStringContainsString('/prognosis/planner-v4/operation', $source); + self::assertStringNotContainsString('/prognosis/schedule', $source); + } +} diff --git a/tests/NetzfahrplanV4BezugszaehlerTest.php b/tests/NetzfahrplanV4BezugszaehlerTest.php new file mode 100644 index 0000000..41a2992 --- /dev/null +++ b/tests/NetzfahrplanV4BezugszaehlerTest.php @@ -0,0 +1,29 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]);self::assertSame(0,proc_close($p),$out.$err);self::assertSame('',$err); + self::assertStringContainsString('TOTAL 42 checks passed',$out); + } +} diff --git a/tests/NetzfahrplanV4DatenTest.php b/tests/NetzfahrplanV4DatenTest.php new file mode 100644 index 0000000..3de873c --- /dev/null +++ b/tests/NetzfahrplanV4DatenTest.php @@ -0,0 +1,21 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($process);fclose($pipes[0]); + $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($process),$out.$err); + self::assertStringContainsString('TOTAL 27 application-data checks passed',$out); + } +} diff --git a/tests/NetzfahrplanV4EmpfangTest.php b/tests/NetzfahrplanV4EmpfangTest.php new file mode 100644 index 0000000..fd22d29 --- /dev/null +++ b/tests/NetzfahrplanV4EmpfangTest.php @@ -0,0 +1,29 @@ + ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + self::assertIsResource($process); + fclose($pipes[0]); + $out = stream_get_contents($pipes[1]);$err = stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0, proc_close($process), $out . $err); + self::assertStringContainsString('Total:', $out); + } + + public function scripts(): array + { + return [['protocol_checks.php'], ['receiver_checks.php']]; + } +} diff --git a/tests/NetzfahrplanV4MessaufnahmeTest.php b/tests/NetzfahrplanV4MessaufnahmeTest.php new file mode 100644 index 0000000..421b42c --- /dev/null +++ b/tests/NetzfahrplanV4MessaufnahmeTest.php @@ -0,0 +1,16 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]);fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err);self::assertStringContainsString('TOTAL',$out); + } + public function scripts(): array { return [['checks.php'],['native_checks.php']]; } +} diff --git a/tests/NetzfahrplanV4MesssichtTest.php b/tests/NetzfahrplanV4MesssichtTest.php new file mode 100644 index 0000000..cc21418 --- /dev/null +++ b/tests/NetzfahrplanV4MesssichtTest.php @@ -0,0 +1,16 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]);$out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]);fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err);self::assertStringContainsString('TOTAL ',$out);self::assertSame('',$err); + } + public function scripts(): array {return [['checks.php'],['native_checks.php']];} +} diff --git a/tests/NetzfahrplanV4RegeltestTest.php b/tests/NetzfahrplanV4RegeltestTest.php new file mode 100644 index 0000000..d5dfcf5 --- /dev/null +++ b/tests/NetzfahrplanV4RegeltestTest.php @@ -0,0 +1,24 @@ + ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + self::assertIsResource($process); + fclose($pipes[0]); + $out = stream_get_contents($pipes[1]);$err = stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0, proc_close($process), $out . $err); + self::assertSame('', $err); + self::assertStringContainsString('TOTAL 96 checks passed.', $out); + } +} diff --git a/tests/NetzfahrplanV4RueckmeldungTest.php b/tests/NetzfahrplanV4RueckmeldungTest.php new file mode 100644 index 0000000..9b1852c --- /dev/null +++ b/tests/NetzfahrplanV4RueckmeldungTest.php @@ -0,0 +1,26 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($p);fclose($pipes[0]); + $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($p),$out.$err); + self::assertSame('',$err);self::assertStringContainsString('TOTAL ',$out); + } + public function testPhysicalFeedback(): void {$this->checkScript('checks.php');} + public function testConfirmedDeviceReads(): void {$this->checkScript('device_read_checks.php');} + public function testActualReceiverIntegration(): void {$this->checkScript('receiver_checks.php');} + public function testActualFallbackWrapper(): void {$this->checkScript('fallback_checks.php');} + public function testMessageWithoutVisibleDiagnostics(): void {$this->checkScript('message_checks.php');} +} diff --git a/tests/PrognoseFormular/checks.php b/tests/PrognoseFormular/checks.php new file mode 100644 index 0000000..b07dee1 --- /dev/null +++ b/tests/PrognoseFormular/checks.php @@ -0,0 +1,116 @@ + true, 'lizenziert' => true, 'senderAktiv' => true, 'empfangAktiv' => true, + 'messdatenAktiv' => true, 'legacyAktiv' => false, 'managerAktiv' => true, 'testErlaubt' => true, + 'aktivtestErlaubt' => true, 'watchdogVerzichtErlaubt' => true, 'aktivtestAktiv' => false, + 'ende' => 1791387487, 'jetzt' => 1791300000, +]; +$form = ManagerPrognoseFormular::integrieren($base, $state); +$f = fields($form); +check(count($form['elements']) === count($base['elements']), 'no additional panel'); +check(!str_contains(json_encode($form), 'V4 Planempfang'), 'separate V4 tab removed'); +foreach (['PrognoseAktiv', 'NetzfahrplanAktiv', 'PrognoseSendeintervall', 'NetzfahrplanV4EmpfangAktiv', + 'PrognoseNetzfahrplanStart', 'PrognoseNetzfahrplanStop', 'PrognoseNetzfahrplanPruefen', + 'PrognoseLizenzVerwalten', 'PrognoseNetzfahrplanStatus'] as $name) { + check(isset($f[$name]), 'one forecast panel contains ' . $name); +} +foreach ($base['elements'] as $index => $element) { + if (($element['name'] ?? '') !== 'PrognoseForecast') { + check($form['elements'][$index] === $element, 'unrelated panel unchanged: ' . $element['caption']); + } +} +foreach (['PrognoseAktiv', 'PrognoseSendeintervall', 'PrognoseLizenzInformation'] as $name) { + check($f[$name] === fields($base)[$name], 'existing forecast field unchanged: ' . $name); +} +check($f['NetzfahrplanAktiv']['visible'] === false, 'inactive legacy control hidden for configured V4'); +check($f['PrognoseNetzfahrplanStart']['enabled'], 'licensed existing consent enables explicit start'); +check(str_contains($f['PrognoseNetzfahrplanStart']['onClick'], "'FormNetzfahrplanSchalten', true"), 'start uses guarded manager action'); +check(str_contains($f['PrognoseNetzfahrplanStop']['onClick'], "'FormNetzfahrplanSchalten', false"), 'stop uses existing stop path'); +check(str_contains($f['PrognoseNetzfahrplanPruefen']['onClick'], "'NetzfahrplanV4Empfangen'"), 'plan check uses existing throttled receiver'); +check(str_contains($f['PrognoseLizenzVerwalten']['onClick'], 'https://license.enelix.ch'), 'existing license portal retained'); +check(str_contains($f['PrognoseNetzfahrplanStatus']['caption'], '07.10.2026 15:38:07 UTC'), 'original expiry displayed without extension'); +check(str_contains($f['PrognoseNetzfahrplanStatus']['caption'], 'Kein freigegebener Dauerbetrieb'), 'test limitation remains explicit'); +foreach (['prognoseAktiv', 'lizenziert', 'senderAktiv', 'empfangAktiv', 'managerAktiv', + 'testErlaubt', 'aktivtestErlaubt', 'watchdogVerzichtErlaubt'] as $key) { + $blocked = array_replace($state, [$key => false]); + $b = fields(ManagerPrognoseFormular::integrieren($base, $blocked)); + check(!$b['PrognoseNetzfahrplanStart']['enabled'], 'start blocked without ' . $key); + check($b['PrognoseNetzfahrplanStop']['enabled'], 'stop remains possible without ' . $key); +} +foreach ([['legacyAktiv' => true], ['aktivtestAktiv' => true], ['ende' => 1791300000], ['ende' => -1]] as $change) { + check(!ManagerPrognoseFormular::startErlaubt(array_replace($state, $change)), 'conflict, replay or expiry blocks start: ' . json_encode($change)); +} +check(!ManagerPrognoseFormular::startErlaubt([]), 'missing state fails closed'); +check(ManagerPrognoseFormular::startErlaubt(array_replace($state, ['ende' => 0])), 'first start still uses existing backend consent procedure'); +$b = fields(ManagerPrognoseFormular::integrieren($base, array_replace($state, ['legacyAktiv' => true, 'lizenziert' => false]))); +check($b['NetzfahrplanAktiv']['visible'] && $b['NetzfahrplanAktiv']['enabled'], 'active legacy controller can be disabled despite lost license'); +check(str_contains($b['PrognoseNetzfahrplanStatus']['caption'], 'konkurrier') === false + && str_contains($b['PrognoseNetzfahrplanStatus']['caption'], 'gesperrt'), 'controller conflict is visible'); +$b = fields(ManagerPrognoseFormular::integrieren($base, array_replace($state, ['lizenziert' => false]))); +check($b['NetzfahrplanV4EmpfangAktiv']['enabled'], 'active receiver remains switchable off without license'); +$b = fields(ManagerPrognoseFormular::integrieren($base, array_replace($state, ['lizenziert' => false, 'empfangAktiv' => false]))); +check(!$b['NetzfahrplanV4EmpfangAktiv']['enabled'], 'unlicensed receiver cannot be enabled through form'); +$b = fields(ManagerPrognoseFormular::integrieren($base, array_replace($state, [ + 'senderAktiv' => false, 'empfangAktiv' => false, 'messdatenAktiv' => false, +]))); +check($b['NetzfahrplanAktiv'] === fields($base)['NetzfahrplanAktiv'], 'non-migrated installation retains legacy control'); +$missing = $base; +$missing['elements'] = array_values(array_filter($missing['elements'], fn($e) => ($e['name'] ?? '') !== 'PrognoseForecast')); +try { + ManagerPrognoseFormular::integrieren($missing, $state); + check(false, 'missing forecast anchor rejected'); +} catch (LogicException $e) { + check(true, 'missing forecast anchor rejected'); +} +$duplicate = $base; +foreach ($base['elements'] as $element) { + if (($element['name'] ?? '') === 'PrognoseForecast') { + $duplicate['elements'][] = $element; + } +} +try { + ManagerPrognoseFormular::integrieren($duplicate, $state); + check(false, 'duplicate forecast anchor rejected'); +} catch (LogicException $e) { + check(true, 'duplicate forecast anchor rejected'); +} +check($form === ManagerPrognoseFormular::integrieren($base, $state), 'repeated rendering is deterministic and does not modify the base form'); +echo "Total: $count checks. Form generation only; no IPS or device calls.\n"; diff --git a/tests/PrognoseFormular/manager_checks.php b/tests/PrognoseFormular/manager_checks.php new file mode 100644 index 0000000..9c408ea --- /dev/null +++ b/tests/PrognoseFormular/manager_checks.php @@ -0,0 +1,131 @@ +props[$key] ?? false); } + protected function ReadPropertyInteger(string $key): int { return (int) ($this->props[$key] ?? 0); } + protected function ReadPropertyString(string $key): string { return (string) ($this->props[$key] ?? ''); } + protected function ReadAttributeString(string $key): string { return (string) ($this->attrs[$key] ?? ''); } + protected function ReadAttributeInteger(string $key): int { return (int) ($this->attrs[$key] ?? 0); } + protected function GetValue(string $key) { return $this->values[$key] ?? false; } + protected function UpdateFormField(string $name, string $field, $value): void { $this->fields[$name][$field] = $value; } +} + +function IPS_GetInstance(int $id): array { return ['InstanceStatus' => $GLOBALS['instanceStatus']]; } + +require_once __DIR__ . '/../../Manager/module.php'; + +final class ManagerFormHarness extends Manager +{ + public array $calls = []; + public bool $reject = false; + public function V4ManagerAktivtestSchalten(bool $aktiv): void + { + $this->calls[] = $aktiv; + if ($this->reject) { + throw new RuntimeException('Existing backend refused start'); + } + $this->values['NetzfahrplanV4Aktivtest'] = $aktiv; + } +} + +$count = 0; +function check(bool $ok, string $name): void +{ + global $count; + if (!$ok) { throw new RuntimeException('FAIL ' . $name); } + $count++; + echo 'PASS ' . $name . PHP_EOL; +} + +function fixture(): ManagerFormHarness +{ + $h = new ManagerFormHarness(); + $h->props = array_fill_keys(['PrognoseAktiv', 'NetzfahrplanV4SchattenAktiv', + 'NetzfahrplanV4EmpfangAktiv', 'NetzfahrplanV4MessdatenAktiv', 'NetzfahrplanV4RegeltestErlaubt', + 'NetzfahrplanV4AktivtestErlaubt', 'NetzfahrplanV4WatchdogVerzichtErlaubt'], true); + $h->props['NetzfahrplanAktiv'] = false; + // Synthetic fixture, not a license or credential from an installation. + $h->props['Lizenzcode'] = 'ENX' . str_repeat('A', 16); + $h->attrs['LizenzcodeHash'] = hash('sha256', $h->props['Lizenzcode']); + $h->attrs['LizenzInstallationID'] = '10000000-0000-4000-8000-000000000001'; + $h->attrs['LizenzLease'] = json_encode([ + 'status' => 'active', 'development' => true, 'installationId' => $h->attrs['LizenzInstallationID'], + 'issuedAt' => gmdate('c', time() - 100), 'refreshAfter' => gmdate('c', time() + 100), + 'offlineUntil' => gmdate('c', time() + 3600), + 'entitlements' => [ + ['catalog_key' => 'manager_peak', 'quantity' => 1, 'status' => 'active'], + ['catalog_key' => 'grid_schedule', 'quantity' => 1, 'status' => 'active'], + ], + ], JSON_THROW_ON_ERROR); + $h->attrs['NetzfahrplanV4AktivtestEnde'] = time() + 600; + $h->values = ['Aktiv' => true, 'NetzfahrplanV4Aktivtest' => false]; + $GLOBALS['instanceStatus'] = 102; + return $h; +} + +function rejected(ManagerFormHarness $h, $value, string $name): void +{ + try { + $h->RequestAction('FormNetzfahrplanSchalten', $value); + } catch (RuntimeException | InvalidArgumentException $e) { + check($h->calls === [], $name); + return; + } + check(false, $name); +} + +$h = fixture(); +$before = [$h->props, $h->attrs]; +$h->RequestAction('FormNetzfahrplanSchalten', true); +check($h->calls === [true], 'real manager action delegates exactly one explicit start'); +check(!$h->fields['PrognoseNetzfahrplanStart']['enabled'], 'started session cannot be restarted from form'); +check([$h->props, $h->attrs] === $before, 'form action changes no configuration, expiry or license'); +$h->props['Lizenzcode'] = ''; +$h->RequestAction('FormNetzfahrplanSchalten', false); +check($h->calls === [true, false], 'stop still delegates after license loss'); +check(!$h->values['NetzfahrplanV4Aktivtest'], 'stop status is refreshed'); +foreach ([1, 'true', 'false', null, []] as $value) { + rejected(fixture(), $value, 'nonboolean input rejected: ' . gettype($value)); +} +foreach (['PrognoseAktiv', 'NetzfahrplanV4SchattenAktiv', 'NetzfahrplanV4EmpfangAktiv', + 'NetzfahrplanV4RegeltestErlaubt', 'NetzfahrplanV4AktivtestErlaubt', 'NetzfahrplanV4WatchdogVerzichtErlaubt'] as $prop) { + $h = fixture(); $h->props[$prop] = false; + rejected($h, true, 'server-side start guard checks ' . $prop); +} +$h = fixture(); $h->props['Lizenzcode'] = ''; +rejected($h, true, 'missing license cannot be bypassed by calling the action directly'); +$h = fixture(); $lease = json_decode($h->attrs['LizenzLease'], true); +$lease['entitlements'][1]['quantity'] = 0; $h->attrs['LizenzLease'] = json_encode($lease); +rejected($h, true, 'manager license alone does not grant grid schedule'); +$h = fixture(); $h->props['NetzfahrplanAktiv'] = true; +rejected($h, true, 'competing legacy controller blocks action'); +$h = fixture(); $h->attrs['NetzfahrplanV4AktivtestEnde'] = time() - 1; +rejected($h, true, 'expired consent cannot be renewed by the form'); +$h = fixture(); $GLOBALS['instanceStatus'] = 203; +rejected($h, true, 'inactive or faulty manager cannot start'); +$h = fixture(); $h->values['Aktiv'] = false; +rejected($h, true, 'disabled manager cannot start'); +$h = fixture(); $h->values['NetzfahrplanV4Aktivtest'] = true; +rejected($h, true, 'existing session cannot be restarted'); +$h = fixture(); $h->reject = true; +try { + $h->RequestAction('FormNetzfahrplanSchalten', true); + check(false, 'backend rejection preserved'); +} catch (RuntimeException $e) { + check($e->getMessage() === 'Existing backend refused start' && $h->calls === [true], 'backend rejection preserved without retry or bypass'); +} +$h = fixture(); $h->RequestAction('FormPrognoseAktiv', false); +check(!$h->fields['PrognoseNetzfahrplanStart']['enabled'] && $h->calls === [], 'unsaved forecast disable prevents start without a control call'); +$h = fixture(); $h->props['PrognoseAktiv'] = false; +$h->RequestAction('FormPrognoseAktiv', true); +check(!$h->fields['PrognoseNetzfahrplanStart']['enabled'] && $h->calls === [], 'unsaved forecast enable does not grant stored permission'); +echo "Total: $count checks. Real RequestAction and license checks; mocked IPS and control delegate.\n"; diff --git a/tests/V4Accounting/checks.php b/tests/V4Accounting/checks.php new file mode 100644 index 0000000..64084f7 --- /dev/null +++ b/tests/V4Accounting/checks.php @@ -0,0 +1,66 @@ +1000.,'pv'=>5000.,'physical_storage'=>3000.,'controlled_virtual'=>2000.,'external_virtual'=>1000.] as $role=>$value){ + $id=101+count($s);$s[]=['key'=>$role,'role'=>$role,'variableId'=>$id,'parentId'=>10,'ident'=>$role,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>[]]; + $m[$id]=['value'=>$value,'updated'=>$now,'parentID'=>10,'ident'=>$role]; + } + $s[3]['dependsOn']=['physical_storage'];$s[4]['dependsOn']=['physical_storage']; + return [['version'=>1,'sources'=>$s,'maxSkewSeconds'=>30,'splitToleranceW'=>100.],$m,$now]; +} +function take(array $c,array $m,int $now):array{return B::aufnehmen($c,static fn($id)=>$m[$id],$now);} +[$c,$m,$n]=fixture();$r=take($c,$m,$n); +check($r['baseLoadCandidateW']===3000.,'subtract physical battery once, not virtual twice'); +check($r['externalEffectW']===1000. && $r['allocationResidualW']===0.,'SDL kept outside household'); +check($r['reconstructedGridW']===1000. && $r['arithmeticBalanceErrorW']===0.,'signed PCC balance closes'); +check($r['controlEligible']===false && !$r['meteringBoundaryVerified'],'algebra never grants control'); +check(!$r['forecastHistoryMigrated'],'old forecasts never relabelled'); +$m[101]['value']=-5000.;$m[103]['value']=-3000.;$m[104]['value']=-2000.;$m[105]['value']=-1000.;$r=take($c,$m,$n); +check($r['baseLoadCandidateW']===3000. && $r['externalEffectW']===-1000.,'discharge and export signs correct'); +[$c,$m,$n]=fixture();$m[103]['value']=2000.;$m[104]['value']=42000.;$m[105]['value']=-40000.;$r=take($c,$m,$n); +check($r['allocationResidualW']===0. && $r['baseLoadCandidateW']===4000.,'opposed virtual accounts are not two physical loads'); +[$c,$m,$n]=fixture();$m[104]['value']=0.;$m[105]['value']=0.;$r=take($c,$m,$n); +check($r['externalEffectW']===3000. && $r['baseLoadCandidateW']===3000.,'held virtual values cannot contaminate base load'); +check(in_array('virtual_physical_allocation_mismatch',$r['issues'],true),'split discrepancy explicit'); +[$c,$m,$n]=fixture();$m[103]['updated']=$n-120;$r=take($c,$m,$n); +check(in_array('stale_origin:physical_storage',$r['issues'],true),'old physical reading rejected despite new acquisition'); +check(in_array('stale_origin:controlled_virtual',$r['issues'],true),'fresh virtual timestamp cannot mask old physical input'); +check($r['sources']['controlled_virtual']['oldestOriginAt']===$n-120,'original dependency timestamp preserved'); +check($r['status']==='quality_hold','bad quality remains a diagnostic, not a training sample'); +[$c,$m,$n]=fixture();$m[102]['updated']=$n-31;$r=take($c,$m,$n);check(in_array('source_time_skew',$r['issues'],true),'skew rejected separately from age'); +$m[102]['updated']=$n+1;$r=take($c,$m,$n);check(in_array('future_source:pv',$r['issues'],true),'future timestamp rejected'); +[$c,$m,$n]=fixture();foreach($m as &$v)$v['value']=0.;unset($v);$r=take($c,$m,$n);check($r['status']==='consistent_candidate' && $r['baseLoadCandidateW']===0.,'true all-zero observation accepted without fake history'); +[$c,$m,$n]=fixture();$m[101]['value']=-6000.;$r=take($c,$m,$n);check($r['baseLoadCandidateW']===-4000. && in_array('negative_base_load',$r['issues'],true),'negative residual not silently clipped to zero'); +[$c,$m,$n]=fixture();$m[102]['value']=-10.;check(in_array('negative_unsigned_source:pv',take($c,$m,$n)['issues'],true),'PV sign validity'); +[$c,$m,$n]=fixture();$m[103]['value']=-3000.;$c['sources'][2]['factorToW']=-1.;check(take($c,$m,$n)['baseLoadCandidateW']===3000.,'explicit vendor sign conversion'); +[$c,$m,$n]=fixture();$m[101]['value']=1.;$c['sources'][0]['factorToW']=1000.;check(take($c,$m,$n)['baseLoadCandidateW']===3000.,'kW to W factor applied once'); +[$c,$m,$n]=fixture();$c['sources'][]=['key'=>'boiler','role'=>'flexible_load','variableId'=>200,'parentId'=>10,'ident'=>'boiler','factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>[]];$m[200]=['value'=>500.,'updated'=>$n,'parentID'=>10,'ident'=>'boiler'];$r=take($c,$m,$n);check($r['baseLoadCandidateW']===2500. && $r['reconstructedGridW']===1000.,'future flexible load excluded exactly once'); +foreach([true,'1000',NAN,INF] as $bad){[$c,$m,$n]=fixture();$m[101]['value']=$bad;fails(fn()=>take($c,$m,$n),'invalid numeric input '.gettype($bad));} +[$c,$m,$n]=fixture();$m[101]['parentID']=99;fails(fn()=>take($c,$m,$n),'different source parent rejected'); +[$c,$m,$n]=fixture();$m[101]['ident']='other';fails(fn()=>take($c,$m,$n),'different source ident rejected'); +[$c,$m,$n]=fixture();$c['sources'][]=$c['sources'][0];fails(fn()=>take($c,$m,$n),'duplicate physical source rejected'); +[$c,$m,$n]=fixture();$c['sources'][3]['dependsOn']=['absent'];fails(fn()=>take($c,$m,$n),'missing origin rejected'); +[$c,$m,$n]=fixture();$c['sources'][2]['dependsOn']=['controlled_virtual'];fails(fn()=>take($c,$m,$n),'dependency loop rejected'); +[$c,$m,$n]=fixture();$a=take($c,$m,$n)['mappingId'];$c['sources']=array_reverse($c['sources']);check(take($c,$m,$n)['mappingId']===$a,'source order does not change mapping identity'); +$c['sources'][0]['factorToW']=2.;check(take($c,$m,$n)['mappingId']!==$a,'factor changes invalidate old mapping identity'); +[$c,$m,$n]=fixture();$calls=[]; +fails(function()use($c,$m,$n,&$calls){B::aufnehmen($c,function($id)use($m,&$calls){$calls[$id]=($calls[$id]??0)+1;$v=$m[$id];if($calls[$id]>1)$v['value']++;return $v;},$n);},'same-second concurrent value change rejected'); +$source=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Bilanzierung.php');check(!preg_match('/\b(?:SetValue|IPS_RequestAction|RequestAction|curl_exec|IPS_SetProperty|AC_SetLoggingStatus)\s*\(/',$source),'adapter has no runtime or external write calls'); +[$c,$m,$n]=fixture();$integer=$c;$integer['splitToleranceW']=100; +check(B::konfigurieren($c)===B::konfigurieren($integer),'integer and float tolerance normalize identically'); +check(take($c,$m,$n)['mappingId']===take($integer,$m,$n)['mappingId'],'equal tolerance gives identical mapping hash'); +$roundtrip=json_decode(json_encode($c,JSON_THROW_ON_ERROR),true,64,JSON_THROW_ON_ERROR); +check(take($roundtrip,$m,$n)['mappingId']===take($c,$m,$n)['mappingId'],'plain JSON property roundtrip preserves mapping'); +$normalized=B::konfigurieren($c);check(B::konfigurieren($normalized)===$normalized,'normalization remains idempotent'); +check(is_int($integer['splitToleranceW']) && is_float($c['splitToleranceW']),'input configurations are not mutated'); +$changed=$c;$changed['splitToleranceW']=101; +check(take($changed,$m,$n)['mappingId']!==take($c,$m,$n)['mappingId'],'real tolerance change still changes identity'); +foreach([true,'100'] as $invalid){$bad=$c;$bad['splitToleranceW']=$invalid;fails(fn()=>B::konfigurieren($bad),'non-numeric tolerance remains rejected '.gettype($invalid));} +echo "TOTAL $count checks passed; synthetic snapshots only.\n"; diff --git a/tests/V4ApplicationData/archive_checks.php b/tests/V4ApplicationData/archive_checks.php new file mode 100644 index 0000000..d5654ef --- /dev/null +++ b/tests/V4ApplicationData/archive_checks.php @@ -0,0 +1,27 @@ +isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($dir);} +$base=sys_get_temp_dir().'/v4-archive-'.bin2hex(random_bytes(6));mkdir($base,0700); +try { + $old='raw-'.gmdate('Ymd',$now-10*86400).'.jsonl';$recent='raw-'.gmdate('Ymd',$now-2*86400).'.jsonl';$day='raw-'.gmdate('Ymd',$now).'.jsonl'; + $raw="{\"sample\":1}\n{\"sample\":2}\n";file_put_contents($base.'/'.$old,$raw);file_put_contents($base.'/'.$recent,$raw);file_put_contents($base.'/'.$day,$raw); + checkArchive(A::run($base,[],$now)['archived']===0,'no acknowledgement means no archival'); + $r=A::run($base,['day'=>$old,'offset'=>5],$now);checkArchive($r['archived']===0&&is_file($base.'/'.$old),'partially acknowledged day retained'); + $r=A::run($base,['day'=>$day,'offset'=>5],$now);checkArchive($r['archived']===1&&!is_file($base.'/'.$old),'only completely acknowledged old day retired'); + $files=glob($base.'/archive/*.gz');checkArchive(count($files)===1&&gzdecode(file_get_contents($files[0]))===$raw,'compressed original verified byte for byte'); + checkArchive(is_file($base.'/'.$recent)&&is_file($base.'/'.$day),'recent and current outbox preserved'); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===0,'repeat is idempotent'); + file_put_contents($base.'/'.$old,$raw);file_put_contents($files[0],gzencode('corrupt')); + try {A::run($base,['day'=>$day,'offset'=>5],$now);throw new LogicException('expected refusal');} catch(RuntimeException $e) {checkArchive(is_file($base.'/'.$old),'archive corruption never loses original');} + unlink($files[0]);$lock=fopen($base.'/.writer.lock','c+b');flock($lock,LOCK_EX); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['status']==='writer_busy','archive serializes with actual capture writer');flock($lock,LOCK_UN);fclose($lock); + foreach(range(11,15)as$d)file_put_contents($base.'/raw-'.gmdate('Ymd',$now-$d*86400).'.jsonl',$raw); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===2,'bounded archive work per invocation'); + $text=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Datenarchiv.php');checkArchive(!preg_match('/\b(?:RequestAction|IPS_SetProperty|curl_exec)\s*\(/',$text),'archive has no control or network calls'); + echo "TOTAL $n archive checks passed; only temporary test data.\n"; +} finally {cleanupArchive($base);} diff --git a/tests/V4ApplicationData/checks.php b/tests/V4ApplicationData/checks.php new file mode 100644 index 0000000..318bbb3 --- /dev/null +++ b/tests/V4ApplicationData/checks.php @@ -0,0 +1,96 @@ +2,'VariableUpdated'=>$sensor[$id]['updated'],'VariableChanged'=>$sensor[$id]['changed']];} +function IPS_GetObject($id):array {return ['ParentID'=>200,'ObjectIdent'=>'s'.$id];} +function GetValue($id) {global $sensor;return $sensor[$id]['value'];} +foreach(['CURLOPT_POST','CURLOPT_RETURNTRANSFER','CURLOPT_FOLLOWLOCATION','CURLOPT_CONNECTTIMEOUT','CURLOPT_TIMEOUT','CURLOPT_SSL_VERIFYPEER','CURLOPT_SSL_VERIFYHOST','CURLOPT_HTTPHEADER','CURLOPT_POSTFIELDS','CURLINFO_HTTP_CODE'] as $i=>$k)if(!defined($k))define($k,$i+100); +function curl_init($url){$o=new \stdClass();$o->url=$url;return $o;} +function curl_setopt_array($h,$options){$h->options=$options;return true;} +function curl_exec($h){global $sent,$mode;$p=json_decode($h->options[CURLOPT_POSTFIELDS],true);$sent[]=$p; + return json_encode(['status'=>'stored','datasetId'=>$p['datasetId'],'acceptedThrough'=>$mode==='wrong_ack'?'2000-01-01T00:00:00+00:00':end($p['records'])['capturedAt']]);} +function curl_getinfo($h,$info){global $mode;return $mode==='429'?429:200;} +function curl_close($h):void {} +class Lizenzpruefung {const NETZFAHRPLAN='test-only';} +class FakeManager { + use ManagerNetzfahrplanV4DatenTrait; + public int $InstanceID=17004;public array $p=[],$a=[],$vars=[],$timers=[];public bool $licensed=true; + public function boot():void {$this->registriereNetzfahrplanV4Daten();} + public function configure():void {$this->konfiguriereNetzfahrplanV4Daten();} + public function folder():string {return $this->v4DatenVerzeichnis();} + public function send():array {return $this->v4DatenUebertragen($this->folder());} + public function RegisterPropertyBoolean($k,$v){$this->p[$k]??=$v;} + public function RegisterPropertyString($k,$v){$this->p[$k]??=$v;} + public function RegisterAttributeString($k,$v){$this->a[$k]??=$v;} + public function RegisterAttributeInteger($k,$v){$this->a[$k]??=$v;} + public function RegisterVariableString($k,...$args){$this->vars[$k]='';} + public function RegisterTimer($k,$v,$code){$this->timers[$k]=$v;} + public function SetTimerInterval($k,$v){$this->timers[$k]=$v;} + public function ReadPropertyBoolean($k){return $this->p[$k];} + public function ReadPropertyString($k){return $this->p[$k];} + public function ReadAttributeString($k){return $this->a[$k];} + public function ReadAttributeInteger($k){return $this->a[$k];} + public function WriteAttributeString($k,$v){$this->a[$k]=$v;} + public function WriteAttributeInteger($k,$v){$this->a[$k]=$v;} + public function SetValue($k,$v){if($k!=='NetzfahrplanV4Datenstatus')throw new \RuntimeException('Unexpected variable write');$this->vars[$k]=$v;} + private function berechtigungLizenziert($k){return $this->licensed;} +} +$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004, + 'reportedInventorySha256'=>str_repeat('a',64),'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]],'extraSources'=>[]]; +foreach(['grid','pv','physical_storage','controlled_virtual','external_virtual'] as $i=>$role){$id=101+$i; + $c['accounting']['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'s'.$id,'factorToW'=>1,'maxAgeSeconds'=>60,'dependsOn'=>[]]; + $sensor[$id]=['value'=>[3000,5000,2000,2000,0][$i],'updated'=>$clock,'changed'=>$clock];} +$m=new FakeManager();$m->boot();$m->a['LizenzInstallationID']=$c['installationId'];$m->a['PrognoseInstallationsToken']='synthetic-not-a-real-token'; +try { + $m->ErfasseNetzfahrplanV4Daten();check($sent===[],'disabled by default makes no network request'); + $m->p['NetzfahrplanV4MessdatenAktiv']=true;$m->p['NetzfahrplanV4Datensatz']='physical-v1';$m->p['NetzfahrplanV4Messkonfiguration']=json_encode($c); + $m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===30000,'only application acquisition timer enabled'); + $m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'native capture delivered to existing device API'); + $cursor=$m->a['NetzfahrplanV4DatenCursor'];check(json_decode($cursor,true)['offset']>0,'matching ack commits durable byte cursor'); + check($sent[0]['records'][0]['raw']['s101']['value']===3000,'physical reading retained'); + check($sent[0]['records'][0]['raw']['s101']['sourceUpdatedAt']===$clock,'original timestamp retained'); + check(!str_contains(json_encode($sent),'synthetic-not-a-real-token'),'token never included in data payload'); + check($sent[0]['records'][0]['controlEligible']===false,'data delivery is not actuator permission'); + $expected=hash('sha256',json_encode(NetzfahrplanV4Messaufnahme::configuration($c),JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + check($sent[0]['records'][0]['mappingSha256']===$expected,'property roundtrip preserves approved mapping identity'); + check(json_decode($m->a['NetzfahrplanV4LetzteDatenquittung'],true)['acceptedThrough']===end($sent[0]['records'])['capturedAt'],'successful acknowledgement persisted independently'); + $clock+=30;$m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'30-second collection does not double request rate'); + check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'unsent rows remain behind cursor'); + check(isset(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['lastAcknowledgement']['acceptedThrough']),'scheduled status retains last successful receipt'); + $clock+=30;$mode='429';$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'429 never advances acknowledgement cursor'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['dataRetained'],'offline data retained'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['httpStatus']===429,'safe HTTP error code visible without credentials'); + $mode='wrong_ack';$clock+=120;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'wrong ack timestamp rejected'); + $mode='ok';$clock+=240;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']!==$cursor,'later success sends the preserved backlog'); + check(count(end($sent)['records'])===4,'all previously unacknowledged rows included once'); + $file=glob($m->folder().'/raw-*.jsonl')[0];$count=count(file($file));check($count===5,'raw journal never cleared by upload'); + $lastCursor=$m->a['NetzfahrplanV4DatenCursor'];file_put_contents($file,'incomplete',FILE_APPEND); + check($m->send()['status']==='up_to_date','in-progress tail not transmitted');check($m->a['NetzfahrplanV4DatenCursor']===$lastCursor,'partial tail not acknowledged'); + // Only temporary test journal: turn the prior incomplete tail into an additional complete queued record. + $buf=file_get_contents($file);$buf=substr($buf,0,-strlen('incomplete')); + $extra=$sent[0]['records'][0];$clock+=60;$extra['capturedAt']=gmdate('c',$clock);$extra['captureStartedAt']=$extra['capturedAt']; + file_put_contents($file,$buf.json_encode($extra)."\n".'crash-tail'); + $clock+=60;$beforeCount=count($sent);$m->ErfasseNetzfahrplanV4Daten(); + $err=json_decode($m->vars['NetzfahrplanV4Datenstatus'],true); + check($err['status']==='capture_error'&&$err['delivery']['status']==='acknowledged','capture failure does not deadlock pending delivery'); + check(count($sent)===$beforeCount+1&&str_ends_with(file_get_contents($file),'crash-tail'),'complete prefix sent while original incomplete tail retained'); + $m->p['NetzfahrplanV4MessdatenAktiv']=false;$m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===0,'disable affects acquisition only'); + check(count($m->vars)===1,'only own manager data-status variable written'); + $text=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4DatenTrait.php'); + check(!preg_match('/\b(?:RequestAction\(|IPS_ApplyChanges\(|MC_Reload\(|schreibeRegister\()/',$text),'data path contains no actuator/module reload calls'); + echo "TOTAL $checks application-data checks passed; mocked IPS and HTTP only.\n"; +} finally { + foreach(new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base,\FilesystemIterator::SKIP_DOTS),\RecursiveIteratorIterator::CHILD_FIRST)as$p){$p->isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base); +} diff --git a/tests/V4ControlTrial/additional_checks.php b/tests/V4ControlTrial/additional_checks.php new file mode 100644 index 0000000..3669c8b --- /dev/null +++ b/tests/V4ControlTrial/additional_checks.php @@ -0,0 +1,30 @@ +op['gridW']=3000.;tick($m); +check(end($b->writes)===-3000,'discharge sign and current net load compensation'); +$b->measurements['Ladezustand']=15.;$b->watchdog();check(end($b->writes)===0,'reserve stops an active discharge independently'); +[$m,$b,$s]=fixture();start($m,$s);$m->op['gridW']=3000.;tick($m);$b->state['HystereseAktiv']=true;$b->watchdog(); +check(end($b->writes)===0,'local discharge hysteresis overrides previous command'); +[$m,$b,$s]=fixture();start($m,$s);$m->op['gridW']=3000.;tick($m);$b->measurements['MaxEntladen']=500.;$b->watchdog(); +check(end($b->writes)===-500,'discharge availability decrease is immediate'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=2.;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'technical minimum violation aborts rather than inventing SOC'); +[$m,$b,$s]=fixture();$s['maxChargeW']=1000.;start($m,$s);tick($m); +check(end($b->writes)===1000,'independent local cap can be stricter than server cap'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=500;$b->attrs['AenderungFreigabeAb']=$clock+4;tick($m); +check(end($b->writes)===500,'switch lock holds power increase'); +advance(5);tick($m);check(end($b->writes)===3000,'power increase only after switching lock'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=500;$b->attrs['AenderungFreigabeAb']=$clock+4;$m->op['gridW']=3000.;tick($m); +check(end($b->writes)===0,'sign reversal first passes zero during switching lock'); +[$m,$b,$s]=fixture();start($m,$s);$b->attrs['LetzterSollwert']=3000;$b->attrs['AenderungFreigabeAb']=$clock+4;$b->measurements['MaxLaden']=100.;tick($m); +check(end($b->writes)===100,'safety reduction not delayed by switching lock'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=15.;$b->props['NachladenMitMaximalleistung']=false;$b->props['MaximaleNachladeleistung']=200;$b->watchdog(); +check(end($b->writes)===200,'configured reserve-recovery charging limit remains enforced'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$m->licensed=false;tick($m); +check(!$m->active()&&!$b->active()&&end($b->writes)===0,'license withdrawal revokes both software sessions'); +[$m,$b,$s]=fixture();$s['durationSeconds']=30;start($m,$s);tick($m);advance(30);tick($m); +check(!$m->active()&&!$b->active()&&end($b->writes)===0,'local shorter test expiration respected'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$m->vars['Aktiv']=false;tick($m); +check(!$m->active()&&!$b->active(),'manual manager disable ends trial'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$b->vars['Aktiv']=false;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'manual battery disable independently ends trial'); diff --git a/tests/V4ControlTrial/cache_checks.php b/tests/V4ControlTrial/cache_checks.php new file mode 100644 index 0000000..465500e --- /dev/null +++ b/tests/V4ControlTrial/cache_checks.php @@ -0,0 +1,20 @@ +cachedFeedback();$reads=$GLOBALS['deviceReads']; +$GLOBALS['locks']['ENELIX.V4.Feedback.44234']=true; +$b->watchdogOnly(); +check($b->active() && $GLOBALS['deviceReads']===$reads,'busy sampler cannot block watchdog or trigger device I/O'); +check($b->cachedFeedback()===$before,'cache preserves original confirmed timestamps and values'); +$b->V4BatterieTestStoppen(); +check(!$b->active() && end($b->writes)===0,'stop reaches zero while the independent sampler is busy'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);advance(3);$b->watchdogOnly(); +check(!$b->active() && end($b->writes)===0,'stale sample revokes the command after two seconds'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$GLOBALS['mono']+=2.01;$b->watchdogOnly(); +check(!$b->active() && end($b->writes)===0,'monotonic sample age blocks frozen wall clock'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$GLOBALS['clock']--;$b->watchdogOnly(); +check(!$b->active() && end($b->writes)===0,'future cache timestamp is never made current'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$deviceReadOk=false;$b->sample(); +rejects(fn()=>$b->cachedFeedback(),'failed acquisition invalidates a still-young cached sample'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$reads=$GLOBALS['deviceReads']; +$c=$m->command();$c['sequence']++;$b->V4BatterieTestBefehl(json_encode($c)); +check($b->active() && $GLOBALS['deviceReads']===$reads,'command independently validates cache and sends without transport I/O'); diff --git a/tests/V4ControlTrial/checks.php b/tests/V4ControlTrial/checks.php new file mode 100644 index 0000000..1a9e8e7 --- /dev/null +++ b/tests/V4ControlTrial/checks.php @@ -0,0 +1,257 @@ +102,'ModuleInfo'=>['ModuleID'=>'{90BF9234-B44C-7EB3-280B-FBD6075ADDAA}']]; + return ['InstanceStatus'=>$GLOBALS['objects'][$id]->ready?102:201, + 'ModuleInfo'=>['ModuleID'=>$id===44234?'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}':'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']]; + } + function ModBus_RequestRead($id) { + if (!empty($GLOBALS['locks']['ENELIX.V4.Battery.44234'])) throw new \LogicException('Device I/O under battery control lock'); + $GLOBALS['deviceReads'] = ($GLOBALS['deviceReads'] ?? 0) + 1; + $GLOBALS['clock'] += $GLOBALS['readDelay'] ?? 0; + return $id===200 && ($GLOBALS['deviceReadOk']??true); + } + function IPS_GetProperty($id,$key) { return $GLOBALS['objects'][$id]->props[$key]; } + function IPS_VariableExists($id) { return in_array($id,[50001,50002],true); } + function IPS_GetVariable($id) { $t=$GLOBALS['feedbackAt'][$id]??$GLOBALS['clock'];return ['VariableUpdated'=>$t,'VariableChanged'=>$t]; } + function IPS_GetObject($id) { return ['ParentID'=>200,'ObjectIdent'=>'source'.$id]; } + function GetValue($id) { return $id===50001?$GLOBALS['objects'][17004]->op['gridW']:$GLOBALS['objects'][17004]->actual['b']; } + + function IPS_RequestAction($id,$key,$value) { + $GLOBALS['calls'][]=[$id,$key];$b=$GLOBALS['objects'][$id]; + if (($GLOBALS['actionRefusal']??null)===$key) return false; + if ($key==='NetzfahrplanV4TestStart') $GLOBALS['readDelay']=$GLOBALS['startReadDelay']??0; + if ($key==='NetzfahrplanV4TestStart') $b->V4BatterieTestStarten($value); + elseif ($key==='NetzfahrplanV4TestCommand') $b->V4BatterieTestBefehl($value); + elseif ($key==='NetzfahrplanV4TestStop') $b->V4BatterieTestStoppen(); + else throw new \RuntimeException('Unexpected external action in offline test'); + return true; + } + trait StoreHarness { + public array $props=[],$attrs=[],$buf=[],$timers=[],$vars=['Aktiv'=>true]; + public bool $ready=true; + private function RegisterPropertyBoolean($k,$v) { $this->props[$k]??=$v; } + private function RegisterPropertyInteger($k,$v) { $this->props[$k]??=$v; } + private function RegisterPropertyString($k,$v) { $this->props[$k]??=$v; } + private function RegisterAttributeString($k,$v) { $this->attrs[$k]??=$v; } + private function RegisterAttributeInteger($k,$v) { $this->attrs[$k]??=$v; } + private function RegisterVariableBoolean($k,$caption,$profile,$position) { $this->vars[$k]??=false; } + private function EnableAction($k) {} + private function RegisterTimer($k,$ms,$script) { $this->timers[$k]=$ms; } + private function SetTimerInterval($k,$ms) { $this->timers[$k]=$ms; } + private function ReadPropertyBoolean($k) { return $this->props[$k]; } + private function ReadPropertyString($k) { return $this->props[$k]; } + private function ReadPropertyInteger($k) { return $this->props[$k]; } + private function ReadPropertyFloat($k) { return (float)$this->props[$k]; } + private function ReadAttributeString($k) { return $this->attrs[$k]??''; } + private function ReadAttributeInteger($k) { return $this->attrs[$k]??0; } + private function WriteAttributeString($k,$v) { $this->attrs[$k]=$v; } + private function WriteAttributeInteger($k,$v) { $this->attrs[$k]=$v; } + private function GetBuffer($k) { return $this->buf[$k]??''; } + private function SetBuffer($k,$v) { $this->buf[$k]=$v; } + private function GetValue($k) { return $this->vars[$k]; } + private function SetValue($k,$v) { $this->vars[$k]=$v; } + } + class BatteryHarness { + use StoreHarness, BatterieNetzfahrplanV4TestTrait; + public int $InstanceID=44234; + public array $state=['HystereseAktiv'=>false],$writes=[],$measurements=[]; + public bool $failRegisters=false; + public function __construct() { + $this->props=['Batteriemanagement'=>2,'ReserveLadezustand'=>15.,'MindestLadezustand'=>3., + 'NachladenMitMaximalleistung'=>true,'MaximaleNachladeleistung'=>1000, + 'LadezustandVariableID'=>1001,'IstleistungVariableID'=>1002]; + $this->measurements=['Gueltig'=>true,'Ladezustand'=>50.,'MaxLaden'=>39000.,'MaxEntladen'=>39000.]; + $this->attrs['LetzterSollwert']=0;$this->attrs['AenderungFreigabeAb']=0; + $this->registriereV4BatterieTest(); + } + private function zugeordneteManagerIDs() { return [17004]; } + private function setzeZustand($k,$v) { $this->state[$k]=$v; } + private function leseZustand($k) { return $this->state[$k]??false; } + private function schreibeRegister($w,$force) { if ($this->failRegisters) throw new \RuntimeException('synthetic register failure');$this->writes[]=$w;$this->attrs['LetzterSollwert']=$w; } + private function aktualisiereIntern($notify) { $w=$this->v4BatterieTestZiel($this->measurements,$this->vars['Aktiv'],time());if ($w!==null) $this->schreibeRegister($w,false); } + public function sample() { try { $this->aktualisiereV4Rueckmeldung(); } catch (\Throwable) {} } + public function watchdog() { $this->sample(); $this->aktualisiereIntern(true); } + public function watchdogOnly() { $this->aktualisiereIntern(true); } + public function cachedFeedback() { return $this->v4BatterieRueckmeldung(); } + public function active() { return $this->v4BatterieTestAktiv(); } + } + class ManagerHarness { + use StoreHarness, ManagerNetzfahrplanV4TestTrait, ManagerNetzfahrplanV4AktivtestTrait; + public int $InstanceID=17004; + public array $op=[],$actual=['b'=>0.];public int $fallback=0; + public bool $licensed=true; + public function __construct($response,$op) { + $this->props=['NetzfahrplanV4SchattenAktiv'=>true,'NetzfahrplanV4EmpfangAktiv'=>true,'NetzfahrplanAktiv'=>false, + 'AnlagenBatterien'=>json_encode([['ID'=>'b','SOCVariableID'=>1001,'LeistungVariableID'=>1002]])]; + $this->op=$op;$this->attrs=['LizenzInstallationID'=>$response['installationId'], + 'NetzfahrplanV4Empfang'=>json_encode(['response'=>$response,'receivedAt'=>time()])]; + $this->registriereV4ManagerTest(); + $this->registriereV4ManagerAktivtest(); + } + private function berechtigungLizenziert($k) { return $this->licensed; } + private function actieveDummy() {} + private function aktiveVerbraucherIDs() { return [44234]; } + private function netzfahrplanV4LokaleVorschauwerte() { + $GLOBALS['objects'][44234]->sample(); // independent acquisition timer, outside control locks + $op=$this->op;$f=json_decode($GLOBALS['objects'][44234]->GetV4BatterieRueckmeldung(),true); + $op['localFeedback']=['b'=>$f];return [$op,$this->actual]; + } + private function regeln($renew) { $this->fallback++; } + public function tick(&$allocation,$consumers=null,$sync=true) { + $this->v4ManagerTestUebergabe($allocation,$consumers??[['InstanzID'=>44234,'Daten'=>[]]],$sync); + } + public function active() { return (bool)$this->v4ManagerTestSession(); } + public function command() { return json_decode($this->attrs['NetzfahrplanV4RegeltestStatus'],true)['command']??null; } + } +} +namespace { + require_once __DIR__.'/../V4Receiver/fixture.php'; + use Belevo\EnelixEMS\NetzfahrplanV4Regeltest as Gate; + use Belevo\EnelixEMS\BatteryHarness; + use Belevo\EnelixEMS\ManagerHarness; + $count=0; + function check($ok,$name) { global $count;if (!$ok) throw new RuntimeException('FAIL '.$name);$count++;echo "PASS $name\n"; } + function rejects(callable $call,$name) { try {$call();}catch(Throwable $e){check(true,$name);return;}throw new RuntimeException('FAIL did not reject '.$name); } + function fixture($permissions=true,$serverAuthority=true): array { + $GLOBALS['actionRefusal']=null;$GLOBALS['readDelay']=0;$GLOBALS['startReadDelay']=0; + [$r,$id,$op,$now]=receiverFixture();$GLOBALS['clock']=$now;$GLOBALS['mono']=1000.;$GLOBALS['locks']=[];$GLOBALS['calls']=[];$GLOBALS['deviceReadOk']=true; + $r['plan']['inputQuality']=['loadBasis'=>'base_load','accountingEvidenceId'=>'synthetic-meter-boundary-v1']; + $r['plan']['peakCostIsEstimate']=true; + $a=['kind'=>'controlled_trial_authority','version'=>1,'sourcePlanRemainsShadow'=>true, + 'sessionId'=>'20000000-0000-4000-8000-000000000001','installationId'=>$id, + 'managerId'=>17004,'batteryInstanceId'=>44234,'assetId'=>'b', + 'sourceShadowPlanId'=>$r['plan']['planId'],'revision'=>1,'family'=>'3', + 'issuedAt'=>gmdate('c',$now),'expiresAt'=>gmdate('c',$now+300),'checkedAt'=>gmdate('c',$now),'validUntil'=>gmdate('c',$now+90), + 'maxChargeW'=>39000.,'maxDischargeW'=>39000.,'actuatorWatchdogEvidenceId'=>'synthetic-hardware-watchdog', + 'accountingEvidenceId'=>'synthetic-meter-boundary-v1','acceptEstimatedPeak'=>true,'controlContext'=>$r['plan']['controlContext']]; + $r['controlledTrial']=$serverAuthority?$a:null; + $m=new ManagerHarness($r,$op);$b=new BatteryHarness();$GLOBALS['objects']=[17004=>$m,44234=>$b];$GLOBALS['feedbackAt']=[]; + $fc=['version'=>1,'installationId'=>$id,'assetId'=>'b','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>'physical_sum','allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60, + 'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0,'sources'=>[]]; + foreach ([50001=>'grid',50002=>'physical'] as $vid=>$role) $fc['sources'][]=['key'=>'source'.$vid,'role'=>$role,'variableId'=>$vid,'parentId'=>200,'ident'=>'source'.$vid,'factorToW'=>1.0]; + $b->props['NetzfahrplanV4RueckmeldungKonfiguration']=json_encode($fc); + $op['localFeedback']=['b'=>json_decode($b->GetV4BatterieRueckmeldung(),true)]; + + $m->props['NetzfahrplanV4RegeltestErlaubt']=$permissions;$b->props['NetzfahrplanV4RegeltestErlaubt']=$permissions; + $b->props['NetzfahrplanV4GeraeteWatchdogNachweis']='synthetic-hardware-watchdog'; + $start=['confirmation'=>'START_BOUNDED_V4_CONTROL_TRIAL','sessionId'=>$a['sessionId'],'durationSeconds'=>300,'maxChargeW'=>39000.,'maxDischargeW'=>39000.]; + return [$m,$b,$start,$r,$op,$now]; + } + function start($m,$s) {$m->V4ManagerTestStarten(json_encode($s));} + function tick($m) {$a=['Sollwerte'=>[44234=>999]];$m->tick($a);return $a;} + function advance($s) {$GLOBALS['clock']+=$s;$GLOBALS['mono']+=$s;} + function activeFixture(): array { + [$m,$b,$s,$r,$op,$now]=fixture(); + $m->props['NetzfahrplanV4AktivtestErlaubt']=true; + $m->props['NetzfahrplanV4WatchdogVerzichtErlaubt']=true; + $b->props['NetzfahrplanV4AktivtestErlaubt']=true; + $b->props['NetzfahrplanV4WatchdogVerzichtErlaubt']=true; + return [$m,$b,$s,$r,$op,$now]; + } + + [$m,$b,$s]=fixture(false);rejects(fn()=>start($m,$s),'default local permission blocks');check($b->writes===[],'no register writes without consent'); + [$m,$b,$s]=fixture(true,false);rejects(fn()=>start($m,$s),'ordinary shadow response never authorizes');check($calls===[],'no driver call for preview alone'); + [$m,$b,$s]=fixture();$b->props['NetzfahrplanV4RegeltestErlaubt']=false;rejects(fn()=>start($m,$s),'second battery gate independent'); + [$m,$b,$s]=fixture();$m->props['NetzfahrplanAktiv']=true;rejects(fn()=>start($m,$s),'old schedule excludes new controller'); + [$m,$b,$s,$r]=fixture();$r['plan']['inputQuality']['loadBasis']='house_total';$m->attrs['NetzfahrplanV4Empfang']=json_encode(['response'=>$r,'receivedAt'=>$clock]);rejects(fn()=>start($m,$s),'unverified aggregate accounting blocks test'); + [$m,$b,$s]=fixture();$b->props['NetzfahrplanV4GeraeteWatchdogNachweis']='';rejects(fn()=>start($m,$s),'unknown device watchdog blocks test'); + [$m,$b,$s]=fixture();$m->ready=false;rejects(fn()=>start($m,$s),'unready manager blocks test'); + [$m,$b,$s,$r]=fixture(); + $r['plan']['points'][0]['gridTargetW']=25000.;$r['plan']['points'][0]['batteryTargetW']=28000.; + $r['plan']['points'][0]['assetTargetsW']['b']=28000.;$r['plan']['points'][0]['intent']='gridCharge'; + $m->attrs['NetzfahrplanV4Empfang']=json_encode(['response'=>$r,'receivedAt'=>$clock]); + start($m,$s);tick($m);check(end($b->writes)===28000&&$m->command()['maxChargeW']===39000,'full dynamic plant power allowed above former 5kW test cap'); + [$m,$b,$s]=fixture();start($m,$s);check($m->active()&&$b->active(),'both sessions armed');check($b->writes===[],'arm itself sends no register command'); + $allocation=tick($m);check(!isset($allocation['Sollwerte'][44234]),'one actuator owner removes ordinary target'); + check(end($b->writes)===3000,'actual trait chain delivers bounded 3kW command'); + $c=$m->command();check($c['kind']==='controlled_trial_command'&&$c['expiresAt']===$clock+10,'independent command type and ten-second lease'); + check(json_decode($b->GetV4BatterieTestStatus(),true)['physicalResponseVerified']===false,'controller acceptance is not physical proof'); + advance(2);tick($m);check($m->command()['sequence']===2,'monotonic command sequence'); + rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'replayed command refused');check(!$b->active()&&end($b->writes)===0,'replay aborts and releases output'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);advance(11);$b->watchdog();check(!$b->active()&&end($b->writes)===0,'watchdog expiry drops to zero'); + [$m,$b,$s]=fixture();start($m,$s);advance(11);$b->watchdog();check(!$b->active()&&end($b->writes)===0,'missing first command expires'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$mono+=11;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'monotonic watchdog works without wall clock advance'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$clock-=1;$b->watchdog();check(!$b->active(),'backwards wall clock aborts'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->V4ManagerTestStoppen();check(!$m->active()&&!$b->active()&&end($b->writes)===0&&$m->fallback===1,'manual stop releases before ordinary fallback'); + rejects(fn()=>start($m,$s),'stopped local session cannot resurrect'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->buf=[];$b->V4BatterieTestStoppen();check(end($b->writes)===0&&!$b->active(),'runtime restart clears persisted command marker'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Gueltig']=false;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'invalid device measurements release command'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['MaxLaden']=1000.;$b->watchdog();check(end($b->writes)===1000,'device availability reduction rechecked'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->measurements['Ladezustand']=99.;$b->watchdog();check(end($b->writes)===0,'full battery cannot keep charging'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->props['NetzfahrplanV4RegeltestErlaubt']=false;$b->watchdog();check(!$b->active()&&end($b->writes)===0,'battery consent withdrawn aborts'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->props['NetzfahrplanV4RegeltestErlaubt']=false;$a=tick($m);check(!$m->active()&&isset($a['Sollwerte'][44234])&&end($b->writes)===0,'manager consent withdrawn resumes ordinary allocation'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$m->attrs['NetzfahrplanV4Empfang']='{}';$a=tick($m);check(!$m->active()&&end($b->writes)===0,'invalidated receiver cache ends trial'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$a=['Sollwerte'=>[44234=>123]];$m->tick($a,null,false);check(!$m->active()&&end($b->writes)===0,'unsynchronized consumers abort'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$b->failRegisters=true;advance(2);tick($m);check(!$m->active()&&!$b->active(),'register failure revokes both sessions');check(json_decode($m->GetV4ManagerTestStatus(),true)['controllerStopAccepted']===false,'stop failure not disguised as safe'); + [$m,$b,$s]=fixture();start($m,$s);$a=['Sollwerte'=>[44234=>999,33333=>1000]];$m->tick($a,[['InstanzID'=>44234,'Daten'=>[]],['InstanzID'=>33333,'Daten'=>['Leistungsquelle'=>'measured','Istleistung_W'=>0]]]); + check(end($b->writes)===2000&&$a['Sollwerte'][33333]===1000,'other consumer delta counted once without takeover'); + [$m,$b,$s]=fixture();start($m,$s);$m->actual['b']=2000.;$m->op['gridW']=-1000.;tick($m);check(end($b->writes)===3000,'command is total battery power not additive'); + [$m,$b,$s]=fixture();start($m,$s);$m->actual['b']=0.;$m->op['gridW']=-40000.;tick($m);check($m->active()&&end($b->writes)===39000,'full dynamic power corrects export while hard grid limit stays enforced'); + [$m,$b,$s]=fixture();start($m,$s);tick($m);$clock+=91;$mono+=91;tick($m);check(!$m->active()&&end($b->writes)===0,'expired server observation cancels even with future prices'); + [$m,$b,$s,$r,$op,$now]=fixture();$local=['enabled'=>true,'legacyEnabled'=>false,'installationId'=>$r['installationId'],'sessionId'=>$s['sessionId'],'managerId'=>17004,'batteryInstanceId'=>44234,'assetId'=>'b','expiresAt'=>$now+300,'maxChargeW'=>5000,'maxDischargeW'=>5000,'sequence'=>1,'actuatorWatchdogEvidenceId'=>'synthetic-hardware-watchdog']; + foreach (['sessionId','installationId','sourceShadowPlanId'] as $key) { + $bad=$r;$bad['controlledTrial'][$key]='ffffffff-ffff-4fff-8fff-ffffffffffff';rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'wrong authority binding '.$key); + } + $bad=$r;$bad['controlledTrial']['validUntil']=gmdate('c',$now+120);rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'oversized server authorization lease'); + $bad=$r;$bad['controlledTrial']['acceptEstimatedPeak']=false;rejects(fn()=>Gate::befehl($bad,$op,['b'=>0.],$local,$now,$now),'peak uncertainty needs explicit acceptance'); + [$m,$b]=activeFixture();$m->V4ManagerAktivtestSchalten(true); + check($m->active()&&$b->active()&&$m->vars['NetzfahrplanV4Aktivtest'],'single runtime switch starts bounded active test'); + $allocation=tick($m); + check(end($b->writes)===3000&&!isset($allocation['Sollwerte'][44234]),'active test uses guarded command path'); + check($m->command()['executionMeaning']==='explicit_time_boxed_test_plant_operation','active command is explicitly distinguished'); + [$m2,$b2]=activeFixture();$m2->V4ManagerAktivtestSchalten(true); + $mixed=['Sollwerte'=>[44234=>999,33333=>1000]]; + $m2->tick($mixed,[['InstanzID'=>44234,'Daten'=>[]],['InstanzID'=>33333,'Daten'=>['Leistungsquelle'=>'calculated','Istleistung_W'=>500]]],false); + check($m2->active()&&$b2->active()&&!isset($mixed['Sollwerte'][44234])&&!isset($mixed['Sollwerte'][33333]),'active test freezes other consumers and tolerates unrelated desync'); + $GLOBALS['objects']=[17004=>$m,44234=>$b]; + check($m->attrs['NetzfahrplanV4AktivtestEnde']-$m->attrs['NetzfahrplanV4AktivtestBeginn']===172800,'active test has fixed 48-hour window'); + advance(91); + $GLOBALS['feedbackAt']=[50001=>$clock,50002=>$clock]; + $m->op['observedAt']=gmdate('c',$clock); + $m->op['batteries'][0]['measuredAt']=gmdate('c',$clock); + $envelope=json_decode($m->attrs['NetzfahrplanV4Empfang'],true); + $envelope['receivedAt']=$clock; + $envelope['response']['checkedAt']=gmdate('c',$clock); + $m->attrs['NetzfahrplanV4Empfang']=json_encode($envelope); + tick($m); + check($m->active()&&$b->active(),'local active authority refreshes while plan and measurements remain valid'); + $m->V4ManagerAktivtestSchalten(false); + check(!$m->active()&&!$b->active()&&!$m->vars['NetzfahrplanV4Aktivtest']&&end($b->writes)===0,'switch off revokes session and requests zero'); + [$m,$b]=activeFixture();$m->props['NetzfahrplanV4WatchdogVerzichtErlaubt']=false; + rejects(fn()=>$m->V4ManagerAktivtestSchalten(true),'active test requires explicit watchdog waiver property'); + [$m,$b]=activeFixture();$b->props['NetzfahrplanV4WatchdogVerzichtErlaubt']=false; + rejects(fn()=>$m->V4ManagerAktivtestSchalten(true),'battery independently requires explicit watchdog waiver property'); + [$m,$b,$s]=fixture();$GLOBALS['actionRefusal']='NetzfahrplanV4TestStart'; + rejects(fn()=>start($m,$s),'FALSE start result is rejected'); + check(!$m->active()&&!$b->active(),'refused start cannot arm manager'); + [$m,$b]=activeFixture();$GLOBALS['actionRefusal']='NetzfahrplanV4TestStart'; + rejects(fn()=>$m->V4ManagerAktivtestSchalten(true),'FALSE active start result is rejected'); + check(!$m->vars['NetzfahrplanV4Aktivtest']&&!$m->active(),'refused active start leaves switch off'); + [$m,$b,$s]=fixture();start($m,$s);$GLOBALS['actionRefusal']='NetzfahrplanV4TestCommand';tick($m); + check(!$m->active()&&!$b->active()&&$m->command()===null,'FALSE command result aborts instead of reporting sent'); + [$m,$b,$s]=fixture();start($m,$s);$GLOBALS['actionRefusal']='NetzfahrplanV4TestStop';$m->V4ManagerTestStoppen(); + check(json_decode($m->GetV4ManagerTestStatus(),true)['controllerStopAccepted']===false,'FALSE stop result stays visibly unconfirmed'); + [$m,$b,$s]=fixture();$GLOBALS['startReadDelay']=1;start($m,$s); + check($m->active()&&$b->active(),'battery arming uses an independently confirmed snapshot without locked I/O'); + [$m,$b,$s]=fixture();start($m,$s);$b->measurements['Gueltig']=false;tick($m); + check(str_contains(json_decode($b->GetV4BatterieTestStatus(),true)['reason'],'Messwerte ungueltig'), + 'command rejection preserves underlying safety reason after zero request'); + require __DIR__ . '/additional_checks.php'; + require __DIR__ . '/feedback_checks.php'; + require __DIR__ . '/cache_checks.php'; + echo "TOTAL $count checks passed. Mocked IPS, clocks and register writes only.\n"; +} diff --git a/tests/V4ControlTrial/feedback_checks.php b/tests/V4ControlTrial/feedback_checks.php new file mode 100644 index 0000000..3c2c83b --- /dev/null +++ b/tests/V4ControlTrial/feedback_checks.php @@ -0,0 +1,23 @@ +props['NetzfahrplanV4RueckmeldungKonfiguration']='{}'; +rejects(fn()=>start($m,$s),'trial cannot use old filtered feedback when mapping missing');check($b->writes===[],'missing corrected feedback sends no command'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$deviceReadOk=false; +$a=tick($m);check(!$m->active()&&!$b->active()&&isset($a['Sollwerte'][44234]),'failed confirmed read returns battery to normal allocation'); +check(end($b->writes)===0,'failed confirmed read revokes prior command'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$deviceReadOk=false;$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'independent battery read catches device failure without manager tick'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=json_decode($b->props['NetzfahrplanV4RueckmeldungKonfiguration'],true);$c['trackingToleranceW']=250; +$b->props['NetzfahrplanV4RueckmeldungKonfiguration']=json_encode($c);$b->watchdog(); +check(!$b->active()&&end($b->writes)===0,'mapping revision change revokes active battery session'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();$c['sequence']++;$c['feedbackConfigHash']=str_repeat('f',64); +rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'changed feedback command binding rejected'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();check(isset($c['feedbackConfigHash'],$c['feedbackSourceOldestAt'])&&array_key_exists('importLimitW',$c),'command includes source provenance and limits'); +$c['sequence']++;unset($c['exportLimitW']);rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'missing explicit grid limit not interpreted as unlimited'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$c=$m->command();$c['sequence']++;$c['importLimitW']=1000.;$c['watts']=0; +$m->op['gridW']=1500.;$b->measurements['Ladezustand']=15.; +$b->sample(); +rejects(fn()=>$b->V4BatterieTestBefehl(json_encode($c)),'final device clipping cannot silently violate accepted grid boundary'); +check(!$b->active()&&end($b->writes)===0,'unreachable final limit aborts software trial'); +[$m,$b,$s]=fixture();start($m,$s);tick($m);$feedbackAt[50002]=$clock+1;tick($m); +check(!$m->active(),'future physical sample is not smoothed into control'); diff --git a/tests/V4ControlTrial/register_checks.php b/tests/V4ControlTrial/register_checks.php new file mode 100644 index 0000000..92a1b78 --- /dev/null +++ b/tests/V4ControlTrial/register_checks.php @@ -0,0 +1,42 @@ +props[$key] ?? 0; } + protected function ReadPropertyBoolean($key) { return $this->props[$key] ?? false; } + protected function ReadAttributeInteger($key) { return $this->attrs[$key] ?? 0; } + protected function ReadAttributeBoolean($key) { return $this->attrs[$key] ?? false; } + protected function ReadAttributeString($key) { return $this->attrs[$key] ?? ''; } + protected function WriteAttributeInteger($key, $value) { $this->attrs[$key] = $value; } + protected function WriteAttributeBoolean($key, $value) { $this->attrs[$key] = $value; } + protected function WriteAttributeString($key, $value) { $this->attrs[$key] = $value; } +} +function IPS_GetObjectIDByIdent($ident, $parent) { return false; } +function RequestAction($id, $value) { $GLOBALS['writes'][] = [$id, $value]; return $GLOBALS['accepted']; } +require_once __DIR__ . '/../../Batterie/module.php'; +$battery = new Batterie(); +$battery->InstanceID = 44234; +$battery->props = [ + 'Batterietyp' => Belevo\EnelixEMS\BatterieRegler::TYP_GOODWE, + 'Batteriemanagement' => Belevo\EnelixEMS\BatterieRegler::STEUERUNG_ENELIX, + 'ManagementRegisterVariableID' => 1, 'ModusRegisterVariableID' => 2, + 'LeistungsRegisterVariableID' => 3, 'Aenderungssperre' => 0, +]; +$method = new ReflectionMethod(Batterie::class, 'schreibeRegister'); +$accepted = false; $writes = []; +try { $method->invoke($battery, 12000, false); throw new LogicException('FALSE was accepted'); } +catch (RuntimeException $error) { + if (!str_contains($error->getMessage(), 'Registeraktion nicht bestaetigt')) throw $error; +} +if (($battery->attrs['RegisterInitialisiert'] ?? false) || ($battery->attrs['RegisterSchreiben'] ?? false)) { + throw new RuntimeException('Failed action cached or lock not released'); +} +$accepted = true; +$method->invoke($battery, 12000, false); +if (($battery->attrs['LetzterSollwert'] ?? 0) !== 12000 || !$battery->attrs['RegisterInitialisiert']) { + throw new RuntimeException('Successful retry not cached'); +} +echo "PASS actual module rejects FALSE register action, releases guard and retries successfully\n"; diff --git a/tests/V4DataCapture/checks.php b/tests/V4DataCapture/checks.php new file mode 100644 index 0000000..4630aff --- /dev/null +++ b/tests/V4DataCapture/checks.php @@ -0,0 +1,62 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004, + 'reportedInventorySha256'=>str_repeat('a',64),'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]], + 'extraSources'=>[['key'=>'soc','variableId'=>106,'parentId'=>200,'ident'=>'soc','unit'=>'percent']]]; +$values=[3000,5000,2000,1500,500]; $readings=[]; +foreach($roles as $i=>$role){ + $key='s'.$i;$id=101+$i; + $c['accounting']['sources'][]=['key'=>$key,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>in_array($i,[3,4],true)?['s2']:[]]; + $readings[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key]; +} +$readings[106]=['value'=>0.,'updated'=>$now,'changed'=>0,'parentID'=>200,'ident'=>'soc']; +function take(array $c,array $r,int $now): array { return Capture::capture($c,static fn(int $id):array=>$r[$id],static fn():int=>$now); } +$r=take($c,$readings,$now); +ck($r['assessment']['baseLoadCandidateW']===6000.,'physical battery subtracted exactly once'); +ck($r['raw']['soc']['value']===0. && $r['raw']['soc']['sourceChangedAt']===0,'real zero and unchanged timestamp retained'); +ck(!$r['trainingEligible'] && !$r['controlEligible'] && !$r['measurementBoundaryVerified'],'algebra never enables training or control'); +ck($r['futureSdl']==='unknown_no_assumption','no fabricated future SDL'); +ck(count($r['raw'])===6,'raw extra sources retained separately'); +$a=$readings;$a[103]['updated']=$now-90;$a[104]['updated']=$now; +$x=take($c,$a,$now);ck($x['assessment']['status']==='quality_hold','old physical measurement quarantines candidate'); +ck($x['raw']['s2']['sourceUpdatedAt']===$now-90,'original old timestamp preserved'); +ck(in_array('stale_origin:s3',$x['assessment']['issues'],true),'fresh virtual write cannot mask old dependency'); +$a=$readings;$a[101]['value']=null;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && $x['assessment']===null,'missing source not zero'); +$a=$readings;$a[101]['value']='sensitive-test-text';$x=take($c,$a,$now);ck(!str_contains(json_encode($x),'sensitive-test-text'),'unexpected strings never logged'); +foreach([true,NAN,INF,1.0e20] as $v){$a=$readings;$a[101]['value']=$v;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && is_string(json_encode($x,JSON_THROW_ON_ERROR)),'invalid numeric represented safely');} +$a=$readings;$a[101]['parentID']=999;$x=take($c,$a,$now);ck($x['raw']['s0']['value']===null && $x['assessment']===null,'reused ID not recorded as correct measurement'); +$a=$readings;$a[102]['updated']=$now+1;$x=take($c,$a,$now);ck($x['assessment']===null && in_array('s1:future_source',$x['issues'],true),'future timestamp marked'); +$a=$readings;$a[101]['updated']=0;$x=take($c,$a,$now);ck($x['assessment']===null,'absent timestamp not invented'); +$counts=[];$x=Capture::capture($c,static function($id)use(&$counts,$readings){$r=$readings[$id];$counts[$id]=($counts[$id]??0)+1;if($id===101 && $counts[$id]===2)$r['value']++;return $r;},static fn()=>$now); +ck($x['assessment']===null && $x['raw']['s0']['secondValue']===3001,'same-second concurrent change visible'); +$x=Capture::capture($c,static function($id)use($readings){if($id===101)throw new RuntimeException('do-not-emit-secret');return $readings[$id];},static fn()=>$now); +ck(!str_contains(json_encode($x),'do-not-emit-secret') && $x['assessment']===null,'source errors bounded without secret messages'); +$a=$readings;$a[106]['value']=null;$x=take($c,$a,$now);ck($x['assessment']!==null && !$x['trainingEligible'],'optional SOC error does not destroy available raw powers'); +$a=$readings;$a[101]['value']=-10000;$x=take($c,$a,$now);ck($x['assessment']['baseLoadCandidateW']<0 && $x['assessment']['status']==='quality_hold','negative candidate not clipped'); +$a=$c;$a['extraSources'][]=$a['extraSources'][0];refuses(fn()=>take($a,$readings,$now),'duplicate source rejected'); +$a=$c;$a['accounting']['sources']=array_reverse($a['accounting']['sources']);ck(take($a,$readings,$now)['mappingSha256']===$r['mappingSha256'],'mapping identity stable'); +$t=0;refuses(fn()=>Capture::capture($c,static fn($id)=>$readings[$id],static function()use(&$t,$now){return $now-$t++;}),'backwards clock rejected'); +$base=sys_get_temp_dir().'/v4-raw-test-'.bin2hex(random_bytes(5));mkdir($base,0700);$dir=$base.'/data';mkdir($dir,0700); +try{ + $file=Capture::append($dir,$r);$first=file_get_contents($file);Capture::append($dir,$r); + ck(file_get_contents($file)===$first.$first,'append retains existing observations'); + ck(count(file($file))===2,'one record per line'); + ck((fileperms($file)&0777)===0640,'data permissions restricted'); + $old=file_get_contents($file);refuses(fn()=>Capture::append($dir,$r,1),'quota refuses without deleting data');ck(file_get_contents($file)===$old,'quota leaves journal unchanged'); + file_put_contents($file,'partial',FILE_APPEND);refuses(fn()=>Capture::append($dir,$r),'crash-truncated tail not silently extended');ck(str_ends_with(file_get_contents($file),'partial'),'incomplete record retained for review'); + unlink($file);file_put_contents($base.'/foreign','untouched');symlink($base.'/foreign',$file);refuses(fn()=>Capture::append($dir,$r),'journal symlink refused');ck(file_get_contents($base.'/foreign')==='untouched','unrelated file preserved');unlink($file); + $lock=fopen($dir.'/.writer.lock','c+b');flock($lock,LOCK_EX);refuses(fn()=>Capture::append($dir,$r),'parallel writer refused');flock($lock,LOCK_UN);fclose($lock); + $a=$r;$a['blob']=str_repeat('x',300000);refuses(fn()=>Capture::append($dir,$a),'oversized record refused'); +}finally{foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($base,FilesystemIterator::SKIP_DOTS),RecursiveIteratorIterator::CHILD_FIRST)as$p){$p->isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base);} +$source=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Messaufnahme.php'); +ck(!preg_match('/\b(?:IPS_|GetValue|SetValue|curl_|RequestAction|shell_exec|exec\s*\()/',$source),'capture core has no device or network calls'); +echo "TOTAL $n checks passed; synthetic values and temporary files only.\n"; diff --git a/tests/V4DataCapture/native_checks.php b/tests/V4DataCapture/native_checks.php new file mode 100644 index 0000000..1b2f062 --- /dev/null +++ b/tests/V4DataCapture/native_checks.php @@ -0,0 +1,61 @@ +$o['ParentID']===$id));} +function IPS_InstanceExists($id){return $id===17004;} +function IPS_GetInstance($id){return ['ModuleInfo'=>['ModuleID'=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']];} +function IPS_VariableExists($id){return isset($GLOBALS['values'][$id]);} +function IPS_GetVariable($id){$v=$GLOBALS['values'][$id];return ['VariableType'=>is_string($v)?3:2,'VariableUpdated'=>time(),'VariableChanged'=>time()-10];} +function GetValue($id){return $GLOBALS['values'][$id];} +function SetValue($id,$v){if($id<60000||IPS_GetObject($id)['ObjectIdent']!=='CaptureStatus')throw new RuntimeException('Foreign variable write');$GLOBALS['values'][$id]=$v;return true;} +function IPS_SemaphoreEnter($key,$wait){if($GLOBALS['locks'][$key]??false)return false;$GLOBALS['locks'][$key]=true;return true;} +function IPS_SemaphoreLeave($key){$GLOBALS['locks'][$key]=false;} +function makeObj($type){$id=$GLOBALS['nextId']++;$GLOBALS['objects'][$id]=['ObjectType'=>$type,'ObjectIdent'=>'','ParentID'=>0,'ObjectInfo'=>''];return $id;} +function IPS_CreateCategory(){return makeObj(0);} +function IPS_CreateScript($type){$id=makeObj(3);$GLOBALS['scripts'][$id]=''synthetic_test','physicalKwh'=>322,'evKwh'=>160];file_put_contents($base.'/USER_REPORTED_INVENTORY.json',json_encode($info)); +$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004,'reportedInventorySha256'=>hash_file('sha256',$base.'/USER_REPORTED_INVENTORY.json'), + 'extraSources'=>[],'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]]]; +foreach(['grid','pv','physical_storage','controlled_virtual','external_virtual']as$i=>$role){$id=101+$i;$c['accounting']['sources'][]=['key'=>'x'.$i,'variableId'=>$id,'parentId'=>200,'ident'=>'v'.$i,'role'=>$role,'factorToW'=>1,'maxAgeSeconds'=>60,'dependsOn'=>[]];$objects[$id]=['ParentID'=>200,'ObjectIdent'=>'v'.$i,'ObjectType'=>2,'ObjectInfo'=>'original'];$values[$id]=[1000,2000,500,500,0][$i];} +file_put_contents($base.'/config.json',json_encode($c));$hashes['config.json']=hash_file('sha256',$base.'/config.json');$hashes['USER_REPORTED_INVENTORY.json']=hash_file('sha256',$base.'/USER_REPORTED_INVENTORY.json'); +file_put_contents($base.'/MANIFEST.json',json_encode(['scope'=>'passive_raw_capture_only','hashes'=>$hashes])); +$before=$objects;$beforeValues=$values; +try{ + ob_start();$r=v4captureInstall($base);ob_end_clean(); + yes($r['status']==='capture_installed','passive installation completes'); + yes(count($r['createdObjects'])===4,'only category, two scripts and one status variable'); + yes($timers[$r['scriptId']]===30,'only new capture timer runs at 30 seconds'); + yes($r['firstCapture']['status']==='recorded'&&count(file($r['firstCapture']['journal']))===1,'first capture actually written before activation'); + yes($r['firstCapture']['accountingQuality']==='consistent_candidate','candidate is not a training permit'); + foreach($before as$id=>$o){yes($objects[$id]===$o && $values[$id]===$beforeValues[$id],'existing source preserved '.$id);} + $size=count($objects);ob_start();$again=v4captureInstall($base);ob_end_clean();yes(count($objects)===$size&&$again['createdObjects']===[],'installation idempotent, no duplicate timers'); + $bad=$scripts[$r['scriptId']];$scripts[$r['scriptId']]="isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base);} +echo "TOTAL $count native-mock checks passed; no Symcon kernel calls.\n"; diff --git a/tests/V4Feedback/checks.php b/tests/V4Feedback/checks.php new file mode 100644 index 0000000..59ba0db --- /dev/null +++ b/tests/V4Feedback/checks.php @@ -0,0 +1,59 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','assetId'=>'ev','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>$mode,'allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60,'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0,'sources'=>[]]; + $roles=['grid','physical','physical'];if($mode==='virtual_split')$roles=array_merge($roles,['ev_request','sdl_request','gateway_active']);$values=[3000.0,-2000.0,-1000.0,-3000.0,0.0,true];$r=[]; + foreach($roles as$i=>$role){$id=100+$i;$c['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'q'.$id,'factorToW'=>1];$r[$id]=['value'=>$values[$i],'updated'=>$now,'changed'=>$now-60,'parentID'=>200,'ident'=>'q'.$id,'confirmedAt'=>$now];} + return [$c,$r,$now]; +} +function readit($c,$r,$n){return F::read($c,fn($id)=>$r[$id],$n);} +[$c,$r,$n]=fx();$f=readit($c,$r,$n); +check($f['batteryW']===-3000.0&&$f['gridW']===3000.0,'physical total and grid signs retained'); +check($f['usableForTrial']&&!$f['estimated']&&!$f['canDispatch'],'usable feedback alone never grants dispatch'); +F::requireTrial($f,$c,$n);check(true,'matching trial binding accepted'); +check(F::fingerprint($c)===F::fingerprint(json_decode(json_encode($c),true)),'configuration hash stable over numeric property roundtrip'); +$d=$c;$d['sources']=array_reverse($d['sources']);check(F::fingerprint($c)===F::fingerprint($d),'source list order canonical'); +$d['trackingToleranceW']=201;check(F::fingerprint($c)!==F::fingerprint($d),'changed tolerance invalidates session hash'); +foreach([true,'1',NAN,INF,null]as$bad){$x=$r;$x[101]['value']=$bad;fails(fn()=>readit($c,$x,$n),'invalid physical number '.gettype($bad));} +$x=$r;$x[101]['updated']=$n-61;$x[101]['changed']=$n-90;$f=readit($c,$x,$n);check($f['sourceVariableOldestAt']===$n-61,'confirmed read does not manufacture VariableUpdated'); +$x=$r;unset($x[101]['confirmedAt']);fails(fn()=>readit($c,$x,$n),'physical source without confirmed device read rejected'); +$x=$r;$x[101]['confirmedAt']=$n-31;fails(fn()=>readit($c,$x,$n),'incoherent confirmed device timestamps rejected'); +$x=$r;$x[101]['updated']=$n+1;fails(fn()=>readit($c,$x,$n),'future original sample rejected'); +$x=$r;$x[101]['parentID']=201;fails(fn()=>readit($c,$x,$n),'reused variable wrong parent rejected'); +$x=$r;$x[101]['ident']='other';fails(fn()=>readit($c,$x,$n),'reused variable wrong ident rejected'); +$d=$c;$d['sources'][1]['variableId']=$d['sources'][0]['variableId'];fails(fn()=>F::configuration($d),'duplicate physical source rejected'); +$d=$c;$d['maxAgeSeconds']=300;fails(fn()=>F::configuration($d),'historical freshness not accepted for live control'); +$calls=[];fails(function()use($c,$r,$n,&$calls){F::read($c,function($id)use($r,&$calls){$calls[$id]=($calls[$id]??0)+1;$x=$r[$id];if($calls[$id]>1)$x['value']+=1;return$x;},$n);},'same-second change during double read rejected'); +$d=$c;$d['sources'][1]['factorToW']=-1;check(readit($d,$r,$n)['batteryW']===1000.0,'vendor sign conversion once'); +$f=readit($c,$r,$n);fails(fn()=>F::requireTrial($f,$c,$n+3),'old validated snapshot not reused'); +$d=$c;$d['managerId']=20000;fails(fn()=>F::requireTrial($f,$d,$n),'different manager cannot use feedback'); +$d=$c;$d['feedbackConfigHash']=str_repeat('a',64);fails(fn()=>F::requireTrial($f,$d,$n),'changed mapping aborts trial'); +[$c,$r,$n]=fx('virtual_split');$r[103]['updated']=$r[104]['updated']=$r[105]['updated']=$n-86400;$r[103]['changed']=$r[104]['changed']=$r[105]['changed']=$n-172800; +$r[101]['value']=$r[102]['value']=$r[103]['value']=$r[104]['value']=0.0;$f=readit($c,$r,$n); +check($f['batteryW']===0.0&&$f['estimated'],'unchanged idle commands need no artificial heartbeat'); +check(!$f['usableForTrial'],'virtual estimate requires explicit acceptance'); +$c['allowEstimatedForTrial']=true;$f=readit($c,$r,$n);check($f['usableForTrial']&&!$f['gatewayWatchdogVerified'],'idle acceptance is not gateway watchdog proof'); +$r[101]['value']=20.0;$f=readit($c,$r,$n);check($f['batteryW']===20.0,'idle residual not clipped to zero'); +$r[101]['value']=60.0;fails(fn()=>readit($c,$r,$n),'unexplained idle power refused'); +$r[101]['value']=-2000.0;$r[102]['value']=-1000.0;$r[103]['value']=2000.0;$r[103]['updated']=$n;$r[103]['changed']=$n; +$f=readit($c,$r,$n);check($f['batteryW']===-3000.0,'single EV feedback follows physics not desired sign or held filter'); +$r[104]['value']=1000.0;$r[104]['updated']=$n;$r[104]['changed']=$n;$r[101]['confirmedAt']=$n-1; +fails(fn()=>readit($c,$r,$n),'SDL transition after physical confirmation cannot be guessed'); +[$c,$r,$n]=fx('virtual_split');$c['allowEstimatedForTrial']=true;$r[103]['value']=3000.0;$r[104]['value']=1000.0;$r[101]['value']=1900.0;$r[102]['value']=2000.0;$f=readit($c,$r,$n); +check($f['batteryW']===2925.0&&$f['externalModelW']===975.0,'small tracking error allocated but explicitly estimated'); +check(!$f['separatelyMeasured']&&$f['heldFilterUsed']===false,'partition never labelled separately measured'); +$r[101]['value']=1000.0;fails(fn()=>readit($c,$r,$n),'large tracking error prevents stale virtual correction'); +$r[103]['value']=42000.0;$r[104]['value']=-40000.0;$r[101]['value']=0.0;$r[102]['value']=2000.0;$f=readit($c,$r,$n); +check(!$f['usableForTrial']&&in_array('opposing_virtual_flows_not_identifiable',$f['reasons'],true),'counterflow cannot be independently attributed'); +$r[105]['value']=false;fails(fn()=>readit($c,$r,$n),'disabled gateway blocks feedback'); +$src=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Rueckmeldung.php'); +check(!preg_match('/\b(?:SetValue|IPS_RequestAction|curl_exec|IPS_SetProperty|AC_SetLoggingStatus)\s*\(/',$src),'pure feedback has no control side effects'); +echo "TOTAL $count physical-feedback checks passed. Synthetic source readings only.\n"; diff --git a/tests/V4Feedback/device_read_checks.php b/tests/V4Feedback/device_read_checks.php new file mode 100644 index 0000000..37df061 --- /dev/null +++ b/tests/V4Feedback/device_read_checks.php @@ -0,0 +1,63 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','assetId'=>'ev','managerId'=>17004,'batteryInstanceId'=>44234, + 'mode'=>'physical_sum','allowEstimatedForTrial'=>false,'maxSkewSeconds'=>30,'maxAgeSeconds'=>60,'idleToleranceW'=>50.0,'trackingToleranceW'=>200.0, + 'sources'=>[ + ['key'=>'grid','role'=>'grid','variableId'=>10,'parentId'=>20,'ident'=>'grid','factorToW'=>1.0], + ['key'=>'battery_a','role'=>'physical','variableId'=>11,'parentId'=>21,'ident'=>'a','factorToW'=>1.0], + ['key'=>'battery_b','role'=>'physical','variableId'=>12,'parentId'=>21,'ident'=>'b','factorToW'=>1.0], + ]]; + $records=[ + 10=>['value'=>3000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>20,'ident'=>'grid'], + 11=>['value'=>-1000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>21,'ident'=>'a'], + 12=>['value'=>-2000.0,'updated'=>$now-100,'changed'=>$now-200,'parentID'=>21,'ident'=>'b'], + ]; + return [$config,$records,$now]; +} +[$c,$r,$n]=dfixture();$calls=[];$times=[$n-1,$n-1,$n]; +$f=D::snapshot($c,fn($id)=>$id===20?D::MBUS_DEVICE:D::MODBUS_DEVICE, + function($id,$module)use(&$calls){$calls[]=$id;return true;}, + fn($id)=>$r[$id],function()use(&$times){return array_shift($times);}); +sort($calls);dcheck($calls===[20,21],'one refresh per physical parent'); +dcheck($f['deviceReadConfirmed']===true&&$f['sourceOldestAt']===$n-1,'successful read confirmation is freshness basis'); +dcheck($f['sourceVariableOldestAt']===$n-100,'original VariableUpdated retained separately'); +dcheck($f['batteryW']===-3000.0&&$f['usableForTrial'],'unchanged values accepted after confirmed device response'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n];$attempts=0;$pauses=[]; +$f=D::snapshot($c,fn($id)=>D::MODBUS_DEVICE, + function()use(&$attempts){$attempts++;return $attempts>=2;},fn($id)=>$r[$id], + function()use(&$times){return array_shift($times);},function($us)use(&$pauses){$pauses[]=$us;}); +dcheck($attempts===3&&$pauses===[200000],'transient busy read is retried once without relaxing confirmation'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n]; +dfails(fn()=>D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>false,fn($id)=>$r[$id],function()use(&$times){return array_shift($times);}), + 'failed device read rejected'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n];$reads=0; +$f=D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>true, + function($id)use(&$r,&$reads){$reads++;$x=$r[$id];if($reads>3&&$id===10){$x['value']++;$x['updated']++;$x['changed']++;}return $x;}, + function()use(&$times){return array_shift($times);}); +dcheck($f['gridW']===3000.0,'newer process value after confirmed response does not invalidate confirmed snapshot'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n];$reads=0; +dfails(fn()=>D::snapshot($c,fn($id)=>D::MODBUS_DEVICE,fn()=>true, + function($id)use(&$r,&$reads){$reads++;$x=$r[$id];if($reads>3&&$id===10)$x['parentID']=999;return $x;}, + function()use(&$times){return array_shift($times);}), 'source identity change after confirmed response rejected'); + +[$c,$r,$n]=dfixture();$times=[$n,$n,$n]; +dfails(fn()=>D::snapshot($c,fn($id)=>'{00000000-0000-0000-0000-000000000000}',fn()=>true,fn($id)=>$r[$id], + function()use(&$times){return array_shift($times);}), 'unsupported device module rejected'); + +echo "TOTAL $count confirmed-device-read checks passed. Synthetic calls only.\n"; diff --git a/tests/V4Feedback/fallback_checks.php b/tests/V4Feedback/fallback_checks.php new file mode 100644 index 0000000..efe7d87 --- /dev/null +++ b/tests/V4Feedback/fallback_checks.php @@ -0,0 +1,23 @@ +$GLOBALS['instance']->status];} +eval('class FallbackHarness { const STATUS_AKTIV=102; public $InstanceID=17004,$trial=true,$active=true,$status=102,$abortFirst=true,$throwFirst=false,$passes=[],$attributes=[],$stops=[];'. + 'public function run(){ $this->regeln(false); } private function v4ManagerTestSession(){return $this->trial?[1]:[];} private function GetValue($k){return $this->active;}'. + 'private function WriteAttributeString($k,$v){$this->attributes[$k]=$v;} private function v4ManagerTestAbbrechen($s){$this->stops[]=$s;$this->trial=false;}'. + 'private function regelnIntern(bool $r):void { $this->passes[]=$r;if(count($this->passes)==1){if($this->throwFirst)throw new RuntimeException("synthetic_failure");if($this->abortFirst)$this->trial=false;}}'. $method.'}'); +$count=0;function ck($v,$s){global$count;if(!$v)throw new RuntimeException('FAIL '.$s);$count++;echo "PASS $s\n";} +$instance=new FallbackHarness();$instance->run();ck($instance->passes===[false,true],'trial abort performs exactly one fresh ordinary recomputation'); +ck($instance->attributes['LetzteSollwerte']==='{}','pre-trial cached target is discarded before fallback');ck(!$locked,'control semaphore released after fallback'); +$instance=new FallbackHarness();$instance->trial=false;$instance->run();ck($instance->passes===[false],'ordinary control path does not gain extra passes'); +$instance=new FallbackHarness();$instance->abortFirst=false;$instance->run();ck($instance->passes===[false],'valid trial not overridden by ordinary pass'); +$instance=new FallbackHarness();$instance->status=201;$instance->run();ck($instance->passes===[false],'invalid measurement status prevents forced ordinary recomputation'); +$instance=new FallbackHarness();$instance->throwFirst=true;try{$instance->run();}catch(RuntimeException $e){} +ck(!$locked&&!$instance->trial&&count($instance->stops)===1,'exception revokes trial and releases semaphore without endless retries'); +echo "TOTAL $count actual manager fallback-wrapper checks passed. No real IPS runtime.\n"; diff --git a/tests/V4Feedback/message_checks.php b/tests/V4Feedback/message_checks.php new file mode 100644 index 0000000..0c85b13 --- /dev/null +++ b/tests/V4Feedback/message_checks.php @@ -0,0 +1,23 @@ +'1'];public bool $valid=true; + protected function GetBuffer($k){return $this->buffers[$k]??'';} + protected function GetValue($k){throw new RuntimeException('No diagnostic variables exist: '.$k);} + protected function ReadAttributeBoolean($k){return $k==='ZustandLadezustandGueltig'?$this->valid:false;} + protected function ReadAttributeInteger($k){return 0;} + protected function ReadAttributeFloat($k){return 0.0;} + protected function ReadAttributeString($k){return $k==='Leistungsangebot'?'[0]':($k==='Betriebsart'?'Auto':'');} + protected function ReadPropertyInteger($k){return 1;} +} +require __DIR__.'/../../Batterie/module.php'; +use Belevo\EnelixEMS\Nachrichtenvertrag as N; +$b=new Batterie();$read=new ReflectionMethod(Batterie::class,'baueVerbraucherdaten');$count=0; +function ck($v,$s){global$count;if(!$v)throw new RuntimeException('FAIL '.$s);$count++;echo "PASS $s\n";} +$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_BERECHNET,'partition source quality independent of hidden diagnostic variables'); +$b->buffers['V4FeedbackEstimated']='0';$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_GEMESSEN,'ordinary physical source marked measured'); +$b->valid=false;$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_NICHT_VORHANDEN&&$d['Istleistung_W']===null,'invalid sample never published as measured'); +$b->valid=true;$b->buffers['V4FeedbackEstimated']='1';$b->buffers['V4BatteryTrialSession']='{}';$d=$read->invoke($b,17004);ck($d['Leistungsquelle']===N::LEISTUNGSQUELLE_BERECHNET,'last estimate not relabelled measured immediately after trial ends'); +$source=file_get_contents(__DIR__.'/../../Batterie/module.php');ck(str_contains($source,"SetBuffer('V4FeedbackEstimated', (\$messwerte['V4FeedbackEstimated'] ?? false) ? '1' : '0')"),'fresh measurement updates internal source quality'); +echo "TOTAL $count actual message-builder checks passed. No real variables or writes.\n"; diff --git a/tests/V4Feedback/receiver_checks.php b/tests/V4Feedback/receiver_checks.php new file mode 100644 index 0000000..18e5b6e --- /dev/null +++ b/tests/V4Feedback/receiver_checks.php @@ -0,0 +1,46 @@ +['ModuleID'=>'{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}']];} + function IPS_GetProperty($id,$k){return ['IstleistungVariableID'=>1002,'LadezustandVariableID'=>1001,'NetzfahrplanV4RueckmeldungKonfiguration'=>$GLOBALS['configured']?'configured':'{}'][$k];} + function IPS_VariableExists($id){return $id===1002;} + function IPS_GetVariable($id){return ['VariableUpdated'=>$GLOBALS['now']];} + function GetValue($id){return -9000.;} // deliberately held/incorrect legacy virtual display + class FeedbackReceiver { + use ManagerNetzfahrplanV4EmpfangTrait; + public int $InstanceID=17004;public array $props,$attrs,$op; + public function __construct($r,$op){$this->op=$op;$this->props=['NetzfahrplanV4EmpfangAktiv'=>true,'NetzfahrplanV4SchattenAktiv'=>true,'NetzfahrplanAktiv'=>false,'MesswertMaxAlter'=>60,'AnlagenBatterien'=>json_encode([['ID'=>'b','LeistungVariableID'=>1002,'SOCVariableID'=>1001,'Leistungsfaktor'=>1]])];$this->attrs=['LizenzInstallationID'=>$r['installationId'],'NetzfahrplanV4Empfang'=>json_encode(['response'=>$r,'receivedAt'=>time()]),'NetzfahrplanV4EmpfangStatus'=>'{}'];} + private function ReadPropertyBoolean($k){return $this->props[$k];} + private function ReadPropertyString($k){return $this->props[$k];} + private function ReadPropertyInteger($k){return $this->props[$k];} + private function ReadAttributeString($k){return $this->attrs[$k];} + private function aktiveVerbraucherIDs(){return [44234];} + private function berechtigungLizenziert($k){return true;} + public function GetNetzfahrplanV4Diagnose(){return json_encode(['operation'=>$this->op]);} + } +} +namespace { + function ENELIX_GetV4BatterieRueckmeldung($id){$GLOBALS['now']+=($GLOBALS['feedbackDelay']??0);$GLOBALS['feedback']['checkedAt']=$GLOBALS['now'];return json_encode($GLOBALS['feedback']);} + require_once __DIR__.'/../V4Receiver/fixture.php'; + use Belevo\EnelixEMS\FeedbackReceiver; + [$response,$id,$op,$now]=receiverFixture();$configured=true;$count=0;$feedbackDelay=0; + $feedback=['kind'=>'local_physical_feedback','version'=>1,'installationId'=>$id,'assetId'=>'b','managerId'=>17004,'batteryInstanceId'=>44234,'batteryW'=>0.,'gridW'=>$op['gridW'],'checkedAt'=>$now,'estimated'=>true,'usableForTrial'=>false,'canDispatch'=>false,'configHash'=>str_repeat('a',64)]; + function ck($v,$t){global$count;if(!$v)throw new RuntimeException('FAIL '.$t);$count++;echo "PASS $t\n";} + $m=new FeedbackReceiver($response,$op);$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='preview_only'&&$p['actualBatteryW']==0,'real receiver uses corrected physical feedback rather than held -9kW'); + ck($p['previewBatteryW']==3000,'net target translated once to total battery demand'); + ck($p['canDispatch']===false&&$p['localFeedback']['b']['estimated']===true,'model feedback remains marked and preview grants no dispatch'); + $feedbackDelay=1;$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true);$feedbackDelay=0; + ck($p['status']==='preview_only','feedback timestamp produced after device read is compared with post-read clock'); + $feedback=['status'=>'unavailable','reason'=>'feedback_device_read_failed:grid','batteryW'=>null,'gridW'=>null];$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='unavailable'&&$p['previewBatteryW']===null,'configured feedback failure never falls back to held virtual display'); + ck(str_contains($p['reason'],'feedback_unavailable:feedback_device_read_failed:grid'),'configured feedback exposes bounded root cause without credentials'); + $configured=false;$p=json_decode($m->GetNetzfahrplanV4Vorschau(),true); + ck($p['status']==='preview_only'&&$p['feedbackMeaning']==='legacy_display_only_not_for_trial','unconfigured old preview stays compatible but explicitly not for control'); + ck($p['canDispatch']===false,'legacy display cannot gain actuation permission'); + echo "TOTAL $count actual receiver checks passed; synthetic read-only controller API.\n"; +} diff --git a/tests/V4Receiver/fixture.php b/tests/V4Receiver/fixture.php new file mode 100644 index 0000000..03f8ed5 --- /dev/null +++ b/tests/V4Receiver/fixture.php @@ -0,0 +1,30 @@ + 'op-original', 'observedAt' => gmdate('c', $now), 'gridW' => -3000.0, + 'batteries' => [['id' => 'b', 'capacityKwh' => 20.0, 'socPercent' => 50.0, 'minSocPercent' => 15.0, + 'maxSocPercent' => 90.0, 'physicalMinSocPercent' => 3.0, 'rearmSocPercent' => 17.0, + 'gridCharging' => true, 'dischargeBlocked' => false, 'maxChargeW' => 39000.0, 'maxDischargeW' => 39000.0, + 'measuredAt' => gmdate('c', $now)]], + 'limits' => ['importW' => 25000.0, 'exportW' => 25000.0, 'managerMonthLimitsW' => []]]; + $points = []; + for ($i = 0; $i < 6; $i++) $points[] = ['time' => gmdate('c', $now + $i*300), 'validUntil' => gmdate('c', $now + ($i+1)*300), + 'baselineGridW' => -3000.0, 'gridTargetW' => 0.0, 'batteryTargetW' => 3000.0, 'pvCurtailmentW' => 0.0, + 'assetTargetsW' => ['b' => 3000.0], 'socEndPercent' => ['b' => 50.0 + $i], 'intent' => 'pvCharge', + 'importLimitW' => 25000.0, 'exportLimitW' => 25000.0]; + $plan = ['schemaVersion' => 2, 'planId' => '10000000-0000-4000-8000-000000000001', 'installationId' => $id, + 'configRevision' => 1, 'sourceFamily' => '3', 'runMode' => 'shadow', 'liveEnabled' => false, 'executable' => true, + 'status' => 'optimal', 'generatedAt' => gmdate('c', $now), 'validFrom' => gmdate('c', $now), + 'validUntil' => gmdate('c', $now + 1800), 'pricesKnownUntil' => gmdate('c', $now + 1800), + 'controlContext' => \Belevo\EnelixEMS\NetzfahrplanV4Planpruefung::kontext($operation), + 'inputRefs' => ['operation' => 'op-original', 'forecast' => 'fc-original', 'tariffs' => 'tar-original'], + 'points' => $points, 'warnings' => ['Estimated peak; shadow only']]; + $response = ['receiverProtocolVersion' => 1, 'installationId' => $id, 'checkedAt' => gmdate('c', $now), + 'liveEnabled' => false, 'fresh' => true, 'pending' => null, + 'settings' => ['revision' => 1, 'family' => '3', 'runMode' => 'shadow', 'roundtripEfficiency' => .9], + 'families' => [['key' => '3'], ['key' => '13'], ['key' => '23']], 'plan' => $plan, + 'lastRun' => ['status' => 'optimal', 'detail' => ['planId' => $plan['planId']]]]; + return [$response, $id, $operation, $now]; +} diff --git a/tests/V4Receiver/protocol_checks.php b/tests/V4Receiver/protocol_checks.php new file mode 100644 index 0000000..de7cd5b --- /dev/null +++ b/tests/V4Receiver/protocol_checks.php @@ -0,0 +1,73 @@ +0.0],$now); +check($v['previewBatteryW']===3000.0 && $v['canDispatch']===false && $v['liveEnabled']===false,'zero grid uses actual surplus, never permission'); +invalid('wrong installation',fn(&$r)=>$r['installationId']='00000000-0000-4000-8000-000000000001'); +invalid('foreign plan',fn(&$r)=>$r['plan']['installationId']='00000000-0000-4000-8000-000000000001'); +invalid('old protocol',function(&$r){unset($r['receiverProtocolVersion']);}); +invalid('revision mismatch',fn(&$r)=>$r['plan']['configRevision']=2); +invalid('boolean revision',fn(&$r)=>$r['plan']['configRevision']=true); +invalid('live response',fn(&$r)=>$r['liveEnabled']=true); +invalid('live plan',fn(&$r)=>$r['plan']['runMode']='live'); +invalid('fake executable',fn(&$r)=>$r['plan']['executable']='true'); +invalid('stale server flag',fn(&$r)=>$r['fresh']=false); +invalid('pending computation',fn(&$r)=>$r['pending']=['reason'=>'new price']); +invalid('last run different plan',fn(&$r)=>$r['lastRun']['detail']['planId']='wrong'); +invalid('last run invalid',fn(&$r)=>$r['lastRun']['status']='invalid_inputs'); +invalid('family mismatch',fn(&$r)=>$r['settings']['family']='23'); +invalid('unknown family',fn(&$r)=>$r['plan']['sourceFamily']='unregistered'); +invalid('receipt stale',function(&$r,&$op,&$now,&$receipt){$receipt-=91;}); +invalid('future receipt',function(&$r,&$op,&$now,&$receipt){$receipt++;}); +invalid('stale server response',function(&$r)use($now){$r['checkedAt']=gmdate('c',$now-91);}); +invalid('future plan',function(&$r)use($now){$r['plan']['generatedAt']=gmdate('c',$now+10);}); +invalid('stale plan',function(&$r)use($now){$r['plan']['generatedAt']=gmdate('c',$now-901);}); +invalid('price horizon too short',function(&$r)use($now){$r['plan']['pricesKnownUntil']=gmdate('c',$now+300);}); +invalid('stale local operation',function(&$r,&$op)use($now){$op['observedAt']=gmdate('c',$now-61);}); +invalid('changed local reserve',fn(&$r,&$op)=>$op['batteries'][0]['minSocPercent']=30.0); +invalid('changed local grid limit',fn(&$r,&$op)=>$op['limits']['importW']=2000.0); +invalid('typed context required',fn(&$r)=>$r['plan']['controlContext']['batteries']['b']['capacityKwh']='20'); +invalid('input provenance missing',function(&$r){unset($r['plan']['inputRefs']);}); +invalid('unknown assets',function(&$r){$r['plan']['points'][0]['assetTargetsW']=['c'=>3000];}); +invalid('overlap',function(&$r){$r['plan']['points'][1]['time']=$r['plan']['points'][0]['time'];}); +invalid('gap',function(&$r)use($now){$r['plan']['points'][1]['time']=gmdate('c',$now+310);}); +invalid('bool target',fn(&$r)=>$r['plan']['points'][0]['gridTargetW']=false); +invalid('string target',fn(&$r)=>$r['plan']['points'][0]['gridTargetW']='0'); +invalid('NaN target',fn(&$r)=>$r['plan']['points'][0]['batteryTargetW']=NAN); +invalid('broken balance',fn(&$r)=>$r['plan']['points'][0]['baselineGridW']=10000.0); +invalid('unknown intent',fn(&$r)=>$r['plan']['points'][0]['intent']='discharge_all'); +invalid('no timezone',fn(&$r)=>$r['plan']['points'][0]['time']='2026-10-02T12:00:00'); +invalid('bad calendar date',fn(&$r)=>$r['plan']['points'][0]['time']='2026-02-30T12:00:00Z'); +invalid('missing step',fn(&$r)=>$r['plan']['points']=[]); +invalid('negative curtailment',fn(&$r)=>$r['plan']['points'][0]['pvCurtailmentW']=-100.0); +[$r,$id,$op,$now]=receiverFixture();$r['checkedAt']=$op['observedAt']=gmdate('c',$now+300);$op['batteries'][0]['measuredAt']=$op['observedAt']; +$x=P::pruefen($r,$id,$op,$now+300,$now+300); +check($x['point']['time']===gmdate('c',$now+300),'half-open boundary picks current, not interpolated'); +[$r,$id,$op,$now]=receiverFixture();$r['plan']['sourceFamily']=$r['settings']['family']='future-family';$r['families'][]=['key'=>'future-family']; +check(P::pruefen($r,$id,$op,$now,$now)['plan']['sourceFamily']==='future-family','registry is extensible'); +[$r,$id,$op,$now]=receiverFixture();$op['gridW']=3000.0;$op['batteries'][0]['dischargeBlocked']=true; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now); +check($v['previewBatteryW']===0.0,'local hysteresis prevents preview discharge'); +[$r,$id,$op,$now]=receiverFixture();$op['gridW']=3000.0;$op['batteries'][0]['socPercent']=15.001; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now); +check($v['previewBatteryW']>-3 && $v['previewBatteryW']<0,'remaining-step energy cannot cross reserve'); +[$r,$id,$op,$now]=receiverFixture();$op['batteries'][0]['socPercent']=90; +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.0,'full battery cannot charge'); +[$r,$id,$op,$now]=receiverFixture();$op['batteries'][0]['maxChargeW']=0;$op['limits']['exportW']=0.;$r['plan']['controlContext']=P::kontext($op); +$x=P::pruefen($r,$id,$op,$now,$now);$v=P::vorschau($x,$op,['b'=>0],$now);check($v['unmetGridLimitW']===3000.0 && !$v['canDispatch'],'zero export limit reports unmet constraint'); +[$r,$id,$op,$now]=receiverFixture();$x=P::pruefen($r,$id,$op,$now,$now);$op['gridW']=2000;$op['batteries'][0]['gridCharging']=false; +$x['point']['gridTargetW']=8000.;$x['point']['intent']='gridCharge';$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.,'grid charging opt-in enforced'); +$op['batteries'][0]['gridCharging']=true;$x['point']['intent']='hold';$v=P::vorschau($x,$op,['b'=>0],$now);check($v['previewBatteryW']===0.,'nonzero hold does not accidentally charge'); +$x['point']['gridTargetW']=0.;$x['point']['intent']='discharge';$op['gridW']=0.;$v=P::vorschau($x,$op,['b'=>-2000],$now);check($v['previewBatteryW']===-2000.,'uses actual battery feedback, not incremental zero'); +try { P::dekodieren(str_repeat('x',P::MAX_RESPONSE_BYTES+1));check(false,'body limit'); } catch (InvalidArgumentException $e) {check(true,'body limit');} +echo "Total: $count checks. Synthetic data only.\n"; diff --git a/tests/V4Receiver/receiver_checks.php b/tests/V4Receiver/receiver_checks.php new file mode 100644 index 0000000..e8fb935 --- /dev/null +++ b/tests/V4Receiver/receiver_checks.php @@ -0,0 +1,140 @@ +$name) { + if (!defined($name)) define($name,50000+$i); + } + function time() { return $GLOBALS['clock']; } + function IPS_SemaphoreEnter($name,$wait) { if ($GLOBALS['lock']) return false;$GLOBALS['lock']=true;return true; } + function IPS_SemaphoreLeave($name) { $GLOBALS['lock']=false; } + function IPS_VariableExists($id) {return $id===100;} + function IPS_GetVariable($id) {return ['VariableUpdated'=>$GLOBALS['clock']];} + function GetValue($id) {return $GLOBALS['actual'];} + function curl_init($url) { return (object)['url'=>$url,'options'=>[],'code'=>0]; } + function curl_setopt_array($h,$options) { $h->options=$options;return true; } + function curl_getinfo($h,$key) { return $h->code; } + function curl_close($h) {} + function curl_exec($h) { + $GLOBALS['requests'][]=['url'=>$h->url,'options'=>$h->options]; + $ack=str_ends_with($h->url,'/ack'); + $h->code=$ack?$GLOBALS['ackCode']:$GLOBALS['getCode']; + $body=$ack?['status'=>'recorded']:$GLOBALS['response']; + $json=json_encode($body,JSON_THROW_ON_ERROR); + $callback=$h->options[CURLOPT_WRITEFUNCTION]; + return $callback($h,$json)===strlen($json); + } + final class ReceiverHarness { + use ManagerNetzfahrplanV4EmpfangTrait; + public int $InstanceID=17004; + public array $props=[],$attrs=[],$vars=[],$timers=[]; + public bool $licensed=true; + public array $session=[]; + public array $op=[]; + public function __construct($id,$op) { + $this->op=$op; + $this->props=['NetzfahrplanV4EmpfangAktiv'=>false,'NetzfahrplanV4SchattenAktiv'=>true, + 'NetzfahrplanAktiv'=>false,'MesswertMaxAlter'=>60, + 'AnlagenBatterien'=>json_encode([['ID'=>'b','LeistungVariableID'=>100,'Leistungsfaktor'=>1]])]; + $this->attrs=['LizenzInstallationID'=>$id,'PrognoseInstallationsToken'=>str_repeat('x',43)]; + $this->registriereNetzfahrplanV4Empfang(); + } + public function receive() {$this->empfangeNetzfahrplanV4();} + public function update() {$this->aktualisiereNetzfahrplanV4Vorschau();} + public function stop() {$this->stoppeNetzfahrplanV4Vorschau();} + public function configure() {$this->konfiguriereNetzfahrplanV4Vorschau();} + public function GetNetzfahrplanV4Diagnose() { return json_encode(['operation'=>$this->op]); } + private function aktiveVerbraucherIDs() {return [];} + private function v4ManagerTestSession() {return $this->session;} + private function berechtigungLizenziert($key) {return $this->licensed;} + private function ReadPropertyBoolean($key) {return (bool)$this->props[$key];} + private function ReadPropertyString($key) {return $this->props[$key];} + private function ReadPropertyInteger($key) {return (int)$this->props[$key];} + private function ReadAttributeString($key) {return $this->attrs[$key]??'';} + private function ReadAttributeInteger($key) {return (int)($this->attrs[$key]??0);} + private function WriteAttributeString($key,$val) { + if (!str_starts_with($key,'NetzfahrplanV4')) throw new \RuntimeException('Unexpected external attribute'); + $this->attrs[$key]=$val; + } + private function WriteAttributeInteger($key,$val) { $this->attrs[$key]=$val; } + private function RegisterPropertyBoolean($key,$default) {$this->props[$key]??=$default;} + private function RegisterAttributeString($key,$default) {$this->attrs[$key]??=$default;} + private function RegisterAttributeInteger($key,$default) {$this->attrs[$key]??=$default;} + private function RegisterVariableString($key,$name,$profile,$pos) {$this->vars[$key]='';} + private function RegisterTimer($key,$interval,$script) {$this->timers[$key]=$interval;} + private function SetTimerInterval($key,$interval) {$this->timers[$key]=$interval;} + private function SetValue($key,$value) { + if (!in_array($key,['NetzfahrplanV4Vorschau','NetzfahrplanV4VorschauJSON'],true)) throw new \RuntimeException('Actuator/foreign variable write'); + $this->vars[$key]=$value; + } + } +} +namespace { + require_once __DIR__.'/fixture.php'; + use Belevo\EnelixEMS\ReceiverHarness; + $count=0; + function check($ok,$name) {global $count;if (!$ok) throw new RuntimeException('FAIL '.$name);$count++;echo 'PASS '.$name."\n";} + function fixture() { + [$r,$id,$op,$now]=receiverFixture(); + $GLOBALS['response']=$r;$GLOBALS['clock']=$now;$GLOBALS['actual']=0.;$GLOBALS['requests']=[]; + $GLOBALS['lock']=false;$GLOBALS['getCode']=200;$GLOBALS['ackCode']=200; + return new ReceiverHarness($id,$op); + } + function advance($h,$seconds) { + $GLOBALS['clock']+=$seconds;$GLOBALS['response']['checkedAt']=gmdate('c',$GLOBALS['clock']); + $h->op['observedAt']=gmdate('c',$GLOBALS['clock']);$h->op['batteries'][0]['measuredAt']=$h->op['observedAt']; + } + $h=fixture();$h->receive();check(count($requests)===0,'default disabled no network'); + $h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->configure();check($h->timers['NetzfahrplanV4VorschauAktualisieren']===10000,'watchdog without network'); + $h->receive();$v=json_decode($h->GetNetzfahrplanV4Vorschau(),true); + check($v['status']==='preview_only' && $v['previewBatteryW']===3000 && $v['canDispatch']===false,'read and show only'); + check(count($requests)===2 && str_ends_with($requests[1]['url'],'/ack'),'one read and one acknowledgement'); + $body=json_decode($requests[1]['options'][CURLOPT_POSTFIELDS],true);check($body['status']==='shadow_seen' && $body['step']===$response['plan']['points'][0]['time'],'ack is shadow_seen, never applied'); + $h->receive();check(count($requests)===2,'repeated click throttled'); + advance($h,60);$h->receive();check(count($requests)===3,'same plan step not acknowledged twice'); + advance($h,91);$h->update();$v=json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true); + check($v['status']==='unavailable' && $v['previewBatteryW']===null,'expired cache clears numeric targets, not fake zero'); + check(!str_contains($h->vars['NetzfahrplanV4Vorschau'],'3000'),'expired HTML clears previous setpoint'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();advance($h,61);$GLOBALS['getCode']=429;$h->receive(); + check($h->attrs['NetzfahrplanV4Empfang']==='{}' && json_decode($h->GetNetzfahrplanV4Vorschau(),true)['previewBatteryW']===null,'429 invalidates cache'); + check(!str_contains(json_encode($h->vars),str_repeat('x',43)),'no credentials in diagnosis'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->props['NetzfahrplanAktiv']=true;$h->receive();check(count($requests)===0,'legacy execution on blocks preview acquisition'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['response']['installationId']='00000000-0000-4000-8000-000000000001';$h->receive(); + check(count($requests)===1 && $h->attrs['NetzfahrplanV4Empfang']==='{}','foreign response not acknowledged'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['ackCode']=503;$h->receive();$v=json_decode($h->GetNetzfahrplanV4Vorschau(),true); + check($v['status']==='preview_only' && !$v['reception']['acknowledged'] && isset($v['reception']['acknowledgementError']),'ack failure is visible, no false execution confirmation'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->op['batteries'][0]['minSocPercent']=30.;$h->update(); + check(json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true)['previewBatteryW']===null,'local policy change invalidates cache'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->stop();check($h->attrs['NetzfahrplanV4Empfang']==='{}' && $h->timers['NetzfahrplanV4VorschauAktualisieren']===0,'apply/restart invalidates cached plan'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->licensed=false;$h->receive();check(count($requests)===0,'no license no read'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['lock']=true;$h->receive();check(count($requests)===0,'concurrent receiver does not send'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->receive();$h->props['NetzfahrplanV4EmpfangAktiv']=false;$h->update();check(json_decode($h->vars['NetzfahrplanV4VorschauJSON'],true)['previewBatteryW']===null,'disable clears displayed target'); + $source=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4EmpfangTrait.php'); + check(!str_contains($source, 'netzfahrplanV4EmpfangFormular'), 'receiver no longer owns a separate form panel'); + check(!preg_match('/\b(?:SendDataToChildren|SendDataToParent|sendeManagerdaten|regeln|SetStatus|EnableAction|IPS_SetProperty|IPS_ApplyChanges)\s*\(/',$source),'no control or external configuration call in receiver'); + check(!str_contains($source,"'status' => 'applied'"),'never acknowledges applied'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->configure(); + check($h->timers['NetzfahrplanV4EmpfangAktualisieren']===10000,'refresh has an independent timer'); + $h->session=['kind'=>'local_active_test_consent'];$h->receive();$saved=$h->attrs['NetzfahrplanV4Empfang']; + check($h->attrs['NetzfahrplanV4NaechsterAbruf']===$clock+45,'active refresh due after 45 seconds'); + advance($h,45);$GLOBALS['getCode']=503;$h->receive(); + check($h->attrs['NetzfahrplanV4Empfang']===$saved,'temporary error retains exact original active envelope'); + check($h->attrs['NetzfahrplanV4NaechsterAbruf']===$clock+15,'temporary error retries after 15 seconds'); + advance($h,46);$h->receive(); + check($h->attrs['NetzfahrplanV4Empfang']==='{}','active envelope older than 90 seconds is discarded'); + $h->stop();check($h->timers['NetzfahrplanV4EmpfangAktualisieren']===0,'stop disables independent refresh timer'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$h->session=['kind'=>'local_active_test_consent'];$h->receive(); + $future=json_decode($h->attrs['NetzfahrplanV4Empfang'],true);$future['receivedAt']=$clock+120; + $h->attrs['NetzfahrplanV4Empfang']=json_encode($future);advance($h,45);$GLOBALS['getCode']=503;$h->receive(); + check($h->attrs['NetzfahrplanV4Empfang']==='{}','future receipt cannot survive failed refresh'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['response']['pending']=['sequence'=>1];$GLOBALS['response']['fresh']=false;$h->receive(); + check($h->attrs['NetzfahrplanV4NaechsterAbruf']===$clock+15 && $h->attrs['NetzfahrplanV4Empfang']==='{}', + 'pending startup retries after 15 seconds without creating a usable envelope'); + $v=json_decode($h->GetNetzfahrplanV4Vorschau(),true); + check(($v['reception']['status']??null)==='rejected_or_unavailable','unavailable preview preserves receiver rejection detail'); + $h=fixture();$h->props['NetzfahrplanV4EmpfangAktiv']=true;$GLOBALS['getCode']=429;$h->receive(); + check($h->attrs['NetzfahrplanV4NaechsterAbruf']===$clock+60,'startup rate-limit response retains ordinary backoff'); + echo "Total: $count checks. Mocked IPS and HTTP only.\n"; +} diff --git a/tests/V4SeparatedObservation/checks.php b/tests/V4SeparatedObservation/checks.php new file mode 100644 index 0000000..45c13c1 --- /dev/null +++ b/tests/V4SeparatedObservation/checks.php @@ -0,0 +1,59 @@ +$r[$id],static fn()=>$now),$c,$p);} +[$c,$p,$r,$now]=separatedFixture();$x=take($c,$p,$r,$now); +ck($x['physicalLoad']['loadCandidateW']===6000.,'physical sources subtracted once'); +ck($x['solarAcAlternative']['loadCandidateW']===6000.,'solar AC terminal used without second battery subtraction'); +ck($x['solarOrigin']['reconstructedPvCandidateW']===1000.,'original mantissa scale and battery retained'); +ck($x['allocationModel']['unfilteredModelEvW']===2500.&&$x['allocationModel']['unfilteredModelSdlW']===500.,'unfiltered partition uses physical feedback'); +ck(!$x['canDispatch']&&!$x['trainingEligible']&&!$x['allocationModel']['separatelyMeasured'],'no measurement or control permission invented'); +$a=$r;$a[106]['value']=-9000;$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='consistent_candidate'&&$x['physicalLoad']['loadCandidateW']===6000.,'held virtual output cannot contaminate physical load'); +ck($x['allocationModel']['unfilteredModelEvW']===2500.&&!$x['allocationModel']['reportedAllocationWithinTolerance'],'held EV value only compared not reused'); +$a=$r;$a[107]['value']='never-export-this';$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='consistent_candidate','invalid SDL output does not veto physical load'); +ck(!str_contains(json_encode($x),'never-export-this'),'unknown string is not propagated'); +$a=$r;$a[108]['value']=42000;$a[109]['value']=-40000;$a[104]['value']=2000;$a[105]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===42000.&&$x['allocationModel']['unfilteredModelSdlW']===-40000.,'opposed commands do not divide by near-zero net'); +$a=$r;$a[108]['value']=0;$a[109]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===null&&$x['allocationModel']['unallocatedPhysicalW']===3000.,'idle physical power kept unallocated not invented as EV'); +$a=$r;$a[104]['value']=-3125;$a[105]['value']=0;$a[108]['value']=-10500;$a[109]['value']=0;$a[106]['value']=-8334;$a[107]['value']=0;$x=take($c,$p,$a,$now); +ck($x['allocationModel']['unfilteredModelEvW']===-3125.,'real export-example candidate is physical -3125 not held -8334'); +ck($x['allocationModel']['status']==='quality_hold'&&in_array('physical_request_tracking_error',$x['allocationModel']['issues'],true),'tracking disagreement remains visible'); +$a=$r;$a[108]['changed']=$now;$a[104]['updated']=$now-10;$x=take($c,$p,$a,$now); +ck(in_array('request_newer_than_feedback:req_ev',$x['allocationModel']['issues'],true),'post-command feedback not pretended'); +$a=$r;unset($a[108]['changed']);$x=take($c,$p,$a,$now); +ck(in_array('request_change_time_unknown:req_ev',$x['allocationModel']['issues'],true),'unknown command change time explicit'); +$a=$r;$a[105]['updated']=$now-70;$x=take($c,$p,$a,$now); +ck($x['storageFeedback']['status']==='quality_hold'&&$x['physicalLoad']['status']==='quality_hold','old physical source blocks physical feedback'); +ck($x['solarAcAlternative']['status']==='consistent_candidate','AC alternative does not inherit unrelated solar battery age'); +$a=$r;$a[111]['updated']=$now-70;$x=take($c,$p,$a,$now); +ck($x['solarAcAlternative']['status']==='quality_hold'&&$x['physicalLoad']['status']==='quality_hold','fresh solar derived output cannot hide old scale origin'); +$a=$r;$a[110]['value']=500;$a[111]['value']=1;$a[103]['value']=6000;$x=take($c,$p,$a,$now); +ck($x['solarOrigin']['acTerminalCandidateW']===5000.&&$x['solarAcAlternative']['loadCandidateW']===11000.,'signed scale applied exactly once'); +$a=$r;$a[110]['value']=-500;$a[111]['value']=0;$a[105]['value']=0;$a[103]['value']=0;$x=take($c,$p,$a,$now); +ck($x['solarOrigin']['reconstructedPvCandidateW']===-500.&&$x['solarAcAlternative']['loadCandidateW']===5500.,'negative AC not clamped or disguised as PV'); +foreach([true,'0',INF,NAN,-32768]as$invalid){$a=$r;$a[111]['value']=$invalid;$x=take($c,$p,$a,$now);ck($x['solarOrigin']['acTerminalCandidateW']===null,'invalid scale has no numeric fallback');} +$a=$r;$a[110]['value']=-32768;$x=take($c,$p,$a,$now);ck($x['solarOrigin']['acTerminalCandidateW']===null,'SunSpec invalid power sentinel rejected'); +$a=$r;$a[102]['updated']=$now-40;$x=take($c,$p,$a,$now); +ck($x['physicalLoad']['status']==='quality_hold'&&$x['storageFeedback']['status']==='consistent_candidate','slow PV does not veto independent storage feedback'); +$a=$r;$a[108]['updated']=$now-90;$x=take($c,$p,$a,$now);ck($x['allocationModel']['status']==='quality_hold','old request timestamp visible'); +$a=$r;$a[104]['updated']=$now+1;$x=take($c,$p,$a,$now);ck($x['storageFeedback']['status']==='quality_hold','future physical value refused'); +$a=$r;$a[104]['parentID']=999;$x=take($c,$p,$a,$now);ck($x['storageFeedback']['physicalStorageCandidateW']===null,'reused measurement identity not accepted'); +$a=$r;$a[101]['value']=-20000;$x=take($c,$p,$a,$now);ck($x['physicalLoad']['loadCandidateW']<0&&$x['physicalLoad']['status']==='quality_hold','negative load candidate retained as invalid not zero'); +$a=$r;foreach($a as &$v)$v['value']=0;unset($v);$x=take($c,$p,$a,$now);ck($x['physicalLoad']['loadCandidateW']===0.&&$x['physicalLoad']['status']==='consistent_candidate','real zero observation accepted'); +$raw=Capture::capture($c,static fn($id)=>$r[$id],static fn()=>$now);$raw['assessment']['baseLoadCandidateW']=999999.;$x=View::calculate($raw,$c,$p);ck($x['physicalLoad']['loadCandidateW']===6000.,'assessment independently recalculated'); +foreach(['installationId','mappingSha256','reportedInventorySha256']as$key){$bad=$raw;$bad[$key]='wrong';refuses(fn()=>View::calculate($bad,$c,$p),'wrong identity '.$key);} +$bad=$p;$bad['sdlRequestKey']='req_ev';refuses(fn()=>View::calculate($raw,$c,$bad),'duplicate request channels refused'); +$bad=$p;$bad['solarReference']['scaleKey']='soc';refuses(fn()=>View::calculate($raw,$c,$bad),'scale cannot be an arbitrary percent channel'); +$before=json_encode([$raw,$c,$p]);View::calculate($raw,$c,$p);ck(json_encode([$raw,$c,$p])===$before,'inputs not mutated'); +$src=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Messsicht.php');ck(!preg_match('/\b(?:IPS_|GetValue\(|SetValue\(|RequestAction\(|curl_|exec\s*\()/',$src),'pure model has no control or native calls'); +echo "TOTAL $n checks passed. Synthetic observations only.\n"; diff --git a/tests/V4SeparatedObservation/fixture.php b/tests/V4SeparatedObservation/fixture.php new file mode 100644 index 0000000..2f9ee5f --- /dev/null +++ b/tests/V4SeparatedObservation/fixture.php @@ -0,0 +1,14 @@ +1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004,'reportedInventorySha256'=>str_repeat('a',64), + 'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]],'extraSources'=>[]]; + $defs=[['grid','grid',3000],['pv_g','pv',5000],['pv_se','pv',1000],['bat_g','physical_storage',2000],['bat_se','physical_storage',1000],['ev','controlled_virtual',2500],['sdl','external_virtual',500]]; + $r=[]; + foreach($defs as $i=>[$key,$role,$value]){$id=101+$i;$c['accounting']['sources'][]=['key'=>$key,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'factorToW'=>1.,'maxAgeSeconds'=>60,'dependsOn'=>in_array($role,['controlled_virtual','external_virtual'],true)?['bat_g','bat_se']:[]];$r[$id]=['value'=>$value,'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key];} + foreach([['req_ev','W',2500],['req_sdl','W',500],['se_ac','raw_power',0],['se_sf','scale',0],['soc','percent',50]] as $i=>[$key,$unit,$value]){ + $id=108+$i;$c['extraSources'][]=['key'=>$key,'variableId'=>$id,'parentId'=>200,'ident'=>$key,'unit'=>$unit];$r[$id]=['value'=>$value,'updated'=>$now,'changed'=>$now-30,'parentID'=>200,'ident'=>$key];} + $p=['version'=>1,'evRequestKey'=>'req_ev','sdlRequestKey'=>'req_sdl','extraMaxAgeSeconds'=>60,'trackingToleranceW'=>100.,'solarReference'=>['pvKey'=>'pv_se','batteryKey'=>'bat_se','rawKey'=>'se_ac','scaleKey'=>'se_sf']]; + return [$c,$p,$r,$now]; +} diff --git a/tests/V4SeparatedObservation/native_checks.php b/tests/V4SeparatedObservation/native_checks.php new file mode 100644 index 0000000..db578a6 --- /dev/null +++ b/tests/V4SeparatedObservation/native_checks.php @@ -0,0 +1,61 @@ +['ModuleID'=>'{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}']];} +function IPS_GetChildrenIDs($parent){return array_keys(array_filter($GLOBALS['objects'],fn($o)=>$o['ParentID']===$parent));} +function IPS_GetObject($id){return $GLOBALS['objects'][$id];} +function IPS_GetVariable($id){return $GLOBALS['variables'][$id];} +function IPS_VariableExists($id){return isset($GLOBALS['variables'][$id]);} +function GetValue($id){return $GLOBALS['values'][$id];} +function SetValue($id,$v){if($id<50000||!is_string($v))throw new RuntimeException('FORBIDDEN source/actuator write');$GLOBALS['values'][$id]=$v;return true;} +function IPS_SemaphoreEnter($name,$wait){if($GLOBALS['locks'][$name]??false)return false;$GLOBALS['locks'][$name]=true;return true;} +function IPS_SemaphoreLeave($name){$GLOBALS['locks'][$name]=false;} +function makeObject($type){$id=++$GLOBALS['next'];$GLOBALS['objects'][$id]=['ParentID'=>0,'ObjectIdent'=>'','ObjectType'=>$type,'ObjectInfo'=>''];return $id;} +function IPS_CreateCategory(){return makeObject(0);} +function IPS_CreateVariable($type){ck($type===3,'only new textual observation variables');$id=makeObject(2);$GLOBALS['variables'][$id]=['VariableType'=>3];$GLOBALS['values'][$id]='';return $id;} +function IPS_CreateScript($type){if($type!==0)throw new RuntimeException('script type');$id=makeObject(3);$GLOBALS['scripts'][$id]='';return $id;} +function changeObject($id,$key,$v){if($id<50000)throw new RuntimeException('FORBIDDEN existing object change');$GLOBALS['objects'][$id][$key]=$v;return true;} +function IPS_SetParent($id,$v){return changeObject($id,'ParentID',$v);} +function IPS_SetIdent($id,$v){return changeObject($id,'ObjectIdent',$v);} +function IPS_SetName($id,$v){return changeObject($id,'ObjectName',$v);} +function IPS_SetInfo($id,$v){return changeObject($id,'ObjectInfo',$v);} +function IPS_GetScriptContent($id){return $GLOBALS['scripts'][$id];} +function IPS_SetScriptContent($id,$s){if($id<50000)throw new RuntimeException('existing script');$GLOBALS['scripts'][$id]=$s;return true;} +function IPS_SetScriptTimer($id,$seconds){if($id<50000||!isset($GLOBALS['scripts'][$id])||!in_array($seconds,[0,30],true))throw new RuntimeException('FORBIDDEN timer change');$GLOBALS['timers'][$id]=$seconds;return true;} +[$c,$p,$r,$ignored]=separatedFixture();$now=time();$objects=[];$variables=[];$values=[];$scripts=[];$timers=[];$locks=[];$next=50000; +foreach($r as$id=>$v){$objects[$id]=['ParentID'=>$v['parentID'],'ObjectIdent'=>$v['ident'],'ObjectType'=>2,'ObjectInfo'=>'original'];$variables[$id]=['VariableType'=>is_int($v['value'])?1:2,'VariableUpdated'=>$now,'VariableChanged'=>$now-30];$values[$id]=$v['value'];} +$sourceValues=$values;$base=sys_get_temp_dir().'/v4views-test-'.bin2hex(random_bytes(6));mkdir($base,0700);mkdir($base.'/data',0700); +$files=['source/libs/NetzfahrplanV4Bilanzierung.php','source/libs/NetzfahrplanV4Messaufnahme.php','source/libs/NetzfahrplanV4Messsicht.php','source/examples/V4SeparatedObservation/runtime.php','source/examples/V4SeparatedObservation/install.php']; +foreach($files as$name){$dest=$base.'/'.$name;if(!is_dir(dirname($dest)))mkdir(dirname($dest),0700,true);copy(__DIR__.'/../../'.substr($name,7),$dest);} +file_put_contents($base.'/INVENTORY.json','{"effectiveEvCapacityKwh":161.44,"evPowerKw":39}');$c['reportedInventorySha256']=hash_file('sha256',$base.'/INVENTORY.json'); +file_put_contents($base.'/config.json',json_encode($c,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION));file_put_contents($base.'/policy.json',json_encode($p,JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); +$manifest=['scope'=>'passive_separated_observation','hashes'=>[]];foreach(array_merge($files,['config.json','policy.json','INVENTORY.json'])as$name)$manifest['hashes'][$name]=hash_file('sha256',$base.'/'.$name);file_put_contents($base.'/MANIFEST.json',json_encode($manifest)); +require_once $base.'/source/examples/V4SeparatedObservation/install.php'; +try{ + ob_start();$report=v4viewsInstall($base);ob_end_clean();ck($report['status']==='observer_installed','native installation completes');$cat=$report['categoryId']; + ck(count($report['createdObjects'])===6,'one new category three read-only variables two scripts'); + ck($timers===[$report['scriptId']=>30],'only new passive observation timer'); + ck($report['firstObservation']['status']==='recorded','actual first record precedes timer'); + $journal=$report['firstObservation']['journal'];$original=file_get_contents($journal); + ck((fileperms($journal)&0777)===0644&&(fileperms(dirname($journal))&0077)===0,'selected new output readable inside private directory'); + foreach($sourceValues as$id=>$value)ck($values[$id]===$value,'source unchanged '.$id); + ob_start();$again=v4viewsInstall($base);ob_end_clean();ck($again['createdObjects']===[],'idempotent no duplicate scripts or variables'); + ck(str_starts_with(file_get_contents($journal),$original),'repeat preserves historical observations'); + $before=file_get_contents($journal);$locks['ENELIX.V4.SeparatedView.'.$c['installationId']]=true;$busy=v4viewsRun($base,$cat);ck($busy['status']==='busy'&&file_get_contents($journal)===$before,'parallel observer does not duplicate writes');$locks=[]; + unset($variables[101]);$failed=v4viewsRun($base,$cat);ck($failed['status']==='recorded','missing measurement explicitly recorded'); + $latest=json_decode(file_get_contents($base.'/data/LATEST.json'),true);ck($latest['views']['physicalLoad']['loadCandidateW']===null&&!$latest['controlEnabled'],'bad source never old target or fake zero');$variables[101]=['VariableType'=>1,'VariableUpdated'=>$now,'VariableChanged'=>$now-30]; + $before=file_get_contents($journal);v4viewsStop($base,$cat);ck($timers[$report['scriptId']]===0&&file_get_contents($journal)===$before,'stop only observer no deletion'); + ck(json_decode(file_get_contents($base.'/data/LATEST.json'),true)['status']==='stopped','stopped view clears latest numbers'); + $scripts[$report['scriptId']].='// parallel edit';ob_start();$bad=v4viewsInstall($base);ob_end_clean();ck($bad['status']==='needs_review'&&$timers[$report['scriptId']]===0,'parallel script edit not overwritten or activated'); + $old=file_get_contents($base.'/config.json');file_put_contents($base.'/config.json',$old.' ');try{v4viewsConfig($base);$refused=false;}catch(Throwable $e){$refused=true;}ck($refused,'changed config refused');file_put_contents($base.'/config.json',$old); + chmod($base,0755);clearstatcache();try{v4viewsConfig($base);$refused=false;}catch(Throwable $e){$refused=true;}ck($refused,'public stage refused before reading output');chmod($base,0700); + foreach(['runtime.php','install.php']as$file){$text=file_get_contents($base.'/source/examples/V4SeparatedObservation/'.$file);ck(!preg_match('/\b(?:RequestAction|IPS_ApplyChanges|IPS_SetProperty|IPS_SetConfiguration|MC_Reload|curl_exec|shell_exec)\s*\(/',$text),'no actuation or configuration calls in '.$file);} +}finally{ + foreach(new RecursiveIteratorIterator(new RecursiveDirectoryIterator($base,FilesystemIterator::SKIP_DOTS),RecursiveIteratorIterator::CHILD_FIRST)as$f){$f->isDir()&&!$f->isLink()?rmdir($f->getPathname()):unlink($f->getPathname());}rmdir($base); +} +echo "TOTAL $n native-mock checks passed. No actual Symcon calls.\n"; diff --git a/tests/fixtures/NativeV4Scenarios.php b/tests/fixtures/NativeV4Scenarios.php new file mode 100644 index 0000000..0fa58a0 --- /dev/null +++ b/tests/fixtures/NativeV4Scenarios.php @@ -0,0 +1,140 @@ + 1, 'NetzbezugEnergieVariableID' => 9, + 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, + 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $m['V4BezugszaehlerQuellen'] = [ + ['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, + 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; + $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, + 'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5, + 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, + 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; + $read = static function (int $id) use ($now): array { + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id], + 'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other']; + }; + $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; + $fail = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid-input rejection'); + }; + $cases = []; + $build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array { + return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now); + }; + $cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void { + $x = $build()['operation']; $b = $x['batteries'][0]; + $check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0 + && $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0); + }; + $cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void { + $x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null); + }; + $cases['empty_maps_are_objects'] = static function () use ($build, $check): void { + $x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW)); + }; + $cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void { + $m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0); + }; + $cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void { + $m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === null && $x['limits']['exportW'] === null); + }; + $cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void { + $m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = []; + for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; } + $x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0); + array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m)); + }; + $cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void { + $a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a)); + $m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]]; + $check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44); + }; + $cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void { + $a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a)); + }; + $cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void { + $c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c)); + }; + $cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void { + $c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c)); + }; + $cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void { + $r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { + $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); + }; + $cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void { + $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; + $b = $build(null, null, null, $r)['operation']['batteries'][0]; + $check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true); + }; + $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { + $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; + $check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0); + }; + $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { + $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); + $check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1); + }; + $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { + $check($build()['operation']['batteries'][0]['gridCharging'] === false); + $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); + }; + $ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now), + 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], + 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; + $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { + $x = $build(null, null, null, null, $ev); + $check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5); + }; + $cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void { + $ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev); + $check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0); + }; + $cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void { + $ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { + $check($build()['operation']['eventId'] === $build()['operation']['eventId']); + }; + $cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void { + unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m)); + }; + $cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void { + $v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203 + && !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/tests/fixtures/V4BezugszaehlerScenarios.php b/tests/fixtures/V4BezugszaehlerScenarios.php new file mode 100644 index 0000000..7e08052 --- /dev/null +++ b/tests/fixtures/V4BezugszaehlerScenarios.php @@ -0,0 +1,113 @@ + 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $samples = [ + 59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'], + 26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'], + ]; + $check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } }; + $reject = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid source/sample rejection'); + }; + $snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array { + $mm = $mm ?? $samples; + return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now); + }; + $cases = []; + $cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void { + $v = $snapshot(); + $check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']); + $check(!isset($v['measuredPeaks']) && !isset($v['quarterPast'])); + }; + $cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void { + $samples[26620]['value'] = 27.25; + $check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9); + }; + $cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void { + $samples[59607]['value'] = 0; + $check($snapshot(null, $samples)['totalKwh'] === 0.0); + }; + $cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void { + $check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources))); + }; + $cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001; + $check($old !== Meter::identitaet($sources)); + }; + $cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345; + $check($old !== Meter::identitaet($sources)); + }; + $cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); }; + $cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources)); + }; + $cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources)); + }; + $cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources)); + }; + $cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void { + unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources)); + }; + $cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void { + unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void { + foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) { + $samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples)); + } + }; + $cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void { + $calls = 0; + $read = static function (int $id) use ($samples, &$calls): array { + $v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v; + }; + $reject(static fn() => Meter::lesen($sources, $read, $now)); + }; + $cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void { + $samples[59607]['updated'] -= 1; + $v = $snapshot(null, $samples); + $check($v['sourceObservationFrom'] === gmdate('c', $now - 1)); + $check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now)); + $check(!$v['billingEvidence']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +}