From fc024fcfba376f797c1bbe60932b87b885279c9c Mon Sep 17 00:00:00 2001 From: ENELIX Agent Date: Fri, 2 Oct 2026 21:00:05 +0000 Subject: [PATCH] feat(application): integrate measured-load ingestion training and planner source --- Manager/module.php | 5 + docs/netplan-v4-application.md | 97 ++ libs/ManagerNetzfahrplanV4DatenTrait.php | 159 +++ libs/ManagerNetzfahrplanV4Trait.php | 3 + services/netplan-v4/.dockerignore | 5 + services/netplan-v4/.gitignore | 7 + services/netplan-v4/Dockerfile | 23 + services/netplan-v4/SOURCE_MANIFEST.json | 93 ++ services/netplan-v4/acceptance/Dockerfile.php | 7 + .../netplan-v4/acceptance/check_forecast.py | 36 + .../forecast-src/SOURCE_MANIFEST.json | 25 + .../acceptance/forecast-src/main.py | 908 ++++++++++++++++++ .../forecast-src/methods/__init__.py | 0 .../forecast-src/methods/battery_optimizer.py | 240 +++++ .../acceptance/forecast-src/methods/common.py | 32 + .../acceptance/forecast-src/methods/var_1.py | 66 ++ .../acceptance/forecast-src/methods/var_10.py | 12 + .../acceptance/forecast-src/methods/var_11.py | 12 + .../acceptance/forecast-src/methods/var_13.py | 13 + .../acceptance/forecast-src/methods/var_2.py | 169 ++++ .../acceptance/forecast-src/methods/var_21.py | 62 ++ .../acceptance/forecast-src/methods/var_22.py | 121 +++ .../acceptance/forecast-src/methods/var_23.py | 13 + .../acceptance/forecast-src/methods/var_3.py | 13 + .../forecast-src/model_isolation.py | 45 + .../forecast-src/netplan_v4_publisher.py | 108 +++ .../acceptance/forecast-src/shared_utils.py | 93 ++ .../forecast-src/soc_diagnostics.py | 66 ++ .../forecast-src/telemetry_quality.py | 106 ++ .../tests/test_battery_optimizer.py | 133 +++ .../forecast-src/tests/test_load_forecast.py | 66 ++ .../tests/test_model_isolation.py | 71 ++ .../tests/test_telemetry_integrity.py | 213 ++++ .../acceptance/php-src/SOURCE_MANIFEST.json | 9 + .../netplan-v4/acceptance/php-src/check.php | 45 + .../libs/ManagerNetzfahrplanV4Trait.php | 164 ++++ .../libs/NetzfahrplanV4Betriebsdaten.php | 219 +++++ .../libs/NetzfahrplanV4Bezugszaehler.php | 108 +++ .../tests/NetzfahrplanV4BetriebsdatenTest.php | 29 + .../tests/NetzfahrplanV4BezugszaehlerTest.php | 29 + .../tests/fixtures/NativeV4Scenarios.php | 140 +++ .../fixtures/V4BezugszaehlerScenarios.php | 113 +++ .../acceptance/read_native_forecasts.py | 100 ++ .../netplan-v4/approved_previous_assets.json | 5 + .../commissioning/APPLICATION_STATUS.md | 80 ++ .../application-source/RELEASE.json | 99 ++ .../application-source/server-dataset.json | 1 + .../commissioning/deploy_application.py | 131 +++ .../netplan-v4/compose.portal-bridge.yaml | 6 + services/netplan-v4/compose.yaml | 37 + .../netplan-v4/deploy_integrated_shadow.py | 175 ++++ services/netplan-v4/forecast_acceptance.py | 38 + services/netplan-v4/gui/netplan-v4.css | 1 + services/netplan-v4/gui/netplan-v4.html | 1 + services/netplan-v4/gui/netplan-v4.js | 107 +++ services/netplan-v4/install_hooks.py | 131 +++ .../integrations/NetzfahrplanV4.php | 71 ++ .../integrations/netplan-v4-bridge.mjs | 39 + .../integrations/netplan_v4_publisher.py | 108 +++ services/netplan-v4/netplan_v4/__init__.py | 1 + .../netplan-v4/netplan_v4/battery_model.py | 44 + .../netplan-v4/netplan_v4/controlled_trial.py | 162 ++++ services/netplan-v4/netplan_v4/domain.py | 170 ++++ .../netplan-v4/netplan_v4/forecast_quality.py | 21 + .../netplan_v4/measurement_pipeline.py | 449 +++++++++ .../netplan-v4/netplan_v4/meter_runtime.py | 145 +++ services/netplan-v4/netplan_v4/metering.py | 245 +++++ services/netplan-v4/netplan_v4/optimizer.py | 199 ++++ services/netplan-v4/netplan_v4/peak_policy.py | 173 ++++ .../netplan_v4/receiver_contract.py | 25 + services/netplan-v4/netplan_v4/selection.py | 53 + services/netplan-v4/netplan_v4/service.py | 432 +++++++++ services/netplan-v4/netplan_v4/store.py | 147 +++ services/netplan-v4/release_preflight.py | 128 +++ services/netplan-v4/requirements.txt | 5 + services/netplan-v4/run_tests.py | 21 + services/netplan-v4/runtime_preflight.py | 46 + .../netplan-v4/tests/manager_protocol.php | 24 + services/netplan-v4/tests/portal.test.mjs | 32 + .../tests/test_application_deployment.py | 39 + .../netplan-v4/tests/test_battery_recovery.py | 43 + .../netplan-v4/tests/test_container_access.py | 60 ++ .../netplan-v4/tests/test_controlled_trial.py | 120 +++ .../tests/test_controlled_trial_pipeline.py | 44 + services/netplan-v4/tests/test_delivery.py | 82 ++ .../tests/test_forecast_acceptance.py | 80 ++ .../netplan-v4/tests/test_forecast_quality.py | 31 + .../tests/test_integrated_deployment.py | 71 ++ .../tests/test_measurement_pipeline.py | 161 ++++ services/netplan-v4/tests/test_metering.py | 134 +++ .../netplan-v4/tests/test_peak_release.py | 155 +++ .../tests/test_receiver_contract.py | 45 + .../tests/test_release_preflight.py | 50 + services/netplan-v4/tests/test_v4.py | 184 ++++ tests/NetzfahrplanV4DatenTest.php | 21 + tests/V4ApplicationData/checks.php | 83 ++ 96 files changed, 8933 insertions(+) create mode 100644 docs/netplan-v4-application.md create mode 100644 libs/ManagerNetzfahrplanV4DatenTrait.php create mode 100644 services/netplan-v4/.dockerignore create mode 100644 services/netplan-v4/.gitignore create mode 100644 services/netplan-v4/Dockerfile create mode 100644 services/netplan-v4/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/Dockerfile.php create mode 100644 services/netplan-v4/acceptance/check_forecast.py create mode 100644 services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/forecast-src/main.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/__init__.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/common.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_1.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_10.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_11.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_13.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_2.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_21.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_22.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_23.py create mode 100644 services/netplan-v4/acceptance/forecast-src/methods/var_3.py create mode 100644 services/netplan-v4/acceptance/forecast-src/model_isolation.py create mode 100644 services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py create mode 100644 services/netplan-v4/acceptance/forecast-src/shared_utils.py create mode 100644 services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py create mode 100644 services/netplan-v4/acceptance/forecast-src/telemetry_quality.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py create mode 100644 services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py create mode 100644 services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json create mode 100644 services/netplan-v4/acceptance/php-src/check.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php create mode 100644 services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/fixtures/NativeV4Scenarios.php create mode 100644 services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php create mode 100644 services/netplan-v4/acceptance/read_native_forecasts.py create mode 100644 services/netplan-v4/approved_previous_assets.json create mode 100644 services/netplan-v4/commissioning/APPLICATION_STATUS.md create mode 100644 services/netplan-v4/commissioning/application-source/RELEASE.json create mode 100644 services/netplan-v4/commissioning/application-source/server-dataset.json create mode 100644 services/netplan-v4/commissioning/deploy_application.py create mode 100644 services/netplan-v4/compose.portal-bridge.yaml create mode 100644 services/netplan-v4/compose.yaml create mode 100644 services/netplan-v4/deploy_integrated_shadow.py create mode 100644 services/netplan-v4/forecast_acceptance.py create mode 100644 services/netplan-v4/gui/netplan-v4.css create mode 100644 services/netplan-v4/gui/netplan-v4.html create mode 100644 services/netplan-v4/gui/netplan-v4.js create mode 100644 services/netplan-v4/install_hooks.py create mode 100644 services/netplan-v4/integrations/NetzfahrplanV4.php create mode 100644 services/netplan-v4/integrations/netplan-v4-bridge.mjs create mode 100644 services/netplan-v4/integrations/netplan_v4_publisher.py create mode 100644 services/netplan-v4/netplan_v4/__init__.py create mode 100644 services/netplan-v4/netplan_v4/battery_model.py create mode 100644 services/netplan-v4/netplan_v4/controlled_trial.py create mode 100644 services/netplan-v4/netplan_v4/domain.py create mode 100644 services/netplan-v4/netplan_v4/forecast_quality.py create mode 100644 services/netplan-v4/netplan_v4/measurement_pipeline.py create mode 100644 services/netplan-v4/netplan_v4/meter_runtime.py create mode 100644 services/netplan-v4/netplan_v4/metering.py create mode 100644 services/netplan-v4/netplan_v4/optimizer.py create mode 100644 services/netplan-v4/netplan_v4/peak_policy.py create mode 100644 services/netplan-v4/netplan_v4/receiver_contract.py create mode 100644 services/netplan-v4/netplan_v4/selection.py create mode 100644 services/netplan-v4/netplan_v4/service.py create mode 100644 services/netplan-v4/netplan_v4/store.py create mode 100644 services/netplan-v4/release_preflight.py create mode 100644 services/netplan-v4/requirements.txt create mode 100644 services/netplan-v4/run_tests.py create mode 100644 services/netplan-v4/runtime_preflight.py create mode 100644 services/netplan-v4/tests/manager_protocol.php create mode 100644 services/netplan-v4/tests/portal.test.mjs create mode 100644 services/netplan-v4/tests/test_application_deployment.py create mode 100644 services/netplan-v4/tests/test_battery_recovery.py create mode 100644 services/netplan-v4/tests/test_container_access.py create mode 100644 services/netplan-v4/tests/test_controlled_trial.py create mode 100644 services/netplan-v4/tests/test_controlled_trial_pipeline.py create mode 100644 services/netplan-v4/tests/test_delivery.py create mode 100644 services/netplan-v4/tests/test_forecast_acceptance.py create mode 100644 services/netplan-v4/tests/test_forecast_quality.py create mode 100644 services/netplan-v4/tests/test_integrated_deployment.py create mode 100644 services/netplan-v4/tests/test_measurement_pipeline.py create mode 100644 services/netplan-v4/tests/test_metering.py create mode 100644 services/netplan-v4/tests/test_peak_release.py create mode 100644 services/netplan-v4/tests/test_receiver_contract.py create mode 100644 services/netplan-v4/tests/test_release_preflight.py create mode 100644 services/netplan-v4/tests/test_v4.py create mode 100644 tests/NetzfahrplanV4DatenTest.php create mode 100644 tests/V4ApplicationData/checks.php diff --git a/Manager/module.php b/Manager/module.php index b2b75ac..82ab561 100644 --- a/Manager/module.php +++ b/Manager/module.php @@ -244,6 +244,7 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->stoppeNetzfahrplanV4Vorschau(); $this->initialisiereLizenzInstallationID(); + $this->konfiguriereNetzfahrplanV4Daten(); $this->aktualisiereVariablen(); try { @@ -480,6 +481,10 @@ class Manager extends IPSModule implements ManagerSchnittstelle $this->V4ManagerTestStoppen(); return; + case 'NetzfahrplanV4DatenErfassen': + $this->ErfasseNetzfahrplanV4Daten(); + return; + case 'NetzfahrplanV4Senden': $this->sendeNetzfahrplanV4(); return; diff --git a/docs/netplan-v4-application.md b/docs/netplan-v4-application.md new file mode 100644 index 0000000..bf964b0 --- /dev/null +++ b/docs/netplan-v4-application.md @@ -0,0 +1,97 @@ +# V4 application data and forecast integration + +## Implemented application path + +`ManagerNetzfahrplanV4DatenTrait` records the configured raw power/SOC/counter sources +inside the existing Manager. This replaces the need for permanent standalone +observation categories once the native delivery path has been accepted. It never +issues device commands. New acquisition properties default disabled. + +The private outbox at `data/enelix-v4/` is append-only and delivery +is acknowledged by dataset and capture time. Unacknowledged data survives network +failures and 429 responses. The native payload budget is below the existing portal +1 MiB request-body limit. Existing authentication and V4 rate limiting are reused. + +Server application source is now versioned under `services/netplan-v4`, rather than +existing only as an untracked server working tree. No live measurements, databases, +credentials, dependency binaries or settings exports are included in this directory. + +The existing worker handles per-plant dataset ingestion, physical five-minute +rollup, model training, model storage and corrected-load substitution into the V4 +optimizer. It continues to consume the existing PV forecasts and price/operation +inputs. The public device route can append measurements but cannot alter source +meaning, register a dataset, change live permissions or arm a controlled trial. + +## Explicit settings and model meaning + +- `forecastSource=legacy` preserves the existing forecast input by default. +- `forecastSource=corrected_profile` plus `measurementDataset` uses the trained + corrected physical load. Missing models are reported; legacy house values are + not substituted silently. +- `trainingCadence=daily|weekly` is now connected to real profile fitting in the + application worker, independently from the optimizer refresh frequency. +- Corrected load profiles are tagged `physical-profile-v1`: robust daily profile, + recent-day profile and weekday/weekend profile paired with PV families 3/13/23. + They are not claimed to be the previous load models unchanged. Economic automatic + selection still requires the separate, not yet finished cost-replay integration. +- Initial models are explicitly bootstrap models. Later candidates record causal + holdout errors; an overlapping validation window cannot certify a promotion. +- Current SDL request persistence is only a labelled shadow scenario. Unknown or + stale SDL is not silently zero, and the scenario is not a guaranteed future + schedule or a robust SDL-aware production control policy. + +## Lihrenmoos package scope + +Effective EV capacity 161.44 kWh and power 39 kW remain unchanged; SDL reserve is +already excluded. The prepared dataset `lihrenmoos-physical-v1` uses a configured +physical estimate with SolarEdge signed terminal power counted once. It requires +at least 24 equivalent usable hours before the first bootstrap profile; the +history window is 28 days. Coverage policy 95% with maximum 10-second unsupported +portion is an explicit modelling assumption. Gaps and source ages remain visible. +It is not an independent electrical metering-boundary proof. + +Server entry point on the service host: + + python3 /home/agent/services/netplan-v4-shadow/commissioning/deploy_application.py \ + --plant e3a08f9e-af12-4695-99bd-8b51c0520021 --apply + +Default without --apply checks source only. The explicit apply command rebuilds +and tests V4 in Python 3.11, tests portal routing, backs up the V4 database and +recreates only V4 and the portal. It configures the dataset but DOES NOT select it +for the existing plan automatically. No forecasts/tariff importer or controllers +are restarted by the server command. A previous-image rollback is prepared. + +Then, inside Lihrenmoos Symcon: + + require '/srv/agent/netplan-v4-application-build/install.php'; + +This is a data-only patch of the currently installed passive manager, with file +backup and library reload. It does NOT install the development control-trial +changes in Manager/Batterie. The first runtime invocation imports at most 48h of +the existing 23-channel observer as a durable backlog, preserving the original +files, then activates acquisition and delivery. If Symcon module registration is +not immediately available, the installer reports that a single repeat is needed. + +The two temporary standalone samplers are not stopped automatically by this +initial package; disable them only after native batch acceptance and history +continuity are confirmed. This package creates no new root diagnostic category. + +## Validation performed before deployment + +219 Python tests passed in the isolated host QA runtime, including actual synthetic +measurement -> profile -> existing optimizer -> stored shadow plan. 16 Node proxy +checks passed. 21 PHP data-path checks and 5 installer scenarios passed with mocked +IPS/HTTP and temporary files on the test host. Full staged PHP syntax passed. +No new container or new manager data integration has yet been run in production. + +Evidence: service `APPLICATION_PIPELINE_TEST_RESULTS.txt`, +`APPLICATION_PORTAL_TEST_RESULTS.txt`, `commissioning/application-source/RELEASE.json`; +test host `netplan-v4-application-build/PREPARATION.json` and `TEST_RESULTS.txt`. + +## Remaining product scope (do not disguise as complete) + +This integrates application data and load forecasting, not completed production +commissioning. Economic replay-based automatic family selection, full corrected +feedback/control integration, long-term outbox/server retention and real live/ +failure acceptance remain open. Existing trial gates are unchanged. No sensor, +accounting or hardware evidence identifier is fabricated by data ingestion. diff --git a/libs/ManagerNetzfahrplanV4DatenTrait.php b/libs/ManagerNetzfahrplanV4DatenTrait.php new file mode 100644 index 0000000..16001c6 --- /dev/null +++ b/libs/ManagerNetzfahrplanV4DatenTrait.php @@ -0,0 +1,159 @@ +RegisterPropertyBoolean('NetzfahrplanV4MessdatenAktiv', false); + $this->RegisterPropertyString('NetzfahrplanV4Messkonfiguration', '{}'); + $this->RegisterPropertyString('NetzfahrplanV4Datensatz', ''); + $this->RegisterAttributeString('NetzfahrplanV4DatenCursor', '{}'); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->RegisterVariableString('NetzfahrplanV4Datenstatus', 'V4 Messdaten und Modellversorgung', '', 199); + $this->RegisterTimer('NetzfahrplanV4DatenErfassen', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4DatenErfassen', true);"); + } + + private function konfiguriereNetzfahrplanV4Daten(): void + { + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 0); + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + try { + $this->v4DatenKonfiguration(); + $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 30000); + } catch (\Throwable $e) { + $this->v4DatenStatus(['status'=>'configuration_error','errorType'=>get_class($e)]); + } + } + + private function v4DatenKonfiguration(): array + { + $id=$this->ReadAttributeString('LizenzInstallationID'); + $dataset=$this->ReadPropertyString('NetzfahrplanV4Datensatz'); + if (!preg_match('/^[a-zA-Z0-9_-]{1,80}$/D',$dataset) + || !preg_match('/^[0-9a-f-]{36}$/D',$id)) throw new \RuntimeException('Dataset/installation not configured'); + $c=json_decode($this->ReadPropertyString('NetzfahrplanV4Messkonfiguration'),true,64,JSON_THROW_ON_ERROR); + $c=NetzfahrplanV4Messaufnahme::configuration($c); + if ($c['installationId']!==$id || ($c['managerId']??null)!==$this->InstanceID) throw new \RuntimeException('Measurement mapping belongs to another manager'); + return $c; + } + + private function v4DatenVerzeichnis(): string + { + $base=rtrim(IPS_GetKernelDir(),'/').'/data/enelix-v4'; + $directory=$base.'/'.$this->ReadAttributeString('LizenzInstallationID'); + // Fixed application-owned location; no configurable path or arbitrary file access. + foreach ([$base,$directory] as $p) { + if (is_link($p)) throw new \RuntimeException('Measurement directory symlink refused'); + if (!is_dir($p) && !mkdir($p,0700,true) && !is_dir($p)) throw new \RuntimeException('Measurement directory unavailable'); + if ((fileperms($p)&0007)!==0) throw new \RuntimeException('Measurement directory must remain private'); + } + return $directory; + } + + private function v4DatenStatus(array $data): void + { + $this->SetValue('NetzfahrplanV4Datenstatus',json_encode($data+['checkedAt'=>gmdate('c'), + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'controlEnabled'=>false],JSON_THROW_ON_ERROR)); + } + + public function ErfasseNetzfahrplanV4Daten(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; + $lock='ENELIX.V4.ApplicationData.'.$this->InstanceID; + if (!IPS_SemaphoreEnter($lock,0)) return; + try { + $c=$this->v4DatenKonfiguration(); + $reader=static function(int $id):array { + if (!IPS_VariableExists($id)) throw new \RuntimeException('Missing measurement'); + $v=IPS_GetVariable($id);$o=IPS_GetObject($id); + if (!in_array($v['VariableType'],[1,2],true)) throw new \RuntimeException('Numeric source required'); + return ['value'=>GetValue($id),'updated'=>(int)$v['VariableUpdated'],'changed'=>(int)$v['VariableChanged'], + 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; + }; + $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); + $dir=$this->v4DatenVerzeichnis(); + NetzfahrplanV4Messaufnahme::append($dir,$r); + $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), + 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + $last=$this->ReadAttributeInteger('NetzfahrplanV4DatenVersuch'); + $failures=min(5,$this->ReadAttributeInteger('NetzfahrplanV4DatenFehlerzahl')); + $retry=min(900,60*(2**$failures)); + if (time()>=$last+$retry) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch',time()); + try { + $result['delivery']=$this->v4DatenUebertragen($dir); + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',0); + } catch (\Throwable $e) { + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',$failures+1); + $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e),'dataRetained'=>true]; + } + } else $result['delivery']=['status'=>'scheduled','nextAttemptAt'=>gmdate('c',$last+$retry)]; + $this->v4DatenStatus($result); + } catch (\Throwable $e) { + // Sensor/configuration exceptions never enter the actuator or module-status path. + $this->v4DatenStatus(['status'=>'capture_error','errorType'=>get_class($e)]); + } finally { IPS_SemaphoreLeave($lock); } + } + + /** Returns only complete records from the append-only prefix; cursor moves after matching ACK. */ + private function v4DatenUebertragen(string $dir): array + { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) throw new \RuntimeException('Forecast licence missing'); + $cursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $day=$cursor['day']??'';$offset=$cursor['offset']??0; + if (!is_string($day) || ($day!==''&&!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day)) || !is_int($offset)||$offset<0) throw new \RuntimeException('Invalid data cursor'); + $files=glob($dir.'/raw-*.jsonl'); + if ($files===false||count($files)>400) throw new \RuntimeException('Measurement journal bounds exceeded'); + sort($files,SORT_STRING); + $records=[];$candidate=$cursor;$bytes=0;$lastTime=null; + foreach ($files as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||($day!==''&&$name<$day)) continue; + if (is_link($file)||!is_file($file)||realpath($file)!==$file) throw new \RuntimeException('Invalid measurement journal'); + $h=fopen($file,'rb');if($h===false)throw new \RuntimeException('Cannot open measurement journal'); + try { + $size=fstat($h)['size'];$position=$name===$day?$offset:0; + if ($position>$size||fseek($h,$position)!==0) throw new \RuntimeException('Measurement journal shortened'); + while (ftell($h)<$size) { + $line=fgets($h,min(262146,$size-ftell($h)+1)); + if ($line===false||!str_ends_with($line,"\n")) break; + if (strlen($line)>262144) throw new \RuntimeException('Oversized measurement'); + if (count($records)>=120||$bytes+strlen($line)>900000) break 2; + $r=json_decode($line,false,64,JSON_THROW_ON_ERROR); + if (!is_object($r)||($r->installationId??null)!==$this->ReadAttributeString('LizenzInstallationID') + ||($r->kind??null)!=='raw_accounting_capture') throw new \RuntimeException('Journal identity invalid'); + $records[]=$r;$bytes+=strlen($line);$lastTime=$r->capturedAt; + $candidate=['day'=>$name,'offset'=>ftell($h)]; + } + } finally { fclose($h); } + } + if ($records===[]) return ['status'=>'up_to_date']; + $id=$this->ReadAttributeString('LizenzInstallationID');$token=$this->ReadAttributeString('PrognoseInstallationsToken'); + if ($token==='') throw new \RuntimeException('Existing device token unavailable'); + $payload=json_encode(['version'=>1,'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'records'=>$records],JSON_THROW_ON_ERROR); + $h=curl_init('https://license.enelix.ch/api/v1/installations/'.rawurlencode($id).'/prognosis/planner-v4/measurements'); + if ($h===false) throw new \RuntimeException('Cannot prepare data transfer'); + try { + curl_setopt_array($h,[CURLOPT_POST=>true,CURLOPT_RETURNTRANSFER=>true,CURLOPT_FOLLOWLOCATION=>false, + CURLOPT_CONNECTTIMEOUT=>2,CURLOPT_TIMEOUT=>8,CURLOPT_SSL_VERIFYPEER=>true,CURLOPT_SSL_VERIFYHOST=>2, + CURLOPT_HTTPHEADER=>['Content-Type: application/json','Accept: application/json','Authorization: Bearer '.$token], + CURLOPT_POSTFIELDS=>$payload]); + $answer=curl_exec($h);$code=(int)curl_getinfo($h,CURLINFO_HTTP_CODE); + if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged'); + $ack=json_decode($answer,true,32,JSON_THROW_ON_ERROR); + if (!in_array($ack['status']??null,['stored','duplicate'],true) + ||($ack['datasetId']??null)!==$this->ReadPropertyString('NetzfahrplanV4Datensatz') + ||($ack['acceptedThrough']??null)!==$lastTime) throw new \RuntimeException('Measurement acknowledgement does not match batch'); + $this->WriteAttributeString('NetzfahrplanV4DatenCursor',json_encode($candidate,JSON_THROW_ON_ERROR)); + return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime]; + } finally { curl_close($h); } + } +} diff --git a/libs/ManagerNetzfahrplanV4Trait.php b/libs/ManagerNetzfahrplanV4Trait.php index 1d32ca6..31aecc6 100644 --- a/libs/ManagerNetzfahrplanV4Trait.php +++ b/libs/ManagerNetzfahrplanV4Trait.php @@ -8,14 +8,17 @@ use RuntimeException; use Throwable; require_once __DIR__ . '/ManagerNetzfahrplanV4EmpfangTrait.php'; +require_once __DIR__ . '/ManagerNetzfahrplanV4DatenTrait.php'; /** Shadow telemetry only. Does not read/apply a V4 schedule or write actuators. */ trait ManagerNetzfahrplanV4Trait { use ManagerNetzfahrplanV4EmpfangTrait; + use ManagerNetzfahrplanV4DatenTrait; private function registriereNetzfahrplanV4(): void { $this->registriereNetzfahrplanV4Empfang(); + $this->registriereNetzfahrplanV4Daten(); $this->RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); diff --git a/services/netplan-v4/.dockerignore b/services/netplan-v4/.dockerignore new file mode 100644 index 0000000..7264271 --- /dev/null +++ b/services/netplan-v4/.dockerignore @@ -0,0 +1,5 @@ +data/ +.git/ +__pycache__/ +**/__pycache__/ +*.log diff --git a/services/netplan-v4/.gitignore b/services/netplan-v4/.gitignore new file mode 100644 index 0000000..eb476e8 --- /dev/null +++ b/services/netplan-v4/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +data/ +*-reports/ +application-releases/ +.env +*.env diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile new file mode 100644 index 0000000..b98928f --- /dev/null +++ b/services/netplan-v4/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY netplan_v4 ./netplan_v4 +COPY tests ./tests +COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile . +COPY integrations ./integrations +COPY gui ./gui +COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./ +COPY acceptance ./acceptance +COPY commissioning/deploy_application.py ./commissioning/deploy_application.py +# Host sources can be 0600/0700. COPY makes them root-owned. +# Normalize only packaged application code; never change host secrets or sockets. +RUN find /app -type d -exec chmod 0755 {} + \ + && find /app -type f -exec chmod 0644 {} + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results +USER 1000:1000 +# Fail the build early if the actual unprivileged runtime cannot read the code. +RUN python /app/runtime_preflight.py +CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"] diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json new file mode 100644 index 0000000..878da98 --- /dev/null +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -0,0 +1,93 @@ +{ + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "packagedAt": "2026-10-02T20:57:00.387145+00:00", + "runtimeChanged": false +} diff --git a/services/netplan-v4/acceptance/Dockerfile.php b/services/netplan-v4/acceptance/Dockerfile.php new file mode 100644 index 0000000..d7aa137 --- /dev/null +++ b/services/netplan-v4/acceptance/Dockerfile.php @@ -0,0 +1,7 @@ +FROM php:8.3-cli +WORKDIR /check +COPY php-src/ ./ +RUN find /check -type d -exec chmod 0755 {} + \ + && find /check -type f -exec chmod 0644 {} + +USER 1000:1000 +CMD ["php", "/check/check.php"] diff --git a/services/netplan-v4/acceptance/check_forecast.py b/services/netplan-v4/acceptance/check_forecast.py new file mode 100644 index 0000000..f42cabb --- /dev/null +++ b/services/netplan-v4/acceptance/check_forecast.py @@ -0,0 +1,36 @@ +"""Offline candidate-forecast tests: no telemetry, model loading or publication. +Run in an unprivileged, networkless test container with no live data volumes. +""" +from pathlib import Path +import ast +import hashlib +import json +import sys +import unittest + + +def main(): + root = Path('/app/forecast') + sys.path.insert(0, str(root)) + manifest = json.loads((root / 'SOURCE_MANIFEST.json').read_text()) + for relative, expected in manifest.items(): + p = root / relative + if Path(relative).is_absolute() or '..' in Path(relative).parts or not p.resolve().is_relative_to(root): + raise ValueError('Unsafe manifest path') + raw = p.read_bytes() + if hashlib.sha256(raw).hexdigest() != expected: + raise ValueError('Test image source checksum mismatch') + if p.suffix == '.py': + ast.parse(raw, filename=str(p)) + import pandas, numpy, scipy, sklearn + versions = {'python': sys.version.split()[0], 'pandas': pandas.__version__, + 'numpy': numpy.__version__, 'scipy': scipy.__version__, 'sklearn': sklearn.__version__} + print('Candidate forecast runtime:', json.dumps(versions), flush=True) + suite = unittest.defaultTestLoader.discover(str(root / 'tests')) + result = unittest.TextTestRunner(verbosity=2).run(suite) + print('Forecast tests only; no live data, no publication, no training job.', flush=True) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..f35881c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/SOURCE_MANIFEST.json @@ -0,0 +1,25 @@ +{ + "main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" +} diff --git a/services/netplan-v4/acceptance/forecast-src/main.py b/services/netplan-v4/acceptance/forecast-src/main.py new file mode 100644 index 0000000..02748d7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/main.py @@ -0,0 +1,908 @@ +from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts +from model_isolation import collect_predictions +import os +import sqlite3 +import json +import time +import datetime +import traceback +import warnings +import subprocess +import sys +import threading + +import numpy as np +import pandas as pd +import pytz +from influxdb_client import InfluxDBClient, Point, WritePrecision +from influxdb_client.client.write_api import SYNCHRONOUS +from soc_diagnostics import battery_soc_points +from telemetry_quality import require_recent_telemetry, sanitize_measured_frame +try: + from influxdb_client.client.warnings import MissingPivotFunction + warnings.simplefilter("ignore", MissingPivotFunction) +except Exception: + pass + +import methods.var_1 as v1 +import methods.var_2 as v2 +import methods.var_3 as v3 +import methods.var_10 as v10 +import methods.var_11 as v11 +import methods.var_13 as v13 +import methods.var_21 as v21 +import methods.var_22 as v22 +import methods.var_23 as v23 + +INFLUX_URL = os.getenv("INFLUX_URL", "http://influxdb:8086") +INFLUX_TOKEN = os.environ["INFLUX_TOKEN"] +INFLUX_ORG = os.getenv("INFLUX_ORG", "belevo") +INFLUX_BUCKET = os.getenv("INFLUX_BUCKET", "energy_data") +SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "/app/data/users.db") +HISTORY_START = os.getenv("FORECAST_HISTORY_START", "1970-01-01T00:00:00Z") +QUALITY_LOOKBACK_DAYS = int(os.getenv("FORECAST_QUALITY_LOOKBACK_DAYS", "14")) +LOCAL_TZ = pytz.timezone(os.getenv("TZ", "Europe/Zurich")) +INFLUX_TIMEOUT_MS = int(os.getenv("FORECAST_INFLUX_TIMEOUT_MS", "120000")) +FORECAST_TIMEOUT_SECONDS = int(os.getenv("FORECAST_RUN_TIMEOUT_SECONDS", "900")) +FORECAST_HORIZON_HOURS = max(24, min(72, int(os.getenv("FORECAST_HORIZON_HOURS", "48")))) +TRAINING_TIMEOUT_SECONDS = int(os.getenv("FORECAST_TRAINING_TIMEOUT_SECONDS", "3600")) +FORECAST_STALE_SECONDS = int(os.getenv("FORECAST_STALE_SECONDS", "5400")) +WATCHDOG_INTERVAL_SECONDS = int(os.getenv("FORECAST_WATCHDOG_INTERVAL_SECONDS", "300")) +TRAINING_HOUR = int(os.getenv("FORECAST_TRAINING_HOUR", "2")) +LAST_SUCCESS_PATH = os.getenv("FORECAST_LAST_SUCCESS_PATH", "/tmp/forecast_engine_last_success.json") +LAST_TRAINING_PATH = os.getenv("FORECAST_LAST_TRAINING_PATH", "/app/data/forecast_training_status.json") +last_trained_day = None +_FORECAST_PROCESS_LOCK = threading.Lock() +_TRAINING_THREAD = None + +MODEL_MODULES = {1: v1, 2: v2, 3: v3, 10: v10, 11: v11, 13: v13, 21: v21, 22: v22, 23: v23} +QUALITY_TARGETS = {1: "PV", 10: "PV", 21: "PV", 2: "Hausverbrauch", 11: "Hausverbrauch", 22: "Hausverbrauch"} + + +def active(config, n): + return bool(int(config.get(f"prog_var_{n}", config.get(f"var_{n}", 0)) or 0)) + + +def get_configs(): + conn = sqlite3.connect(SQLITE_DB_PATH) + conn.row_factory = sqlite3.Row + columns = {row["name"] for row in conn.execute("PRAGMA table_info(anlagen_meta)")} + if "batt_grid_charging_enabled" not in columns: + conn.execute( + "ALTER TABLE anlagen_meta ADD COLUMN batt_grid_charging_enabled INTEGER NOT NULL DEFAULT 0" + ) + conn.commit() + rows = conn.execute("SELECT * FROM anlagen_meta").fetchall() + conn.close() + configs = [] + for r in rows: + d = dict(r) + try: + d["daecher"] = json.loads(d.get("daecher") or "[]") + except Exception: + d["daecher"] = [] + for key, default in [ + ("ac_leistung", 10.0), + ("batt_capacity_kwh", 0.0), + ("batt_power_kw", 0.0), + ("tarif_bezug_fest", 0.30), + ("tarif_einspeisung_fest", 0.10), + ("tarif_peak_fest", 5.0), + ]: + raw_value = d.get(key) + d[key] = float(default if raw_value is None or raw_value == "" else raw_value) + configs.append(d) + return configs + + +def _time_literal(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _query_df(query): + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + try: + df = client.query_api().query_data_frame(org=INFLUX_ORG, query=query) + finally: + client.close() + if isinstance(df, list): + df = pd.concat(df, ignore_index=True) if df else pd.DataFrame() + if df is None or df.empty or "_time" not in df.columns: + return pd.DataFrame() + df["_time"] = pd.to_datetime(df["_time"], utc=True).dt.tz_localize(None) + return df + + +def _pivot_frame(df, fields): + if df.empty: + return pd.DataFrame() + out = df.set_index("_time") + keep = [c for c in fields if c in out.columns] + out = out[keep] if keep else pd.DataFrame(index=out.index) + out = out.apply(pd.to_numeric, errors="coerce") + out = out[~out.index.duplicated(keep="last")] + return out.sort_index().resample("5min").mean(numeric_only=True) + + +def _tariff_frame(df, config=None): + if df is None or df.empty: + return pd.DataFrame() + df = df.copy() + if "_time" in df.columns: + df["_time"] = pd.to_datetime(df["_time"], errors="coerce") + df = df.dropna(subset=["_time"]).set_index("_time") + if df.empty: + return pd.DataFrame() + + price = df["price_chf_kwh"] if "price_chf_kwh" in df.columns else df.get("_value") + if price is None: + return pd.DataFrame() + price = pd.to_numeric(price, errors="coerce") + model = df.get("tariff_model", pd.Series("", index=df.index)).astype(str).str.lower() + typ = df.get("type", pd.Series("", index=df.index)).astype(str).str.lower() + tariff_name = df.get("tariff_name", pd.Series("", index=df.index)).astype(str).str.lower() + provider = df.get("provider", pd.Series("", index=df.index)).astype(str).str.lower() + key = (model + " " + tariff_name + " " + provider).str.lower() + + rows = pd.DataFrame({"price": price, "key": key, "type": typ}, index=df.index) + rows = rows[pd.notna(rows["price"])].sort_index() + if rows.empty: + return pd.DataFrame() + + cfg = config or {} + import_choice = str(cfg.get("tarif_bezug", "") or "").lower() + export_choice = str(cfg.get("tarif_einspeisung", "") or "").lower() + + out = pd.DataFrame(index=rows.index.unique().sort_values()) + + import_base = ( + rows["type"].str.contains("consumption|import|bezug", regex=True, na=False) + | rows["key"].str.contains("dynamic|dynamisch|home|business", regex=True, na=False) + ) + if "business" in import_choice: + import_mask = import_base & rows["key"].str.contains("business|gewerbe|commercial", regex=True, na=False) + elif "home" in import_choice or "privat" in import_choice: + import_mask = import_base & rows["key"].str.contains("home|privat|private", regex=True, na=False) + elif "dynam" in import_choice: + import_mask = import_base + else: + import_mask = pd.Series(False, index=rows.index) + if not import_mask.any() and "dynam" in import_choice: + import_mask = import_base + + export_base = ( + rows["type"].str.contains("feed|einspeis|export", regex=True, na=False) + | rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|feed", regex=True, na=False) + ) + if "referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice: + export_mask = export_base & rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|belevo", regex=True, na=False) + else: + export_mask = rows["key"].str.contains("standard_feedin|ckw statisch", regex=True, na=False) & export_base + if not export_mask.any() and ("referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice): + export_mask = export_base + + if import_mask.any(): + out["import_price"] = rows.loc[import_mask, "price"].groupby(level=0).last() + if export_mask.any(): + out["export_price"] = rows.loc[export_mask, "price"].groupby(level=0).last() + + if out.empty: + return out + return out.sort_index().resample("5min").mean().ffill().bfill() + +def fetch_influx_frames(config, training): + aid = config["anlagen_id"] + start = HISTORY_START if training else "-14d" + future_stop = _time_literal(datetime.datetime.utcnow() + datetime.timedelta(hours=FORECAST_HORIZON_HOURS)) + + q_tel = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "PV" or r["_field"] == "Hausverbrauch" or r["_field"] == "Netzleistung" or r["_field"] == "SOC") + |> filter(fn: (r) => not exists r["data_type"] or (r["data_type"] != "forecast" and r["data_type"] != "forecast_snapshot")) + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false, timeSrc: "_start") + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_tel = _pivot_frame(_query_df(q_tel), ["PV", "Hausverbrauch", "Netzleistung", "SOC"]) + + q_wea = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: {start}, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "weather_forecast") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + df_wea = _pivot_frame(_query_df(q_wea), ["temp_c", "temperature", "cloud", "cloud_cover", "precip_mm", "wind_kph", "chance_of_snow"]) + if "temperature" in df_wea.columns and "temp_c" not in df_wea.columns: + df_wea["temp_c"] = df_wea["temperature"] + if "cloud_cover" in df_wea.columns and "cloud" not in df_wea.columns: + df_wea["cloud"] = df_wea["cloud_cover"] + if "cloud" in df_wea.columns and "cloud_cover" not in df_wea.columns: + df_wea["cloud_cover"] = df_wea["cloud"] + + q_tar = f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -7d, stop: {future_stop}) + |> filter(fn: (r) => r["_measurement"] == "tariffs") + |> filter(fn: (r) => r["_field"] == "price_chf_kwh") +''' + df_tar = _tariff_frame(_query_df(q_tar), config) + return df_tel, df_wea, df_tar + + +def _consistent_tail(df): + if df.empty or "PV" not in df.columns or "Hausverbrauch" not in df.columns: + return df + probe = df[["PV", "Hausverbrauch"]].copy() + filled = probe.interpolate(limit=3, limit_direction="both") + valid = filled.notna().all(axis=1) + if not valid.any(): + return df.iloc[0:0] + run = 0 + last_break = -1 + for i, ok in enumerate(valid.to_numpy()): + if ok: + run = 0 + else: + run += 1 + if run >= 4: + last_break = i + if last_break >= 0: + after = np.where(valid.iloc[last_break + 1:].to_numpy())[0] + if len(after): + return df.iloc[last_break + 1 + after[0]:] + return df.iloc[0:0] + return df.loc[valid[valid].index[0]:] + + + +def _longest_consistent_segment(df, column): + if df.empty or column not in df.columns: + return pd.DataFrame() + series = pd.to_numeric(df[column], errors="coerce") + valid = series.notna().to_numpy() + if not valid.any(): + return pd.DataFrame() + best_start = best_end = None + start = 0 + gap = 0 + for i, ok in enumerate(valid): + if ok: + gap = 0 + else: + gap += 1 + if gap >= 4: + end = i - gap + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + start = i + 1 + gap = 0 + end = len(valid) - 1 + if end >= start and (best_start is None or end - start > best_end - best_start): + best_start, best_end = start, end + if best_start is None: + return pd.DataFrame() + segment = df.iloc[best_start:best_end + 1].copy() + first = pd.to_numeric(segment[column], errors="coerce").first_valid_index() + last = pd.to_numeric(segment[column], errors="coerce").last_valid_index() + if first is None or last is None: + return pd.DataFrame() + return segment.loc[first:last] + + +def _add_time_features(df): + hour = df.index.hour + df.index.minute / 60.0 + doy = df.index.dayofyear + df["hour_float"] = hour + df["hour_sin"] = np.sin(2 * np.pi * hour / 24.0) + df["hour_cos"] = np.cos(2 * np.pi * hour / 24.0) + df["sin_year"] = np.sin(2 * np.pi * doy / 365.25) + df["cos_year"] = np.cos(2 * np.pi * doy / 365.25) + df["weekday"] = df.index.weekday + df["is_weekday"] = (df.index.weekday < 5).astype(int) + return df + + +def _fill_defaults(df, history): + defaults = { + "PV": np.nan, + "Hausverbrauch": np.nan, + "SOC": np.nan, + "Netzleistung": np.nan, + "temp_c": 15.0, + "cloud": 20.0, + "cloud_cover": 20.0, + "precip_mm": 0.0, + "wind_kph": 0.0, + "chance_of_snow": 0.0, + "import_price": np.nan, + "export_price": np.nan, + } + for col, default in defaults.items(): + if col not in df.columns: + df[col] = default + df[col] = pd.to_numeric(df[col], errors="coerce") + # Measurements are never interpolated/filled here. Outages are not zero load, + # zero PV, a fresh SOC, or a measured grid peak. Weather defaults are separate. + df = sanitize_measured_frame(df) + for col, default in defaults.items(): + if col not in ["PV", "Hausverbrauch", "SOC", "Netzleistung"]: + df[col] = df[col].ffill().bfill() + if np.isfinite(default): + df[col] = df[col].fillna(default) + if "cloud_cover" in df.columns: + df["cloud"] = df["cloud"].fillna(df["cloud_cover"]) + df["import_price"] = df["import_price"].fillna(0.30) + df["export_price"] = df["export_price"].fillna(0.10) + return _add_time_features(df) + + +def build_data_object(config, training=False): + now = datetime.datetime.utcnow().replace(second=0, microsecond=0) + now = now - datetime.timedelta(minutes=now.minute % 5) + df_tel, df_wea, df_tar = fetch_influx_frames(config, training) + + frames = [f for f in [df_tel, df_wea, df_tar] if not f.empty] + combined = frames[0] if frames else pd.DataFrame() + for f in frames[1:]: + combined = combined.join(f, how="outer") + combined = combined.sort_index() + + full_hist_raw = combined.loc[:now - datetime.timedelta(minutes=5)] if not combined.empty else pd.DataFrame() + # A trailing telemetry gap must not discard all earlier valid observations. + hist_raw = full_hist_raw.copy() + # Freshness is determined from measurements, not the outer-joined weather grid. + recent_raw = sanitize_measured_frame(df_tel.loc[:now - datetime.timedelta(minutes=5)].copy()) if not df_tel.empty else pd.DataFrame() + start_hist = hist_raw.index.min().floor("5min") if training and not hist_raw.empty else now - datetime.timedelta(days=14) + + idx_hist = pd.date_range(start=start_hist, end=now - datetime.timedelta(minutes=5), freq="5min") + df_hist = pd.DataFrame(index=idx_hist).join(hist_raw, how="left") + df_hist = _fill_defaults(df_hist, history=True) + + df_load_training = pd.DataFrame() + if training and "Hausverbrauch" in full_hist_raw.columns: + load_segment = _longest_consistent_segment(full_hist_raw, "Hausverbrauch") + if not load_segment.empty: + idx_load = pd.date_range( + start=load_segment.index.min().floor("5min"), + end=load_segment.index.max().floor("5min"), + freq="5min", + ) + df_load_training = pd.DataFrame(index=idx_load).join(load_segment, how="left") + df_load_training = _fill_defaults(df_load_training, history=True) + + df_pv_training = pd.DataFrame() + if training and "PV" in full_hist_raw.columns: + pv_first = full_hist_raw["PV"].first_valid_index() + if pv_first is not None: + idx_pv = pd.date_range( + start=pv_first.floor("5min"), + end=now - datetime.timedelta(minutes=5), + freq="5min", + ) + df_pv_training = pd.DataFrame(index=idx_pv).join(full_hist_raw, how="left") + df_pv_training = _fill_defaults(df_pv_training, history=True) + + idx_fut = pd.date_range(start=now, periods=FORECAST_HORIZON_HOURS * 12, freq="5min") + fut_raw = combined.reindex(combined.index.union(idx_fut)).sort_index() if not combined.empty else pd.DataFrame(index=idx_fut) + df_fut = pd.DataFrame(index=idx_fut).join(fut_raw, how="left") + df_fut = _fill_defaults(df_fut, history=False) + + month_hist = df_hist[(df_hist.index.year == now.year) & (df_hist.index.month == now.month)] + current_peak_kw = 0.0 + if not month_hist.empty and "Netzleistung" in month_hist.columns: + measured_grid = pd.to_numeric(month_hist["Netzleistung"], errors="coerce").dropna() + measured_peak = measured_grid.resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not measured_peak.empty: + current_peak_kw = max(0.0, float(measured_peak.max()) / 1000.0) + if current_peak_kw <= 0.0 and not month_hist.empty and {"Hausverbrauch", "PV"}.issubset(month_hist.columns): + fallback_residual = (month_hist["Hausverbrauch"] - month_hist["PV"]).resample( + "15min", origin="start_day", label="left", closed="left" + ).mean() + if not fallback_residual.empty: + current_peak_kw = max(0.0, float(fallback_residual.max()) / 1000.0) + + min_soc = float( + config.get("batt_min_soc", config.get("batt_min_soc_percent", 0.0)) or 0.0 + ) + current_soc = max(0.0, min(100.0, min_soc)) + current_soc_source = "safe_minimum" + current_soc_age_minutes = None + if "SOC" in full_hist_raw.columns: + soc_values = pd.to_numeric(full_hist_raw["SOC"], errors="coerce").dropna() + if not soc_values.empty: + latest_soc_time = pd.Timestamp(soc_values.index[-1]) + current_soc_age_minutes = max( + 0.0, + (pd.Timestamp(now) - latest_soc_time).total_seconds() / 60.0, + ) + try: + max_soc_age_minutes = max( + 5.0, + float(config.get("batt_soc_max_age_minutes", 30.0) or 30.0), + ) + except (TypeError, ValueError): + max_soc_age_minutes = 30.0 + if current_soc_age_minutes <= max_soc_age_minutes: + current_soc = float(soc_values.iloc[-1]) + current_soc_source = "telemetry" + current_soc = max(0.0, min(100.0, current_soc)) + + return { + "config": config, + "metadata": config, + "now": now, + "df_hist": df_hist, + "df_recent_raw": recent_raw, + "df_load_training": df_load_training, + "df_pv_training": df_pv_training, + "df_fut": df_fut, + "current_soc_perc": current_soc, + "current_soc_source": current_soc_source, + "current_soc_age_minutes": current_soc_age_minutes, + "current_month_peak_kw": current_peak_kw, + } + + +def run_training(): + for config in get_configs(): + if not any(active(config, n) for n in MODEL_MODULES): + continue + print(f"Training Anlage {config['anlagen_id']}...") + data_obj = build_data_object(config, training=True) + for n, module in MODEL_MODULES.items(): + if active(config, n) and hasattr(module, "train"): + try: + target = QUALITY_TARGETS.get(n) + if target: + require_recent_telemetry(data_obj, [target]) + print(f" var_{n}: {module.train(data_obj)}") + except Exception: + print(f" var_{n}: Training fehlgeschlagen") + traceback.print_exc() + + +def _forecast_point(aid, field, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast") + .field(field, float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _snapshot_point(aid, field, run_hour, t, value): + return ( + Point("api_telemetry") + .tag("anlagen_id", aid) + .tag("data_type", "forecast_snapshot") + .tag("run_hour", run_hour) + .field(f"{field}_run_{run_hour}", float(value)) + .time(t.to_pydatetime(), WritePrecision.S) + ) + + +def _quality_query(aid, forecast_field, target_field): + stop = _time_literal(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) + return f''' +from(bucket: "{INFLUX_BUCKET}") + |> range(start: -{QUALITY_LOOKBACK_DAYS}d, stop: {stop}) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == "{aid}") + |> filter(fn: (r) => r["_field"] == "{target_field}" or r["_field"] == "{forecast_field}") + |> aggregateWindow(every: 5m, fn: mean, createEmpty: false) + |> group() + |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value") +''' + + +def _forecast_quality(aid, variant, target_field): + forecast_field = f"prog_var_{variant}" + df = _query_df(_quality_query(aid, forecast_field, target_field)) + if df.empty or forecast_field not in df.columns or target_field not in df.columns: + return None + pair = df[[target_field, forecast_field]].apply(pd.to_numeric, errors="coerce").dropna() + if len(pair) < 3: + return None + actual = pair[target_field].to_numpy(dtype=float) + pred = pair[forecast_field].to_numpy(dtype=float) + ss_res = float(np.sum((actual - pred) ** 2)) + ss_tot = float(np.sum((actual - np.mean(actual)) ** 2)) + r2 = None if ss_tot <= 0 else 1.0 - (ss_res / ss_tot) + mae = float(np.mean(np.abs(actual - pred))) + rmse = float(np.sqrt(np.mean((actual - pred) ** 2))) + return {"r2": r2, "mae": mae, "rmse": rmse, "samples": int(len(pair))} + + +def _quality_point(aid, variant, target, metrics): + p = ( + Point("forecast_metrics") + .tag("anlagen_id", aid) + .tag("forecast", f"prog_var_{variant}") + .tag("target", target) + .field("samples", int(metrics["samples"])) + .field("mae", float(metrics["mae"])) + .field("rmse", float(metrics["rmse"])) + .time(datetime.datetime.utcnow(), WritePrecision.S) + ) + if metrics["r2"] is not None: + p.field("r2", float(metrics["r2"])) + return p + + +def write_quality_metrics(write_api, aid, active_variants): + points = [] + + for variant, target in QUALITY_TARGETS.items(): + if variant not in active_variants: + continue + try: + metrics = _forecast_quality(aid, variant, target) + if metrics: + points.append(_quality_point(aid, variant, target, metrics)) + r2_text = "nan" if metrics["r2"] is None else f"{metrics['r2']:.3f}" + print(f"R2 Anlage {aid} prog_var_{variant}: {r2_text} / samples={metrics['samples']}") + except Exception: + print(f"R2 Anlage {aid} prog_var_{variant}: Berechnung fehlgeschlagen") + traceback.print_exc() + if points: + try: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + except Exception: + print(f"R2 Anlage {aid}: Schreiben der Qualitaetswerte fehlgeschlagen") + traceback.print_exc() + + +def run_forecast(only_anlagen_id=None, manual=False): + configs = get_configs() + if only_anlagen_id: + configs = [c for c in configs if c.get("anlagen_id") == only_anlagen_id] + client = InfluxDBClient( + url=INFLUX_URL, + token=INFLUX_TOKEN, + org=INFLUX_ORG, + timeout=INFLUX_TIMEOUT_MS, + ) + write_api = client.write_api(write_options=SYNCHRONOUS) + completed = [] + failures = [] + try: + for config in configs: + aid = config["anlagen_id"] + try: + data_obj = build_data_object(config, training=False) + targets = [] + if any(active(config, n) for n in (1, 3, 10, 13, 21, 23)): + targets.append("PV") + if any(active(config, n) for n in (2, 3, 11, 13, 22, 23)): + targets.append("Hausverbrauch") + if float(config.get("batt_capacity_kwh", 0.0) or 0.0) > 0 and any(active(config, n) for n in (3, 13, 23)): + targets.append("SOC") + # Fail BEFORE model prediction, snapshot publication or V1/V4 plan writes. + data_obj["telemetry_quality"] = require_recent_telemetry(data_obj, targets) + soc_age = data_obj.get("current_soc_age_minutes") + soc_age_text = "keine Messung" if soc_age is None else f"{soc_age:.1f} min" + print( + f"Forecast Anlage {aid}: Batterie-SOC {data_obj['current_soc_perc']:.1f}% " + f"({data_obj['current_soc_source']}, Alter {soc_age_text})." + ) + forecasts, model_errors = collect_predictions( + data_obj, config, + {1: v1, 2: v2, 10: v10, 11: v11, 21: v21, 22: v22}, active, + ) + data_obj['forecast_model_status'] = model_errors + for variant, detail in model_errors.items(): + print(f"Forecast Anlage {aid} prog_var_{variant}: unavailable ({detail['errorType']}); keine Nullwerte eingesetzt.") + p_1, p_2, p_10, p_11, p_21, p_22 = ( + forecasts[n] for n in (1, 2, 10, 11, 21, 22) + ) + + # ENELIX_V4_SHADOW_BRIDGE + _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)]) + + p_3 = v3.predict(data_obj, p_1, p_2) if active(config, 3) and p_1 and p_2 else {} + p_13 = v13.predict(data_obj, p_10, p_11) if active(config, 13) and p_10 and p_11 else {} + p_23 = v23.predict(data_obj, p_21, p_22) if active(config, 23) and p_21 and p_22 else {} + + forecast_sets = [ + (1, p_1), (2, p_2), (3, p_3), + (10, p_10), (11, p_11), (13, p_13), + (21, p_21), (22, p_22), (23, p_23), + ] + run_hour = str(int(datetime.datetime.now(LOCAL_TZ).strftime("%H"))) + points = [] + for variant_id, pv_src, load_src, grid_src in [ + (3, p_1, p_2, p_3), + (13, p_10, p_11, p_13), + (23, p_21, p_22, p_23), + ]: + if grid_src and pv_src and load_src: + points.extend(battery_soc_points(data_obj, variant_id, pv_src, load_src, grid_src)) + active_variants = set() + battery_plans = data_obj.get("battery_plans", {}) + for t in data_obj["df_fut"].index: + for n, values in forecast_sets: + if t in values: + field = f"prog_var_{n}" + value = float(values[t]) + points.append(_forecast_point(aid, field, t, value)) + points.append(_snapshot_point(aid, field, run_hour, t, value)) + if n in battery_plans and t in battery_plans[n].get("battery", {}): + points.append(_forecast_point( + aid, + f"prog_var_{n}_battery", + t, + float(battery_plans[n]["battery"][t]), + )) + active_variants.add(n) + if points: + write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points) + print(f"Forecast Anlage {aid}: {len(points)} Punkte geschrieben inkl. run_{run_hour}.") + else: + print(f"Forecast Anlage {aid}: keine aktiven Prognosen oder keine Daten.") + write_quality_metrics(write_api, aid, active_variants) + completed.append(aid) + except Exception as exc: + failures.append({"anlagen_id": aid, "error": f"{type(exc).__name__}: {exc}"}) + print(f"Forecast Anlage {aid}: Lauf fehlgeschlagen.") + traceback.print_exc() + finally: + client.close() + if failures: + raise RuntimeError(f"Forecast-Fehler: {failures}") + return {"completed": completed, "manual": bool(manual)} + + +def _write_json_atomic(path, payload): + directory = os.path.dirname(path) or "." + os.makedirs(directory, exist_ok=True) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + with open(tmp_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +def _read_json(path): + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + return {} + + +def _last_success_age_seconds(): + stamp = _read_json(LAST_SUCCESS_PATH).get("timestamp") + if not stamp: + return None + try: + then = datetime.datetime.fromisoformat(str(stamp).replace("Z", "+00:00")) + now = datetime.datetime.now(datetime.timezone.utc) + return max(0.0, (now - then).total_seconds()) + except Exception: + return None + + +def _child_environment(training=False): + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + if training: + max_threads = str(max(1, int(env.get("FORECAST_TRAINING_MAX_THREADS", "1")))) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "LOKY_MAX_CPU_COUNT", + ): + env[name] = max_threads + return env + + +def _run_child(mode, timeout_seconds, only_anlagen_id=None): + command = [sys.executable, "-u", os.path.abspath(__file__), mode] + if only_anlagen_id: + command.append(str(only_anlagen_id)) + label = "Training" if mode == "--train-once" else "Forecast" + print(f"{label}-Kindprozess startet (Timeout {timeout_seconds}s).") + process = subprocess.Popen(command, env=_child_environment(training=mode == "--train-once")) + try: + return_code = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + print(f"{label}-Kindprozess hat das Zeitlimit erreicht und wird beendet.") + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=15) + return 124 + if return_code != 0: + print(f"{label}-Kindprozess beendet mit Status {return_code}.") + return return_code + + +def run_forecast_isolated(only_anlagen_id=None, source="scheduler"): + if not _FORECAST_PROCESS_LOCK.acquire(blocking=False): + print(f"Forecast-Aufruf ({source}) uebersprungen: bereits ein Lauf aktiv.") + return {"status": "busy", "source": source} + try: + return_code = _run_child("--forecast-once", FORECAST_TIMEOUT_SECONDS, only_anlagen_id) + return { + "status": "ok" if return_code == 0 else "error", + "source": source, + "return_code": return_code, + } + finally: + _FORECAST_PROCESS_LOCK.release() + + +def _training_worker(day_text): + global _TRAINING_THREAD + started = datetime.datetime.now(datetime.timezone.utc).isoformat() + _write_json_atomic(LAST_TRAINING_PATH, { + "day": day_text, + "status": "running", + "started_at": started, + }) + return_code = 1 + error = None + try: + return_code = _run_child("--train-once", TRAINING_TIMEOUT_SECONDS) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + finally: + status = "ok" if return_code == 0 else ("timeout" if return_code == 124 else "error") + payload = { + "day": day_text, + "status": status, + "started_at": started, + "finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "return_code": return_code, + } + if error: + payload["error"] = error + _write_json_atomic(LAST_TRAINING_PATH, payload) + print(f"Nachttraining beendet: {status}.") + _TRAINING_THREAD = None + + +def start_training_async(day): + global _TRAINING_THREAD + if _TRAINING_THREAD is not None and _TRAINING_THREAD.is_alive(): + print("Nachttraining laeuft bereits.") + return False + day_text = day.isoformat() + _TRAINING_THREAD = threading.Thread(target=_training_worker, args=(day_text,), daemon=True) + _TRAINING_THREAD.start() + return True + + +def _watchdog_loop(): + while True: + time.sleep(max(60, WATCHDOG_INTERVAL_SECONDS)) + try: + age = _last_success_age_seconds() + if age is None or age > FORECAST_STALE_SECONDS: + age_text = "unbekannt" if age is None else f"{age / 60.0:.1f} Minuten" + print(f"Forecast-Watchdog: letzter erfolgreicher Lauf {age_text}; neuer Lauf wird gestartet.") + run_forecast_isolated(source="watchdog") + except Exception: + print("Forecast-Watchdog: Pruefung fehlgeschlagen.") + traceback.print_exc() + + +# BEGIN EMS RESIMULATE HTTP SERVER +_RESIMULATE_SERVER_STARTED = False + + +def _start_resimulate_server(): + global _RESIMULATE_SERVER_STARTED + if _RESIMULATE_SERVER_STARTED: + return + _RESIMULATE_SERVER_STARTED = True + + import json as _json + import os as _os + import threading as _threading + import traceback as _traceback + import urllib.parse as _urlparse + from http.server import BaseHTTPRequestHandler as _BaseHTTPRequestHandler, ThreadingHTTPServer as _ThreadingHTTPServer + + class _Handler(_BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def _send(self, code, body): + raw = _json.dumps(body).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def _handle(self): + try: + parsed = _urlparse.urlparse(self.path) + if parsed.path == "/health": + age = _last_success_age_seconds() + healthy = age is not None and age <= FORECAST_STALE_SECONDS + self._send(200 if healthy else 503, { + "status": "ok" if healthy else "stale", + "last_success_age_seconds": age, + "training": _read_json(LAST_TRAINING_PATH), + }) + return + if parsed.path not in ("/run_now", "/resimulate"): + self._send(404, {"detail": "not found"}) + return + params = _urlparse.parse_qs(parsed.query) + anlagen_id = (params.get("anlagen_id") or [None])[0] + result = run_forecast_isolated(only_anlagen_id=anlagen_id, source="http") + result["anlagen_id"] = anlagen_id + code = 200 if result["status"] == "ok" else (409 if result["status"] == "busy" else 500) + self._send(code, result) + except Exception as exc: + _traceback.print_exc() + self._send(500, {"detail": str(exc)}) + + port = int(_os.getenv("FORECAST_ENGINE_HTTP_PORT", "9000")) + server = _ThreadingHTTPServer(("0.0.0.0", port), _Handler) + thread = _threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + print(f"Forecast Resimulate HTTP Server startet auf Port {port}.") +# END EMS RESIMULATE HTTP SERVER + +def main(): + _start_resimulate_server() + global last_trained_day + print("Forecast Engine startet.") + threading.Thread(target=_watchdog_loop, daemon=True).start() + run_forecast_isolated(source="startup") + while True: + try: + now_local = datetime.datetime.now(LOCAL_TZ) + if now_local.hour == TRAINING_HOUR and last_trained_day != now_local.date(): + if start_training_async(now_local.date()): + last_trained_day = now_local.date() + next_run = (now_local + datetime.timedelta(hours=1)).replace(minute=0, second=0, microsecond=0) + time.sleep(max(60.0, (next_run - now_local).total_seconds())) + run_forecast_isolated(source="scheduler") + except Exception: + traceback.print_exc() + time.sleep(60) + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "--forecast-once": + selected_anlage = sys.argv[2] if len(sys.argv) >= 3 else None + run_forecast(only_anlagen_id=selected_anlage, manual=True) + _write_json_atomic(LAST_SUCCESS_PATH, { + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "anlagen_id": selected_anlage, + "pid": os.getpid(), + }) + elif len(sys.argv) >= 2 and sys.argv[1] == "--train-once": + run_training() + else: + main() diff --git a/services/netplan-v4/acceptance/forecast-src/methods/__init__.py b/services/netplan-v4/acceptance/forecast-src/methods/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py new file mode 100644 index 0000000..8615394 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/battery_optimizer.py @@ -0,0 +1,240 @@ +import numpy as np +import pandas as pd +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import lil_matrix + +DT_H = 5.0 / 60.0 + + +def train_artifact(kind): + return {"trained": True, "type": "battery_48h_cost_milp_v3", "source": kind} + + +def _cfg_float(config, key, default): + try: + value = config.get(key, default) + return float(default if value is None or value == "" else value) + except Exception: + return float(default) + + +def _cfg_bool(config, key, default=False): + value = config.get(key, default) + if value is None or value == "": + return bool(default) + if isinstance(value, bool): + return value + return str(value).strip().lower() in {"1", "true", "yes", "ja", "on"} + + +def _use_dynamic(config, key): + value = str(config.get(key, "") or "").strip().lower() + return any(token in value for token in ("dynam", "marktpreis", "referenzmarktpreis", "market")) + + +def _price(data_obj, timestamp, column, fallback, dynamic_enabled): + if not dynamic_enabled: + return fallback + frame = data_obj["df_fut"] + if column in frame.columns and timestamp in frame.index: + try: + value = float(frame.at[timestamp, column]) + if np.isfinite(value): + return value + except Exception: + pass + return fallback + + +def _battery_meta(config, data_obj): + cap_kwh = _cfg_float(config, "batt_capacity_kwh", 0.0) + max_power_w = _cfg_float(config, "batt_power_kw", 0.0) * 1000.0 + min_soc = _cfg_float(config, "batt_min_soc", _cfg_float(config, "batt_min_soc_percent", 0.0)) + max_soc = _cfg_float(config, "batt_max_soc", _cfg_float(config, "batt_max_soc_percent", 100.0)) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None: + start_soc_value = min_soc + try: + start_soc = float(start_soc_value) + except (TypeError, ValueError): + start_soc = min_soc + min_soc = max(0.0, min(100.0, min_soc)) + max_soc = max(min_soc, min(100.0, max_soc)) + start_soc = max(min_soc, min(max_soc, start_soc)) + charge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_discharge_efficiency", 0.95))) + return cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff + + +def _quarter_groups(index): + groups = {} + for position, timestamp in enumerate(index): + quarter = timestamp.floor("15min") if hasattr(timestamp, "floor") else position // 3 + groups.setdefault(quarter, []).append(position) + return list(groups.values()) + + +def _fallback_plan(index, residual_w): + grid = {timestamp: float(value) for timestamp, value in zip(index, residual_w)} + battery = {timestamp: 0.0 for timestamp in index} + return {"grid": grid, "battery": battery, "solver": "fallback"} + + +def optimize_battery_plan(data_obj, pv_dict, load_dict): + config = data_obj["config"] + cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff = _battery_meta(config, data_obj) + index = list(data_obj["df_fut"].index) + if not index: + return {"grid": {}, "battery": {}, "solver": "empty"} + + load_w = np.array([max(0.0, float(load_dict.get(t, 0.0))) for t in index]) + pv_w = np.array([max(0.0, float(pv_dict.get(t, 0.0))) for t in index]) + residual_w = load_w - pv_w + if cap_kwh <= 0.0 or max_power_w <= 0.0: + return _fallback_plan(index, residual_w) + + import_fixed = _cfg_float(config, "tarif_bezug_fest", 0.30) + export_fixed = _cfg_float(config, "tarif_einspeisung_fest", 0.10) + import_dynamic = _use_dynamic(config, "tarif_bezug") + export_dynamic = _use_dynamic(config, "tarif_einspeisung") + import_price = np.array([ + _price(data_obj, t, "import_price", import_fixed, import_dynamic) for t in index + ]) + export_price = np.array([ + _price(data_obj, t, "export_price", export_fixed, export_dynamic) for t in index + ]) + + n = len(index) + imp, exp, charge, discharge, curtail, soc = 0, n, 2 * n, 3 * n, 4 * n, 5 * n + peak = 6 * n + 1 + battery_mode = peak + 1 + grid_mode = battery_mode + n + variable_count = grid_mode + n + + max_import_w = max(float(load_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_import_limit = _cfg_float(config, "grid_import_limit_w", 0.0) + if configured_import_limit > 0.0: + max_import_w = min(max_import_w, configured_import_limit) + max_export_w = max(float(pv_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0) + configured_export_limit = _cfg_float(config, "grid_export_limit_w", 0.0) + if configured_export_limit > 0.0: + max_export_w = min(max_export_w, configured_export_limit) + + lower = np.zeros(variable_count) + upper = np.full(variable_count, np.inf) + upper[imp:imp + n] = max_import_w + upper[exp:exp + n] = max_export_w + upper[charge:charge + n] = max_power_w + if not _cfg_bool(config, "batt_grid_charging_enabled", False): + upper[charge:charge + n] = np.minimum(max_power_w, np.maximum(0.0, pv_w - load_w)) + upper[discharge:discharge + n] = max_power_w + upper[curtail:curtail + n] = pv_w + reserve_soc = max( + min_soc, + min(100.0, _cfg_float(config, "batt_economic_reserve_soc_percent", 10.0)), + ) + economic_min_soc = max(min_soc, min(start_soc, reserve_soc)) + min_kwh = cap_kwh * economic_min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + lower[soc:soc + n + 1] = min_kwh + upper[soc:soc + n + 1] = max_kwh + current_peak_kw = max(0.0, float(data_obj.get("current_month_peak_kw", 0.0) or 0.0)) + lower[peak] = current_peak_kw + upper[peak] = max(current_peak_kw, max_import_w / 1000.0) + upper[battery_mode:battery_mode + n] = 1.0 + upper[grid_mode:grid_mode + n] = 1.0 + + objective = np.zeros(variable_count) + objective[imp:imp + n] = import_price * DT_H / 1000.0 + objective[exp:exp + n] = -export_price * DT_H / 1000.0 + degradation = max(0.0, _cfg_float(config, "batt_degradation_chf_kwh", 0.03)) + objective[charge:charge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[discharge:discharge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0 + objective[curtail:curtail + n] = 1e-9 * DT_H / 1000.0 + objective[peak] = max(0.0, _cfg_float(config, "tarif_peak_fest", 0.0)) + terminal_value = _cfg_float(config, "batt_terminal_value_chf_kwh", np.median(import_price)) + objective[soc + n] = -max(0.0, terminal_value) * discharge_eff + + equality_rows = 2 * n + 1 + equality = lil_matrix((equality_rows, variable_count), dtype=float) + equality_rhs = np.zeros(equality_rows) + for i in range(n): + equality[i, imp + i] = 1.0 + equality[i, exp + i] = -1.0 + equality[i, charge + i] = -1.0 + equality[i, discharge + i] = 1.0 + equality[i, curtail + i] = -1.0 + equality_rhs[i] = residual_w[i] + + row = n + i + equality[row, soc + i] = -1.0 + equality[row, soc + i + 1] = 1.0 + equality[row, charge + i] = -charge_eff * DT_H / 1000.0 + equality[row, discharge + i] = DT_H / (1000.0 * discharge_eff) + equality[2 * n, soc] = 1.0 + equality_rhs[2 * n] = cap_kwh * start_soc / 100.0 + + quarter_groups = _quarter_groups(pd.Index(index)) + inequality_rows = 4 * n + len(quarter_groups) + inequality = lil_matrix((inequality_rows, variable_count), dtype=float) + inequality_upper = np.zeros(inequality_rows) + row = 0 + for i in range(n): + inequality[row, charge + i] = 1.0 + inequality[row, battery_mode + i] = -max_power_w + row += 1 + inequality[row, discharge + i] = 1.0 + inequality[row, battery_mode + i] = max_power_w + inequality_upper[row] = max_power_w + row += 1 + inequality[row, imp + i] = 1.0 + inequality[row, grid_mode + i] = -max_import_w + row += 1 + inequality[row, exp + i] = 1.0 + inequality[row, grid_mode + i] = max_export_w + inequality_upper[row] = max_export_w + row += 1 + for group in quarter_groups: + for i in group: + inequality[row, imp + i] = 1.0 / (len(group) * 1000.0) + inequality[row, peak] = -1.0 + row += 1 + + integrality = np.zeros(variable_count, dtype=int) + integrality[battery_mode:battery_mode + n] = 1 + integrality[grid_mode:grid_mode + n] = 1 + constraints = [ + LinearConstraint(equality.tocsr(), equality_rhs, equality_rhs), + LinearConstraint(inequality.tocsr(), -np.inf, inequality_upper), + ] + result = milp( + objective, + integrality=integrality, + bounds=Bounds(lower, upper), + constraints=constraints, + options={"time_limit": max(5.0, _cfg_float(config, "batt_optimizer_timeout_seconds", 30.0))}, + ) + if not result.success or result.x is None: + return _fallback_plan(index, residual_w) + + grid_values = result.x[imp:imp + n] - result.x[exp:exp + n] + battery_values = result.x[charge:charge + n] - result.x[discharge:discharge + n] + threshold_w = max(25.0, max_power_w * 0.005) + grid_values[np.abs(grid_values) < threshold_w] = 0.0 + battery_values[np.abs(battery_values) < threshold_w] = 0.0 + return { + "grid": {t: float(v) for t, v in zip(index, grid_values)}, + "battery": {t: float(v) for t, v in zip(index, battery_values)}, + "solver": "scipy-milp", + "objective_chf": float(result.fun), + "planned_peak_kw": float(result.x[peak]), + "start_soc_percent": float(start_soc), + "economic_min_soc_percent": float(economic_min_soc), + } + + +def optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=None): + plan = optimize_battery_plan(data_obj, pv_dict, load_dict) + if forecast_id is not None: + data_obj.setdefault("battery_plans", {})[int(forecast_id)] = plan + return plan["grid"] diff --git a/services/netplan-v4/acceptance/forecast-src/methods/common.py b/services/netplan-v4/acceptance/forecast-src/methods/common.py new file mode 100644 index 0000000..a0ab5e9 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/common.py @@ -0,0 +1,32 @@ +import os +import joblib + +MODEL_DIR = "/app/data/models" + + +def model_path(aid, forecast_id): + os.makedirs(MODEL_DIR, exist_ok=True) + return os.path.join(MODEL_DIR, f"forecast_var_{forecast_id}_{aid}.pkl") + + +def save_model(aid, forecast_id, artifact): + path = model_path(aid, forecast_id) + tmp_path = f"{path}.tmp.{os.getpid()}" + try: + joblib.dump(artifact, tmp_path) + os.replace(tmp_path, path) + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + return path + + +def load_model(aid, forecast_id): + path = model_path(aid, forecast_id) + if not os.path.exists(path): + return None + try: + return joblib.load(path) + except Exception as exc: + print(f"Modell {path} konnte nicht geladen werden: {exc}") + return None diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_1.py b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py new file mode 100644 index 0000000..98cbe44 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_1.py @@ -0,0 +1,66 @@ +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model +from shared_utils import calc_pure_math_pv, roof_features + +FORECAST_ID = 1 +FEATURES = [ + "math_pv", + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "sin_year", + "cos_year", + "pv_kwp_total", + "roof_azimuth_sin", + "roof_azimuth_cos", + "roof_tilt_avg", + "roof_south_factor", +] + + +def _features(frame, config): + out = frame.copy() + out["math_pv"] = [calc_pure_math_pv(config, t) for t in out.index] + rf = roof_features(config) + for k, v in rf.items(): + out[k] = v + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out[FEATURES].astype(float) + + +def train(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + df = data_obj.get("df_pv_training", data_obj["df_hist"]).copy() + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + X = _features(df, config) + y = pd.to_numeric(df["PV"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + model = RandomForestRegressor(n_estimators=400, max_depth=18, min_samples_leaf=2, random_state=42, n_jobs=-1) + model.fit(X, y) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES}) + return {"trained": True, "samples": int(len(X)), "path": path} + + +def predict(data_obj): + config = data_obj["config"] + aid = config["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + X = _features(data_obj["df_fut"], config) + if artifact and "model" in artifact: + values = artifact["model"].predict(X) + else: + values = X["math_pv"].to_numpy() + ac_limit = float(config.get("ac_leistung", 10.0) or 10.0) * 1000.0 + return {t: max(0.0, min(float(v), ac_limit)) for t, v in zip(data_obj["df_fut"].index, values)} diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_10.py b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py new file mode 100644 index 0000000..02ea37d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_10.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 10, {"type": "repeat_pv_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'PV') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_11.py b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py new file mode 100644 index 0000000..7e1dc3c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_11.py @@ -0,0 +1,12 @@ +from telemetry_quality import repeat_daily_profile +import datetime +from methods.common import save_model + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, 11, {"type": "repeat_load_24h"})} + + +def predict(data_obj): + return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'Hausverbrauch') diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_13.py b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py new file mode 100644 index 0000000..9775eeb --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_13.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 13 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_10_11"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=13) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_2.py b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py new file mode 100644 index 0000000..fbb3d59 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_2.py @@ -0,0 +1,169 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor + +from methods.common import load_model, save_model +from telemetry_quality import profile_source_value + +FORECAST_ID = 2 +FEATURES = [ + "temp_c", + "cloud", + "hour_sin", + "hour_cos", + "weekday", + "is_weekday", + "load_24h_ago", + "load_7d_ago", + "energy_24h_rolling", + "load_3d_same_time_mean", + "load_7d_same_time_mean", +] + + +def _history_features(df): + out = df.copy() + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_24h_rolling"] = (out["Hausverbrauch"] * 5 / 60 / 1000).shift(1).rolling(288).sum() + same_time_lags = [out["Hausverbrauch"].shift(288 * d) for d in range(1, 8)] + out["load_3d_same_time_mean"] = pd.concat(same_time_lags[:3], axis=1).mean(axis=1) + out["load_7d_same_time_mean"] = pd.concat(same_time_lags, axis=1).mean(axis=1) + for col, default in [("temp_c", 15.0), ("cloud", 20.0)]: + if col not in out.columns: + out[col] = default + out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = data_obj.get("df_load_training", data_obj["df_hist"]).copy() + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = _history_features(df) + X = df[FEATURES].astype(float) + y = pd.to_numeric(df["Hausverbrauch"], errors="coerce") + valid = X.notna().all(axis=1) & y.notna() + X, y = X.loc[valid], y.loc[valid] + if len(X) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))} + validation_day = X.index.max().normalize() + validation_mask = X.index >= validation_day + if int(validation_mask.sum()) < 144: + validation_day -= datetime.timedelta(days=1) + validation_mask = (X.index >= validation_day) & (X.index < validation_day + datetime.timedelta(days=1)) + score = None + if int((~validation_mask).sum()) >= 288 and int(validation_mask.sum()) >= 96: + eval_model = HistGradientBoostingRegressor( + max_iter=900, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + eval_model.fit(X.loc[~validation_mask], y.loc[~validation_mask]) + score = float(eval_model.score(X.loc[validation_mask], y.loc[validation_mask])) + + model = HistGradientBoostingRegressor( + max_iter=1200, + max_depth=12, + learning_rate=0.02, + min_samples_leaf=8, + random_state=42, + ) + model.fit(X, y) + path = save_model( + aid, + FORECAST_ID, + {"model": model, "features": FEATURES, "r2_last_day": score}, + ) + return { + "trained": True, + "samples": int(len(X)), + "r2_last_day": score, + "path": path, + } + + +def _history_value(history, ts): + if history is None or history.empty or ts not in history.index or "Hausverbrauch" not in history.columns: + return None + value = history.at[ts, "Hausverbrauch"] + return float(value) if pd.notna(value) and np.isfinite(value) and value >= 0 else None + + +def _same_time_values(history, t, days): + values = [] + for day in range(1, days + 1): + value = _history_value(history, t - datetime.timedelta(days=day)) + if value is not None and np.isfinite(value): + values.append(max(0.0, value)) + return values + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + + frames = [ + frame + for frame in ( + data_obj.get("df_load_training"), + data_obj.get("df_hist"), + data_obj.get("df_recent_raw"), + ) + if frame is not None and not frame.empty and "Hausverbrauch" in frame.columns + ] + history = pd.concat(frames).sort_index() if frames else pd.DataFrame(columns=["Hausverbrauch"]) + history = history[~history.index.duplicated(keep="last")] + fut = data_obj["df_fut"] + res = {} + + for t in fut.index: + same_values = _same_time_values(history, t, 7) + load_24 = _history_value(history, t - datetime.timedelta(days=1)) + load_7d = _history_value(history, t - datetime.timedelta(days=7)) + if load_24 is None: + load_24 = load_7d if load_7d is not None else profile_source_value(history, t, "Hausverbrauch") + if load_7d is None: + load_7d = load_24 + same_median = float(np.median(same_values)) if same_values else load_7d + profile = max(0.0, (0.50 * load_24) + (0.30 * load_7d) + (0.20 * same_median)) + + window = history.loc[ + t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), + "Hausverbrauch", + ] + energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + same_3 = same_values[:3] + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "cloud"]), + float(fut.at[t, "hour_sin"]), + float(fut.at[t, "hour_cos"]), + int(fut.at[t, "weekday"]), + int(fut.at[t, "is_weekday"]), + load_24, + load_7d, + energy, + float(np.mean(same_3)) if same_3 else profile, + float(np.mean(same_values)) if same_values else profile, + ]], columns=FEATURES) + + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + model_pred = min(max(model_pred, profile * 0.25), max(500.0, profile * 3.0)) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + + res[t] = pred + history.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_21.py b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py new file mode 100644 index 0000000..1c84981 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_21.py @@ -0,0 +1,62 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import RandomForestRegressor + +from methods.common import load_model, save_model + +FORECAST_ID = 21 +FEATURES = ["temp_c", "hour_cos", "pv_24h_ago"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + if "PV" in out.columns: + out["pv_24h_ago"] = out["PV"].shift(288) + return out + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_pv_training", data_obj["df_hist"]).copy()) + if "PV" not in df.columns: + return {"trained": False, "reason": "PV fehlt"} + df = df.dropna(subset=FEATURES + ["PV"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["PV"]) + score = float(eval_model.score(test_df[FEATURES], test_df["PV"])) + print(f"[var_21] R2 PV letzter kompletter Tag: {score:.3f}") + + model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42) + model.fit(df[FEATURES], df["PV"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + hist = data_obj["df_hist"] + fut = data_obj["df_fut"].copy() + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + pv_24 = float(hist.at[t_24, "PV"]) if t_24 in hist.index else 0.0 + row = pd.DataFrame([[float(fut.at[t, "temp_c"]), float(fut.at[t, "hour_cos"]), pv_24]], columns=FEATURES) + pred = float(model.predict(row)[0]) if model is not None else pv_24 + res[t] = max(0.0, pred) + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_22.py b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py new file mode 100644 index 0000000..3f1d6d8 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_22.py @@ -0,0 +1,121 @@ +import datetime +import numpy as np +import pandas as pd +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.inspection import permutation_importance + +from methods.common import load_model, save_model + +FORECAST_ID = 22 +FEATURES = ["temp_c", "hour_cos", "load_24h_ago", "load_7d_ago", "energy_24h_rolling", "weekday"] + + +def _feature_frame(df): + out = df.copy() + if "temp_c" not in out.columns: + out["temp_c"] = 15.0 + out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0) + out["hour_float"] = out.index.hour + out.index.minute / 60.0 + out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0) + out["weekday"] = out.index.weekday + if "Hausverbrauch" in out.columns: + out["load_24h_ago"] = out["Hausverbrauch"].shift(288) + out["load_7d_ago"] = out["Hausverbrauch"].shift(2016) + out["energy_5m_kwh"] = out["Hausverbrauch"] * (5 / 60) / 1000 + out["energy_24h_rolling"] = out["energy_5m_kwh"].shift(1).rolling(window=288).sum() + out = out.drop(columns=["energy_5m_kwh"]) + return out + + +def _history_value(history, recent, reference, ts): + for frame in (history, recent, reference): + if frame is not None and not frame.empty and ts in frame.index and "Hausverbrauch" in frame.columns: + value = frame.at[ts, "Hausverbrauch"] + if pd.notna(value): + return float(value) + return None + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + df = _feature_frame(data_obj.get("df_load_training", data_obj["df_hist"]).copy()) + if "Hausverbrauch" not in df.columns: + return {"trained": False, "reason": "Hausverbrauch fehlt"} + df = df.dropna(subset=FEATURES + ["Hausverbrauch"]) + if len(df) < 288: + return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))} + + last_day = df.index.max().normalize() + train_df = df[df.index < last_day] + test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))] + score = None + importance = {} + if len(train_df) >= 288 and len(test_df) >= 12: + eval_model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + eval_model.fit(train_df[FEATURES], train_df["Hausverbrauch"]) + score = float(eval_model.score(test_df[FEATURES], test_df["Hausverbrauch"])) + print(f"[var_22] R2 Hausverbrauch letzter kompletter Tag: {score:.3f}") + try: + perm = permutation_importance(eval_model, test_df[FEATURES], test_df["Hausverbrauch"], n_repeats=10, random_state=42) + order = perm.importances_mean.argsort()[::-1] + importance = {FEATURES[i]: float(perm.importances_mean[i]) for i in order} + print("[var_22] Feature-Wichtigkeit Hausverbrauch:") + for name, val in importance.items(): + print(f" {name}: {val:.4f}") + except Exception as exc: + print(f"[var_22] permutation_importance nicht berechnet: {exc}") + + model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42) + model.fit(df[FEATURES], df["Hausverbrauch"]) + path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score, "importance": importance}) + return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path} + + +def predict(data_obj): + aid = data_obj["config"]["anlagen_id"] + artifact = load_model(aid, FORECAST_ID) + model = artifact["model"] if artifact and "model" in artifact else None + try: + score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0 + except (TypeError, ValueError): + score = 0.0 + model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0 + hist = data_obj["df_hist"].copy() + recent = data_obj.get("df_recent_raw", pd.DataFrame()) + reference = data_obj.get("df_load_training", pd.DataFrame()) + fut = data_obj["df_fut"] + res = {} + for t in fut.index: + t_24 = t - datetime.timedelta(days=1) + t_7d = t - datetime.timedelta(days=7) + load_7d = _history_value(hist, recent, reference, t_7d) + load_24 = _history_value(hist, recent, reference, t_24) + if load_24 is None: + load_24 = load_7d if load_7d is not None else 0.0 + if load_7d is None: + load_7d = load_24 + ref_end = t - datetime.timedelta(days=7) + ref_start = ref_end - datetime.timedelta(days=1) + if not reference.empty and "Hausverbrauch" in reference.columns: + window = reference.loc[ref_start:ref_end - datetime.timedelta(minutes=5), "Hausverbrauch"] + else: + window = pd.Series(dtype=float) + if window.empty and "Hausverbrauch" in recent.columns: + window = recent.loc[t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), "Hausverbrauch"] + roll_energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0 + row = pd.DataFrame([[ + float(fut.at[t, "temp_c"]), + float(fut.at[t, "hour_cos"]), + load_24, + load_7d, + roll_energy, + int(fut.at[t, "weekday"]), + ]], columns=FEATURES) + profile = max(0.0, (0.65 * load_24) + (0.35 * load_7d)) + pred = profile + if model is not None and model_weight > 0.0: + model_pred = max(0.0, float(model.predict(row)[0])) + pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred) + res[t] = pred + hist.loc[t, "Hausverbrauch"] = pred + return res diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_23.py b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py new file mode 100644 index 0000000..3cafe6d --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_23.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 23 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_21_22"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=23) diff --git a/services/netplan-v4/acceptance/forecast-src/methods/var_3.py b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py new file mode 100644 index 0000000..db46d6f --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/methods/var_3.py @@ -0,0 +1,13 @@ +from methods.battery_optimizer import optimize_grid_setpoint, train_artifact +from methods.common import save_model + +FORECAST_ID = 3 + + +def train(data_obj): + aid = data_obj["config"]["anlagen_id"] + return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_1_2"))} + + +def predict(data_obj, pv_dict, load_dict): + return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=3) diff --git a/services/netplan-v4/acceptance/forecast-src/model_isolation.py b/services/netplan-v4/acceptance/forecast-src/model_isolation.py new file mode 100644 index 0000000..e64d3d1 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/model_isolation.py @@ -0,0 +1,45 @@ +"""Forecast family fault isolation; no IO, fabricated samples or model switching.""" +from collections.abc import Mapping +from math import isfinite +from numbers import Real + + +def collect_predictions(data, config, models, enabled): + """Return complete finite PV/load series independently for each requested model. + + A model with insufficient history must not prevent other valid families being + published. It is omitted, not filled with zeros or replaced by another model. + Global raw-telemetry checks still run BEFORE this helper in run_forecast. + """ + expected = tuple(data['df_fut'].index) + if not expected or len(set(expected)) != len(expected): + raise ValueError('Nonempty unique future interval index required') + expected_keys = set(expected) + predictions, errors = {}, {} + requested = 0 + for variant, model in models.items(): + predictions[variant] = {} + if not enabled(config, variant): + continue + requested += 1 + try: + values = model.predict(data) + if not isinstance(values, Mapping) or set(values) != expected_keys: + raise ValueError('Missing, extra or non-matching forecast timestamps') + cleaned = {} + for at in expected: + value = values[at] + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError('Non-numeric forecast power') + value = float(value) + if not isfinite(value) or value < 0: + raise ValueError('Invalid nonnegative forecast power') + cleaned[at] = value + predictions[variant] = cleaned + except Exception as error: + # Expected data failures and unexpected model failures are visible, + # but model exception text may contain filesystem paths. Do not leak it. + errors[variant] = {'status': 'unavailable', 'errorType': type(error).__name__} + if requested and all(not p for p in predictions.values()): + raise ValueError('All requested PV/load models failed; no forecasts published') + return predictions, errors diff --git a/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/acceptance/forecast-src/shared_utils.py b/services/netplan-v4/acceptance/forecast-src/shared_utils.py new file mode 100644 index 0000000..fb40dce --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/shared_utils.py @@ -0,0 +1,93 @@ +import math + + +def as_float(value, default=0.0): + try: + if value is None: + return default + return float(value) + except Exception: + return default + + +def roof_kwp(roof): + return as_float(roof.get("kwp", roof.get("leistung", roof.get("pv_kwp", 0.0))), 0.0) + + +def roof_azimuth_deg(roof): + return as_float(roof.get("azimut", roof.get("azimuth", roof.get("ausrichtung", 180.0))), 180.0) + + +def roof_tilt_deg(roof): + return as_float(roof.get("neigung", roof.get("tilt", 30.0)), 30.0) + + +def roof_features(config): + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "azimut": 180.0, "neigung": 30.0}] + total_kwp = sum(roof_kwp(r) for r in roofs) + if total_kwp <= 0: + total_kwp = as_float(config.get("ac_leistung", 0.0), 0.0) + weighted_az = 0.0 + weighted_tilt = 0.0 + south_factor = 0.0 + for roof in roofs: + w = roof_kwp(roof) / total_kwp if total_kwp > 0 else 0.0 + az = roof_azimuth_deg(roof) + tilt = roof_tilt_deg(roof) + weighted_az += w * az + weighted_tilt += w * tilt + south_factor += w * max(0.0, math.cos(math.radians(az - 180.0))) + return { + "pv_kwp_total": total_kwp, + "roof_azimuth_sin": math.sin(math.radians(weighted_az)), + "roof_azimuth_cos": math.cos(math.radians(weighted_az)), + "roof_tilt_avg": weighted_tilt, + "roof_south_factor": south_factor, + } + + +def calc_pure_math_pv(config, dt): + ac_limit = as_float(config.get("ac_leistung", 10.0), 10.0) * 1000.0 + roofs = config.get("daecher", []) or [] + if not roofs: + roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "neigung": 30.0, "azimut": 180.0}] + + day = dt.timetuple().tm_yday + hour = dt.hour + dt.minute / 60.0 + lat = math.radians(as_float(config.get("latitude", 47.0), 47.0)) + decl = math.radians(23.45 * math.sin(math.radians(360.0 * (day - 81) / 365.0))) + hour_angle = math.radians(15.0 * (hour - 12.0)) + + sin_alt = math.sin(lat) * math.sin(decl) + math.cos(lat) * math.cos(decl) * math.cos(hour_angle) + sun_alt = math.asin(max(-1.0, min(1.0, sin_alt))) + if sun_alt <= 0.0: + return 0.0 + + sun_az = math.atan2( + math.sin(hour_angle), + math.cos(hour_angle) * math.sin(lat) - math.tan(decl) * math.cos(lat), + ) + + total = 0.0 + for roof in roofs: + kwp = roof_kwp(roof) + if kwp <= 0: + continue + tilt = math.radians(roof_tilt_deg(roof)) + az = math.radians(roof_azimuth_deg(roof)) + cos_inc = math.sin(sun_alt) * math.cos(tilt) + math.cos(sun_alt) * math.sin(tilt) * math.cos(sun_az - az) + if cos_inc > 0: + total += kwp * 1000.0 * cos_inc + + return max(0.0, min(total, ac_limit)) + + +def calc_pure_math_load(dt): + hour = dt.hour + dt.minute / 60.0 + base = 650.0 + morning = 220.0 * math.exp(-((hour - 7.0) ** 2) / 5.0) + evening = 380.0 * math.exp(-((hour - 19.0) ** 2) / 8.0) + weekend = 1.12 if dt.weekday() >= 5 else 1.0 + return max(0.0, (base + morning + evening) * weekend) diff --git a/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py new file mode 100644 index 0000000..f2bf2e7 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/soc_diagnostics.py @@ -0,0 +1,66 @@ +from influxdb_client import Point, WritePrecision + +DT_H = 5.0 / 60.0 + + +def _f(config, key, default): + try: + return float(config.get(key, default) or default) + except Exception: + return float(default) + + +def battery_soc_points(data_obj, forecast_var, pv_dict, load_dict, grid_dict): + config = data_obj["config"] + aid = str(config["anlagen_id"]) + cap_kwh = _f(config, "batt_capacity_kwh", 0.0) + if cap_kwh <= 0 or not grid_dict: + return [] + + min_soc = _f(config, "batt_min_soc", _f(config, "batt_min_soc_percent", 0.0)) + max_soc = _f(config, "batt_max_soc", _f(config, "batt_max_soc_percent", 100.0)) + charge_eff = max(0.01, min(1.0, _f(config, "batt_charge_efficiency", 0.95))) + discharge_eff = max(0.01, min(1.0, _f(config, "batt_discharge_efficiency", 0.95))) + start_soc_value = data_obj.get("current_soc_perc") + if start_soc_value is None or start_soc_value == "": + start_soc_value = config.get("batt_soc_percent", 50.0) + start_soc = float(start_soc_value) + start_soc = max(min_soc, min(max_soc, start_soc)) + + min_kwh = cap_kwh * min_soc / 100.0 + max_kwh = cap_kwh * max_soc / 100.0 + soc_kwh = max(min_kwh, min(max_kwh, cap_kwh * start_soc / 100.0)) + + planned_battery = ( + data_obj.get("battery_plans", {}) + .get(int(forecast_var), {}) + .get("battery", {}) + ) + points = [] + for t in data_obj["df_fut"].index: + if t not in grid_dict: + continue + if t in planned_battery: + battery_target_w = float(planned_battery[t]) # positiv = laden + else: + residual_w = float(load_dict.get(t, 0.0)) - float(pv_dict.get(t, 0.0)) + grid_w = float(grid_dict.get(t, 0.0)) + battery_target_w = grid_w - residual_w + + if battery_target_w > 0: + soc_kwh += battery_target_w * DT_H / 1000.0 * charge_eff + elif battery_target_w < 0: + soc_kwh += battery_target_w * DT_H / 1000.0 / discharge_eff + + soc_kwh = max(min_kwh, min(max_kwh, soc_kwh)) + soc_percent = max(0.0, min(100.0, soc_kwh / cap_kwh * 100.0)) + + points.append( + Point("forecast_diagnostics") + .tag("anlagen_id", aid) + .tag("data_type", "battery_soc_simulation") + .tag("forecast_var", f"prog_var_{forecast_var}") + .field("soc_percent", float(soc_percent)) + .time(t.to_pydatetime() if hasattr(t, "to_pydatetime") else t, WritePrecision.S) + ) + return points diff --git a/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py new file mode 100644 index 0000000..bf316d2 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/telemetry_quality.py @@ -0,0 +1,106 @@ +"""Measured telemetry integrity and repeat-profile helpers. +No IO, no fabricated measurements and no fixed household-load fallback. +Forecast freshness is checked against raw telemetry, never against filled features. +""" +from __future__ import annotations +import datetime +import math +import numpy as np +import pandas as pd + +MEASURED_COLUMNS = ('PV', 'Hausverbrauch', 'Netzleistung', 'SOC') + +class TelemetryUnavailable(ValueError): + """No publishable forecast can be derived from the supplied observations.""" + + +def sanitize_measured_frame(frame): + out = frame.copy() + for column in MEASURED_COLUMNS: + if column not in out: + continue + raw = out[column] + numeric = pd.to_numeric(raw, errors='coerce').astype(float) + bad = ~np.isfinite(numeric) | raw.map(lambda x: isinstance(x, (bool, np.bool_))) + if column in ('PV', 'Hausverbrauch', 'SOC'): + bad |= numeric < 0 + if column == 'SOC': + bad |= numeric > 100 + out[column] = numeric.mask(bad) + return out + + +def _naive_utc(value): + stamp = pd.Timestamp(value) + if stamp.tzinfo is not None: + stamp = stamp.tz_convert('UTC').tz_localize(None) + return stamp + + +def require_recent_telemetry(data_obj, fields=('PV','Hausverbrauch'), max_age_minutes=30.0): + """Refuse missing/stale inputs without changing real zeros or raw samples.""" + if isinstance(max_age_minutes, bool) or not math.isfinite(max_age_minutes) or max_age_minutes <= 0: + raise ValueError('Positive telemetry age limit required') + now = _naive_utc(data_obj['now']) + raw = data_obj.get('df_recent_raw') + raw = sanitize_measured_frame(raw) if raw is not None else pd.DataFrame() + if not raw.empty: + raw.index = pd.DatetimeIndex([_naive_utc(t) for t in raw.index]) + raw = raw.loc[raw.index < now].sort_index() + report = {} + errors = [] + for field in fields: + if field not in MEASURED_COLUMNS: + raise ValueError('Unknown telemetry target') + values = raw[field].dropna() if field in raw else pd.Series(dtype=float) + if values.empty: + errors.append(field + ': keine gemessenen Werte') + continue + stamp = values.index[-1] + age = (now-stamp).total_seconds()/60.0 + report[field] = {'lastObservedInterval': stamp.isoformat()+'Z', 'ageMinutes': age, + 'observedIntervals': int(len(values)), 'lastValue': float(values.iloc[-1])} + if age > max_age_minutes: + errors.append(field + ': Messdaten veraltet (' + format(age,'.1f') + ' min)') + if errors: + raise TelemetryUnavailable('; '.join(errors) + '. Keine neuen Prognosen/Fahrplaene veroeffentlicht.') + return report + + +def _finite_nonnegative(value): + if isinstance(value, (bool, np.bool_)): + return None + try: + value = float(value) + except (ValueError, TypeError): + return None + return value if math.isfinite(value) and value >= 0.0 else None + + +def profile_source_value(history, at, column, predictions=None): + """Repeat yesterday; prefer same weekday if yesterday is missing, then older days. + Explicitly generated first-day values may be repeated on the second forecast day. + Missing historical values are not zeros. No future measurement is read. + """ + if column not in ('PV','Hausverbrauch'): + raise ValueError('Unsupported repeat-profile target') + predictions = {} if predictions is None else predictions + at = pd.Timestamp(at) + for day in (1, 7, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14): + source = at - datetime.timedelta(days=day) + if source in predictions: + value = _finite_nonnegative(predictions[source]) + if value is not None: + return value + if history is not None and column in history and source in history.index: + value = _finite_nonnegative(history.at[source,column]) + if value is not None: + return value + raise TelemetryUnavailable(column + ': kein gemessener Tagesprofilwert fuer ' + str(at)) + + +def repeat_daily_profile(history, future_index, column): + result = {} + for at in future_index: + result[at] = profile_source_value(history, at, column, result) + return result diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py new file mode 100644 index 0000000..e533d76 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_battery_optimizer.py @@ -0,0 +1,133 @@ +import unittest + +import pandas as pd + +from methods.battery_optimizer import DT_H, optimize_battery_plan + + +class BatteryCostOptimizerTest(unittest.TestCase): + def plan(self, load, pv, import_prices=None, export_prices=None, **overrides): + index = pd.date_range("2026-09-29T00:00:00", periods=len(load), freq="5min") + config = { + "batt_capacity_kwh": 2.0, + "batt_power_kw": 1.0, + "batt_min_soc": 0.0, + "batt_max_soc": 100.0, + "batt_charge_efficiency": 1.0, + "batt_discharge_efficiency": 1.0, + "batt_soc_percent": 0.0, + "batt_grid_charging_enabled": False, + "batt_degradation_chf_kwh": 0.0, + "tarif_bezug": "dynamic", + "tarif_einspeisung": "dynamic", + "tarif_peak_fest": 0.0, + } + config.update(overrides) + data = { + "config": config, + "df_fut": pd.DataFrame({ + "import_price": import_prices or [0.30] * len(index), + "export_price": export_prices or [0.10] * len(index), + }, index=index), + "current_soc_perc": config["batt_soc_percent"], + "current_month_peak_kw": overrides.get("current_month_peak_kw", 0.0), + } + return index, optimize_battery_plan(data, dict(zip(index, pv)), dict(zip(index, load))) + + def test_grid_charging_is_opt_in(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + ) + self.assertTrue(all(plan["battery"][t] <= 1e-6 for t in index)) + + def test_cheap_grid_energy_is_shifted_to_expensive_period(self): + index, plan = self.plan( + [500.0] * 4, + [0.0] * 4, + import_prices=[0.05, 0.05, 0.50, 0.50], + batt_grid_charging_enabled=True, + ) + self.assertGreater(plan["battery"][index[0]], 0.0) + self.assertLess(plan["battery"][index[-1]], 0.0) + self.assertGreater(plan["grid"][index[0]], 500.0) + self.assertAlmostEqual(plan["grid"][index[-1]], 0.0, places=5) + + def test_high_feed_in_value_prefers_export(self): + index, plan = self.plan( + [0.0, 1000.0], + [1000.0, 0.0], + import_prices=[0.20, 0.20], + export_prices=[0.60, 0.60], + ) + self.assertAlmostEqual(plan["grid"][index[0]], -1000.0, places=5) + self.assertAlmostEqual(plan["grid"][index[1]], 1000.0, places=5) + + def test_peak_tariff_prevents_grid_charge_above_existing_peak(self): + index, plan = self.plan( + [1000.0] * 6, + [0.0] * 6, + import_prices=[0.05] * 3 + [0.50] * 3, + batt_grid_charging_enabled=True, + tarif_peak_fest=20.0, + current_month_peak_kw=1.0, + ) + self.assertLessEqual(plan["planned_peak_kw"], 1.0 + 1e-7) + self.assertTrue(all(plan["grid"][t] <= 1000.0 + 1e-5 for t in index)) + + def test_nearly_empty_battery_is_not_discharged_further_at_low_value(self): + index, plan = self.plan( + [1000.0] * 4, + [0.0] * 4, + import_prices=[0.01] * 4, + export_prices=[0.0] * 4, + batt_soc_percent=5.0, + batt_economic_reserve_soc_percent=10.0, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(all(plan["battery"][timestamp] >= -1e-6 for timestamp in index)) + self.assertTrue(all(abs(plan["grid"][timestamp] - 1000.0) <= 1e-5 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 5.0) + + def test_profitable_export_discharge_remains_allowed_above_reserve(self): + index, plan = self.plan( + [100.0, 100.0], + [1000.0, 1000.0], + import_prices=[0.20, 0.20], + export_prices=[0.80, 0.80], + batt_soc_percent=100.0, + batt_economic_reserve_soc_percent=10.0, + batt_degradation_chf_kwh=0.03, + batt_terminal_value_chf_kwh=0.0, + ) + self.assertTrue(any(plan["battery"][timestamp] < -1e-6 for timestamp in index)) + self.assertTrue(all(plan["grid"][timestamp] <= -899.0 for timestamp in index)) + self.assertAlmostEqual(plan["economic_min_soc_percent"], 10.0) + + def test_power_and_soc_limits_hold(self): + load = [0.0] * 12 + [1000.0] * 12 + pv = [1000.0] * 12 + [0.0] * 12 + index, plan = self.plan( + load, + pv, + batt_capacity_kwh=1.0, + batt_power_kw=0.5, + batt_min_soc=20.0, + batt_max_soc=80.0, + batt_soc_percent=20.0, + ) + soc = 0.2 + for timestamp in index: + target = plan["battery"][timestamp] + self.assertLessEqual(abs(target), 500.0 + 1e-6) + if target >= 0.0: + soc += target * DT_H / 1000.0 + else: + soc += target * DT_H / 1000.0 + self.assertGreaterEqual(soc, 0.2 - 1e-8) + self.assertLessEqual(soc, 0.8 + 1e-8) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py new file mode 100644 index 0000000..42767ef --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_load_forecast.py @@ -0,0 +1,66 @@ +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from methods.var_2 import predict +from methods.var_11 import predict as predict_repeat + + +class LoadProfileForecastTest(unittest.TestCase): + def test_profile_keeps_daily_shape_across_48_hours(self): + history_index = pd.date_range("2026-09-21", periods=8 * 288, freq="5min") + phase = 2 * np.pi * ( + history_index.hour.to_numpy() + history_index.minute.to_numpy() / 60.0 + ) / 24.0 + history = pd.DataFrame( + {"Hausverbrauch": 1800.0 + 900.0 * np.cos(phase - np.pi)}, + index=history_index, + ) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=2 * 288, freq="5min") + future_phase = 2 * np.pi * ( + future_index.hour.to_numpy() + future_index.minute.to_numpy() / 60.0 + ) / 24.0 + future = pd.DataFrame( + { + "temp_c": 15.0, + "cloud": 20.0, + "hour_sin": np.sin(future_phase), + "hour_cos": np.cos(future_phase), + "weekday": future_index.weekday, + "is_weekday": (future_index.weekday < 5).astype(int), + }, + index=future_index, + ) + data = { + "config": {"anlagen_id": "test"}, + "df_hist": history, + "df_load_training": history, + "df_recent_raw": history.iloc[-2 * 288 :], + "df_fut": future, + } + + with patch("methods.var_2.load_model", return_value=None): + values = np.array(list(predict(data).values())) + + self.assertGreater(float(values.max() - values.min()), 1200.0) + np.testing.assert_allclose(values[:288], values[288:], rtol=0.0, atol=1e-6) + + def test_repeat_profile_remains_available_on_second_day(self): + history_index = pd.date_range("2026-09-29", periods=288, freq="5min") + daily_values = np.arange(288, dtype=float) + 1000.0 + history = pd.DataFrame({"Hausverbrauch": daily_values}, index=history_index) + future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=576, freq="5min") + + values = np.array(list(predict_repeat({ + "df_hist": history, + "df_fut": pd.DataFrame(index=future_index), + }).values())) + + np.testing.assert_allclose(values[:288], daily_values) + np.testing.assert_allclose(values[288:], daily_values) + + +if __name__ == "__main__": + unittest.main() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py new file mode 100644 index 0000000..dd6220c --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_model_isolation.py @@ -0,0 +1,71 @@ +import ast +import contextlib +import datetime +import io +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock +from model_isolation import collect_predictions + + +class ModelIsolationTest(unittest.TestCase): + def setUp(self): + self.index = [datetime.datetime(2026,10,2,6,0) + datetime.timedelta(minutes=5*i) for i in range(3)] + self.data = {'df_fut': SimpleNamespace(index=self.index)} + self.good = {t: 1000.0 for t in self.index} + def model(self, values=None): + return SimpleNamespace(predict=Mock(return_value=self.good if values is None else values)) + def call(self, models, enabled=lambda c,n:True): + return collect_predictions(self.data, {}, models, enabled) + def test_one_failed_model_does_not_remove_other_families(self): + bad=self.model();bad.predict.side_effect=ValueError('private path not logged') + forecasts,errors=self.call({1:self.model(),10:bad,21:self.model()}) + self.assertEqual(forecasts[1],self.good);self.assertEqual(forecasts[21],self.good) + self.assertEqual(forecasts[10],{});self.assertEqual(errors[10]['errorType'],'ValueError') + self.assertNotIn('private',str(errors)) + def test_missing_timestamp_does_not_get_filled(self): + forecasts,errors=self.call({1:self.model(),10:self.model({self.index[0]:20.0})}) + self.assertFalse(forecasts[10]);self.assertIn(10,errors) + def test_nonfinite_negative_and_boolean_rejected(self): + for value in (float('nan'),float('inf'),-1.0,True,'2'): + with self.subTest(value=value): + result,errors=self.call({1:self.model(),10:self.model({t:value for t in self.index})}) + self.assertFalse(result[10]);self.assertIn(10,errors) + def test_explicit_zero_forecast_is_not_imputed(self): + result,errors=self.call({10:self.model({t:0.0 for t in self.index})}) + self.assertEqual(errors,{});self.assertEqual(sum(result[10].values()),0.) + def test_all_requested_models_fail_closed(self): + with self.assertRaisesRegex(ValueError,'All requested'): + self.call({10:self.model({})}) + def test_disabled_models_not_called(self): + model=self.model();result,errors=self.call({10:model},lambda c,n:False) + model.predict.assert_not_called();self.assertEqual(result,{10:{}});self.assertEqual(errors,{}) + def test_input_time_order_preserved(self): + result,_=self.call({1:self.model(dict(reversed(list(self.good.items()))))}) + self.assertEqual(list(result[1]),self.index) + def test_run_forecast_publishes_valid_pairs_after_repeat_failure(self): + root=Path(__file__).resolve().parents[1] + tree=ast.parse((root/'main.py').read_text()) + fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run_forecast') + modules={n:self.model() for n in (1,2,3,10,11,13,21,22,23)} + modules[10].predict.side_effect=ValueError('profile gap') + client=Mock();published=Mock() + data={**self.data,'current_soc_perc':20.,'current_soc_source':'telemetry'} + ns={'datetime':datetime,'LOCAL_TZ':datetime.timezone.utc,'traceback':Mock(), + 'get_configs':lambda:[{'anlagen_id':'test','batt_capacity_kwh':10.}], + 'InfluxDBClient':Mock(return_value=client),'SYNCHRONOUS':object(), + 'INFLUX_URL':'offline','INFLUX_TOKEN':'synthetic','INFLUX_ORG':'offline', + 'INFLUX_BUCKET':'offline','INFLUX_TIMEOUT_MS':1, + 'build_data_object':lambda *a,**k:data,'active':lambda c,n:True, + 'require_recent_telemetry':lambda *a,**k:{},'collect_predictions':collect_predictions, + '_v4_publish_forecasts':published,'battery_soc_points':lambda *a:[], + '_forecast_point':lambda *a:a,'_snapshot_point':lambda *a:a, + 'write_quality_metrics':Mock(),**{'v'+str(n):m for n,m in modules.items()}} + exec(compile(ast.Module(body=[fn],type_ignores=[]),'source-run-forecast','exec'),ns) + with contextlib.redirect_stdout(io.StringIO()):r=ns['run_forecast']() + self.assertEqual(r['completed'],['test']) + families=published.call_args.args[1] + self.assertTrue(families[0][1] and families[0][2]);self.assertFalse(families[1][1]) + self.assertTrue(families[2][1] and families[2][2]);modules[13].predict.assert_not_called() + client.write_api.return_value.write.assert_called_once() diff --git a/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py new file mode 100644 index 0000000..693c530 --- /dev/null +++ b/services/netplan-v4/acceptance/forecast-src/tests/test_telemetry_integrity.py @@ -0,0 +1,213 @@ +"""Offline regressions against the exact reviewed orchestration source. +Only selected function definitions are compiled; main is NOT imported, and there +is no database, network, model-file, prediction-publication or device access. +""" +import ast +import contextlib +import datetime +import io +from pathlib import Path +import unittest +from unittest.mock import Mock, patch + +import numpy as np +import pandas as pd +from telemetry_quality import (TelemetryUnavailable, require_recent_telemetry, + sanitize_measured_frame, repeat_daily_profile, profile_source_value) + +ROOT = Path(__file__).resolve().parents[1] +AT = pd.Timestamp('2026-10-01T20:00:00') + + +def functions(*names, **extra): + tree = ast.parse((ROOT/'main.py').read_text()) + nodes = [n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names] + if {n.name for n in nodes} != set(names): + raise AssertionError('Reviewed function missing') + ns = {'pd':pd,'np':np,'datetime':datetime, + 'sanitize_measured_frame':sanitize_measured_frame, + 'require_recent_telemetry':require_recent_telemetry} + ns.update(extra) + exec(compile(ast.Module(body=nodes,type_ignores=[]),str(ROOT/'main.py'),'exec'),ns) + return ns + + +def raw_frame(start=None, periods=288): + index=pd.date_range(start if start is not None else AT-pd.Timedelta(days=1),periods=periods,freq='5min') + return pd.DataFrame({'PV':200.,'Hausverbrauch':3200.,'SOC':25.,'Netzleistung':3000.},index=index) + + +class MeasuredTelemetryTest(unittest.TestCase): + def test_empty_history_is_not_zero_consumption(self): + ns=functions('_fill_defaults','_add_time_features') + frame=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=3,freq='5min')),True) + self.assertTrue(frame[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + self.assertEqual(frame['temp_c'].tolist(),[15.]*3) + + def test_recorded_zero_is_preserved(self): + frame=raw_frame(periods=3);frame[:]=0. + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(out['Hausverbrauch'].tolist(),[0.]*3) + self.assertEqual(out['PV'].tolist(),[0.]*3) + + def test_internal_and_trailing_measurement_gaps_remain_missing(self): + frame=raw_frame(periods=7) + frame.loc[frame.index[[0,2,3,6]],['PV','Hausverbrauch','SOC','Netzleistung']]=np.nan + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](frame,True) + self.assertEqual(int(out['Hausverbrauch'].isna().sum()),4) + self.assertEqual(int(out['SOC'].isna().sum()),4) + self.assertEqual(int(out['Netzleistung'].isna().sum()),4) + + def test_bad_values_not_real_measurements(self): + frame=pd.DataFrame({'Hausverbrauch':[np.inf,-1.,True,0.,250.], 'Netzleistung':[-100.,np.nan,0.,1.,2.], 'SOC':[101.,-1.,np.inf,0.,100.]}) + out=sanitize_measured_frame(frame) + self.assertTrue(out['Hausverbrauch'].iloc[:3].isna().all()) + self.assertEqual(out['Hausverbrauch'].iloc[3],0.) + self.assertEqual(out['Netzleistung'].iloc[0],-100.) + self.assertTrue(out['SOC'].iloc[:3].isna().all()) + + def test_recent_recorded_zero_passes(self): + frame=raw_frame();frame[['PV','Hausverbrauch']]=0. + report=require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_missing_raw_cannot_be_hidden_by_filled_feature_grid(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':pd.DataFrame(),'df_hist':raw_frame()}) + + def test_stale_values_fail(self): + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':raw_frame(AT-pd.Timedelta(days=2))}) + + def test_freshness_is_checked_per_field(self): + frame=raw_frame();frame.loc[frame.index[-12:],'Hausverbrauch']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'Hausverbrauch'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_future_measurements_do_not_rescue_freshness(self): + frame=raw_frame(AT,periods=3) + with self.assertRaises(TelemetryUnavailable): + require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + + def test_aware_timestamps_normalized_to_utc(self): + frame=raw_frame();frame.index=frame.index.tz_localize('UTC').tz_convert('Europe/Zurich') + report=require_recent_telemetry({'now':AT.tz_localize('UTC'),'df_recent_raw':frame}) + self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.) + + def test_soc_staleness_cannot_be_hidden(self): + frame=raw_frame();frame.loc[frame.index[-24:],'SOC']=np.nan + with self.assertRaisesRegex(TelemetryUnavailable,'SOC'): + require_recent_telemetry({'now':AT,'df_recent_raw':frame},['SOC']) + + def test_only_exogenous_future_defaults_are_filled(self): + ns=functions('_fill_defaults','_add_time_features') + out=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=6,freq='5min')),False) + self.assertTrue(out[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all()) + + def test_input_unchanged(self): + frame=raw_frame();before=frame.copy(deep=True) + sanitize_measured_frame(frame);require_recent_telemetry({'now':AT,'df_recent_raw':frame}) + pd.testing.assert_frame_equal(frame,before) + + +class HistoryOrchestrationTest(unittest.TestCase): + def data(self,telemetry): + class FrozenDateTime(datetime.datetime): + @classmethod + def utcnow(cls):return AT.to_pydatetime() + import types + dates=types.SimpleNamespace(datetime=FrozenDateTime,timedelta=datetime.timedelta) + weather=pd.DataFrame({'temp_c':15.,'cloud':20.},index=pd.date_range(AT-pd.Timedelta(days=14),AT+pd.Timedelta(days=2),freq='5min')) + fetch=Mock(return_value=(telemetry,weather,pd.DataFrame())) + ns=functions('build_data_object','_fill_defaults','_add_time_features','_longest_consistent_segment',datetime=dates,fetch_influx_frames=fetch,FORECAST_HORIZON_HOURS=48) + return ns['build_data_object']({'anlagen_id':'offline'},False) + + def test_weather_tail_does_not_erase_historical_load(self): + telemetry=raw_frame(AT-pd.Timedelta(days=2),periods=288) + data=self.data(telemetry) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw']['Hausverbrauch'].count(),len(telemetry)) + self.assertEqual(data['df_recent_raw'].index.max(),telemetry.index.max()) + self.assertTrue(data['df_hist']['Hausverbrauch'].iloc[-24:].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_no_measurements_preserves_all_missing(self): + data=self.data(pd.DataFrame()) + self.assertTrue(data['df_hist']['Hausverbrauch'].isna().all()) + with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data) + + def test_reconnected_short_tail_does_not_delete_past_profile(self): + old=raw_frame(AT-pd.Timedelta(days=2),periods=288) + new=raw_frame(AT-pd.Timedelta(minutes=10),periods=2) + data=self.data(pd.concat([old,new])) + self.assertEqual(data['df_hist']['Hausverbrauch'].count(),290) + self.assertEqual(require_recent_telemetry(data)['Hausverbrauch']['ageMinutes'],5.) + + def test_stale_forecast_never_predicts_or_publishes(self): + import types + models={n:types.SimpleNamespace(predict=Mock()) for n in (1,2,3,10,11,13,21,22,23)} + client=Mock();write=Mock();client.write_api.return_value=write + publish=Mock();trace=Mock() + ns=functions('run_forecast',get_configs=lambda:[{'anlagen_id':'offline','batt_capacity_kwh':10}], + InfluxDBClient=Mock(return_value=client),SYNCHRONOUS=object(), + INFLUX_URL='offline',INFLUX_TOKEN='synthetic',INFLUX_ORG='offline',INFLUX_BUCKET='offline',INFLUX_TIMEOUT_MS=1, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},active=lambda *args:True, + _v4_publish_forecasts=publish,traceback=trace,**{'v'+str(k):v for k,v in models.items()}) + with contextlib.redirect_stdout(io.StringIO()),self.assertRaises(RuntimeError):ns['run_forecast']() + for model in models.values():model.predict.assert_not_called() + publish.assert_not_called();write.write.assert_not_called();client.close.assert_called_once() + + def test_stale_training_does_not_overwrite_model(self): + import types + train=Mock();mod=types.SimpleNamespace(train=train) + ns=functions('run_training',get_configs=lambda:[{'anlagen_id':'offline'}], + MODEL_MODULES={2:mod},QUALITY_TARGETS={2:'Hausverbrauch'},active=lambda *a:True, + build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},traceback=Mock()) + with contextlib.redirect_stdout(io.StringIO()):ns['run_training']() + train.assert_not_called() + + def test_queries_exclude_forecasts_and_align_interval_starts(self): + source=(ROOT/'main.py').read_text();ns=functions('fetch_influx_frames', + HISTORY_START='1970-01-01T00:00:00Z',FORECAST_HORIZON_HOURS=48,INFLUX_BUCKET='offline', + _query_df=Mock(return_value=pd.DataFrame()),_pivot_frame=lambda df,fields:df, + _tariff_frame=lambda df,cfg:df,_time_literal=lambda t:t.isoformat()) + ns['fetch_influx_frames']({'anlagen_id':'offline'},False) + query=ns['_query_df'].call_args_list[0].args[0] + self.assertIn('timeSrc: "_start"',query) + self.assertIn('!= "forecast_snapshot"',query) + self.assertIn('!= "forecast"',query) + + +class RepeatProfileIntegrityTest(unittest.TestCase): + def test_pv_repeats_on_second_day(self): + history=raw_frame();history['PV']=np.maximum(0.,np.sin(np.arange(288)*2*np.pi/288))*12000. + idx=pd.date_range(AT,periods=576,freq='5min') + actual=np.array(list(repeat_daily_profile(history,idx,'PV').values())) + np.testing.assert_allclose(actual[:288],history['PV']) + np.testing.assert_allclose(actual[288:],history['PV']) + + def test_missing_yesterday_uses_older_finite_day(self): + t=AT + history=pd.DataFrame({'Hausverbrauch':[3500.,np.nan]},index=[t-pd.Timedelta(days=7),t-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,t,'Hausverbrauch'),3500.) + + def test_missing_profile_is_not_zero(self): + history=pd.DataFrame({'Hausverbrauch':[np.nan]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):repeat_daily_profile(history,[AT],'Hausverbrauch') + + def test_recorded_profile_zero_is_valid(self): + history=pd.DataFrame({'PV':[0.]},index=[AT-pd.Timedelta(days=1)]) + self.assertEqual(profile_source_value(history,AT,'PV'),0.) + + def test_negative_profile_cannot_be_silently_clamped(self): + history=pd.DataFrame({'Hausverbrauch':[-100.]},index=[AT-pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + def test_future_value_is_never_used_as_history(self): + history=pd.DataFrame({'Hausverbrauch':[3300.]},index=[AT+pd.Timedelta(days=1)]) + with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch') + + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json new file mode 100644 index 0000000..c667975 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/SOURCE_MANIFEST.json @@ -0,0 +1,9 @@ +{ + "libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" +} diff --git a/services/netplan-v4/acceptance/php-src/check.php b/services/netplan-v4/acceptance/php-src/check.php new file mode 100644 index 0000000..feb59a8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/check.php @@ -0,0 +1,45 @@ + $expected) { + $path = __DIR__ . '/' . $relative; + if (!is_file($path) || hash_file('sha256', $path) !== $expected) { + throw new RuntimeException('Source manifest mismatch: ' . $relative); + } + $command = escapeshellarg(PHP_BINARY) . ' -l ' . escapeshellarg($path); + $lines = []; + exec($command, $lines, $code); + if ($code !== 0) { + throw new RuntimeException('PHP syntax failure: ' . $relative); + } + } + require __DIR__ . '/libs/NetzfahrplanV4Bezugszaehler.php'; + require __DIR__ . '/libs/NetzfahrplanV4Betriebsdaten.php'; + require __DIR__ . '/libs/ManagerNetzfahrplanV4Trait.php'; + require __DIR__ . '/tests/fixtures/NativeV4Scenarios.php'; + require __DIR__ . '/tests/fixtures/V4BezugszaehlerScenarios.php'; + $native = \Belevo\EnelixEMS\Tests\NativeV4Scenarios::run(); + $meters = \Belevo\EnelixEMS\Tests\V4BezugszaehlerScenarios::run(); + if (count($native) !== 24 || count($meters) !== 20) { + throw new RuntimeException('Expected 24 native and 20 meter scenarios.'); + } + foreach (array_merge($native, $meters) as $name) { + echo 'PASS ', $name, "\n"; + } + echo 'PHP ', PHP_VERSION, ': 44 offline conversion scenarios passed.', "\n"; + echo "NOT a full EMS suite and NOT an IP-Symcon runtime/physical test.\n"; +} catch (Throwable $error) { + fwrite(STDERR, 'FAIL: ' . $error->getMessage() . "\n"); + exit(1); +} diff --git a/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php new file mode 100644 index 0000000..c6e7412 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php @@ -0,0 +1,164 @@ +RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false); + $this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}'); + $this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0); + $this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]'); + $this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + $this->RegisterTimer('NetzfahrplanV4Senden', 0, + "IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);"); + } + + public function GetNetzfahrplanV4Diagnose(): string + { + try { + $manager = []; + foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout', + 'Lastspitzenmodus'] as $key) { + $manager[$key] = $this->ReadPropertyInteger($key); + } + foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) { + $manager[$key] = $this->ReadPropertyFloat($key); + } + $manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified'; + $manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv'); + $manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt'); + $manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR); + $manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR); + $manager['V4BezugszaehlerQuellen'] = json_decode( + $this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR + ); + $assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR); + $cache = $this->leseJsonAttribut('VerbraucherCache'); + $controllers = []; + foreach ($this->aktiveVerbraucherIDs() as $id) { + if (!IPS_InstanceExists($id) + || IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') { + continue; + } + $c = ['InstanzID' => $id]; + foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID', + 'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) { + $c[$key] = IPS_GetProperty($id, $key); + } + $entry = $cache[(string) $id] ?? []; + $c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0; + $c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false; + foreach (($entry['Daten']['Zustand'] ?? []) as $state) { + if (($state['Kennung'] ?? '') === 'HystereseAktiv') { + $c['HystereseAktiv'] = $state['Wert']; + } + } + $controllers[] = $c; + } + $evidence = []; + $evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID'); + if ($evidenceID > 0) { + if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) { + throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.'); + } + $evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR); + if (!is_array($evidence)) { + throw new RuntimeException('Ungueltiger Messnachweis.'); + } + } + $read = static function (int $id): array { + if (!IPS_VariableExists($id)) { + throw new RuntimeException('Messvariable ' . $id . ' fehlt.'); + } + $before = IPS_GetVariable($id); + $value = GetValue($id); + $after = IPS_GetVariable($id); + if ($before['VariableUpdated'] !== $after['VariableUpdated']) { + throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.'); + } + $object = IPS_GetObject($id); + return ['value' => $value, 'updated' => (int) $after['VariableUpdated'], + 'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']]; + }; + return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers, + $read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } catch (Throwable $error) { + return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR); + } + } + + public function GetNetzfahrplanV4Sendestatus(): string + { + return $this->ReadAttributeString('NetzfahrplanV4Sendestatus'); + } + + private function sendeNetzfahrplanV4(): void + { + if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) { + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}'); + return; + } + try { + if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) { + throw new RuntimeException('Netzfahrplanberechtigung fehlt.'); + } + $snapshot = $this->GetNetzfahrplanV4Diagnose(); + $data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR); + if (!isset($data['operation'])) { + throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.'); + } + // Decoding as objects preserves empty JSON dictionaries in the API contract. + $payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR); + if (!isset($payload->operation)) { + throw new RuntimeException('Keine konsistenten Betriebsdaten.'); + } + $id = $this->ReadAttributeString('LizenzInstallationID'); + $token = $this->ReadAttributeString('PrognoseInstallationsToken'); + if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') { + throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.'); + } + $url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation'; + $handle = curl_init($url); + if ($handle === false) { + throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.'); + } + try { + curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true, + CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5, + CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, + CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR), + CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]); + $answer = curl_exec($handle); + $status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); + if ($answer === false || $status !== 200) { + throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.'); + } + if (strlen($answer) > 65536) { + throw new RuntimeException('V4-Antwort ist zu gross.'); + } + $ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR); + if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) { + throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.'); + } + } finally { + curl_close($handle); + } + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt, + 'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } catch (Throwable $error) { + // Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands. + $this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([ + 'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR)); + } + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php new file mode 100644 index 0000000..35495da --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php @@ -0,0 +1,219 @@ + $max) { + throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.'); + } + return (float) $value; + } + + private static function timestamp($value): int + { + if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) { + throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.'); + } + return (new DateTimeImmutable($value))->getTimestamp(); + } + + /** $read returns ['value' => int|float, 'updated' => int], without changing a device. */ + private static function measurement(callable $read, int $id, int $now, int $age, string $label): array + { + if ($id <= 0) { + throw new InvalidArgumentException($label . ': Messquelle fehlt.'); + } + $m = $read($id); + if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now + || $now - $m['updated'] > $age) { + throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.'); + } + return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']]; + } + + /** No inference of paid peak or elapsed energy from power, reserve or forecast. */ + private static function evidence(array $input, int $now, string $meterID, array &$warnings): array + { + $peaks = []; + $past = null; + if ($input === []) { + $warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.'; + return [(object) [], null]; + } + if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) { + throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.'); + } + $measured = self::timestamp($input['measuredAt'] ?? null); + if ($measured > $now || $now - $measured > 120) { + throw new InvalidArgumentException('Messnachweis ist nicht aktuell.'); + } + $month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m'); + foreach (($input['measuredPeaks'] ?? []) as $key => $p) { + if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month + || !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) { + throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.'); + } + $peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']]; + } + if (!array_key_exists($month, $peaks)) { + $warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.'; + } + $q = intdiv($now, 900) * 900; + if (isset($input['quarterPast'])) { + $p = $input['quarterPast']; + // Stale quarter evidence remains absent, not extrapolated to the decision time. + if ($measured === $now && self::timestamp($p['start'] ?? null) === $q + && ($p['measuredSeconds'] ?? null) === $now - $q) { + $past = ['start' => gmdate('c', $q), + 'measuredSeconds' => $now - $q, + 'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)]; + } else { + $warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.'; + } + } + if ($now !== $q && $past === null) { + $warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.'; + } + return [(object) $peaks, $past]; + } + + /** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */ + public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array + { + $warnings = []; + $meter = NetzfahrplanV4Bezugszaehler::lesen( + $manager['V4BezugszaehlerQuellen'] ?? [], $read, $now + ); + $grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now, + (int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung'); + $gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor'); + $mode = $manager['Lastspitzenmodus'] ?? 0; + $import = null; + $months = []; + if ($mode === 1) { + $import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0); + } elseif ($mode === 2) { + foreach (($manager['Monatsgrenzen'] ?? []) as $row) { + $m = $row['MonatIndex'] ?? null; + if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) { + throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.'); + } + $months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0); + } + if (count($months) !== 12) { + throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.'); + } + } elseif ($mode !== 0) { + throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.'); + } + $export = !empty($manager['EinspeisebegrenzungAktiv']) + ? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null; + $result = []; + $seen = []; + $used = []; + foreach ($assets as $asset) { + $id = $asset['ID'] ?? ''; + if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) { + throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.'); + } + $seen[$id] = true; + $socID = (int) ($asset['SOCVariableID'] ?? 0); + $matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool { + return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID; + })); + if (count($matches) !== 1) { + throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.'); + } + $c = $matches[0]; + $controllerID = $c['InstanzID']; + if (isset($used[$controllerID])) { + throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.'); + } + $used[$controllerID] = true; + if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) { + throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.'); + } + $opts = $manager['BatterieOptionen'][$id] ?? []; + $nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001); + $usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal); + if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) { + throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.'); + } + $capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal); + $age = (int) ($c['MesswertMaxAlter'] ?? 30); + $soc = self::measurement($read, $socID, $now, $age, $id . ' SOC'); + $charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden'); + $discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen'); + $min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100), + self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100)); + $max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100); + $socValue = self::number($soc['value'], 'SOC', 0, 100); + $physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min); + if ($socValue < $physicalMin || $socValue > $max) { + throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.'); + } + if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) { + throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.'); + } + $maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0)); + $maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0)); + if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) { + $maxCharge = $maxDischarge = 0.0; + $warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.'; + } + $rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100)); + $blocked = !empty($c['HystereseAktiv']) || $socValue <= $min; + if ($blocked) { + $warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.'; + } + if ($socValue < $min) { + $warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.'; + } + $result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue, + 'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge, + 'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])), + 'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true, + 'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true, + 'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm]; + } + if ($result === []) { + throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.'); + } + [$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings); + $payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW, + 'meteringBoundary' => 'common_pcc', 'batteries' => $result, + 'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months], + 'measuredPeaks' => $peaks, 'quarterPast' => $past]; + // A fresh acquisition sample is an operational estimate. It does NOT + // claim a calibrated billing-period boundary or full historical coverage. + $policy = [ + 'adapterVersion' => 'v4-native-estimated-meter-1', + 'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified', + 'mode' => $mode, 'importW' => $import, 'exportW' => $export, + 'months' => $months, + 'batteryPolicy' => array_map(static function (array $b): array { + return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent'])); + }, $result), + ]; + $payload['meterObservation'] = [ + 'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now), + 'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'], + 'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)), + ]; + $payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR)); + return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php new file mode 100644 index 0000000..dda9b72 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php @@ -0,0 +1,108 @@ + 8 + || array_keys($sources) !== range(0, count($sources) - 1)) { + throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.'); + } + $result = []; + $parent = null; + foreach ($sources as $source) { + if (!is_array($source) || count($source) !== 5 + || ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') { + throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.'); + } + $id = $source['VariableID'] ?? null; + $pid = $source['ElternID'] ?? null; + $ident = $source['Ident'] ?? null; + $factor = $source['FaktorZuKWh'] ?? null; + if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0 + || !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident) + || (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor) + || $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) { + throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.'); + } + if ($parent !== null && $parent !== $pid) { + throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.'); + } + $parent = $pid; + foreach ($result as $existing) { + if ($existing['Ident'] === $ident) { + throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.'); + } + } + $result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident, + 'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug']; + } + ksort($result, SORT_NUMERIC); + return array_values($result); + } + + public static function identitaet(array $sources): string + { + // Source/factor changes invalidate old measurement evidence; names do not matter. + return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources), + JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION)); + } + + private static function probe(array $source, callable $read, int $now, int $maxAge): array + { + $m = $read($source['VariableID']); + if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID'] + || ($m['ident'] ?? null) !== $source['Ident']) { + throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.'); + } + $value = $m['value'] ?? null; + $updated = $m['updated'] ?? null; + if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0 + || !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) { + throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.'); + } + $kwh = (float) $value * $source['FaktorZuKWh']; + if (!is_finite($kwh) || $kwh > 1.0e12) { + throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.'); + } + return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value, + 'totalKwh' => $kwh, 'updatedAt' => $updated]; + } + + /** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */ + public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array + { + if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) { + throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.'); + } + $sources = self::quellen($sources); + $samples = []; + foreach ($sources as $source) { + $samples[] = self::probe($source, $read, $now, $maxAge); + } + // Reread all channels to reject concurrent updates, including changes in the same second. + foreach ($sources as $i => $source) { + if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) { + throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.'); + } + } + $times = array_column($samples, 'updatedAt'); + if (max($times) - min($times) > $maxSkew) { + throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.'); + } + return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh', + 'totalKwh' => array_sum(array_column($samples, 'totalKwh')), + 'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)), + 'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples, + 'billingEvidence' => false, 'historyComplete' => false]; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php new file mode 100644 index 0000000..adb80c8 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php @@ -0,0 +1,29 @@ +regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) { + self::assertStringNotContainsString($forbidden, $source); + } + self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source); + self::assertStringContainsString('/prognosis/planner-v4/operation', $source); + self::assertStringNotContainsString('/prognosis/schedule', $source); + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php new file mode 100644 index 0000000..41a2992 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php @@ -0,0 +1,29 @@ + 1, 'NetzbezugEnergieVariableID' => 9, + 'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1, + 'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0]; + $m['V4BezugszaehlerQuellen'] = [ + ['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44, + 'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]]; + $c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3, + 'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5, + 'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30, + 'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]]; + $read = static function (int $id) use ($now): array { + return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id], + 'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other']; + }; + $check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } }; + $fail = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid-input rejection'); + }; + $cases = []; + $build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array { + return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now); + }; + $cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void { + $x = $build()['operation']; $b = $x['batteries'][0]; + $check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0 + && $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0); + }; + $cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void { + $x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null); + }; + $cases['empty_maps_are_objects'] = static function () use ($build, $check): void { + $x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW)); + }; + $cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void { + $m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0); + }; + $cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void { + $m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation']; + $check($x['limits']['importW'] === null && $x['limits']['exportW'] === null); + }; + $cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void { + $m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = []; + for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; } + $x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0); + array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m)); + }; + $cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void { + $a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a)); + $m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]]; + $check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44); + }; + $cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void { + $a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a)); + }; + $cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void { + $c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c)); + }; + $cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void { + $c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c)); + }; + $cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void { + $r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; }; + $fail(static fn() => $build(null, null, null, $r)); + }; + $cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void { + $c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c)); + }; + $cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void { + $r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; }; + $b = $build(null, null, null, $r)['operation']['batteries'][0]; + $check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true); + }; + $cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void { + $c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0]; + $check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0); + }; + $cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void { + $c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c); + $check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1); + }; + $cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void { + $check($build()['operation']['batteries'][0]['gridCharging'] === false); + $m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true); + }; + $ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now), + 'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']], + 'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]]; + $cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void { + $x = $build(null, null, null, null, $ev); + $check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5); + }; + $cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void { + $ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev); + $check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0); + }; + $cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void { + $ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['deterministic_retry_event_id'] = static function () use ($build, $check): void { + $check($build()['operation']['eventId'] === $build()['operation']['eventId']); + }; + $cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void { + $ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev)); + }; + $cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void { + unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m)); + }; + $cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void { + $v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203 + && !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php new file mode 100644 index 0000000..7e08052 --- /dev/null +++ b/services/netplan-v4/acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php @@ -0,0 +1,113 @@ + 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'], + ]; + $samples = [ + 59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'], + 26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'], + ]; + $check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } }; + $reject = static function (callable $fn): void { + try { $fn(); } catch (InvalidArgumentException $e) { return; } + throw new RuntimeException('Expected invalid source/sample rejection'); + }; + $snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array { + $mm = $mm ?? $samples; + return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now); + }; + $cases = []; + $cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void { + $v = $snapshot(); + $check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']); + $check(!isset($v['measuredPeaks']) && !isset($v['quarterPast'])); + }; + $cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void { + $samples[26620]['value'] = 27.25; + $check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9); + }; + $cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void { + $samples[59607]['value'] = 0; + $check($snapshot(null, $samples)['totalKwh'] === 0.0); + }; + $cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void { + $check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources))); + }; + $cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001; + $check($old !== Meter::identitaet($sources)); + }; + $cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void { + $old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345; + $check($old !== Meter::identitaet($sources)); + }; + $cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); }; + $cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources)); + }; + $cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources)); + }; + $cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void { + $sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources)); + }; + $cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void { + unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources)); + }; + $cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void { + unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void { + foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) { + $samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples)); + } + }; + $cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void { + $samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples)); + }; + $cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void { + $calls = 0; + $read = static function (int $id) use ($samples, &$calls): array { + $v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v; + }; + $reject(static fn() => Meter::lesen($sources, $read, $now)); + }; + $cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void { + $samples[59607]['updated'] -= 1; + $v = $snapshot(null, $samples); + $check($v['sourceObservationFrom'] === gmdate('c', $now - 1)); + $check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now)); + $check(!$v['billingEvidence']); + }; + $passed = []; + foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; } + return $passed; + } +} diff --git a/services/netplan-v4/acceptance/read_native_forecasts.py b/services/netplan-v4/acceptance/read_native_forecasts.py new file mode 100644 index 0000000..abae9e8 --- /dev/null +++ b/services/netplan-v4/acceptance/read_native_forecasts.py @@ -0,0 +1,100 @@ +"""Read-only probe executed INSIDE the existing forecast-engine container. + +No get_configs() (it may migrate SQL), no training/prediction, no forecast writing, +no manual /run_now request, and no user credentials in output. Reads one plant's +stored raw 5-minute forecast values and the input frames used by the engine. +""" +import contextlib +from datetime import datetime, timezone +import hashlib +import io +import json +import math +import os +from pathlib import Path +import sqlite3 +from urllib.parse import quote +from uuid import UUID + +class Discard(io.TextIOBase): + def write(self,text):return len(text) + +def clean_time(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def run(): + aid=str(UUID(os.environ['ENELIX_ACCEPTANCE_PLANT'])) + os.environ['FORECAST_INFLUX_TIMEOUT_MS']='20000' + # These assignments affect only this diagnostic process, not the service. + with contextlib.redirect_stdout(Discard()),contextlib.redirect_stderr(Discard()): + import main + import numpy as np + import pandas as pd + path=Path(main.SQLITE_DB_PATH) + if not path.is_file():raise RuntimeError('Existing configuration database missing') + con=sqlite3.connect('file:'+quote(str(path))+'?mode=ro',uri=True) + con.row_factory=sqlite3.Row + try:row=con.execute('SELECT * FROM anlagen_meta WHERE anlagen_id=?',(aid,)).fetchone() + finally:con.close() + if row is None:raise RuntimeError('Installation not found') + cfg=dict(row) + cfg['daecher']=json.loads(cfg.get('daecher') or '[]') + for key,default in [('ac_leistung',10.),('batt_capacity_kwh',0.),('batt_power_kw',0.),('tarif_bezug_fest',.3),('tarif_einspeisung_fest',.1),('tarif_peak_fest',5.)]: + value=cfg.get(key);cfg[key]=float(default if value is None or value=='' else value) + data=main.build_data_object(cfg,training=False) + frames={} + for key in ('df_hist','df_recent_raw','df_load_training','df_fut'): + frame=data.get(key) + if frame is None:frames[key]={'available':False};continue + item={'rows':len(frame),'from':clean_time(frame.index.min()) if len(frame) else None,'until':clean_time(frame.index.max()) if len(frame) else None,'columns':{}} + for col in ('Hausverbrauch','PV','Netzleistung','SOC'): + if col not in frame.columns:continue + values=pd.to_numeric(frame[col],errors='coerce');valid=values[np.isfinite(values)] + item['columns'][col]={'finite':len(valid),'zeros':int((valid==0).sum()),'median':float(valid.median()) if len(valid) else None,'maximum':float(valid.max()) if len(valid) else None,'lastFiniteAt':clean_time(valid.index[-1]) if len(valid) else None,'recentValues':[{'time':clean_time(t),'value':float(v)} for t,v in valid.tail(12).items()]} + frames[key]=item + start=datetime.now(timezone.utc).replace(second=0,microsecond=0) + from datetime import timedelta + end=start+timedelta(hours=48) + fields=('prog_var_1','prog_var_2','prog_var_10','prog_var_11','prog_var_21','prog_var_22') + field_filter=' or '.join('r["_field"] == '+json.dumps(f) for f in fields) + query='''from(bucket: %s) + |> range(start: %s, stop: %s) + |> filter(fn: (r) => r["_measurement"] == "api_telemetry") + |> filter(fn: (r) => r["anlagen_id"] == %s) + |> filter(fn: (r) => r["data_type"] == "forecast") + |> filter(fn: (r) => %s) + |> keep(columns: ["_time", "_field", "_value"]) + ''' % (json.dumps(main.INFLUX_BUCKET),start.isoformat(),end.isoformat(),json.dumps(aid),field_filter) + client=main.InfluxDBClient(url=main.INFLUX_URL,token=main.INFLUX_TOKEN,org=main.INFLUX_ORG,timeout=20000) + series={field:[] for field in fields} + try: + tables=client.query_api().query(org=main.INFLUX_ORG,query=query) + for table in tables: + for record in table.records: + value=record.get_value();field=record.get_field() + if field in series and isinstance(value,(int,float)) and math.isfinite(value): + series[field].append({'time':clean_time(record.get_time()),'value':float(value)}) + finally:client.close() + for field in series:series[field].sort(key=lambda p:p['time']) + versions={} + for relative in ('main.py','methods/var_1.py','methods/var_2.py','methods/var_10.py','methods/var_11.py','methods/var_21.py','methods/var_22.py'): + source=Path('/app')/relative + if source.is_file():versions[relative]=hashlib.sha256(source.read_bytes()).hexdigest() + summary={} + for field,points in series.items(): + values=[p['value'] for p in points] + summary[field]={'points':len(values),'allZero':bool(values) and max(abs(v) for v in values)==0,'maximumW':max(values) if values else None,'from':points[0]['time'] if points else None,'until':points[-1]['time'] if points else None} + native=True + for points in series.values(): + stamps=[datetime.fromisoformat(p['time']).timestamp() for p in points] + if not stamps or any(t%300 for t in stamps) or any(b-a!=300 for a,b in zip(stamps,stamps[1:])):native=False + return {'status':'read_only_acquired','installationId':aid,'observedAt':datetime.now(timezone.utc).isoformat(),'queryResolution':'raw_no_chart_resampling','nativeFiveMinuteForecast':native,'generationTimeVerified':False,'inputFrames':frames,'forecastSummary':summary,'forecastSeries':series,'runningSourceHashes':versions,'modelWrite':False,'liveControlChanged':False} + +try: + result=run() +except Exception as error: + # Never echo exception details containing SQL payloads, credentials or URLs. + result={'status':'read_only_probe_failed','errorType':type(error).__name__,'liveControlChanged':False} +print(json.dumps(result,allow_nan=False)) diff --git a/services/netplan-v4/approved_previous_assets.json b/services/netplan-v4/approved_previous_assets.json new file mode 100644 index 0000000..3c59814 --- /dev/null +++ b/services/netplan-v4/approved_previous_assets.json @@ -0,0 +1,5 @@ +{ + "license/public/netplan-v4.js": [ + "e859e91a52dd1d6c96f411d16caed17375f6925bf867d09992ab32ce93bff5d7" + ] +} diff --git a/services/netplan-v4/commissioning/APPLICATION_STATUS.md b/services/netplan-v4/commissioning/APPLICATION_STATUS.md new file mode 100644 index 0000000..b716bd1 --- /dev/null +++ b/services/netplan-v4/commissioning/APPLICATION_STATUS.md @@ -0,0 +1,80 @@ +# Integrated application data/model path - 2026-10-02 + +User request: finish application implementation, stop asking for repeated hardware +confirmations and adding standalone diagnostic samplers. No actuator permission is +implied by that development request. Confirmed EV allocation remains 161.44 kWh / +39 kW each direction, with SDL reserve already excluded. + +## Implemented in the existing application + +- Native ManagerNetzfahrplanV4DatenTrait: 30-second acquisition of configured numeric + sources; durable private app-owned outbox; original timestamps, no new Modbus + polls; acknowledged per-plant batches with retries/backoff. Cursor advances only + after exact dataset/acceptedThrough acknowledgement. Batches bounded below the + existing portal's 1 MiB body limit. No credentials in data payload or reports. +- Existing device proxy now forwards /planner-v4/measurements through the existing + authentication, plant binding, licence check and separate V4 rate limiter. + Device route cannot modify the dataset mapping or issue a trial grant. +- Existing V4 database stores immutable versioned source mappings and observations. + Same-time conflicts reject the whole batch; repeated identical data is idempotent. + No writes to portal/users/legacy measurement databases. +- Existing V4 worker aggregates physical load using original source times. Virtual + EV/SDL filter outputs are not load inputs. Solar terminal variant counts signed + terminal power once. Small unobserved parts remain quantified, never zero-filled. + This configured formula remains an estimate, not independent AC/DC certification. +- Application profile training actually runs daily/weekly. Models are immutable, + bootstrap/holdout status explicit. Incumbent replacement requires a causal + holdout when available. Existing data and model versions are not relabelled. +- Three corrected load variants (daily robust profile, last-day profile, + weekday/weekend profile) are paired with the existing PV families 3/13/23. + They are tagged physical-profile-v1; they are NOT claimed to be the old trained + load algorithms under a new data name. Automatic future family registry work + and economic replay are still distinct from this load-profile implementation. +- Selecting corrected_profile in the existing GUI/setting now actually changes + the load entering the existing optimizer. Missing trained data cannot silently + fall back to the contaminated legacy household series. +- A current SDL request is an explicitly labelled PERSISTENCE SCENARIO for the + shadow economic calculation, not a published future schedule. Missing/stale SDL + is not zero. This is not a robust/full SDL production-dispatch policy. +- GUI now exposes dataset/source selection, records, usable equivalent hours, + model version, validation and real training cadence. Existing normal portal + app.js/index files are not changed. + +## Deployment package + +Server: commissioning/deploy_application.py --plant --apply +Rebuild/test V4 with Python 3.11, portal routing tests, consistent SQLite backup, +replace ONLY V4 and portal, configure the new per-plant dataset. No automatic +forecast-source switch and no control grant. Atomic file-bind replacement requires +portal recreation, not a simple restart. Additive DB rollback retains new data. +Default invocation without --apply validates source manifests only. + +Test host: /srv/agent/netplan-v4-application-build/install.php inside Symcon. +Installs a DATA-ONLY patch of the currently installed passive manager (not the +unreleased controller-trial candidate), reloads EMS, configures acquisition and +imports up to 48h of the existing 23-source observer as a delivery backlog. +Original observer and old raw journals remain intact. Existing separate observers +are NOT automatically stopped by this initial installer; consolidation should +follow confirmed native batch acceptance. No new root diagnostic category. +An asynchronous module registration may require one repeat, handled explicitly. + +Dataset lihrenmoos-physical-v1: minimum coverage 95%, longest unsupported portion +10s, at least 24 equivalent usable hours, 28-day profile history. The 95% rule is +an explicit modelling policy, not a statement that missing energy was measured. +Insufficient data remains collecting; no promised time to good forecasts. +All original acquisition thresholds remain unchanged. + +## Tests and honest limits + +Host tests use existing isolated QA dependencies; native tests mock IPS/HTTP and +use temporary files. Real container and kernel tests occur on the user-run deploy +commands, not during preparation. No services/modules/settings/devices were modified +by this development session. + +Still NOT a complete production commissioning: the corrected profile-to-local +feedback/dispatch integration, full automatic cost replay, independent long-term +outbox/server retention and staged live/failure acceptance remain open. The trial +code from previous commits remains gated and is not included in the data-only +native installation. No accountingEvidenceId/device watchdog proof is fabricated. +This release is an integrated application build, NOT a claim that the full original +multi-plant production scope is done. No main/beta release is authorized by tests alone. diff --git a/services/netplan-v4/commissioning/application-source/RELEASE.json b/services/netplan-v4/commissioning/application-source/RELEASE.json new file mode 100644 index 0000000..596cf5c --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/RELEASE.json @@ -0,0 +1,99 @@ +{ + "scope": "integrated_measurement_application", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "sourceHashes": { + ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713", + "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743", + "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + }, + "portalBefore": { + "netplan-v4-bridge.mjs": "f58c652b04cd2ac20f2bec11e1161c44b72c26267b4dfb42096a1fad36a3c9c2", + "public/netplan-v4.js": "6cd1d887bb08c73a815c92eb684a0992e8cb092f748770cc94debcfd9705a628" + }, + "createdAt": "2026-10-02T20:57:00.387145+00:00", + "controlEnabled": false +} diff --git a/services/netplan-v4/commissioning/application-source/server-dataset.json b/services/netplan-v4/commissioning/application-source/server-dataset.json new file mode 100644 index 0000000..7fd1190 --- /dev/null +++ b/services/netplan-v4/commissioning/application-source/server-dataset.json @@ -0,0 +1 @@ +{"datasetId":"lihrenmoos-physical-v1","mappingSha256":"517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b","inventorySha256":"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc","sources":[{"key":"grid","variableId":40348,"factorToW":1000,"maxAgeSeconds":60,"role":"grid"},{"key":"pv_goodwe1","variableId":48459,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_goodwe2","variableId":53802,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_solaredge","variableId":20335,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"physical_goodwe1","variableId":47725,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_goodwe2","variableId":35724,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_solaredge","variableId":21447,"factorToW":1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"ev_account","variableId":52020,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"sdl_account","variableId":25085,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"grid_display","variableId":49301,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe1","variableId":27361,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe2","variableId":23109,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_solaredge","variableId":51938,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_ev","variableId":32871,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_sdl","variableId":23879,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_charge","variableId":50230,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_discharge","variableId":43899,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t1","variableId":59607,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t2","variableId":26620,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_requested","variableId":19651,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"sdl_requested","variableId":38943,"role":"sdl_request","factorToW":1,"maxAgeSeconds":120},{"key":"solar_ac_value","variableId":37975,"role":"solar_raw","factorToW":1,"maxAgeSeconds":60},{"key":"solar_ac_scale","variableId":41853,"role":"solar_scale","factorToW":1,"maxAgeSeconds":60}],"formula":"solar_terminal_v1","solarReference":{"pvKey":"pv_solaredge","batteryKey":"physical_solaredge","rawKey":"solar_ac_value","scaleKey":"solar_ac_scale"},"minimumCoverage":0.95,"maximumGapSeconds":10,"minimumTrainingHours":24,"historyDays":28} diff --git a/services/netplan-v4/commissioning/deploy_application.py b/services/netplan-v4/commissioning/deploy_application.py new file mode 100644 index 0000000..4e27762 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_application.py @@ -0,0 +1,131 @@ +"""Deploy the integrated measurement/training application, NOT actuator permission. + +Only V4 and the portal are recreated. Existing legacy forecasts, tariffs, Symcon and +battery dispatch are unchanged. Source-only checks are the default; --apply is explicit. +""" +from pathlib import Path +from datetime import datetime, timezone +from uuid import UUID +import argparse,hashlib,json,os,sqlite3,subprocess,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PORTAL=ROOT.parent/'license' +MANIFEST=Path(__file__).with_name('application-source')/'RELEASE.json' +DATASET=Path(__file__).with_name('application-source')/'server-dataset.json' +PORTAL_FILES={'integrations/netplan-v4-bridge.mjs':'netplan-v4-bridge.mjs','gui/netplan-v4.js':'public/netplan-v4.js'} + +def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest() + +def verify(): + m=json.loads(MANIFEST.read_text()) + if m.get('scope')!='integrated_measurement_application' or not m.get('sourceHashes'): + raise ValueError('Unexpected application release manifest') + for n,h in m['sourceHashes'].items(): + p=ROOT/n + if Path(n).is_absolute() or '..' in Path(n).parts or p.is_symlink() or not p.is_file() or digest(p)!=h: + raise ValueError('Source drift: '+n) + for n,old in m['portalBefore'].items(): + if n not in PORTAL_FILES.values():raise ValueError('Unexpected portal target') + p=PORTAL/n + if p.is_symlink() or not p.is_file():raise ValueError('Portal target changed') + source=next(s for s,t in PORTAL_FILES.items() if t==n) + if digest(p) not in (old,m['sourceHashes'][source]):raise ValueError('Concurrent portal change; not overwritten') + return m + +def atomic(path,data,mode=0o644): + if path.is_symlink():raise ValueError('Symlink refused') + fd,name=tempfile.mkstemp(prefix='.v4-app-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno()) + os.chmod(name,mode);os.replace(name,path) + finally: + if os.path.exists(name):os.unlink(name) + +def backup_database(source,destination): + if not source.is_file() or source.is_symlink() or destination.exists():raise ValueError('Explicit existing database and new backup path required') + with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as a,sqlite3.connect(destination) as b: + a.backup(b) + if b.execute('PRAGMA integrity_check').fetchone()[0]!='ok':raise ValueError('Backup failed') + destination.chmod(0o600) + +BOOTSTRAP='''import json,os,urllib.request,sys +p=json.load(sys.stdin) +base='http://127.0.0.1:9100/internal/v2/prognosis/'+p['plant']+'/planner' +h={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +req=urllib.request.Request(base+'/datasets/'+p['dataset']['datasetId'],data=json.dumps(p['dataset']).encode(),headers=h,method='PUT') +receipt=json.load(urllib.request.urlopen(req,timeout=10)) +state=json.load(urllib.request.urlopen(urllib.request.Request(base,headers=h),timeout=10)) +assert state['liveEnabled'] is False +print(json.dumps({'dataset':receipt,'existingForecastSource':state['settings']['forecastSource'],'liveEnabled':False})) +''' + +def run(plant,apply=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Use the prepared installation-specific mapping') + if not apply: + print('APPLICATION SOURCE CHECK PASSED:',len(m['sourceHashes']),'files. No deployment.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker permissions') + folder=ROOT/'application-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ') + folder.mkdir(parents=True,mode=0o700) + env={**os.environ,'NETPLAN_V4_PLANTS':plant} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')] + portal=['docker','compose','-f',str(PORTAL/'compose.yaml'),'-f',str(ROOT/'compose.portal-bridge.yaml')] + result={'scope':'integrated_measurement_application','startedAt':datetime.now(timezone.utc).isoformat(), + 'liveEnabled':False,'productionCommissioned':False,'steps':[],'sourceHashes':m['sourceHashes']} + changed={};old_image=None;v4_replaced=False;portal_replaced=False + def cmd(args,timeout=180,capture=False,input=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=input, + stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + try: + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip() + result['previousV4Image']=old_image + cmd(compose+['build','netplan-v4'],timeout=900) + cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240) + cmd(['node','--test',str(ROOT/'tests/portal.test.mjs')]) + cmd(['node','--check',str(ROOT/'gui/netplan-v4.js')]) + result['steps'].append('target_python_and_portal_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite') + result['steps'].append('consistent_database_backup') + for source,target in PORTAL_FILES.items(): + p=PORTAL/target;old=p.read_bytes();new=(ROOT/source).read_bytes() + if old==new:continue + if hashlib.sha256(old).hexdigest()!=m['portalBefore'][target]:raise ValueError('Concurrent portal change') + dest=folder/'portal-before'/target;dest.parent.mkdir(parents=True,exist_ok=True);dest.write_bytes(old) + atomic(p,new);changed[target]=(old,new) + v4_replaced=True;cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4']) + # Recreate rather than restart: atomic replacement of a file bind mount needs a new mount. + portal_replaced=True;cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + payload=json.dumps({'plant':plant,'dataset':json.loads(DATASET.read_text())}) + receipt=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,input=payload) + result['application']=json.loads(receipt.stdout) + result['status']='application_deployed_no_actuator_permission' + result['steps'].append('configured_dataset_and_health_verified') + except Exception as e: + result['status']='needs_review';result['errorType']=type(e).__name__ + restored=[] + for target,(old,new) in changed.items(): + p=PORTAL/target + if p.read_bytes()==new:atomic(p,old);restored.append(target) + result['restoredPortalFiles']=restored + try: + if v4_replaced and old_image: + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + cmd(compose+['-f',str(rollback),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + if portal_replaced:cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal']) + result['rollback']='previous_runtime_restored_additive_data_retained' + except Exception:result['rollback']='manual_review_required' + raise + finally: + result['finishedAt']=datetime.now(timezone.utc).isoformat() + report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n') + uid=ROOT.stat().st_uid;gid=ROOT.stat().st_gid + os.chown(folder,uid,gid);os.chown(folder.parent,uid,gid);os.chown(report,uid,gid);report.chmod(0o640) + print('APPLICATION RELEASE REPORT:',report) + print('V4 data/model application and portal updated. Install manager data integration separately. No V4 actuation enabled.') + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True,type=lambda v:str(UUID(v)));p.add_argument('--apply',action='store_true');a=p.parse_args() + try:run(a.plant,a.apply) + except Exception as e:raise SystemExit('Stopped: '+type(e).__name__+'. See the application release report.') diff --git a/services/netplan-v4/compose.portal-bridge.yaml b/services/netplan-v4/compose.portal-bridge.yaml new file mode 100644 index 0000000..0bf8355 --- /dev/null +++ b/services/netplan-v4/compose.portal-bridge.yaml @@ -0,0 +1,6 @@ +services: + license-portal: + environment: + NETPLAN_V4_URL: http://netplan-v4:9100 + volumes: + - /home/agent/services/license/netplan-v4-bridge.mjs:/app/netplan-v4-bridge.mjs:ro diff --git a/services/netplan-v4/compose.yaml b/services/netplan-v4/compose.yaml new file mode 100644 index 0000000..84c7075 --- /dev/null +++ b/services/netplan-v4/compose.yaml @@ -0,0 +1,37 @@ +name: enelix-netplan-v4-shadow +services: + netplan-v4: + build: . + restart: unless-stopped + init: true + user: "1000:1000" + read_only: true + env_file: + - /home/agent/services/prognosis-integration.env + environment: + NETPLAN_V4_DB: /data/netplan-v4.sqlite + NETPLAN_V4_PLANTS: ${NETPLAN_V4_PLANTS:?Explicit installation ID required} + NETPLAN_V4_TEST_REPORT_DIR: /tmp/test-results + volumes: + - ./data:/data + tmpfs: + - /tmp:size=64m,mode=1777,noexec,nosuid + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + networks: + prognosis_integration: + aliases: + - netplan-v4 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9100/health', timeout=3).read()"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + # No public port, no Docker socket, no access to legacy databases. +networks: + prognosis_integration: + external: true + name: enelix-platform-internal diff --git a/services/netplan-v4/deploy_integrated_shadow.py b/services/netplan-v4/deploy_integrated_shadow.py new file mode 100644 index 0000000..6e4d614 --- /dev/null +++ b/services/netplan-v4/deploy_integrated_shadow.py @@ -0,0 +1,175 @@ +"""Install the reviewed server-side SHADOW integration; no Symcon/device writes. + +Default: verify source and print scope only. --apply requires root. Build and test +all changed code before recreating services. Save old image/config references and +an integrity-checked online backup of the V4 DB. Does not enable native telemetry +or dispatch: operation ingress may still be awaiting installation on Symcon. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse +import hashlib +import json +import os +import sqlite3 +import subprocess +import sys +from uuid import UUID + +ROOT = Path(__file__).resolve().parent +SERVICES = ROOT.parent +PROJECT = SERVICES / 'prognosis-manager-enelix2' +PORTAL = SERVICES / 'license' +PREFLIGHT = ROOT / 'acceptance-reports/20261002T043021Z/REPORT.json' +MANIFEST = ROOT / 'INTEGRATED_SHADOW_SOURCE.json' + + +def verify_sources(): + report = json.loads(PREFLIGHT.read_text()) + if report.get('preflightChecksPassed') is not True: + raise ValueError('Expected successful earlier runtime preflight is missing') + expected = json.loads(MANIFEST.read_text())['files'] + for relative, digest in expected.items(): + path = SERVICES / relative + if not path.resolve().is_relative_to(SERVICES) or path.is_symlink() or not path.is_file(): + raise ValueError('Invalid release source path: ' + relative) + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError('Source changed since preparation: ' + relative) + from forecast_acceptance import verify_forecast_bundle + verify_forecast_bundle(ROOT/'acceptance/forecast-src', PROJECT/'forecast_engine') + from release_preflight import verify_native_bundle + verify_native_bundle() + return expected + + +def compose_groups(): + return { + 'v4': (ROOT, [ROOT/'compose.yaml'], ['netplan-v4']), + 'forecast': (PROJECT, [PROJECT/'compose.yaml', ROOT/'compose.forecast-bridge.yaml'], ['api','forecast-engine','tariff-importer']), + 'portal': (PORTAL, [PORTAL/'compose.yaml', ROOT/'compose.portal-bridge.yaml'], ['license-portal']), + } + + +def compose_args(files): + result=['docker','compose'] + for path in files:result.extend(['-f',str(path)]) + return result + + +def backup_database(source, destination): + if not source.exists():return {'exists':False} + src=sqlite3.connect(source.resolve().as_uri()+'?mode=ro',uri=True,timeout=15) + dst=sqlite3.connect(destination,timeout=15) + try: + src.backup(dst) + if dst.execute('PRAGMA integrity_check').fetchone()[0]!='ok': + raise ValueError('V4 database backup failed integrity check') + finally:dst.close();src.close() + os.chmod(destination,0o640) + if os.geteuid()==0:os.chown(destination,1000,1000) + return {'exists':True,'path':str(destination),'sha256':hashlib.sha256(destination.read_bytes()).hexdigest(),'integrity':'ok'} + + +def save_json(path, value): + path.write_text(json.dumps(value,indent=2,allow_nan=False)+'\n') + os.chmod(path,0o640) + if os.geteuid()==0:os.chown(path,1000,1000) + + +def run(plant): + sources=verify_sources() + env=dict(os.environ,NETPLAN_V4_PLANTS=plant) + stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + folder=ROOT/'deployment-reports'/stamp + folder.mkdir(parents=True,exist_ok=False) + if os.geteuid()==0: + os.chown(folder.parent,1000,1000);os.chown(folder,1000,1000) + report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant, + 'scope':'server_shadow_only','liveEnabled':False,'productionReady':False, + 'sourceHashes':sources,'steps':[],'before':{},'existingServiceRecreated':False} + def command(args,cwd=ROOT,capture=False,stdin=None,timeout=900): + result=subprocess.run(args,cwd=cwd,env=env,input=stdin,text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None,timeout=timeout,check=False) + if result.returncode: + raise RuntimeError('Step failed (exit '+str(result.returncode)+'): '+args[0]) + return result.stdout if capture else '' + def step(name,args,cwd=ROOT,timeout=900): + print('\n=== '+name+' ===',flush=True) + command(args,cwd,timeout=timeout) + report['steps'].append(name);save_json(folder/'DEPLOYMENT.json',report) + try: + # Capture only selected nonsecret Docker fields, never the environment. + for group,(cwd,files,services) in compose_groups().items(): + oldfiles=[files[0]] + items=[] + for service in services: + cid=command(compose_args(oldfiles)+['ps','-q',service],cwd,True).strip() + if not cid or '\n' in cid:raise ValueError('Expected exactly one existing container: '+service) + image=command(['docker','inspect','--format','{{.Image}}',cid],cwd,True).strip() + configs=command(['docker','inspect','--format','{{index .Config.Labels "com.docker.compose.project.config_files"}}',cid],cwd,True).strip() + items.append({'service':service,'containerId':cid,'imageId':image,'previousComposeFiles':configs}) + report['before'][group]=items + save_json(folder/'DEPLOYMENT.json',report) + step('review optional source installer',[sys.executable,str(ROOT/'install_hooks.py')]) + # All new forecasting tests run offline in a disposable Python 3.11 image. + step('build forecast acceptance image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')]) + step('forecast candidate offline tests',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local']) + step('V4 and bridge target tests',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only']) + verify_sources() + from install_hooks import plan,apply + receipt=apply(plan()) + report['sourceInstallReceipt']=str(receipt) if receipt else None + step('portal syntax',['node','--check',str(PORTAL/'server.mjs')]) + # Build every production-shaped candidate before changing a running container. + cwd,files,svcs=compose_groups()['forecast'] + step('build server-side images',compose_args(files)+['build']+svcs,cwd) + report['v4DatabaseBackup']=backup_database(ROOT/'data/netplan-v4.sqlite',folder/'netplan-v4-before.sqlite') + report['existingServiceRecreated']=True + save_json(folder/'DEPLOYMENT.json',report) + for group,(cwd,files,svcs) in compose_groups().items(): + step('start shadow integration '+group,compose_args(files)+['up','-d','--no-deps','--no-build','--wait','--wait-timeout','180']+svcs,cwd,timeout=300) + cwd,files,_=compose_groups()['v4'] + # Check shadow guard and report incoming types without disclosing payloads. + probe='''import json,os,sqlite3,urllib.request +from datetime import datetime,timezone +plant=os.environ['NETPLAN_V4_PLANTS'] +health=json.loads(urllib.request.urlopen('http://127.0.0.1:9100/health',timeout=5).read()) +assert health.get('mode')=='shadow' and health.get('liveEnabled') is False +con=sqlite3.connect('file:/data/netplan-v4.sqlite?mode=ro',uri=True) +rows=con.execute('SELECT kind,count(*) FROM planner_inputs WHERE plant=? GROUP BY kind',(plant,)).fetchall() +status=con.execute('SELECT status FROM planner_run_status WHERE plant=?',(plant,)).fetchone() +con.close() +print(json.dumps({'checkedAt':datetime.now(timezone.utc).isoformat(),'health':health,'inputEventsByKind':dict(rows),'lastRunStatus':status[0] if status else None,'nativeSenderInstalledByThisCommand':False})) +''' + report['readiness']=json.loads(command(compose_args(files)+['exec','-T','netplan-v4','python','-B','-'],cwd,True,probe,60)) + report['status']='server_shadow_installed' + report['remaining']='Native operation sender, plant-level initialization and real shadow-plan acceptance still required; no live dispatch installed' + except Exception as error: + report['status']='failed';report['errorType']=type(error).__name__;report['error']=str(error)[:500] + report['remaining']='Inspect completed steps and before-image references. No automatic database restore or device changes.' + raise + finally: + save_json(folder/'DEPLOYMENT.json',report) + print('\nDEPLOYMENT REPORT: '+str(folder/'DEPLOYMENT.json'),flush=True) + print('SERVER SHADOW ONLY. Symcon, device permissions and actuators were not modified.',flush=True) + print(json.dumps(report['readiness'],indent=2)) + print('Native operating-state input may still be missing; this is NOT production commissioning.') + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda p:str(UUID(p))) + parser.add_argument('--apply',action='store_true') + args=parser.parse_args() + if args.plant!='e3a08f9e-af12-4695-99bd-8b51c0520021': + raise SystemExit('This first integration rollout is allowlisted for Lihrenmoos only.') + if not args.apply: + files=verify_sources() + print('CHECK ONLY:',len(files),'reviewed source files. No running service changed.') + for group,(_,_,svcs) in compose_groups().items():print(group,', '.join(svcs)) + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not change Docker socket access.') + try:run(args.plant) + except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + raise SystemExit('Deployment stopped: '+str(error)) diff --git a/services/netplan-v4/forecast_acceptance.py b/services/netplan-v4/forecast_acceptance.py new file mode 100644 index 0000000..686765f --- /dev/null +++ b/services/netplan-v4/forecast_acceptance.py @@ -0,0 +1,38 @@ +"""Verify the separately staged forecast repair before target-runtime testing. +Only explicit Python source and requirements are copied. No models, credentials, +live databases or telemetry are needed by this offline test image. +""" +from pathlib import Path +import hashlib +import json + + +def source_paths(root): + root = Path(root) + paths = [root / 'requirements.txt'] + list(root.glob('*.py')) + for folder in ('methods', 'tests'): + paths.extend((root / folder).glob('*.py')) + if not (root / 'telemetry_quality.py').is_file(): + raise ValueError('Forecast telemetry repair missing') + for path in paths: + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError('External forecast source is not allowed') + return sorted(paths) + + +def verify_forecast_bundle(bundle, source): + bundle, source = Path(bundle), Path(source) + manifest_path = bundle / 'SOURCE_MANIFEST.json' + manifest = json.loads(manifest_path.read_text()) + expected_paths = {str(p.relative_to(source)) for p in source_paths(source)} + if not isinstance(manifest, dict) or not manifest or set(manifest) != expected_paths: + raise ValueError('Forecast source set changed; rebuild reviewed test bundle') + if {str(p.relative_to(bundle)) for p in source_paths(bundle)} != expected_paths: + raise ValueError('Staged forecast source set does not match development source') + for relative, expected in manifest.items(): + if Path(relative).is_absolute() or '..' in Path(relative).parts: + raise ValueError('Unsafe forecast manifest path') + for path in (source / relative, bundle / relative): + if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise ValueError('Forecast source changed after staging; review before test') + return dict(manifest) diff --git a/services/netplan-v4/gui/netplan-v4.css b/services/netplan-v4/gui/netplan-v4.css new file mode 100644 index 0000000..2460732 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.css @@ -0,0 +1 @@ +body{font:16px system-ui,sans-serif;background:#eef4f4;color:#243c42}main{max-width:1100px;margin:2rem auto;padding:1.5rem;background:white}a{color:#006b83}label{display:block;margin:.8rem 0}select,button{font:inherit;padding:.6rem;margin:.3rem;border:1px solid #bacaca;border-radius:.25rem}button{background:#006b83;color:white;cursor:pointer}.notice{background:#e9f7fa;padding:1rem;border-left:4px solid #006b83}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#eef4f4;padding:1rem}svg{width:100%;height:auto}.curve{fill:none;stroke-width:2}.baseline{stroke:#7d888c}.optimized{stroke:#006b83}.axis{stroke:#b7c8cc}button:disabled{opacity:.5} diff --git a/services/netplan-v4/gui/netplan-v4.html b/services/netplan-v4/gui/netplan-v4.html new file mode 100644 index 0000000..1a96991 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.html @@ -0,0 +1 @@ +ENELIX Netzfahrplan V4
Zurueck zum Portal

Netzfahrplan V4

Schattenbetrieb. Diese Ansicht steuert keine Verbraucher. Die bestehende Regelung bleibt unveraendert.

diff --git a/services/netplan-v4/gui/netplan-v4.js b/services/netplan-v4/gui/netplan-v4.js new file mode 100644 index 0000000..a67ac55 --- /dev/null +++ b/services/netplan-v4/gui/netplan-v4.js @@ -0,0 +1,107 @@ +const message = document.getElementById('message'); +const panel = document.getElementById('panel'); +const plants = document.getElementById('plants'); +let session, generation = 0; +const element = (tag, text) => { const e = document.createElement(tag); if (text !== undefined) e.textContent = text; return e; }; +const time = value => value ? new Date(value).toLocaleString('de-CH', { timeZone: 'Europe/Zurich' }) : 'nicht vorhanden'; +const num = (v, digits = 2) => typeof v === 'number' && Number.isFinite(v) ? v.toLocaleString('de-CH', { maximumFractionDigits: digits }) : 'unbekannt'; + +async function request(url, method = 'GET', body) { + const headers = {}; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + if (method !== 'GET') headers['X-CSRF-Token'] = session.csrfToken; + const response = await fetch(url, { method, headers, credentials: 'same-origin', body: body === undefined ? undefined : JSON.stringify(body) }); + const data = await response.json(); + if (!response.ok) throw new Error(data.error || data.detail || 'Anfrage fehlgeschlagen'); + return data; +} +function select(label, choices, selected) { + const wrap = element('label', label + ' '), field = element('select'); + for (const [value, caption] of choices) { const option = element('option', caption); option.value = value; field.append(option); } + field.value = selected; wrap.append(field); return { wrap, field }; +} +function table(title, rows) { + panel.append(element('h3', title)); const grid = element('table'); + for (const [name, value] of rows) { const row = element('tr'); row.append(element('th', name), element('td', String(value))); grid.append(row); } + panel.append(grid); +} +function curve(title, points, series) { + panel.append(element('h3', title)); + if (!points.length) { panel.append(element('p', 'Keine Daten.')); return; } + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + svg.setAttribute('viewBox', '0 0 1000 210'); svg.setAttribute('role', 'img'); svg.setAttribute('aria-label', title); + const values = points.flatMap(p => series.map(s => s.value(p))).filter(v => typeof v === 'number' && Number.isFinite(v)); + if (!values.length) { panel.append(element('p', 'Keine gueltigen Werte.')); return; } + const low = Math.min(0, ...values), high = Math.max(1, ...values); + const start = Date.parse(points[0].time), end = Date.parse(points[points.length - 1].validUntil || points[points.length - 1].time); + for (const [i, source] of series.entries()) { + let section = []; + const flush = () => { if (!section.length) return; const line = document.createElementNS(svg.namespaceURI, 'polyline'); line.setAttribute('class', 'curve ' + (i ? 'optimized' : 'baseline')); line.setAttribute('points', section.join(' ')); svg.append(line); section = []; }; + for (const p of points) { + const value = source.value(p); + if (typeof value !== 'number' || !Number.isFinite(value)) { flush(); continue; } + const x = 10 + 980 * (Date.parse(p.time) - start) / Math.max(1, end - start); + const y = 190 - (value - low) / (high - low) * 180; section.push(`${x},${y}`); + } + flush(); + } + panel.append(svg, element('p', `${num(low)} bis ${num(high)}. ${series.map((s, i) => (i ? 'Blau: ' : 'Grau: ') + s.label).join('; ')}. ${time(points[0].time)} – ${time(points[points.length - 1].validUntil)}.`)); +} +async function render() { + const ticket = ++generation, id = plants.value; if (!id) return; + const base = `/api/plants/${encodeURIComponent(id)}/prognosis/planner-v4`; + message.textContent = 'Lade Planungsstand ...'; + try { + const state = await request(base); if (ticket !== generation) return; panel.replaceChildren(); + panel.append(element('p', 'SCHATTENBETRIEB: Die angezeigten Vorgaben steuern keine Geraete.')); + const form = element('form'); + const family = select('Modellfamilie', [['auto', 'Automatisch – historischer Kostenvergleich'], ...state.families.map(f => [f.key, f.label])], state.settings.family); + const measurement = select('Messdatenbasis', [['verified_only', 'Nur verifizierte Peak-/Viertelstundenwerte'], ['allow_estimates', 'Gekennzeichnete Schaetzungen fuer die Planung zulassen']], state.settings.measurementPolicy || 'verified_only'); + const outlook = select('Peakbewertung', [['full_incremental', 'Voller zusaetzlicher Monatstarif'], ['empirical_if_available', 'Restmonats-Szenarien bei ausreichender Historie']], state.settings.peakOutlookPolicy || 'full_incremental'); + const dataSource = select('Verbrauchsprognose', [['legacy', 'Bisherige Datenquelle'], ['corrected_profile', 'Bereinigte physische Lastprofile']], state.settings.forecastSource || 'legacy'); + const datasets = state.dataPipeline?.datasets || []; + const dataset = select('Messdatensatz', [['', 'Datensatz auswaehlen'], ...datasets.map(d => [d.datasetId, d.datasetId + ' (' + d.status + ')'])], state.settings.measurementDataset || ''); + const cadence = select('Nachtraining', [['daily', 'Taeglich'], ['weekly', 'Woechentlich']], state.settings.trainingCadence); + const relianceWrap = element('label', 'Gewicht der Restmonatsprognose (0 bis 100 %) '), reliance = element('input'); + reliance.type = 'number'; reliance.min = '0'; reliance.max = '100'; reliance.step = '1'; reliance.value = String(100 * (state.settings.peakOutlookReliance ?? .5)); relianceWrap.append(reliance); + const save = element('button', 'Speichern und neu berechnen'), replan = element('button', 'Neu berechnen'), refresh = element('button', 'Status aktualisieren'); + save.type = 'submit'; replan.type = refresh.type = 'button'; + form.append(family.wrap, dataSource.wrap, dataset.wrap, measurement.wrap, outlook.wrap, relianceWrap, cadence.wrap, save, replan, refresh); panel.append(form); + let revision = state.settings.revision; + form.addEventListener('submit', async event => { + event.preventDefault(); save.disabled = true; + try { + const r = await request(base + '/settings', 'PUT', { expectedRevision: revision, changes: { family: family.field.value, forecastSource: dataSource.field.value, measurementDataset: dataset.field.value, measurementPolicy: measurement.field.value, peakOutlookPolicy: outlook.field.value, peakOutlookReliance: Number(reliance.value) / 100, trainingCadence: cadence.field.value } }); + revision = r.revision; message.textContent = `Revision ${revision} gespeichert. Neuberechnung angefordert, noch nicht abgeschlossen.`; + } catch (error) { message.textContent = error.message; } finally { save.disabled = false; } + }); + replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } }); + refresh.addEventListener('click', render); + panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.')); + panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik wartet weiterhin auf vergleichbare Kosten-Replays, nicht bloss auf gute R2-Werte.')); + for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.')); + if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.')); + for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]); + const plan = state.plan; + if (plan) { + if (!state.fresh) panel.append(element('p', 'ACHTUNG: Plan veraltet oder Neuberechnung ausstehend.')); + table('Plan und Kosten', [['Plan-ID', plan.planId], ['Modellfamilie', plan.sourceFamily], ['Revision', plan.configRevision], ['Prognose bis', time(plan.forecastUntil)], ['Bekannte Preise bis', time(plan.pricesKnownUntil)], ['Energiekosten', num(plan.energyCostChf) + ' CHF'], ['Zusaetzliche Peakkosten zum vollen Tarif', num(plan.additionalPeakCostChf) + ' CHF'], ['Peakbewertung unter Restmonats-Szenarien', num(plan.planningPeakCostChf) + ' CHF'], ['Planung mit geschaetzter Peakbasis', plan.peakCostIsEstimate ? 'Ja – kein Abrechnungsnachweis' : 'Nein'], ['V4 steuert die Anlage', 'Nein']]); + for (const warning of plan.warnings || []) panel.append(element('p', warning)); + for (const [month, basis] of Object.entries(plan.peakBasis || {})) table(`Peak ${month}`, [['Ausgangsbasis', num(basis.kw) + ' kW (' + basis.quality + ')'], ['Geplant', num(plan.plannedPeaksKw?.[month]) + ' kW'], ['Bewertung', plan.peakOutlook]]); + curve('Netzleistung (W)', plan.points, [{ label: 'ohne Optimierung', value: p => p.baselineGridW }, { label: 'optimiert', value: p => p.gridTargetW }]); + curve('Batterieleistung (W; positiv = Laden)', plan.points, [{ label: 'Batterieplan', value: p => p.batteryTargetW }]); + for (const asset of Object.keys(plan.points[0]?.socEndPercent || {})) curve(`SOC ${asset} (%)`, plan.points, [{ label: 'geplanter SOC', value: p => p.socEndPercent?.[asset] }]); + curve('Kumulierte Zahlungen inklusive vollem Peakpreis (CHF)', plan.points, [{ label: 'ohne Optimierung', value: p => p.cumulativeBaselineCashCostChf }, { label: 'optimiert', value: p => p.cumulativeCashCostChf }]); + panel.append(element('p', 'Kostenverlaeufe sind Prognosen. Restwert der Batterie und Restmonatsbewertung sind keine bereits erzielten Erloese.')); + } + message.textContent = ''; + } catch (error) { if (ticket === generation) { panel.replaceChildren(); message.textContent = error.message; } } +} +try { + session = await request('/api/session'); if (!session.authenticated) throw new Error('Bitte zuerst im Portal anmelden.'); + const data = await request('/api/plants'); + for (const plant of data.plants || []) { if (!plant.installation_id) continue; const option = element('option', plant.name); option.value = plant.id; plants.append(option); } + if (!plants.options.length) throw new Error('Keine verknuepfte Anlage vorhanden.'); + plants.addEventListener('change', render); await render(); +} catch (error) { message.textContent = error.message; } diff --git a/services/netplan-v4/install_hooks.py b/services/netplan-v4/install_hooks.py new file mode 100644 index 0000000..b6e8c76 --- /dev/null +++ b/services/netplan-v4/install_hooks.py @@ -0,0 +1,131 @@ +"""Count-checked optional bridge installation. Default CHECK ONLY, no service action. +Only dedicated V4 GUI files are added. app.js/index.html/styles.css stay untouched. +Every changed legacy source is backed up with hashes. Concurrent changes abort. +""" +from pathlib import Path +from datetime import datetime,timezone +import argparse,ast,hashlib,json,os,tempfile +ROOT=Path(__file__).resolve().parent +SERVICES=Path('/home/agent/services') + +def once(text,old,new): + if text.count(old)!=1:raise ValueError('Source anchor changed/ambiguous; review before updating') + return text.replace(old,new,1) + +def patch_forecast(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts\n'+text + anchor=' p_3 = v3.predict(data_obj, p_1, p_2)' + text=once(text,anchor,' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)])\n\n'+anchor) + text=once(text,' d[key] = float(d.get(key) or default)',' raw_value = d.get(key)\n d[key] = float(default if raw_value is None or raw_value == "" else raw_value)') + ast.parse(text);return text + +def patch_api(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_tariffs as _v4_publish_tariffs\n'+text + anchor=' _portal_store_configuration(anlagen_id, configuration)\n' + text=once(text,anchor,anchor+' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_tariffs(anlagen_id, configuration)\n') + ast.parse(text);return text + +def patch_tariff(text): + if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text + text='from netplan_v4_publisher import publish_ckw as _v4_publish_ckw\n'+text + anchor=' return candidate_points\n' + text=once(text,anchor," # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_ckw(source['label'], rows, data.get('publication_timestamp') if isinstance(data, dict) else None, tariff_type)\n"+anchor) + for name in ('row','integrated'): + old=f'{name}.get("value") or {name}.get("price") or {name}.get("amount")' + if old in text:text=once(text,old,f'next(({name}[key] for key in ("value", "price", "amount") if {name}.get(key) is not None), None)') + ast.parse(text);return text + +def patch_portal(text): + if '// ENELIX_V4_SHADOW_BRIDGE' in text:return text + anchor='const server = createServer(async (req, res) => {' + extra="""// ENELIX_V4_SHADOW_BRIDGE +// No dependency or changed route while NETPLAN_V4_URL is absent. +const plannerV4Bridge = process.env.NETPLAN_V4_URL + ? (await import('./netplan-v4-bridge.mjs')).createPlannerV4Bridge({ + configuredPrognosisPlant, roleAllowed, bodyJson, json, deviceActivation, + checkDeviceRate, ownedLicenseState, serviceToken: prognosisServiceToken, + upstreamUrl: process.env.NETPLAN_V4_URL + }) + : null; + +""" + text=once(text,anchor,extra+anchor) + anchor=' if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {' + return once(text,anchor,' if (plannerV4Bridge && await plannerV4Bridge(req, res, url)) return;\n'+anchor) + +def plan(root=SERVICES, approved_assets=None): + root=Path(root).resolve();project=root/'prognosis-manager-enelix2';result={} + if approved_assets is None: + manifest=ROOT/'approved_previous_assets.json' + approved_assets=json.loads(manifest.read_text()) if manifest.is_file() else {} + for path,patch in [(project/'api/main.py',patch_api),(project/'forecast_engine/main.py',patch_forecast),(project/'tariff_importer/main.py',patch_tariff),(root/'license/server.mjs',patch_portal)]: + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes();new=patch(old.decode()).encode() + if old!=new:result[path]=(old,new) + files={project/service/'netplan_v4_publisher.py':ROOT/'integrations/netplan_v4_publisher.py' for service in ('api','forecast_engine','tariff_importer')} + files[root/'license/netplan-v4-bridge.mjs']=ROOT/'integrations/netplan-v4-bridge.mjs' + for name in ('netplan-v4.html','netplan-v4.js','netplan-v4.css'):files[root/'license/public'/name]=ROOT/'gui'/name + for path,source in files.items(): + if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused') + old=path.read_bytes() if path.exists() else None;new=source.read_bytes() + if old is not None and old!=new: + approved=approved_assets.get(str(path.relative_to(root)),[]) + if hashlib.sha256(old).hexdigest() not in approved: + raise ValueError('Existing V4 file differs from the reviewed previous release; explicit merge required: '+str(path)) + if old!=new:result[path]=(old,new) + return result + +def atomic(path,data,mode=0o644): + path.parent.mkdir(parents=True,exist_ok=True);fd,tmp=tempfile.mkstemp(prefix=path.name+'.v4-',dir=path.parent) + try: + with os.fdopen(fd,'wb') as out:out.write(data);out.flush();os.fsync(out.fileno()) + os.chmod(tmp,mode);os.replace(tmp,path) + finally: + if os.path.exists(tmp):os.unlink(tmp) + +def apply(entries,root=SERVICES): + root=Path(root).resolve() + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent change: no files modified') + if not entries:return None + backup=root/'change-backups'/'netplan-v4'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + backup.mkdir(parents=True);receipt={'root':str(root),'files':[]};written=[] + try: + for path,(old,new) in entries.items(): + if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent source change during install') + rel=path.relative_to(root);mode=path.stat().st_mode&0o777 if path.exists() else 0o644 + if old is not None: + original=backup/'originals'/rel;original.parent.mkdir(parents=True,exist_ok=True);original.write_bytes(old) + atomic(path,new,mode);written.append(path) + receipt['files'].append({'path':str(rel),'before':hashlib.sha256(old).hexdigest() if old is not None else None,'after':hashlib.sha256(new).hexdigest(),'mode':mode}) + atomic(backup/'receipt.json',json.dumps(receipt,indent=2).encode());return backup/'receipt.json' + except Exception: + for path in reversed(written): + old,new=entries[path] + if path.read_bytes()==new: + if old is None:path.unlink() + else:atomic(path,old) + raise + +def rollback(receipt_path): + path=Path(receipt_path).resolve();data=json.loads(path.read_text());root=Path(data['root']).resolve() + for item in data['files']: + target=(root/item['path']).resolve() + if not target.is_relative_to(root) or hashlib.sha256(target.read_bytes()).hexdigest()!=item['after']:raise ValueError('Later change detected; automatic rollback refused') + for item in reversed(data['files']): + target=root/item['path'] + if item['before'] is None:target.unlink() + else: + original=(path.parent/'originals'/item['path']).read_bytes() + if hashlib.sha256(original).hexdigest()!=item['before']:raise ValueError('Backup checksum mismatch') + atomic(target,original,item['mode']) + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--apply',action='store_true');parser.add_argument('--rollback');args=parser.parse_args() + if args.rollback:rollback(args.rollback);print('Source rollback completed. No service restart.') + else: + entries=plan() + for path in entries:print(path) + print('Receipt:',apply(entries)) if args.apply else print('CHECK ONLY:',len(entries),'files. No source/database/service changed.') diff --git a/services/netplan-v4/integrations/NetzfahrplanV4.php b/services/netplan-v4/integrations/NetzfahrplanV4.php new file mode 100644 index 0000000..5ffe258 --- /dev/null +++ b/services/netplan-v4/integrations/NetzfahrplanV4.php @@ -0,0 +1,71 @@ +getTimestamp(); + } + private static function number(mixed $value): float + { + if ((!is_float($value) && !is_int($value)) || !is_finite((float)$value)) { + throw new InvalidArgumentException('Finite numeric value required'); + } + return (float)$value; + } + public static function currentPoint(array $plan, int $now, int $revision, bool $shadowInspection=false): ?array + { + if (($plan['schemaVersion']??null)!==2 || ($plan['configRevision']??null)!==$revision + || ($plan['executable']??false)!==true || !is_array($plan['points']??null) + || !in_array($plan['status']??'', ['optimal','feasible_time_limit'], true) + || !is_string($plan['planId']??null) || $plan['planId']==='') return null; + if ($shadowInspection ? ($plan['runMode']??'')!=='shadow' + : (($plan['runMode']??'')!=='live' || ($plan['liveEnabled']??false)!==true)) return null; + try { + $age=$now-self::stamp($plan['generatedAt']); + if ($age < -30 || $age>900 || self::stamp($plan['validFrom'])>$now || self::stamp($plan['validUntil'])<=$now) return null; + $selected=null; $previousEnd=null; + foreach ($plan['points'] as $p) { + $start=self::stamp($p['time']); $end=self::stamp($p['validUntil']); + self::number($p['gridTargetW']); + if ($end<=$start || $end-$start>300 || ($previousEnd!==null && $previousEnd!==$start)) return null; + $previousEnd=$end; + if ($start<=$now && $now<$end) $selected=$p; + } + return $selected; + } catch (\Throwable $e) { return null; } + } + public static function correction(array $point, float $grid, float $battery, float $chargeAvailable, float $dischargeAvailable, ?float $hardImportLimit=null): array + { + foreach ([$grid,$battery,$chargeAvailable,$dischargeAvailable] as $v) self::number($v); + if ($chargeAvailable<0 || $dischargeAvailable<0) throw new InvalidArgumentException('Invalid available power'); + $target=self::number($point['gridTargetW']??null); + if ($hardImportLimit!==null) { + if (self::number($hardImportLimit)<0) throw new InvalidArgumentException('Negative hard limit'); + $target=min($target,$hardImportLimit); + } + $raw=$battery+$target-$grid; + if (abs($target)>1) { + $raw=match ($point['intent']??'') { + 'gridCharge'=>max(0.,$raw), + 'pvCharge'=>max(0.,min($battery-$grid,$raw)), + 'discharge','export'=>min(0.,$raw), + 'hold'=>0., + default=>throw new InvalidArgumentException('Unknown plan intent'), + }; + } + if ($hardImportLimit!==null) $raw=min($raw,$battery+$hardImportLimit-$grid); + $wanted=min($chargeAvailable,max(-$dischargeAvailable,$raw)); + $expectedGrid=$grid+$wanted-$battery; + return ['batteryTargetW'=>$wanted,'gridTargetW'=>$target,'limited'=>abs($wanted-$raw)>1, + 'expectedGridW'=>$expectedGrid,'trackingErrorW'=>$expectedGrid-$target]; + } +} diff --git a/services/netplan-v4/integrations/netplan-v4-bridge.mjs b/services/netplan-v4/integrations/netplan-v4-bridge.mjs new file mode 100644 index 0000000..1e8786e --- /dev/null +++ b/services/netplan-v4/integrations/netplan-v4-bridge.mjs @@ -0,0 +1,39 @@ +/** Scoped opt-in proxy. Never intercept the existing V1 schedule. */ +export function createPlannerV4Bridge({configuredPrognosisPlant,roleAllowed,bodyJson,json, + deviceActivation,checkDeviceRate,ownedLicenseState,serviceToken, + upstreamUrl='http://netplan-v4:9100',fetchImpl=fetch}){ + const base=new URL(upstreamUrl); + if(!['http:','https:'].includes(base.protocol)||base.username||base.password)throw new Error('Invalid V4 URL'); + async function forward(req,res,installation,suffix,method){ + if(!serviceToken){json(res,503,{error:'V4 ist nicht angebunden.'});return;} + const url=new URL(`/internal/v2/prognosis/${encodeURIComponent(installation)}/planner${suffix}`,base); + let response; + const payload=method==='GET'?undefined:await bodyJson(req); + try{response=await fetchImpl(url,{method,headers:{'X-Enelix-Service-Token':serviceToken,'Content-Type':'application/json'},body:payload===undefined?undefined:JSON.stringify(payload),signal:AbortSignal.timeout(10000),redirect:'error'});} + catch{json(res,503,{error:'V4 nicht erreichbar. Bestehende Regelung bleibt unveraendert.'});return;} + const data=await response.json().catch(()=>null); + if(!response.ok){const code=[400,401,403,404,409,413].includes(response.status)?response.status:503;json(res,code,{error:code===409?'Versionskonflikt: Ansicht neu laden.':'V4-Anfrage abgelehnt oder Dienst nicht verfuegbar.'});return;} + if(!data||typeof data!=='object'){json(res,503,{error:'Ungueltige V4-Antwort.'});return;} + json(res,200,data); + } + return async function(req,res,url){ + const c=url.pathname.match(/^\/api\/plants\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(settings|replan))?$/i); + if(c){ + const action=c[2]||'',method={'':'GET',settings:'PUT',replan:'POST'}[action]; + if(req.method!==method){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + const access=configuredPrognosisPlant(req,res,c[1],method!=='GET');if(!access)return true; + if(Number(access.license.quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + if(method!=='GET'&&!roleAllowed(access.session,['owner','admin','operator'])){json(res,403,{error:'Keine Aenderungsberechtigung.'});return true;} + await forward(req,res,access.plant.installation_id,action?`/${action}`:'',method);return true; + } + const d=url.pathname.match(/^\/api\/v1\/installations\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(operation|ack|measurements))?$/i); + if(d){ + const action=d[2]||'';if(req.method!==(action?'POST':'GET')){json(res,405,{error:'Methode nicht erlaubt.'});return true;} + if(!checkDeviceRate(req,d[1],'planner-v4')){json(res,429,{error:'Zu viele Anfragen.'});return true;} + const activation=deviceActivation(req,res,d[1]);if(!activation)return true; + if(Number(ownedLicenseState(activation.plant_id).quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;} + await forward(req,res,d[1],action==='operation'?'/inputs/operation':action==='measurements'?'/measurements':action?'/ack':'',req.method);return true; + } + return false; + }; +} diff --git a/services/netplan-v4/integrations/netplan_v4_publisher.py b/services/netplan-v4/integrations/netplan_v4_publisher.py new file mode 100644 index 0000000..aeb9273 --- /dev/null +++ b/services/netplan-v4/integrations/netplan_v4_publisher.py @@ -0,0 +1,108 @@ +"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services. +No credentials are printed. Existing V1 computation survives shadow-service errors. +Legacy UTC-naive forecast indices are explicitly interpreted as UTC here. +""" +from datetime import datetime,timezone +from hashlib import sha256 +from uuid import uuid4 +from urllib.parse import urlparse +from urllib.request import Request,urlopen +import json,logging,math,os + +def timestamp(value): + if hasattr(value,'to_pydatetime'):value=value.to_pydatetime() + if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00')) + if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + +def tariff_id(label): + if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required') + return 'legacy:'+sha256(label.encode()).hexdigest()[:32] + +def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))} + +def tariff_payload(config,at=None): + result=envelope(at) + for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')): + mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus')) + if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name') + item={'mode':mode,'tariffId':tariff_id(config[prefix])} + if mode=='static': + value=config.get(prefix+'_fest') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid') + item['staticChfKwh']=float(value) + result[side]=item + peak=config.get('tarif_peak_fest') + if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required') + result['peakChfKwMonth']=float(peak) + return result + +def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None): + result=envelope(at);result['families']={} + for key,pv,load in forecasts: + if not pv or not load:continue + if set(pv)!=set(load):raise ValueError('PV/load timestamps differ') + result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until, + 'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]} + if not result['families']:raise ValueError('No forecast families supplied') + return result + +def ckw_payload(label,rows,publication_timestamp=None,at=None): + """Provider-delimited integrated price intervals only. No scalar price replication.""" + result=envelope(at);periods=[] + units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'} + for row in rows: + if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required') + integrated=row.get('integrated') + if isinstance(integrated,list): + if len(integrated)!=1:raise ValueError('Ambiguous integrated price components') + integrated=integrated[0] + if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required') + value=integrated.get('value') + if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required') + item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']), + 'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']], + 'observedAt':result['observedAt'],'sourceKind':'published_interval'} + if publication_timestamp: + published=timestamp(publication_timestamp) + if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication') + item['publishedAt']=published + periods.append(item) + result['periods']=periods + return result + +def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()}) + +def send(plant,kind,payload): + if not enabled(plant):return {'status':'disabled'} + token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base) + if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured') + UUID=__import__('uuid').UUID;UUID(plant) + if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind') + req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind, + json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST') + with urlopen(req,timeout=5) as response: + data=response.read(65537) + if len(data)>65536:raise ValueError('Oversized service response') + return json.loads(data) + +def publish_forecasts(config,forecasts): + plant=config['anlagen_id'] + if not enabled(plant):return + try: + send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__) + +def publish_tariffs(plant,config): + if not enabled(plant):return + try:send(plant,'tariffs',tariff_payload(config)) + except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__) + +def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'): + plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()] + if not os.getenv('NETPLAN_V4_URL') or not plants:return + try: + if tariff_type!='integrated':raise ValueError('Full integrated tariff required') + payload=ckw_payload(label,rows,publication_timestamp) + for plant in plants:send(plant,'prices',payload) + except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__) diff --git a/services/netplan-v4/netplan_v4/__init__.py b/services/netplan-v4/netplan_v4/__init__.py new file mode 100644 index 0000000..21e0e8d --- /dev/null +++ b/services/netplan-v4/netplan_v4/__init__.py @@ -0,0 +1 @@ +"""ENELIX V4 shadow planner. No live actuator interface.""" diff --git a/services/netplan-v4/netplan_v4/battery_model.py b/services/netplan-v4/netplan_v4/battery_model.py new file mode 100644 index 0000000..0c7813e --- /dev/null +++ b/services/netplan-v4/netplan_v4/battery_model.py @@ -0,0 +1,44 @@ +"""SOC-aware battery model, including reserve recovery and hysteresis rearming.""" +from math import sqrt + + +class BatteryModel: + @staticmethod + def build(model,battery,steps,direction): + cap=battery.capacity_kwh;eta=sqrt(battery.roundtrip_efficiency) + reserve=cap*battery.min_soc_percent/100 + initial=cap*battery.soc_percent/100 + lower=min(initial,reserve) if battery.recovery_allowed else reserve + upper=cap*battery.max_soc_percent/100 + energies=[model.variable(lower,upper) for _ in range(len(steps)+1)] + model.constraint({energies[0]:1},initial,initial) + terminal=battery.terminal_soc_min_percent + if terminal is None:terminal=max(battery.soc_percent,battery.min_soc_percent) + model.constraint({energies[-1]:1},cap*terminal/100) + model.cost[energies[-1]]=-battery.terminal_value_chf_kwh + rearm=cap*(battery.rearm_soc_percent if battery.rearm_soc_percent is not None else battery.min_soc_percent)/100 + enabled=[model.variable(0,1,integer=True) for _ in steps] + # Only a numerical threshold, not an undisclosed extra operating reserve. + epsilon=min(1e-5,max(0.,upper-reserve)/1000) + initially_enabled=not battery.discharge_blocked and initial>=reserve+max(epsilon,1e-8) + model.constraint({enabled[0]:1},int(initially_enabled),int(initially_enabled)) + big=max(upper-lower+epsilon,1.) + charge,discharge=[],[] + for i,step in enumerate(steps): + dt=step.seconds/3600;cmax=battery.max_charge_w/1000;dmax=battery.max_discharge_w/1000 + c=model.variable(0,cmax,battery.throughput_chf_kwh*dt) + d=model.variable(0,dmax,battery.throughput_chf_kwh*dt) + charge.append(c);discharge.append(d) + model.constraint({c:1,direction[i]:-cmax},upper=0) + model.constraint({d:1,direction[i]:dmax},upper=dmax) + model.constraint({d:1,enabled[i]:-dmax},upper=0) + model.constraint({energies[i]:1,enabled[i]:-big},lower=reserve+epsilon-big) + # Once enabled, discharge may consume only energy above reserve. + model.constraint({energies[i+1]:1,enabled[i]:-big},lower=reserve-big) + if i: + # A disabled battery can rearm only AFTER prior charging has + # actually reached the configured hysteresis threshold. + model.constraint({energies[i]:1,enabled[i]:-big,enabled[i-1]:big},lower=rearm-big) + model.constraint({energies[i+1]:1,energies[i]:-1,c:-eta*dt,d:dt/eta},0,0) + return {'charge':charge,'discharge':discharge,'energy':energies, + 'terminal_min_percent':terminal,'discharge_enabled':enabled} diff --git a/services/netplan-v4/netplan_v4/controlled_trial.py b/services/netplan-v4/netplan_v4/controlled_trial.py new file mode 100644 index 0000000..cf9afcb --- /dev/null +++ b/services/netplan-v4/netplan_v4/controlled_trial.py @@ -0,0 +1,162 @@ +"""Explicit, time-limited commissioning authority; NEVER enables shadow execution. + +Only a reviewed internal operator call can arm a trial, and only for a separate +allowlist (empty by default). The manager and battery must additionally consent +locally. Existing shadow plans/acknowledgements keep their original meaning. +""" +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from uuid import UUID +import json + +MAX_SECONDS = 1800 +MAX_POWER_W = 5000 +AUTHORITY_TTL_SECONDS = 90 + + +def schema(con): + con.execute('''CREATE TABLE IF NOT EXISTS planner_controlled_trials( + plant TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE, + value TEXT NOT NULL, revoked_at TEXT)''') + + +def timestamp(v): + if not isinstance(v, str): + raise ValueError('Explicit timestamp required') + t = datetime.fromisoformat(v.replace('Z', '+00:00')) + if t.tzinfo is None: + raise ValueError('Timezone required') + return t.astimezone(timezone.utc) + + +def uuid(v): + if not isinstance(v, str) or str(UUID(v)) != v: + raise ValueError('Canonical UUID required') + return v + + +def power(v): + if type(v) not in (int, float) or not 0 < v <= MAX_POWER_W: + raise ValueError('Pilot limit must be explicit and at most 5000 W') + return float(v) + + +def accounting(plan): + # A human checkbox alone must not relabel aggregate house consumption. + quality = plan.get('inputQuality', {}) + if quality.get('loadBasis') != 'base_load': + raise ValueError('Verified base-load/SDL adapter required before a control trial') + evidence = quality.get('accountingEvidenceId') + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Plan lacks traceable base-load accounting evidence') + return evidence + + +def eligibility(view, plant): + if view.get('installationId') != plant or view.get('liveEnabled') is not False: + raise ValueError('Wrong plant or service mode') + p = view.get('plan') + if view.get('fresh') is not True or not isinstance(p, dict): + raise ValueError('Fresh independently validated planning input required') + if (p.get('installationId') != plant or p.get('runMode') != 'shadow' + or p.get('liveEnabled') is not False or p.get('executable') is not True): + raise ValueError('Wrong source plan identity or mode') + settings = view.get('settings', {}) + if settings.get('family') == 'auto' or p.get('sourceFamily') != settings.get('family'): + raise ValueError('First controlled trial requires a fixed model family') + if p.get('configRevision') != settings.get('revision'): + raise ValueError('Configuration revision changed') + if len(p.get('controlContext', {}).get('batteries', {})) != 1: + raise ValueError('First controlled trial supports exactly one battery') + accounting(p) + return p + + +def arm(store, plant, request, view, now, allowed_plants): + """Caller is authenticated internal operator; not exposed via device proxy.""" + if plant not in allowed_plants: + raise ValueError('Controlled trial disabled by server allowlist') + fields = {'sessionId', 'expectedPlanId', 'expectedRevision', 'durationSeconds', + 'maxChargeW', 'maxDischargeW', 'assetId', 'managerId', 'batteryInstanceId', + 'acceptEstimatedPeak', 'actuatorWatchdogEvidenceId', 'confirmation'} + if set(request) != fields or request['confirmation'] != 'ARM_BOUNDED_CONTROL_TRIAL': + raise ValueError('Explicit reviewed commissioning request required') + session = uuid(request['sessionId']); p = eligibility(view, plant) + if type(request['expectedRevision']) is not int or request['expectedRevision'] < 0: + raise ValueError('Expected revision must be an integer') + if request['expectedPlanId'] != p['planId'] or request['expectedRevision'] != p['configRevision']: + raise ValueError('Source plan/revision changed; review again') + seconds = request['durationSeconds'] + if type(seconds) is not int or not 30 <= seconds <= MAX_SECONDS: + raise ValueError('Trial duration must be 30..1800 seconds') + for k in ('managerId', 'batteryInstanceId'): + if type(request[k]) is not int or not 1 <= request[k] <= 99999: + raise ValueError('Explicit local instance binding required') + if request['managerId'] == request['batteryInstanceId']: + raise ValueError('Manager and battery instance must differ') + if request['assetId'] not in p['controlContext']['batteries']: + raise ValueError('Wrong trial battery') + if type(request['acceptEstimatedPeak']) is not bool: + raise ValueError('Explicit estimated-peak policy required') + if p.get('peakCostIsEstimate') and not request['acceptEstimatedPeak']: + raise ValueError('Estimated peak has not been accepted for this trial') + evidence = request['actuatorWatchdogEvidenceId'] + if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160: + raise ValueError('Device-side command-loss watchdog proof required') + value = {'kind': 'controlled_trial_grant', 'version': 1, 'sessionId': session, + 'installationId': plant, 'issuedAt': now.isoformat(), + 'expiresAt': (now + timedelta(seconds=seconds)).isoformat(), + 'revision': p['configRevision'], 'family': p['sourceFamily'], + 'assetId': request['assetId'], 'managerId': request['managerId'], + 'batteryInstanceId': request['batteryInstanceId'], + 'maxChargeW': power(request['maxChargeW']), + 'maxDischargeW': power(request['maxDischargeW']), + 'acceptEstimatedPeak': request['acceptEstimatedPeak'], + 'accountingEvidenceId': accounting(p), + 'actuatorWatchdogEvidenceId': evidence, + 'controlContext': deepcopy(p['controlContext'])} + with store.con: + existing = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if existing and existing[1] is None and timestamp(json.loads(existing[0])['expiresAt']) > now: + raise ValueError('Existing trial must first end; implicit extension forbidden') + # Reusing an expired/revoked session ID must never resurrect it. + used = store.con.execute("SELECT 1 FROM planner_audit WHERE plant=? AND kind='trial_arm' AND detail=?", (plant, session)).fetchone() + if used: + raise ValueError('Session ID already used') + store.con.execute('INSERT INTO planner_controlled_trials VALUES(?,?,?,NULL) ON CONFLICT(plant) DO UPDATE SET session_id=excluded.session_id,value=excluded.value,revoked_at=NULL', + (plant, session, json.dumps(value, sort_keys=True, allow_nan=False))) + store.con.execute("INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,'trial_arm',?)", (plant, now.isoformat(), session)) + return value + + +def revoke(store, plant, session, now): + uuid(session) + with store.con: + store.con.execute('UPDATE planner_controlled_trials SET revoked_at=? WHERE plant=? AND session_id=?', (now.isoformat(), plant, session)) + return {'status': 'revoked', 'sessionId': session, 'remoteRevocationMaxSeconds': AUTHORITY_TTL_SECONDS} + + +def authority(store, plant, view, now, allowed_plants): + """Separate authorization envelope, not a mutation of the shadow plan.""" + if plant not in allowed_plants: + return None + row = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone() + if not row or row[1] is not None: + return None + grant = json.loads(row[0]) + try: + p = eligibility(view, plant) + if not timestamp(grant['issuedAt']) <= now < timestamp(grant['expiresAt']): + return None + if (p['configRevision'] != grant['revision'] or p['sourceFamily'] != grant['family'] + or p['controlContext'] != grant['controlContext'] + or accounting(p) != grant['accountingEvidenceId'] + or p.get('peakCostIsEstimate') and not grant['acceptEstimatedPeak']): + return None + except (ValueError, KeyError, TypeError): + return None + until = min(timestamp(grant['expiresAt']), timestamp(p['validUntil']), + now + timedelta(seconds=AUTHORITY_TTL_SECONDS)) + return {**grant, 'kind': 'controlled_trial_authority', + 'sourceShadowPlanId': p['planId'], 'checkedAt': now.isoformat(), + 'validUntil': until.isoformat(), 'sourcePlanRemainsShadow': True} diff --git a/services/netplan-v4/netplan_v4/domain.py b/services/netplan-v4/netplan_v4/domain.py new file mode 100644 index 0000000..140acb8 --- /dev/null +++ b/services/netplan-v4/netplan_v4/domain.py @@ -0,0 +1,170 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from math import isfinite +from typing import Mapping +from zoneinfo import ZoneInfo + +UTC = timezone.utc +ZURICH = ZoneInfo('Europe/Zurich') + +def utc(value): + if isinstance(value, str): + value = datetime.fromisoformat(value.replace('Z', '+00:00')) + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError('Timezone required') + return value.astimezone(UTC) + +def number(value, name, minimum=None, maximum=None): + if isinstance(value, bool) or not isinstance(value, (float, int)): + raise ValueError(f'{name}: finite number required') + value = float(value) + if not isfinite(value) or minimum is not None and value < minimum or maximum is not None and value > maximum: + raise ValueError(f'{name}: outside permitted range') + return value + +def quarter_start(value): + value = utc(value) + return value.replace(minute=value.minute // 15 * 15, second=0, microsecond=0) + +def month_key(value): + return utc(value).astimezone(ZURICH).strftime('%Y-%m') + +@dataclass(frozen=True) +class Family: + key: str + pv: str + load: str + schedule: str + label: str + +class FamilyRegistry: + def __init__(self, families=()): + self._families = {} + for family in families: + self.register(family) + def register(self, family): + if family.key in self._families or any(f.schedule == family.schedule for f in self._families.values()): + raise ValueError('Duplicate family or schedule identifier') + self._families[family.key] = family + def get(self, key): + if key not in self._families: + raise ValueError(f'Unknown model family: {key}') + return self._families[key] + def entries(self): + return tuple(self._families.values()) + +def default_registry(): + return FamilyRegistry([ + Family('3','prog_var_1','prog_var_2','prog_var_3','Variante 1 (1 / 2 / 3)'), + Family('13','prog_var_10','prog_var_11','prog_var_13','Variante 2 (10 / 11 / 13)'), + Family('23','prog_var_21','prog_var_22','prog_var_23','Variante 3 (21 / 22 / 23)'), + ]) + +@dataclass(frozen=True) +class Price: + chf_kwh: float + published_at: datetime | None = None + mode: str = 'static' + def known_at(self, at): + number(self.chf_kwh, 'energy price') + if self.mode not in ('static','dynamic'): + raise ValueError('Explicit static/dynamic price mode required') + return self.mode == 'static' or self.published_at is not None and utc(self.published_at) <= utc(at) + +@dataclass(frozen=True) +class Step: + start: datetime + base_load_w: float + pv_w: float + import_price: Price | None + export_price: Price | None + external_w: float = 0.0 + seconds: int = 300 + @property + def end(self): + return utc(self.start) + timedelta(seconds=self.seconds) + @property + def residual_w(self): + return self.base_load_w + self.external_w - self.pv_w + +def split_base_load(measured_house_w, flexible_w, external_w=0.0, external_already_removed=False): + measured = number(measured_house_w, 'measured_house_w') + flex = sum(number(v, 'flexible measurement') for v in flexible_w) + external = number(external_w, 'external measurement') + base = measured - flex - (0.0 if external_already_removed else external) + if base < -1.0: + raise ValueError('Negative base load: inconsistent measurement boundary or double subtraction') + return max(0.0, base) + +def priced_prefix(steps, at): + result = [] + for step in steps: + if not step.import_price or not step.export_price: + break + if not step.import_price.known_at(at) or not step.export_price.known_at(at): + break + result.append(step) + while result and result[-1].end != quarter_start(result[-1].end): + result.pop() + return result + +@dataclass(frozen=True) +class Battery: + asset_id: str + capacity_kwh: float + soc_percent: float + min_soc_percent: float + max_soc_percent: float + max_charge_w: float + max_discharge_w: float + measured_at: datetime + roundtrip_efficiency: float = 0.90 + grid_charging: bool = False + throughput_chf_kwh: float = 0.0 + terminal_soc_min_percent: float | None = None + terminal_value_chf_kwh: float = 0.0 + recovery_allowed: bool = False + physical_min_soc_percent: float = 0.0 + discharge_blocked: bool = False + rearm_soc_percent: float | None = None + def validate(self, at): + if not self.asset_id: + raise ValueError('Battery asset_id required') + number(self.capacity_kwh, 'capacity_kwh', 0.001) + lo = number(self.min_soc_percent, 'min_soc_percent', 0, 100) + hi = number(self.max_soc_percent, 'max_soc_percent', lo, 100) + physical=number(self.physical_min_soc_percent,'physical minimum SOC',0,lo) + if type(self.recovery_allowed) is not bool or type(self.discharge_blocked) is not bool: + raise ValueError('Explicit battery recovery and hysteresis flags required') + number(self.soc_percent, 'SOC', physical if self.recovery_allowed else lo, hi) + if self.rearm_soc_percent is not None:number(self.rearm_soc_percent,'hysteresis rearm SOC',lo,hi) + number(self.max_charge_w, 'max_charge_w', 0) + number(self.max_discharge_w, 'max_discharge_w', 0) + number(self.roundtrip_efficiency, 'roundtrip_efficiency', 0.01, 1) + number(self.throughput_chf_kwh, 'throughput_chf_kwh', 0) + number(self.terminal_value_chf_kwh, 'terminal_value_chf_kwh', 0) + if self.terminal_soc_min_percent is not None: + number(self.terminal_soc_min_percent, 'terminal_soc_min_percent', lo, hi) + age = (utc(at) - utc(self.measured_at)).total_seconds() + if age < -30 or age > 1800: + raise ValueError('SOC is stale or from the future') + +@dataclass(frozen=True) +class QuarterPast: + import_kwh: float + measured_seconds: int + +@dataclass(frozen=True) +class Limits: + export_w: float | None = None + import_w: float | None = None + manager_month_limits_w: Mapping[int,float] = field(default_factory=dict) + def import_limit(self, timestamp): + values = [] + if self.import_w is not None: + values.append(number(self.import_w, 'import limit', 0)) + m = utc(timestamp).astimezone(ZURICH).month + if m in self.manager_month_limits_w: + values.append(number(self.manager_month_limits_w[m], 'monthly manager limit', 0)) + return min(values) if values else None diff --git a/services/netplan-v4/netplan_v4/forecast_quality.py b/services/netplan-v4/netplan_v4/forecast_quality.py new file mode 100644 index 0000000..4a83446 --- /dev/null +++ b/services/netplan-v4/netplan_v4/forecast_quality.py @@ -0,0 +1,21 @@ +"""Minimal validity gates, not a claim of forecast accuracy or model ranking.""" +from math import isfinite + + +def assess_family(family, *, minimum_steps=3): + points=family.get('points',[]) + if len(points) physical load -> trained profiles. + +Lives in the existing planner service/database; no separate diagnostic service. +The device may append only to an operator-configured dataset. Original observations, +model revisions and prediction vintages are preserved. Output is never an actuator grant. +""" +from __future__ import annotations +from bisect import bisect_right +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from math import isfinite +from statistics import median +from zoneinfo import ZoneInfo +import json + +UTC = timezone.utc +LOCAL = ZoneInfo('Europe/Zurich') +FAMILIES = ('3', '13', '23') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False) + + +def epoch(value): + if not isinstance(value, str): + raise ValueError('UTC timestamp required') + t = datetime.fromisoformat(value.replace('Z', '+00:00')) + if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond: + raise ValueError('Explicit whole-second UTC timestamp required') + return int(t.timestamp()) + + +def iso(t): + return datetime.fromtimestamp(t, UTC).isoformat() + + +def numeric(value, bound=1e12): + return type(value) in (int, float) and isfinite(value) and abs(value) <= bound + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_data_sets( + plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_observations( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,captured_at)); + CREATE TABLE IF NOT EXISTS planner_load_windows( + plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL, + available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,start)); + CREATE TABLE IF NOT EXISTS planner_load_models( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,dataset,model_id)); + CREATE TABLE IF NOT EXISTS planner_model_current( + plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL, + PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_pipeline_state( + plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL, + status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset)); + CREATE TABLE IF NOT EXISTS planner_prediction_vintages( + plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL, + target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL, + load_w REAL NOT NULL, pv_w REAL NOT NULL, + PRIMARY KEY(plant,dataset,issued_at,target,family)); + CREATE INDEX IF NOT EXISTS planner_observation_window + ON planner_observations(plant,dataset,captured_at); + CREATE INDEX IF NOT EXISTS planner_prediction_target + ON planner_prediction_vintages(plant,dataset,target); + ''') + + +def validate_config(c): + fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources', + 'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds', + 'minimumTrainingHours', 'historyDays'} + if not isinstance(c, dict) or set(c) != fields: + raise ValueError('Explicit dataset configuration required') + name = c['datasetId'] + if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name): + raise ValueError('Invalid dataset ID') + for field in ('mappingSha256', 'inventorySha256'): + h = c[field] + if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h): + raise ValueError('Explicit mapping/inventory fingerprint required') + if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'): + raise ValueError('Unknown physical formula') + if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1: + raise ValueError('Coverage must be .90..1; recorded gaps remain visible') + for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)): + if type(c[field]) is not int or not lo <= c[field] <= hi: + raise ValueError('Invalid '+field) + sources = c['sources'] + if not isinstance(sources, list) or not 3 <= len(sources) <= 80: + raise ValueError('Source list required') + seen, ids = set(), set() + roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'} + for s in sources: + if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}: + raise ValueError('Explicit source definition required') + k = s['key'] + if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids: + raise ValueError('Duplicate/invalid source') + if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0: + raise ValueError('Source role/factor invalid') + if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300: + raise ValueError('Source lifetime invalid') + seen.add(k); ids.add(s['variableId']) + if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources): + raise ValueError('Grid and PV measurement sources required') + sr = c['solarReference'] + if c['formula'] == 'solar_terminal_v1': + if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}: + raise ValueError('Solar terminal sources required') + bykey = {s['key']: s['role'] for s in sources} + if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))): + raise ValueError('Solar origin roles mismatch') + elif sr is not None: + raise ValueError('No unused solar mapping allowed') + canonical(c) + return c + + +def register_dataset(con, plant, c, now): + """Operator endpoint only; device append endpoint cannot change units or limits.""" + validate_config(c) + value = canonical(c) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone() + if old and old[0] != value: + raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning') + con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False} + + +def configuration(con, plant, dataset): + row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone() + if row is None: + raise ValueError('Dataset not configured for this installation') + return json.loads(row[0]) + + +def project(record, c, plant, now): + if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: + raise ValueError('Wrong capture identity') + if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + raise ValueError('Wrong capture mapping or inventory') + t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) + if start > t or t > now+30 or t < now-90*86400: + raise ValueError('Capture timestamp outside permitted range') + if not isinstance(record.get('raw'), dict): + raise ValueError('Numeric raw observations required') + out = {}; issues = [] + for s in c['sources']: + r = record['raw'].get(s['key'], {}) + if not isinstance(r, dict): + r = {} + v, at = r.get('value'), r.get('sourceUpdatedAt') + good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == [] + if not good: + v = at = None + issues.append(s['key']) + # Unknown/free-text fields, credentials, client quality claims never persisted. + out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)} + return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues} + + +def ingest_batch(con, plant, payload, now): + if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1: + raise ValueError('Measurement batch version/fields invalid') + c = configuration(con,plant,payload['datasetId']) + records = payload['records'] + if not isinstance(records,list) or not 1 <= len(records) <= 120: + raise ValueError('Batch requires 1..120 captures') + rows = [project(r,c,plant,now) for r in records] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): + raise ValueError('Batch must be in increasing capture order') + stored = duplicate = 0 + con.execute('BEGIN IMMEDIATE') + try: + for r in rows: + value = canonical(r); digest = sha256(value.encode()).hexdigest() + old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() + if old: + if old[0] != digest: + raise ValueError('Conflicting immutable observation') + duplicate += 1 + else: + con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + con.commit() + except Exception: + con.rollback(); raise + return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate, + 'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False} + + +def physical_value(values, c): + """Same explicit sign convention as configured acquisition. No virtual power in load.""" + total = defaultdict(float) + sr = c['solarReference'] + for s in c['sources']: + if s['role'] not in ('grid','pv','physical_storage','flexible_load'): + continue + if sr and s['key'] in (sr['pvKey'],sr['batteryKey']): + continue + val = values[s['key']]*s['factorToW'] + if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0): + raise ValueError('Invalid physical power') + total[s['role']] += val + solar = 0. + if sr: + raw, sf = values[sr['rawKey']], values[sr['scaleKey']] + if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6: + raise ValueError('Invalid solar power/scaling sentinel') + solar = raw*10**int(sf) + load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar + if not numeric(load,1e9) or load < 0: + raise ValueError('Negative/nonfinite physical load') + return load + + +def reconstruct(records, c): + """Bounded retrospective estimation, never a real-time feedback signal. + +Missing observations split support. Source timestamps are not refreshed. Small +uncovered portions remain quantified and are never filled with zero. +""" + if len(records) < 2: + return [] + if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])): + raise ValueError('Capture sequence not ordered') + sr = c['solarReference'] + primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')} + if sr: + primary.pop(sr['pvKey']); primary.pop(sr['batteryKey']) + for s in c['sources']: + if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s + first,last = records[0]['capturedAt'],records[-1]['capturedAt'] + series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] + pending = {k:None for k in primary}; high = {k:0 for k in primary} + edges = {first,last} + for a,b in zip(records,records[1:]): + if b['capturedAt']-a['capturedAt'] > 45: + gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1]) + for r in records: + at = r['capturedAt'] + for k in primary: + v = r['raw'].get(k,{}) + t = v.get('sourceUpdatedAt') + if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5: + if pending[k] is None: pending[k] = at + continue + high[k] = max(high[k],t) + if pending[k] is not None: + blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None + if t in series[k] and series[k][t] != v['value']: + series[k][t] = None + else: + series[k].setdefault(t,v['value']) + for k,s in primary.items(): + if pending[k] is not None: + blocks[k].append((pending[k],last));edges.update(blocks[k][-1]) + for t in series[k]: edges.update((t,t+s['maxAgeSeconds'])) + edges.update(range(first//300*300+300,last,300)) + edges = sorted(x for x in edges if first <= x <= last) + knots = {k:sorted(v) for k,v in series.items()} + bins = {} + for a,b in zip(edges,edges[1:]): + start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0}) + vals = {}; usable = not any(x <= a < y for x,y in gaps) + for k,s in primary.items(): + pos = bisect_right(knots[k],a)-1 + t = knots[k][pos] if pos >= 0 else None + if t is None or a >= t+s['maxAgeSeconds'] or series[k][t] is None or any(x <= a < y for x,y in blocks[k]): + usable = False + else: vals[k] = series[k][t] + load = None + if usable: + try: load = physical_value(vals,c) + except ValueError: usable = False + if usable: + item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 + else: + item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap']) + out=[] + for t,item in sorted(bins.items()): + # Partial beginning/end bins remain diagnostic and cannot train. + complete_extent = first <= t and last >= t+300 + coverage = item['seconds']/300 + eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds'] + out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'], + 'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None, + 'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False, + 'meterBoundaryVerified':False,'method':c['formula']}) + return out + + +def slot(t): + local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL) + return local.hour*12+local.minute//5 + + +def build_profiles(rows): + samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)} + anchor=max(r['start'] for r in rows) + for r in rows: + i=slot(r['start']); v=r['loadW']; samples[i].append(v) + if anchor-r['start'] < 86400: recent[i].append(v) + weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v) + overall = median([r['loadW'] for r in rows]) + def profile(values): + # Missing calendar slots are a model estimate, not invented historical measurements. + result=[] + for i in range(288): + local=values.get(i,[]) + if not local: + local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])] + result.append(float(median(local)) if local else float(overall)) + return result + return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)}, + 'slotCoverage':len(samples)/288} + + +def predict(model, family, t): + p=model['profiles'][family] + if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday'] + return p[slot(t)] + + +def advance(con, plant, dataset, settings, now): + """Called by the existing worker; bounded data/model update once per five-minute tick.""" + c=configuration(con,plant,dataset); tick=now//300 + old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone() + if old and old[0]==tick: return + fetched=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at', + (plant,dataset,now-172800-300,now,now)).fetchall() + records=[json.loads(r[0]) for r in fetched] + windows=reconstruct(records,c) + with con: + for w in windows: + if w['start']+300 > now-30: continue + con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value', + (plant,dataset,w['start'],now,w['coverage'],canonical(w))) + rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))] + good=[r for r in rows if r['profileUsable']] + active=current_model(con,plant,dataset,now) + cadence=86400 if settings['trainingCadence']=='daily' else 604800 + detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'], + 'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'], + 'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True} + state='collecting' + if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12: + state='model_ready' if active else 'training' + attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0] + if attempted is None or now-attempted>=cadence: + profiles=build_profiles(good) + candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good), + 'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'], + 'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'}, + 'measurementBoundaryVerified':False} + # Causal held-out validation: build validation profiles without the final day. + split=good[-1]['start']-86400 + train=[r for r in good if r['start']+300<=split]; test=[r for r in good if r['start']>=split] + if len(train)>=288 and len(test)>=240: + val={'profiles':build_profiles(train)} + errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES} + candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors} + # Initial model is labelled bootstrap, never a production measurement proof. + # Existing model can be replaced only with held-out evidence and no aggregate regression. + promote=active is None + if active and candidate['validation']['status']=='causal_holdout': + past_model_eligible=active['trainedThrough']<=split + if past_model_eligible: + incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test) + promote=challenger<=incumbent + candidate['validation']['incumbentCompared']=True + else: + candidate['validation']['status']='holdout_overlaps_active_training' + ident=sha256(canonical(candidate).encode()).hexdigest() + with con: + con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate))) + if promote: + con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident)) + detail['candidateModelId']=ident;detail['candidatePromoted']=promote + state='model_ready' if promote or active else 'candidate_pending' + active=current_model(con,plant,dataset,now) + if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']}) + with con: + con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail', + (plant,dataset,tick,state,canonical(detail))) + + +def current_model(con,plant,dataset,at): + row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone() + return {**json.loads(row[1]),'modelId':row[0]} if row else None + + +def apply_load_forecast(con,plant,dataset,forecast,decision): + model=current_model(con,plant,dataset,decision) + if not model: + raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused') + last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,dataset,decision,decision)).fetchone() + if not last: raise ValueError('No recent corrected observation') + last=json.loads(last[0]); c=configuration(con,plant,dataset) + if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds') + sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request'] + if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario') + s=sdl_sources[0];r=last['raw'][s['key']] + if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']: + raise ValueError('No current external SDL request for the persistence scenario') + sdl=r['value']*s['factorToW'] + result=json.loads(canonical(forecast)); result['families']={} + result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt'])) + for family,old in forecast['families'].items(): + if family not in FAMILIES: continue + points=[] + for p in old['points']: + t=epoch(p['time']) + points.append({**p,'loadW':predict(model,family,t),'externalW':sdl}) + result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']), + 'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'], + 'loadMethodVersion':model['methodVersion'],'loadVariant':family, + 'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'), + 'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False, + 'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']), + 'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation']}} + return result + + +def pipeline_status(con,plant): + out=[] + for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): + ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,ds)).fetchone() + out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], + 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, + 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds']}) + return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False} diff --git a/services/netplan-v4/netplan_v4/meter_runtime.py b/services/netplan-v4/netplan_v4/meter_runtime.py new file mode 100644 index 0000000..db92eda --- /dev/null +++ b/services/netplan-v4/netplan_v4/meter_runtime.py @@ -0,0 +1,145 @@ +"""Persistent, explicitly estimated operational demand tracking. + +No inference from a configured cap. Samples are device-reception observations; +last-value integration is a labelled control estimate, not settlement metering. +No reset, stale sample or long communication gap is bridged silently. +""" +from datetime import datetime,timedelta,timezone +import json +from .domain import number,utc,month_key,quarter_start,ZURICH +from .peak_policy import basis_record + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_peak_assumptions( + plant TEXT NOT NULL,month TEXT NOT NULL,kw REAL NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_runtime_samples( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,at INTEGER NOT NULL, + power_w REAL NOT NULL,total_kwh REAL,policy_id TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,at)); + CREATE TABLE IF NOT EXISTS planner_runtime_quarters( + plant TEXT NOT NULL,meter_id TEXT NOT NULL,start INTEGER NOT NULL, + import_kwh REAL NOT NULL,policy_id TEXT NOT NULL,value TEXT NOT NULL, + PRIMARY KEY(plant,meter_id,start)); + CREATE INDEX IF NOT EXISTS idx_runtime_samples_time ON planner_runtime_samples(plant,at); + ''') + + +def validate_observation(obs, observed_at): + if not isinstance(obs,dict) or set(obs)!={'meterId','sampleAt','powerW','totalImportKwh','controlPolicyId'}: + raise ValueError('Explicit meter observation schema required') + import re + if not isinstance(obs['meterId'],str) or not re.fullmatch(r'symcon-active-import:[0-9a-f]{64}',obs['meterId']): + raise ValueError('Active import identity required') + if not isinstance(obs['controlPolicyId'],str) or not 1<=len(obs['controlPolicyId'])<=160: + raise ValueError('Control policy identity required') + t=utc(obs['sampleAt']) + if t.microsecond or not 0<=(utc(observed_at)-t).total_seconds()<=60: + raise ValueError('Fresh whole-second acquisition timestamp required') + number(obs['powerW'],'meter power',-1e9,1e9) + number(obs['totalImportKwh'],'active import total',0,1e12) + + +def integrate_power(samples,start,end,max_gap=120): + """samples sorted tuples (epoch,power,total,policy). Return None on gaps/reset. + + A source can keep the same value while receiving fresh telemetry. The caller + records all acquisitions, not only changes. A quarter is never labelled exact. + """ + if end<=start:return None + rows=sorted(samples,key=lambda r:r[0]) + if len(rows)<2:return None + covered=energy=0.;policies=set();largest_gap=0 + previous=None + for row in rows: + if previous is not None: + ta,pa,ea,pola=previous;tb,pb,eb,polb=row + if tb<=ta:return None + left=max(start,ta);right=min(end,tb) + if right>left: + if tb-ta>max_gap or pola!=polb:return None + if ea is not None and eb is not None and eb=validated['kw']:return old + con.execute('INSERT INTO planner_peak_assumptions VALUES(?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET kw=excluded.kw,value=excluded.value', + (plant,month,validated['kw'],json.dumps(validated,sort_keys=True,allow_nan=False))) + return validated + + +def assumptions(con,plant): + return {r[0]:json.loads(r[1]) for r in con.execute('SELECT month,value FROM planner_peak_assumptions WHERE plant=?',(plant,))} + + +def observe(con,plant,obs,at): + validate_observation(obs,at) + mid=obs['meterId'];stamp=int(utc(obs['sampleAt']).timestamp()) + existing=con.execute('SELECT power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at=?',(plant,mid,stamp)).fetchone() + values=(float(obs['powerW']),float(obs['totalImportKwh']),obs['controlPolicyId']) + if existing and tuple(existing)!=values:raise ValueError('Conflicting meter acquisition at same time') + con.execute('INSERT OR IGNORE INTO planner_runtime_samples VALUES(?,?,?,?,?,?)',(plant,mid,stamp,*values)) + # Only a just-completed quarter is finalised; late acquisition never promotes + # a historical gap to complete data without all supporting samples. + current=stamp//900*900 + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,mid,current-900-120,stamp))] + report=integrate_power(rows,current-900,current) + if report: + quarter=current-900 + previous=con.execute('SELECT import_kwh FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start=?',(plant,mid,quarter)).fetchone() + if previous is None: + con.execute('INSERT INTO planner_runtime_quarters VALUES(?,?,?,?,?,?)',(plant,mid,quarter,report['importKwh'],report['controlPolicyId'],json.dumps(report))) + m=month_key(datetime.fromtimestamp(quarter,timezone.utc)) + save_assumption(con,plant,m,{'kw':report['importKwh']/.25,'quality':'estimated','source':'sampled_power_estimate', + 'observedAt':utc(at).isoformat(),'meterId':mid,'notes':'Maximum of available sampled quarters; earlier month may be incomplete'},at) + # Samples are retained for reproducibility; production retention job required. + return report + + +def current_quarter(con,plant,obs,decision): + q=quarter_start(decision);start=int(q.timestamp());end=int(utc(decision).timestamp()) + if start==end:return None + rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at', + (plant,obs['meterId'],start-120,end))] + report=integrate_power(rows,start,end) + if not report:return None + return {'start':q.isoformat(),'measuredSeconds':end-start,'importKwh':report['importKwh'], + 'quality':'estimated','source':'sampled_power_estimate','coverage':1.0,'notes':'Acquisition power estimate, not an exact billing counter boundary'} + + +def daily_peaks(con,plant,meter_id,now): + rows=con.execute('SELECT start,import_kwh,policy_id FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start>=? ORDER BY start', + (plant,meter_id,int((utc(now)-timedelta(days=91)).timestamp()))) + grouped={} + for start,energy,policy in rows: + t=datetime.fromtimestamp(start,timezone.utc).astimezone(ZURICH) + grouped.setdefault((t.strftime('%Y-%m-%d'),policy),[]).append((start,energy)) + result=[] + for (day,policy),items in grouped.items(): + begin=datetime.strptime(day,'%Y-%m-%d').replace(tzinfo=ZURICH);end=begin+timedelta(days=1) + if utc(end)>utc(now):continue + expected=set(range(int(begin.timestamp()),int(end.timestamp()),900)) + complete={t for t,e in items}==expected + result.append({'day':day,'peakKw':max(e/.25 for t,e in items),'controlPolicyId':policy,'complete':complete, + 'observedAt':utc(end).isoformat(),'quality':'estimated'}) + return result diff --git a/services/netplan-v4/netplan_v4/metering.py b/services/netplan-v4/netplan_v4/metering.py new file mode 100644 index 0000000..ecca015 --- /dev/null +++ b/services/netplan-v4/netplan_v4/metering.py @@ -0,0 +1,245 @@ +"""Auditable import-energy evidence. Pure calculations; no device/database writes. + +Raw change archives are NOT proof of uninterrupted meter communication. Boundaries +without an exact counter record yield intervals, never silently interpolated +billing facts. Only a separately verified source and chronology can be promoted +into the existing strict planner evidence contract. +""" +from __future__ import annotations +from bisect import bisect_left, bisect_right +from dataclasses import dataclass +from datetime import datetime, timedelta +import hashlib +import json +from math import isfinite +from .domain import utc, month_key, quarter_start, ZURICH + + +@dataclass(frozen=True) +class Reading: + at: datetime + kwh: float + + +class CounterSeries: + def __init__(self, source_id: str, rows, *, max_bracket_seconds=180, + quantity='active_import', unit='kWh', timestamp_verified=False, + identity_verified=False, chronology_verified=False): + if quantity != 'active_import' or unit != 'kWh': + raise ValueError('Active import energy in kWh required') + if not isinstance(source_id, str) or not source_id: + raise ValueError('Nonempty stable source identity required') + if type(max_bracket_seconds) is not int or max_bracket_seconds < 1: + raise ValueError('Positive bracket age required') + self.source_id = source_id + self.max_gap = max_bracket_seconds + self.verified = all(x is True for x in + (timestamp_verified, identity_verified, chronology_verified)) + values = {} + for row in rows: + at = utc(row.at) + value = row.kwh + if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value < 0: + raise ValueError('Invalid cumulative active import value') + if at in values and abs(values[at] - value) > 1e-9: + raise ValueError('Conflicting duplicate timestamp') + values[at] = float(value) + self.times = sorted(values) + self.values = [values[t] for t in self.times] + self.resets = [self.times[i] for i in range(1,len(self.times)) + if self.values[i] < self.values[i-1] - 1e-9] + + def boundary(self, at): + at = utc(at) + pos = bisect_left(self.times, at) + if pos < len(self.times) and self.times[pos] == at: + value = self.values[pos] + return {'lowerKwh':value, 'upperKwh':value, 'exactRecord':True, + 'before':at.isoformat(), 'after':at.isoformat()} + if pos == 0 or pos == len(self.times): + return None + before, after = self.times[pos-1], self.times[pos] + if (after-before).total_seconds() > self.max_gap: + return None + if self.values[pos] < self.values[pos-1] - 1e-9: + return None + return {'lowerKwh':self.values[pos-1], 'upperKwh':self.values[pos], + 'exactRecord':False, 'before':before.isoformat(), 'after':after.isoformat()} + + def energy(self, start, end): + start, end = utc(start), utc(end) + if end <= start: + raise ValueError('Positive interval required') + left, right = self.boundary(start), self.boundary(end) + if not left or not right: + return None + support_start = utc(left['before']) + support_end = utc(right['after']) + if any(support_start < reset <= support_end for reset in self.resets): + return None + lower = max(0., right['lowerKwh'] - left['upperKwh']) + upper = right['upperKwh'] - left['lowerKwh'] + if upper < lower - 1e-9: + return None + exact = bool(left['exactRecord'] and right['exactRecord']) + return {'lowerKwh':lower, 'upperKwh':max(lower, upper), + 'exactRecords':exact, 'billingEvidence':exact and self.verified} + + +def native_meter_id(sources): + """Same ordered JSON and SHA256 as NetzfahrplanV4Bezugszaehler::identitaet.""" + import re + if not isinstance(sources,list) or not 1<=len(sources)<=8: + raise ValueError('Explicit native import sources required') + normalized=[];ids=set();idents=set();parents=set() + for source in sources: + if set(source)!={'VariableID','ElternID','Ident','FaktorZuKWh','Messgroesse'}: + raise ValueError('Unknown native source fields') + sid,pid,ident,factor=(source[k] for k in ('VariableID','ElternID','Ident','FaktorZuKWh')) + if (type(sid) is not int or sid<=0 or type(pid) is not int or pid<=0 + or sid in ids or ident in idents or not isinstance(ident,str) + or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}',ident) + or isinstance(factor,bool) or not isinstance(factor,(float,int)) + or not isfinite(factor) or not 0=0 else '')+str(power) + text=re.sub(r'"FaktorZuKWh":([0-9]+(?:\.[0-9]+)?)e([+-]?[0-9]+)',exponent,text) + return 'symcon-active-import:'+hashlib.sha256(text.encode()).hexdigest() + + +class MeterEvidence: + def __init__(self, series, *, native_sources=None): + self.series = list(series) + if not self.series or len({s.source_id for s in self.series}) != len(self.series): + raise ValueError('Unique, nonempty set of counter sources required') + self.native_bound = native_sources is not None + if self.native_bound: + self.meter_id=native_meter_id(native_sources) + if {s.source_id for s in self.series}!={'symcon:'+str(s['VariableID']) for s in native_sources}: + raise ValueError('Evidence stream does not match native source set') + else: + signature = json.dumps(sorted(s.source_id for s in self.series),separators=(',',':')) + self.meter_id = 'audit-only:' + hashlib.sha256(signature.encode()).hexdigest() + + def interval(self, start, end): + start, end = utc(start), utc(end) + parts = [s.energy(start,end) for s in self.series] + missing = [s.source_id for s,p in zip(self.series,parts) if p is None] + if missing: + return {'status':'missing', 'start':start.isoformat(),'end':end.isoformat(), + 'missingSources':missing, 'billingEvidence':False} + lo = sum(p['lowerKwh'] for p in parts) + hi = sum(p['upperKwh'] for p in parts) + seconds = (end-start).total_seconds() + exact = all(p['exactRecords'] for p in parts) + return {'status':'exact_records' if exact else 'bounded_records', + 'start':start.isoformat(),'end':end.isoformat(),'lowerKwh':lo,'upperKwh':hi, + 'lowerAverageKw':lo*3600/seconds,'upperAverageKw':hi*3600/seconds, + 'billingEvidence':all(p['billingEvidence'] for p in parts)} + + def month(self, at): + at = utc(at) + local = at.astimezone(ZURICH) + begin = utc(local.replace(day=1,hour=0,minute=0,second=0,microsecond=0)) + stop = quarter_start(at) + count = int((stop-begin).total_seconds())//900 + covered = exact = verified = 0 + lo = hi = 0. + gaps = [] + quarters = [] + for n in range(count): + start = begin + timedelta(minutes=15*n) + result = self.interval(start,start+timedelta(minutes=15)) + quarters.append(result) + if result['status'] == 'missing': + if len(gaps)<8:gaps.append(start.isoformat()) + continue + covered += 1 + exact += result['status']=='exact_records' + verified += result['billingEvidence'] + lo = max(lo,result['lowerAverageKw']) + hi = max(hi,result['upperAverageKw']) + complete = count>0 and covered==count + certified = count>0 and verified==count + return {'meterId':self.meter_id,'month':month_key(at),'completedQuarters':count, + 'coveredQuarters':covered,'exactRecordQuarters':exact,'verifiedQuarters':verified, + 'historyComplete':complete,'billingEvidence':certified, + 'observedPeakLowerKw':lo if covered else None, + 'observedPeakUpperKw':hi if covered else None, + 'monthPeakUpperKw':hi if complete else None, + 'verifiedMonthPeakKw':hi if certified else None, + 'firstMissingQuarters':gaps,'quarters':quarters} + + def strict_contract(self, at): + """Fail closed; do not promote archive gaps, estimated boundaries or a cap.""" + at = utc(at) + if not self.native_bound: + raise ValueError('Explicit native counter identity required') + result = self.month(at) + if not result['billingEvidence']: + raise ValueError('Full verified month history missing') + q = quarter_start(at) + past = None + if at != q: + partial = self.interval(q,at) + if not partial['billingEvidence']: + raise ValueError('Verified elapsed-quarter energy missing') + past = {'start':q.isoformat(),'measuredSeconds':int((at-q).total_seconds()), + 'importKwh':partial['upperKwh']} + return {'version':1,'meterId':self.meter_id,'measuredAt':at.isoformat(), + 'measuredPeaks':{month_key(at):{'kw':result['verifiedMonthPeakKw'], + 'source':'verified_month_history'}}, + 'quarterPast':past} + + +def audit_capture(payload): + """Read-only report from the dedicated Symcon export, never planner ingestion.""" + if payload.get('schemaVersion') != 1 or payload.get('kind') != 'v4_meter_capture': + raise ValueError('Unknown capture contract') + now = utc(payload['capturedAt']) + channels = payload['channels'] + series = [] + summaries = [] + for variable in ('59607','26620'): + channel = channels[variable] + rows = channel.get('history',[]) + readings = [Reading(utc(datetime.fromtimestamp(r['TimeStamp'],now.tzinfo)),r['Value']) for r in rows] + # An explicitly captured current value is usable from its capture, not + # from an earlier last-change timestamp, and never before logging began. + current = channel.get('snapshot') + if current and channel.get('snapshotConsistent'): + readings.append(Reading(utc(current['capturedAt']),current['value'])) + s = CounterSeries('symcon:'+variable, readings, timestamp_verified=False, + identity_verified=False, chronology_verified=False) + series.append(s) + summaries.append({'variableId':int(variable),'ident':channel['ident'], + 'logging':channel.get('logging'), 'rows':len(rows), + 'queryComplete':channel.get('queryComplete',False), + 'firstRecord':s.times[0].isoformat() if s.times else None, + 'lastRecord':s.times[-1].isoformat() if s.times else None, + 'counterDecreases':[t.isoformat() for t in s.resets]}) + meter = MeterEvidence(series) + month = meter.month(now) + current = None + q = quarter_start(now) + if now>q:current=meter.interval(q,now) + blockers=['Meter register identity/unit and acquisition timestamps still require verification.', + 'Change-only archive does not prove uninterrupted meter acquisition.'] + if any(not s['queryComplete'] for s in summaries):blockers.append('Archive export incomplete; do not infer complete history.') + if not month['historyComplete']:blockers.append('At least one completed billing quarter lacks bounded readings for every tariff.') + return {'capturedAt':now.isoformat(),'status':'audit_only','sourceSummary':summaries, + 'month':{k:v for k,v in month.items() if k!='quarters'}, + 'recentQuarters':month['quarters'][-8:], 'currentQuarter':current, + 'billingEvidence':False, 'liveEnabled':False,'blockers':blockers} diff --git a/services/netplan-v4/netplan_v4/optimizer.py b/services/netplan-v4/netplan_v4/optimizer.py new file mode 100644 index 0000000..8dabed5 --- /dev/null +++ b/services/netplan-v4/netplan_v4/optimizer.py @@ -0,0 +1,199 @@ +from __future__ import annotations +from math import sqrt +from uuid import uuid4 +import numpy as np +from scipy.optimize import Bounds, LinearConstraint, milp +from scipy.sparse import coo_matrix +from .domain import Battery, Limits, Step, month_key, number, quarter_start, utc +from .peak_policy import basis_record, RestMonthOutlook + +class Model: + def __init__(self): + self.lower,self.upper,self.cost,self.integer=[],[],[],[] + self.rows,self.row_lo,self.row_hi=[],[],[] + def variable(self, lower=0., upper=np.inf, cost=0., integer=False): + index=len(self.lower) + self.lower.append(lower);self.upper.append(upper);self.cost.append(cost);self.integer.append(int(integer)) + return index + def constraint(self, coefficients, lower=-np.inf, upper=np.inf): + self.rows.append(coefficients);self.row_lo.append(lower);self.row_hi.append(upper) + def matrices(self): + rr,cc,vv=[],[],[] + for row,coefficients in enumerate(self.rows): + for col,value in coefficients.items(): + rr.append(row);cc.append(col);vv.append(value) + matrix=coo_matrix((vv,(rr,cc)),shape=(len(self.rows),len(self.lower))).tocsr() + return matrix,np.array(self.row_lo),np.array(self.row_hi) + +from .battery_model import BatteryModel + +class AssetRegistry: + """Reviewed asset classes only; future EV/thermal models add their own constraints.""" + def __init__(self):self.models={Battery:BatteryModel} + def register(self,asset_type,implementation): + if asset_type in self.models:raise ValueError('Asset model already registered') + self.models[asset_type]=implementation + def build(self,model,asset,steps,direction): + if type(asset) not in self.models:raise ValueError('Unsupported asset model') + return self.models[type(asset)].build(model,asset,steps,direction) + +def _error(reason,status='invalid_inputs'): + return {'schemaVersion':2,'status':status,'executable':False,'points':[],'reason':str(reason)} + +def _validate(steps,assets,at,past,observed_peaks,peak_prices,limits): + if not steps or len(steps)>576:raise ValueError('Need 1..576 steps') + for i,step in enumerate(steps): + start=utc(step.start) + if type(step.seconds) is not int or not 1<=step.seconds<=300:raise ValueError('Interval duration must be 1..300 seconds') + if start.microsecond or step.end.second or step.end.microsecond or step.end.minute%5:raise ValueError('Intervals must end on a 5-minute boundary') + if i and (step.seconds!=300 or start.second or start.minute%5):raise ValueError('Only first interval may be shortened') + if i and start!=steps[i-1].end:raise ValueError('Missing/duplicated/overlapping interval') + number(step.base_load_w,'base load',0);number(step.pv_w,'PV',0);number(step.external_w,'external flow') + if not step.import_price or not step.export_price or not step.import_price.known_at(at) or not step.export_price.known_at(at):raise ValueError('Unknown or unpublished prices') + limits.import_limit(start) + if utc(at)!=utc(steps[0].start):raise ValueError('First interval must start at decision time') + if steps[-1].end!=quarter_start(steps[-1].end):raise ValueError('End horizon on complete billing quarter') + if limits.export_w is not None:number(limits.export_w,'export limit',0) + ids=set() + for asset in assets: + asset.validate(at) + if asset.asset_id in ids:raise ValueError('Duplicate asset_id') + ids.add(asset.asset_id) + q=quarter_start(steps[0].start);elapsed=int((utc(steps[0].start)-q).total_seconds()) + if any(key!=q for key in past):raise ValueError('Only elapsed energy of first quarter permitted') + if not elapsed and q in past and (past[q].import_kwh!=0 or past[q].measured_seconds!=0):raise ValueError('No elapsed energy at quarter boundary') + if elapsed: + if q not in past or past[q].measured_seconds!=elapsed:raise ValueError('Actual elapsed quarter import energy missing') + number(past[q].import_kwh,'quarter import energy',0) + for month in {month_key(s.start) for s in steps}: + if month not in observed_peaks or month not in peak_prices:raise ValueError(f'Measured peak state or tariff missing for {month}') + number(observed_peaks[month],'measured peak',0);number(peak_prices[month],'peak tariff',0) + +def optimize(steps,batteries,*,at,limits=None,observed_peaks=None,peak_prices=None,quarter_history=None,config_revision=1,family='3',timeout_seconds=30.,asset_registry=None,peak_context=None,peak_outlooks=None): + """Pure MILP. Peak state is measured, never a configured cap. No device calls.""" + limits=limits or Limits();observed_peaks=observed_peaks or {};peak_prices=peak_prices or {} + past={utc(k):v for k,v in (quarter_history or {}).items()} + try: + _validate(steps,batteries,at,past,observed_peaks,peak_prices,limits) + number(timeout_seconds,'solver timeout',.01,600) + contexts={} + for month in {month_key(s.start) for s in steps}: + if peak_context is None: + contexts[month]={'kw':observed_peaks[month],'quality':'verified','source':'legacy_verified_contract','observedAt':utc(at).isoformat()} + else: + contexts[month]=basis_record(peak_context[month],month,at,allow_estimates=True) + if abs(contexts[month]['kw']-observed_peaks[month])>1e-8:raise ValueError('Peak context differs from numerical basis') + outlooks=peak_outlooks or {} + if set(outlooks)-set(contexts):raise ValueError('Outlook for month outside horizon') + for month,outlook in outlooks.items(): + if not isinstance(outlook,RestMonthOutlook) or outlook.month!=month:raise ValueError('Invalid peak outlook') + outlook.validate(at,steps[-1].end) + except (ValueError,TypeError,KeyError,AttributeError) as exc:return _error(exc) + model=Model();direction=[model.variable(0,1,integer=True) for _ in steps];registry=asset_registry or AssetRegistry() + try:handles={b.asset_id:registry.build(model,b,steps,direction) for b in batteries} + except ValueError as exc:return _error(exc) + total_charge=sum(b.max_charge_w for b in batteries)/1000 + total_discharge=sum(b.max_discharge_w for b in batteries)/1000 + imp,exp,curtail=[],[],[];quarters={} + for i,step in enumerate(steps): + residual=step.residual_w/1000;dt=step.seconds/3600 + imax=max(0.,(step.base_load_w+step.external_w)/1000)+total_charge + emax=max(0.,-residual)+total_discharge;limit=limits.import_limit(step.start) + if limit is not None:imax=min(imax,limit/1000) + if limits.export_w is not None:emax=min(emax,limits.export_w/1000) + pi=model.variable(0,imax,step.import_price.chf_kwh*dt) + pe=model.variable(0,emax,-step.export_price.chf_kwh*dt) + pc=model.variable(0,step.pv_w/1000) + imp.append(pi);exp.append(pe);curtail.append(pc) + gm=model.variable(0,1,integer=True) + model.constraint({pi:1,gm:-imax},upper=0);model.constraint({pe:1,gm:emax},upper=emax) + balance={pi:1,pe:-1,pc:-1};pv_only={} + for b in batteries: + h=handles[b.asset_id];balance[h['charge'][i]]=-1;balance[h['discharge'][i]]=1 + if not b.grid_charging:pv_only[h['charge'][i]]=1 + model.constraint(balance,residual,residual) + if pv_only: + model.constraint(pv_only,upper=max(0.,-residual)) + no_grid_max=sum(b.max_charge_w for b in batteries if not b.grid_charging)/1000 + model.constraint({**pv_only,gm:no_grid_max},upper=no_grid_max) + quarters.setdefault(quarter_start(step.start),[]).append(i) + months=sorted({month_key(s.start) for s in steps}) + peaks={m:model.variable(observed_peaks[m]) for m in months} + for m in months: + if m in outlooks:outlooks[m].add_to_model(model,peaks[m],observed_peaks[m],peak_prices[m]) + else:model.cost[peaks[m]]=peak_prices[m] + for quarter,positions in quarters.items(): + coefficients={imp[i]:steps[i].seconds/900 for i in positions};coefficients[peaks[month_key(quarter)]]=-1 + used=past[quarter].import_kwh if quarter in past else 0. + model.constraint(coefficients,upper=-used/.25) + matrix,row_lo,row_hi=model.matrices() + try: + result=milp(np.array(model.cost),integrality=np.array(model.integer),bounds=Bounds(model.lower,model.upper),constraints=LinearConstraint(matrix,row_lo,row_hi),options={'time_limit':float(timeout_seconds),'mip_rel_gap':1e-4}) + except Exception as exc:return _error(f'Solver exception: {type(exc).__name__}','solver_error') + x=result.x + if result.status not in (0,1) or x is None:return _error(result.message,'no_feasible_plan') + if not np.isfinite(x).all():return _error('Non-finite solver result','validation_failed') + ax=matrix@x;tol=1e-6 + if (np.any(xnp.array(model.upper)+tol) or np.any(axrow_hi+tol) or any(abs(x[i]-round(x[i]))>tol for i,flag in enumerate(model.integer) if flag)): + return _error('Solver incumbent violates constraints','validation_failed') + running_peaks=dict(observed_peaks);baseline_peaks=dict(observed_peaks);peak_deltas={};baseline_peak_deltas={} + for quarter,positions in quarters.items(): + used=past[quarter].import_kwh if quarter in past else 0.;m=month_key(quarter) + demand=(used+sum(x[imp[i]]*steps[i].seconds/3600 for i in positions))/.25 + baseline=(used+sum(max(0.,steps[i].residual_w)*steps[i].seconds/3600000 for i in positions))/.25 + before=running_peaks[m];running_peaks[m]=max(before,demand);peak_deltas[positions[-1]]=(running_peaks[m]-before)*peak_prices[m] + before=baseline_peaks[m];baseline_peaks[m]=max(before,baseline);baseline_peak_deltas[positions[-1]]=(baseline_peaks[m]-before)*peak_prices[m] + points=[];cumulative_energy=cumulative_cash=cumulative_baseline_energy=cumulative_baseline_cash=cumulative_throughput=0. + for i,step in enumerate(steps): + dt=step.seconds/3600;p_import=max(0.,float(x[imp[i]]));p_export=max(0.,float(x[exp[i]])) + energy_cost=(p_import*step.import_price.chf_kwh-p_export*step.export_price.chf_kwh)*dt + baseline_import=max(0.,step.residual_w)/1000;baseline_export=max(0.,-step.residual_w)/1000 + if limits.export_w is not None:baseline_export=min(baseline_export,limits.export_w/1000) + baseline_cost=(baseline_import*step.import_price.chf_kwh-baseline_export*step.export_price.chf_kwh)*dt + targets,soc_end={},{};throughput_cost=0. + for b in batteries: + h=handles[b.asset_id];targets[b.asset_id]=float((x[h['charge'][i]]-x[h['discharge'][i]])*1000) + soc_end[b.asset_id]=float(100*x[h['energy'][i+1]]/b.capacity_kwh) + throughput_cost+=float((x[h['charge'][i]]+x[h['discharge'][i]])*dt*b.throughput_chf_kwh) + cumulative_energy+=energy_cost;cumulative_cash+=energy_cost+peak_deltas.get(i,0.) + cumulative_baseline_energy+=baseline_cost;cumulative_baseline_cash+=baseline_cost+baseline_peak_deltas.get(i,0.) + cumulative_throughput+=throughput_cost;battery_w=sum(targets.values()) + if battery_w>1:intent='gridCharge' if p_import>.001 else 'pvCharge' + elif battery_w< -1:intent='export' if p_export>.001 else 'discharge' + else:intent='hold' + points.append({'time':utc(step.start).isoformat(),'validUntil':step.end.isoformat(), + 'baselineGridW':float(step.residual_w),'gridTargetW':(p_import-p_export)*1000, + 'batteryTargetW':battery_w,'assetTargetsW':targets,'socEndPercent':soc_end, + 'pvCurtailmentW':float(x[curtail[i]]*1000),'intent':intent, + 'importLimitW':limits.import_limit(step.start),'exportLimitW':limits.export_w, + 'importPriceChfKwh':step.import_price.chf_kwh,'exportPriceChfKwh':step.export_price.chf_kwh, + 'energyCostChf':energy_cost,'additionalPeakCostChf':peak_deltas.get(i,0.), + 'throughputCostChf':throughput_cost,'cumulativeEnergyCostChf':cumulative_energy, + 'cumulativeCashCostChf':cumulative_cash,'cumulativeBaselineEnergyCostChf':cumulative_baseline_energy, + 'cumulativeBaselineCashCostChf':cumulative_baseline_cash,'baselineAdditionalPeakCostChf':baseline_peak_deltas.get(i,0.)}) + end_value=sum(float(x[handles[b.asset_id]['energy'][-1]])*b.terminal_value_chf_kwh for b in batteries) + def planning_cost(chosen): + return sum(outlooks[m].incremental_cost(observed_peaks[m],chosen[m],peak_prices[m]) if m in outlooks + else max(0.,chosen[m]-observed_peaks[m])*peak_prices[m] for m in months) + planning_peak=planning_cost(running_peaks) + full_peak=cumulative_cash-cumulative_energy + estimated=any(c['quality']=='estimated' for c in contexts.values()) + return {'schemaVersion':2,'planId':str(uuid4()),'configRevision':config_revision,'sourceFamily':family, + 'status':'optimal' if result.status==0 else 'feasible_time_limit','executable':True, + 'generatedAt':utc(at).isoformat(),'validFrom':points[0]['time'],'validUntil':points[-1]['validUntil'], + 'intervalMinutes':5,'points':points,'solverGap':float(result.mip_gap) if getattr(result,'mip_gap',None) is not None else None, + 'measuredPeaksKw':{m:observed_peaks[m] for m in months if contexts[m]['quality']=='verified'}, + 'peakBasisKw':{m:observed_peaks[m] for m in months},'peakBasis':contexts, + 'peakCostIsEstimate':estimated,'planningPeakCostChf':planning_peak, + 'restMonthAdjustmentChf':planning_peak-full_peak, + 'peakScenarioOutlook':{m:o.as_dict() for m,o in outlooks.items()}, + 'plannedPeaksKw':{m:running_peaks[m] for m in months}, + 'additionalPeakCostChf':cumulative_cash-cumulative_energy,'energyCostChf':cumulative_energy,'cashCostChf':cumulative_cash, + 'baselineEnergyCostChf':cumulative_baseline_energy,'baselineCashCostChf':cumulative_baseline_cash, + 'baselineAdditionalPeakCostChf':sum(baseline_peak_deltas.values()),'baselinePeaksKw':baseline_peaks, + 'throughputCostChf':cumulative_throughput,'terminalValueChf':end_value, + 'objectiveChf':cumulative_energy+planning_peak+cumulative_throughput-end_value, + 'baselinePlanningPeakCostChf':planning_cost(baseline_peaks), + 'cashCostMeaning':'Projected horizon energy plus full incremental monthly tariff, relative to the labelled peak basis; not an invoice', + 'terminalMinSocPercent':{b.asset_id:handles[b.asset_id]['terminal_min_percent'] for b in batteries}, + 'peakOutlook':'rest_month_scenarios' if outlooks else 'full_incremental_tariff'} diff --git a/services/netplan-v4/netplan_v4/peak_policy.py b/services/netplan-v4/netplan_v4/peak_policy.py new file mode 100644 index 0000000..30dad71 --- /dev/null +++ b/services/netplan-v4/netplan_v4/peak_policy.py @@ -0,0 +1,173 @@ +"""Economic peak policy. Planning assumptions NEVER become metering facts. + +Monthly marginal cost is convex. A rest-month scenario is a peak expected after +this horizon under a comparable control policy, not a configured limit, free +allowance, or a promise of savings. With insufficient evidence use full tariff. +""" +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime, timedelta +from math import isclose +from .domain import number, utc, month_key, ZURICH + +VERIFIED_SOURCES = frozenset({'meter_month_register','verified_month_history','verified_new_month'}) +ESTIMATE_SOURCES = frozenset({'power_history_estimate','sampled_power_estimate','counter_interval_estimate','operator_estimate','new_month'}) + + +def valid_month(value): + if not isinstance(value,str) or len(value)!=7: + raise ValueError('Calendar month YYYY-MM required') + if datetime.strptime(value,'%Y-%m').strftime('%Y-%m') != value: + raise ValueError('Invalid calendar month') + return value + + +def basis_record(value, month, at, allow_estimates=False): + """Strict provenance; missing stays missing. Caller decides explicit opt-in.""" + valid_month(month) + if not isinstance(value,dict) or set(value)-{'kw','quality','source','observedAt','notes','coverage','meterId'}: + raise ValueError('Invalid peak-basis schema') + kw=number(value.get('kw'),'peak basis kW',0,1e6) + quality=value.get('quality') + source=value.get('source') + if quality=='verified': + if source not in VERIFIED_SOURCES:raise ValueError('Not a verified peak source') + elif quality=='estimated': + if not allow_estimates or source not in ESTIMATE_SOURCES: + raise ValueError('Estimated planning basis not permitted or source invalid') + elif quality=='new_month': + if source!='new_month' or kw!=0 or month<=month_key(at): + raise ValueError('Zero future-month state is not a past measured peak') + else:raise ValueError('Peak quality must be explicit') + stamp=utc(value.get('observedAt')) + if stamp>utc(at):raise ValueError('Peak evidence from the future') + if quality!='new_month' and month>month_key(at): + raise ValueError('Future peak is an outlook, not historical evidence') + notes=value.get('notes','') + if not isinstance(notes,str) or len(notes)>500:raise ValueError('Invalid peak notes') + coverage=value.get('coverage') + if coverage is not None:number(coverage,'peak coverage',0,1) + mid=value.get('meterId') + if mid is not None and (not isinstance(mid,str) or not 1<=len(mid)<=160):raise ValueError('Invalid meter identity') + return {**value,'kw':kw,'quality':quality,'source':source,'observedAt':stamp.isoformat()} + + +@dataclass(frozen=True) +class PeakScenario: + future_peak_kw: float + probability: float + + +@dataclass(frozen=True) +class RestMonthOutlook: + month: str + scenarios: tuple[PeakScenario, ...] + issued_at: datetime + future_from: datetime + history_until: datetime + control_policy_id: str + method: str + evidence_id: str + reliance: float = 0.5 + + def validate(self, at, horizon_end): + valid_month(self.month) + at=utc(at);end=utc(horizon_end) + if utc(self.issued_at)>at or utc(self.history_until)>utc(self.issued_at): + raise ValueError('Rest-month outlook contains future information') + if at-utc(self.issued_at)>timedelta(days=2):raise ValueError('Stale rest-month outlook') + if utc(self.future_from)=end:return None + good={} + for r in daily_records: + if r.get('controlPolicyId')!=control_policy_id or r.get('complete') is not True:continue + if r.get('quality') not in ('verified','estimated'):continue + day=datetime.strptime(r['day'],'%Y-%m-%d').replace(tzinfo=ZURICH) + finished=utc(day+timedelta(days=1)) + observed=utc(r['observedAt']) + if finished>utc(at) or observed>utc(at) or observedtimedelta(days=90):continue + kw=number(r['peakKw'],'historical daily peak',0,1e6) + if r['day'] in good and good[r['day']]!=kw:raise ValueError('Conflicting daily peak evidence') + good[r['day']]=kw + if len(good)len(longest):longest=segment + segment=[] + segment.append(day) + if len(segment)>len(longest):longest=segment + if len(longest)=utc(self.last_decision) and self.terminal_normalized and self.constraint_breaches==0) + +def choose_family(setting,current,scores,*,registry=None,lookback_days=14,minimum_days=7,minimum_coverage=.9,margin_chf=1.,now): + registry=registry or default_registry() + if setting!='auto': + registry.get(setting) + return {'family':setting,'mode':'configured','reason':'Explicit installation setting'} + registry.get(current);number(margin_chf,'margin',0);valid=[] + for score in scores: + registry.get(score.family) + if (score.valid() and score.coverage>=minimum_coverage and score.days>=minimum_days + and utc(score.first_decision)>=utc(now)-timedelta(days=lookback_days) + and utc(score.last_decision)<=utc(now) and utc(score.actual_available_at)<=utc(now)): + valid.append(score) + groups={} + for score in valid: + key=(score.comparison_key,utc(score.first_decision),utc(score.last_decision),score.days) + groups.setdefault(key,{})[score.family]=score + complete=[g for g in groups.values() if len(g)==len(registry.entries())] + if not complete:return {'family':current,'mode':'collecting','reason':'Insufficient comparable out-of-sample replay evidence'} + group=max(complete,key=lambda g:utc(next(iter(g.values())).last_decision)) + best=min(group,key=lambda f:(group[f].cost_chf,f));improvement=group[current].cost_chf-group[best].cost_chf + chosen=best if improvement>margin_chf else current + return {'family':chosen,'mode':'economic_replay','improvementChf':improvement,'reason':'Matched historical cost replay; switching margin applied','costByFamilyChf':{k:v.cost_chf for k,v in group.items()}} + +def training_due(last_trained_at,now,cadence='daily'): + if cadence not in ('daily','weekly'):raise ValueError('Unknown training cadence') + if last_trained_at is None:return True + now,last=utc(now).astimezone(ZURICH),utc(last_trained_at).astimezone(ZURICH) + return (now.date()-last.date()).days >= (1 if cadence=='daily' else 7) + +def promote_candidate(*,active_cost,candidate_cost,valid_coverage,no_data_leakage,constraints_passed): + number(active_cost,'active cost');number(candidate_cost,'candidate cost') + return bool(valid_coverage and no_data_leakage and constraints_passed and candidate_cost20 or len({b.asset_id for b in result})!=len(result):raise ValueError('Duplicate/too many batteries') + return result + +def validate(kind,value,now,registry): + if kind not in KINDS or type(value.get('version')) is not int or value['version']!=1:raise ValueError('Unsupported event version/kind') + identifier=value['eventId'] + if not isinstance(identifier,str) or not 1<=len(identifier)<=160:raise ValueError('Event ID required') + observed=utc(value['observedAt']) + if observed>now+timedelta(seconds=30):raise ValueError('Observation from future') + fields={'version','eventId','observedAt'} + if kind=='forecast': + fields|={'families','modelVersions'} + if not value['families']:raise ValueError('No forecast families') + for key,family in value['families'].items(): + registry.get(key) + if family['loadBasis'] not in ('base_load','house_total'):raise ValueError('Explicit metering basis required') + evidence=family.get('accountingEvidenceId') + if evidence is not None and (family['loadBasis']!='base_load' or not isinstance(evidence,str) or not 8<=len(evidence)<=160):raise ValueError('Explicit base-load accounting evidence required') + if family.get('trainedUntil') and utc(family['trainedUntil'])>observed:raise ValueError('Training leakage') + if not 1<=len(family['points'])<=576:raise ValueError('Need 1..576 forecast intervals') + previous=None + for p in family['points']: + t=utc(p['time']) + if t.second or t.microsecond or t.minute%5:raise ValueError('Forecast interval alignment') + if previous and t-previous!=timedelta(minutes=5):raise ValueError('Forecast gap/overlap') + previous=t;number(p['pvW'],'PV',0,1e9);number(p['loadW'],'load',0,1e9);number(p.get('externalW',0.),'external',-1e9,1e9) + elif kind=='operation': + fields|={'gridW','meteringBoundary','batteries','limits','measuredPeaks','quarterPast','planningPeaks','quarterEstimate','meterObservation'} + if value['meteringBoundary']!='common_pcc':raise ValueError('Common metering boundary required') + number(value['gridW'],'grid W',-1e9,1e9) + for b in batteries(value['batteries']):b.validate(observed) + limits=value['limits'] + if set(limits)!={'importW','exportW','managerMonthLimitsW'}:raise ValueError('Explicit limits required; null unlimited, zero zero') + for k in ('importW','exportW'): + if limits[k] is not None:number(limits[k],k,0,1e9) + for k,v in limits['managerMonthLimitsW'].items(): + if str(int(k))!=k or not 1<=int(k)<=12:raise ValueError('Invalid manager month') + number(v,'manager limit',0,1e9) + for m,p in value.get('measuredPeaks',{}).items(): + datetime.strptime(m,'%Y-%m');number(p['kw'],'peak kW',0) + if m>month_key(observed) or p['source'] not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Measured peak source invalid; cap is not paid peak') + past=value.get('quarterPast') + if past: + q=quarter_start(observed) + if utc(past['start'])!=q or type(past['measuredSeconds']) is not int or past['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Quarter measurement timestamp mismatch') + number(past['importKwh'],'quarter energy',0) + for m,p in value.get('planningPeaks',{}).items(): + record=basis_record(p,m,observed,allow_estimates=True) + if record['quality']!='estimated':raise ValueError('planningPeaks contains estimates only') + estimate=value.get('quarterEstimate') + if estimate: + if set(estimate)-{'start','measuredSeconds','importKwh','quality','source','coverage','notes'}:raise ValueError('Unknown quarter estimate field') + if estimate.get('quality')!='estimated' or estimate.get('source') not in ('power_history_estimate','sampled_power_estimate','counter_interval_estimate'):raise ValueError('Explicit quarter estimate provenance required') + q=quarter_start(observed) + if utc(estimate['start'])!=q or type(estimate['measuredSeconds']) is not int or estimate['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Estimated quarter timing mismatch') + number(estimate['importKwh'],'estimated quarter energy',0) + if number(estimate.get('coverage'),'quarter estimate coverage',0,1)<1.:raise ValueError('Missing current-quarter coverage') + if value.get('meterObservation') is not None:meter_runtime.validate_observation(value['meterObservation'],observed) + elif kind=='planning_basis': + fields|={'peaks'} + if not isinstance(value.get('peaks'),dict) or not value['peaks']:raise ValueError('Explicit peak estimates required') + for m,p in value['peaks'].items(): + if basis_record(p,m,observed,allow_estimates=True)['quality']!='estimated':raise ValueError('Planning basis is not a metering import') + elif kind=='peak_outlook': + fields|={'outlooks'} + for m,v in value['outlooks'].items(): + outlook=RestMonthOutlook.from_dict(v) + if m!=outlook.month:raise ValueError('Outlook month mismatch') + outlook.validate(observed,observed) + elif kind=='tariffs': + fields|={'import','export','peakChfKwMonth'} + for side in ('import','export'): + p=value[side] + if p['mode'] not in ('static','dynamic') or not p['tariffId']:raise ValueError('Explicit price mode/id required') + if p['mode']=='static':number(p['staticChfKwh'],'static price') + peaks=value['peakChfKwMonth'] + if isinstance(peaks,dict): + for m,v in peaks.items():datetime.strptime(m,'%Y-%m');number(v,'peak tariff',0) + else:number(peaks,'peak tariff',0) + else: + fields|={'periods'} + if len(value['periods'])>3000:raise ValueError('Too many price intervals') + for p in value['periods']: + if p['unit'] not in ('CHF/kWh','CHF_kWh','Rp/kWh','CHF/MWh') or p['side'] not in ('import','export'):raise ValueError('Explicit price unit/direction required') + if p['sourceKind'] not in ('published_interval','estimate','carried_forward'):raise ValueError('Explicit price provenance required') + number(p['value'],'price') + if utc(p['end'])<=utc(p['start']) or utc(p['observedAt'])>observed:raise ValueError('Invalid price interval or observation') + if p.get('publishedAt') and utc(p['publishedAt'])>utc(p['observedAt']):raise ValueError('Price not published when observed') + if set(value)-fields:raise ValueError('Unknown fields: extra device data/credentials must not be submitted') + canonical(value) + +def ingest(store,plant,kind,value,now): + validate(kind,value,now,store.registry);data=canonical(value);con=store.con;con.execute('BEGIN IMMEDIATE') + try: + old=con.execute('SELECT value FROM planner_inputs WHERE plant=? AND kind=? AND event_id=?',(plant,kind,value['eventId'])).fetchone() + if old: + if old[0]!=data:raise ValueError('Immutable event conflict') + con.commit();return {'status':'duplicate','queued':False} + current=latest(store,plant,kind) + if current and utc(current['observedAt'])==utc(value['observedAt']): + left,right=dict(current),dict(value);left.pop('eventId');right.pop('eventId') + if canonical(left)!=canonical(right):raise ValueError('Conflicting simultaneous observations') + con.execute('INSERT INTO planner_inputs VALUES(?,?,?,?,?)',(plant,kind,value['eventId'],utc(value['observedAt']).isoformat(timespec='microseconds'),data)) + newer=not current or utc(current['observedAt'])<=utc(value['observedAt']) + if newer: + if kind=='operation': + known=store.peaks(plant) + for m,p in value.get('measuredPeaks',{}).items(): + if m in known and p['kw']=end and utc(p['observedAt'])<=now): + factor={'CHF/kWh':1.,'CHF_kWh':1.,'Rp/kWh':.01,'CHF/MWh':.001}[p['unit']] + candidates.append((utc(p['observedAt']),p['value']*factor)) + if not candidates:return None + last=max(t for t,v in candidates);values={v for t,v in candidates if t==last} + if len(values)!=1:raise ValueError('Conflicting published price intervals') + return Price(values.pop(),last,'dynamic') + +class AwaitingInput(ValueError):pass + +def assemble(store,plant,family,now): + values={k:latest(store,plant,k) for k in ('operation','forecast','tariffs')} + for k,v in values.items(): + if not v:raise AwaitingInput('Missing '+k+' input') + op,forecast,tariffs=(values[k] for k in ('operation','forecast','tariffs')) + decision=utc(op['observedAt']).replace(microsecond=0) + settings=store.settings(plant);allow_estimates=settings['measurementPolicy']=='allow_estimates' + input_quality={'quarter':'verified','warnings':[]} + if not 0<=(now-decision).total_seconds()<=120:raise AwaitingInput('Fresh manager observation required (120s maximum)') + if (now-utc(forecast['observedAt'])).total_seconds()>5400:raise AwaitingInput('Forecast older than 90 minutes') + if utc(forecast['observedAt'])>decision or utc(tariffs['observedAt'])>decision:raise AwaitingInput('New data awaiting fresh manager observation') + if settings['forecastSource']=='corrected_profile': + try:forecast=measurement_pipeline.apply_load_forecast(store.con,plant,settings['measurementDataset'],forecast,int(decision.timestamp())) + except ValueError as exc:raise AwaitingInput(str(exc)) from exc + if family not in forecast['families']:raise AwaitingInput('Chosen family is unavailable; no silent switch') + source=forecast['families'][family];steps=[] + future_source={**source,'points':[p for p in source['points'] if utc(p['time'])+timedelta(minutes=5)>decision]} + assessment=assess_family(future_source) + if not assessment['valid']:raise AwaitingInput(assessment['reason']) + input_quality['forecastAssessment']=assessment + if source.get('dataPipeline'): + input_quality['dataPipeline']=source['dataPipeline'] + input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule') + for p in source['points']: + start=utc(p['time']);end=start+timedelta(minutes=5) + if end<=decision:continue + start=max(start,decision) + external=p.get('externalW',0.) if source['loadBasis']=='base_load' else 0. + steps.append(Step(start,p['loadW'],p['pvW'],price_at(store,plant,'import',tariffs['import'],start,end,decision),price_at(store,plant,'export',tariffs['export'],start,end,decision),external,int((end-start).total_seconds()))) + if not steps or steps[0].start!=decision:raise AwaitingInput('Forecast has no current interval') + full_end=steps[-1].end;steps=priced_prefix(steps,decision) + if not steps:raise AwaitingInput('No complete published-price billing quarter') + q=quarter_start(decision);elapsed=int((decision-q).total_seconds());past={} + if elapsed: + p=op.get('quarterPast') + if not p and allow_estimates: + p=op.get('quarterEstimate') + if not p and op.get('meterObservation'):p=meter_runtime.current_quarter(store.con,plant,op['meterObservation'],decision) + if p: + input_quality['quarter']='estimated' + input_quality['warnings'].append('Current quarter uses an explicitly estimated energy value, not a billing measurement') + if not p or utc(p['start'])!=q or p['measuredSeconds']!=elapsed:raise AwaitingInput('Current-quarter energy missing (measured or explicitly permitted estimate)') + past[q]=QuarterPast(p['importKwh'],elapsed) + peaks=store.peaks(plant);months={month_key(s.start) for s in steps};contexts={} + estimates=meter_runtime.assumptions(store.con,plant) + for m in months: + if m>month_key(decision): + peaks[m]=0.;contexts[m]={'kw':0.,'quality':'new_month','source':'new_month','observedAt':decision.isoformat()} + elif m in peaks: + r=store.con.execute('SELECT source,updated_at FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone() + known_at=op['observedAt'] if m in op.get('measuredPeaks',{}) else r['updated_at'] + contexts[m]={'kw':peaks[m],'quality':'verified','source':r['source'],'observedAt':known_at} + # A later acquired larger quarter may raise an older verified baseline, + # but the resulting combined planning basis must then say estimated. + if allow_estimates and m in estimates and estimates[m]['kw']>peaks[m] and utc(estimates[m]['observedAt'])>utc(known_at): + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True) + peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('A newer sampled quarter increased the earlier verified peak baseline; current planning maximum is estimated') + elif allow_estimates and m in estimates: + contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True);peaks[m]=contexts[m]['kw'] + input_quality['warnings'].append('Monthly peak '+m+' is a planning estimate, not an authoritative billing maximum') + else:raise AwaitingInput('Peak basis missing; supply a verified maximum or explicitly permit a labelled estimate') + peak_prices=tariffs['peakChfKwMonth'] + if not isinstance(peak_prices,dict):peak_prices={m:peak_prices for m in months} + lim=op['limits'];limits=Limits(lim['exportW'],lim['importW'],{int(k):v for k,v in lim['managerMonthLimitsW'].items()}) + outlooks={};horizon_end=steps[-1].end + if settings['peakOutlookPolicy']=='empirical_if_available': + explicit=latest(store,plant,'peak_outlook') + for m in months: + if explicit and m in explicit['outlooks']: + candidate=RestMonthOutlook.from_dict(explicit['outlooks'][m]) + try: + candidate.validate(decision,horizon_end) + policy=op.get('meterObservation',{}).get('controlPolicyId') + if not policy or candidate.control_policy_id!=policy:raise ValueError('Different or unknown control policy') + except ValueError:input_quality['warnings'].append('Stale, overlapping or incomparable rest-month outlook ignored; full incremental tariff used') + else:outlooks[m]=replace(candidate,reliance=min(candidate.reliance,settings['peakOutlookReliance'])) + elif op.get('meterObservation'): + observation=op['meterObservation'] + candidate=empirical_rest_month(meter_runtime.daily_peaks(store.con,plant,observation['meterId'],decision), + month=m,at=decision,horizon_end=horizon_end,control_policy_id=observation['controlPolicyId'], + reliance=settings['peakOutlookReliance']) + if candidate is not None:outlooks[m]=candidate + if not outlooks:input_quality['warnings'].append('Insufficient comparable rest-month history; full incremental peak tariff used, no arbitrary free peak allowance') + data={'steps':steps,'batteries':batteries(op['batteries']),'limits':limits,'observed_peaks':peaks,'peak_prices':peak_prices, + 'quarter_history':past,'at':decision,'peak_context':contexts,'peak_outlooks':outlooks} + if source['loadBasis']=='base_load' and source.get('accountingEvidenceId'): + input_quality['accountingEvidenceId']=source['accountingEvidenceId'] + return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)} + +def run_once(store,now): + stamp=int(now.timestamp())//300 + plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')] + for plant in plants: + config=store.settings(plant) + datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))] + for dataset in datasets: + try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp())) + except ValueError as exc:logging.getLogger(__name__).warning('Data pipeline unavailable for configured dataset: %s',type(exc).__name__) + with store.con: + old=store.con.execute('SELECT tick FROM planner_ticks WHERE plant=?',(plant,)).fetchone() + if not old or old[0]!=stamp: + store._request(plant,store.settings(plant)['revision'],'five_minute_tick',now) + store.con.execute('INSERT INTO planner_ticks VALUES(?,?) ON CONFLICT(plant) DO UPDATE SET tick=excluded.tick',(plant,stamp)) + claim=store.claim(now) + if not claim:return {'status':'idle'} + plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]} + try: + settings=store.settings(plant);previous=store.current(plant) + current=previous['sourceFamily'] if previous else store.registry.entries()[0].key + # Productive replay ingestion is intentionally not fabricated from R2 metrics. + selection=choose_family(settings['family'],current,(),registry=store.registry,now=now) + data,full_end,quality=assemble(store,plant,selection['family'],now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + result=optimize(**data,config_revision=settings['revision'],family=selection['family']) + if result['executable']: + result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])}) + store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token']) + return result + except (ValueError,TypeError,KeyError) as exc: + result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]} + return result + finally: + detail={k:v for k,v in result.items() if k in ('status','reason','planId','configRevision','sourceFamily')} + with store.con:store.con.execute('INSERT INTO planner_run_status VALUES(?,?,?,?) ON CONFLICT(plant) DO UPDATE SET updated_at=excluded.updated_at,status=excluded.status,detail=excluded.detail',(plant,now.isoformat(),result['status'],canonical(detail))) + store.finish(claim) + +def status(store,plant,now): + plan=store.current(plant);settings=store.settings(plant) + row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone() + pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone() + ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone() + fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit')) + return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)} + +def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()): + allowed={str(UUID(p)) for p in plants} + trial_allowed={str(UUID(p)) for p in controlled_trial_plants} + if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist') + if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required') + path=Path(db_path).resolve() + if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required') + path.parent.mkdir(parents=True,exist_ok=True);stop=Event() + def factory():return PlannerStore(str(path)) + def loop(): + while not stop.is_set(): + s=factory() + try:run_once(s,datetime.now(timezone.utc).replace(microsecond=0)) + except Exception as exc:logging.getLogger(__name__).error('V4 worker error: %s',type(exc).__name__) + finally:s.close() + stop.wait(1.) + @asynccontextmanager + async def lifespan(app): + thread=Thread(target=loop,name='v4-shadow',daemon=True) + if start_worker:thread.start() + yield + stop.set() + if start_worker:thread.join(35) + app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan) + app.state.store_factory=factory + def authorize(plant,token): + if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized') + try:plant=str(UUID(plant)) + except ValueError:raise HTTPException(400,'Invalid installation ID') + if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial') + return factory() + @app.get('/health') + def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1} + @app.get('/internal/v2/prognosis/{plant}/planner') + def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc);view=status(s,plant,now) + view['controlledTrial']=controlled_trial.authority(s,plant,view,now,trial_allowed) + view['controlledTrialAuthorized']=view['controlledTrial'] is not None + return view + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/arm') + def arm_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + return controlled_trial.arm(s,plant,payload,status(s,plant,now),now,trial_allowed) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409,str(exc)[:300]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/trial/revoke') + def revoke_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if set(payload)!={'sessionId'}:raise ValueError('Session ID only') + return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/settings') + def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return s.save_settings(plant,payload['changes'],payload['expectedRevision'],datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409 if 'Revision conflict' in str(exc) else 400,str(exc)) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/replan') + def replan(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.request(plant,'manual',datetime.now(timezone.utc));return {'status':'queued','liveEnabled':False} + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/inputs/{kind}') + def input_event(plant:str,kind:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:return ingest(s,plant,kind,payload,datetime.now(timezone.utc)) + except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}') + def configure_dataset(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + if payload.get('datasetId')!=dataset:raise ValueError('Dataset path/payload mismatch') + return measurement_pipeline.register_dataset(s.con,plant,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/measurements') + def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + now=datetime.now(timezone.utc) + result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp())) + # Existing five-minute worker handles rollup/training; no per-record optimizer flood. + return result + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200]) + finally:s.close() + @app.post('/internal/v2/prognosis/{plant}/planner/ack') + def ack(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try:s.acknowledge(plant,payload['planId'],payload['revision'],datetime.now(timezone.utc),payload.get('step'),payload.get('status','shadow_seen'));return {'status':'recorded'} + except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) + finally:s.close() + from starlette.responses import JSONResponse + class BodyLimit: + def __init__(self,app):self.app=app + async def __call__(self,scope,receive,send): + if scope['type']!='http' or scope['method'] not in ('POST','PUT'):return await self.app(scope,receive,send) + chunks=[];total=0 + while True: + msg=await receive() + if msg['type']=='http.disconnect':return + total+=len(msg.get('body',b'')) + if total>2000000:return await JSONResponse({'detail':'Request too large'},status_code=413)(scope,receive,send) + chunks.append(msg) + if not msg.get('more_body',False):break + async def replay():return chunks.pop(0) if chunks else await receive() + return await self.app(scope,replay,send) + app.add_middleware(BodyLimit) + return app + +def from_environment(): + return create_app(os.environ.get('NETPLAN_V4_DB','/data/netplan-v4.sqlite'),os.environ.get('PROGNOSIS_SERVICE_TOKEN',''),[p.strip() for p in os.environ.get('NETPLAN_V4_PLANTS','').split(',') if p.strip()], controlled_trial_plants=[p.strip() for p in os.environ.get('NETPLAN_V4_CONTROL_TRIAL_PLANTS','').split(',') if p.strip()]) diff --git a/services/netplan-v4/netplan_v4/store.py b/services/netplan-v4/netplan_v4/store.py new file mode 100644 index 0000000..07d5254 --- /dev/null +++ b/services/netplan-v4/netplan_v4/store.py @@ -0,0 +1,147 @@ +from __future__ import annotations +import json +import sqlite3 +from datetime import datetime,timedelta +from uuid import uuid4 +from .domain import default_registry,month_key,number,quarter_start,utc +from . import meter_runtime, controlled_trial, measurement_pipeline + +def canonical(value): + return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False) + +DEFAULT_SETTINGS={'family':'3','autoLookbackDays':14,'autoMinimumDays':7,'autoMinimumCoverage':.9,'autoSwitchMarginChf':1.,'tariffPolicy':'published_only','trainingCadence':'daily','trainingPromotion':'validated_only','runMode':'shadow','roundtripEfficiency':.90,'measurementPolicy':'verified_only','peakOutlookPolicy':'empirical_if_available','peakOutlookReliance':.5,'forecastSource':'legacy','measurementDataset':''} + +class PlannerStore: + """Own SQLite file, no mutation of legacy application databases.""" + def __init__(self,path,registry=None): + self.registry=registry or default_registry();self.con=sqlite3.connect(path,timeout=10) + self.con.row_factory=sqlite3.Row;self.con.execute('PRAGMA foreign_keys=ON') + self.con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_settings(plant TEXT PRIMARY KEY,revision INTEGER NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_audit(id INTEGER PRIMARY KEY,plant TEXT NOT NULL,at TEXT NOT NULL,kind TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_work(plant TEXT PRIMARY KEY,sequence INTEGER NOT NULL,revision INTEGER NOT NULL,reasons TEXT NOT NULL,requested_at TEXT NOT NULL,lease_until TEXT,lease_token TEXT); + CREATE TABLE IF NOT EXISTS planner_measurements(plant TEXT NOT NULL,start TEXT NOT NULL,import_kwh REAL NOT NULL,PRIMARY KEY(plant,start)); + CREATE TABLE IF NOT EXISTS planner_month_peaks(plant TEXT NOT NULL,month TEXT NOT NULL,peak_kw REAL NOT NULL,source TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(plant,month)); + CREATE TABLE IF NOT EXISTS planner_snapshots(id TEXT PRIMARY KEY,plant TEXT NOT NULL,issued_at TEXT NOT NULL,value TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_plans(plan_id TEXT PRIMARY KEY,plant TEXT NOT NULL,revision INTEGER NOT NULL,mode TEXT NOT NULL,value TEXT NOT NULL,created_at TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_current(plant TEXT NOT NULL,mode TEXT NOT NULL,plan_id TEXT NOT NULL REFERENCES planner_plans(plan_id),PRIMARY KEY(plant,mode)); + CREATE TABLE IF NOT EXISTS planner_ack(plant TEXT PRIMARY KEY,plan_id TEXT NOT NULL,revision INTEGER NOT NULL,received_at TEXT NOT NULL,applied_step TEXT,status TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_inputs(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,observed_at TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(plant,kind,event_id)); + CREATE TABLE IF NOT EXISTS planner_input_current(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,PRIMARY KEY(plant,kind)); + CREATE TABLE IF NOT EXISTS planner_run_status(plant TEXT PRIMARY KEY,updated_at TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_ticks(plant TEXT PRIMARY KEY,tick INTEGER NOT NULL); + ''') + meter_runtime.schema(self.con) + controlled_trial.schema(self.con) + measurement_pipeline.schema(self.con) + def close(self):self.con.close() + def settings(self,plant): + row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone() + return {'revision':row['revision'],**DEFAULT_SETTINGS,**json.loads(row['value'])} if row else {'revision':0,**DEFAULT_SETTINGS} + def _validate_settings(self,value): + if set(value)!=set(DEFAULT_SETTINGS):raise ValueError('Unknown or missing setting') + if value['family']!='auto':self.registry.get(value['family']) + for key in ('autoLookbackDays','autoMinimumDays'): + if type(value[key]) is not int:raise ValueError('Days must be integers') + number(value['autoLookbackDays'],'lookback',7,90);number(value['autoMinimumDays'],'minimum days',1,value['autoLookbackDays']) + number(value['autoMinimumCoverage'],'coverage',.5,1);number(value['autoSwitchMarginChf'],'margin',0) + if value['tariffPolicy']!='published_only':raise ValueError('Only published-price policy implemented') + if value['trainingCadence'] not in ('daily','weekly') or value['trainingPromotion']!='validated_only':raise ValueError('Training must use validated promotion') + if value['runMode']!='shadow':raise ValueError('Shadow-only: live release requires separate validation') + number(value['roundtripEfficiency'],'roundtrip efficiency',.01,1) + if value['measurementPolicy'] not in ('verified_only','allow_estimates'):raise ValueError('Invalid measurement policy') + if value['peakOutlookPolicy'] not in ('full_incremental','empirical_if_available'):raise ValueError('Invalid peak outlook policy') + number(value['peakOutlookReliance'],'peak outlook reliance',0,1) + if value['forecastSource'] not in ('legacy','corrected_profile'):raise ValueError('Unknown forecast source') + if not isinstance(value['measurementDataset'],str) or len(value['measurementDataset'])>80:raise ValueError('Invalid measurement dataset') + if value['forecastSource']=='corrected_profile' and not value['measurementDataset']:raise ValueError('Corrected forecast requires an explicit dataset') + def _request(self,plant,revision,reason,now): + row=self.con.execute('SELECT * FROM planner_work WHERE plant=?',(plant,)).fetchone() + reasons=set(json.loads(row['reasons'])) if row else set();reasons.add(reason) + seq=row['sequence']+1 if row else 1 + self.con.execute('''INSERT INTO planner_work(plant,sequence,revision,reasons,requested_at) VALUES(?,?,?,?,?) + ON CONFLICT(plant) DO UPDATE SET sequence=excluded.sequence,revision=excluded.revision,reasons=excluded.reasons,requested_at=excluded.requested_at''',(plant,seq,revision,canonical(sorted(reasons)),utc(now).isoformat())) + def save_settings(self,plant,changes,expected_revision,now): + if type(expected_revision) is not int or expected_revision<0:raise ValueError('Invalid expected revision') + self.con.execute('BEGIN IMMEDIATE') + try: + current=self.settings(plant) + if current.pop('revision')!=expected_revision:raise ValueError('Revision conflict; reload before saving') + current.update(changes);self._validate_settings(current);revision=expected_revision+1 + self.con.execute('''INSERT INTO planner_settings VALUES(?,?,?) ON CONFLICT(plant) DO UPDATE SET revision=excluded.revision,value=excluded.value''',(plant,revision,canonical(current))) + self._request(plant,revision,'configuration_changed',now) + self.con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',(plant,utc(now).isoformat(),'settings',canonical({'revision':revision,'changes':changes}))) + self.con.commit();return {'revision':revision,**current} + except Exception:self.con.rollback();raise + def request(self,plant,reason,now): + if reason not in ('prices_changed','telemetry_changed','five_minute_tick','manual','model_promoted','forecast_changed','operation_changed','tariffs_changed'):raise ValueError('Unknown trigger') + with self.con:self._request(plant,self.settings(plant)['revision'],reason,now) + def claim(self,now,lease_seconds=120): + at=utc(now);self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT * FROM planner_work WHERE lease_until IS NULL OR lease_until < ? ORDER BY requested_at LIMIT 1',(at.isoformat(),)).fetchone() + if not row:self.con.commit();return None + token=str(uuid4());self.con.execute('UPDATE planner_work SET lease_until=?,lease_token=? WHERE plant=?',((at+timedelta(seconds=lease_seconds)).isoformat(),token,row['plant'])) + self.con.commit();return {**dict(row),'lease_token':token} + except Exception:self.con.rollback();raise + def finish(self,claim): + self.con.execute('BEGIN IMMEDIATE') + try: + row=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(claim['plant'],)).fetchone() + if not row or row['lease_token']!=claim['lease_token']:self.con.commit();return False + if row['sequence']==claim['sequence']:self.con.execute('DELETE FROM planner_work WHERE plant=?',(claim['plant'],)) + else:self.con.execute('UPDATE planner_work SET lease_until=NULL,lease_token=NULL WHERE plant=?',(claim['plant'],)) + self.con.commit();return True + except Exception:self.con.rollback();raise + def initialize_peak(self,plant,month,peak_kw,source,now): + number(peak_kw,'authoritative measured peak',0) + if source not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Configured cap is NOT measured peak') + datetime.strptime(month,'%Y-%m') + if month>month_key(now):raise ValueError('Future month cannot have a measured peak') + with self.con: + old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,month)).fetchone() + if old and peak_kwreceived_at:raise ValueError('Completed aligned intervals required') + with self.con: + old=self.con.execute('SELECT import_kwh FROM planner_measurements WHERE plant=? AND start=?',(plant,start.isoformat())).fetchone() + if old and abs(old[0]-import_kwh)>1e-9:raise ValueError('Conflicting metering fact') + self.con.execute('INSERT OR IGNORE INTO planner_measurements VALUES(?,?,?)',(plant,start.isoformat(),import_kwh)) + q=quarter_start(start);rows=self.con.execute('SELECT start,import_kwh FROM planner_measurements WHERE plant=? AND start>=? AND start8000000:raise RuntimeError('Probe execution failed or oversized') + probe=json.loads(result.stdout) + save(reports/'native-inputs.json',probe) + report['nativeInputStatus']=probe.get('status') + report['forecastSummary']=probe.get('forecastSummary',{}) + report['dataBlockers']=[] + for key,value in report['forecastSummary'].items(): + if key in ('prog_var_2','prog_var_11','prog_var_22') and (not value.get('points') or value.get('allZero')): + report['dataBlockers'].append(key+': missing or unconfirmed all-zero load forecast') + recent=probe.get('inputFrames',{}).get('df_recent_raw',{}).get('columns',{}).get('Hausverbrauch',{}).get('lastFiniteAt') + if not recent or (datetime.now(timezone.utc)-datetime.fromisoformat(recent)).total_seconds()>1800: + report['dataBlockers'].append('Recent raw load telemetry missing or older than 30 minutes') + if probe.get('status')!='read_only_acquired' or report['dataBlockers']:success=False + except (ValueError,RuntimeError,subprocess.TimeoutExpired) as error: + report['nativeInputStatus']='failed';report['nativeInputErrorType']=type(error).__name__;success=False + try: + verify_native_bundle() + report['sourceStableDuringTests']=(source_manifest()==report['sourceHashes'] and verify_forecast_bundle(ROOT/'acceptance/forecast-src',PROJECT/'forecast_engine')==forecast_hashes) + except (OSError,ValueError): + report['sourceStableDuringTests']=False + if not report['sourceStableDuringTests']: + report.setdefault('dataBlockers',[]).append('Source changed during checks; results cannot certify current candidate') + success=False + report['preflightChecksPassed']=success + save(reports/'REPORT.json',report) + save(ROOT/'acceptance-reports'/'LATEST.json',{'report':str(reports/'REPORT.json')}) + print('\nPRECHECK '+('PASSED' if success else 'NEEDS REVIEW')+'; NOT a production release.') + print('REPORT: '+str(reports/'REPORT.json')) + for key,value in report.get('forecastSummary',{}).items():print(key,json.dumps(value)) + for blocker in report.get('dataBlockers',[]):print('DATA BLOCKER:',blocker) + print('Existing containers, Symcon settings, timers and actuators unchanged.') + return 0 if success else 1 + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plant',required=True,type=lambda v:str(UUID(v))) + parser.add_argument('--show-commands',action='store_true') + args=parser.parse_args() + if args.show_commands: + for name,command,cwd,timeout in command_list(args.plant):print(name,json.dumps(command)) + print('Native input probe: read-only docker compose exec forecast-engine python with reviewed stdin script') + else: + if os.geteuid()!=0:raise SystemExit('Run as root; do not alter Docker socket permissions.') + try:raise SystemExit(run(args.plant)) + except (OSError,ValueError) as error:raise SystemExit('Preflight stopped safely: '+str(error)) diff --git a/services/netplan-v4/requirements.txt b/services/netplan-v4/requirements.txt new file mode 100644 index 0000000..595f8e7 --- /dev/null +++ b/services/netplan-v4/requirements.txt @@ -0,0 +1,5 @@ +numpy>=2.2,<3 +scipy>=1.15,<1.18 +fastapi>=0.115,<0.129 +httpx>=0.27,<0.29 +uvicorn>=0.30,<0.41 diff --git a/services/netplan-v4/run_tests.py b/services/netplan-v4/run_tests.py new file mode 100644 index 0000000..4876825 --- /dev/null +++ b/services/netplan-v4/run_tests.py @@ -0,0 +1,21 @@ +"""Run against the exact server working tree; no live data or credentials used.""" +from pathlib import Path +import hashlib,json,os,sys,unittest +ROOT=Path(__file__).resolve().parent +DEPS=Path('/home/agent/services/qa/netplan-v4-candidate-20261001/python-dependencies') +if DEPS.exists():sys.path.insert(0,str(DEPS)) +sys.path.insert(0,str(ROOT)) +if __name__=='__main__': + import numpy,scipy,fastapi,httpx + versions={'python':sys.version.split()[0],**{m.__name__:m.__version__ for m in (numpy,scipy,fastapi,httpx)}} + print('Runtime',json.dumps(versions),flush=True) + reports=Path(os.getenv('NETPLAN_V4_TEST_REPORT_DIR',str(ROOT))) + reports.mkdir(parents=True,exist_ok=True) + suite=unittest.defaultTestLoader.discover(str(ROOT/'tests')) + with (reports/'TEST_RESULTS.txt').open('w') as output: + output.write('Actual server/target tests; synthetic data only.\n'+json.dumps(versions)+'\n\n') + result=unittest.TextTestRunner(stream=output,verbosity=2).run(suite) + print((reports/'TEST_RESULTS.txt').read_text()) + hashes={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(ROOT.rglob('*.py')) if '__pycache__' not in p.parts} + (reports/'TESTED_SOURCE_HASHES.json').write_text(json.dumps(hashes,indent=2)+'\n') + raise SystemExit(0 if result.wasSuccessful() else 1) diff --git a/services/netplan-v4/runtime_preflight.py b/services/netplan-v4/runtime_preflight.py new file mode 100644 index 0000000..8991424 --- /dev/null +++ b/services/netplan-v4/runtime_preflight.py @@ -0,0 +1,46 @@ +"""Check packaged code readability and writable temporary test output as non-root.""" +import os +from pathlib import Path +import tempfile + +SOURCE_DIRS = ("netplan_v4", "tests", "integrations", "gui") +SOURCE_FILES = ("run_tests.py", "install_hooks.py", "runtime_preflight.py", "requirements.txt") + + +def verify_access(root, reports): + root, reports = Path(root).resolve(), Path(reports).resolve() + if reports == root or root in reports.parents: + raise ValueError("Test reports must be outside the packaged application tree") + if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK): + raise PermissionError(f"Application directory is not accessible: {root}") + paths = [root / name for name in SOURCE_FILES] + for name in SOURCE_DIRS: + directory = root / name + if not directory.is_dir(): + raise FileNotFoundError(f"Missing packaged directory: {directory}") + def on_error(error): + raise error + for parent, directories, files in os.walk(directory, onerror=on_error): + base = Path(parent) + if not os.access(base, os.R_OK | os.X_OK): + raise PermissionError(f"Packaged directory is not accessible: {base}") + if any((base / child).is_symlink() for child in directories + files): + raise ValueError(f"Unexpected symlink in packaged source: {base}") + paths.extend(base / name for name in files) + for path in paths: + with path.open("rb") as handle: + handle.read(1) + reports.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryFile(dir=reports) as probe: + probe.write(b"permission check") + probe.flush() + return len(paths) + + +if __name__ == "__main__": + if os.geteuid() == 0: + raise SystemExit("Runtime preflight must run as the unprivileged container user") + reports = Path(os.getenv("NETPLAN_V4_TEST_REPORT_DIR", "/tmp/test-results")) + count = verify_access(Path(__file__).resolve().parent, reports) + import numpy, scipy, fastapi, httpx + print(f"Non-root runtime check OK: uid={os.geteuid()}, readable_files={count}, reports={reports}") diff --git a/services/netplan-v4/tests/manager_protocol.php b/services/netplan-v4/tests/manager_protocol.php new file mode 100644 index 0000000..9a56099 --- /dev/null +++ b/services/netplan-v4/tests/manager_protocol.php @@ -0,0 +1,24 @@ +'2026-10-01T12:00:00Z','validUntil'=>'2026-10-01T12:05:00Z','gridTargetW'=>0.,'intent'=>'pvCharge']; +$plan=['schemaVersion'=>2,'planId'=>'test','configRevision'=>7,'status'=>'optimal','executable'=>true, + 'runMode'=>'shadow','liveEnabled'=>false,'generatedAt'=>$point['time'],'validFrom'=>$point['time'],'validUntil'=>'2026-10-01T12:10:00Z', + 'points'=>[$point,['time'=>'2026-10-01T12:05:00Z','validUntil'=>'2026-10-01T12:10:00Z','gridTargetW'=>9000.,'intent'=>'gridCharge']]]; +$now=strtotime('2026-10-01T12:03:00Z'); +check(NetzfahrplanV4::currentPoint($plan,$now,7)===null,'Shadow never live'); +check(NetzfahrplanV4::currentPoint($plan,$now,7,true)['gridTargetW']===0.,'Current interval, not nearest future point'); +check(NetzfahrplanV4::currentPoint($plan,$now,8,true)===null,'Revision mismatch'); +check(NetzfahrplanV4::currentPoint($plan,$now+3600,7,true)===null,'Expired plan'); +$bad=$plan;$bad['points'][1]['time']=$point['time'];check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Overlapping intervals'); +$bad=$plan;$bad['points'][0]['gridTargetW']=NAN;check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'NaN rejected'); +$bad=$plan;$bad['generatedAt']='2026-10-01T12:00:00';check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Timezone required'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,39000.,0.,15000.);check($r['batteryTargetW']===5332.,'Surplus charges');check($r['gridTargetW']===0.,'Peak cap never replaces zero target');check($r['expectedGridW']===0.,'Expected grid balanced'); +$r=NetzfahrplanV4::correction($point,-5332.,0.,3000.,0.);check($r['batteryTargetW']===3000.&&$r['limited'],'Charging availability enforced'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],1000.,0.,39000.,0.);check($r['batteryTargetW']===0.,'No inadvertent grid charge from load forecast'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],7000.,0.,39000.,1000.,5000.);check($r['batteryTargetW']===-1000.&&$r['limited'],'Safety override still respects available discharge'); +$r=NetzfahrplanV4::correction(['gridTargetW'=>-40000.,'intent'=>'pvCharge'],-10000.,0.,39000.,10000.);check($r['batteryTargetW']===0.,'No unintended discharge in PV charge intent');check($r['trackingErrorW']===30000.,'Intent saturation reports unmet target'); +echo "Netzfahrplan V4: {$count} assertions passed. No IPS or device calls.\n"; diff --git a/services/netplan-v4/tests/portal.test.mjs b/services/netplan-v4/tests/portal.test.mjs new file mode 100644 index 0000000..05bb0fa --- /dev/null +++ b/services/netplan-v4/tests/portal.test.mjs @@ -0,0 +1,32 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createPlannerV4Bridge} from '../integrations/netplan-v4-bridge.mjs'; +const PLANT='00000000-0000-4000-8000-000000000001',INSTALL='e3a08f9e-af12-4695-99bd-8b51c0520021'; +function fixture(options={}){ + const sent=[],auth=[],upstream=[]; + const bridge=createPlannerV4Bridge({ + configuredPrognosisPlant:(req,res,id,csrf)=>{auth.push({id,csrf});return options.denied?null:{plant:{installation_id:INSTALL},license:{quantities:{grid_schedule:options.unlicensed?0:1}},session:{}};}, + roleAllowed:()=>!options.viewer,bodyJson:async()=>({expectedRevision:0,changes:{family:'23'}}), + json:(res,status,payload)=>sent.push({status,payload}),deviceActivation:()=>options.deviceDenied?null:{plant_id:PLANT}, + checkDeviceRate:()=>!options.ratelimited,ownedLicenseState:()=>({quantities:{grid_schedule:1}}),serviceToken:'synthetic-test-only', + fetchImpl:async(url,args)=>{upstream.push({url:String(url),args});if(options.down)throw new Error('secret-host-detail');return {ok:!options.conflict,status:options.conflict?409:200,json:async()=>options.conflict?{detail:'internal-field'}:{liveEnabled:false,plan:{planId:'p1'}}};} + });return {bridge,sent,auth,upstream}; +} +const url=suffix=>new URL(`https://portal.test/api/plants/${PLANT}/prognosis/planner-v4${suffix}`); +test('read scoped by customer plant, upstream uses installation id',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},url('')),true);assert.equal(f.auth[0].csrf,false);assert.match(f.upstream[0].url,new RegExp(INSTALL));assert.equal(f.sent[0].status,200);}); +test('save requires existing CSRF checks',async()=>{const f=fixture();await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.auth[0].csrf,true);assert.equal(f.upstream[0].args.method,'PUT');}); +test('viewer cannot mutate',async()=>{const f=fixture({viewer:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,403);assert.equal(f.upstream.length,0);}); +test('unowned plant blocked',async()=>{const f=fixture({denied:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.upstream.length,0);}); +test('license required',async()=>{const f=fixture({unlicensed:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,403);}); +test('revision conflict preserved without internal error disclosure',async()=>{const f=fixture({conflict:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,409);assert.doesNotMatch(JSON.stringify(f.sent),/internal-field/);}); +test('service failure neither exposes details nor changes live plan',async()=>{const f=fixture({down:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,503);assert.doesNotMatch(JSON.stringify(f.sent),/secret-host-detail|synthetic-test-only/);}); +test('V1 live schedule is NOT intercepted',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/schedule`)),false);assert.equal(f.upstream.length,0);}); +test('device telemetry mapped only to operation ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/operation`));assert.match(f.upstream[0].url,/\/inputs\/operation$/);}); +test('unauthenticated device blocked',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.upstream.length,0);}); +test('device rate limit reused',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); +test('unexpected method rejected',async()=>{const f=fixture();await f.bridge({method:'DELETE'},{},url(''));assert.equal(f.sent[0].status,405);assert.equal(f.upstream.length,0);}); + +test('measurement batches use authenticated dedicated data ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.match(f.upstream[0].url,/\/planner\/measurements$/);assert.equal(f.upstream[0].args.method,'POST');}); +test('unauthenticated measurement upload cannot reach storage',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.upstream.length,0);}); +test('device cannot change dataset mapping through public proxy',async()=>{const f=fixture();assert.equal(await f.bridge({method:'PUT'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/datasets/physical-v1`)),false);assert.equal(f.upstream.length,0);}); +test('measurement upload keeps device rate protection',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);}); diff --git a/services/netplan-v4/tests/test_application_deployment.py b/services/netplan-v4/tests/test_application_deployment.py new file mode 100644 index 0000000..44b2d8e --- /dev/null +++ b/services/netplan-v4/tests/test_application_deployment.py @@ -0,0 +1,39 @@ +import importlib.util +from pathlib import Path +import sqlite3,tempfile,unittest +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('app_deploy',ROOT/'commissioning/deploy_application.py');d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) +class ApplicationDeploymentTest(unittest.TestCase): + def test_backup_consistent_and_original_not_changed(self): + with tempfile.TemporaryDirectory() as tmp: + src=Path(tmp)/'source.sqlite';out=Path(tmp)/'backup.sqlite' + with sqlite3.connect(src) as c:c.execute('CREATE TABLE sample(value)');c.execute('INSERT INTO sample VALUES(3)') + before=src.read_bytes();d.backup_database(src,out) + with sqlite3.connect(out) as c:self.assertEqual(c.execute('SELECT value FROM sample').fetchone()[0],3) + self.assertEqual(before,src.read_bytes()) + def test_backup_cannot_invent_missing_database(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError):d.backup_database(Path(tmp)/'missing.sqlite',Path(tmp)/'new.sqlite') + def test_atomic_refuses_symlink(self): + with tempfile.TemporaryDirectory() as tmp: + a=Path(tmp)/'a';b=Path(tmp)/'b';a.write_text('original');b.symlink_to(a) + with self.assertRaises(ValueError):d.atomic(b,b'changed') + self.assertEqual(a.read_text(),'original') + def test_only_two_reviewed_portal_files(self): + self.assertEqual(set(d.PORTAL_FILES.values()),{'netplan-v4-bridge.mjs','public/netplan-v4.js'}) + def test_build_tests_backup_before_runtime_replacement(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertLess(s.index("'target_python_and_portal_tests_passed'"),s.index('v4_replaced=True')) + self.assertLess(s.index("'consistent_database_backup'"),s.index('v4_replaced=True')) + def test_does_not_change_optimizer_or_device_permissions(self): + self.assertNotIn('/settings',d.BOOTSTRAP) + self.assertNotIn('trial/arm',d.BOOTSTRAP) + self.assertNotIn('IPS_',d.BOOTSTRAP) + self.assertIn("state['liveEnabled'] is False",d.BOOTSTRAP) + def test_image_contains_application_installer_imported_by_tests(self): + self.assertIn('COPY commissioning/deploy_application.py ./commissioning/deploy_application.py',(ROOT/'Dockerfile').read_text()) + def test_file_bind_mount_recreated_not_only_restarted(self): + s=(ROOT/'commissioning/deploy_application.py').read_text() + self.assertIn("'--force-recreate','--wait','license-portal'",s) + self.assertIn('additive_data_retained',s) +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_battery_recovery.py b/services/netplan-v4/tests/test_battery_recovery.py new file mode 100644 index 0000000..e82974c --- /dev/null +++ b/services/netplan-v4/tests/test_battery_recovery.py @@ -0,0 +1,43 @@ +from dataclasses import replace +import unittest +from netplan_v4.optimizer import optimize +from netplan_v4.domain import Limits +from test_v4 import AT,seq,batt + +class BatteryRecoveryTest(unittest.TestCase): + def plan(self,b,steps=None): + steps=steps or seq([0.]*3+[1000.]*3,pv=[1000.]*3+[0.]*3,sell=[0.]*6) + return optimize(steps,[b],at=AT,observed_peaks={'2026-10':1.},peak_prices={'2026-10':0.}) + def test_reserve_recovery_preserves_real_initial_energy(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + energy=.05 + for x in p['points']: + target=x['batteryTargetW'];energy+=target/1000/12*(.9**.5 if target>=0 else 1/(.9**.5)) + self.assertAlmostEqual(100*energy,x['socEndPercent']['b'],places=5) + self.assertGreaterEqual(energy,.05-1e-6) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],15.-1e-5) + def test_below_hardware_min_not_fabricated(self): + b=batt(soc_percent=1.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True) + self.assertFalse(self.plan(b)['executable']) + def test_disabled_can_rearm_after_charging(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.) + p=self.plan(b);self.assertTrue(p['executable'],p) + self.assertAlmostEqual(min(0.,p['points'][0]['batteryTargetW']),0.) + self.assertTrue(any(x['batteryTargetW']<-1. for x in p['points'][3:])) + first=next(i for i,x in enumerate(p['points']) if x['batteryTargetW']<-1.) + self.assertGreaterEqual(p['points'][first-1]['socEndPercent']['b'],17.-1e-5) + def test_cannot_discharge_below_rearm_while_blocked(self): + b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False) + p=self.plan(b,seq([1000.]*6));self.assertTrue(p['executable']) + self.assertTrue(all(x['batteryTargetW']>=-1e-5 for x in p['points'])) + def test_unavailable_power_does_not_get_invented(self): + b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,max_charge_w=0.,max_discharge_w=0.) + p=self.plan(b);self.assertFalse(p['executable']);self.assertEqual(p['status'],'no_feasible_plan') + def test_invalid_hysteresis_range_rejected(self): + b=batt(rearm_soc_percent=95.) + self.assertFalse(self.plan(b)['executable']) + def test_no_recovery_permission_keeps_strict_behavior(self): + self.assertFalse(self.plan(batt(soc_percent=5.))['executable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_container_access.py b/services/netplan-v4/tests/test_container_access.py new file mode 100644 index 0000000..8a24ebc --- /dev/null +++ b/services/netplan-v4/tests/test_container_access.py @@ -0,0 +1,60 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import os +import unittest +from runtime_preflight import SOURCE_DIRS, SOURCE_FILES, verify_access + + +class ContainerAccessTest(unittest.TestCase): + def fixture(self, tmp): + root = Path(tmp) / "app" + root.mkdir() + for name in SOURCE_DIRS: + (root / name).mkdir() + (root / name / "sample.txt").write_text("readable") + for name in SOURCE_FILES: + (root / name).write_text("readable") + return root, Path(tmp) / "reports" + + def test_code_readable_and_reports_outside_readonly_tree(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + self.assertEqual(verify_access(root, reports), 8) + self.assertEqual(list(reports.iterdir()), []) + + def test_missing_runner_is_reported(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + (root / "run_tests.py").unlink() + with self.assertRaises(FileNotFoundError): + verify_access(root, reports) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_unreadable_source_fails(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "netplan_v4" / "sample.txt" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o600) + + @unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user") + def test_untraversable_directory_is_not_silently_skipped(self): + with TemporaryDirectory() as tmp: + root, reports = self.fixture(tmp) + path = root / "tests" + path.chmod(0) + try: + with self.assertRaises(PermissionError): + verify_access(root, reports) + finally: + path.chmod(0o700) + + def test_reports_cannot_target_application_tree(self): + with TemporaryDirectory() as tmp: + root, _ = self.fixture(tmp) + with self.assertRaises(ValueError): + verify_access(root, root / "test-results") diff --git a/services/netplan-v4/tests/test_controlled_trial.py b/services/netplan-v4/tests/test_controlled_trial.py new file mode 100644 index 0000000..ddb1f2a --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial.py @@ -0,0 +1,120 @@ +import copy +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app +from netplan_v4 import controlled_trial as t + +AT=datetime(2026,10,2,12,tzinfo=timezone.utc) +PLANT='00000000-0000-4000-8000-000000000001' +SESSION='00000000-0000-4000-8000-000000000002' +PLAN='00000000-0000-4000-8000-000000000003' +TOKEN='synthetic-only-control-test-token' + +def fixture(): + context={'batteries':{'b':{'capacityKwh':10.}},'limits':{'importW':10000.,'exportW':None,'managerMonthLimitsW':{}}} + plan={'planId':PLAN,'installationId':PLANT,'sourceFamily':'3','configRevision':1, + 'runMode':'shadow','liveEnabled':False,'executable':True,'validUntil':(AT+timedelta(hours=1)).isoformat(), + 'controlContext':context,'inputQuality':{'loadBasis':'base_load','accountingEvidenceId':'synthetic-meter-boundary-v1'}, + 'peakCostIsEstimate':True} + view={'installationId':PLANT,'liveEnabled':False,'fresh':True,'plan':plan,'settings':{'family':'3','revision':1}} + request={'sessionId':SESSION,'expectedPlanId':PLAN,'expectedRevision':1,'durationSeconds':300, + 'maxChargeW':5000.,'maxDischargeW':5000.,'assetId':'b','managerId':11111,'batteryInstanceId':22222, + 'acceptEstimatedPeak':True,'actuatorWatchdogEvidenceId':'synthetic-hardware-watchdog', + 'confirmation':'ARM_BOUNDED_CONTROL_TRIAL'} + return view,request + +class ControlledTrialTest(unittest.TestCase): + def setUp(self): + self.s=PlannerStore(':memory:');self.v,self.r=fixture() + def tearDown(self): self.s.close() + def arm(self,**kw): + return t.arm(self.s,PLANT,kw.get('request',self.r),kw.get('view',self.v),kw.get('now',AT),kw.get('allowed',{PLANT})) + def test_default_has_no_authority(self): + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,set())) + with self.assertRaises(ValueError):self.arm(allowed=set()) + def test_shadow_plan_never_becomes_live(self): + before=copy.deepcopy(self.v);self.arm() + a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(self.v,before);self.assertEqual(a['kind'],'controlled_trial_authority') + self.assertTrue(a['sourcePlanRemainsShadow']);self.assertFalse(self.v['plan']['liveEnabled']) + def test_short_authority_lease(self): + self.arm();a=t.authority(self.s,PLANT,self.v,AT,{PLANT}) + self.assertEqual(t.timestamp(a['validUntil'])-AT,timedelta(seconds=90)) + def test_session_expiry(self): + self.arm();self.assertIsNone(t.authority(self.s,PLANT,self.v,AT+timedelta(seconds=300),{PLANT})) + def test_revoke(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT);self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_expired_or_revoked_session_not_resurrected(self): + self.arm();t.revoke(self.s,PLANT,SESSION,AT) + with self.assertRaises(ValueError):self.arm() + def test_active_session_cannot_be_extended(self): + self.arm() + with self.assertRaises(ValueError):self.arm(now=AT+timedelta(seconds=30)) + def test_other_plant(self): + self.arm();self.assertIsNone(t.authority(self.s,SESSION,self.v,AT,{SESSION})) + def test_revocation_mismatched_id_leaves_real_session(self): + self.arm();t.revoke(self.s,PLANT,PLAN,AT);self.assertIsNotNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_stale_source_plan_cannot_arm(self): + self.v['fresh']=False + with self.assertRaises(ValueError):self.arm() + def test_stale_source_removes_authority(self): + self.arm();self.v['fresh']=False;self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_revision_bool_not_an_integer(self): + self.r['expectedRevision']=True + with self.assertRaises(ValueError):self.arm() + def test_wrong_revision(self): + self.r['expectedRevision']=2 + with self.assertRaises(ValueError):self.arm() + def test_revision_change_invalidates_authority(self): + self.arm();self.v['settings']['revision']=2;self.v['plan']['configRevision']=2 + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_policy_change_invalidates_authority(self): + self.arm();self.v['plan']['controlContext']['limits']['importW']=12000. + self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT})) + def test_newer_plan_same_policy_supported(self): + self.arm();self.v['plan']['planId']=SESSION + self.assertEqual(t.authority(self.s,PLANT,self.v,AT,{PLANT})['sourceShadowPlanId'],SESSION) + def test_auto_family_not_enabled_for_first_trial(self): + self.v['settings']['family']='auto' + with self.assertRaises(ValueError):self.arm() + def test_aggregate_house_refused(self): + self.v['plan']['inputQuality']['loadBasis']='house_total' + with self.assertRaises(ValueError):self.arm() + def test_no_accounting_evidence_no_trial(self): + del self.v['plan']['inputQuality']['accountingEvidenceId'] + with self.assertRaises(ValueError):self.arm() + def test_no_watchdog_evidence_no_trial(self): + self.r['actuatorWatchdogEvidenceId']='' + with self.assertRaises(ValueError):self.arm() + def test_no_implicit_estimate_acceptance(self): + self.r['acceptEstimatedPeak']=False + with self.assertRaises(ValueError):self.arm() + def test_power_cap_and_nonfinite(self): + for v in (5001, float('nan'),float('inf'),True,0,'5000'): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'maxChargeW':v}) + def test_duration_bound(self): + for v in (0,1801,True,30.5): + with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'durationSeconds':v}) + def test_cannot_replace_asset(self): + self.r['assetId']='other' + with self.assertRaises(ValueError):self.arm() + def test_two_assets_not_supported(self): + self.v['plan']['controlContext']['batteries']['other']={} + with self.assertRaises(ValueError):self.arm() + def test_operator_endpoint_requires_auth(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False) + with TestClient(app) as c: + self.assertEqual(c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',json=self.r).status_code,401) + r=c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',headers={'X-Enelix-Service-Token':TOKEN},json=self.r) + self.assertEqual(r.status_code,409);self.assertIn('allowlist',r.text) + self.assertIsNone(c.get(f'/internal/v2/prognosis/{PLANT}/planner',headers={'X-Enelix-Service-Token':TOKEN}).json()['controlledTrial']) + def test_trial_allowlist_cannot_expand_installation_access(self): + with tempfile.TemporaryDirectory() as td,self.assertRaises(ValueError): + create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[SESSION]) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_controlled_trial_pipeline.py b/services/netplan-v4/tests/test_controlled_trial_pipeline.py new file mode 100644 index 0000000..894242e --- /dev/null +++ b/services/netplan-v4/tests/test_controlled_trial_pipeline.py @@ -0,0 +1,44 @@ +import copy +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch +from fastapi.testclient import TestClient +from netplan_v4.service import create_app, ingest, run_once +from test_v4 import inputs, AT +from test_controlled_trial import fixture, PLANT, SESSION, TOKEN + +class ControlledTrialPipelineTest(unittest.TestCase): + def test_real_planner_to_internal_trial_api_and_revoke(self): + with tempfile.TemporaryDirectory() as td: + app=create_app(Path(td)/'trial.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[PLANT]) + store=app.state.store_factory() + op,fc,tar=inputs() + for f in fc['families'].values():f['accountingEvidenceId']='synthetic-accounting-review' + for k,v in [('operation',op),('forecast',fc),('tariffs',tar)]:ingest(store,PLANT,k,v,AT) + plan=run_once(store,AT);self.assertEqual(plan['status'],'optimal');store.close() + headers={'X-Enelix-Service-Token':TOKEN};prefix=f'/internal/v2/prognosis/{PLANT}/planner' + with patch('netplan_v4.service.datetime',wraps=datetime) as clock,TestClient(app) as c: + clock.now.return_value=AT + view=c.get(prefix,headers=headers).json() + self.assertIsNone(view['controlledTrial']);self.assertTrue(view['fresh']) + _,request=fixture();request.update(expectedPlanId=plan['planId'],expectedRevision=0,acceptEstimatedPeak=False) + grant=c.post(prefix+'/trial/arm',headers=headers,json=request) + self.assertEqual(grant.status_code,200,grant.text) + view=c.get(prefix,headers=headers).json() + self.assertEqual(view['controlledTrial']['sourceShadowPlanId'],plan['planId']) + self.assertFalse(view['liveEnabled']);self.assertFalse(view['plan']['liveEnabled']) + self.assertEqual(view['plan']['runMode'],'shadow') + self.assertEqual(c.post(prefix+'/ack',headers=headers,json={'planId':plan['planId'],'revision':0,'status':'applied'}).status_code,400) + self.assertEqual(c.post(prefix+'/trial/revoke',headers=headers,json={'sessionId':SESSION}).status_code,200) + self.assertIsNone(c.get(prefix,headers=headers).json()['controlledTrial']) + self.assertEqual(c.post(prefix+'/trial/arm',headers=headers,json=request).status_code,409) + def test_accounting_marker_cannot_label_aggregate_load_verified(self): + from netplan_v4.store import PlannerStore + s=PlannerStore(':memory:');op,fc,tar=inputs() + f=fc['families']['3'];f['loadBasis']='house_total';f['accountingEvidenceId']='synthetic-incorrect-label' + with self.assertRaises(ValueError):ingest(s,PLANT,'forecast',fc,AT) + s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_delivery.py b/services/netplan-v4/tests/test_delivery.py new file mode 100644 index 0000000..3cbe890 --- /dev/null +++ b/services/netplan-v4/tests/test_delivery.py @@ -0,0 +1,82 @@ +import importlib.util +from pathlib import Path +from tempfile import TemporaryDirectory +from datetime import datetime,timedelta,timezone +import unittest +ROOT=Path(__file__).resolve().parents[1] +def module(name,path): + spec=importlib.util.spec_from_file_location(name,path);value=importlib.util.module_from_spec(spec);spec.loader.exec_module(value);return value +publisher=module('publisher',ROOT/'integrations/netplan_v4_publisher.py') +hooks=module('hooks',ROOT/'install_hooks.py') +AT=datetime(2026,10,1,12,0,tzinfo=timezone.utc) + +class PublisherTest(unittest.TestCase): + def test_tariff_null_preserved_and_mode_explicit(self): + cfg={'tarif_bezug_modus':'statisch','tarif_bezug':'custom','tarif_bezug_fest':0.,'tarif_einspeisung_modus':'dynamisch','tarif_einspeisung':'dynamic','tarif_peak_fest':0.} + p=publisher.tariff_payload(cfg,AT);self.assertEqual(p['import']['staticChfKwh'],0.);self.assertEqual(p['peakChfKwMonth'],0.) + del cfg['tarif_bezug_modus'] + with self.assertRaises(ValueError):publisher.tariff_payload(cfg,AT) + def test_native_ckw_unit_zero_and_delivery_range(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'CHF_kWh','value':0.}]}] + p=publisher.ckw_payload('CKW Dynamisch Home',rows,AT-timedelta(hours=1),AT) + self.assertEqual(p['periods'][0]['value'],0.);self.assertEqual(p['periods'][0]['unit'],'CHF/kWh');self.assertEqual(p['periods'][0]['sourceKind'],'published_interval') + del rows[0]['end_timestamp'] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + def test_unknown_unit_and_future_publication_rejected(self): + rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'unknown','value':100.}]}] + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT) + rows[0]['integrated'][0]['unit']='CHF_kWh' + with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,AT+timedelta(minutes=1),AT) + def test_forecast_not_relabelled_as_base_load(self): + p=publisher.forecast_payload([(23,{AT:5000.},{AT:2000.})],AT) + self.assertEqual(p['families']['23']['loadBasis'],'house_total');self.assertIsNone(p['families']['23']['trainedUntil']) + +class InstallerTest(unittest.TestCase): + def fixtures(self,root): + files={'prognosis-manager-enelix2/api/main.py':'def f():\n _portal_store_configuration(anlagen_id, configuration)\n', + 'prognosis-manager-enelix2/forecast_engine/main.py':'def f():\n for a in []:\n for key, default in []:\n d[key] = float(d.get(key) or default)\n for c in []:\n p_3 = v3.predict(data_obj, p_1, p_2)\n', + 'prognosis-manager-enelix2/tariff_importer/main.py':'def f():\n for a in []:\n for b in []:\n for c in []:\n if True:\n return candidate_points\n', + 'license/server.mjs':'const server = createServer(async (req, res) => {\n if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {status:"ok"});\n});', + 'license/public/app.js':'// Parallel GUI work. Must remain byte-identical.'} + for name,data in files.items():p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(data) + return files + def test_install_idempotent_and_verified_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);originals=self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root) + self.assertFalse(hooks.plan(root));self.assertEqual((root/'license/public/app.js').read_text(),originals['license/public/app.js']) + hooks.rollback(receipt) + for name,data in originals.items():self.assertEqual((root/name).read_text(),data) + self.assertFalse((root/'license/public/netplan-v4.html').exists()) + def test_concurrent_changes_never_overwritten(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);entries=hooks.plan(root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Concurrent') + with self.assertRaises(ValueError):hooks.apply(entries,root) + self.assertTrue(p.read_text().endswith('// Concurrent')) + def test_later_changes_block_rollback(self): + with TemporaryDirectory() as tmp: + root=Path(tmp);self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Later') + with self.assertRaises(ValueError):hooks.rollback(receipt) + self.assertTrue(p.read_text().endswith('// Later')) + def test_changed_or_ambiguous_anchor_fails_closed(self): + with self.assertRaises(ValueError):hooks.once('x x','x','y') + with self.assertRaises(ValueError):hooks.patch_api('def f():pass') + +class HorizonValidationTest(unittest.TestCase): + def test_synthetic_48h_energy_soc_limits(self): + import math + from netplan_v4.domain import Battery,Step,Price,Limits + from netplan_v4.optimizer import optimize + steps=[] + for i in range(576): + solar=max(0.,math.sin(math.pi*((i%288)/12-6)/12))*10000. + steps.append(Step(AT+timedelta(minutes=5*i),3000.,solar,Price(.15 if i%288<72 else .35),Price(.1))) + b=Battery('synthetic',40.,20.,10.,90.,10000.,8000.,AT,grid_charging=True) + p=optimize(steps,[b],at=AT,limits=Limits(import_w=12000.,export_w=10000.),observed_peaks={'2026-10':8.},peak_prices={'2026-10':5.}) + self.assertTrue(p['executable']);self.assertEqual(len(p['points']),576) + for s,x in zip(steps,p['points']): + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=3) + self.assertLessEqual(x['gridTargetW'],12000.001);self.assertGreaterEqual(x['gridTargetW'],-10000.001) + self.assertTrue(9.999<=x['socEndPercent']['synthetic']<=90.001) + self.assertGreaterEqual(p['points'][-1]['socEndPercent']['synthetic'],19.999) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_acceptance.py b/services/netplan-v4/tests/test_forecast_acceptance.py new file mode 100644 index 0000000..eb89b2e --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_acceptance.py @@ -0,0 +1,80 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +from forecast_acceptance import source_paths, verify_forecast_bundle + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('forecast_release_test', ROOT / 'release_preflight.py') +preflight = importlib.util.module_from_spec(spec) +spec.loader.exec_module(preflight) + + +class ForecastAcceptanceTest(unittest.TestCase): + def fixture(self, root): + src, bundle = root / 'source', root / 'bundle' + for path in (src, bundle): + path.mkdir() + for name, value in {'main.py': 'x=1\n', 'telemetry_quality.py': 'x=2\n', 'requirements.txt': 'pandas\n'}.items(): + (path / name).write_text(value) + manifest = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in source_paths(src)} + (bundle / 'SOURCE_MANIFEST.json').write_text(json.dumps(manifest)) + return src, bundle + + def test_exact_source_is_verified(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_modified_development_source_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (src / 'main.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_extra_staged_python_file_blocks_test(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'surprise.py').write_text('x=3\n') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_symlink_is_rejected(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + (bundle / 'main.py').unlink() + (bundle / 'main.py').symlink_to(src / 'main.py') + with self.assertRaises(ValueError): + verify_forecast_bundle(bundle, src) + + def test_models_secrets_and_databases_not_staged(self): + with TemporaryDirectory() as d: + src, bundle = self.fixture(Path(d)) + for name in ('.env', 'model.pkl', 'users.db'): + (src / name).write_text('not a source file') + self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3) + + def test_forecast_checks_have_no_network_or_live_mounts(self): + args = next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='forecast_candidate_python311') + self.assertEqual(args[args.index('--network')+1], 'none') + self.assertEqual(args[args.index('--user')+1], '1000:1000') + self.assertIn('--read-only', args) + for forbidden in ('--volume', '-v', '--env-file', '--privileged', 'up', 'restart'): + self.assertNotIn(forbidden, args) + + def test_runner_does_not_import_application_main(self): + source = (ROOT / 'acceptance/check_forecast.py').read_text() + tree = ast.parse(source) + imports = {a.name for node in ast.walk(tree) if isinstance(node, ast.Import) for a in node.names} + self.assertNotIn('main', imports) + self.assertNotIn('run_forecast(', source) + self.assertNotIn('get_configs(', source) + + +if __name__ == '__main__': + unittest.main() diff --git a/services/netplan-v4/tests/test_forecast_quality.py b/services/netplan-v4/tests/test_forecast_quality.py new file mode 100644 index 0000000..0a81f63 --- /dev/null +++ b/services/netplan-v4/tests/test_forecast_quality.py @@ -0,0 +1,31 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from netplan_v4.forecast_quality import assess_family +from netplan_v4.service import ingest,run_once +from netplan_v4.store import PlannerStore +from test_v4 import inputs,AT,AID + +class ForecastQualityTest(unittest.TestCase): + def test_zero_load_not_free_energy_assumption(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_confirmed_pv_only_plant_is_supported(self): + f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)],'zeroLoadConfirmed':True} + self.assertTrue(assess_family(f)['valid']) + def test_boolean_and_nan_are_not_forecast_powers(self): + for value in (float('nan'),True,-1.,None): + f={'points':[{'loadW':value,'pvW':1000.} for _ in range(3)]} + self.assertFalse(assess_family(f)['valid']) + def test_selected_invalid_family_not_silently_replaced(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));op,fc,tar=inputs() + for p in fc['families']['3']['points']:p['loadW']=0. + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(s,AID,k,v,AT) + plan=run_once(s,AT) + self.assertEqual(plan['status'],'awaiting_inputs');self.assertIn('all-zero',plan['reason']);self.assertIsNone(s.current(AID));s.close() + def test_nonzero_family_remains_usable(self): + f={'points':[{'loadW':50.,'pvW':0.} for _ in range(3)]} + self.assertTrue(assess_family(f)['valid']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_integrated_deployment.py b/services/netplan-v4/tests/test_integrated_deployment.py new file mode 100644 index 0000000..d9ed2d7 --- /dev/null +++ b/services/netplan-v4/tests/test_integrated_deployment.py @@ -0,0 +1,71 @@ +import ast +import hashlib +import importlib.util +import io +import contextlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('integrated_deployment_test',ROOT/'deploy_integrated_shadow.py') +d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d) + + +class IntegratedDeploymentTest(unittest.TestCase): + def test_exact_server_scope_and_no_symcon(self): + scope=d.compose_groups() + self.assertEqual(set(scope),{'v4','forecast','portal'}) + self.assertEqual(scope['forecast'][2],['api','forecast-engine','tariff-importer']) + self.assertNotIn('license-admin',scope['portal'][2]) + self.assertNotIn('/var/lib/symcon',(ROOT/'deploy_integrated_shadow.py').read_text()) + def test_sqlite_backup_is_consistent_and_source_unchanged(self): + with tempfile.TemporaryDirectory() as temp: + source=Path(temp)/'before.sqlite';dest=Path(temp)/'backup.sqlite' + con=sqlite3.connect(source);con.execute('CREATE TABLE test(value INTEGER)') + con.execute('INSERT INTO test VALUES (37)');con.commit();con.close() + before=source.read_bytes() + with patch.object(d.os,'geteuid',return_value=1000):result=d.backup_database(source,dest) + self.assertEqual(result['integrity'],'ok');self.assertEqual(before,source.read_bytes()) + check=sqlite3.connect(dest);self.assertEqual(check.execute('SELECT value FROM test').fetchone()[0],37);check.close() + self.assertEqual(result['sha256'],hashlib.sha256(dest.read_bytes()).hexdigest()) + def test_missing_database_not_fabricated(self): + with tempfile.TemporaryDirectory() as temp: + dst=Path(temp)/'backup' + self.assertEqual(d.backup_database(Path(temp)/'missing',dst),{'exists':False}) + self.assertFalse(dst.exists()) + def test_scope_uses_no_environment_dump_or_control_activation(self): + text=(ROOT/'deploy_integrated_shadow.py').read_text() + self.assertNotIn('.Config.Env',text) + self.assertNotIn('NetzfahrplanAktiv',text) + self.assertNotIn('IPS_RequestAction',text) + self.assertNotIn('PROGNOSIS_SERVICE_TOKEN',text) + ast.parse(text) + def test_build_and_tests_precede_all_service_recreation(self): + with tempfile.TemporaryDirectory() as temp: + root=Path(temp)/'netplan';root.mkdir();project=Path(temp)/'forecast';project.mkdir() + portal=Path(temp)/'portal';portal.mkdir();(root/'acceptance').mkdir() + commands=[] + def fake(args,**kwargs): + commands.append(list(args));out='' + if 'ps' in args:out='0123456789abcdef\n' + elif 'inspect' in args: + out='sha256:'+('a'*64) if '{{.Image}}' in args else '/some/compose.yaml' + elif 'exec' in args: + out=json.dumps({'health':{'mode':'shadow','liveEnabled':False},'inputEventsByKind':{},'lastRunStatus':None}) + return SimpleNamespace(returncode=0,stdout=out) + with patch.object(d,'ROOT',root),patch.object(d,'PROJECT',project),patch.object(d,'PORTAL',portal),patch.object(d,'verify_sources',return_value={}),patch.object(d.os,'geteuid',return_value=1000),patch.object(d.subprocess,'run',side_effect=fake),patch('install_hooks.plan',return_value={}),patch('install_hooks.apply',return_value=None),contextlib.redirect_stdout(io.StringIO()): + d.run('e3a08f9e-af12-4695-99bd-8b51c0520021') + ups=[i for i,c in enumerate(commands) if 'up' in c] + self.assertEqual(len(ups),3) + test_at=next(i for i,c in enumerate(commands) if '--test-only' in c) + build_at=max(i for i,c in enumerate(commands) if 'build' in c) + self.assertLess(test_at,min(ups));self.assertLess(build_at,min(ups)) + self.assertTrue(all('--no-build' in commands[i] and '--no-deps' in commands[i] for i in ups)) + report=json.loads(next((root/'deployment-reports').glob('*/DEPLOYMENT.json')).read_text()) + self.assertEqual(report['status'],'server_shadow_installed') + self.assertFalse(report['liveEnabled']);self.assertFalse(report['productionReady']) diff --git a/services/netplan-v4/tests/test_measurement_pipeline.py b/services/netplan-v4/tests/test_measurement_pipeline.py new file mode 100644 index 0000000..ab8cfa6 --- /dev/null +++ b/services/netplan-v4/tests/test_measurement_pipeline.py @@ -0,0 +1,161 @@ +import copy +import json +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from fastapi.testclient import TestClient +from netplan_v4 import measurement_pipeline as m +from netplan_v4.store import PlannerStore +from netplan_v4.service import create_app, ingest, run_once, status +from test_v4 import inputs, AID, TOKEN + +NOW=int(datetime(2026,10,2,12,tzinfo=timezone.utc).timestamp()) + +def config(**changes): + c={'datasetId':'physical-v1','mappingSha256':'a'*64,'inventorySha256':'b'*64, + 'formula':'physical_sum_v1','solarReference':None,'minimumCoverage':.95, + 'maximumGapSeconds':10,'minimumTrainingHours':1,'historyDays':14,'sources':[]} + for i,(k,role) in enumerate((('grid','grid'),('pv','pv'),('battery','physical_storage'),('sdl','sdl_request'))): + c['sources'].append({'key':k,'role':role,'variableId':101+i,'factorToW':1,'maxAgeSeconds':60}) + c.update(changes);return c + +def record(t, c=None, **values): + c=c or config(); v={'grid':3000.,'pv':5000.,'battery':2000.,'sdl':0.,**values} + return {'schemaVersion':1,'kind':'raw_accounting_capture','installationId':AID, + 'capturedAt':m.iso(t),'captureStartedAt':m.iso(t),'mappingSha256':c['mappingSha256'], + 'reportedInventorySha256':c['inventorySha256'],'raw':{s['key']:{'variableId':s['variableId'],'value':v.get(s['key'],0), + 'sourceUpdatedAt':t,'issues':[]} for s in c['sources']}} + +class MeasurementPipelineTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.path=str(Path(self.tmp.name)/'pipeline.sqlite') + self.s=PlannerStore(self.path);self.c=config();m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self):self.s.close();self.tmp.cleanup() + def batch(self,rows,now=NOW,plant=AID):return m.ingest_batch(self.s.con,plant,{'version':1,'datasetId':'physical-v1','records':rows},now) + def history(self,hours=2): + rows=[record(t) for t in range(NOW-hours*3600,NOW+1,30)] + for i in range(0,len(rows),120):self.batch(rows[i:i+120]) + def test_append_ack_idempotence(self): + r=record(NOW);self.assertEqual(self.batch([r])['stored'],1) + a=self.batch([r]);self.assertEqual(a['duplicates'],1);self.assertEqual(a['acceptedThrough'],m.iso(NOW)) + def test_conflict_rolls_back_whole_batch(self): + self.batch([record(NOW)]) + with self.assertRaises(ValueError):self.batch([record(NOW-30),record(NOW,grid=9000)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_registration_immutable(self): + self.assertEqual(m.register_dataset(self.s.con,AID,self.c,NOW)['status'],'configured') + with self.assertRaises(ValueError):m.register_dataset(self.s.con,AID,{**self.c,'minimumCoverage':1.},NOW) + def test_plants_isolated(self): + with self.assertRaises(ValueError):self.batch([record(NOW)],plant='30509683-4569-49e4-848f-4905e4cc813a') + def test_mapping_is_not_device_supplied(self): + r=record(NOW);r['mappingSha256']='c'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_no_unknown_text_persisted(self): + r=record(NOW);r['unknown']='do-not-store';r['raw']['grid']['value']='do-not-store';r['assessment']={'status':'verified'} + self.batch([r]);data=self.s.con.execute('SELECT value FROM planner_observations').fetchone()[0] + self.assertNotIn('do-not-store',data);self.assertIn('null',data);self.assertNotIn('verified',data) + def test_missing_zero_not_invented(self): + r=record(NOW);r['raw'].pop('grid');p=m.project(r,self.c,AID,NOW) + self.assertIsNone(p['raw']['grid']['value']);self.assertFalse(p['raw']['grid']['valid']) + def test_boolean_not_numeric(self): + p=m.project(record(NOW,grid=True),self.c,AID,NOW);self.assertFalse(p['raw']['grid']['valid']) + def test_batch_order_required(self): + with self.assertRaises(ValueError):self.batch([record(NOW),record(NOW-30)]) + def test_batch_size_limit(self): + with self.assertRaises(ValueError):self.batch([record(NOW)]*121) + def test_no_naive_time(self): + r=record(NOW);r['capturedAt']='2026-10-02T12:00:00' + with self.assertRaises(ValueError):self.batch([r]) + def test_future_source_not_rejuvenated(self): + r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1 + self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid']) + def test_profile_uses_physical_not_virtual(self): + r=record(NOW);r['raw']['virtual_ev']={'value':-90000} + self.assertEqual(m.physical_value({'grid':3000,'pv':5000,'battery':2000},self.c),6000) + def test_zero_numeric_valid(self): + p=m.project(record(NOW,grid=0,pv=0,battery=0),self.c,AID,NOW);self.assertTrue(p['raw']['grid']['valid']) + def projected(self,rows):return [m.project(r,self.c,AID,NOW) for r in rows] + def test_rollup_time_weighted(self): + rows=[record(t,grid=3000 if t=a-1e-10 for a,b in zip(slopes,slopes[1:])));self.assertAlmostEqual(slopes[-1],5.) + self.assertAlmostEqual(o.incremental_cost(1.,.8,5.),0.) + def test_hard_cap_beats_outlook(self): + p=self.solve(limits=Limits(import_w=1000.),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(max(x['gridTargetW'] for x in p['points']),1000.001) + def test_no_fixed_cap_is_supported(self): + p=self.solve(limits=Limits(),peak_outlooks={'2026-10':outlook()}) + self.assertTrue(p['executable']);self.assertTrue(all(x['importLimitW'] is None for x in p['points'])) + def test_monthly_cap_beats_outlook(self): + p=self.solve(limits=Limits(manager_month_limits_w={10:1000.}),peak_outlooks={'2026-10':outlook()}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_estimated_basis_not_in_measured_map(self): + p=self.solve(peak_context={'2026-10':estimate(1.)}) + self.assertEqual(p['measuredPeaksKw'],{});self.assertEqual(p['peakBasisKw']['2026-10'],1.) + self.assertTrue(p['peakCostIsEstimate']) + def test_context_cannot_lie_about_numerical_basis(self): + self.assertFalse(self.solve(peak_context={'2026-10':estimate(3.)})['executable']) + def test_no_free_cap_source(self): + with self.assertRaises(ValueError):basis_record(estimate(source='manager_cap'),'2026-10',AT,True) + def test_overlap_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),future_from=AT)})['executable']) + def test_future_observation_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),issued_at=AT+timedelta(seconds=1))})['executable']) + def test_wrong_month_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-11':outlook()})['executable']) + def test_invalid_probabilities_rejected(self): + self.assertFalse(self.solve(peak_outlooks={'2026-10':outlook(scenarios=((5.,.8),))})['executable']) + def test_no_history_no_outlook_not_fake_allowance(self): + self.assertIsNone(empirical_rest_month([],month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p')) + def test_empirical_requires_comparable_complete_history(self): + records=[] + for n in range(20): + d=AT.astimezone(ZURICH).replace(hour=0,minute=0)-timedelta(days=n+1) + records.append({'day':d.strftime('%Y-%m-%d'),'peakKw':3.+n%4,'observedAt':(d+timedelta(days=1)).isoformat(),'quality':'estimated','complete':True,'controlPolicyId':'p'}) + args=dict(month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p') + o=empirical_rest_month(records,**args);self.assertIsNotNone(o);o.validate(AT,args['horizon_end']) + for r in records:r['controlPolicyId']='different' + self.assertIsNone(empirical_rest_month(records,**args)) + + +class EstimatedBasisServiceTest(unittest.TestCase): + def setUp(self):self.tmp=TemporaryDirectory();self.path=str(Path(self.tmp.name)/'planner.sqlite');self.s=PlannerStore(self.path) + def tearDown(self):self.s.close();self.tmp.cleanup() + def load(self,when=AT,quarter=False): + op,fc,tar=inputs(when);op['measuredPeaks']={};op['planningPeaks']={'2026-10':estimate(at=when)} + if quarter: + op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.2,'measuredSeconds':int((when-AT).total_seconds()),'quality':'estimated','source':'power_history_estimate','coverage':1.} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,when) + return op + def allow(self):self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},self.s.settings(AID)['revision'],AT) + def test_opt_in_required(self): + self.load();self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + self.allow();p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertTrue(p['peakCostIsEstimate']) + self.assertEqual(self.s.peaks(AID),{}) + def test_no_peak_not_invented_even_when_opted_in(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.allow() + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs') + def test_historical_seed_independent_from_live_operation(self): + self.allow();op,fc,tar=inputs();op['measuredPeaks']={} + for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT) + ingest(self.s,AID,'planning_basis',{'version':1,'eventId':'seed','observedAt':AT.isoformat(),'peaks':{'2026-10':estimate()}},AT) + p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertEqual(p['measuredPeaksKw'],{}) + def test_estimates_persist_but_plans_do_not_increase_them(self): + self.load();self.allow();p=run_once(self.s,AT) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['kw'],5.) + self.s.close();self.s=PlannerStore(self.path) + self.assertEqual(assumptions(self.s.con,AID)['2026-10']['quality'],'estimated') + def test_verified_source_has_precedence_even_if_lower_than_estimate(self): + self.load();self.allow();self.s.initialize_peak(AID,'2026-10',4.,'verified_month_history',AT) + p=run_once(self.s,AT);self.assertEqual(p['measuredPeaksKw'],{'2026-10':4.});self.assertFalse(p['peakCostIsEstimate']) + def test_quarter_estimate_never_disguised(self): + self.load(AT+timedelta(minutes=5),True);self.allow();p=run_once(self.s,AT+timedelta(minutes=5)) + self.assertTrue(p['executable']);self.assertEqual(p['inputQuality']['quarter'],'estimated') + def test_incomplete_quarter_refused(self): + op,_,_=inputs();op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.1,'measuredSeconds':0,'quality':'estimated','source':'power_history_estimate','coverage':.8} + with self.assertRaises(ValueError):ingest(self.s,AID,'operation',op,AT) + def test_new_settings_load_old_rows(self): + old=self.s.settings(AID);old.pop('revision');old.pop('measurementPolicy');old.pop('peakOutlookPolicy');old.pop('peakOutlookReliance') + with self.s.con:self.s.con.execute('INSERT INTO planner_settings VALUES(?,?,?)',(AID,1,json.dumps(old))) + self.assertEqual(self.s.settings(AID)['measurementPolicy'],'verified_only') + self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},1,AT) + def test_http_explicit_estimate_optin_and_no_live(self): + app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as c: + url=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':0,'changes':{'measurementPolicy':'allow_estimates'}}).status_code,200) + self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + + +class RuntimeMeterTest(unittest.TestCase): + def test_exact_samples_still_estimates(self): + r=integrate_power([(0,1000.,0.,'p'),(60,1000.,.02,'p'),(120,1000.,.04,'p')],0,120) + self.assertAlmostEqual(r['importKwh'],1/30);self.assertFalse(r['billingEvidence']) + def test_gap_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,0.,'p'),(180,1000.,.1,'p')],0,180)) + def test_reset_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,1.,'p'),(60,1000.,0.,'p')],0,60)) + def test_control_policy_change_not_comparable(self):self.assertIsNone(integrate_power([(0,1000.,0.,'a'),(60,1000.,1.,'b')],0,60)) + def test_only_completed_quarters_update_peak(self): + with TemporaryDirectory() as d: + s=PlannerStore(str(Path(d)/'p.sqlite'));mid='symcon-active-import:'+'a'*64 + for i in range(16): + at=AT+timedelta(minutes=i) + obs={'meterId':mid,'sampleAt':at.isoformat(),'powerW':2000.,'totalImportKwh':10+i/30,'controlPolicyId':'p'} + with s.con:observe(s.con,AID,obs,at) + if i<15:self.assertEqual(assumptions(s.con,AID),{}) + self.assertAlmostEqual(assumptions(s.con,AID)['2026-10']['kw'],2.) + self.assertEqual(s.peaks(AID),{});s.close() + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_receiver_contract.py b/services/netplan-v4/tests/test_receiver_contract.py new file mode 100644 index 0000000..1377ae8 --- /dev/null +++ b/services/netplan-v4/tests/test_receiver_contract.py @@ -0,0 +1,45 @@ +import copy +from datetime import timedelta +import tempfile +from pathlib import Path +import unittest +from netplan_v4.receiver_contract import control_context, provenance +from netplan_v4.store import PlannerStore +from netplan_v4.service import ingest, run_once, status +from test_v4 import inputs, AT, AID + +class ReceiverContractTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'qa.sqlite')) + op,fc,tar=inputs() + for kind,value in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,kind,value,AT) + def tearDown(self):self.s.close();self.tmp.cleanup() + def test_published_plan_and_response_bound_to_same_plant(self): + p=run_once(self.s,AT);r=status(self.s,AID,AT) + self.assertEqual(p['installationId'],AID);self.assertEqual(r['installationId'],AID) + self.assertEqual(r['receiverProtocolVersion'],1);self.assertTrue(r['fresh']) + self.assertFalse(r['liveEnabled']);self.assertFalse(p['liveEnabled']) + self.assertEqual(p['inputRefs'],{'operation':'op1','forecast':'fc1','tariffs':'tar1'}) + def test_control_context_tracks_stable_policy_not_volatile_soc(self): + op=inputs()[0];a=control_context(op);op['batteries'][0]['socPercent']=80.;op['gridW']=1000. + self.assertEqual(a,control_context(op));op['batteries'][0]['minSocPercent']=15. + self.assertNotEqual(a,control_context(op)) + def test_context_is_snapshot_not_reference_to_future_changes(self): + op,fc,tar=inputs();p=provenance({'operation':op,'forecast':fc,'tariffs':tar}) + op['limits']['importW']=100.;self.assertEqual(p['controlContext']['limits']['importW'],10000.) + def test_no_plan_for_another_installation(self): + run_once(self.s,AT);r=status(self.s,'00000000-0000-4000-8000-000000000002',AT) + self.assertIsNone(r['plan']);self.assertFalse(r['fresh']) + def test_shadow_ack_is_not_execution(self): + p=run_once(self.s,AT) + self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,step=p['points'][0]['time'],status='shadow_seen') + self.assertEqual(status(self.s,AID,AT)['acknowledgement']['status'],'shadow_seen') + with self.assertRaises(ValueError):self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,status='applied') + def test_new_configuration_invalidates_fresh_flag(self): + run_once(self.s,AT);self.s.save_settings(AID,{'family':'23'},0,AT+timedelta(seconds=1)) + self.assertFalse(status(self.s,AID,AT+timedelta(seconds=2))['fresh']) + def test_rearm_default_matches_php_contract(self): + op=inputs()[0];op['batteries'][0]['rearmSocPercent']=None + self.assertEqual(control_context(op)['batteries']['b']['rearmSocPercent'],10.) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_release_preflight.py b/services/netplan-v4/tests/test_release_preflight.py new file mode 100644 index 0000000..977a292 --- /dev/null +++ b/services/netplan-v4/tests/test_release_preflight.py @@ -0,0 +1,50 @@ +import ast +import hashlib +import importlib.util +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +import unittest + +ROOT=Path(__file__).resolve().parents[1] +spec=importlib.util.spec_from_file_location('release_preflight_test',ROOT/'release_preflight.py') +preflight=importlib.util.module_from_spec(spec);spec.loader.exec_module(preflight) + +class ReleasePreflightTest(unittest.TestCase): + def test_preflight_never_restarts_existing_services(self): + for name,args,cwd,timeout in preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021'): + self.assertNotIn('up',args);self.assertNotIn('restart',args);self.assertNotIn('--with-bridges',args) + self.assertGreater(timeout,0) + first=preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021')[0][1] + self.assertIn('--test-only',first) + def test_php_runs_without_network_or_writeable_root(self): + args=next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='php83_offline') + self.assertIn('--read-only',args);self.assertEqual(args[args.index('--network')+1],'none') + self.assertEqual(args[args.index('--user')+1],'1000:1000') + def test_native_bundle_requires_same_development_source(self): + with TemporaryDirectory() as d: + root=Path(d)/'root';repo=Path(d)/'repo';out=root/'acceptance/php-src' + out.mkdir(parents=True);repo.mkdir() + source=b'=-1e-5 for x in p['points'][:3])) + def test_roundtrip_90_percent(self): + p=solve(seq([0.]*3+[5000.]*3,pv=[5000.]*3+[0.]*3,buy=[.3]*6,sell=[0.]*6),[batt(soc_percent=10.,grid_charging=False,max_discharge_w=5000.)]) + c=sum(max(0,x['batteryTargetW']) for x in p['points']);d=sum(max(0,-x['batteryTargetW']) for x in p['points']);self.assertAlmostEqual(d/c,.9,places=5) + def test_nulleinspeisung(self): + p=solve(seq([0.]*3,pv=[10000.]*3),limits=Limits(export_w=0.));self.assertTrue(p['executable']);self.assertTrue(all(abs(x['gridTargetW'])<1e-4 for x in p['points'])) + def test_nulltariffs(self): + p=solve(seq([1000.]*3,buy=[0.]*3,sell=[0.]*3));self.assertEqual(p['cashCostChf'],0.) + def test_negative_prices(self): + p=solve(seq([1000.]*3,buy=[-.1]*3));self.assertLess(p['energyCostChf'],0.) + def test_asymmetric_limits_and_balance(self): + steps=seq([0.]*3+[3000.]*3,pv=[5000.]*3+[0.]*3) + p=solve(steps,[batt(max_charge_w=700.,max_discharge_w=300.)]);self.assertTrue(p['executable']) + for s,x in zip(steps,p['points']): + self.assertLessEqual(x['batteryTargetW'],700.001);self.assertGreaterEqual(x['batteryTargetW'],-300.001) + self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=4) + self.assertTrue(9.999<=x['socEndPercent']['b']<=90.001) + def test_stale_soc_rejected(self):self.assertFalse(solve(seq([0.]*3),[batt(measured_at=AT-timedelta(hours=1))])['executable']) + def test_soc_not_fabricated(self):self.assertFalse(solve(seq([0.]*3),[batt(soc_percent=5.)])['executable']) + def test_peak_increment_only_and_baseline(self): + p=solve(seq([2000.]*3),observed_peaks={'2026-10':1.5},peak_prices={'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],2.5);self.assertAlmostEqual(p['baselineCashCostChf'],2.65) + def test_peak_blocks_unprofitable_charging(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.)],observed_peaks={'2026-10':1.},peak_prices={'2026-10':20.}) + self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001) + def test_month_switch(self): + at=datetime(2026,9,30,21,45,tzinfo=timezone.utc) + p=solve(seq([2000.]*6,at=at),observed_peaks={'2026-09':2.,'2026-10':.5},peak_prices={'2026-09':5.,'2026-10':5.}) + self.assertAlmostEqual(p['additionalPeakCostChf'],7.5) + def test_manager_cap_is_not_measured_peak(self): + p=solve(seq([1000.]*3),limits=Limits(manager_month_limits_w={10:1500.}),observed_peaks={'2026-10':0.},peak_prices={'2026-10':5.}) + self.assertEqual(p['measuredPeaksKw']['2026-10'],0.);self.assertAlmostEqual(p['additionalPeakCostChf'],5.) + def test_infeasible_not_fake_residual(self): + p=solve(seq([10000.]*3),limits=Limits(import_w=1000.));self.assertFalse(p['executable']);self.assertEqual(p['points'],[]) + def test_elapsed_quarter_required(self): + s=seq([1000.],at=AT+timedelta(minutes=10));self.assertFalse(solve(s)['executable']) + p=solve(s,quarter_history={AT:QuarterPast(.5,600)},peak_prices={'2026-10':5.},observed_peaks={'2026-10':0.}) + self.assertAlmostEqual(p['plannedPeaksKw']['2026-10'],7/3) + def test_partial_first_step(self): + s=seq([1000.]*3);s[0]=replace(s[0],start=AT+timedelta(seconds=10),seconds=290) + p=solve(s,quarter_history={AT:QuarterPast(.02,10)});self.assertTrue(p['executable']);self.assertEqual(p['validFrom'],s[0].start.isoformat()) + def test_terminal_guard(self): + p=solve(seq([0.]*3,sell=[10.]*3),[batt(soc_percent=50.,grid_charging=False)]);self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],49.999) + def test_grid_charging_opt_in(self): + p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.,grid_charging=False)]);self.assertTrue(all(x['batteryTargetW']<=1e-5 for x in p['points'])) + +class StoreAndSelectionTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def test_settings_revision_and_shadow_only(self): + s=self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(s['revision'],1) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'family':'3'},0,AT) + with self.assertRaises(ValueError):self.store.save_settings(AID,{'runMode':'live'},1,AT) + def test_request_during_calculation_survives(self): + self.store.request(AID,'manual',AT);claim=self.store.claim(AT);self.store.request(AID,'prices_changed',AT) + p=solve(seq([0.]*3),config_revision=0) + with self.assertRaises(ValueError):self.store.publish_shadow(AID,p,0,AT,claim['sequence'],claim['lease_token']) + self.store.finish(claim);self.assertIsNotNone(self.store.claim(AT)) + def test_month_peak_persistent_and_monotonic(self): + self.store.initialize_peak(AID,'2026-10',20.,'meter_month_register',AT) + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',15.,'meter_month_register',AT) + self.store.close();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'));self.assertEqual(self.store.peaks(AID)['2026-10'],20.) + def test_cap_is_not_valid_peak_source(self): + with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',20.,'configured_limit',AT) + def test_meter_peak_requires_complete_quarter(self): + self.store.initialize_peak(AID,'2026-10',0.,'verified_new_month',AT) + for i in range(3):r=self.store.record_import_interval(AID,AT+timedelta(minutes=5*i),.5,AT+timedelta(hours=1)) + self.assertEqual(r['quarterPeakKw'],6.);self.assertEqual(self.store.peaks(AID)['2026-10'],6.) + def test_shadow_ack_cannot_claim_applied(self): + p=solve(seq([0.]*3),config_revision=0);self.store.publish_shadow(AID,p,0,AT) + with self.assertRaises(ValueError):self.store.acknowledge(AID,p['planId'],0,AT,status='applied') + self.store.acknowledge(AID,p['planId'],0,AT,status='shadow_seen') + def test_family_registry_extensible(self): + r=default_registry();r.register(Family('42','p42','l42','g42','Future'));self.assertEqual(r.get('42').pv,'p42') + def test_sdl_removed_once(self): + self.assertEqual(split_base_load(15000,[3000],2000),10000);self.assertEqual(split_base_load(13000,[3000],2000,True),10000) + with self.assertRaises(ValueError):split_base_load(100,[1000],0) + def test_auto_uses_comparable_cost_and_margin(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT,c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['family'],'23');self.assertEqual(choose_family('auto','3',scores,now=AT,margin_chf=6.)['family'],'3') + def test_auto_no_future_outcomes(self): + scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT+timedelta(hours=1),c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]] + self.assertEqual(choose_family('auto','3',scores,now=AT)['mode'],'collecting') + def test_training_schedule_and_validation(self): + self.assertTrue(training_due(AT-timedelta(days=1),AT));self.assertFalse(training_due(AT-timedelta(days=1),AT,'weekly')) + self.assertFalse(promote_candidate(active_cost=10,candidate_cost=1,valid_coverage=True,no_data_leakage=False,constraints_passed=True)) + +class ServiceIntegrationTest(unittest.TestCase): + def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite')) + def tearDown(self):self.store.close();self.temp.cleanup() + def load(self,values=None,now=AT): + values=values or inputs() + for kind,value in zip(('operation','forecast','tariffs'),values):ingest(self.store,AID,kind,value,now) + def test_pipeline_shadow_plan(self): + self.load();p=run_once(self.store,AT);self.assertTrue(p['executable']);self.assertEqual(p['runMode'],'shadow');self.assertTrue(status(self.store,AID,AT)['fresh']);self.assertEqual(self.store.current(AID)['planId'],p['planId']) + def test_exact_model_selection(self): + self.load();self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(run_once(self.store,AT)['sourceFamily'],'23') + def test_one_tick_not_every_poll(self): + self.load();run_once(self.store,AT);self.assertEqual(run_once(self.store,AT)['status'],'idle') + def test_stale_operation_visible(self): + self.load();p=run_once(self.store,AT+timedelta(minutes=5));self.assertEqual(p['status'],'awaiting_inputs');self.assertIsNone(self.store.current(AID)) + def test_missing_peak_not_invented(self): + op,fc,tar=inputs();op['measuredPeaks']={};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_static_zero_roundtrip(self): + op,fc,tar=inputs();tar['import']['staticChfKwh']=0.;tar['export']['staticChfKwh']=0.;tar['peakChfKwMonth']=0.;self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['cashCostChf'],0.) + def test_immutable_id_conflict(self): + op,fc,tar=inputs();self.load((op,fc,tar));self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'duplicate');op['gridW']=123. + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_older_input_never_overwrites_current(self): + op,fc,tar=inputs();self.load((op,fc,tar));op['observedAt']=(AT-timedelta(seconds=1)).isoformat();op['eventId']='old' + self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'archived_older');self.assertEqual(latest(self.store,AID,'operation')['eventId'],'op1') + def test_unpublished_dynamic_tail_not_executed(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_published_negative_price_and_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(minutes=15)).isoformat(),'value':-2.,'unit':'Rp/kWh','observedAt':AT.isoformat(),'sourceKind':'published_interval'}]} + ingest(self.store,AID,'prices',price,AT);p=run_once(self.store,AT);self.assertEqual(len(p['points']),3);self.assertEqual(p['points'][0]['importPriceChfKwh'],-.02) + def test_carried_price_does_not_extend_horizon(self): + op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar)) + price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(hours=48)).isoformat(),'value':.1,'unit':'CHF/kWh','observedAt':AT.isoformat(),'sourceKind':'carried_forward'}]} + ingest(self.store,AID,'prices',price,AT);self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs') + def test_aggregate_load_warning(self): + op,fc,tar=inputs();fc['families']['3']['loadBasis']='house_total';self.load((op,fc,tar));self.assertTrue(run_once(self.store,AT)['warnings']) + def test_extra_credentials_not_stored(self): + op,fc,tar=inputs();op['password']='do-not-store' + with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT) + def test_http_auth_allowlist_and_revision_conflict(self): + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + path=f'/internal/v2/prognosis/{AID}/planner';headers={'X-Enelix-Service-Token':TOKEN} + self.assertEqual(client.get(path).status_code,401);self.assertEqual(client.get(path,headers=headers).status_code,200) + other='30509683-4569-49e4-848f-4905e4cc813a';self.assertEqual(client.get(path.replace(AID,other),headers=headers).status_code,403) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'23'}}).status_code,200) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'3'}}).status_code,409) + self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400) + def test_body_limit_and_no_legacy_db(self): + with self.assertRaises(ValueError):create_app(str(Path(self.temp.name)/'users.db'),TOKEN,[AID]) + app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + r=client.post(f'/internal/v2/prognosis/{AID}/planner/inputs/forecast',json={'x':'a'*2000001});self.assertEqual(r.status_code,413) + +if __name__=='__main__':unittest.main() diff --git a/tests/NetzfahrplanV4DatenTest.php b/tests/NetzfahrplanV4DatenTest.php new file mode 100644 index 0000000..d4c69b9 --- /dev/null +++ b/tests/NetzfahrplanV4DatenTest.php @@ -0,0 +1,21 @@ +['pipe','r'],1=>['pipe','w'],2=>['pipe','w']],$pipes); + self::assertIsResource($process);fclose($pipes[0]); + $out=stream_get_contents($pipes[1]);$err=stream_get_contents($pipes[2]); + fclose($pipes[1]);fclose($pipes[2]); + self::assertSame(0,proc_close($process),$out.$err); + self::assertStringContainsString('TOTAL 21 application-data checks passed',$out); + } +} diff --git a/tests/V4ApplicationData/checks.php b/tests/V4ApplicationData/checks.php new file mode 100644 index 0000000..5f018e9 --- /dev/null +++ b/tests/V4ApplicationData/checks.php @@ -0,0 +1,83 @@ +2,'VariableUpdated'=>$sensor[$id]['updated'],'VariableChanged'=>$sensor[$id]['changed']];} +function IPS_GetObject($id):array {return ['ParentID'=>200,'ObjectIdent'=>'s'.$id];} +function GetValue($id) {global $sensor;return $sensor[$id]['value'];} +foreach(['CURLOPT_POST','CURLOPT_RETURNTRANSFER','CURLOPT_FOLLOWLOCATION','CURLOPT_CONNECTTIMEOUT','CURLOPT_TIMEOUT','CURLOPT_SSL_VERIFYPEER','CURLOPT_SSL_VERIFYHOST','CURLOPT_HTTPHEADER','CURLOPT_POSTFIELDS','CURLINFO_HTTP_CODE'] as $i=>$k)if(!defined($k))define($k,$i+100); +function curl_init($url){$o=new \stdClass();$o->url=$url;return $o;} +function curl_setopt_array($h,$options){$h->options=$options;return true;} +function curl_exec($h){global $sent,$mode;$p=json_decode($h->options[CURLOPT_POSTFIELDS],true);$sent[]=$p; + return json_encode(['status'=>'stored','datasetId'=>$p['datasetId'],'acceptedThrough'=>$mode==='wrong_ack'?'2000-01-01T00:00:00+00:00':end($p['records'])['capturedAt']]);} +function curl_getinfo($h,$info){global $mode;return $mode==='429'?429:200;} +function curl_close($h):void {} +class Lizenzpruefung {const NETZFAHRPLAN='test-only';} +class FakeManager { + use ManagerNetzfahrplanV4DatenTrait; + public int $InstanceID=17004;public array $p=[],$a=[],$vars=[],$timers=[];public bool $licensed=true; + public function boot():void {$this->registriereNetzfahrplanV4Daten();} + public function configure():void {$this->konfiguriereNetzfahrplanV4Daten();} + public function folder():string {return $this->v4DatenVerzeichnis();} + public function send():array {return $this->v4DatenUebertragen($this->folder());} + public function RegisterPropertyBoolean($k,$v){$this->p[$k]??=$v;} + public function RegisterPropertyString($k,$v){$this->p[$k]??=$v;} + public function RegisterAttributeString($k,$v){$this->a[$k]??=$v;} + public function RegisterAttributeInteger($k,$v){$this->a[$k]??=$v;} + public function RegisterVariableString($k,...$args){$this->vars[$k]='';} + public function RegisterTimer($k,$v,$code){$this->timers[$k]=$v;} + public function SetTimerInterval($k,$v){$this->timers[$k]=$v;} + public function ReadPropertyBoolean($k){return $this->p[$k];} + public function ReadPropertyString($k){return $this->p[$k];} + public function ReadAttributeString($k){return $this->a[$k];} + public function ReadAttributeInteger($k){return $this->a[$k];} + public function WriteAttributeString($k,$v){$this->a[$k]=$v;} + public function WriteAttributeInteger($k,$v){$this->a[$k]=$v;} + public function SetValue($k,$v){if($k!=='NetzfahrplanV4Datenstatus')throw new \RuntimeException('Unexpected variable write');$this->vars[$k]=$v;} + private function berechtigungLizenziert($k){return $this->licensed;} +} +$c=['version'=>1,'installationId'=>'e3a08f9e-af12-4695-99bd-8b51c0520021','managerId'=>17004, + 'reportedInventorySha256'=>str_repeat('a',64),'accounting'=>['version'=>1,'maxSkewSeconds'=>30,'splitToleranceW'=>100.,'sources'=>[]],'extraSources'=>[]]; +foreach(['grid','pv','physical_storage','controlled_virtual','external_virtual'] as $i=>$role){$id=101+$i; + $c['accounting']['sources'][]=['key'=>'s'.$id,'role'=>$role,'variableId'=>$id,'parentId'=>200,'ident'=>'s'.$id,'factorToW'=>1,'maxAgeSeconds'=>60,'dependsOn'=>[]]; + $sensor[$id]=['value'=>[3000,5000,2000,2000,0][$i],'updated'=>$clock,'changed'=>$clock];} +$m=new FakeManager();$m->boot();$m->a['LizenzInstallationID']=$c['installationId'];$m->a['PrognoseInstallationsToken']='synthetic-not-a-real-token'; +try { + $m->ErfasseNetzfahrplanV4Daten();check($sent===[],'disabled by default makes no network request'); + $m->p['NetzfahrplanV4MessdatenAktiv']=true;$m->p['NetzfahrplanV4Datensatz']='physical-v1';$m->p['NetzfahrplanV4Messkonfiguration']=json_encode($c); + $m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===30000,'only application acquisition timer enabled'); + $m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'native capture delivered to existing device API'); + $cursor=$m->a['NetzfahrplanV4DatenCursor'];check(json_decode($cursor,true)['offset']>0,'matching ack commits durable byte cursor'); + check($sent[0]['records'][0]['raw']['s101']['value']===3000,'physical reading retained'); + check($sent[0]['records'][0]['raw']['s101']['sourceUpdatedAt']===$clock,'original timestamp retained'); + check(!str_contains(json_encode($sent),'synthetic-not-a-real-token'),'token never included in data payload'); + check($sent[0]['records'][0]['controlEligible']===false,'data delivery is not actuator permission'); + $clock+=30;$m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'30-second collection does not double request rate'); + check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'unsent rows remain behind cursor'); + $clock+=30;$mode='429';$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'429 never advances acknowledgement cursor'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['dataRetained'],'offline data retained'); + $mode='wrong_ack';$clock+=120;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'wrong ack timestamp rejected'); + $mode='ok';$clock+=240;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']!==$cursor,'later success sends the preserved backlog'); + check(count(end($sent)['records'])===4,'all previously unacknowledged rows included once'); + $file=glob($m->folder().'/raw-*.jsonl')[0];$count=count(file($file));check($count===5,'raw journal never cleared by upload'); + $lastCursor=$m->a['NetzfahrplanV4DatenCursor'];file_put_contents($file,'incomplete',FILE_APPEND); + check($m->send()['status']==='up_to_date','in-progress tail not transmitted');check($m->a['NetzfahrplanV4DatenCursor']===$lastCursor,'partial tail not acknowledged'); + $m->p['NetzfahrplanV4MessdatenAktiv']=false;$m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===0,'disable affects acquisition only'); + check(count($m->vars)===1,'only own manager data-status variable written'); + $text=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4DatenTrait.php'); + check(!preg_match('/\b(?:RequestAction\(|IPS_ApplyChanges\(|MC_Reload\(|schreibeRegister\()/',$text),'data path contains no actuator/module reload calls'); + echo "TOTAL $checks application-data checks passed; mocked IPS and HTTP only.\n"; +} finally { + foreach(new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($base,\FilesystemIterator::SKIP_DOTS),\RecursiveIteratorIterator::CHILD_FIRST)as$p){$p->isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($base); +}