diff --git a/docs/netplan-v4-unified-release.md b/docs/netplan-v4-unified-release.md new file mode 100644 index 0000000..09bd7e0 --- /dev/null +++ b/docs/netplan-v4-unified-release.md @@ -0,0 +1,77 @@ +# V4 Unified RC1 - actual scope and release boundary + +This release consolidates application data recovery, native delivery status, economic +family comparison, and optional verified archival. It is NOT a production actuation +release. No claim that all original product requirements or hardware acceptance are +complete. Sources are in the existing develop branch, not feature branches. + +## Delivered code + +* Audited recovery for the exact 100.0/100 splitToleranceW serialization defect. + The internal operator endpoint validates the two complete JSON representations, + their exact SHA256, installation, inventory, and every field. Only that numeric + representation may differ. Device-supplied arbitrary aliases are never accepted. + Existing rows/measurements are not rewritten. The original received mapping and + proof reference are recorded with the observation. A mixed 45/75 block and retries + are regression tested. Adoption requires an explicit deployment flag. +* Native normalization prevents recurrence. The sender preserves a separate last + successful receipt even during scheduled/backoff states, reports safe numeric HTTP + status (no response or credential text), and continues trying to deliver a durable + prefix if the current capture cannot be appended. No cursor reset or skipped block. +* Economic comparison is actually connected, not an empty score list. At the start + of a local calendar day the worker freezes comparable plans for families 3/13/23 + with the then-known forecasts, prices and common battery/limit state. Completed + days are evaluated against subsequently acquired physical residual data. Initial + energy, losses, common terminal energy valuation and 15-minute demand charge are + included; monthly maxima are charged once over a comparison period, not per day. + GUI lookback/minimum days/coverage/switch margin are applied by the selector. +* v1 economic scope: one grid-charge-enabled battery, frozen day-ahead plans, with + local grid-target tracking simulated during the day. It is NOT a full receding- + horizon replay. Historical external SDL uses the then-recorded SDL request and is + clearly marked as an estimate, not independently measured delivery. Missing + actuals, unmatched contexts and breaches cannot become zero cost or a successful + comparison. Broader device topologies stay collecting; no fake equivalence. +* Lossless archive implementation and tests are present for acknowledged old native + outbox days and old server observations/plans. Archives are verified before plain + rows/files are retired. Active plans, acknowledged plans and unacknowledged data + are excluded. Archiving is OFF by default; enable only with an explicit storage + policy (native NetzfahrplanV4ArchivAktiv; server NETPLAN_V4_ARCHIVE_ENABLED=1). + Compressed archives are retained: this is not an external backup or infinite disk. + +## Deployment is not data/model/actuator acceptance + +Server entry: commissioning/deploy_unified_release.py. Default checks sources only. +--apply builds and tests a separate candidate context with no network/production +mounts before replacing the V4 service. It preserves the running plant allowlist, +backs up SQLite consistently, and uses a pinned built image. Only the V4 planner JS +asset is updated; unrelated GUI work, portal runtime, tariffs, forecasting, Symcon +and actuators are untouched. Rollback restores previous source/image; additive audit +rows remain. Database schema changes do not overwrite old observations. + +--approve-numeric-mapping-compatibility explicitly authorizes the demonstrated pair +of equivalent configuration representations. Without it, strict original validation +remains and backlog containing the legacy serialization is still rejected. + +Lihrenmoos native stage: /srv/agent/netplan-v4-unified-release/install.php. +This patches three data-only libraries, reloads the existing library, compares all +pre-existing properties and lets the existing data timer resume. No raw replay, +force-skip or actuator callback from the installer. Archiving stays disabled by default. + +## Remaining product work - do not label as complete + +1. The full production dispatcher is not installed. Earlier controlled-trial code + still has bounded pilot grants; continuous production authority and complete + corrected physical/virtual feedback integration are NOT delivered by this RC. +2. The actual host/container/kernel and field acceptance of this release are not + replaced by CLI mocks. Real source validity and independent command-loss behavior + are not asserted by a measurement algebra or an override. +3. The replay limitation above must be accepted as a beta feature, or extended to + continuous rolling reoptimization and additional asset types before advertising + the original full automatic-selection requirement. +4. Native multi-plant commissioning, external backups, retained archive lifecycle, + model quality on field data and publication authentication remain operational + release criteria. The tests do not demonstrate actual cost savings. + +Do not re-run obsolete piecemeal installers after this release, bypass their source +checks, turn the previous grid schedule on, or turn 'productionReady' true manually. +The retained EV capacity is 161.44 kWh / 39 kW; SDL reserve is already outside EV. diff --git a/examples/V4UnifiedRelease/file_installer.php b/examples/V4UnifiedRelease/file_installer.php new file mode 100644 index 0000000..b19ffe3 --- /dev/null +++ b/examples/V4UnifiedRelease/file_installer.php @@ -0,0 +1,35 @@ +'already_installed']; + foreach($names as$name)if(($old[$name]===null?null:hash('sha256',$old[$name]))!==$m['files'][$name]['before'])throw new RuntimeException('Teilstand zuerst pruefen.'); + $backup=$stage.'/backups/'.gmdate('YmdTHis').'-'.bin2hex(random_bytes(4)); + if(!mkdir($backup.'/libs',0700,true))throw new RuntimeException('Sicherung fehlt.'); + foreach($names as$name)if($old[$name]!==null)v4UnifiedWrite($backup.'/'.$name,$old[$name],0600); + v4UnifiedWrite($backup.'/MANIFEST.json',json_encode($m,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0600); + $written=[]; + try {foreach($names as$name){$p=$target.'/'.$name;if((is_file($p)?file_get_contents($p):null)!==$old[$name])throw new RuntimeException('Parallele Dateiaenderung.');v4UnifiedWrite($p,$new[$name]);$written[]=$name;}} + catch(Throwable $e){foreach(array_reverse($written)as$name){$p=$target.'/'.$name;if(hash_file('sha256',$p)!==$m['files'][$name]['after'])continue;if($old[$name]===null)unlink($p);else v4UnifiedWrite($p,$old[$name]);}throw $e;} + return ['status'=>'installed','backup'=>$backup]; +} diff --git a/examples/V4UnifiedRelease/install.php b/examples/V4UnifiedRelease/install.php new file mode 100644 index 0000000..6062f0c --- /dev/null +++ b/examples/V4UnifiedRelease/install.php @@ -0,0 +1,33 @@ +'unified_native_data_only','startedAt'=>gmdate('c'),'newActuatorPermission'=>false,'cursorReset'=>false,'outboxRewritten'=>false]; +try { + $package=json_decode(file_get_contents(__DIR__.'/PACKAGE_HASHES.json'),true,64,JSON_THROW_ON_ERROR); + foreach($package as$name=>$hash)if(str_contains($name,'..')||str_starts_with($name,'/')||is_link(__DIR__.'/'.$name)||hash_file('sha256',__DIR__.'/'.$name)!==$hash)throw new RuntimeException('Installationspaket wurde geaendert.'); + if(!IPS_InstanceExists($manager)||IPS_GetInstance($manager)['ModuleInfo']['ModuleID']!=='{6F771B18-59D4-4C8A-B951-3B2FE9F6A2C4}')throw new RuntimeException('Erwarteter Manager fehlt.'); + if(IPS_HasChanges($manager))throw new RuntimeException('Offene Manageraenderungen zuerst speichern oder verwerfen.'); + if(IPS_GetProperty($manager,'NetzfahrplanAktiv')!==false||IPS_GetProperty($manager,'NetzfahrplanV4MessdatenAktiv')!==true)throw new RuntimeException('Erwartet: Datenaufnahme EIN, alter Netzfahrplan AUS.'); + $controls=[];foreach(IPS_GetModuleList()as$mid)if((IPS_GetModule($mid)['Prefix']??'')==='MC')$controls=array_merge($controls,IPS_GetInstanceListByModuleID($mid)); + if(count($controls)!==1||!function_exists('MC_ReloadModule'))throw new RuntimeException('Module Control nicht eindeutig.'); + $lock=IPS_SemaphoreEnter('ENELIX.V4.UnifiedRelease',3000);if(!$lock)throw new RuntimeException('Installation bereits aktiv.'); + $before=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + $report['files']=v4UnifiedFiles(__DIR__,'/var/lib/symcon/modules/Enelix-EMS'); + // No staged application classes are loaded into this script context. + if(MC_ReloadModule($controls[0],'Enelix-EMS')===false)throw new RuntimeException('Bibliotheks-Reload fehlgeschlagen.'); + $after=json_decode(IPS_GetConfiguration($manager),true,128,JSON_THROW_ON_ERROR); + foreach($before as$key=>$value)if(!array_key_exists($key,$after)||$after[$key]!==$value)throw new RuntimeException('Bestehender Parameter unerwartet geaendert: '.$key); + $report['status']='native_data_candidate_installed';$report['parametersUnchanged']=true; + echo "OK: Einheitliche Messdatenkennung und dauerhafte Versandbestaetigung installiert. Archivierung vorbereitet, standardmaessig AUS.\n"; + echo "Der bestehende Datentimer setzt den Versand fort. Sendezeiger und Originalbestand wurden nicht umgeschrieben.\n"; + echo "Keine V4-Stellfreigabe und kein Wechsel der Prognosequelle.\n"; +} catch(Throwable $e) { + $report['status']='needs_review';$report['reason']=substr($e->getMessage(),0,250); + echo 'FEHLER: '.$report['reason']."\nKeine neue Stellfreigabe.\n"; +} finally { + $report['finishedAt']=gmdate('c');v4UnifiedWrite(__DIR__.'/INSTALL_RESULT.json',json_encode($report,JSON_THROW_ON_ERROR|JSON_PRETTY_PRINT),0644); + if($lock)IPS_SemaphoreLeave('ENELIX.V4.UnifiedRelease'); + echo 'UNIFIED MANAGER REPORT: '.__DIR__."/INSTALL_RESULT.json\n"; +} diff --git a/libs/ManagerNetzfahrplanV4DatenTrait.php b/libs/ManagerNetzfahrplanV4DatenTrait.php index 16001c6..a718525 100644 --- a/libs/ManagerNetzfahrplanV4DatenTrait.php +++ b/libs/ManagerNetzfahrplanV4DatenTrait.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Belevo\EnelixEMS; require_once __DIR__ . '/NetzfahrplanV4Messaufnahme.php'; +require_once __DIR__ . '/NetzfahrplanV4Datenarchiv.php'; /** Application measurement acquisition + acknowledged durable delivery. No device writes. */ trait ManagerNetzfahrplanV4DatenTrait @@ -11,9 +12,13 @@ trait ManagerNetzfahrplanV4DatenTrait private function registriereNetzfahrplanV4Daten(): void { $this->RegisterPropertyBoolean('NetzfahrplanV4MessdatenAktiv', false); + $this->RegisterPropertyBoolean('NetzfahrplanV4ArchivAktiv', false); $this->RegisterPropertyString('NetzfahrplanV4Messkonfiguration', '{}'); $this->RegisterPropertyString('NetzfahrplanV4Datensatz', ''); $this->RegisterAttributeString('NetzfahrplanV4DatenCursor', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4LetzteDatenquittung', '{}'); + $this->RegisterAttributeString('NetzfahrplanV4ArchivTag', ''); + $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersion', 0); $this->RegisterAttributeInteger('NetzfahrplanV4DatenVersuch', 0); $this->RegisterAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); $this->RegisterVariableString('NetzfahrplanV4Datenstatus', 'V4 Messdaten und Modellversorgung', '', 199); @@ -27,6 +32,12 @@ trait ManagerNetzfahrplanV4DatenTrait if (!$this->ReadPropertyBoolean('NetzfahrplanV4MessdatenAktiv')) return; try { $this->v4DatenKonfiguration(); + if ($this->ReadAttributeInteger('NetzfahrplanV4DatenVersion')<1) { + // One retry after the data-only upgrade, never a cursor reset or a device write. + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl', 0); + $this->WriteAttributeInteger('NetzfahrplanV4DatenVersion', 1); + } $this->SetTimerInterval('NetzfahrplanV4DatenErfassen', 30000); } catch (\Throwable $e) { $this->v4DatenStatus(['status'=>'configuration_error','errorType'=>get_class($e)]); @@ -60,6 +71,8 @@ trait ManagerNetzfahrplanV4DatenTrait private function v4DatenStatus(array $data): void { + $receipt=json_decode($this->ReadAttributeString('NetzfahrplanV4LetzteDatenquittung'),true,32,JSON_THROW_ON_ERROR); + $data['lastAcknowledgement']=$receipt; $this->SetValue('NetzfahrplanV4Datenstatus',json_encode($data+['checkedAt'=>gmdate('c'), 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz'),'controlEnabled'=>false],JSON_THROW_ON_ERROR)); } @@ -78,11 +91,25 @@ trait ManagerNetzfahrplanV4DatenTrait return ['value'=>GetValue($id),'updated'=>(int)$v['VariableUpdated'],'changed'=>(int)$v['VariableChanged'], 'parentID'=>$o['ParentID'],'ident'=>$o['ObjectIdent']]; }; - $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); $dir=$this->v4DatenVerzeichnis(); - NetzfahrplanV4Messaufnahme::append($dir,$r); - $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), - 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + $archiveResult=null; + if ($this->ReadPropertyBoolean('NetzfahrplanV4ArchivAktiv') && $this->ReadAttributeString('NetzfahrplanV4ArchivTag')!==gmdate('Y-m-d')) { + try { + $archiveCursor=json_decode($this->ReadAttributeString('NetzfahrplanV4DatenCursor'),true,32,JSON_THROW_ON_ERROR); + $archiveResult=NetzfahrplanV4Datenarchiv::run($dir,$archiveCursor,time()); + if ($archiveResult['archived']<2 && ($archiveResult['status']??null)!=='writer_busy')$this->WriteAttributeString('NetzfahrplanV4ArchivTag',gmdate('Y-m-d')); + } catch (\Throwable $e) { $archiveResult=['status'=>'archive_error','errorType'=>get_class($e),'originalsRetained'=>true]; } + } + try { + $r=NetzfahrplanV4Messaufnahme::capture($c,$reader,static fn():int=>time()); + NetzfahrplanV4Messaufnahme::append($dir,$r); + $result=['status'=>'recorded','capturedAt'=>$r['capturedAt'],'sourceCount'=>count($r['raw']), + 'rawIssues'=>count($r['issues']),'measurementBoundaryVerified'=>false]; + } catch (\Throwable $e) { + // Storage/capture failure must not prevent retrying the already durable outbox. + $result=['status'=>'capture_error','errorType'=>get_class($e),'dataRetained'=>true]; + } + if ($archiveResult!==null)$result['archive']=$archiveResult; $last=$this->ReadAttributeInteger('NetzfahrplanV4DatenVersuch'); $failures=min(5,$this->ReadAttributeInteger('NetzfahrplanV4DatenFehlerzahl')); $retry=min(900,60*(2**$failures)); @@ -90,10 +117,16 @@ trait ManagerNetzfahrplanV4DatenTrait $this->WriteAttributeInteger('NetzfahrplanV4DatenVersuch',time()); try { $result['delivery']=$this->v4DatenUebertragen($dir); + if (($result['delivery']['status']??null)==='acknowledged') { + $this->WriteAttributeString('NetzfahrplanV4LetzteDatenquittung',json_encode($result['delivery']+['receivedAt'=>gmdate('c')],JSON_THROW_ON_ERROR)); + } $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',0); } catch (\Throwable $e) { $this->WriteAttributeInteger('NetzfahrplanV4DatenFehlerzahl',$failures+1); - $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e),'dataRetained'=>true]; + $httpCode=(int)$e->getCode(); + $result['delivery']=['status'=>'retry_pending','errorType'=>get_class($e), + 'httpStatus'=>$httpCode>=100&&$httpCode<=599?$httpCode:null,'dataRetained'=>true, + 'nextAttemptAt'=>gmdate('c',time()+min(900,60*(2**min(5,$failures+1))))]; } } else $result['delivery']=['status'=>'scheduled','nextAttemptAt'=>gmdate('c',$last+$retry)]; $this->v4DatenStatus($result); @@ -147,13 +180,14 @@ trait ManagerNetzfahrplanV4DatenTrait CURLOPT_HTTPHEADER=>['Content-Type: application/json','Accept: application/json','Authorization: Bearer '.$token], CURLOPT_POSTFIELDS=>$payload]); $answer=curl_exec($h);$code=(int)curl_getinfo($h,CURLINFO_HTTP_CODE); - if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged'); + if ($answer===false||$code!==200||strlen($answer)>65536) throw new \RuntimeException('Measurement delivery not acknowledged',$code); $ack=json_decode($answer,true,32,JSON_THROW_ON_ERROR); if (!in_array($ack['status']??null,['stored','duplicate'],true) ||($ack['datasetId']??null)!==$this->ReadPropertyString('NetzfahrplanV4Datensatz') ||($ack['acceptedThrough']??null)!==$lastTime) throw new \RuntimeException('Measurement acknowledgement does not match batch'); $this->WriteAttributeString('NetzfahrplanV4DatenCursor',json_encode($candidate,JSON_THROW_ON_ERROR)); - return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime]; + return ['status'=>'acknowledged','records'=>count($records),'acceptedThrough'=>$lastTime, + 'datasetId'=>$this->ReadPropertyString('NetzfahrplanV4Datensatz')]; } finally { curl_close($h); } } } diff --git a/libs/NetzfahrplanV4Datenarchiv.php b/libs/NetzfahrplanV4Datenarchiv.php new file mode 100644 index 0000000..3211530 --- /dev/null +++ b/libs/NetzfahrplanV4Datenarchiv.php @@ -0,0 +1,78 @@ +0]; + if (!is_string($day)||!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$day) + ||!is_int($cursor['offset']??null)||$cursor['offset']<0 + ||is_link($directory)||realpath($directory)!==$directory) { + throw new \RuntimeException('Invalid archive cursor/directory'); + } + $cutoff='raw-'.gmdate('Ymd',$now-7*86400).'.jsonl'; + $archive=$directory.'/archive'; + if (is_link($archive)) throw new \RuntimeException('Archive symlink refused'); + if (!is_dir($archive)&&!mkdir($archive,0700)) throw new \RuntimeException('Archive directory unavailable'); + if ((fileperms($archive)&0007)!==0) throw new \RuntimeException('Archive must remain private'); + $lockPath=$directory.'/.writer.lock'; + if (is_link($lockPath)) throw new \RuntimeException('Writer lock symlink refused'); + $lock=fopen($lockPath,'c+b'); + if ($lock===false) throw new \RuntimeException('Archive lock unavailable'); + $done=[]; + try { + if (!flock($lock,LOCK_EX|LOCK_NB)) return ['archived'=>0,'status'=>'writer_busy']; + foreach (glob($directory.'/raw-*.jsonl')?:[] as $file) { + $name=basename($file); + if (!preg_match('/^raw-[0-9]{8}\.jsonl$/D',$name)||$name>=$day||$name>=$cutoff) continue; + if (is_link($file)||realpath($file)!==$file||!is_file($file)) throw new \RuntimeException('Unexpected journal'); + $before=stat($file); + if (!$before||$before['size']>67108864) throw new \RuntimeException('Archive input bounds'); + $digest=hash_file('sha256',$file); + if (!is_string($digest)) throw new \RuntimeException('Cannot hash journal'); + $target=$archive.'/'.$name.'.'.$digest.'.gz'; + if (is_link($target)) throw new \RuntimeException('Archive target symlink'); + if (!is_file($target)) { + $tmp=tempnam($archive,'.pending-'); + if ($tmp===false) throw new \RuntimeException('Cannot create archive'); + try { + chmod($tmp,0600);$in=fopen($file,'rb');$gz=gzopen($tmp,'wb6'); + if ($in===false||$gz===false) {if(is_resource($in))fclose($in);if(is_resource($gz))gzclose($gz);throw new \RuntimeException('Cannot open archive streams');} + try { + while (!feof($in)) { + $block=fread($in,65536); + if ($block===false) throw new \RuntimeException('Archive read failed'); + $offset=0; + while($offset$before['size'])throw new \RuntimeException('Oversized decompressed archive');hash_update($hash,$block);}} + finally {gzclose($gz);} + clearstatcache(true,$file);$after=stat($file); + if ($bytes!==$before['size']||hash_final($hash)!==$digest||!$after + ||$after['size']!==$before['size']||$after['ino']!==$before['ino']||$after['mtime']!==$before['mtime'] + ||hash_file('sha256',$file)!==$digest) throw new \RuntimeException('Archive mismatch; original retained'); + $directoryHandle=fopen($archive,'r'); + if ($directoryHandle===false) throw new \RuntimeException('Cannot sync archive directory'); + try {if(!fsync($directoryHandle))throw new \RuntimeException('Archive directory sync failed');} finally {fclose($directoryHandle);} + if (!unlink($file)) throw new \RuntimeException('Verified original could not be retired'); + $done[]=$name; + if (count($done)>=2) break; + } + } finally {flock($lock,LOCK_UN);fclose($lock);} + return ['archived'=>count($done),'compressedOriginalsRetained'=>true,'unacknowledgedDaysTouched'=>false]; + } +} diff --git a/services/netplan-v4/Dockerfile b/services/netplan-v4/Dockerfile index ceff456..33e5701 100644 --- a/services/netplan-v4/Dockerfile +++ b/services/netplan-v4/Dockerfile @@ -11,6 +11,7 @@ COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py app COPY acceptance ./acceptance COPY commissioning/deploy_application.py ./commissioning/deploy_application.py COPY commissioning/deploy_history_timing.py ./commissioning/deploy_history_timing.py +COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py # Host sources can be 0600/0700. COPY makes them root-owned. # Normalize only packaged application code; never change host secrets or sockets. RUN find /app -type d -exec chmod 0755 {} + \ diff --git a/services/netplan-v4/SOURCE_MANIFEST.json b/services/netplan-v4/SOURCE_MANIFEST.json index 34ed6eb..9cbe60b 100644 --- a/services/netplan-v4/SOURCE_MANIFEST.json +++ b/services/netplan-v4/SOURCE_MANIFEST.json @@ -1,7 +1,7 @@ { "sourceHashes": { ".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", - "Dockerfile": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", + "Dockerfile": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33", "acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", "acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", "acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", @@ -42,13 +42,15 @@ "commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", "commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", "commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "commissioning/deploy_unified_release.py": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", "compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", "compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", "deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", "forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", "gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", "gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", - "gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "gui/netplan-v4.js": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9", "install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", "integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", "integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", @@ -57,16 +59,20 @@ "netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", "netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", "netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "netplan_v4/economic_replay.py": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302", "netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", - "netplan_v4/measurement_pipeline.py": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", + "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "netplan_v4/mapping_identity.py": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d", + "netplan_v4/measurement_pipeline.py": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f", "netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", "netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", "netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", "netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", "netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "netplan_v4/retention.py": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3", "netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", - "netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", - "netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "netplan_v4/service.py": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f", + "netplan_v4/store.py": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2", "release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", "requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", "run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", @@ -79,22 +85,22 @@ "tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", "tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", "tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "tests/test_economic_replay.py": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3", "tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", "tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", "tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "tests/test_mapping_recovery.py": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e", "tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", "tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", "tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", "tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", - "tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", - "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", - "netplan_v4/history_timing.py": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", - "tests/test_history_timing.py": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", - "tests/test_history_timing_deployment.py": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", - "commissioning/deploy_history_timing.py": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", - "commissioning/history-timing-source/dataset.json": "671216e1cc04148f4f553103077bcb4500bf0120cb74780703bfcf26cd1121d3", - "commissioning/history-timing-source/RELEASE.json": "14cb690a357c40f698c2e9b4190933318396b9c08e15cc4ad9f9f4439d6054ca" + "tests/test_release_preflight.py": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68", + "tests/test_retention.py": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51", + "tests/test_unified_release.py": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236", + "tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" }, - "packagedAt": "2026-10-02T21:00:35.231591+00:00", - "runtimeChanged": false + "scope": "application_source_only", + "productionReady": false } diff --git a/services/netplan-v4/commissioning/deploy_unified_release.py b/services/netplan-v4/commissioning/deploy_unified_release.py new file mode 100644 index 0000000..edf2f93 --- /dev/null +++ b/services/netplan-v4/commissioning/deploy_unified_release.py @@ -0,0 +1,124 @@ +"""Install the reviewed V4 release candidate as one transaction-like deployment. + +Tests run in an isolated candidate image BEFORE replacing any running service. +No device control permission, forecast-source switch, telemetry rewrite or cursor +jump. Exactly proven numeric representations are registered for backlog recovery. +""" +from datetime import datetime, timezone +from pathlib import Path +import argparse,hashlib,json,os,subprocess,sys,tempfile + +ROOT=Path(__file__).resolve().parents[1] +PACKAGE=ROOT/'commissioning/unified-release' +GUI=ROOT.parent/'license/public/netplan-v4.js' + + +def digest(p): + if p.is_symlink():raise ValueError('Symlink refused') + return hashlib.sha256(p.read_bytes()).hexdigest() if p.is_file() else None + + +def verify(): + m=json.loads((PACKAGE/'RELEASE.json').read_text()) + if m.get('scope')!='unified_v4_candidate' or not m.get('files'):raise ValueError('Unexpected release') + if digest(Path(__file__).resolve())!=m['installerSha256']:raise ValueError('Installer changed') + for n,v in m['files'].items(): + if Path(n).is_absolute() or '..' in Path(n).parts:raise ValueError('Unsafe release path') + if digest(PACKAGE/'context'/n)!=v['after']:raise ValueError('Candidate drift: '+n) + if digest(ROOT/n) not in (v['before'],v['after']):raise ValueError('Concurrent runtime source edit: '+n) + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent GUI edit') + if digest(PACKAGE/'mapping-proof.json')!=m['proofSha256']:raise ValueError('Compatibility proof changed') + if digest(ROOT/'commissioning/deploy_application.py')!=m['backupHelperSha256']:raise ValueError('Backup helper changed') + return m + + +BOOTSTRAP=r'''import json,os,sys,urllib.request +x=json.load(sys.stdin);base='http://127.0.0.1:9100' +health=json.load(urllib.request.urlopen(base+'/health',timeout=5)) +assert health.get('applicationRelease')=='unified-rc1' and health.get('liveEnabled') is False +headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'} +url=base+'/internal/v2/prognosis/'+x['plant']+'/planner' +def get():return json.load(urllib.request.urlopen(urllib.request.Request(url,headers=headers),timeout=20)) +before=get() +receipt={'status':'not_registered_requires_explicit_approval'} +if x['approveMapping']: + request=urllib.request.Request(url+'/datasets/'+x['dataset']+'/mapping-compatibility',data=json.dumps(x['proof']).encode(),headers=headers,method='PUT') + receipt=json.load(urllib.request.urlopen(request,timeout=15)) +after=get();assert before['settings']==after['settings'] and after['liveEnabled'] is False +print(json.dumps({'health':health,'mappingRecovery':receipt,'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'],'economicReplayConnected':after['economicComparison']['connected'],'dataSets':[{'datasetId':d['datasetId'],'records':d['records'],'lastCapture':d['lastCapture'],'lastReceived':d.get('lastReceived'),'status':d['status']} for d in after['dataPipeline']['datasets']]})) +''' + + +def run(plant,apply=False,approve_mapping=False): + m=verify() + if plant!=m['installationId']:raise ValueError('Wrong prepared installation') + if not apply: + print('UNIFIED RELEASE SOURCE CHECK PASSED; no installation.');return + if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker socket permissions') + from deploy_application import atomic,backup_database + folder=ROOT/'unified-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ');folder.mkdir(parents=True,mode=0o700) + report={'scope':'unified_v4_candidate','releaseId':m['releaseId'],'startedAt':datetime.now(timezone.utc).isoformat(),'liveEnabled':False,'productionCommissioned':False,'steps':[]} + compose=['docker','compose','-f',str(ROOT/'compose.yaml')];env=dict(os.environ) + def cmd(args,timeout=180,capture=False,data=None): + return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data,stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None) + changed={};old_gui=None;new_gui=None;replaced=False;old_image=None + try: + # Read only this non-secret allowlist; never print or persist the complete environment. + env['NETPLAN_V4_PLANTS']=plant + ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split() + if len(ids)!=1:raise ValueError('Expected existing V4 service') + allowed=cmd(['docker','exec',ids[0],'python','-c',"import os;print(os.getenv('NETPLAN_V4_PLANTS',''))"],capture=True).stdout.strip() + if plant not in allowed.split(','):raise ValueError('Installation not in running allowlist') + env['NETPLAN_V4_PLANTS']=allowed + old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip();report['previousImage']=old_image + tag='enelix-netplan-v4:rc-'+m['releaseId'][:16] + cmd(['docker','build','-t',tag,str(PACKAGE/'context')],timeout=900) + candidate=cmd(['docker','image','inspect','--format','{{.Id}}',tag],capture=True).stdout.strip() + cmd(['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--tmpfs','/tmp:rw,nosuid,nodev,noexec,size=256m','--entrypoint','python',candidate,'/app/run_tests.py'],timeout=600) + cmd(['node','--test',str(PACKAGE/'context/tests/portal.test.mjs')]);cmd(['node','--check',str(PACKAGE/'context/gui/netplan-v4.js')]) + report['steps'].append('isolated_candidate_tests_passed');verify() + backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite');report['steps'].append('consistent_backup') + for name,v in m['files'].items(): + target=ROOT/name;after=(PACKAGE/'context'/name).read_bytes();before=target.read_bytes() if target.is_file() else None + if before==after:continue + if digest(target)!=v['before']:raise ValueError('Source changed during tests') + backup=folder/'source-before'/name;backup.parent.mkdir(parents=True,exist_ok=True) + if before is not None:backup.write_bytes(before) + target.parent.mkdir(parents=True,exist_ok=True);atomic(target,after);changed[name]=(before,after) + override=folder/'candidate.yaml';override.write_text('services:\n netplan-v4:\n image: '+candidate+'\n') + rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n') + replaced=True;cmd(compose+['-f',str(override),'up','-d','--no-deps','--no-build','--wait','netplan-v4'],timeout=180) + data=json.dumps({'plant':plant,'dataset':'lihrenmoos-physical-v1','proof':json.loads((PACKAGE/'mapping-proof.json').read_text()),'approveMapping':approve_mapping}) + result=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=data) + report['application']=json.loads(result.stdout) + old_gui=GUI.read_bytes();new_gui=(PACKAGE/'context/gui/netplan-v4.js').read_bytes() + if digest(GUI) not in (m['guiBefore'],m['files']['gui/netplan-v4.js']['after']):raise ValueError('Concurrent portal UI edit') + (folder/'gui-before.js').write_bytes(old_gui);atomic(GUI,new_gui) + report['steps'].append('audited_backlog_compatibility_and_application_verified');report['candidateImage']=candidate + report['status']='unified_candidate_installed_no_actuation' + except Exception as exc: + report['status']='needs_review';report['errorType']=type(exc).__name__ + if old_gui is not None and GUI.read_bytes()==new_gui:atomic(GUI,old_gui) + for name,(before,after) in reversed(list(changed.items())): + target=ROOT/name + if target.read_bytes()!=after:continue + if before is None:target.unlink() + else:atomic(target,before) + if replaced and old_image: + try: + cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4']) + report['rollback']='previous_image_restored_additive_audit_tables_retained' + except Exception:report['rollback']='manual_review_required' + raise + finally: + report['finishedAt']=datetime.now(timezone.utc).isoformat();p=folder/'REPORT.json';p.write_text(json.dumps(report,indent=2)+'\n') + uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid + for x in (folder.parent,folder,p):os.chown(x,uid,gid) + p.chmod(0o640);print('UNIFIED RELEASE REPORT:',p) + print('V4 candidate and planner UI installed. Mapping equivalence is activated only with explicit approval. No data/cursor rewrite and no device-control grant.') + + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True);p.add_argument('--apply',action='store_true');p.add_argument('--approve-numeric-mapping-compatibility',action='store_true',help='Explicitly approve only the audited 100 versus 100.0 mapping equivalence; never rewrite original records');a=p.parse_args() + try:run(a.plant,a.apply,a.approve_numeric_mapping_compatibility) + except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. Review UNIFIED RELEASE REPORT; no actuator permission enabled.') diff --git a/services/netplan-v4/commissioning/unified-release/.gitignore b/services/netplan-v4/commissioning/unified-release/.gitignore new file mode 100644 index 0000000..2b3f20c --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/.gitignore @@ -0,0 +1 @@ +context/ diff --git a/services/netplan-v4/commissioning/unified-release/RELEASE.json b/services/netplan-v4/commissioning/unified-release/RELEASE.json new file mode 100644 index 0000000..611d7e0 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/RELEASE.json @@ -0,0 +1,414 @@ +{ + "scope": "unified_v4_candidate", + "installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021", + "files": { + ".dockerignore": { + "before": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790", + "after": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790" + }, + "Dockerfile": { + "before": "f61404d65d9c031d0ddd5db9d2fa52279e39b1bd6015249a375b9167c744ae90", + "after": "7d5945ad5b17396bb3554e9c1602510cc2d89fa784f2f634514d076ef9f2ec33" + }, + "acceptance/Dockerfile.php": { + "before": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062", + "after": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062" + }, + "acceptance/check_forecast.py": { + "before": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5", + "after": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5" + }, + "acceptance/forecast-src/SOURCE_MANIFEST.json": { + "before": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5", + "after": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5" + }, + "acceptance/forecast-src/main.py": { + "before": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c", + "after": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c" + }, + "acceptance/forecast-src/methods/__init__.py": { + "before": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "after": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "acceptance/forecast-src/methods/battery_optimizer.py": { + "before": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6", + "after": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6" + }, + "acceptance/forecast-src/methods/common.py": { + "before": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8", + "after": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8" + }, + "acceptance/forecast-src/methods/var_1.py": { + "before": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4", + "after": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4" + }, + "acceptance/forecast-src/methods/var_10.py": { + "before": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7", + "after": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7" + }, + "acceptance/forecast-src/methods/var_11.py": { + "before": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc", + "after": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc" + }, + "acceptance/forecast-src/methods/var_13.py": { + "before": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9", + "after": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9" + }, + "acceptance/forecast-src/methods/var_2.py": { + "before": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a", + "after": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a" + }, + "acceptance/forecast-src/methods/var_21.py": { + "before": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15", + "after": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15" + }, + "acceptance/forecast-src/methods/var_22.py": { + "before": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6", + "after": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6" + }, + "acceptance/forecast-src/methods/var_23.py": { + "before": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678", + "after": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678" + }, + "acceptance/forecast-src/methods/var_3.py": { + "before": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984", + "after": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984" + }, + "acceptance/forecast-src/model_isolation.py": { + "before": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5", + "after": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5" + }, + "acceptance/forecast-src/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "acceptance/forecast-src/requirements.txt": { + "before": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083", + "after": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083" + }, + "acceptance/forecast-src/shared_utils.py": { + "before": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57", + "after": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57" + }, + "acceptance/forecast-src/soc_diagnostics.py": { + "before": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a", + "after": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a" + }, + "acceptance/forecast-src/telemetry_quality.py": { + "before": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810", + "after": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810" + }, + "acceptance/forecast-src/tests/test_battery_optimizer.py": { + "before": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8", + "after": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8" + }, + "acceptance/forecast-src/tests/test_load_forecast.py": { + "before": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c", + "after": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c" + }, + "acceptance/forecast-src/tests/test_model_isolation.py": { + "before": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30", + "after": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30" + }, + "acceptance/forecast-src/tests/test_telemetry_integrity.py": { + "before": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce", + "after": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce" + }, + "acceptance/php-src/SOURCE_MANIFEST.json": { + "before": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50", + "after": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50" + }, + "acceptance/php-src/check.php": { + "before": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473", + "after": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473" + }, + "acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": { + "before": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7", + "after": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7" + }, + "acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": { + "before": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9", + "after": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9" + }, + "acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": { + "before": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a", + "after": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a" + }, + "acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": { + "before": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc", + "after": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc" + }, + "acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": { + "before": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c", + "after": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c" + }, + "acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": { + "before": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6", + "after": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6" + }, + "acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": { + "before": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2", + "after": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2" + }, + "acceptance/read_native_forecasts.py": { + "before": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f", + "after": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f" + }, + "approved_previous_assets.json": { + "before": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0", + "after": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0" + }, + "commissioning/APPLICATION_STATUS.md": { + "before": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc", + "after": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc" + }, + "commissioning/application-source/server-dataset.json": { + "before": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d", + "after": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d" + }, + "commissioning/deploy_application.py": { + "before": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "after": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e" + }, + "commissioning/deploy_history_timing.py": { + "before": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb", + "after": "7769a473927bb10acf1ba1aef902e6b14fdf26b548d58056b557e6e4c33f62eb" + }, + "commissioning/deploy_unified_release.py": { + "before": null, + "after": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e" + }, + "compose.portal-bridge.yaml": { + "before": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197", + "after": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197" + }, + "compose.yaml": { + "before": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607", + "after": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607" + }, + "deploy_integrated_shadow.py": { + "before": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898", + "after": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898" + }, + "forecast_acceptance.py": { + "before": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9", + "after": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9" + }, + "gui/netplan-v4.css": { + "before": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd", + "after": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd" + }, + "gui/netplan-v4.html": { + "before": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755", + "after": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755" + }, + "gui/netplan-v4.js": { + "before": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "after": "245af626f3322dcaec5afbbc9772707ee6de00f07fd939e59709e7349f2d12d9" + }, + "install_hooks.py": { + "before": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3", + "after": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3" + }, + "integrations/NetzfahrplanV4.php": { + "before": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3", + "after": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3" + }, + "integrations/netplan-v4-bridge.mjs": { + "before": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7", + "after": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7" + }, + "integrations/netplan_v4_publisher.py": { + "before": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161", + "after": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161" + }, + "netplan_v4/__init__.py": { + "before": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48", + "after": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48" + }, + "netplan_v4/battery_model.py": { + "before": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332", + "after": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332" + }, + "netplan_v4/controlled_trial.py": { + "before": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4", + "after": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4" + }, + "netplan_v4/domain.py": { + "before": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b", + "after": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b" + }, + "netplan_v4/economic_replay.py": { + "before": null, + "after": "bcf1822486e5330495775fe3df63b27d3159147a09a24759d0f9227547561302" + }, + "netplan_v4/forecast_quality.py": { + "before": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840", + "after": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840" + }, + "netplan_v4/history_timing.py": { + "before": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f", + "after": "d2992e15d8d1a4f79d068405eefba4d4b37a4326afb09064cfd7c098822ec63f" + }, + "netplan_v4/mapping_identity.py": { + "before": null, + "after": "199bd4b80c618e0d3d8c5f0a7a9168820022cffa403ee6861ac2e3b933f9fc4d" + }, + "netplan_v4/measurement_pipeline.py": { + "before": "17e59c8e8d7403f107e95620260c7b71d9f51e8165c0c29944bb931c55c4001d", + "after": "32e9b245c0e35f1047f08cd6286193ebcb2db267c11cefe36885bcc3d9c8ca7f" + }, + "netplan_v4/meter_runtime.py": { + "before": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547", + "after": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547" + }, + "netplan_v4/metering.py": { + "before": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987", + "after": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987" + }, + "netplan_v4/optimizer.py": { + "before": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063", + "after": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063" + }, + "netplan_v4/peak_policy.py": { + "before": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576", + "after": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576" + }, + "netplan_v4/receiver_contract.py": { + "before": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c", + "after": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c" + }, + "netplan_v4/retention.py": { + "before": null, + "after": "a902024f00fdc22452e391917c6064a9fa30d60c30cebd0cdd192ce8bc1a0cc3" + }, + "netplan_v4/selection.py": { + "before": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad", + "after": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad" + }, + "netplan_v4/service.py": { + "before": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4", + "after": "80f0c835a8269a78ba62d52dea3da67a664f6b786627ed38ac02674b6f0f491f" + }, + "netplan_v4/store.py": { + "before": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99", + "after": "cb6a87e6f08f8f36840e6e1fd0e7b8f09c1bcb012193ce1d8819c5f963eab3b2" + }, + "release_preflight.py": { + "before": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5", + "after": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5" + }, + "requirements.txt": { + "before": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464", + "after": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464" + }, + "run_tests.py": { + "before": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4", + "after": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4" + }, + "runtime_preflight.py": { + "before": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1", + "after": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1" + }, + "tests/manager_protocol.php": { + "before": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93", + "after": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93" + }, + "tests/portal.test.mjs": { + "before": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a", + "after": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a" + }, + "tests/test_application_deployment.py": { + "before": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5", + "after": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5" + }, + "tests/test_battery_recovery.py": { + "before": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3", + "after": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3" + }, + "tests/test_container_access.py": { + "before": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9", + "after": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9" + }, + "tests/test_controlled_trial.py": { + "before": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64", + "after": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64" + }, + "tests/test_controlled_trial_pipeline.py": { + "before": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d", + "after": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d" + }, + "tests/test_delivery.py": { + "before": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09", + "after": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09" + }, + "tests/test_economic_replay.py": { + "before": null, + "after": "a3b41f9d8d296f889814c985c357d16efa3b6fd6ae683f2324ec53cd662629c3" + }, + "tests/test_forecast_acceptance.py": { + "before": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76", + "after": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76" + }, + "tests/test_forecast_quality.py": { + "before": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82", + "after": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82" + }, + "tests/test_history_timing.py": { + "before": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2", + "after": "7a11d52c779cd25f44bc2bb56a758b862154e2c8df6987046e18eace46ed51a2" + }, + "tests/test_history_timing_deployment.py": { + "before": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e", + "after": "7508d24b70c1fa4c33e21fbad4e99d5bff23e262d91222df5b0405e937c96d2e" + }, + "tests/test_integrated_deployment.py": { + "before": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883", + "after": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883" + }, + "tests/test_mapping_recovery.py": { + "before": null, + "after": "5c85eedf7ae780c8efa11ced856a7816ce24df0347d01e1a08e19f6dc101468e" + }, + "tests/test_measurement_pipeline.py": { + "before": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4", + "after": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4" + }, + "tests/test_metering.py": { + "before": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d", + "after": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d" + }, + "tests/test_peak_release.py": { + "before": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d", + "after": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d" + }, + "tests/test_receiver_contract.py": { + "before": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e", + "after": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e" + }, + "tests/test_release_preflight.py": { + "before": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8", + "after": "e9f0dfc3dd752c2cc25908e47781554bf71e2e1af4f4d2514621fab37205ca68" + }, + "tests/test_retention.py": { + "before": null, + "after": "dfe62da159be4b103643d61c10307395e22612c65017aaadd469d50a65398b51" + }, + "tests/test_unified_release.py": { + "before": null, + "after": "3665b3ffd0872e1956312ac16ab9527802ebc396e52ae0d2954bad53890d5236" + }, + "tests/test_v4.py": { + "before": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763", + "after": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763" + } + }, + "installerSha256": "d5f7831fa556288f23747b6b186a77b18836d6c22d88acce2f1b4dc786ccbf4e", + "guiBefore": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5", + "proofSha256": "11cb827ab65611bbb06bf8a181d0f44a16202ae957d82f56e0a18a916325128d", + "backupHelperSha256": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e", + "autoArchiveDefault": false, + "mappingRegistrationRequiresExplicitApproval": true, + "productionReady": false, + "releaseId": "aeff5f094f6c8fbd7dc962d2d091d364a14c45e10fc7bd2f4847dd6dba329a22" +} diff --git a/services/netplan-v4/commissioning/unified-release/build_context.py b/services/netplan-v4/commissioning/unified-release/build_context.py new file mode 100644 index 0000000..ae11ab4 --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/build_context.py @@ -0,0 +1,28 @@ +"""Reproduce the frozen candidate context from the versioned application sources. + +No Docker/runtime/secret reads. Existing output is verified, never silently replaced. +""" +from pathlib import Path +import hashlib,json,shutil + +PACKAGE=Path(__file__).resolve().parent +ROOT=PACKAGE.parents[1] + +def build(): + manifest=json.loads((PACKAGE/'RELEASE.json').read_text()) + context=PACKAGE/'context' + for name,v in manifest['files'].items(): + if Path(name).is_absolute() or '..' in Path(name).parts:raise ValueError('Unsafe source path') + source=ROOT/name;target=context/name + if source.is_symlink() or not source.is_file() or hashlib.sha256(source.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Checked-out source does not match release: '+name) + if target.is_symlink():raise ValueError('Target symlink refused') + if target.exists() and hashlib.sha256(target.read_bytes()).hexdigest()!=v['after']: + raise ValueError('Existing context differs: '+name) + for name in manifest['files']: + target=context/name + if target.exists():continue + target.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(ROOT/name,target);target.chmod(0o644) + print('Frozen build context verified:',len(manifest['files']),'source files; no installation.') + +if __name__=='__main__':build() diff --git a/services/netplan-v4/commissioning/unified-release/mapping-proof.json b/services/netplan-v4/commissioning/unified-release/mapping-proof.json new file mode 100644 index 0000000..382cdca --- /dev/null +++ b/services/netplan-v4/commissioning/unified-release/mapping-proof.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "canonicalJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100.0},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}", + "legacyJson": "{\"version\":1,\"installationId\":\"e3a08f9e-af12-4695-99bd-8b51c0520021\",\"managerId\":17004,\"reportedInventorySha256\":\"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc\",\"accounting\":{\"version\":1,\"sources\":[{\"key\":\"ev_account\",\"role\":\"controlled_virtual\",\"variableId\":52020,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_EV\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]},{\"key\":\"grid\",\"role\":\"grid\",\"variableId\":40348,\"parentId\":11490,\"ident\":\"Power_8\",\"factorToW\":1000.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe1\",\"role\":\"physical_storage\",\"variableId\":47725,\"parentId\":19742,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_goodwe2\",\"role\":\"physical_storage\",\"variableId\":35724,\"parentId\":57658,\"ident\":\"A_6_3_35182\",\"factorToW\":-1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"physical_solaredge\",\"role\":\"physical_storage\",\"variableId\":21447,\"parentId\":30789,\"ident\":\"Value\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe1\",\"role\":\"pv\",\"variableId\":48459,\"parentId\":19742,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_goodwe2\",\"role\":\"pv\",\"variableId\":53802,\"parentId\":57658,\"ident\":\"A_3_3_35301\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"pv_solaredge\",\"role\":\"pv\",\"variableId\":20335,\"parentId\":36915,\"ident\":\"\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[]},{\"key\":\"sdl_account\",\"role\":\"external_virtual\",\"variableId\":25085,\"parentId\":58448,\"ident\":\"Aktuelle_Leistung_SDL\",\"factorToW\":1.0,\"maxAgeSeconds\":60,\"dependsOn\":[\"physical_goodwe1\",\"physical_goodwe2\",\"physical_solaredge\"]}],\"maxSkewSeconds\":30,\"splitToleranceW\":100},\"extraSources\":[{\"key\":\"energy_t1\",\"variableId\":59607,\"parentId\":11490,\"ident\":\"Energy_0\",\"unit\":\"kWh\"},{\"key\":\"energy_t2\",\"variableId\":26620,\"parentId\":11490,\"ident\":\"Energy_1\",\"unit\":\"kWh\"},{\"key\":\"ev_available_charge\",\"variableId\":50230,\"parentId\":58448,\"ident\":\"P_EV_laden\",\"unit\":\"W\"},{\"key\":\"ev_available_discharge\",\"variableId\":43899,\"parentId\":58448,\"ident\":\"P_EV_entladen\",\"unit\":\"W\"},{\"key\":\"ev_requested\",\"variableId\":19651,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_EV\",\"unit\":\"W\"},{\"key\":\"grid_display\",\"variableId\":49301,\"parentId\":42773,\"ident\":\"\",\"unit\":\"W\"},{\"key\":\"sdl_requested\",\"variableId\":38943,\"parentId\":58448,\"ident\":\"Nennleistung_Soll_SDL\",\"unit\":\"W\"},{\"key\":\"soc_ev\",\"variableId\":32871,\"parentId\":58448,\"ident\":\"SoC_EV\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe1\",\"variableId\":27361,\"parentId\":19742,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_goodwe2\",\"variableId\":23109,\"parentId\":57658,\"ident\":\"A_2_3_47908\",\"unit\":\"percent\"},{\"key\":\"soc_sdl\",\"variableId\":23879,\"parentId\":58448,\"ident\":\"SDL_Pos\",\"unit\":\"percent\"},{\"key\":\"soc_solaredge\",\"variableId\":51938,\"parentId\":37468,\"ident\":\"Value\",\"unit\":\"percent\"},{\"key\":\"solar_ac_scale\",\"variableId\":41853,\"parentId\":48996,\"ident\":\"Value\",\"unit\":\"scale\"},{\"key\":\"solar_ac_value\",\"variableId\":37975,\"parentId\":35514,\"ident\":\"Value\",\"unit\":\"raw_power\"}]}" +} diff --git a/services/netplan-v4/gui/netplan-v4.js b/services/netplan-v4/gui/netplan-v4.js index a67ac55..7e76572 100644 --- a/services/netplan-v4/gui/netplan-v4.js +++ b/services/netplan-v4/gui/netplan-v4.js @@ -78,8 +78,14 @@ async function render() { replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } }); refresh.addEventListener('click', render); panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.')); - panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik wartet weiterhin auf vergleichbare Kosten-Replays, nicht bloss auf gute R2-Werte.')); - for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik erstellt vergleichbare Kosten-Replays aus eingefrorenen Tagesplaenen und spaeter eingegangenen Messwerten. Ohne ausreichende Vergleichstage bleibt die bisherige Familie aktiv.')); + for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Letzter Serverempfang', time(d.lastReceived)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] ); + if (state.economicComparison?.connected) { + table('Wirtschaftlicher Modellvergleich', [['Gespeicherte Tagesvergleiche', state.economicComparison.cohorts], ['Abgeschlossene Vergleiche', state.economicComparison.completedComparisons.length], ['Methode', 'Eingefrorener Tagesplan mit simulierter Netzregelung'], ['Grenze dieses Vergleichs', 'Eine netzladefaehige Batterie; keine vollstaendige Nachbildung laufender Neuoptimierungen'], ['Bewertung', 'Energiekosten + zusaetzlicher Peak + Verluste + gemeinsame Restenergiebewertung']]); + panel.append(element('p', 'Der SDL-Anteil der historischen Bilanz ist eine gekennzeichnete Schaetzung aus dem damaligen Auftrag. Die Vergleichskosten sind keine nachgewiesenen Einsparungen auf einer Stromrechnung.')); + for (const item of state.economicComparison.completedComparisons) table('Vergleich ' + item.day, Object.entries(item.results).map(([family, result]) => ['Familie ' + family, num(result.costChf) + ' CHF; Datenabdeckung ' + num(100 * result.coverage, 1) + ' %'])); + } + if (state.maintenance) table('Datenaufbewahrung', [['Archivierung', state.maintenance.status], ['Originale', 'Nach verifizierter Kompression weiterhin wiederherstellbar; externe Sicherung separat betreiben']]); panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.')); if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.')); for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]); diff --git a/services/netplan-v4/netplan_v4/economic_replay.py b/services/netplan-v4/netplan_v4/economic_replay.py new file mode 100644 index 0000000..239647f --- /dev/null +++ b/services/netplan-v4/netplan_v4/economic_replay.py @@ -0,0 +1,207 @@ +"""Causal daily economic family comparison using frozen plans and later physical data. + +This is a labelled *simulation*, not savings on an invoice. A cohort is captured +near local midnight, uses only prices/forecasts then available, and all families +share initial storage, physical outcomes, tariff and constraints. No device calls. +Supported v1: one grid-charge-enabled battery; other topologies remain explicit. +""" +from dataclasses import asdict, replace +from datetime import datetime, timedelta +from hashlib import sha256 +from math import sqrt +import json +from .domain import ZURICH, utc, month_key, quarter_start +from .selection import ReplayScore +from . import measurement_pipeline as m + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_economic_cohorts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, issued_at INTEGER NOT NULL, + ends_at INTEGER NOT NULL, policy_id TEXT NOT NULL, value TEXT NOT NULL, + PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_results( + plant TEXT NOT NULL, local_day TEXT NOT NULL, evaluated_at INTEGER NOT NULL, + policy_id TEXT NOT NULL, value TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + CREATE TABLE IF NOT EXISTS planner_economic_attempts( + plant TEXT NOT NULL, local_day TEXT NOT NULL, checked_at INTEGER NOT NULL, + reason TEXT NOT NULL, PRIMARY KEY(plant,local_day)); + ''') + + +def _policy(data, cfg): + batteries=[] + for b in data['batteries']: + v=asdict(b) + for k in ('soc_percent','measured_at','discharge_blocked'):v.pop(k) + batteries.append(v) + return sha256(m.canonical({'battery':batteries,'limits':asdict(data['limits']), + 'mapping':cfg['mappingSha256'],'formula':cfg['formula'],'dataset':cfg['datasetId'], + 'method':'frozen_day_ahead_grid_tracking_v1','external':'sdl_request_estimate','peakTariffs':data['peak_prices']}).encode()).hexdigest() + + +def capture(store, plant, now, assemble, optimizer): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile': return + local=utc(now).astimezone(ZURICH); day=local.date().isoformat() + # Freeze only at the beginning of a local day. Never reconstruct an old forecast from hindsight. + if local.hour!=0 or local.minute>=15:return + if store.con.execute('SELECT 1 FROM planner_economic_cohorts WHERE plant=? AND local_day=?',(plant,day)).fetchone():return + attempted=store.con.execute('SELECT checked_at FROM planner_economic_attempts WHERE plant=? AND local_day=?',(plant,day)).fetchone() + if attempted and now.timestamp()-attempted[0]<300:return + reason='awaiting_comparable_snapshot' + try: + cfg=m.configuration(store.con,plant,settings['measurementDataset']);plans={}; common=None;policy=None + end=datetime.combine(local.date()+timedelta(days=1),datetime.min.time(),tzinfo=ZURICH).astimezone(utc(now).tzinfo) + for family in store.registry.entries(): + data,_,quality=assemble(store,plant,family.key,now) + data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] + if len(data['batteries'])!=1 or not data['batteries'][0].grid_charging: + raise ValueError('unsupported_replay_topology') + if data['steps'][-1].end=rearm-1e-9:blocked=False + discharge=0. if blocked else min(b['max_discharge_w'],max(0.,(energy-low)*eta/dt*1000)) + target=p['gridTargetW'];cap=limits['import_w'];monthly=limits['manager_month_limits_w'].get(str(start.astimezone(ZURICH).month),limits['manager_month_limits_w'].get(start.astimezone(ZURICH).month)) + if monthly is not None:cap=monthly if cap is None else min(cap,monthly) + if cap is not None:target=min(target,cap) + if limits['export_w'] is not None:target=max(target,-limits['export_w']) + battery=min(charge,max(-discharge,target-residual));grid=residual+battery + energy+=battery/1000*dt*(eta if battery>=0 else 1/eta) + if energyhigh+1e-6:raise ValueError('Replay SOC invariant') + if energy<=low+1e-9:blocked=True + if cap is not None and grid>cap+1.:breaches+=1 + if limits['export_w'] is not None and grid < -limits['export_w']-1.:breaches+=1 + money+=(max(grid,0)*p['importPriceChfKwh']-max(-grid,0)*p['exportPriceChfKwh'])/1000*dt + wear+=abs(battery)/1000*dt*b['throughput_chf_kwh'] + q=quarter_start(start).isoformat();v=quarters.setdefault(q,[0.,0]);v[0]+=max(grid,0)/1000*dt;v[1]+=seconds + if end==quarter_start(start)+timedelta(minutes=15): + if v[1]!=900:raise ValueError('Incomplete simulated billing quarter') + month=month_key(start);peaks[month]=max(peaks[month],v[0]/.25);quarter_max[month]=max(quarter_max.get(month,0.),v[0]/.25) + additional=sum(max(0,v-ctx['peaks'][month])*ctx['peakTariffs'][month] for month,v in peaks.items()) + # Identical terminal valuation for all families, known at cohort creation; separate from cash. + last=plan['points'][-1];buy=max(0.,last['importPriceChfKwh']);sell=max(0.,min(buy,last['exportPriceChfKwh'])) + terminal=max(initial-energy,0)/eta*buy-max(energy-initial,0)*eta*sell + return {'cashCostChf':money+additional,'energyCostChf':money,'peakCostChf':additional,'throughputCostChf':wear, + 'terminalAdjustmentChf':terminal,'costChf':money+additional+wear+terminal, + 'initialEnergyKwh':initial,'finalEnergyKwh':energy,'coverage':min(coverage), + 'quarterMaximaKw':quarter_max,'initialPeaksKw':ctx['peaks'],'peakTariffs':ctx['peakTariffs'], + 'constraintBreaches':breaches,'terminalNormalized':True, + 'terminalMethod':'common_known_end_price_inventory_valuation_not_physical_restoration'} + + +def advance(store,plant,now): + timestamp=int(now.timestamp()) + rows=store.con.execute('SELECT c.* FROM planner_economic_cohorts c LEFT JOIN planner_economic_results r USING(plant,local_day) WHERE c.plant=? AND c.ends_at<=? AND c.ends_at>=? AND r.local_day IS NULL AND NOT EXISTS(SELECT 1 FROM planner_economic_attempts a WHERE a.plant=c.plant AND a.local_day=c.local_day AND a.checked_at>?) ORDER BY c.issued_at LIMIT 1',(plant,timestamp-120,timestamp-90*86400,timestamp-300)).fetchall() + for row in rows: + with store.con:store.con.execute('INSERT INTO planner_economic_attempts VALUES(?,?,?,?) ON CONFLICT(plant,local_day) DO UPDATE SET checked_at=excluded.checked_at,reason=excluded.reason',(plant,row['local_day'],timestamp,'evaluating_actuals')) + try: + cohort=json.loads(row['value']);cfg=m.configuration(store.con,plant,cohort['datasetId']);source=cfg.get('sourceDatasetId',cfg['datasetId']);start=m.epoch(cohort['start']);end=m.epoch(cohort['end']) + observed=store.con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',(plant,source,start-600,end+600,timestamp)) + windows=actuals([json.loads(r[0]) for r in observed],cfg);outcomes={w['start']:w for w in windows} + results={f:simulate(cohort,f,outcomes) for f in cohort['plans']} + value={'status':'evaluated','results':results,'start':cohort['start'],'end':cohort['end'], + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','method':cohort['method'], + 'actualCashSavings':False,'controlEnabled':False} + with store.con:store.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(plant,row['local_day'],timestamp,row['policy_id'],m.canonical(value))) + except (ValueError,KeyError,TypeError): + # Retain unscored cohort. Lack of actual data must never become a zero cost. + pass + + +def scores(store,plant,now): + settings=store.settings(plant) + if settings.get('forecastSource')!='corrected_profile':return [] + since=int((utc(now)-timedelta(days=settings['autoLookbackDays'])).timestamp()) + recent=store.con.execute('SELECT policy_id,value FROM planner_economic_cohorts WHERE plant=? ORDER BY issued_at DESC LIMIT 1',(plant,)).fetchone() + if not recent or json.loads(recent[1])['datasetId']!=settings.get('measurementDataset'):return [] + rows=store.con.execute('SELECT c.issued_at,c.ends_at,r.evaluated_at,r.value FROM planner_economic_results r JOIN planner_economic_cohorts c USING(plant,local_day) WHERE r.plant=? AND r.policy_id=? AND c.issued_at>=? AND r.evaluated_at<=? ORDER BY c.issued_at',(plant,recent[0],since,int(now.timestamp()))).fetchall() + if not rows:return [] + keys=[f.key for f in store.registry.entries()];valid=[] + for r in rows: + v=json.loads(r['value']);out=v['results'] + if set(out)!=set(keys) or any(out[k]['constraintBreaches'] for k in keys):continue + valid.append((r,v)) + if not valid:return [] + result=[] + for k in keys: + # Monthly demand cost is paid ONCE for the maximum, not once per replay day. + total=sum(v['results'][k]['energyCostChf']+v['results'][k]['throughputCostChf']+v['results'][k]['terminalAdjustmentChf'] for _,v in valid) + bases={}; maxima={}; rates={} + for _,v in valid: + d=v['results'][k] + for month,peak in d['quarterMaximaKw'].items(): + bases.setdefault(month,d['initialPeaksKw'][month]) + maxima[month]=max(maxima.get(month,0.),peak);rates[month]=d['peakTariffs'][month] + total+=sum(max(0.,maxima[month]-bases[month])*rates[month] for month in maxima) + cover=min(v['results'][k]['coverage'] for _,v in valid) + first=utc(valid[0][1]['start']);end=utc(valid[-1][1]['end']);available=datetime.fromtimestamp(max(r['evaluated_at'] for r,_ in valid),utc(now).tzinfo) + result.append(ReplayScore(k,recent[0],first,end,first,available,total,cover,len(valid))) + return result + + +def status(store,plant): + rows=store.con.execute('SELECT local_day,value FROM planner_economic_results WHERE plant=? ORDER BY local_day DESC LIMIT 14',(plant,)) + evaluated=[{'day':r[0],**json.loads(r[1])} for r in rows] + count=store.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts WHERE plant=?',(plant,)).fetchone()[0] + return {'connected':True,'method':'frozen_day_ahead_grid_tracking_v1','cohorts':count,'completedComparisons':evaluated, + 'evaluationBasis':'physical_balance_with_sdl_request_estimate','isBillingEvidence':False, + 'limitations':['one_grid_charging_battery','frozen_daily_plan_not_receding_horizon_field_replay']} diff --git a/services/netplan-v4/netplan_v4/mapping_identity.py b/services/netplan-v4/netplan_v4/mapping_identity.py new file mode 100644 index 0000000..255f5f1 --- /dev/null +++ b/services/netplan-v4/netplan_v4/mapping_identity.py @@ -0,0 +1,100 @@ +"""Audited numeric-representation compatibility, not relaxed device identity checks. + +Only the internal operator API can register equivalence. Raw journals are never +rewritten, and the original received fingerprint is retained with each receipt. +""" +from hashlib import sha256 +import json +import math +import re + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_mapping_compatibility( + plant TEXT NOT NULL, dataset TEXT NOT NULL, alias TEXT NOT NULL, + canonical TEXT NOT NULL, inventory TEXT NOT NULL, evidence TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset,alias)); + CREATE TABLE IF NOT EXISTS planner_observation_origins( + plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL, + received_mapping TEXT NOT NULL, inventory TEXT NOT NULL, + evidence_id TEXT, received_at INTEGER NOT NULL, + PRIMARY KEY(plant,dataset,captured_at,received_mapping)); + ''') + + +def _decode(text): + if not isinstance(text, str) or len(text.encode()) > 131072: + raise ValueError('Bounded configuration evidence required') + def pairs(items): + d = {} + for k, v in items: + if k in d: raise ValueError('Duplicate configuration key') + d[k] = v + return d + def constant(_): raise ValueError('Nonfinite configuration') + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + + +def _same(a, b): + if type(a) is not type(b): return False + if isinstance(a, dict): + return list(a) == list(b) and all(_same(a[k], b[k]) for k in a) + if isinstance(a, list): return len(a) == len(b) and all(_same(x, y) for x, y in zip(a, b)) + return a == b + + +def validate_evidence(plant, config, payload): + if not isinstance(payload, dict) or set(payload) != {'version', 'canonicalJson', 'legacyJson'} or type(payload['version']) is not int or payload['version'] != 1: + raise ValueError('Explicit versioned representation evidence required') + ca, le = payload['canonicalJson'], payload['legacyJson'] + a, b = _decode(ca), _decode(le) + canonical_hash, alias = sha256(ca.encode()).hexdigest(), sha256(le.encode()).hexdigest() + if canonical_hash != config['mappingSha256'] or alias == canonical_hash: + raise ValueError('Evidence does not match configured mapping') + for c in (a, b): + if not isinstance(c, dict) or c.get('installationId') != plant or c.get('reportedInventorySha256') != config['inventorySha256']: + raise ValueError('Evidence belongs to another installation or inventory') + x = a.get('accounting', {}).get('splitToleranceW') + y = b.get('accounting', {}).get('splitToleranceW') + if type(x) is not float or type(y) is not int or not math.isfinite(x) or x != y or not 0 <= x <= 500: + raise ValueError('Only demonstrated float/integer tolerance representation is compatible') + b['accounting']['splitToleranceW'] = float(y) + if not _same(a, b): + raise ValueError('Other configuration differences are not representation compatibility') + # Proof includes the exact hashed JSON strings; no arbitrary labels as evidence. + evidence = json.dumps(payload, sort_keys=True, separators=(',', ':'), allow_nan=False) + return {'alias': alias, 'canonical': canonical_hash, 'inventory': config['inventorySha256'], + 'evidenceId': sha256(evidence.encode()).hexdigest(), 'evidence': evidence} + + +def register(con, plant, config, payload, now): + if config.get('sourceDatasetId'): raise ValueError('Register compatibility on original dataset only') + v = validate_evidence(plant, config, payload) + con.execute('BEGIN IMMEDIATE') + try: + old = con.execute('SELECT canonical,inventory,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND alias=?', + (plant, config['datasetId'], v['alias'])).fetchone() + expected = (v['canonical'], v['inventory'], v['evidence']) + if old and tuple(old) != expected: raise ValueError('Immutable mapping compatibility conflict') + con.execute('INSERT OR IGNORE INTO planner_mapping_compatibility VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], v['alias'], *expected, now)) + con.commit() + except Exception: + con.rollback(); raise + return {'status': 'registered', 'datasetId': config['datasetId'], 'evidenceId': v['evidenceId'], + 'compatibleMapping': v['alias'], 'canonicalMapping': v['canonical'], 'controlEnabled': False} + + +def approved(con, plant, config): + rows = con.execute('SELECT alias,evidence FROM planner_mapping_compatibility WHERE plant=? AND dataset=? AND canonical=? AND inventory=?', + (plant, config['datasetId'], config['mappingSha256'], config['inventorySha256'])) + return {r['alias']: sha256(r['evidence'].encode()).hexdigest() for r in rows} + + +def save_origin(con, plant, config, record, captured_at, received_at, aliases): + source = record['mappingSha256'] + if source != config['mappingSha256'] and source not in aliases: + raise ValueError('Unknown mapping; no receipt written') + con.execute('INSERT OR IGNORE INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)', + (plant, config['datasetId'], captured_at, source, config['inventorySha256'], aliases.get(source), received_at)) diff --git a/services/netplan-v4/netplan_v4/measurement_pipeline.py b/services/netplan-v4/netplan_v4/measurement_pipeline.py index 0579467..61ca5a9 100644 --- a/services/netplan-v4/netplan_v4/measurement_pipeline.py +++ b/services/netplan-v4/netplan_v4/measurement_pipeline.py @@ -14,6 +14,7 @@ from statistics import median from zoneinfo import ZoneInfo import json from .history_timing import validate_policy, endpoint_bridges +from . import mapping_identity UTC = timezone.utc LOCAL = ZoneInfo('Europe/Zurich') @@ -42,6 +43,7 @@ def numeric(value, bound=1e12): def schema(con): + mapping_identity.schema(con) con.executescript(''' CREATE TABLE IF NOT EXISTS planner_data_sets( plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL, @@ -162,10 +164,11 @@ def configuration(con, plant, dataset): return json.loads(row[0]) -def project(record, c, plant, now): +def project(record, c, plant, now, approved_mappings=()): if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant: raise ValueError('Wrong capture identity') - if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']: + incoming = record.get('mappingSha256') + if not isinstance(incoming, str) or (incoming != c['mappingSha256'] and incoming not in approved_mappings) or record.get('reportedInventorySha256') != c['inventorySha256']: raise ValueError('Wrong capture mapping or inventory') t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt')) if start > t or t > now+30 or t < now-90*86400: @@ -196,13 +199,14 @@ def ingest_batch(con, plant, payload, now): records = payload['records'] if not isinstance(records,list) or not 1 <= len(records) <= 120: raise ValueError('Batch requires 1..120 captures') - rows = [project(r,c,plant,now) for r in records] + aliases = mapping_identity.approved(con, plant, c) + rows = [project(r,c,plant,now,aliases) for r in records] if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])): raise ValueError('Batch must be in increasing capture order') stored = duplicate = 0 con.execute('BEGIN IMMEDIATE') try: - for r in rows: + for original, r in zip(records, rows): value = canonical(r); digest = sha256(value.encode()).hexdigest() old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone() if old: @@ -211,6 +215,7 @@ def ingest_batch(con, plant, payload, now): duplicate += 1 else: con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1 + mapping_identity.save_origin(con, plant, c, original, r['capturedAt'], now, aliases) con.commit() except Exception: con.rollback(); raise @@ -243,7 +248,7 @@ def physical_value(values, c): return load -def reconstruct(records, c): +def reconstruct(records, c, *, projection=None): """Bounded retrospective estimation, never a real-time feedback signal. Missing observations split support. Source timestamps are not refreshed. Small @@ -260,6 +265,11 @@ uncovered portions remain quantified and are never filled with zero. for s in c['sources']: if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s timing_policy = validate_policy(c.get('historyTimingPolicy'), c['sources']) + if projection is not None: + configured={s['key']:s for s in c['sources']} + if set(projection)!= {'keys','calculate'} or not callable(projection['calculate']) or not projection['keys'] or not set(projection['keys']) <= set(configured): + raise ValueError('Invalid internal projection') + primary={k:configured[k] for k in projection['keys']} first,last = records[0]['capturedAt'],records[-1]['capturedAt'] series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = [] first_observed = {k:{} for k in primary} @@ -313,7 +323,9 @@ uncovered portions remain quantified and are never filled with zero. extended.append(k); known_at = max(known_at, bridge['availableAt']) load = None if usable: - try: load = physical_value(vals,c) + try: + load = physical_value(vals,c) if projection is None else projection['calculate'](vals,c) + if not numeric(load,1e9):raise ValueError('Invalid historical projection') except ValueError: usable = False if usable: item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0 @@ -490,9 +502,10 @@ def pipeline_status(con,plant): out=[] for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)): ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone() - count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() + count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at),MAX(received_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,c.get('sourceDatasetId',ds))).fetchone() out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0], 'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None, + 'lastReceived':iso(count[3]) if count[3] else None, 'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {}, 'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds'], 'observationDatasetId':c.get('sourceDatasetId',ds), diff --git a/services/netplan-v4/netplan_v4/retention.py b/services/netplan-v4/netplan_v4/retention.py new file mode 100644 index 0000000..219a9a8 --- /dev/null +++ b/services/netplan-v4/netplan_v4/retention.py @@ -0,0 +1,106 @@ +"""Bounded online tables with verified lossless archives; never prune unarchived data. + +Runs at most hourly. Raw input history: 120 days (longer than 90-day training and +comparison limits). Ordinary plans: 7 days, excluding active/acknowledged plans. +Archives are retained; external backup/long-term archive lifecycle is operational. +""" +from pathlib import Path +import gzip +import hashlib +import json +import os +import sqlite3 +import tempfile + + +def schema(con): + con.executescript(''' + CREATE TABLE IF NOT EXISTS planner_maintenance( + name TEXT PRIMARY KEY, checked_at INTEGER NOT NULL, status TEXT NOT NULL, detail TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS planner_archives( + id TEXT PRIMARY KEY, created_at INTEGER NOT NULL, filename TEXT NOT NULL, + sha256 TEXT NOT NULL, records INTEGER NOT NULL, kind TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS planner_plan_created ON planner_plans(created_at); + ''') + + +def archive_batch(con,directory,now,kind,limit=100): + if kind not in ('observations','plans') or type(limit) is not int or not 1<=limit<=1000: + raise ValueError('Invalid archival scope') + directory=Path(directory) + if directory.is_symlink():raise ValueError('Archive symlink refused') + directory.mkdir(mode=0o700,parents=True,exist_ok=True) + if directory.stat().st_mode & 0o007:raise ValueError('Archive directory must not be public') + from datetime import datetime,timezone + with_context=False + con.execute('BEGIN IMMEDIATE') + try: + if kind=='observations': + rows=con.execute('SELECT * FROM planner_observations WHERE captured_at67108864:raise ValueError('Archive batch exceeds memory budget') + ident=hashlib.sha256(raw).hexdigest();name=kind+'-'+ident+'.jsonl.gz';path=directory/name + if path.is_symlink():raise ValueError('Archive target symlink refused') + if not path.exists(): + fd,tmp=tempfile.mkstemp(prefix='.archive-',dir=directory) + try: + os.fchmod(fd,0o600) + with os.fdopen(fd,'wb') as out: + with gzip.GzipFile(fileobj=out,mode='wb',mtime=0) as zipped:zipped.write(raw) + out.flush();os.fsync(out.fileno()) + os.replace(tmp,path) + dfd=os.open(directory,os.O_RDONLY) + try:os.fsync(dfd) + finally:os.close(dfd) + finally: + if os.path.exists(tmp):os.unlink(tmp) + # Read back exact bytes before removing any database row. + with gzip.open(path,'rb') as f:verified=f.read(len(raw)+1) + if verified!=raw:raise ValueError('Archive verification failed; original rows retained') + for r in rows: + if kind=='observations': + key=(r['plant'],r['dataset'],r['captured_at']) + con.execute('DELETE FROM planner_observation_origins WHERE plant=? AND dataset=? AND captured_at=?',key) + con.execute('DELETE FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=? AND fingerprint=?',(*key,r['fingerprint'])) + else:con.execute('DELETE FROM planner_plans WHERE plan_id=?',(r['plan_id'],)) + con.execute('INSERT OR IGNORE INTO planner_archives VALUES(?,?,?,?,?,?)',(ident,now,name,ident,len(rows),kind)) + con.commit() + return {'archived':len(rows),'kind':kind,'file':name,'sha256':ident,'sourceRecoverable':True} + except Exception: + con.rollback();raise + + +def maintain(store,now): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return + con=store.con;stamp=int(now.timestamp());old=con.execute("SELECT checked_at FROM planner_maintenance WHERE name='archive'").fetchone() + if old and stamp-old[0]<3600:return + db=con.execute('PRAGMA database_list').fetchone()[2] + if not db:return # In-memory test/ephemeral databases have no archival location. + try: + directory=Path(db).resolve().parent/'archives' + results=[archive_batch(con,directory,stamp,kind) for kind in ('plans','observations')] + if any(r['archived']>=100 for r in results):stamp-=3300 + state='ok';detail={'results':results,'archivesRetained':True,'rawRetentionDays':120,'ordinaryPlanRetentionDays':7} + except (OSError,ValueError,sqlite3.Error) as exc: + state='archive_error';detail={'errorType':type(exc).__name__,'unverifiedDataNotDeleted':True} + with con:con.execute('INSERT INTO planner_maintenance VALUES(?,?,?,?) ON CONFLICT(name) DO UPDATE SET checked_at=excluded.checked_at,status=excluded.status,detail=excluded.detail', + ('archive',stamp,state,json.dumps(detail,separators=(',',':')))) + + +def status(con): + if os.environ.get('NETPLAN_V4_ARCHIVE_ENABLED','0')!='1':return {'status':'disabled_requires_operator_opt_in'} + r=con.execute("SELECT checked_at,status,detail FROM planner_maintenance WHERE name='archive'").fetchone() + # Public per-plant state must not expose archival filenames/rows of other tenants. + return {'checkedAtEpoch':r[0],'status':r[1],'rawRetentionDays':120,'ordinaryPlanRetentionDays':7,'archivesRetained':True} if r else {'status':'not_yet_run'} diff --git a/services/netplan-v4/netplan_v4/service.py b/services/netplan-v4/netplan_v4/service.py index ceabaee..0143c79 100644 --- a/services/netplan-v4/netplan_v4/service.py +++ b/services/netplan-v4/netplan_v4/service.py @@ -16,7 +16,7 @@ from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_star from .store import PlannerStore,canonical from .selection import choose_family from .optimizer import optimize -from . import meter_runtime, controlled_trial, measurement_pipeline +from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline from .forecast_quality import assess_family from .receiver_contract import provenance from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month @@ -267,10 +267,12 @@ def assemble(store,plant,family,now): return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)} def run_once(store,now): + retention.maintain(store,now) stamp=int(now.timestamp())//300 plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')] for plant in plants: config=store.settings(plant) + economic_replay.advance(store,plant,now) datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))] for dataset in datasets: try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp())) @@ -286,14 +288,16 @@ def run_once(store,now): try: settings=store.settings(plant);previous=store.current(plant) current=previous['sourceFamily'] if previous else store.registry.entries()[0].key - # Productive replay ingestion is intentionally not fabricated from R2 metrics. - selection=choose_family(settings['family'],current,(),registry=store.registry,now=now) + selection=choose_family(settings['family'],current,economic_replay.scores(store,plant,now),registry=store.registry,now=now, + lookback_days=settings['autoLookbackDays'],minimum_days=settings['autoMinimumDays'], + minimum_coverage=settings['autoMinimumCoverage'],margin_chf=settings['autoSwitchMarginChf']) data,full_end,quality=assemble(store,plant,selection['family'],now) data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']] result=optimize(**data,config_revision=settings['revision'],family=selection['family']) if result['executable']: result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])}) store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token']) + economic_replay.capture(store,plant,now,assemble,optimize) return result except (ValueError,TypeError,KeyError) as exc: result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]} @@ -309,7 +313,7 @@ def status(store,plant,now): pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone() ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone() fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit')) - return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)} + return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant),'economicComparison':economic_replay.status(store,plant),'maintenance':retention.status(store.con)} def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()): allowed={str(UUID(p)) for p in plants} @@ -343,7 +347,7 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial') return factory() @app.get('/health') - def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1} + def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1,'applicationRelease':'unified-rc1','mappingCompatibilityVersion':1,'economicReplayVersion':1,'archiveVersion':1} @app.get('/internal/v2/prognosis/{plant}/planner') def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')): s=authorize(plant,token) @@ -369,6 +373,17 @@ def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc)) except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300]) finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}/mapping-compatibility') + def mapping_compatibility(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): + s=authorize(plant,token) + try: + from .mapping_identity import register + cfg=measurement_pipeline.configuration(s.con,plant,dataset) + return register(s.con,plant,cfg,payload,int(datetime.now(timezone.utc).timestamp())) + except (ValueError,KeyError,TypeError,AttributeError): + raise HTTPException(400, 'Mapping compatibility proof rejected') + finally:s.close() + @app.put('/internal/v2/prognosis/{plant}/planner/settings') def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')): s=authorize(plant,token) diff --git a/services/netplan-v4/netplan_v4/store.py b/services/netplan-v4/netplan_v4/store.py index 07d5254..236e376 100644 --- a/services/netplan-v4/netplan_v4/store.py +++ b/services/netplan-v4/netplan_v4/store.py @@ -4,7 +4,7 @@ import sqlite3 from datetime import datetime,timedelta from uuid import uuid4 from .domain import default_registry,month_key,number,quarter_start,utc -from . import meter_runtime, controlled_trial, measurement_pipeline +from . import meter_runtime, controlled_trial, economic_replay, retention, measurement_pipeline def canonical(value): return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False) @@ -33,6 +33,8 @@ class PlannerStore: ''') meter_runtime.schema(self.con) controlled_trial.schema(self.con) + economic_replay.schema(self.con) + retention.schema(self.con) measurement_pipeline.schema(self.con) def close(self):self.con.close() def settings(self,plant): diff --git a/services/netplan-v4/tests/test_economic_replay.py b/services/netplan-v4/tests/test_economic_replay.py new file mode 100644 index 0000000..3b5dc43 --- /dev/null +++ b/services/netplan-v4/tests/test_economic_replay.py @@ -0,0 +1,144 @@ +import copy +import json +import tempfile +import unittest +from dataclasses import asdict +from datetime import datetime,timedelta,timezone +from pathlib import Path +from netplan_v4 import economic_replay as e, measurement_pipeline as m +from netplan_v4.domain import Battery,Limits,Step,Price +from netplan_v4.store import PlannerStore +from netplan_v4.selection import choose_family +from test_measurement_pipeline import config,record +from test_v4 import AID + +AT=datetime(2026,10,2,22,tzinfo=timezone.utc) +END=AT+timedelta(days=1) +TS=int(AT.timestamp()) + + +def fixture(): + battery=Battery('b',20.,50.,10.,90.,5000.,5000.,AT,grid_charging=True) + steps=[Step(AT+timedelta(minutes=5*i),1000.,0.,Price(.1 if i<144 else .4),Price(.03)) for i in range(288)] + data={'steps':steps,'batteries':[battery],'limits':Limits(import_w=10000.),'observed_peaks':{'2026-10':5.},'peak_prices':{'2026-10':5.},'quarter_history':{},'at':AT} + def optimizer(**kw): + points=[] + family=kw['family'] + for i,s in enumerate(kw['steps']): + target=1000. if family in ('3','13') else (1600. if i<144 else 500.) + points.append({'time':s.start.isoformat(),'validUntil':s.end.isoformat(),'gridTargetW':target, + 'importPriceChfKwh':s.import_price.chf_kwh,'exportPriceChfKwh':s.export_price.chf_kwh}) + return {'executable':True,'points':points} + def assemble(*args):return copy.deepcopy(data),END,{} + return data,assemble,optimizer + + +class EconomicReplayTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite'));self.c=config() + m.register_dataset(self.s.con,AID,self.c,TS) + self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':self.c['datasetId'],'autoMinimumDays':1},0,AT) + self.data,self.assemble,self.optimizer=fixture() + def tearDown(self):self.s.close();self.tmp.cleanup() + def capture(self):e.capture(self.s,AID,AT,self.assemble,self.optimizer) + def cohort(self):return json.loads(self.s.con.execute('SELECT value FROM planner_economic_cohorts').fetchone()[0]) + def populate(self): + now=int(END.timestamp())+180 + rows=[record(t,self.c) for t in range(TS-60,now+1,30)] + for i in range(0,len(rows),120):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows[i:i+120]},now) + def outcomes(self,value=1000.):return {TS+300*i:{'loadW':value,'coverage':1.,'profileUsable':True} for i in range(288)} + def test_snapshot_frozen_once(self): + self.capture();first=self.cohort();self.capture();self.assertEqual(first,self.cohort()) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_no_retroactive_midday_snapshot(self): + e.capture(self.s,AID,AT+timedelta(hours=12),self.assemble,self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_silent_legacy_replay(self): + self.s.save_settings(AID,{'forecastSource':'legacy'},1,AT);self.capture() + self.assertEqual(e.scores(self.s,AID,END),[]) + def test_all_families_same_initial_energy(self): + self.capture();c=self.cohort();values=[e.simulate(c,f,self.outcomes()) for f in c['plans']] + self.assertEqual(len({v['initialEnergyKwh'] for v in values}),1) + for v in values:self.assertGreaterEqual(v['finalEnergyKwh'],2.-1e-6);self.assertLessEqual(v['finalEnergyKwh'],18.00001) + def test_profitable_time_shift_has_lower_normalized_cost(self): + self.capture();c=self.cohort();a=e.simulate(c,'3',self.outcomes());b=e.simulate(c,'23',self.outcomes()) + self.assertLess(b['costChf'],a['costChf']);self.assertTrue(b['terminalNormalized']) + def test_end_energy_not_free_savings(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=0.;p['importPriceChfKwh']=.4 + d=e.simulate(c,'3',self.outcomes());self.assertGreater(d['terminalAdjustmentChf'],0) + def test_missing_actual_not_zero(self): + self.capture();out=self.outcomes();out.pop(TS+300) + with self.assertRaises(ValueError):e.simulate(self.cohort(),'3',out) + def test_export_and_import_limits_not_ignored(self): + self.capture();c=self.cohort();c['context']['limits']['import_w']=500. + result=e.simulate(c,'3',self.outcomes(20000.));self.assertGreater(result['constraintBreaches'],0) + def test_peak_increment_not_total_billed_twice(self): + self.capture();c=self.cohort() + for p in c['plans']['3']['points']:p['gridTargetW']=6000. + result=e.simulate(c,'3',self.outcomes(6000.));self.assertAlmostEqual(result['peakCostChf'],5.) + def test_no_actual_results_before_day_end(self): + self.capture();self.populate();e.advance(self.s,AID,AT+timedelta(hours=12)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + def test_real_pipeline_from_frozen_plans_to_actuals_to_selection(self): + self.capture();self.populate();now=END+timedelta(minutes=4) + e.advance(self.s,AID,now);scores=e.scores(self.s,AID,now) + self.assertEqual(len(scores),3);self.assertEqual(scores[0].days,1) + chosen=choose_family('auto','3',scores,minimum_days=1,margin_chf=0,now=now) + self.assertEqual(chosen['family'],'23');self.assertEqual(chosen['mode'],'economic_replay') + self.assertTrue(e.status(self.s,AID)['connected']);self.assertFalse(e.status(self.s,AID)['isBillingEvidence']) + def test_evaluation_idempotent(self): + self.capture();self.populate();now=END+timedelta(minutes=4);e.advance(self.s,AID,now);e.advance(self.s,AID,now) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],1) + def test_missing_outcomes_preserve_cohort(self): + self.capture();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_results').fetchone()[0],0) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],1) + def test_changed_tariffs_between_families_reject_cohort(self): + def changed(**kwargs): + v=self.optimizer(**kwargs) + if kwargs['family']=='23':v['points'][0]['importPriceChfKwh']+=.5 + return v + e.capture(self.s,AID,AT,self.assemble,changed) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_no_grid_charging_unsupported_is_explicit(self): + self.data['batteries'][0]=__import__('dataclasses').replace(self.data['batteries'][0],grid_charging=False) + e.capture(self.s,AID,AT,lambda *args:(self.data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_short_price_horizon_no_fake_extension(self): + data=copy.deepcopy(self.data);data['steps']=data['steps'][:12] + e.capture(self.s,AID,AT,lambda *args:(data,END,{}),self.optimizer) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_economic_cohorts').fetchone()[0],0) + def test_other_plant_no_results(self): + self.capture();self.populate();e.advance(self.s,AID,END+timedelta(minutes=4)) + self.assertEqual(e.scores(self.s,'30509683-4569-49e4-848f-4905e4cc813a',END+timedelta(minutes=4)),[]) + def test_residual_projection_does_not_change_load_profile(self): + rows=[m.project(record(t,self.c),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1000.) + self.assertEqual(m.reconstruct(rows,self.c)[0]['loadW'],6000.) + def test_monthly_peak_not_charged_again_every_comparison_day(self): + self.capture();c=self.cohort() + for family in c['plans']: + for point in c['plans'][family]['points']:point['gridTargetW']=6000. + result={f:e.simulate(c,f,self.outcomes(6000.)) for f in c['plans']} + self.assertEqual(result['3']['peakCostChf'],5.) + policy=self.s.con.execute('SELECT policy_id FROM planner_economic_cohorts').fetchone()[0] + with self.s.con: + for n in (0,1): + start=AT+timedelta(days=n);end=END+timedelta(days=n);day=start.astimezone(__import__('zoneinfo').ZoneInfo('Europe/Zurich')).date().isoformat() + if n: + self.s.con.execute('INSERT INTO planner_economic_cohorts VALUES(?,?,?,?,?,?)',(AID,day,int(start.timestamp()),int(end.timestamp()),policy,json.dumps(c))) + v={'results':result,'start':start.isoformat(),'end':end.isoformat()} + self.s.con.execute('INSERT INTO planner_economic_results VALUES(?,?,?,?,?)',(AID,day,int(end.timestamp())+120,policy,json.dumps(v))) + scores=e.scores(self.s,AID,END+timedelta(days=1,minutes=4)) + value=next(x.cost_chf for x in scores if x.family=='3') + expected=2*(result['3']['energyCostChf']+result['3']['throughputCostChf']+result['3']['terminalAdjustmentChf'])+5. + self.assertAlmostEqual(value,expected) + + def test_external_sdl_not_assumed_zero(self): + rows=[m.project(record(t,self.c,sdl=250.),self.c,AID,TS) for t in range(TS-300,TS+1,30)] + self.assertEqual(e.actuals(rows,self.c)[0]['loadW'],1250.) + for r in rows:r['raw']['sdl']['valid']=False + self.assertFalse(e.actuals(rows,self.c)[0]['profileUsable']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_mapping_recovery.py b/services/netplan-v4/tests/test_mapping_recovery.py new file mode 100644 index 0000000..1532535 --- /dev/null +++ b/services/netplan-v4/tests/test_mapping_recovery.py @@ -0,0 +1,102 @@ +import copy +import json +import tempfile +import unittest +from pathlib import Path +from hashlib import sha256 +from netplan_v4.store import PlannerStore +from netplan_v4 import mapping_identity as mi, measurement_pipeline as m +from netplan_v4.service import create_app +from fastapi.testclient import TestClient +from test_measurement_pipeline import config, record, NOW +from test_v4 import AID, TOKEN + + +def proof(c): + a = {'version':1, 'installationId':AID, 'reportedInventorySha256':c['inventorySha256'], + 'accounting':{'splitToleranceW':100.0,'sources':copy.deepcopy(c['sources'])}} + b=copy.deepcopy(a); b['accounting']['splitToleranceW']=100 + p={'version':1, 'canonicalJson':json.dumps(a,separators=(',',':')), 'legacyJson':json.dumps(b,separators=(',',':'))} + c['mappingSha256']=sha256(p['canonicalJson'].encode()).hexdigest() + return p + + +class MappingRecoveryTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(); self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.c=config(); self.proof=proof(self.c); self.alias=sha256(self.proof['legacyJson'].encode()).hexdigest() + m.register_dataset(self.s.con,AID,self.c,NOW) + def tearDown(self): self.s.close(); self.tmp.cleanup() + def register(self): return mi.register(self.s.con,AID,self.c,self.proof,NOW) + def batch(self,rows): return m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':self.c['datasetId'],'records':rows},NOW) + def r(self,t,legacy=False): + r=record(t,self.c) + if legacy:r['mappingSha256']=self.alias + return r + def test_original_mixed_batch_defect_and_atomic_recovery(self): + rows=[self.r(NOW-3600+i*30,i>=45) for i in range(120)]; before=copy.deepcopy(rows) + with self.assertRaises(ValueError):self.batch(rows) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],0) + self.register(); result=self.batch(rows) + self.assertEqual(result['stored'],120); self.assertEqual(rows,before) + self.assertEqual(result['acceptedThrough'],rows[-1]['capturedAt']) + origins=self.s.con.execute('SELECT received_mapping,COUNT(*) FROM planner_observation_origins GROUP BY received_mapping').fetchall() + self.assertEqual(dict(origins)[self.alias],75) + def test_retry_does_not_duplicate(self): + self.register(); self.batch([self.r(NOW,True)]) + self.assertEqual(self.batch([self.r(NOW,True)])['duplicates'],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1) + def test_canonical_retry_preserves_both_origins(self): + self.register(); self.batch([self.r(NOW,True)]); self.batch([self.r(NOW)]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],2) + def test_unknown_alias_rolls_back_entire_block(self): + self.register(); bad=self.r(NOW,True);bad['mappingSha256']='f'*64 + with self.assertRaises(ValueError): self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],0) + def test_conflicting_measurement_keeps_existing_and_does_not_ack_prefix(self): + self.register(); self.batch([self.r(NOW)]) + bad=self.r(NOW,True);bad['raw']['grid']['value']+=500 + with self.assertRaises(ValueError):self.batch([self.r(NOW-30),bad]) + self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1) + def test_proof_wrong_canonical_hash_rejected(self): + p=copy.deepcopy(self.proof);p['canonicalJson']+=' ' + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_numeric_change_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":101,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_boolean_rejected(self): + p=copy.deepcopy(self.proof);p['legacyJson']=p['legacyJson'].replace('"splitToleranceW":100,','"splitToleranceW":true,') + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_other_field_change_rejected(self): + p=copy.deepcopy(self.proof);b=json.loads(p['legacyJson']);b['accounting']['sources'][0]['factorToW']=2;p['legacyJson']=json.dumps(b) + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,p,NOW) + def test_inventory_remains_strict(self): + self.register();r=self.r(NOW,True);r['reportedInventorySha256']='d'*64 + with self.assertRaises(ValueError):self.batch([r]) + def test_other_plant_does_not_inherit_proof(self): + self.register();other='30509683-4569-49e4-848f-4905e4cc813a';m.register_dataset(self.s.con,other,self.c,NOW) + r=self.r(NOW,True);r['installationId']=other + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,other,{'version':1,'datasetId':self.c['datasetId'],'records':[r]},NOW) + def test_other_dataset_does_not_inherit_proof(self): + self.register();c=copy.deepcopy(self.c);c['datasetId']='unrelated';m.register_dataset(self.s.con,AID,c,NOW) + with self.assertRaises(ValueError):m.ingest_batch(self.s.con,AID,{'version':1,'datasetId':'unrelated','records':[self.r(NOW,True)]},NOW) + def test_direct_project_still_strict(self): + self.register() + with self.assertRaises(ValueError):m.project(self.r(NOW,True),self.c,AID,NOW) + def test_registration_idempotent_and_persistent(self): + a=self.register();b=self.register();self.assertEqual(a,b) + self.s.close();self.s=PlannerStore(str(Path(self.tmp.name)/'db.sqlite')) + self.assertEqual(self.batch([self.r(NOW,True)])['stored'],1) + def test_malformed_proof_no_registration(self): + for x in ({}, {'version':True,'canonicalJson':'{}','legacyJson':'{}'}): + with self.assertRaises(ValueError):mi.register(self.s.con,AID,self.c,x,NOW) + def test_internal_api_authentication(self): + app=create_app(str(Path(self.tmp.name)/'db.sqlite'),TOKEN,[AID],start_worker=False) + with TestClient(app) as client: + url=f'/internal/v2/prognosis/{AID}/planner/datasets/{self.c["datasetId"]}/mapping-compatibility' + self.assertEqual(client.put(url,json=self.proof).status_code,401) + response=client.put(url,json=self.proof,headers={'X-Enelix-Service-Token':TOKEN}) + self.assertEqual(response.status_code,200,response.text);self.assertFalse(response.json()['controlEnabled']) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_release_preflight.py b/services/netplan-v4/tests/test_release_preflight.py index 977a292..d4942a9 100644 --- a/services/netplan-v4/tests/test_release_preflight.py +++ b/services/netplan-v4/tests/test_release_preflight.py @@ -42,7 +42,8 @@ class ReleasePreflightTest(unittest.TestCase): text=(ROOT/'gui/netplan-v4.js').read_text() self.assertIn('SCHATTENBETRIEB',text) self.assertIn('measurementPolicy',text) - self.assertIn('wartet weiterhin auf vergleichbare Kosten-Replays',text) + self.assertIn('erstellt vergleichbare Kosten-Replays',text) + self.assertIn('keine vollstaendige Nachbildung laufender Neuoptimierungen',text) self.assertIn('forecastSource',text) self.assertIn('trainingCadence',text) self.assertIn('Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe',text) diff --git a/services/netplan-v4/tests/test_retention.py b/services/netplan-v4/tests/test_retention.py new file mode 100644 index 0000000..910b56e --- /dev/null +++ b/services/netplan-v4/tests/test_retention.py @@ -0,0 +1,56 @@ +import gzip,hashlib,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +from netplan_v4.store import PlannerStore +from netplan_v4 import retention +from test_v4 import AID + +NOW=1791028800 + +class RetentionTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name);self.s=PlannerStore(str(self.root/'db.sqlite'));self.archive=self.root/'archive' + def tearDown(self):self.s.close();self.tmp.cleanup() + def old(self,ago=121,received_ago=10): + v='{"numeric":123}';fp=hashlib.sha256(v.encode()).hexdigest() + with self.s.con: + self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,NOW-received_ago*86400,fp,v)) + self.s.con.execute('INSERT INTO planner_observation_origins VALUES(?,?,?,?,?,?,?)',(AID,'ds',NOW-ago*86400,'m','i','e',NOW-received_ago*86400)) + def count(self):return self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0] + def test_lossless_archive_before_removal(self): + self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations');self.assertEqual(r['archived'],1);self.assertEqual(self.count(),0) + raw=gzip.decompress((self.archive/r['file']).read_bytes());row=json.loads(raw) + self.assertEqual(row['row']['value'],'{"numeric":123}');self.assertEqual(row['origins'][0]['received_mapping'],'m') + self.assertEqual(hashlib.sha256(raw).hexdigest(),r['sha256']) + def test_recent_and_late_received_rows_not_pruned(self): + self.old(ago=100);self.old(ago=121,received_ago=1) + self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0);self.assertEqual(self.count(),2) + def test_failed_durability_keeps_original(self): + self.old() + with patch('netplan_v4.retention.os.fsync',side_effect=OSError('synthetic')): + with self.assertRaises(OSError):retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(self.count(),1);self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observation_origins').fetchone()[0],1) + def test_symlink_archive_rejected(self): + self.old();other=self.root/'other';other.mkdir();self.archive.symlink_to(other) + with self.assertRaises(ValueError):retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(self.count(),1) + def test_current_and_acknowledged_plans_retained(self): + with self.s.con: + for key in ('old','current','ack'): + self.s.con.execute('INSERT INTO planner_plans VALUES(?,?,?,?,?,?)',(key,AID,0,'shadow','{}','2020-01-01T00:00:00+00:00')) + self.s.con.execute('INSERT INTO planner_current VALUES(?,?,?)',(AID,'shadow','current')) + self.s.con.execute('INSERT INTO planner_ack VALUES(?,?,?,?,?,?)',(AID,'ack',0,'2020-01-01T00:00:00+00:00',None,'shadow_seen')) + r=retention.archive_batch(self.s.con,self.archive,NOW,'plans');self.assertEqual(r['archived'],1) + self.assertEqual({r[0] for r in self.s.con.execute('SELECT plan_id FROM planner_plans')},{'current','ack'}) + def test_archived_rows_can_be_restored_exactly(self): + self.old();r=retention.archive_batch(self.s.con,self.archive,NOW,'observations') + obj=json.loads(gzip.decompress((self.archive/r['file']).read_bytes()));row=obj['row'] + with self.s.con:self.s.con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',tuple(row[k] for k in ('plant','dataset','captured_at','received_at','fingerprint','value'))) + self.assertEqual(self.count(),1) + self.assertEqual(self.s.con.execute('SELECT fingerprint FROM planner_observations').fetchone()[0],hashlib.sha256(row['value'].encode()).hexdigest()) + def test_no_duplicate_archive_after_repeat(self): + self.old();retention.archive_batch(self.s.con,self.archive,NOW,'observations') + self.assertEqual(retention.archive_batch(self.s.con,self.archive,NOW,'observations')['archived'],0) + self.assertEqual(len(list(self.archive.glob('*.gz'))),1) + +if __name__=='__main__':unittest.main() diff --git a/services/netplan-v4/tests/test_unified_release.py b/services/netplan-v4/tests/test_unified_release.py new file mode 100644 index 0000000..e010e34 --- /dev/null +++ b/services/netplan-v4/tests/test_unified_release.py @@ -0,0 +1,81 @@ +import importlib.util,json,hashlib,tempfile,unittest,sqlite3,subprocess,sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from test_v4 import AID + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'commissioning')) +import deploy_unified_release as dep + + +class UnifiedReleaseTest(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.root=Path(self.tmp.name)/'service';self.root.mkdir() + self.package=self.root/'commissioning/unified-release';(self.package/'context/netplan_v4').mkdir(parents=True) + (self.package/'context/gui').mkdir();(self.root/'netplan_v4').mkdir();(self.root/'gui').mkdir();(self.root/'data').mkdir() + self.gui=Path(self.tmp.name)/'portal.js';self.gui.write_text('old ui') + self.proof=self.package/'mapping-proof.json';self.proof.write_text('{}') + helper=self.root/'commissioning/deploy_application.py';helper.write_bytes((ROOT/'commissioning/deploy_application.py').read_bytes()) + self.files={} + for name,old,new in [('netplan_v4/code.py','old','new'),('gui/netplan-v4.js','old ui','new ui')]: + (self.root/name).write_text(old);(self.package/'context'/name).write_text(new) + self.files[name]={'before':hashlib.sha256(old.encode()).hexdigest(),'after':hashlib.sha256(new.encode()).hexdigest()} + with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:c.execute('CREATE TABLE original(value TEXT)');c.execute("INSERT INTO original VALUES('kept')") + manifest={'scope':'unified_v4_candidate','releaseId':'a'*64,'installationId':AID,'files':self.files, + 'installerSha256':dep.digest(Path(dep.__file__).resolve()),'guiBefore':dep.digest(self.gui),'proofSha256':dep.digest(self.proof),'backupHelperSha256':dep.digest(helper)} + (self.package/'RELEASE.json').write_text(json.dumps(manifest));self.calls=[];self.fail=None + self.patches=[patch.object(dep,'ROOT',self.root),patch.object(dep,'PACKAGE',self.package),patch.object(dep,'GUI',self.gui),patch.object(dep.os,'geteuid',return_value=0),patch.object(dep.os,'chown'),patch.object(dep.subprocess,'run',side_effect=self.command)] + for p in self.patches:p.start() + def tearDown(self): + for p in reversed(self.patches):p.stop() + self.tmp.cleanup() + def command(self,args,**kwargs): + self.calls.append(args) + if self.fail=='build' and args[:2]==['docker','build']:raise subprocess.CalledProcessError(1,args) + if args[0]=='node':return SimpleNamespace(stdout='') + if args[:2]==['docker','exec']:return SimpleNamespace(stdout=AID+'\n') + if args[:2]==['docker','inspect']:return SimpleNamespace(stdout='sha256:'+'1'*64+'\n') + if args[:3]==['docker','image','inspect']:return SimpleNamespace(stdout='sha256:'+'2'*64+'\n') + if 'ps' in args:return SimpleNamespace(stdout='container\n') + if 'exec' in args: + if self.fail=='bootstrap':raise subprocess.CalledProcessError(1,args) + return SimpleNamespace(stdout=json.dumps({'settingsUnchanged':True,'health':{'liveEnabled':False}})) + return SimpleNamespace(stdout='') + def test_source_only_never_deploys(self): + dep.run(AID);self.assertEqual(self.calls,[]);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old') + def test_drift_stops_before_docker(self): + (self.root/'netplan_v4/code.py').write_text('other work') + with self.assertRaises(ValueError):dep.run(AID,True) + self.assertEqual(self.calls,[]) + def test_gui_parallel_change_preserved(self): + self.gui.write_text('parallel gui') + with self.assertRaises(ValueError):dep.run(AID,True) + self.assertEqual(self.gui.read_text(),'parallel gui') + def test_tests_before_replacement_and_no_other_service(self): + dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new') + tests=next(i for i,c in enumerate(self.calls) if c[:2]==['docker','run']) + start=next(i for i,c in enumerate(self.calls) if 'up' in c) + self.assertLess(tests,start);self.assertIn('none',self.calls[tests]);self.assertIn('--read-only',self.calls[tests]) + for c in self.calls: + if 'up' in c:self.assertEqual(c[-1],'netplan-v4') + self.assertEqual(self.gui.read_text(),'new ui') + with sqlite3.connect(self.root/'data/netplan-v4.sqlite') as c:self.assertEqual(c.execute('SELECT value FROM original').fetchone()[0],'kept') + def test_failed_build_changes_no_running_sources(self): + self.fail='build' + with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True) + self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertFalse(any('up' in c for c in self.calls)) + def test_bootstrap_failure_restores_previous_runtime_sources(self): + self.fail='bootstrap' + with self.assertRaises(subprocess.CalledProcessError):dep.run(AID,True) + self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'old');self.assertEqual(self.gui.read_text(),'old ui') + self.assertTrue(any('--pull' in c and 'never' in c for c in self.calls)) + def test_repeat_is_safe(self): + dep.run(AID,True);dep.run(AID,True);self.assertEqual((self.root/'netplan_v4/code.py').read_text(),'new') + def test_no_test_or_production_dispatch_grant(self): + text=Path(dep.__file__).read_text() + self.assertNotIn('/trial/arm',text);self.assertNotIn("'runMode':'live'",text);self.assertNotIn('SET NetzfahrplanV4DatenCursor',text) + def test_dockerfile_copies_installers_needed_by_test_suite(self): + self.assertIn('COPY commissioning/deploy_unified_release.py ./commissioning/deploy_unified_release.py',(ROOT/'Dockerfile').read_text()) + +if __name__=='__main__':unittest.main() diff --git a/tests/V4ApplicationData/archive_checks.php b/tests/V4ApplicationData/archive_checks.php new file mode 100644 index 0000000..d5654ef --- /dev/null +++ b/tests/V4ApplicationData/archive_checks.php @@ -0,0 +1,27 @@ +isDir()&&!$p->isLink()?rmdir($p->getPathname()):unlink($p->getPathname());}rmdir($dir);} +$base=sys_get_temp_dir().'/v4-archive-'.bin2hex(random_bytes(6));mkdir($base,0700); +try { + $old='raw-'.gmdate('Ymd',$now-10*86400).'.jsonl';$recent='raw-'.gmdate('Ymd',$now-2*86400).'.jsonl';$day='raw-'.gmdate('Ymd',$now).'.jsonl'; + $raw="{\"sample\":1}\n{\"sample\":2}\n";file_put_contents($base.'/'.$old,$raw);file_put_contents($base.'/'.$recent,$raw);file_put_contents($base.'/'.$day,$raw); + checkArchive(A::run($base,[],$now)['archived']===0,'no acknowledgement means no archival'); + $r=A::run($base,['day'=>$old,'offset'=>5],$now);checkArchive($r['archived']===0&&is_file($base.'/'.$old),'partially acknowledged day retained'); + $r=A::run($base,['day'=>$day,'offset'=>5],$now);checkArchive($r['archived']===1&&!is_file($base.'/'.$old),'only completely acknowledged old day retired'); + $files=glob($base.'/archive/*.gz');checkArchive(count($files)===1&&gzdecode(file_get_contents($files[0]))===$raw,'compressed original verified byte for byte'); + checkArchive(is_file($base.'/'.$recent)&&is_file($base.'/'.$day),'recent and current outbox preserved'); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===0,'repeat is idempotent'); + file_put_contents($base.'/'.$old,$raw);file_put_contents($files[0],gzencode('corrupt')); + try {A::run($base,['day'=>$day,'offset'=>5],$now);throw new LogicException('expected refusal');} catch(RuntimeException $e) {checkArchive(is_file($base.'/'.$old),'archive corruption never loses original');} + unlink($files[0]);$lock=fopen($base.'/.writer.lock','c+b');flock($lock,LOCK_EX); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['status']==='writer_busy','archive serializes with actual capture writer');flock($lock,LOCK_UN);fclose($lock); + foreach(range(11,15)as$d)file_put_contents($base.'/raw-'.gmdate('Ymd',$now-$d*86400).'.jsonl',$raw); + checkArchive(A::run($base,['day'=>$day,'offset'=>5],$now)['archived']===2,'bounded archive work per invocation'); + $text=file_get_contents(__DIR__.'/../../libs/NetzfahrplanV4Datenarchiv.php');checkArchive(!preg_match('/\b(?:RequestAction|IPS_SetProperty|curl_exec)\s*\(/',$text),'archive has no control or network calls'); + echo "TOTAL $n archive checks passed; only temporary test data.\n"; +} finally {cleanupArchive($base);} diff --git a/tests/V4ApplicationData/checks.php b/tests/V4ApplicationData/checks.php index 5f018e9..318bbb3 100644 --- a/tests/V4ApplicationData/checks.php +++ b/tests/V4ApplicationData/checks.php @@ -63,16 +63,29 @@ try { check($sent[0]['records'][0]['raw']['s101']['sourceUpdatedAt']===$clock,'original timestamp retained'); check(!str_contains(json_encode($sent),'synthetic-not-a-real-token'),'token never included in data payload'); check($sent[0]['records'][0]['controlEligible']===false,'data delivery is not actuator permission'); + $expected=hash('sha256',json_encode(NetzfahrplanV4Messaufnahme::configuration($c),JSON_THROW_ON_ERROR|JSON_PRESERVE_ZERO_FRACTION)); + check($sent[0]['records'][0]['mappingSha256']===$expected,'property roundtrip preserves approved mapping identity'); + check(json_decode($m->a['NetzfahrplanV4LetzteDatenquittung'],true)['acceptedThrough']===end($sent[0]['records'])['capturedAt'],'successful acknowledgement persisted independently'); $clock+=30;$m->ErfasseNetzfahrplanV4Daten();check(count($sent)===1,'30-second collection does not double request rate'); check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'unsent rows remain behind cursor'); + check(isset(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['lastAcknowledgement']['acceptedThrough']),'scheduled status retains last successful receipt'); $clock+=30;$mode='429';$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'429 never advances acknowledgement cursor'); check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['dataRetained'],'offline data retained'); + check(json_decode($m->vars['NetzfahrplanV4Datenstatus'],true)['delivery']['httpStatus']===429,'safe HTTP error code visible without credentials'); $mode='wrong_ack';$clock+=120;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']===$cursor,'wrong ack timestamp rejected'); $mode='ok';$clock+=240;$m->ErfasseNetzfahrplanV4Daten();check($m->a['NetzfahrplanV4DatenCursor']!==$cursor,'later success sends the preserved backlog'); check(count(end($sent)['records'])===4,'all previously unacknowledged rows included once'); $file=glob($m->folder().'/raw-*.jsonl')[0];$count=count(file($file));check($count===5,'raw journal never cleared by upload'); $lastCursor=$m->a['NetzfahrplanV4DatenCursor'];file_put_contents($file,'incomplete',FILE_APPEND); check($m->send()['status']==='up_to_date','in-progress tail not transmitted');check($m->a['NetzfahrplanV4DatenCursor']===$lastCursor,'partial tail not acknowledged'); + // Only temporary test journal: turn the prior incomplete tail into an additional complete queued record. + $buf=file_get_contents($file);$buf=substr($buf,0,-strlen('incomplete')); + $extra=$sent[0]['records'][0];$clock+=60;$extra['capturedAt']=gmdate('c',$clock);$extra['captureStartedAt']=$extra['capturedAt']; + file_put_contents($file,$buf.json_encode($extra)."\n".'crash-tail'); + $clock+=60;$beforeCount=count($sent);$m->ErfasseNetzfahrplanV4Daten(); + $err=json_decode($m->vars['NetzfahrplanV4Datenstatus'],true); + check($err['status']==='capture_error'&&$err['delivery']['status']==='acknowledged','capture failure does not deadlock pending delivery'); + check(count($sent)===$beforeCount+1&&str_ends_with(file_get_contents($file),'crash-tail'),'complete prefix sent while original incomplete tail retained'); $m->p['NetzfahrplanV4MessdatenAktiv']=false;$m->configure();check($m->timers['NetzfahrplanV4DatenErfassen']===0,'disable affects acquisition only'); check(count($m->vars)===1,'only own manager data-status variable written'); $text=file_get_contents(__DIR__.'/../../libs/ManagerNetzfahrplanV4DatenTrait.php');