feat(application): integrate measured-load ingestion training and planner source

2026-10-02 21:00:05 +00:00
parent 44362e4bf5
commit fc024fcfba
96 changed files with 8933 additions and 0 deletions
@@ -0,0 +1,175 @@
+"""Install the reviewed server-side SHADOW integration; no Symcon/device writes.
+
+Default: verify source and print scope only. --apply requires root. Build and test
+all changed code before recreating services. Save old image/config references and
+an integrity-checked online backup of the V4 DB. Does not enable native telemetry
+or dispatch: operation ingress may still be awaiting installation on Symcon.
+"""
+from datetime import datetime, timezone
+from pathlib import Path
+import argparse
+import hashlib
+import json
+import os
+import sqlite3
+import subprocess
+import sys
+from uuid import UUID
+
+ROOT = Path(__file__).resolve().parent
+SERVICES = ROOT.parent
+PROJECT = SERVICES / 'prognosis-manager-enelix2'
+PORTAL = SERVICES / 'license'
+PREFLIGHT = ROOT / 'acceptance-reports/20261002T043021Z/REPORT.json'
+MANIFEST = ROOT / 'INTEGRATED_SHADOW_SOURCE.json'
+
+
+def verify_sources():
+    report = json.loads(PREFLIGHT.read_text())
+    if report.get('preflightChecksPassed') is not True:
+        raise ValueError('Expected successful earlier runtime preflight is missing')
+    expected = json.loads(MANIFEST.read_text())['files']
+    for relative, digest in expected.items():
+        path = SERVICES / relative
+        if not path.resolve().is_relative_to(SERVICES) or path.is_symlink() or not path.is_file():
+            raise ValueError('Invalid release source path: ' + relative)
+        if hashlib.sha256(path.read_bytes()).hexdigest() != digest:
+            raise ValueError('Source changed since preparation: ' + relative)
+    from forecast_acceptance import verify_forecast_bundle
+    verify_forecast_bundle(ROOT/'acceptance/forecast-src', PROJECT/'forecast_engine')
+    from release_preflight import verify_native_bundle
+    verify_native_bundle()
+    return expected
+
+
+def compose_groups():
+    return {
+        'v4': (ROOT, [ROOT/'compose.yaml'], ['netplan-v4']),
+        'forecast': (PROJECT, [PROJECT/'compose.yaml', ROOT/'compose.forecast-bridge.yaml'], ['api','forecast-engine','tariff-importer']),
+        'portal': (PORTAL, [PORTAL/'compose.yaml', ROOT/'compose.portal-bridge.yaml'], ['license-portal']),
+    }
+
+
+def compose_args(files):
+    result=['docker','compose']
+    for path in files:result.extend(['-f',str(path)])
+    return result
+
+
+def backup_database(source, destination):
+    if not source.exists():return {'exists':False}
+    src=sqlite3.connect(source.resolve().as_uri()+'?mode=ro',uri=True,timeout=15)
+    dst=sqlite3.connect(destination,timeout=15)
+    try:
+        src.backup(dst)
+        if dst.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
+            raise ValueError('V4 database backup failed integrity check')
+    finally:dst.close();src.close()
+    os.chmod(destination,0o640)
+    if os.geteuid()==0:os.chown(destination,1000,1000)
+    return {'exists':True,'path':str(destination),'sha256':hashlib.sha256(destination.read_bytes()).hexdigest(),'integrity':'ok'}
+
+
+def save_json(path, value):
+    path.write_text(json.dumps(value,indent=2,allow_nan=False)+'\n')
+    os.chmod(path,0o640)
+    if os.geteuid()==0:os.chown(path,1000,1000)
+
+
+def run(plant):
+    sources=verify_sources()
+    env=dict(os.environ,NETPLAN_V4_PLANTS=plant)
+    stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
+    folder=ROOT/'deployment-reports'/stamp
+    folder.mkdir(parents=True,exist_ok=False)
+    if os.geteuid()==0:
+        os.chown(folder.parent,1000,1000);os.chown(folder,1000,1000)
+    report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant,
+            'scope':'server_shadow_only','liveEnabled':False,'productionReady':False,
+            'sourceHashes':sources,'steps':[],'before':{},'existingServiceRecreated':False}
+    def command(args,cwd=ROOT,capture=False,stdin=None,timeout=900):
+        result=subprocess.run(args,cwd=cwd,env=env,input=stdin,text=True,
+                              stdout=subprocess.PIPE if capture else None,
+                              stderr=subprocess.PIPE if capture else None,timeout=timeout,check=False)
+        if result.returncode:
+            raise RuntimeError('Step failed (exit '+str(result.returncode)+'): '+args[0])
+        return result.stdout if capture else ''
+    def step(name,args,cwd=ROOT,timeout=900):
+        print('\n=== '+name+' ===',flush=True)
+        command(args,cwd,timeout=timeout)
+        report['steps'].append(name);save_json(folder/'DEPLOYMENT.json',report)
+    try:
+        # Capture only selected nonsecret Docker fields, never the environment.
+        for group,(cwd,files,services) in compose_groups().items():
+            oldfiles=[files[0]]
+            items=[]
+            for service in services:
+                cid=command(compose_args(oldfiles)+['ps','-q',service],cwd,True).strip()
+                if not cid or '\n' in cid:raise ValueError('Expected exactly one existing container: '+service)
+                image=command(['docker','inspect','--format','{{.Image}}',cid],cwd,True).strip()
+                configs=command(['docker','inspect','--format','{{index .Config.Labels "com.docker.compose.project.config_files"}}',cid],cwd,True).strip()
+                items.append({'service':service,'containerId':cid,'imageId':image,'previousComposeFiles':configs})
+            report['before'][group]=items
+        save_json(folder/'DEPLOYMENT.json',report)
+        step('review optional source installer',[sys.executable,str(ROOT/'install_hooks.py')])
+        # All new forecasting tests run offline in a disposable Python 3.11 image.
+        step('build forecast acceptance image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')])
+        step('forecast candidate offline tests',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local'])
+        step('V4 and bridge target tests',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only'])
+        verify_sources()
+        from install_hooks import plan,apply
+        receipt=apply(plan())
+        report['sourceInstallReceipt']=str(receipt) if receipt else None
+        step('portal syntax',['node','--check',str(PORTAL/'server.mjs')])
+        # Build every production-shaped candidate before changing a running container.
+        cwd,files,svcs=compose_groups()['forecast']
+        step('build server-side images',compose_args(files)+['build']+svcs,cwd)
+        report['v4DatabaseBackup']=backup_database(ROOT/'data/netplan-v4.sqlite',folder/'netplan-v4-before.sqlite')
+        report['existingServiceRecreated']=True
+        save_json(folder/'DEPLOYMENT.json',report)
+        for group,(cwd,files,svcs) in compose_groups().items():
+            step('start shadow integration '+group,compose_args(files)+['up','-d','--no-deps','--no-build','--wait','--wait-timeout','180']+svcs,cwd,timeout=300)
+        cwd,files,_=compose_groups()['v4']
+        # Check shadow guard and report incoming types without disclosing payloads.
+        probe='''import json,os,sqlite3,urllib.request
+from datetime import datetime,timezone
+plant=os.environ['NETPLAN_V4_PLANTS']
+health=json.loads(urllib.request.urlopen('http://127.0.0.1:9100/health',timeout=5).read())
+assert health.get('mode')=='shadow' and health.get('liveEnabled') is False
+con=sqlite3.connect('file:/data/netplan-v4.sqlite?mode=ro',uri=True)
+rows=con.execute('SELECT kind,count(*) FROM planner_inputs WHERE plant=? GROUP BY kind',(plant,)).fetchall()
+status=con.execute('SELECT status FROM planner_run_status WHERE plant=?',(plant,)).fetchone()
+con.close()
+print(json.dumps({'checkedAt':datetime.now(timezone.utc).isoformat(),'health':health,'inputEventsByKind':dict(rows),'lastRunStatus':status[0] if status else None,'nativeSenderInstalledByThisCommand':False}))
+'''
+        report['readiness']=json.loads(command(compose_args(files)+['exec','-T','netplan-v4','python','-B','-'],cwd,True,probe,60))
+        report['status']='server_shadow_installed'
+        report['remaining']='Native operation sender, plant-level initialization and real shadow-plan acceptance still required; no live dispatch installed'
+    except Exception as error:
+        report['status']='failed';report['errorType']=type(error).__name__;report['error']=str(error)[:500]
+        report['remaining']='Inspect completed steps and before-image references. No automatic database restore or device changes.'
+        raise
+    finally:
+        save_json(folder/'DEPLOYMENT.json',report)
+        print('\nDEPLOYMENT REPORT: '+str(folder/'DEPLOYMENT.json'),flush=True)
+        print('SERVER SHADOW ONLY. Symcon, device permissions and actuators were not modified.',flush=True)
+    print(json.dumps(report['readiness'],indent=2))
+    print('Native operating-state input may still be missing; this is NOT production commissioning.')
+
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--plant',required=True,type=lambda p:str(UUID(p)))
+    parser.add_argument('--apply',action='store_true')
+    args=parser.parse_args()
+    if args.plant!='e3a08f9e-af12-4695-99bd-8b51c0520021':
+        raise SystemExit('This first integration rollout is allowlisted for Lihrenmoos only.')
+    if not args.apply:
+        files=verify_sources()
+        print('CHECK ONLY:',len(files),'reviewed source files. No running service changed.')
+        for group,(_,_,svcs) in compose_groups().items():print(group,', '.join(svcs))
+    else:
+        if os.geteuid()!=0:raise SystemExit('Run as root; do not change Docker socket access.')
+        try:run(args.plant)
+        except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as error:
+            raise SystemExit('Deployment stopped: '+str(error))