fix(history): model sparse publication without relaxing live freshness
This commit is contained in:
ENELIX Agent
@@ -0,0 +1,146 @@
|
||||
"""Install the tested historical-publication correction in the V4 service only.
|
||||
|
||||
Default is source validation. --apply requires root. No Symcon changes, polling,
|
||||
portal restart, old-data rewrite, forecast-source switch or actuator permission.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PACKAGE = Path(__file__).with_name('history-timing-source')
|
||||
TARGETS = ('Dockerfile', 'netplan_v4/history_timing.py', 'netplan_v4/measurement_pipeline.py',
|
||||
'tests/test_history_timing.py', 'tests/test_history_timing_deployment.py')
|
||||
|
||||
|
||||
def digest(path):
|
||||
if path.is_symlink():
|
||||
raise ValueError('Source symlink refused')
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
|
||||
|
||||
|
||||
def verify():
|
||||
manifest = json.loads((PACKAGE/'RELEASE.json').read_text())
|
||||
if manifest.get('scope') != 'historical_publication_only' or tuple(manifest.get('files', {})) != TARGETS:
|
||||
raise ValueError('Unexpected timing release scope')
|
||||
for name, versions in manifest['files'].items():
|
||||
if digest(PACKAGE/'source'/name) != versions['after']:
|
||||
raise ValueError('Timing candidate changed: '+name)
|
||||
if digest(ROOT/name) not in (versions['before'], versions['after']):
|
||||
raise ValueError('Concurrent application change: '+name)
|
||||
if digest(PACKAGE/'dataset.json') != manifest['datasetSha256']:
|
||||
raise ValueError('Dataset candidate changed')
|
||||
if digest(ROOT/'commissioning/deploy_application.py') != manifest['existingDeploymentHelperSha256']:
|
||||
raise ValueError('Existing deployment helper changed')
|
||||
if digest(Path(__file__).resolve()) != manifest['installerSha256']:
|
||||
raise ValueError('Timing installer changed')
|
||||
return manifest
|
||||
|
||||
|
||||
BOOTSTRAP = r'''import json,os,sys,time,urllib.request
|
||||
payload=json.load(sys.stdin)
|
||||
base='http://127.0.0.1:9100/internal/v2/prognosis/'+payload['plant']+'/planner'
|
||||
headers={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'}
|
||||
def get():
|
||||
return json.load(urllib.request.urlopen(urllib.request.Request(base,headers=headers),timeout=10))
|
||||
before=get()
|
||||
assert before['dataPipeline'].get('historyTimingVersion')==1
|
||||
assert before['liveEnabled'] is False
|
||||
req=urllib.request.Request(base+'/datasets/'+payload['dataset']['datasetId'],
|
||||
data=json.dumps(payload['dataset']).encode(),headers=headers,method='PUT')
|
||||
receipt=json.load(urllib.request.urlopen(req,timeout=10))
|
||||
after=get()
|
||||
assert before['settings']==after['settings'] and after['liveEnabled'] is False
|
||||
view=next(d for d in after['dataPipeline']['datasets'] if d['datasetId']==payload['dataset']['datasetId'])
|
||||
print(json.dumps({'dataset':receipt,'originalObservationDataset':view['observationDatasetId'],
|
||||
'availableRecords':view['records'],'pipelineStatus':view['status'],
|
||||
'settingsUnchanged':True,'forecastSource':after['settings']['forecastSource'],
|
||||
'liveEnabled':False}))
|
||||
'''
|
||||
|
||||
|
||||
def run(plant, apply=False):
|
||||
manifest = verify()
|
||||
if plant != manifest['installationId']:
|
||||
raise ValueError('Use the reviewed installation')
|
||||
if not apply:
|
||||
print('HISTORY TIMING SOURCE CHECK PASSED; no runtime changes.')
|
||||
return
|
||||
if os.geteuid() != 0:
|
||||
raise ValueError('Run as root; do not change Docker permissions')
|
||||
# Reuse the reviewed atomic-write and consistent-backup implementation.
|
||||
from deploy_application import atomic, backup_database
|
||||
folder = ROOT/'history-timing-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ')
|
||||
folder.mkdir(parents=True, mode=0o700)
|
||||
result={'scope':'historical_publication_only','startedAt':datetime.now(timezone.utc).isoformat(),
|
||||
'steps':[],'liveEnabled':False,'symconChanged':False,'pollingChanged':False,
|
||||
'sourceFreshnessLimitsChanged':False,'forecastSelectionChanged':False}
|
||||
compose=['docker','compose','-f',str(ROOT/'compose.yaml')]
|
||||
env={**os.environ,'NETPLAN_V4_PLANTS':plant}
|
||||
def cmd(args, timeout=180, capture=False, data=None):
|
||||
return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=data,
|
||||
stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None)
|
||||
changed={}; previous_image=None; replaced=False
|
||||
try:
|
||||
ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split()
|
||||
if len(ids)!=1: raise ValueError('Expected one running V4 container')
|
||||
previous_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip()
|
||||
result['previousImage']=previous_image
|
||||
for name, versions in manifest['files'].items():
|
||||
target=ROOT/name; content=(PACKAGE/'source'/name).read_bytes()
|
||||
before=target.read_bytes() if target.is_file() else None
|
||||
if digest(target)==versions['after']: continue
|
||||
if digest(target)!=versions['before']: raise ValueError('Concurrent source change')
|
||||
backup=folder/'source-before'/name; backup.parent.mkdir(parents=True,exist_ok=True)
|
||||
if before is not None: backup.write_bytes(before)
|
||||
atomic(target,content); changed[name]=(before,content)
|
||||
verify()
|
||||
cmd(compose+['build','netplan-v4'],timeout=900)
|
||||
cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240)
|
||||
result['steps'].append('target_tests_passed'); verify()
|
||||
backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite')
|
||||
result['steps'].append('consistent_database_backup')
|
||||
rollback=folder/'rollback.yaml'
|
||||
rollback.write_text('services:\n netplan-v4:\n image: '+previous_image+'\n')
|
||||
replaced=True
|
||||
cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4'])
|
||||
payload=json.dumps({'plant':plant,'dataset':json.loads((PACKAGE/'dataset.json').read_text())})
|
||||
response=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,data=payload)
|
||||
result['application']=json.loads(response.stdout)
|
||||
result['status']='historical_timing_installed_no_actuation'
|
||||
result['steps'].append('versioned_dataset_registered_original_observations_retained')
|
||||
except Exception as exc:
|
||||
result['status']='needs_review';result['errorType']=type(exc).__name__
|
||||
conflicts=[]
|
||||
for name,(before,after) in changed.items():
|
||||
path=ROOT/name
|
||||
if digest(path)!=hashlib.sha256(after).hexdigest(): conflicts.append(name);continue
|
||||
if before is None:path.unlink()
|
||||
else:atomic(path,before)
|
||||
result['concurrentFilesNotOverwritten']=conflicts
|
||||
if replaced and previous_image:
|
||||
try:
|
||||
cmd(compose+['-f',str(folder/'rollback.yaml'),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4'])
|
||||
result['rollback']='previous_image_restored_additive_dataset_retained'
|
||||
except Exception: result['rollback']='manual_review_required'
|
||||
raise
|
||||
finally:
|
||||
result['finishedAt']=datetime.now(timezone.utc).isoformat()
|
||||
report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n')
|
||||
uid,gid=ROOT.stat().st_uid,ROOT.stat().st_gid
|
||||
for path in (folder.parent,folder,report):os.chown(path,uid,gid)
|
||||
report.chmod(0o640)
|
||||
print('HISTORY TIMING REPORT:',report)
|
||||
print('Only V4 updated. Original data, manager, polling, forecast selection and actuator permissions unchanged.')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--plant',required=True);parser.add_argument('--apply',action='store_true')
|
||||
args=parser.parse_args()
|
||||
try:run(args.plant,args.apply)
|
||||
except Exception as exc:raise SystemExit('Stopped: '+type(exc).__name__+'. See HISTORY TIMING REPORT; do not enable actuators.')
|
||||
Reference in New Issue
Block a user