feat(application): integrate measured-load ingestion training and planner source
This commit is contained in:
ENELIX Agent
@@ -0,0 +1,5 @@
|
||||
data/
|
||||
.git/
|
||||
__pycache__/
|
||||
**/__pycache__/
|
||||
*.log
|
||||
@@ -0,0 +1,7 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
data/
|
||||
*-reports/
|
||||
application-releases/
|
||||
.env
|
||||
*.env
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM python:3.11-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY netplan_v4 ./netplan_v4
|
||||
COPY tests ./tests
|
||||
COPY run_tests.py install_hooks.py runtime_preflight.py Dockerfile .
|
||||
COPY integrations ./integrations
|
||||
COPY gui ./gui
|
||||
COPY release_preflight.py forecast_acceptance.py deploy_integrated_shadow.py approved_previous_assets.json ./
|
||||
COPY acceptance ./acceptance
|
||||
COPY commissioning/deploy_application.py ./commissioning/deploy_application.py
|
||||
# Host sources can be 0600/0700. COPY makes them root-owned.
|
||||
# Normalize only packaged application code; never change host secrets or sockets.
|
||||
RUN find /app -type d -exec chmod 0755 {} + \
|
||||
&& find /app -type f -exec chmod 0644 {} +
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
NETPLAN_V4_TEST_REPORT_DIR=/tmp/test-results
|
||||
USER 1000:1000
|
||||
# Fail the build early if the actual unprivileged runtime cannot read the code.
|
||||
RUN python /app/runtime_preflight.py
|
||||
CMD ["uvicorn", "netplan_v4.service:from_environment", "--factory", "--host", "0.0.0.0", "--port", "9100", "--workers", "1"]
|
||||
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"sourceHashes": {
|
||||
".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790",
|
||||
"Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713",
|
||||
"acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062",
|
||||
"acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5",
|
||||
"acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5",
|
||||
"acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c",
|
||||
"acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6",
|
||||
"acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8",
|
||||
"acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4",
|
||||
"acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7",
|
||||
"acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc",
|
||||
"acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9",
|
||||
"acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a",
|
||||
"acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15",
|
||||
"acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6",
|
||||
"acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678",
|
||||
"acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984",
|
||||
"acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5",
|
||||
"acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161",
|
||||
"acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57",
|
||||
"acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a",
|
||||
"acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810",
|
||||
"acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8",
|
||||
"acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c",
|
||||
"acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30",
|
||||
"acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce",
|
||||
"acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50",
|
||||
"acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473",
|
||||
"acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7",
|
||||
"acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9",
|
||||
"acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a",
|
||||
"acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc",
|
||||
"acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c",
|
||||
"acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6",
|
||||
"acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2",
|
||||
"acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f",
|
||||
"approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0",
|
||||
"commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc",
|
||||
"commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d",
|
||||
"commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e",
|
||||
"compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197",
|
||||
"compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607",
|
||||
"deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898",
|
||||
"forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9",
|
||||
"gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd",
|
||||
"gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755",
|
||||
"gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5",
|
||||
"install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3",
|
||||
"integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3",
|
||||
"integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7",
|
||||
"integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161",
|
||||
"netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48",
|
||||
"netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332",
|
||||
"netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4",
|
||||
"netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b",
|
||||
"netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840",
|
||||
"netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743",
|
||||
"netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547",
|
||||
"netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987",
|
||||
"netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063",
|
||||
"netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576",
|
||||
"netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c",
|
||||
"netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad",
|
||||
"netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4",
|
||||
"netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99",
|
||||
"release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5",
|
||||
"requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464",
|
||||
"run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4",
|
||||
"runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1",
|
||||
"tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93",
|
||||
"tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a",
|
||||
"tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5",
|
||||
"tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3",
|
||||
"tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9",
|
||||
"tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64",
|
||||
"tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d",
|
||||
"tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09",
|
||||
"tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76",
|
||||
"tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82",
|
||||
"tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883",
|
||||
"tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4",
|
||||
"tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d",
|
||||
"tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d",
|
||||
"tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e",
|
||||
"tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8",
|
||||
"tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763"
|
||||
},
|
||||
"packagedAt": "2026-10-02T20:57:00.387145+00:00",
|
||||
"runtimeChanged": false
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM php:8.3-cli
|
||||
WORKDIR /check
|
||||
COPY php-src/ ./
|
||||
RUN find /check -type d -exec chmod 0755 {} + \
|
||||
&& find /check -type f -exec chmod 0644 {} +
|
||||
USER 1000:1000
|
||||
CMD ["php", "/check/check.php"]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Offline candidate-forecast tests: no telemetry, model loading or publication.
|
||||
Run in an unprivileged, networkless test container with no live data volumes.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
|
||||
def main():
|
||||
root = Path('/app/forecast')
|
||||
sys.path.insert(0, str(root))
|
||||
manifest = json.loads((root / 'SOURCE_MANIFEST.json').read_text())
|
||||
for relative, expected in manifest.items():
|
||||
p = root / relative
|
||||
if Path(relative).is_absolute() or '..' in Path(relative).parts or not p.resolve().is_relative_to(root):
|
||||
raise ValueError('Unsafe manifest path')
|
||||
raw = p.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != expected:
|
||||
raise ValueError('Test image source checksum mismatch')
|
||||
if p.suffix == '.py':
|
||||
ast.parse(raw, filename=str(p))
|
||||
import pandas, numpy, scipy, sklearn
|
||||
versions = {'python': sys.version.split()[0], 'pandas': pandas.__version__,
|
||||
'numpy': numpy.__version__, 'scipy': scipy.__version__, 'sklearn': sklearn.__version__}
|
||||
print('Candidate forecast runtime:', json.dumps(versions), flush=True)
|
||||
suite = unittest.defaultTestLoader.discover(str(root / 'tests'))
|
||||
result = unittest.TextTestRunner(verbosity=2).run(suite)
|
||||
print('Forecast tests only; no live data, no publication, no training job.', flush=True)
|
||||
return 0 if result.wasSuccessful() else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c",
|
||||
"methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6",
|
||||
"methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8",
|
||||
"methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4",
|
||||
"methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7",
|
||||
"methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc",
|
||||
"methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9",
|
||||
"methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a",
|
||||
"methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15",
|
||||
"methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6",
|
||||
"methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678",
|
||||
"methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984",
|
||||
"model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5",
|
||||
"netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161",
|
||||
"requirements.txt": "1f4731380246b4b08e5666ce736d9f24063128d2fc0737a061b36ac93952c083",
|
||||
"shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57",
|
||||
"soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a",
|
||||
"telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810",
|
||||
"tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8",
|
||||
"tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c",
|
||||
"tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30",
|
||||
"tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce"
|
||||
}
|
||||
@@ -0,0 +1,908 @@
|
||||
from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts
|
||||
from model_isolation import collect_predictions
|
||||
import os
|
||||
import sqlite3
|
||||
import json
|
||||
import time
|
||||
import datetime
|
||||
import traceback
|
||||
import warnings
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytz
|
||||
from influxdb_client import InfluxDBClient, Point, WritePrecision
|
||||
from influxdb_client.client.write_api import SYNCHRONOUS
|
||||
from soc_diagnostics import battery_soc_points
|
||||
from telemetry_quality import require_recent_telemetry, sanitize_measured_frame
|
||||
try:
|
||||
from influxdb_client.client.warnings import MissingPivotFunction
|
||||
warnings.simplefilter("ignore", MissingPivotFunction)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import methods.var_1 as v1
|
||||
import methods.var_2 as v2
|
||||
import methods.var_3 as v3
|
||||
import methods.var_10 as v10
|
||||
import methods.var_11 as v11
|
||||
import methods.var_13 as v13
|
||||
import methods.var_21 as v21
|
||||
import methods.var_22 as v22
|
||||
import methods.var_23 as v23
|
||||
|
||||
INFLUX_URL = os.getenv("INFLUX_URL", "http://influxdb:8086")
|
||||
INFLUX_TOKEN = os.environ["INFLUX_TOKEN"]
|
||||
INFLUX_ORG = os.getenv("INFLUX_ORG", "belevo")
|
||||
INFLUX_BUCKET = os.getenv("INFLUX_BUCKET", "energy_data")
|
||||
SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "/app/data/users.db")
|
||||
HISTORY_START = os.getenv("FORECAST_HISTORY_START", "1970-01-01T00:00:00Z")
|
||||
QUALITY_LOOKBACK_DAYS = int(os.getenv("FORECAST_QUALITY_LOOKBACK_DAYS", "14"))
|
||||
LOCAL_TZ = pytz.timezone(os.getenv("TZ", "Europe/Zurich"))
|
||||
INFLUX_TIMEOUT_MS = int(os.getenv("FORECAST_INFLUX_TIMEOUT_MS", "120000"))
|
||||
FORECAST_TIMEOUT_SECONDS = int(os.getenv("FORECAST_RUN_TIMEOUT_SECONDS", "900"))
|
||||
FORECAST_HORIZON_HOURS = max(24, min(72, int(os.getenv("FORECAST_HORIZON_HOURS", "48"))))
|
||||
TRAINING_TIMEOUT_SECONDS = int(os.getenv("FORECAST_TRAINING_TIMEOUT_SECONDS", "3600"))
|
||||
FORECAST_STALE_SECONDS = int(os.getenv("FORECAST_STALE_SECONDS", "5400"))
|
||||
WATCHDOG_INTERVAL_SECONDS = int(os.getenv("FORECAST_WATCHDOG_INTERVAL_SECONDS", "300"))
|
||||
TRAINING_HOUR = int(os.getenv("FORECAST_TRAINING_HOUR", "2"))
|
||||
LAST_SUCCESS_PATH = os.getenv("FORECAST_LAST_SUCCESS_PATH", "/tmp/forecast_engine_last_success.json")
|
||||
LAST_TRAINING_PATH = os.getenv("FORECAST_LAST_TRAINING_PATH", "/app/data/forecast_training_status.json")
|
||||
last_trained_day = None
|
||||
_FORECAST_PROCESS_LOCK = threading.Lock()
|
||||
_TRAINING_THREAD = None
|
||||
|
||||
MODEL_MODULES = {1: v1, 2: v2, 3: v3, 10: v10, 11: v11, 13: v13, 21: v21, 22: v22, 23: v23}
|
||||
QUALITY_TARGETS = {1: "PV", 10: "PV", 21: "PV", 2: "Hausverbrauch", 11: "Hausverbrauch", 22: "Hausverbrauch"}
|
||||
|
||||
|
||||
def active(config, n):
|
||||
return bool(int(config.get(f"prog_var_{n}", config.get(f"var_{n}", 0)) or 0))
|
||||
|
||||
|
||||
def get_configs():
|
||||
conn = sqlite3.connect(SQLITE_DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
columns = {row["name"] for row in conn.execute("PRAGMA table_info(anlagen_meta)")}
|
||||
if "batt_grid_charging_enabled" not in columns:
|
||||
conn.execute(
|
||||
"ALTER TABLE anlagen_meta ADD COLUMN batt_grid_charging_enabled INTEGER NOT NULL DEFAULT 0"
|
||||
)
|
||||
conn.commit()
|
||||
rows = conn.execute("SELECT * FROM anlagen_meta").fetchall()
|
||||
conn.close()
|
||||
configs = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
try:
|
||||
d["daecher"] = json.loads(d.get("daecher") or "[]")
|
||||
except Exception:
|
||||
d["daecher"] = []
|
||||
for key, default in [
|
||||
("ac_leistung", 10.0),
|
||||
("batt_capacity_kwh", 0.0),
|
||||
("batt_power_kw", 0.0),
|
||||
("tarif_bezug_fest", 0.30),
|
||||
("tarif_einspeisung_fest", 0.10),
|
||||
("tarif_peak_fest", 5.0),
|
||||
]:
|
||||
raw_value = d.get(key)
|
||||
d[key] = float(default if raw_value is None or raw_value == "" else raw_value)
|
||||
configs.append(d)
|
||||
return configs
|
||||
|
||||
|
||||
def _time_literal(dt):
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _query_df(query):
|
||||
client = InfluxDBClient(
|
||||
url=INFLUX_URL,
|
||||
token=INFLUX_TOKEN,
|
||||
org=INFLUX_ORG,
|
||||
timeout=INFLUX_TIMEOUT_MS,
|
||||
)
|
||||
try:
|
||||
df = client.query_api().query_data_frame(org=INFLUX_ORG, query=query)
|
||||
finally:
|
||||
client.close()
|
||||
if isinstance(df, list):
|
||||
df = pd.concat(df, ignore_index=True) if df else pd.DataFrame()
|
||||
if df is None or df.empty or "_time" not in df.columns:
|
||||
return pd.DataFrame()
|
||||
df["_time"] = pd.to_datetime(df["_time"], utc=True).dt.tz_localize(None)
|
||||
return df
|
||||
|
||||
|
||||
def _pivot_frame(df, fields):
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
out = df.set_index("_time")
|
||||
keep = [c for c in fields if c in out.columns]
|
||||
out = out[keep] if keep else pd.DataFrame(index=out.index)
|
||||
out = out.apply(pd.to_numeric, errors="coerce")
|
||||
out = out[~out.index.duplicated(keep="last")]
|
||||
return out.sort_index().resample("5min").mean(numeric_only=True)
|
||||
|
||||
|
||||
def _tariff_frame(df, config=None):
|
||||
if df is None or df.empty:
|
||||
return pd.DataFrame()
|
||||
df = df.copy()
|
||||
if "_time" in df.columns:
|
||||
df["_time"] = pd.to_datetime(df["_time"], errors="coerce")
|
||||
df = df.dropna(subset=["_time"]).set_index("_time")
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
price = df["price_chf_kwh"] if "price_chf_kwh" in df.columns else df.get("_value")
|
||||
if price is None:
|
||||
return pd.DataFrame()
|
||||
price = pd.to_numeric(price, errors="coerce")
|
||||
model = df.get("tariff_model", pd.Series("", index=df.index)).astype(str).str.lower()
|
||||
typ = df.get("type", pd.Series("", index=df.index)).astype(str).str.lower()
|
||||
tariff_name = df.get("tariff_name", pd.Series("", index=df.index)).astype(str).str.lower()
|
||||
provider = df.get("provider", pd.Series("", index=df.index)).astype(str).str.lower()
|
||||
key = (model + " " + tariff_name + " " + provider).str.lower()
|
||||
|
||||
rows = pd.DataFrame({"price": price, "key": key, "type": typ}, index=df.index)
|
||||
rows = rows[pd.notna(rows["price"])].sort_index()
|
||||
if rows.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
cfg = config or {}
|
||||
import_choice = str(cfg.get("tarif_bezug", "") or "").lower()
|
||||
export_choice = str(cfg.get("tarif_einspeisung", "") or "").lower()
|
||||
|
||||
out = pd.DataFrame(index=rows.index.unique().sort_values())
|
||||
|
||||
import_base = (
|
||||
rows["type"].str.contains("consumption|import|bezug", regex=True, na=False)
|
||||
| rows["key"].str.contains("dynamic|dynamisch|home|business", regex=True, na=False)
|
||||
)
|
||||
if "business" in import_choice:
|
||||
import_mask = import_base & rows["key"].str.contains("business|gewerbe|commercial", regex=True, na=False)
|
||||
elif "home" in import_choice or "privat" in import_choice:
|
||||
import_mask = import_base & rows["key"].str.contains("home|privat|private", regex=True, na=False)
|
||||
elif "dynam" in import_choice:
|
||||
import_mask = import_base
|
||||
else:
|
||||
import_mask = pd.Series(False, index=rows.index)
|
||||
if not import_mask.any() and "dynam" in import_choice:
|
||||
import_mask = import_base
|
||||
|
||||
export_base = (
|
||||
rows["type"].str.contains("feed|einspeis|export", regex=True, na=False)
|
||||
| rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|feed", regex=True, na=False)
|
||||
)
|
||||
if "referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice:
|
||||
export_mask = export_base & rows["key"].str.contains("referenzmarktpreis|marktpreis|reference|belevo", regex=True, na=False)
|
||||
else:
|
||||
export_mask = rows["key"].str.contains("standard_feedin|ckw statisch", regex=True, na=False) & export_base
|
||||
if not export_mask.any() and ("referenz" in export_choice or "marktpreis" in export_choice or "market" in export_choice):
|
||||
export_mask = export_base
|
||||
|
||||
if import_mask.any():
|
||||
out["import_price"] = rows.loc[import_mask, "price"].groupby(level=0).last()
|
||||
if export_mask.any():
|
||||
out["export_price"] = rows.loc[export_mask, "price"].groupby(level=0).last()
|
||||
|
||||
if out.empty:
|
||||
return out
|
||||
return out.sort_index().resample("5min").mean().ffill().bfill()
|
||||
|
||||
def fetch_influx_frames(config, training):
|
||||
aid = config["anlagen_id"]
|
||||
start = HISTORY_START if training else "-14d"
|
||||
future_stop = _time_literal(datetime.datetime.utcnow() + datetime.timedelta(hours=FORECAST_HORIZON_HOURS))
|
||||
|
||||
q_tel = f'''
|
||||
from(bucket: "{INFLUX_BUCKET}")
|
||||
|> range(start: {start})
|
||||
|> filter(fn: (r) => r["_measurement"] == "api_telemetry")
|
||||
|> filter(fn: (r) => r["anlagen_id"] == "{aid}")
|
||||
|> filter(fn: (r) => r["_field"] == "PV" or r["_field"] == "Hausverbrauch" or r["_field"] == "Netzleistung" or r["_field"] == "SOC")
|
||||
|> filter(fn: (r) => not exists r["data_type"] or (r["data_type"] != "forecast" and r["data_type"] != "forecast_snapshot"))
|
||||
|> aggregateWindow(every: 5m, fn: mean, createEmpty: false, timeSrc: "_start")
|
||||
|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
|
||||
'''
|
||||
df_tel = _pivot_frame(_query_df(q_tel), ["PV", "Hausverbrauch", "Netzleistung", "SOC"])
|
||||
|
||||
q_wea = f'''
|
||||
from(bucket: "{INFLUX_BUCKET}")
|
||||
|> range(start: {start}, stop: {future_stop})
|
||||
|> filter(fn: (r) => r["_measurement"] == "weather_forecast")
|
||||
|> aggregateWindow(every: 5m, fn: mean, createEmpty: false)
|
||||
|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
|
||||
'''
|
||||
df_wea = _pivot_frame(_query_df(q_wea), ["temp_c", "temperature", "cloud", "cloud_cover", "precip_mm", "wind_kph", "chance_of_snow"])
|
||||
if "temperature" in df_wea.columns and "temp_c" not in df_wea.columns:
|
||||
df_wea["temp_c"] = df_wea["temperature"]
|
||||
if "cloud_cover" in df_wea.columns and "cloud" not in df_wea.columns:
|
||||
df_wea["cloud"] = df_wea["cloud_cover"]
|
||||
if "cloud" in df_wea.columns and "cloud_cover" not in df_wea.columns:
|
||||
df_wea["cloud_cover"] = df_wea["cloud"]
|
||||
|
||||
q_tar = f'''
|
||||
from(bucket: "{INFLUX_BUCKET}")
|
||||
|> range(start: -7d, stop: {future_stop})
|
||||
|> filter(fn: (r) => r["_measurement"] == "tariffs")
|
||||
|> filter(fn: (r) => r["_field"] == "price_chf_kwh")
|
||||
'''
|
||||
df_tar = _tariff_frame(_query_df(q_tar), config)
|
||||
return df_tel, df_wea, df_tar
|
||||
|
||||
|
||||
def _consistent_tail(df):
|
||||
if df.empty or "PV" not in df.columns or "Hausverbrauch" not in df.columns:
|
||||
return df
|
||||
probe = df[["PV", "Hausverbrauch"]].copy()
|
||||
filled = probe.interpolate(limit=3, limit_direction="both")
|
||||
valid = filled.notna().all(axis=1)
|
||||
if not valid.any():
|
||||
return df.iloc[0:0]
|
||||
run = 0
|
||||
last_break = -1
|
||||
for i, ok in enumerate(valid.to_numpy()):
|
||||
if ok:
|
||||
run = 0
|
||||
else:
|
||||
run += 1
|
||||
if run >= 4:
|
||||
last_break = i
|
||||
if last_break >= 0:
|
||||
after = np.where(valid.iloc[last_break + 1:].to_numpy())[0]
|
||||
if len(after):
|
||||
return df.iloc[last_break + 1 + after[0]:]
|
||||
return df.iloc[0:0]
|
||||
return df.loc[valid[valid].index[0]:]
|
||||
|
||||
|
||||
|
||||
def _longest_consistent_segment(df, column):
|
||||
if df.empty or column not in df.columns:
|
||||
return pd.DataFrame()
|
||||
series = pd.to_numeric(df[column], errors="coerce")
|
||||
valid = series.notna().to_numpy()
|
||||
if not valid.any():
|
||||
return pd.DataFrame()
|
||||
best_start = best_end = None
|
||||
start = 0
|
||||
gap = 0
|
||||
for i, ok in enumerate(valid):
|
||||
if ok:
|
||||
gap = 0
|
||||
else:
|
||||
gap += 1
|
||||
if gap >= 4:
|
||||
end = i - gap
|
||||
if end >= start and (best_start is None or end - start > best_end - best_start):
|
||||
best_start, best_end = start, end
|
||||
start = i + 1
|
||||
gap = 0
|
||||
end = len(valid) - 1
|
||||
if end >= start and (best_start is None or end - start > best_end - best_start):
|
||||
best_start, best_end = start, end
|
||||
if best_start is None:
|
||||
return pd.DataFrame()
|
||||
segment = df.iloc[best_start:best_end + 1].copy()
|
||||
first = pd.to_numeric(segment[column], errors="coerce").first_valid_index()
|
||||
last = pd.to_numeric(segment[column], errors="coerce").last_valid_index()
|
||||
if first is None or last is None:
|
||||
return pd.DataFrame()
|
||||
return segment.loc[first:last]
|
||||
|
||||
|
||||
def _add_time_features(df):
|
||||
hour = df.index.hour + df.index.minute / 60.0
|
||||
doy = df.index.dayofyear
|
||||
df["hour_float"] = hour
|
||||
df["hour_sin"] = np.sin(2 * np.pi * hour / 24.0)
|
||||
df["hour_cos"] = np.cos(2 * np.pi * hour / 24.0)
|
||||
df["sin_year"] = np.sin(2 * np.pi * doy / 365.25)
|
||||
df["cos_year"] = np.cos(2 * np.pi * doy / 365.25)
|
||||
df["weekday"] = df.index.weekday
|
||||
df["is_weekday"] = (df.index.weekday < 5).astype(int)
|
||||
return df
|
||||
|
||||
|
||||
def _fill_defaults(df, history):
|
||||
defaults = {
|
||||
"PV": np.nan,
|
||||
"Hausverbrauch": np.nan,
|
||||
"SOC": np.nan,
|
||||
"Netzleistung": np.nan,
|
||||
"temp_c": 15.0,
|
||||
"cloud": 20.0,
|
||||
"cloud_cover": 20.0,
|
||||
"precip_mm": 0.0,
|
||||
"wind_kph": 0.0,
|
||||
"chance_of_snow": 0.0,
|
||||
"import_price": np.nan,
|
||||
"export_price": np.nan,
|
||||
}
|
||||
for col, default in defaults.items():
|
||||
if col not in df.columns:
|
||||
df[col] = default
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
# Measurements are never interpolated/filled here. Outages are not zero load,
|
||||
# zero PV, a fresh SOC, or a measured grid peak. Weather defaults are separate.
|
||||
df = sanitize_measured_frame(df)
|
||||
for col, default in defaults.items():
|
||||
if col not in ["PV", "Hausverbrauch", "SOC", "Netzleistung"]:
|
||||
df[col] = df[col].ffill().bfill()
|
||||
if np.isfinite(default):
|
||||
df[col] = df[col].fillna(default)
|
||||
if "cloud_cover" in df.columns:
|
||||
df["cloud"] = df["cloud"].fillna(df["cloud_cover"])
|
||||
df["import_price"] = df["import_price"].fillna(0.30)
|
||||
df["export_price"] = df["export_price"].fillna(0.10)
|
||||
return _add_time_features(df)
|
||||
|
||||
|
||||
def build_data_object(config, training=False):
|
||||
now = datetime.datetime.utcnow().replace(second=0, microsecond=0)
|
||||
now = now - datetime.timedelta(minutes=now.minute % 5)
|
||||
df_tel, df_wea, df_tar = fetch_influx_frames(config, training)
|
||||
|
||||
frames = [f for f in [df_tel, df_wea, df_tar] if not f.empty]
|
||||
combined = frames[0] if frames else pd.DataFrame()
|
||||
for f in frames[1:]:
|
||||
combined = combined.join(f, how="outer")
|
||||
combined = combined.sort_index()
|
||||
|
||||
full_hist_raw = combined.loc[:now - datetime.timedelta(minutes=5)] if not combined.empty else pd.DataFrame()
|
||||
# A trailing telemetry gap must not discard all earlier valid observations.
|
||||
hist_raw = full_hist_raw.copy()
|
||||
# Freshness is determined from measurements, not the outer-joined weather grid.
|
||||
recent_raw = sanitize_measured_frame(df_tel.loc[:now - datetime.timedelta(minutes=5)].copy()) if not df_tel.empty else pd.DataFrame()
|
||||
start_hist = hist_raw.index.min().floor("5min") if training and not hist_raw.empty else now - datetime.timedelta(days=14)
|
||||
|
||||
idx_hist = pd.date_range(start=start_hist, end=now - datetime.timedelta(minutes=5), freq="5min")
|
||||
df_hist = pd.DataFrame(index=idx_hist).join(hist_raw, how="left")
|
||||
df_hist = _fill_defaults(df_hist, history=True)
|
||||
|
||||
df_load_training = pd.DataFrame()
|
||||
if training and "Hausverbrauch" in full_hist_raw.columns:
|
||||
load_segment = _longest_consistent_segment(full_hist_raw, "Hausverbrauch")
|
||||
if not load_segment.empty:
|
||||
idx_load = pd.date_range(
|
||||
start=load_segment.index.min().floor("5min"),
|
||||
end=load_segment.index.max().floor("5min"),
|
||||
freq="5min",
|
||||
)
|
||||
df_load_training = pd.DataFrame(index=idx_load).join(load_segment, how="left")
|
||||
df_load_training = _fill_defaults(df_load_training, history=True)
|
||||
|
||||
df_pv_training = pd.DataFrame()
|
||||
if training and "PV" in full_hist_raw.columns:
|
||||
pv_first = full_hist_raw["PV"].first_valid_index()
|
||||
if pv_first is not None:
|
||||
idx_pv = pd.date_range(
|
||||
start=pv_first.floor("5min"),
|
||||
end=now - datetime.timedelta(minutes=5),
|
||||
freq="5min",
|
||||
)
|
||||
df_pv_training = pd.DataFrame(index=idx_pv).join(full_hist_raw, how="left")
|
||||
df_pv_training = _fill_defaults(df_pv_training, history=True)
|
||||
|
||||
idx_fut = pd.date_range(start=now, periods=FORECAST_HORIZON_HOURS * 12, freq="5min")
|
||||
fut_raw = combined.reindex(combined.index.union(idx_fut)).sort_index() if not combined.empty else pd.DataFrame(index=idx_fut)
|
||||
df_fut = pd.DataFrame(index=idx_fut).join(fut_raw, how="left")
|
||||
df_fut = _fill_defaults(df_fut, history=False)
|
||||
|
||||
month_hist = df_hist[(df_hist.index.year == now.year) & (df_hist.index.month == now.month)]
|
||||
current_peak_kw = 0.0
|
||||
if not month_hist.empty and "Netzleistung" in month_hist.columns:
|
||||
measured_grid = pd.to_numeric(month_hist["Netzleistung"], errors="coerce").dropna()
|
||||
measured_peak = measured_grid.resample(
|
||||
"15min", origin="start_day", label="left", closed="left"
|
||||
).mean()
|
||||
if not measured_peak.empty:
|
||||
current_peak_kw = max(0.0, float(measured_peak.max()) / 1000.0)
|
||||
if current_peak_kw <= 0.0 and not month_hist.empty and {"Hausverbrauch", "PV"}.issubset(month_hist.columns):
|
||||
fallback_residual = (month_hist["Hausverbrauch"] - month_hist["PV"]).resample(
|
||||
"15min", origin="start_day", label="left", closed="left"
|
||||
).mean()
|
||||
if not fallback_residual.empty:
|
||||
current_peak_kw = max(0.0, float(fallback_residual.max()) / 1000.0)
|
||||
|
||||
min_soc = float(
|
||||
config.get("batt_min_soc", config.get("batt_min_soc_percent", 0.0)) or 0.0
|
||||
)
|
||||
current_soc = max(0.0, min(100.0, min_soc))
|
||||
current_soc_source = "safe_minimum"
|
||||
current_soc_age_minutes = None
|
||||
if "SOC" in full_hist_raw.columns:
|
||||
soc_values = pd.to_numeric(full_hist_raw["SOC"], errors="coerce").dropna()
|
||||
if not soc_values.empty:
|
||||
latest_soc_time = pd.Timestamp(soc_values.index[-1])
|
||||
current_soc_age_minutes = max(
|
||||
0.0,
|
||||
(pd.Timestamp(now) - latest_soc_time).total_seconds() / 60.0,
|
||||
)
|
||||
try:
|
||||
max_soc_age_minutes = max(
|
||||
5.0,
|
||||
float(config.get("batt_soc_max_age_minutes", 30.0) or 30.0),
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
max_soc_age_minutes = 30.0
|
||||
if current_soc_age_minutes <= max_soc_age_minutes:
|
||||
current_soc = float(soc_values.iloc[-1])
|
||||
current_soc_source = "telemetry"
|
||||
current_soc = max(0.0, min(100.0, current_soc))
|
||||
|
||||
return {
|
||||
"config": config,
|
||||
"metadata": config,
|
||||
"now": now,
|
||||
"df_hist": df_hist,
|
||||
"df_recent_raw": recent_raw,
|
||||
"df_load_training": df_load_training,
|
||||
"df_pv_training": df_pv_training,
|
||||
"df_fut": df_fut,
|
||||
"current_soc_perc": current_soc,
|
||||
"current_soc_source": current_soc_source,
|
||||
"current_soc_age_minutes": current_soc_age_minutes,
|
||||
"current_month_peak_kw": current_peak_kw,
|
||||
}
|
||||
|
||||
|
||||
def run_training():
|
||||
for config in get_configs():
|
||||
if not any(active(config, n) for n in MODEL_MODULES):
|
||||
continue
|
||||
print(f"Training Anlage {config['anlagen_id']}...")
|
||||
data_obj = build_data_object(config, training=True)
|
||||
for n, module in MODEL_MODULES.items():
|
||||
if active(config, n) and hasattr(module, "train"):
|
||||
try:
|
||||
target = QUALITY_TARGETS.get(n)
|
||||
if target:
|
||||
require_recent_telemetry(data_obj, [target])
|
||||
print(f" var_{n}: {module.train(data_obj)}")
|
||||
except Exception:
|
||||
print(f" var_{n}: Training fehlgeschlagen")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def _forecast_point(aid, field, t, value):
|
||||
return (
|
||||
Point("api_telemetry")
|
||||
.tag("anlagen_id", aid)
|
||||
.tag("data_type", "forecast")
|
||||
.field(field, float(value))
|
||||
.time(t.to_pydatetime(), WritePrecision.S)
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_point(aid, field, run_hour, t, value):
|
||||
return (
|
||||
Point("api_telemetry")
|
||||
.tag("anlagen_id", aid)
|
||||
.tag("data_type", "forecast_snapshot")
|
||||
.tag("run_hour", run_hour)
|
||||
.field(f"{field}_run_{run_hour}", float(value))
|
||||
.time(t.to_pydatetime(), WritePrecision.S)
|
||||
)
|
||||
|
||||
|
||||
def _quality_query(aid, forecast_field, target_field):
|
||||
stop = _time_literal(datetime.datetime.utcnow() - datetime.timedelta(minutes=10))
|
||||
return f'''
|
||||
from(bucket: "{INFLUX_BUCKET}")
|
||||
|> range(start: -{QUALITY_LOOKBACK_DAYS}d, stop: {stop})
|
||||
|> filter(fn: (r) => r["_measurement"] == "api_telemetry")
|
||||
|> filter(fn: (r) => r["anlagen_id"] == "{aid}")
|
||||
|> filter(fn: (r) => r["_field"] == "{target_field}" or r["_field"] == "{forecast_field}")
|
||||
|> aggregateWindow(every: 5m, fn: mean, createEmpty: false)
|
||||
|> group()
|
||||
|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
|
||||
'''
|
||||
|
||||
|
||||
def _forecast_quality(aid, variant, target_field):
|
||||
forecast_field = f"prog_var_{variant}"
|
||||
df = _query_df(_quality_query(aid, forecast_field, target_field))
|
||||
if df.empty or forecast_field not in df.columns or target_field not in df.columns:
|
||||
return None
|
||||
pair = df[[target_field, forecast_field]].apply(pd.to_numeric, errors="coerce").dropna()
|
||||
if len(pair) < 3:
|
||||
return None
|
||||
actual = pair[target_field].to_numpy(dtype=float)
|
||||
pred = pair[forecast_field].to_numpy(dtype=float)
|
||||
ss_res = float(np.sum((actual - pred) ** 2))
|
||||
ss_tot = float(np.sum((actual - np.mean(actual)) ** 2))
|
||||
r2 = None if ss_tot <= 0 else 1.0 - (ss_res / ss_tot)
|
||||
mae = float(np.mean(np.abs(actual - pred)))
|
||||
rmse = float(np.sqrt(np.mean((actual - pred) ** 2)))
|
||||
return {"r2": r2, "mae": mae, "rmse": rmse, "samples": int(len(pair))}
|
||||
|
||||
|
||||
def _quality_point(aid, variant, target, metrics):
|
||||
p = (
|
||||
Point("forecast_metrics")
|
||||
.tag("anlagen_id", aid)
|
||||
.tag("forecast", f"prog_var_{variant}")
|
||||
.tag("target", target)
|
||||
.field("samples", int(metrics["samples"]))
|
||||
.field("mae", float(metrics["mae"]))
|
||||
.field("rmse", float(metrics["rmse"]))
|
||||
.time(datetime.datetime.utcnow(), WritePrecision.S)
|
||||
)
|
||||
if metrics["r2"] is not None:
|
||||
p.field("r2", float(metrics["r2"]))
|
||||
return p
|
||||
|
||||
|
||||
def write_quality_metrics(write_api, aid, active_variants):
|
||||
points = []
|
||||
|
||||
for variant, target in QUALITY_TARGETS.items():
|
||||
if variant not in active_variants:
|
||||
continue
|
||||
try:
|
||||
metrics = _forecast_quality(aid, variant, target)
|
||||
if metrics:
|
||||
points.append(_quality_point(aid, variant, target, metrics))
|
||||
r2_text = "nan" if metrics["r2"] is None else f"{metrics['r2']:.3f}"
|
||||
print(f"R2 Anlage {aid} prog_var_{variant}: {r2_text} / samples={metrics['samples']}")
|
||||
except Exception:
|
||||
print(f"R2 Anlage {aid} prog_var_{variant}: Berechnung fehlgeschlagen")
|
||||
traceback.print_exc()
|
||||
if points:
|
||||
try:
|
||||
write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points)
|
||||
except Exception:
|
||||
print(f"R2 Anlage {aid}: Schreiben der Qualitaetswerte fehlgeschlagen")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def run_forecast(only_anlagen_id=None, manual=False):
|
||||
configs = get_configs()
|
||||
if only_anlagen_id:
|
||||
configs = [c for c in configs if c.get("anlagen_id") == only_anlagen_id]
|
||||
client = InfluxDBClient(
|
||||
url=INFLUX_URL,
|
||||
token=INFLUX_TOKEN,
|
||||
org=INFLUX_ORG,
|
||||
timeout=INFLUX_TIMEOUT_MS,
|
||||
)
|
||||
write_api = client.write_api(write_options=SYNCHRONOUS)
|
||||
completed = []
|
||||
failures = []
|
||||
try:
|
||||
for config in configs:
|
||||
aid = config["anlagen_id"]
|
||||
try:
|
||||
data_obj = build_data_object(config, training=False)
|
||||
targets = []
|
||||
if any(active(config, n) for n in (1, 3, 10, 13, 21, 23)):
|
||||
targets.append("PV")
|
||||
if any(active(config, n) for n in (2, 3, 11, 13, 22, 23)):
|
||||
targets.append("Hausverbrauch")
|
||||
if float(config.get("batt_capacity_kwh", 0.0) or 0.0) > 0 and any(active(config, n) for n in (3, 13, 23)):
|
||||
targets.append("SOC")
|
||||
# Fail BEFORE model prediction, snapshot publication or V1/V4 plan writes.
|
||||
data_obj["telemetry_quality"] = require_recent_telemetry(data_obj, targets)
|
||||
soc_age = data_obj.get("current_soc_age_minutes")
|
||||
soc_age_text = "keine Messung" if soc_age is None else f"{soc_age:.1f} min"
|
||||
print(
|
||||
f"Forecast Anlage {aid}: Batterie-SOC {data_obj['current_soc_perc']:.1f}% "
|
||||
f"({data_obj['current_soc_source']}, Alter {soc_age_text})."
|
||||
)
|
||||
forecasts, model_errors = collect_predictions(
|
||||
data_obj, config,
|
||||
{1: v1, 2: v2, 10: v10, 11: v11, 21: v21, 22: v22}, active,
|
||||
)
|
||||
data_obj['forecast_model_status'] = model_errors
|
||||
for variant, detail in model_errors.items():
|
||||
print(f"Forecast Anlage {aid} prog_var_{variant}: unavailable ({detail['errorType']}); keine Nullwerte eingesetzt.")
|
||||
p_1, p_2, p_10, p_11, p_21, p_22 = (
|
||||
forecasts[n] for n in (1, 2, 10, 11, 21, 22)
|
||||
)
|
||||
|
||||
# ENELIX_V4_SHADOW_BRIDGE
|
||||
_v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)])
|
||||
|
||||
p_3 = v3.predict(data_obj, p_1, p_2) if active(config, 3) and p_1 and p_2 else {}
|
||||
p_13 = v13.predict(data_obj, p_10, p_11) if active(config, 13) and p_10 and p_11 else {}
|
||||
p_23 = v23.predict(data_obj, p_21, p_22) if active(config, 23) and p_21 and p_22 else {}
|
||||
|
||||
forecast_sets = [
|
||||
(1, p_1), (2, p_2), (3, p_3),
|
||||
(10, p_10), (11, p_11), (13, p_13),
|
||||
(21, p_21), (22, p_22), (23, p_23),
|
||||
]
|
||||
run_hour = str(int(datetime.datetime.now(LOCAL_TZ).strftime("%H")))
|
||||
points = []
|
||||
for variant_id, pv_src, load_src, grid_src in [
|
||||
(3, p_1, p_2, p_3),
|
||||
(13, p_10, p_11, p_13),
|
||||
(23, p_21, p_22, p_23),
|
||||
]:
|
||||
if grid_src and pv_src and load_src:
|
||||
points.extend(battery_soc_points(data_obj, variant_id, pv_src, load_src, grid_src))
|
||||
active_variants = set()
|
||||
battery_plans = data_obj.get("battery_plans", {})
|
||||
for t in data_obj["df_fut"].index:
|
||||
for n, values in forecast_sets:
|
||||
if t in values:
|
||||
field = f"prog_var_{n}"
|
||||
value = float(values[t])
|
||||
points.append(_forecast_point(aid, field, t, value))
|
||||
points.append(_snapshot_point(aid, field, run_hour, t, value))
|
||||
if n in battery_plans and t in battery_plans[n].get("battery", {}):
|
||||
points.append(_forecast_point(
|
||||
aid,
|
||||
f"prog_var_{n}_battery",
|
||||
t,
|
||||
float(battery_plans[n]["battery"][t]),
|
||||
))
|
||||
active_variants.add(n)
|
||||
if points:
|
||||
write_api.write(bucket=INFLUX_BUCKET, org=INFLUX_ORG, record=points)
|
||||
print(f"Forecast Anlage {aid}: {len(points)} Punkte geschrieben inkl. run_{run_hour}.")
|
||||
else:
|
||||
print(f"Forecast Anlage {aid}: keine aktiven Prognosen oder keine Daten.")
|
||||
write_quality_metrics(write_api, aid, active_variants)
|
||||
completed.append(aid)
|
||||
except Exception as exc:
|
||||
failures.append({"anlagen_id": aid, "error": f"{type(exc).__name__}: {exc}"})
|
||||
print(f"Forecast Anlage {aid}: Lauf fehlgeschlagen.")
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
client.close()
|
||||
if failures:
|
||||
raise RuntimeError(f"Forecast-Fehler: {failures}")
|
||||
return {"completed": completed, "manual": bool(manual)}
|
||||
|
||||
|
||||
def _write_json_atomic(path, payload):
|
||||
directory = os.path.dirname(path) or "."
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
tmp_path = f"{path}.tmp.{os.getpid()}"
|
||||
try:
|
||||
with open(tmp_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
os.replace(tmp_path, path)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _last_success_age_seconds():
|
||||
stamp = _read_json(LAST_SUCCESS_PATH).get("timestamp")
|
||||
if not stamp:
|
||||
return None
|
||||
try:
|
||||
then = datetime.datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
return max(0.0, (now - then).total_seconds())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _child_environment(training=False):
|
||||
env = os.environ.copy()
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
if training:
|
||||
max_threads = str(max(1, int(env.get("FORECAST_TRAINING_MAX_THREADS", "1"))))
|
||||
for name in (
|
||||
"OMP_NUM_THREADS",
|
||||
"OPENBLAS_NUM_THREADS",
|
||||
"MKL_NUM_THREADS",
|
||||
"NUMEXPR_NUM_THREADS",
|
||||
"LOKY_MAX_CPU_COUNT",
|
||||
):
|
||||
env[name] = max_threads
|
||||
return env
|
||||
|
||||
|
||||
def _run_child(mode, timeout_seconds, only_anlagen_id=None):
|
||||
command = [sys.executable, "-u", os.path.abspath(__file__), mode]
|
||||
if only_anlagen_id:
|
||||
command.append(str(only_anlagen_id))
|
||||
label = "Training" if mode == "--train-once" else "Forecast"
|
||||
print(f"{label}-Kindprozess startet (Timeout {timeout_seconds}s).")
|
||||
process = subprocess.Popen(command, env=_child_environment(training=mode == "--train-once"))
|
||||
try:
|
||||
return_code = process.wait(timeout=timeout_seconds)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"{label}-Kindprozess hat das Zeitlimit erreicht und wird beendet.")
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=15)
|
||||
return 124
|
||||
if return_code != 0:
|
||||
print(f"{label}-Kindprozess beendet mit Status {return_code}.")
|
||||
return return_code
|
||||
|
||||
|
||||
def run_forecast_isolated(only_anlagen_id=None, source="scheduler"):
|
||||
if not _FORECAST_PROCESS_LOCK.acquire(blocking=False):
|
||||
print(f"Forecast-Aufruf ({source}) uebersprungen: bereits ein Lauf aktiv.")
|
||||
return {"status": "busy", "source": source}
|
||||
try:
|
||||
return_code = _run_child("--forecast-once", FORECAST_TIMEOUT_SECONDS, only_anlagen_id)
|
||||
return {
|
||||
"status": "ok" if return_code == 0 else "error",
|
||||
"source": source,
|
||||
"return_code": return_code,
|
||||
}
|
||||
finally:
|
||||
_FORECAST_PROCESS_LOCK.release()
|
||||
|
||||
|
||||
def _training_worker(day_text):
|
||||
global _TRAINING_THREAD
|
||||
started = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
_write_json_atomic(LAST_TRAINING_PATH, {
|
||||
"day": day_text,
|
||||
"status": "running",
|
||||
"started_at": started,
|
||||
})
|
||||
return_code = 1
|
||||
error = None
|
||||
try:
|
||||
return_code = _run_child("--train-once", TRAINING_TIMEOUT_SECONDS)
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
status = "ok" if return_code == 0 else ("timeout" if return_code == 124 else "error")
|
||||
payload = {
|
||||
"day": day_text,
|
||||
"status": status,
|
||||
"started_at": started,
|
||||
"finished_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"return_code": return_code,
|
||||
}
|
||||
if error:
|
||||
payload["error"] = error
|
||||
_write_json_atomic(LAST_TRAINING_PATH, payload)
|
||||
print(f"Nachttraining beendet: {status}.")
|
||||
_TRAINING_THREAD = None
|
||||
|
||||
|
||||
def start_training_async(day):
|
||||
global _TRAINING_THREAD
|
||||
if _TRAINING_THREAD is not None and _TRAINING_THREAD.is_alive():
|
||||
print("Nachttraining laeuft bereits.")
|
||||
return False
|
||||
day_text = day.isoformat()
|
||||
_TRAINING_THREAD = threading.Thread(target=_training_worker, args=(day_text,), daemon=True)
|
||||
_TRAINING_THREAD.start()
|
||||
return True
|
||||
|
||||
|
||||
def _watchdog_loop():
|
||||
while True:
|
||||
time.sleep(max(60, WATCHDOG_INTERVAL_SECONDS))
|
||||
try:
|
||||
age = _last_success_age_seconds()
|
||||
if age is None or age > FORECAST_STALE_SECONDS:
|
||||
age_text = "unbekannt" if age is None else f"{age / 60.0:.1f} Minuten"
|
||||
print(f"Forecast-Watchdog: letzter erfolgreicher Lauf {age_text}; neuer Lauf wird gestartet.")
|
||||
run_forecast_isolated(source="watchdog")
|
||||
except Exception:
|
||||
print("Forecast-Watchdog: Pruefung fehlgeschlagen.")
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
# BEGIN EMS RESIMULATE HTTP SERVER
|
||||
_RESIMULATE_SERVER_STARTED = False
|
||||
|
||||
|
||||
def _start_resimulate_server():
|
||||
global _RESIMULATE_SERVER_STARTED
|
||||
if _RESIMULATE_SERVER_STARTED:
|
||||
return
|
||||
_RESIMULATE_SERVER_STARTED = True
|
||||
|
||||
import json as _json
|
||||
import os as _os
|
||||
import threading as _threading
|
||||
import traceback as _traceback
|
||||
import urllib.parse as _urlparse
|
||||
from http.server import BaseHTTPRequestHandler as _BaseHTTPRequestHandler, ThreadingHTTPServer as _ThreadingHTTPServer
|
||||
|
||||
class _Handler(_BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
def _send(self, code, body):
|
||||
raw = _json.dumps(body).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.end_headers()
|
||||
self.wfile.write(raw)
|
||||
|
||||
def do_GET(self):
|
||||
self._handle()
|
||||
|
||||
def do_POST(self):
|
||||
self._handle()
|
||||
|
||||
def _handle(self):
|
||||
try:
|
||||
parsed = _urlparse.urlparse(self.path)
|
||||
if parsed.path == "/health":
|
||||
age = _last_success_age_seconds()
|
||||
healthy = age is not None and age <= FORECAST_STALE_SECONDS
|
||||
self._send(200 if healthy else 503, {
|
||||
"status": "ok" if healthy else "stale",
|
||||
"last_success_age_seconds": age,
|
||||
"training": _read_json(LAST_TRAINING_PATH),
|
||||
})
|
||||
return
|
||||
if parsed.path not in ("/run_now", "/resimulate"):
|
||||
self._send(404, {"detail": "not found"})
|
||||
return
|
||||
params = _urlparse.parse_qs(parsed.query)
|
||||
anlagen_id = (params.get("anlagen_id") or [None])[0]
|
||||
result = run_forecast_isolated(only_anlagen_id=anlagen_id, source="http")
|
||||
result["anlagen_id"] = anlagen_id
|
||||
code = 200 if result["status"] == "ok" else (409 if result["status"] == "busy" else 500)
|
||||
self._send(code, result)
|
||||
except Exception as exc:
|
||||
_traceback.print_exc()
|
||||
self._send(500, {"detail": str(exc)})
|
||||
|
||||
port = int(_os.getenv("FORECAST_ENGINE_HTTP_PORT", "9000"))
|
||||
server = _ThreadingHTTPServer(("0.0.0.0", port), _Handler)
|
||||
thread = _threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
print(f"Forecast Resimulate HTTP Server startet auf Port {port}.")
|
||||
# END EMS RESIMULATE HTTP SERVER
|
||||
|
||||
def main():
|
||||
_start_resimulate_server()
|
||||
global last_trained_day
|
||||
print("Forecast Engine startet.")
|
||||
threading.Thread(target=_watchdog_loop, daemon=True).start()
|
||||
run_forecast_isolated(source="startup")
|
||||
while True:
|
||||
try:
|
||||
now_local = datetime.datetime.now(LOCAL_TZ)
|
||||
if now_local.hour == TRAINING_HOUR and last_trained_day != now_local.date():
|
||||
if start_training_async(now_local.date()):
|
||||
last_trained_day = now_local.date()
|
||||
next_run = (now_local + datetime.timedelta(hours=1)).replace(minute=0, second=0, microsecond=0)
|
||||
time.sleep(max(60.0, (next_run - now_local).total_seconds()))
|
||||
run_forecast_isolated(source="scheduler")
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "--forecast-once":
|
||||
selected_anlage = sys.argv[2] if len(sys.argv) >= 3 else None
|
||||
run_forecast(only_anlagen_id=selected_anlage, manual=True)
|
||||
_write_json_atomic(LAST_SUCCESS_PATH, {
|
||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"anlagen_id": selected_anlage,
|
||||
"pid": os.getpid(),
|
||||
})
|
||||
elif len(sys.argv) >= 2 and sys.argv[1] == "--train-once":
|
||||
run_training()
|
||||
else:
|
||||
main()
|
||||
@@ -0,0 +1,240 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.optimize import Bounds, LinearConstraint, milp
|
||||
from scipy.sparse import lil_matrix
|
||||
|
||||
DT_H = 5.0 / 60.0
|
||||
|
||||
|
||||
def train_artifact(kind):
|
||||
return {"trained": True, "type": "battery_48h_cost_milp_v3", "source": kind}
|
||||
|
||||
|
||||
def _cfg_float(config, key, default):
|
||||
try:
|
||||
value = config.get(key, default)
|
||||
return float(default if value is None or value == "" else value)
|
||||
except Exception:
|
||||
return float(default)
|
||||
|
||||
|
||||
def _cfg_bool(config, key, default=False):
|
||||
value = config.get(key, default)
|
||||
if value is None or value == "":
|
||||
return bool(default)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "ja", "on"}
|
||||
|
||||
|
||||
def _use_dynamic(config, key):
|
||||
value = str(config.get(key, "") or "").strip().lower()
|
||||
return any(token in value for token in ("dynam", "marktpreis", "referenzmarktpreis", "market"))
|
||||
|
||||
|
||||
def _price(data_obj, timestamp, column, fallback, dynamic_enabled):
|
||||
if not dynamic_enabled:
|
||||
return fallback
|
||||
frame = data_obj["df_fut"]
|
||||
if column in frame.columns and timestamp in frame.index:
|
||||
try:
|
||||
value = float(frame.at[timestamp, column])
|
||||
if np.isfinite(value):
|
||||
return value
|
||||
except Exception:
|
||||
pass
|
||||
return fallback
|
||||
|
||||
|
||||
def _battery_meta(config, data_obj):
|
||||
cap_kwh = _cfg_float(config, "batt_capacity_kwh", 0.0)
|
||||
max_power_w = _cfg_float(config, "batt_power_kw", 0.0) * 1000.0
|
||||
min_soc = _cfg_float(config, "batt_min_soc", _cfg_float(config, "batt_min_soc_percent", 0.0))
|
||||
max_soc = _cfg_float(config, "batt_max_soc", _cfg_float(config, "batt_max_soc_percent", 100.0))
|
||||
start_soc_value = data_obj.get("current_soc_perc")
|
||||
if start_soc_value is None:
|
||||
start_soc_value = min_soc
|
||||
try:
|
||||
start_soc = float(start_soc_value)
|
||||
except (TypeError, ValueError):
|
||||
start_soc = min_soc
|
||||
min_soc = max(0.0, min(100.0, min_soc))
|
||||
max_soc = max(min_soc, min(100.0, max_soc))
|
||||
start_soc = max(min_soc, min(max_soc, start_soc))
|
||||
charge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_charge_efficiency", 0.95)))
|
||||
discharge_eff = max(0.01, min(1.0, _cfg_float(config, "batt_discharge_efficiency", 0.95)))
|
||||
return cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff
|
||||
|
||||
|
||||
def _quarter_groups(index):
|
||||
groups = {}
|
||||
for position, timestamp in enumerate(index):
|
||||
quarter = timestamp.floor("15min") if hasattr(timestamp, "floor") else position // 3
|
||||
groups.setdefault(quarter, []).append(position)
|
||||
return list(groups.values())
|
||||
|
||||
|
||||
def _fallback_plan(index, residual_w):
|
||||
grid = {timestamp: float(value) for timestamp, value in zip(index, residual_w)}
|
||||
battery = {timestamp: 0.0 for timestamp in index}
|
||||
return {"grid": grid, "battery": battery, "solver": "fallback"}
|
||||
|
||||
|
||||
def optimize_battery_plan(data_obj, pv_dict, load_dict):
|
||||
config = data_obj["config"]
|
||||
cap_kwh, max_power_w, min_soc, max_soc, start_soc, charge_eff, discharge_eff = _battery_meta(config, data_obj)
|
||||
index = list(data_obj["df_fut"].index)
|
||||
if not index:
|
||||
return {"grid": {}, "battery": {}, "solver": "empty"}
|
||||
|
||||
load_w = np.array([max(0.0, float(load_dict.get(t, 0.0))) for t in index])
|
||||
pv_w = np.array([max(0.0, float(pv_dict.get(t, 0.0))) for t in index])
|
||||
residual_w = load_w - pv_w
|
||||
if cap_kwh <= 0.0 or max_power_w <= 0.0:
|
||||
return _fallback_plan(index, residual_w)
|
||||
|
||||
import_fixed = _cfg_float(config, "tarif_bezug_fest", 0.30)
|
||||
export_fixed = _cfg_float(config, "tarif_einspeisung_fest", 0.10)
|
||||
import_dynamic = _use_dynamic(config, "tarif_bezug")
|
||||
export_dynamic = _use_dynamic(config, "tarif_einspeisung")
|
||||
import_price = np.array([
|
||||
_price(data_obj, t, "import_price", import_fixed, import_dynamic) for t in index
|
||||
])
|
||||
export_price = np.array([
|
||||
_price(data_obj, t, "export_price", export_fixed, export_dynamic) for t in index
|
||||
])
|
||||
|
||||
n = len(index)
|
||||
imp, exp, charge, discharge, curtail, soc = 0, n, 2 * n, 3 * n, 4 * n, 5 * n
|
||||
peak = 6 * n + 1
|
||||
battery_mode = peak + 1
|
||||
grid_mode = battery_mode + n
|
||||
variable_count = grid_mode + n
|
||||
|
||||
max_import_w = max(float(load_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0)
|
||||
configured_import_limit = _cfg_float(config, "grid_import_limit_w", 0.0)
|
||||
if configured_import_limit > 0.0:
|
||||
max_import_w = min(max_import_w, configured_import_limit)
|
||||
max_export_w = max(float(pv_w.max(initial=0.0)) + max_power_w, max_power_w, 1.0)
|
||||
configured_export_limit = _cfg_float(config, "grid_export_limit_w", 0.0)
|
||||
if configured_export_limit > 0.0:
|
||||
max_export_w = min(max_export_w, configured_export_limit)
|
||||
|
||||
lower = np.zeros(variable_count)
|
||||
upper = np.full(variable_count, np.inf)
|
||||
upper[imp:imp + n] = max_import_w
|
||||
upper[exp:exp + n] = max_export_w
|
||||
upper[charge:charge + n] = max_power_w
|
||||
if not _cfg_bool(config, "batt_grid_charging_enabled", False):
|
||||
upper[charge:charge + n] = np.minimum(max_power_w, np.maximum(0.0, pv_w - load_w))
|
||||
upper[discharge:discharge + n] = max_power_w
|
||||
upper[curtail:curtail + n] = pv_w
|
||||
reserve_soc = max(
|
||||
min_soc,
|
||||
min(100.0, _cfg_float(config, "batt_economic_reserve_soc_percent", 10.0)),
|
||||
)
|
||||
economic_min_soc = max(min_soc, min(start_soc, reserve_soc))
|
||||
min_kwh = cap_kwh * economic_min_soc / 100.0
|
||||
max_kwh = cap_kwh * max_soc / 100.0
|
||||
lower[soc:soc + n + 1] = min_kwh
|
||||
upper[soc:soc + n + 1] = max_kwh
|
||||
current_peak_kw = max(0.0, float(data_obj.get("current_month_peak_kw", 0.0) or 0.0))
|
||||
lower[peak] = current_peak_kw
|
||||
upper[peak] = max(current_peak_kw, max_import_w / 1000.0)
|
||||
upper[battery_mode:battery_mode + n] = 1.0
|
||||
upper[grid_mode:grid_mode + n] = 1.0
|
||||
|
||||
objective = np.zeros(variable_count)
|
||||
objective[imp:imp + n] = import_price * DT_H / 1000.0
|
||||
objective[exp:exp + n] = -export_price * DT_H / 1000.0
|
||||
degradation = max(0.0, _cfg_float(config, "batt_degradation_chf_kwh", 0.03))
|
||||
objective[charge:charge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0
|
||||
objective[discharge:discharge + n] = (degradation / 2.0 + 1e-7) * DT_H / 1000.0
|
||||
objective[curtail:curtail + n] = 1e-9 * DT_H / 1000.0
|
||||
objective[peak] = max(0.0, _cfg_float(config, "tarif_peak_fest", 0.0))
|
||||
terminal_value = _cfg_float(config, "batt_terminal_value_chf_kwh", np.median(import_price))
|
||||
objective[soc + n] = -max(0.0, terminal_value) * discharge_eff
|
||||
|
||||
equality_rows = 2 * n + 1
|
||||
equality = lil_matrix((equality_rows, variable_count), dtype=float)
|
||||
equality_rhs = np.zeros(equality_rows)
|
||||
for i in range(n):
|
||||
equality[i, imp + i] = 1.0
|
||||
equality[i, exp + i] = -1.0
|
||||
equality[i, charge + i] = -1.0
|
||||
equality[i, discharge + i] = 1.0
|
||||
equality[i, curtail + i] = -1.0
|
||||
equality_rhs[i] = residual_w[i]
|
||||
|
||||
row = n + i
|
||||
equality[row, soc + i] = -1.0
|
||||
equality[row, soc + i + 1] = 1.0
|
||||
equality[row, charge + i] = -charge_eff * DT_H / 1000.0
|
||||
equality[row, discharge + i] = DT_H / (1000.0 * discharge_eff)
|
||||
equality[2 * n, soc] = 1.0
|
||||
equality_rhs[2 * n] = cap_kwh * start_soc / 100.0
|
||||
|
||||
quarter_groups = _quarter_groups(pd.Index(index))
|
||||
inequality_rows = 4 * n + len(quarter_groups)
|
||||
inequality = lil_matrix((inequality_rows, variable_count), dtype=float)
|
||||
inequality_upper = np.zeros(inequality_rows)
|
||||
row = 0
|
||||
for i in range(n):
|
||||
inequality[row, charge + i] = 1.0
|
||||
inequality[row, battery_mode + i] = -max_power_w
|
||||
row += 1
|
||||
inequality[row, discharge + i] = 1.0
|
||||
inequality[row, battery_mode + i] = max_power_w
|
||||
inequality_upper[row] = max_power_w
|
||||
row += 1
|
||||
inequality[row, imp + i] = 1.0
|
||||
inequality[row, grid_mode + i] = -max_import_w
|
||||
row += 1
|
||||
inequality[row, exp + i] = 1.0
|
||||
inequality[row, grid_mode + i] = max_export_w
|
||||
inequality_upper[row] = max_export_w
|
||||
row += 1
|
||||
for group in quarter_groups:
|
||||
for i in group:
|
||||
inequality[row, imp + i] = 1.0 / (len(group) * 1000.0)
|
||||
inequality[row, peak] = -1.0
|
||||
row += 1
|
||||
|
||||
integrality = np.zeros(variable_count, dtype=int)
|
||||
integrality[battery_mode:battery_mode + n] = 1
|
||||
integrality[grid_mode:grid_mode + n] = 1
|
||||
constraints = [
|
||||
LinearConstraint(equality.tocsr(), equality_rhs, equality_rhs),
|
||||
LinearConstraint(inequality.tocsr(), -np.inf, inequality_upper),
|
||||
]
|
||||
result = milp(
|
||||
objective,
|
||||
integrality=integrality,
|
||||
bounds=Bounds(lower, upper),
|
||||
constraints=constraints,
|
||||
options={"time_limit": max(5.0, _cfg_float(config, "batt_optimizer_timeout_seconds", 30.0))},
|
||||
)
|
||||
if not result.success or result.x is None:
|
||||
return _fallback_plan(index, residual_w)
|
||||
|
||||
grid_values = result.x[imp:imp + n] - result.x[exp:exp + n]
|
||||
battery_values = result.x[charge:charge + n] - result.x[discharge:discharge + n]
|
||||
threshold_w = max(25.0, max_power_w * 0.005)
|
||||
grid_values[np.abs(grid_values) < threshold_w] = 0.0
|
||||
battery_values[np.abs(battery_values) < threshold_w] = 0.0
|
||||
return {
|
||||
"grid": {t: float(v) for t, v in zip(index, grid_values)},
|
||||
"battery": {t: float(v) for t, v in zip(index, battery_values)},
|
||||
"solver": "scipy-milp",
|
||||
"objective_chf": float(result.fun),
|
||||
"planned_peak_kw": float(result.x[peak]),
|
||||
"start_soc_percent": float(start_soc),
|
||||
"economic_min_soc_percent": float(economic_min_soc),
|
||||
}
|
||||
|
||||
|
||||
def optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=None):
|
||||
plan = optimize_battery_plan(data_obj, pv_dict, load_dict)
|
||||
if forecast_id is not None:
|
||||
data_obj.setdefault("battery_plans", {})[int(forecast_id)] = plan
|
||||
return plan["grid"]
|
||||
@@ -0,0 +1,32 @@
|
||||
import os
|
||||
import joblib
|
||||
|
||||
MODEL_DIR = "/app/data/models"
|
||||
|
||||
|
||||
def model_path(aid, forecast_id):
|
||||
os.makedirs(MODEL_DIR, exist_ok=True)
|
||||
return os.path.join(MODEL_DIR, f"forecast_var_{forecast_id}_{aid}.pkl")
|
||||
|
||||
|
||||
def save_model(aid, forecast_id, artifact):
|
||||
path = model_path(aid, forecast_id)
|
||||
tmp_path = f"{path}.tmp.{os.getpid()}"
|
||||
try:
|
||||
joblib.dump(artifact, tmp_path)
|
||||
os.replace(tmp_path, path)
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
return path
|
||||
|
||||
|
||||
def load_model(aid, forecast_id):
|
||||
path = model_path(aid, forecast_id)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
return joblib.load(path)
|
||||
except Exception as exc:
|
||||
print(f"Modell {path} konnte nicht geladen werden: {exc}")
|
||||
return None
|
||||
@@ -0,0 +1,66 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
from methods.common import load_model, save_model
|
||||
from shared_utils import calc_pure_math_pv, roof_features
|
||||
|
||||
FORECAST_ID = 1
|
||||
FEATURES = [
|
||||
"math_pv",
|
||||
"temp_c",
|
||||
"cloud",
|
||||
"hour_sin",
|
||||
"hour_cos",
|
||||
"sin_year",
|
||||
"cos_year",
|
||||
"pv_kwp_total",
|
||||
"roof_azimuth_sin",
|
||||
"roof_azimuth_cos",
|
||||
"roof_tilt_avg",
|
||||
"roof_south_factor",
|
||||
]
|
||||
|
||||
|
||||
def _features(frame, config):
|
||||
out = frame.copy()
|
||||
out["math_pv"] = [calc_pure_math_pv(config, t) for t in out.index]
|
||||
rf = roof_features(config)
|
||||
for k, v in rf.items():
|
||||
out[k] = v
|
||||
for col, default in [("temp_c", 15.0), ("cloud", 20.0)]:
|
||||
if col not in out.columns:
|
||||
out[col] = default
|
||||
out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default)
|
||||
return out[FEATURES].astype(float)
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
config = data_obj["config"]
|
||||
aid = config["anlagen_id"]
|
||||
df = data_obj.get("df_pv_training", data_obj["df_hist"]).copy()
|
||||
if "PV" not in df.columns:
|
||||
return {"trained": False, "reason": "PV fehlt"}
|
||||
X = _features(df, config)
|
||||
y = pd.to_numeric(df["PV"], errors="coerce")
|
||||
valid = X.notna().all(axis=1) & y.notna()
|
||||
X, y = X.loc[valid], y.loc[valid]
|
||||
if len(X) < 288:
|
||||
return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))}
|
||||
model = RandomForestRegressor(n_estimators=400, max_depth=18, min_samples_leaf=2, random_state=42, n_jobs=-1)
|
||||
model.fit(X, y)
|
||||
path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES})
|
||||
return {"trained": True, "samples": int(len(X)), "path": path}
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
config = data_obj["config"]
|
||||
aid = config["anlagen_id"]
|
||||
artifact = load_model(aid, FORECAST_ID)
|
||||
X = _features(data_obj["df_fut"], config)
|
||||
if artifact and "model" in artifact:
|
||||
values = artifact["model"].predict(X)
|
||||
else:
|
||||
values = X["math_pv"].to_numpy()
|
||||
ac_limit = float(config.get("ac_leistung", 10.0) or 10.0) * 1000.0
|
||||
return {t: max(0.0, min(float(v), ac_limit)) for t, v in zip(data_obj["df_fut"].index, values)}
|
||||
@@ -0,0 +1,12 @@
|
||||
from telemetry_quality import repeat_daily_profile
|
||||
import datetime
|
||||
from methods.common import save_model
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
return {"trained": True, "path": save_model(aid, 10, {"type": "repeat_pv_24h"})}
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'PV')
|
||||
@@ -0,0 +1,12 @@
|
||||
from telemetry_quality import repeat_daily_profile
|
||||
import datetime
|
||||
from methods.common import save_model
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
return {"trained": True, "path": save_model(aid, 11, {"type": "repeat_load_24h"})}
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
return repeat_daily_profile(data_obj["df_hist"], data_obj["df_fut"].index, 'Hausverbrauch')
|
||||
@@ -0,0 +1,13 @@
|
||||
from methods.battery_optimizer import optimize_grid_setpoint, train_artifact
|
||||
from methods.common import save_model
|
||||
|
||||
FORECAST_ID = 13
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_10_11"))}
|
||||
|
||||
|
||||
def predict(data_obj, pv_dict, load_dict):
|
||||
return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=13)
|
||||
@@ -0,0 +1,169 @@
|
||||
import datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
|
||||
from methods.common import load_model, save_model
|
||||
from telemetry_quality import profile_source_value
|
||||
|
||||
FORECAST_ID = 2
|
||||
FEATURES = [
|
||||
"temp_c",
|
||||
"cloud",
|
||||
"hour_sin",
|
||||
"hour_cos",
|
||||
"weekday",
|
||||
"is_weekday",
|
||||
"load_24h_ago",
|
||||
"load_7d_ago",
|
||||
"energy_24h_rolling",
|
||||
"load_3d_same_time_mean",
|
||||
"load_7d_same_time_mean",
|
||||
]
|
||||
|
||||
|
||||
def _history_features(df):
|
||||
out = df.copy()
|
||||
out["load_24h_ago"] = out["Hausverbrauch"].shift(288)
|
||||
out["load_7d_ago"] = out["Hausverbrauch"].shift(2016)
|
||||
out["energy_24h_rolling"] = (out["Hausverbrauch"] * 5 / 60 / 1000).shift(1).rolling(288).sum()
|
||||
same_time_lags = [out["Hausverbrauch"].shift(288 * d) for d in range(1, 8)]
|
||||
out["load_3d_same_time_mean"] = pd.concat(same_time_lags[:3], axis=1).mean(axis=1)
|
||||
out["load_7d_same_time_mean"] = pd.concat(same_time_lags, axis=1).mean(axis=1)
|
||||
for col, default in [("temp_c", 15.0), ("cloud", 20.0)]:
|
||||
if col not in out.columns:
|
||||
out[col] = default
|
||||
out[col] = pd.to_numeric(out[col], errors="coerce").ffill().bfill().fillna(default)
|
||||
return out
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
df = data_obj.get("df_load_training", data_obj["df_hist"]).copy()
|
||||
if "Hausverbrauch" not in df.columns:
|
||||
return {"trained": False, "reason": "Hausverbrauch fehlt"}
|
||||
df = _history_features(df)
|
||||
X = df[FEATURES].astype(float)
|
||||
y = pd.to_numeric(df["Hausverbrauch"], errors="coerce")
|
||||
valid = X.notna().all(axis=1) & y.notna()
|
||||
X, y = X.loc[valid], y.loc[valid]
|
||||
if len(X) < 288:
|
||||
return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(X))}
|
||||
validation_day = X.index.max().normalize()
|
||||
validation_mask = X.index >= validation_day
|
||||
if int(validation_mask.sum()) < 144:
|
||||
validation_day -= datetime.timedelta(days=1)
|
||||
validation_mask = (X.index >= validation_day) & (X.index < validation_day + datetime.timedelta(days=1))
|
||||
score = None
|
||||
if int((~validation_mask).sum()) >= 288 and int(validation_mask.sum()) >= 96:
|
||||
eval_model = HistGradientBoostingRegressor(
|
||||
max_iter=900,
|
||||
max_depth=12,
|
||||
learning_rate=0.02,
|
||||
min_samples_leaf=8,
|
||||
random_state=42,
|
||||
)
|
||||
eval_model.fit(X.loc[~validation_mask], y.loc[~validation_mask])
|
||||
score = float(eval_model.score(X.loc[validation_mask], y.loc[validation_mask]))
|
||||
|
||||
model = HistGradientBoostingRegressor(
|
||||
max_iter=1200,
|
||||
max_depth=12,
|
||||
learning_rate=0.02,
|
||||
min_samples_leaf=8,
|
||||
random_state=42,
|
||||
)
|
||||
model.fit(X, y)
|
||||
path = save_model(
|
||||
aid,
|
||||
FORECAST_ID,
|
||||
{"model": model, "features": FEATURES, "r2_last_day": score},
|
||||
)
|
||||
return {
|
||||
"trained": True,
|
||||
"samples": int(len(X)),
|
||||
"r2_last_day": score,
|
||||
"path": path,
|
||||
}
|
||||
|
||||
|
||||
def _history_value(history, ts):
|
||||
if history is None or history.empty or ts not in history.index or "Hausverbrauch" not in history.columns:
|
||||
return None
|
||||
value = history.at[ts, "Hausverbrauch"]
|
||||
return float(value) if pd.notna(value) and np.isfinite(value) and value >= 0 else None
|
||||
|
||||
|
||||
def _same_time_values(history, t, days):
|
||||
values = []
|
||||
for day in range(1, days + 1):
|
||||
value = _history_value(history, t - datetime.timedelta(days=day))
|
||||
if value is not None and np.isfinite(value):
|
||||
values.append(max(0.0, value))
|
||||
return values
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
artifact = load_model(aid, FORECAST_ID)
|
||||
model = artifact["model"] if artifact and "model" in artifact else None
|
||||
try:
|
||||
score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
score = 0.0
|
||||
model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0
|
||||
|
||||
frames = [
|
||||
frame
|
||||
for frame in (
|
||||
data_obj.get("df_load_training"),
|
||||
data_obj.get("df_hist"),
|
||||
data_obj.get("df_recent_raw"),
|
||||
)
|
||||
if frame is not None and not frame.empty and "Hausverbrauch" in frame.columns
|
||||
]
|
||||
history = pd.concat(frames).sort_index() if frames else pd.DataFrame(columns=["Hausverbrauch"])
|
||||
history = history[~history.index.duplicated(keep="last")]
|
||||
fut = data_obj["df_fut"]
|
||||
res = {}
|
||||
|
||||
for t in fut.index:
|
||||
same_values = _same_time_values(history, t, 7)
|
||||
load_24 = _history_value(history, t - datetime.timedelta(days=1))
|
||||
load_7d = _history_value(history, t - datetime.timedelta(days=7))
|
||||
if load_24 is None:
|
||||
load_24 = load_7d if load_7d is not None else profile_source_value(history, t, "Hausverbrauch")
|
||||
if load_7d is None:
|
||||
load_7d = load_24
|
||||
same_median = float(np.median(same_values)) if same_values else load_7d
|
||||
profile = max(0.0, (0.50 * load_24) + (0.30 * load_7d) + (0.20 * same_median))
|
||||
|
||||
window = history.loc[
|
||||
t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5),
|
||||
"Hausverbrauch",
|
||||
]
|
||||
energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0
|
||||
same_3 = same_values[:3]
|
||||
row = pd.DataFrame([[
|
||||
float(fut.at[t, "temp_c"]),
|
||||
float(fut.at[t, "cloud"]),
|
||||
float(fut.at[t, "hour_sin"]),
|
||||
float(fut.at[t, "hour_cos"]),
|
||||
int(fut.at[t, "weekday"]),
|
||||
int(fut.at[t, "is_weekday"]),
|
||||
load_24,
|
||||
load_7d,
|
||||
energy,
|
||||
float(np.mean(same_3)) if same_3 else profile,
|
||||
float(np.mean(same_values)) if same_values else profile,
|
||||
]], columns=FEATURES)
|
||||
|
||||
pred = profile
|
||||
if model is not None and model_weight > 0.0:
|
||||
model_pred = max(0.0, float(model.predict(row)[0]))
|
||||
model_pred = min(max(model_pred, profile * 0.25), max(500.0, profile * 3.0))
|
||||
pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred)
|
||||
|
||||
res[t] = pred
|
||||
history.loc[t, "Hausverbrauch"] = pred
|
||||
return res
|
||||
@@ -0,0 +1,62 @@
|
||||
import datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.ensemble import RandomForestRegressor
|
||||
|
||||
from methods.common import load_model, save_model
|
||||
|
||||
FORECAST_ID = 21
|
||||
FEATURES = ["temp_c", "hour_cos", "pv_24h_ago"]
|
||||
|
||||
|
||||
def _feature_frame(df):
|
||||
out = df.copy()
|
||||
if "temp_c" not in out.columns:
|
||||
out["temp_c"] = 15.0
|
||||
out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0)
|
||||
out["hour_float"] = out.index.hour + out.index.minute / 60.0
|
||||
out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0)
|
||||
if "PV" in out.columns:
|
||||
out["pv_24h_ago"] = out["PV"].shift(288)
|
||||
return out
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
df = _feature_frame(data_obj.get("df_pv_training", data_obj["df_hist"]).copy())
|
||||
if "PV" not in df.columns:
|
||||
return {"trained": False, "reason": "PV fehlt"}
|
||||
df = df.dropna(subset=FEATURES + ["PV"])
|
||||
if len(df) < 288:
|
||||
return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))}
|
||||
|
||||
last_day = df.index.max().normalize()
|
||||
train_df = df[df.index < last_day]
|
||||
test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))]
|
||||
score = None
|
||||
if len(train_df) >= 288 and len(test_df) >= 12:
|
||||
eval_model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42)
|
||||
eval_model.fit(train_df[FEATURES], train_df["PV"])
|
||||
score = float(eval_model.score(test_df[FEATURES], test_df["PV"]))
|
||||
print(f"[var_21] R2 PV letzter kompletter Tag: {score:.3f}")
|
||||
|
||||
model = RandomForestRegressor(n_estimators=300, max_depth=15, random_state=42)
|
||||
model.fit(df[FEATURES], df["PV"])
|
||||
path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score})
|
||||
return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path}
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
artifact = load_model(aid, FORECAST_ID)
|
||||
model = artifact["model"] if artifact and "model" in artifact else None
|
||||
hist = data_obj["df_hist"]
|
||||
fut = data_obj["df_fut"].copy()
|
||||
res = {}
|
||||
for t in fut.index:
|
||||
t_24 = t - datetime.timedelta(days=1)
|
||||
pv_24 = float(hist.at[t_24, "PV"]) if t_24 in hist.index else 0.0
|
||||
row = pd.DataFrame([[float(fut.at[t, "temp_c"]), float(fut.at[t, "hour_cos"]), pv_24]], columns=FEATURES)
|
||||
pred = float(model.predict(row)[0]) if model is not None else pv_24
|
||||
res[t] = max(0.0, pred)
|
||||
return res
|
||||
@@ -0,0 +1,121 @@
|
||||
import datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from sklearn.inspection import permutation_importance
|
||||
|
||||
from methods.common import load_model, save_model
|
||||
|
||||
FORECAST_ID = 22
|
||||
FEATURES = ["temp_c", "hour_cos", "load_24h_ago", "load_7d_ago", "energy_24h_rolling", "weekday"]
|
||||
|
||||
|
||||
def _feature_frame(df):
|
||||
out = df.copy()
|
||||
if "temp_c" not in out.columns:
|
||||
out["temp_c"] = 15.0
|
||||
out["temp_c"] = pd.to_numeric(out["temp_c"], errors="coerce").ffill().bfill().fillna(15.0)
|
||||
out["hour_float"] = out.index.hour + out.index.minute / 60.0
|
||||
out["hour_cos"] = np.cos(2 * np.pi * out["hour_float"] / 24.0)
|
||||
out["weekday"] = out.index.weekday
|
||||
if "Hausverbrauch" in out.columns:
|
||||
out["load_24h_ago"] = out["Hausverbrauch"].shift(288)
|
||||
out["load_7d_ago"] = out["Hausverbrauch"].shift(2016)
|
||||
out["energy_5m_kwh"] = out["Hausverbrauch"] * (5 / 60) / 1000
|
||||
out["energy_24h_rolling"] = out["energy_5m_kwh"].shift(1).rolling(window=288).sum()
|
||||
out = out.drop(columns=["energy_5m_kwh"])
|
||||
return out
|
||||
|
||||
|
||||
def _history_value(history, recent, reference, ts):
|
||||
for frame in (history, recent, reference):
|
||||
if frame is not None and not frame.empty and ts in frame.index and "Hausverbrauch" in frame.columns:
|
||||
value = frame.at[ts, "Hausverbrauch"]
|
||||
if pd.notna(value):
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
df = _feature_frame(data_obj.get("df_load_training", data_obj["df_hist"]).copy())
|
||||
if "Hausverbrauch" not in df.columns:
|
||||
return {"trained": False, "reason": "Hausverbrauch fehlt"}
|
||||
df = df.dropna(subset=FEATURES + ["Hausverbrauch"])
|
||||
if len(df) < 288:
|
||||
return {"trained": False, "reason": "zu wenig Daten", "samples": int(len(df))}
|
||||
|
||||
last_day = df.index.max().normalize()
|
||||
train_df = df[df.index < last_day]
|
||||
test_df = df[(df.index >= last_day) & (df.index < last_day + datetime.timedelta(days=1))]
|
||||
score = None
|
||||
importance = {}
|
||||
if len(train_df) >= 288 and len(test_df) >= 12:
|
||||
eval_model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42)
|
||||
eval_model.fit(train_df[FEATURES], train_df["Hausverbrauch"])
|
||||
score = float(eval_model.score(test_df[FEATURES], test_df["Hausverbrauch"]))
|
||||
print(f"[var_22] R2 Hausverbrauch letzter kompletter Tag: {score:.3f}")
|
||||
try:
|
||||
perm = permutation_importance(eval_model, test_df[FEATURES], test_df["Hausverbrauch"], n_repeats=10, random_state=42)
|
||||
order = perm.importances_mean.argsort()[::-1]
|
||||
importance = {FEATURES[i]: float(perm.importances_mean[i]) for i in order}
|
||||
print("[var_22] Feature-Wichtigkeit Hausverbrauch:")
|
||||
for name, val in importance.items():
|
||||
print(f" {name}: {val:.4f}")
|
||||
except Exception as exc:
|
||||
print(f"[var_22] permutation_importance nicht berechnet: {exc}")
|
||||
|
||||
model = HistGradientBoostingRegressor(max_iter=2500, max_depth=25, learning_rate=0.01, min_samples_leaf=1, random_state=42)
|
||||
model.fit(df[FEATURES], df["Hausverbrauch"])
|
||||
path = save_model(aid, FORECAST_ID, {"model": model, "features": FEATURES, "r2_last_day": score, "importance": importance})
|
||||
return {"trained": True, "samples": int(len(df)), "features": FEATURES, "r2_last_day": score, "path": path}
|
||||
|
||||
|
||||
def predict(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
artifact = load_model(aid, FORECAST_ID)
|
||||
model = artifact["model"] if artifact and "model" in artifact else None
|
||||
try:
|
||||
score = float(artifact.get("r2_last_day")) if artifact and artifact.get("r2_last_day") is not None else 0.0
|
||||
except (TypeError, ValueError):
|
||||
score = 0.0
|
||||
model_weight = min(0.35, max(0.0, score) * 0.35) if np.isfinite(score) else 0.0
|
||||
hist = data_obj["df_hist"].copy()
|
||||
recent = data_obj.get("df_recent_raw", pd.DataFrame())
|
||||
reference = data_obj.get("df_load_training", pd.DataFrame())
|
||||
fut = data_obj["df_fut"]
|
||||
res = {}
|
||||
for t in fut.index:
|
||||
t_24 = t - datetime.timedelta(days=1)
|
||||
t_7d = t - datetime.timedelta(days=7)
|
||||
load_7d = _history_value(hist, recent, reference, t_7d)
|
||||
load_24 = _history_value(hist, recent, reference, t_24)
|
||||
if load_24 is None:
|
||||
load_24 = load_7d if load_7d is not None else 0.0
|
||||
if load_7d is None:
|
||||
load_7d = load_24
|
||||
ref_end = t - datetime.timedelta(days=7)
|
||||
ref_start = ref_end - datetime.timedelta(days=1)
|
||||
if not reference.empty and "Hausverbrauch" in reference.columns:
|
||||
window = reference.loc[ref_start:ref_end - datetime.timedelta(minutes=5), "Hausverbrauch"]
|
||||
else:
|
||||
window = pd.Series(dtype=float)
|
||||
if window.empty and "Hausverbrauch" in recent.columns:
|
||||
window = recent.loc[t - datetime.timedelta(days=1):t - datetime.timedelta(minutes=5), "Hausverbrauch"]
|
||||
roll_energy = float((window.sum() * 5 / 60) / 1000.0) if not window.empty else 0.0
|
||||
row = pd.DataFrame([[
|
||||
float(fut.at[t, "temp_c"]),
|
||||
float(fut.at[t, "hour_cos"]),
|
||||
load_24,
|
||||
load_7d,
|
||||
roll_energy,
|
||||
int(fut.at[t, "weekday"]),
|
||||
]], columns=FEATURES)
|
||||
profile = max(0.0, (0.65 * load_24) + (0.35 * load_7d))
|
||||
pred = profile
|
||||
if model is not None and model_weight > 0.0:
|
||||
model_pred = max(0.0, float(model.predict(row)[0]))
|
||||
pred = ((1.0 - model_weight) * profile) + (model_weight * model_pred)
|
||||
res[t] = pred
|
||||
hist.loc[t, "Hausverbrauch"] = pred
|
||||
return res
|
||||
@@ -0,0 +1,13 @@
|
||||
from methods.battery_optimizer import optimize_grid_setpoint, train_artifact
|
||||
from methods.common import save_model
|
||||
|
||||
FORECAST_ID = 23
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_21_22"))}
|
||||
|
||||
|
||||
def predict(data_obj, pv_dict, load_dict):
|
||||
return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=23)
|
||||
@@ -0,0 +1,13 @@
|
||||
from methods.battery_optimizer import optimize_grid_setpoint, train_artifact
|
||||
from methods.common import save_model
|
||||
|
||||
FORECAST_ID = 3
|
||||
|
||||
|
||||
def train(data_obj):
|
||||
aid = data_obj["config"]["anlagen_id"]
|
||||
return {"trained": True, "path": save_model(aid, FORECAST_ID, train_artifact("var_1_2"))}
|
||||
|
||||
|
||||
def predict(data_obj, pv_dict, load_dict):
|
||||
return optimize_grid_setpoint(data_obj, pv_dict, load_dict, forecast_id=3)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Forecast family fault isolation; no IO, fabricated samples or model switching."""
|
||||
from collections.abc import Mapping
|
||||
from math import isfinite
|
||||
from numbers import Real
|
||||
|
||||
|
||||
def collect_predictions(data, config, models, enabled):
|
||||
"""Return complete finite PV/load series independently for each requested model.
|
||||
|
||||
A model with insufficient history must not prevent other valid families being
|
||||
published. It is omitted, not filled with zeros or replaced by another model.
|
||||
Global raw-telemetry checks still run BEFORE this helper in run_forecast.
|
||||
"""
|
||||
expected = tuple(data['df_fut'].index)
|
||||
if not expected or len(set(expected)) != len(expected):
|
||||
raise ValueError('Nonempty unique future interval index required')
|
||||
expected_keys = set(expected)
|
||||
predictions, errors = {}, {}
|
||||
requested = 0
|
||||
for variant, model in models.items():
|
||||
predictions[variant] = {}
|
||||
if not enabled(config, variant):
|
||||
continue
|
||||
requested += 1
|
||||
try:
|
||||
values = model.predict(data)
|
||||
if not isinstance(values, Mapping) or set(values) != expected_keys:
|
||||
raise ValueError('Missing, extra or non-matching forecast timestamps')
|
||||
cleaned = {}
|
||||
for at in expected:
|
||||
value = values[at]
|
||||
if isinstance(value, bool) or not isinstance(value, Real):
|
||||
raise ValueError('Non-numeric forecast power')
|
||||
value = float(value)
|
||||
if not isfinite(value) or value < 0:
|
||||
raise ValueError('Invalid nonnegative forecast power')
|
||||
cleaned[at] = value
|
||||
predictions[variant] = cleaned
|
||||
except Exception as error:
|
||||
# Expected data failures and unexpected model failures are visible,
|
||||
# but model exception text may contain filesystem paths. Do not leak it.
|
||||
errors[variant] = {'status': 'unavailable', 'errorType': type(error).__name__}
|
||||
if requested and all(not p for p in predictions.values()):
|
||||
raise ValueError('All requested PV/load models failed; no forecasts published')
|
||||
return predictions, errors
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services.
|
||||
No credentials are printed. Existing V1 computation survives shadow-service errors.
|
||||
Legacy UTC-naive forecast indices are explicitly interpreted as UTC here.
|
||||
"""
|
||||
from datetime import datetime,timezone
|
||||
from hashlib import sha256
|
||||
from uuid import uuid4
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request,urlopen
|
||||
import json,logging,math,os
|
||||
|
||||
def timestamp(value):
|
||||
if hasattr(value,'to_pydatetime'):value=value.to_pydatetime()
|
||||
if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00'))
|
||||
if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat()
|
||||
|
||||
def tariff_id(label):
|
||||
if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required')
|
||||
return 'legacy:'+sha256(label.encode()).hexdigest()[:32]
|
||||
|
||||
def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))}
|
||||
|
||||
def tariff_payload(config,at=None):
|
||||
result=envelope(at)
|
||||
for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')):
|
||||
mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus'))
|
||||
if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name')
|
||||
item={'mode':mode,'tariffId':tariff_id(config[prefix])}
|
||||
if mode=='static':
|
||||
value=config.get(prefix+'_fest')
|
||||
if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid')
|
||||
item['staticChfKwh']=float(value)
|
||||
result[side]=item
|
||||
peak=config.get('tarif_peak_fest')
|
||||
if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required')
|
||||
result['peakChfKwMonth']=float(peak)
|
||||
return result
|
||||
|
||||
def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None):
|
||||
result=envelope(at);result['families']={}
|
||||
for key,pv,load in forecasts:
|
||||
if not pv or not load:continue
|
||||
if set(pv)!=set(load):raise ValueError('PV/load timestamps differ')
|
||||
result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until,
|
||||
'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]}
|
||||
if not result['families']:raise ValueError('No forecast families supplied')
|
||||
return result
|
||||
|
||||
def ckw_payload(label,rows,publication_timestamp=None,at=None):
|
||||
"""Provider-delimited integrated price intervals only. No scalar price replication."""
|
||||
result=envelope(at);periods=[]
|
||||
units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'}
|
||||
for row in rows:
|
||||
if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required')
|
||||
integrated=row.get('integrated')
|
||||
if isinstance(integrated,list):
|
||||
if len(integrated)!=1:raise ValueError('Ambiguous integrated price components')
|
||||
integrated=integrated[0]
|
||||
if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required')
|
||||
value=integrated.get('value')
|
||||
if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required')
|
||||
item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']),
|
||||
'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']],
|
||||
'observedAt':result['observedAt'],'sourceKind':'published_interval'}
|
||||
if publication_timestamp:
|
||||
published=timestamp(publication_timestamp)
|
||||
if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication')
|
||||
item['publishedAt']=published
|
||||
periods.append(item)
|
||||
result['periods']=periods
|
||||
return result
|
||||
|
||||
def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()})
|
||||
|
||||
def send(plant,kind,payload):
|
||||
if not enabled(plant):return {'status':'disabled'}
|
||||
token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base)
|
||||
if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured')
|
||||
UUID=__import__('uuid').UUID;UUID(plant)
|
||||
if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind')
|
||||
req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind,
|
||||
json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST')
|
||||
with urlopen(req,timeout=5) as response:
|
||||
data=response.read(65537)
|
||||
if len(data)>65536:raise ValueError('Oversized service response')
|
||||
return json.loads(data)
|
||||
|
||||
def publish_forecasts(config,forecasts):
|
||||
plant=config['anlagen_id']
|
||||
if not enabled(plant):return
|
||||
try:
|
||||
send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts))
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__)
|
||||
|
||||
def publish_tariffs(plant,config):
|
||||
if not enabled(plant):return
|
||||
try:send(plant,'tariffs',tariff_payload(config))
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__)
|
||||
|
||||
def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'):
|
||||
plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()]
|
||||
if not os.getenv('NETPLAN_V4_URL') or not plants:return
|
||||
try:
|
||||
if tariff_type!='integrated':raise ValueError('Full integrated tariff required')
|
||||
payload=ckw_payload(label,rows,publication_timestamp)
|
||||
for plant in plants:send(plant,'prices',payload)
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__)
|
||||
@@ -0,0 +1,93 @@
|
||||
import math
|
||||
|
||||
|
||||
def as_float(value, default=0.0):
|
||||
try:
|
||||
if value is None:
|
||||
return default
|
||||
return float(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def roof_kwp(roof):
|
||||
return as_float(roof.get("kwp", roof.get("leistung", roof.get("pv_kwp", 0.0))), 0.0)
|
||||
|
||||
|
||||
def roof_azimuth_deg(roof):
|
||||
return as_float(roof.get("azimut", roof.get("azimuth", roof.get("ausrichtung", 180.0))), 180.0)
|
||||
|
||||
|
||||
def roof_tilt_deg(roof):
|
||||
return as_float(roof.get("neigung", roof.get("tilt", 30.0)), 30.0)
|
||||
|
||||
|
||||
def roof_features(config):
|
||||
roofs = config.get("daecher", []) or []
|
||||
if not roofs:
|
||||
roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "azimut": 180.0, "neigung": 30.0}]
|
||||
total_kwp = sum(roof_kwp(r) for r in roofs)
|
||||
if total_kwp <= 0:
|
||||
total_kwp = as_float(config.get("ac_leistung", 0.0), 0.0)
|
||||
weighted_az = 0.0
|
||||
weighted_tilt = 0.0
|
||||
south_factor = 0.0
|
||||
for roof in roofs:
|
||||
w = roof_kwp(roof) / total_kwp if total_kwp > 0 else 0.0
|
||||
az = roof_azimuth_deg(roof)
|
||||
tilt = roof_tilt_deg(roof)
|
||||
weighted_az += w * az
|
||||
weighted_tilt += w * tilt
|
||||
south_factor += w * max(0.0, math.cos(math.radians(az - 180.0)))
|
||||
return {
|
||||
"pv_kwp_total": total_kwp,
|
||||
"roof_azimuth_sin": math.sin(math.radians(weighted_az)),
|
||||
"roof_azimuth_cos": math.cos(math.radians(weighted_az)),
|
||||
"roof_tilt_avg": weighted_tilt,
|
||||
"roof_south_factor": south_factor,
|
||||
}
|
||||
|
||||
|
||||
def calc_pure_math_pv(config, dt):
|
||||
ac_limit = as_float(config.get("ac_leistung", 10.0), 10.0) * 1000.0
|
||||
roofs = config.get("daecher", []) or []
|
||||
if not roofs:
|
||||
roofs = [{"kwp": as_float(config.get("ac_leistung", 0.0), 0.0), "neigung": 30.0, "azimut": 180.0}]
|
||||
|
||||
day = dt.timetuple().tm_yday
|
||||
hour = dt.hour + dt.minute / 60.0
|
||||
lat = math.radians(as_float(config.get("latitude", 47.0), 47.0))
|
||||
decl = math.radians(23.45 * math.sin(math.radians(360.0 * (day - 81) / 365.0)))
|
||||
hour_angle = math.radians(15.0 * (hour - 12.0))
|
||||
|
||||
sin_alt = math.sin(lat) * math.sin(decl) + math.cos(lat) * math.cos(decl) * math.cos(hour_angle)
|
||||
sun_alt = math.asin(max(-1.0, min(1.0, sin_alt)))
|
||||
if sun_alt <= 0.0:
|
||||
return 0.0
|
||||
|
||||
sun_az = math.atan2(
|
||||
math.sin(hour_angle),
|
||||
math.cos(hour_angle) * math.sin(lat) - math.tan(decl) * math.cos(lat),
|
||||
)
|
||||
|
||||
total = 0.0
|
||||
for roof in roofs:
|
||||
kwp = roof_kwp(roof)
|
||||
if kwp <= 0:
|
||||
continue
|
||||
tilt = math.radians(roof_tilt_deg(roof))
|
||||
az = math.radians(roof_azimuth_deg(roof))
|
||||
cos_inc = math.sin(sun_alt) * math.cos(tilt) + math.cos(sun_alt) * math.sin(tilt) * math.cos(sun_az - az)
|
||||
if cos_inc > 0:
|
||||
total += kwp * 1000.0 * cos_inc
|
||||
|
||||
return max(0.0, min(total, ac_limit))
|
||||
|
||||
|
||||
def calc_pure_math_load(dt):
|
||||
hour = dt.hour + dt.minute / 60.0
|
||||
base = 650.0
|
||||
morning = 220.0 * math.exp(-((hour - 7.0) ** 2) / 5.0)
|
||||
evening = 380.0 * math.exp(-((hour - 19.0) ** 2) / 8.0)
|
||||
weekend = 1.12 if dt.weekday() >= 5 else 1.0
|
||||
return max(0.0, (base + morning + evening) * weekend)
|
||||
@@ -0,0 +1,66 @@
|
||||
from influxdb_client import Point, WritePrecision
|
||||
|
||||
DT_H = 5.0 / 60.0
|
||||
|
||||
|
||||
def _f(config, key, default):
|
||||
try:
|
||||
return float(config.get(key, default) or default)
|
||||
except Exception:
|
||||
return float(default)
|
||||
|
||||
|
||||
def battery_soc_points(data_obj, forecast_var, pv_dict, load_dict, grid_dict):
|
||||
config = data_obj["config"]
|
||||
aid = str(config["anlagen_id"])
|
||||
cap_kwh = _f(config, "batt_capacity_kwh", 0.0)
|
||||
if cap_kwh <= 0 or not grid_dict:
|
||||
return []
|
||||
|
||||
min_soc = _f(config, "batt_min_soc", _f(config, "batt_min_soc_percent", 0.0))
|
||||
max_soc = _f(config, "batt_max_soc", _f(config, "batt_max_soc_percent", 100.0))
|
||||
charge_eff = max(0.01, min(1.0, _f(config, "batt_charge_efficiency", 0.95)))
|
||||
discharge_eff = max(0.01, min(1.0, _f(config, "batt_discharge_efficiency", 0.95)))
|
||||
start_soc_value = data_obj.get("current_soc_perc")
|
||||
if start_soc_value is None or start_soc_value == "":
|
||||
start_soc_value = config.get("batt_soc_percent", 50.0)
|
||||
start_soc = float(start_soc_value)
|
||||
start_soc = max(min_soc, min(max_soc, start_soc))
|
||||
|
||||
min_kwh = cap_kwh * min_soc / 100.0
|
||||
max_kwh = cap_kwh * max_soc / 100.0
|
||||
soc_kwh = max(min_kwh, min(max_kwh, cap_kwh * start_soc / 100.0))
|
||||
|
||||
planned_battery = (
|
||||
data_obj.get("battery_plans", {})
|
||||
.get(int(forecast_var), {})
|
||||
.get("battery", {})
|
||||
)
|
||||
points = []
|
||||
for t in data_obj["df_fut"].index:
|
||||
if t not in grid_dict:
|
||||
continue
|
||||
if t in planned_battery:
|
||||
battery_target_w = float(planned_battery[t]) # positiv = laden
|
||||
else:
|
||||
residual_w = float(load_dict.get(t, 0.0)) - float(pv_dict.get(t, 0.0))
|
||||
grid_w = float(grid_dict.get(t, 0.0))
|
||||
battery_target_w = grid_w - residual_w
|
||||
|
||||
if battery_target_w > 0:
|
||||
soc_kwh += battery_target_w * DT_H / 1000.0 * charge_eff
|
||||
elif battery_target_w < 0:
|
||||
soc_kwh += battery_target_w * DT_H / 1000.0 / discharge_eff
|
||||
|
||||
soc_kwh = max(min_kwh, min(max_kwh, soc_kwh))
|
||||
soc_percent = max(0.0, min(100.0, soc_kwh / cap_kwh * 100.0))
|
||||
|
||||
points.append(
|
||||
Point("forecast_diagnostics")
|
||||
.tag("anlagen_id", aid)
|
||||
.tag("data_type", "battery_soc_simulation")
|
||||
.tag("forecast_var", f"prog_var_{forecast_var}")
|
||||
.field("soc_percent", float(soc_percent))
|
||||
.time(t.to_pydatetime() if hasattr(t, "to_pydatetime") else t, WritePrecision.S)
|
||||
)
|
||||
return points
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Measured telemetry integrity and repeat-profile helpers.
|
||||
No IO, no fabricated measurements and no fixed household-load fallback.
|
||||
Forecast freshness is checked against raw telemetry, never against filled features.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import datetime
|
||||
import math
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
MEASURED_COLUMNS = ('PV', 'Hausverbrauch', 'Netzleistung', 'SOC')
|
||||
|
||||
class TelemetryUnavailable(ValueError):
|
||||
"""No publishable forecast can be derived from the supplied observations."""
|
||||
|
||||
|
||||
def sanitize_measured_frame(frame):
|
||||
out = frame.copy()
|
||||
for column in MEASURED_COLUMNS:
|
||||
if column not in out:
|
||||
continue
|
||||
raw = out[column]
|
||||
numeric = pd.to_numeric(raw, errors='coerce').astype(float)
|
||||
bad = ~np.isfinite(numeric) | raw.map(lambda x: isinstance(x, (bool, np.bool_)))
|
||||
if column in ('PV', 'Hausverbrauch', 'SOC'):
|
||||
bad |= numeric < 0
|
||||
if column == 'SOC':
|
||||
bad |= numeric > 100
|
||||
out[column] = numeric.mask(bad)
|
||||
return out
|
||||
|
||||
|
||||
def _naive_utc(value):
|
||||
stamp = pd.Timestamp(value)
|
||||
if stamp.tzinfo is not None:
|
||||
stamp = stamp.tz_convert('UTC').tz_localize(None)
|
||||
return stamp
|
||||
|
||||
|
||||
def require_recent_telemetry(data_obj, fields=('PV','Hausverbrauch'), max_age_minutes=30.0):
|
||||
"""Refuse missing/stale inputs without changing real zeros or raw samples."""
|
||||
if isinstance(max_age_minutes, bool) or not math.isfinite(max_age_minutes) or max_age_minutes <= 0:
|
||||
raise ValueError('Positive telemetry age limit required')
|
||||
now = _naive_utc(data_obj['now'])
|
||||
raw = data_obj.get('df_recent_raw')
|
||||
raw = sanitize_measured_frame(raw) if raw is not None else pd.DataFrame()
|
||||
if not raw.empty:
|
||||
raw.index = pd.DatetimeIndex([_naive_utc(t) for t in raw.index])
|
||||
raw = raw.loc[raw.index < now].sort_index()
|
||||
report = {}
|
||||
errors = []
|
||||
for field in fields:
|
||||
if field not in MEASURED_COLUMNS:
|
||||
raise ValueError('Unknown telemetry target')
|
||||
values = raw[field].dropna() if field in raw else pd.Series(dtype=float)
|
||||
if values.empty:
|
||||
errors.append(field + ': keine gemessenen Werte')
|
||||
continue
|
||||
stamp = values.index[-1]
|
||||
age = (now-stamp).total_seconds()/60.0
|
||||
report[field] = {'lastObservedInterval': stamp.isoformat()+'Z', 'ageMinutes': age,
|
||||
'observedIntervals': int(len(values)), 'lastValue': float(values.iloc[-1])}
|
||||
if age > max_age_minutes:
|
||||
errors.append(field + ': Messdaten veraltet (' + format(age,'.1f') + ' min)')
|
||||
if errors:
|
||||
raise TelemetryUnavailable('; '.join(errors) + '. Keine neuen Prognosen/Fahrplaene veroeffentlicht.')
|
||||
return report
|
||||
|
||||
|
||||
def _finite_nonnegative(value):
|
||||
if isinstance(value, (bool, np.bool_)):
|
||||
return None
|
||||
try:
|
||||
value = float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return value if math.isfinite(value) and value >= 0.0 else None
|
||||
|
||||
|
||||
def profile_source_value(history, at, column, predictions=None):
|
||||
"""Repeat yesterday; prefer same weekday if yesterday is missing, then older days.
|
||||
Explicitly generated first-day values may be repeated on the second forecast day.
|
||||
Missing historical values are not zeros. No future measurement is read.
|
||||
"""
|
||||
if column not in ('PV','Hausverbrauch'):
|
||||
raise ValueError('Unsupported repeat-profile target')
|
||||
predictions = {} if predictions is None else predictions
|
||||
at = pd.Timestamp(at)
|
||||
for day in (1, 7, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14):
|
||||
source = at - datetime.timedelta(days=day)
|
||||
if source in predictions:
|
||||
value = _finite_nonnegative(predictions[source])
|
||||
if value is not None:
|
||||
return value
|
||||
if history is not None and column in history and source in history.index:
|
||||
value = _finite_nonnegative(history.at[source,column])
|
||||
if value is not None:
|
||||
return value
|
||||
raise TelemetryUnavailable(column + ': kein gemessener Tagesprofilwert fuer ' + str(at))
|
||||
|
||||
|
||||
def repeat_daily_profile(history, future_index, column):
|
||||
result = {}
|
||||
for at in future_index:
|
||||
result[at] = profile_source_value(history, at, column, result)
|
||||
return result
|
||||
@@ -0,0 +1,133 @@
|
||||
import unittest
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from methods.battery_optimizer import DT_H, optimize_battery_plan
|
||||
|
||||
|
||||
class BatteryCostOptimizerTest(unittest.TestCase):
|
||||
def plan(self, load, pv, import_prices=None, export_prices=None, **overrides):
|
||||
index = pd.date_range("2026-09-29T00:00:00", periods=len(load), freq="5min")
|
||||
config = {
|
||||
"batt_capacity_kwh": 2.0,
|
||||
"batt_power_kw": 1.0,
|
||||
"batt_min_soc": 0.0,
|
||||
"batt_max_soc": 100.0,
|
||||
"batt_charge_efficiency": 1.0,
|
||||
"batt_discharge_efficiency": 1.0,
|
||||
"batt_soc_percent": 0.0,
|
||||
"batt_grid_charging_enabled": False,
|
||||
"batt_degradation_chf_kwh": 0.0,
|
||||
"tarif_bezug": "dynamic",
|
||||
"tarif_einspeisung": "dynamic",
|
||||
"tarif_peak_fest": 0.0,
|
||||
}
|
||||
config.update(overrides)
|
||||
data = {
|
||||
"config": config,
|
||||
"df_fut": pd.DataFrame({
|
||||
"import_price": import_prices or [0.30] * len(index),
|
||||
"export_price": export_prices or [0.10] * len(index),
|
||||
}, index=index),
|
||||
"current_soc_perc": config["batt_soc_percent"],
|
||||
"current_month_peak_kw": overrides.get("current_month_peak_kw", 0.0),
|
||||
}
|
||||
return index, optimize_battery_plan(data, dict(zip(index, pv)), dict(zip(index, load)))
|
||||
|
||||
def test_grid_charging_is_opt_in(self):
|
||||
index, plan = self.plan(
|
||||
[500.0] * 4,
|
||||
[0.0] * 4,
|
||||
import_prices=[0.05, 0.05, 0.50, 0.50],
|
||||
)
|
||||
self.assertTrue(all(plan["battery"][t] <= 1e-6 for t in index))
|
||||
|
||||
def test_cheap_grid_energy_is_shifted_to_expensive_period(self):
|
||||
index, plan = self.plan(
|
||||
[500.0] * 4,
|
||||
[0.0] * 4,
|
||||
import_prices=[0.05, 0.05, 0.50, 0.50],
|
||||
batt_grid_charging_enabled=True,
|
||||
)
|
||||
self.assertGreater(plan["battery"][index[0]], 0.0)
|
||||
self.assertLess(plan["battery"][index[-1]], 0.0)
|
||||
self.assertGreater(plan["grid"][index[0]], 500.0)
|
||||
self.assertAlmostEqual(plan["grid"][index[-1]], 0.0, places=5)
|
||||
|
||||
def test_high_feed_in_value_prefers_export(self):
|
||||
index, plan = self.plan(
|
||||
[0.0, 1000.0],
|
||||
[1000.0, 0.0],
|
||||
import_prices=[0.20, 0.20],
|
||||
export_prices=[0.60, 0.60],
|
||||
)
|
||||
self.assertAlmostEqual(plan["grid"][index[0]], -1000.0, places=5)
|
||||
self.assertAlmostEqual(plan["grid"][index[1]], 1000.0, places=5)
|
||||
|
||||
def test_peak_tariff_prevents_grid_charge_above_existing_peak(self):
|
||||
index, plan = self.plan(
|
||||
[1000.0] * 6,
|
||||
[0.0] * 6,
|
||||
import_prices=[0.05] * 3 + [0.50] * 3,
|
||||
batt_grid_charging_enabled=True,
|
||||
tarif_peak_fest=20.0,
|
||||
current_month_peak_kw=1.0,
|
||||
)
|
||||
self.assertLessEqual(plan["planned_peak_kw"], 1.0 + 1e-7)
|
||||
self.assertTrue(all(plan["grid"][t] <= 1000.0 + 1e-5 for t in index))
|
||||
|
||||
def test_nearly_empty_battery_is_not_discharged_further_at_low_value(self):
|
||||
index, plan = self.plan(
|
||||
[1000.0] * 4,
|
||||
[0.0] * 4,
|
||||
import_prices=[0.01] * 4,
|
||||
export_prices=[0.0] * 4,
|
||||
batt_soc_percent=5.0,
|
||||
batt_economic_reserve_soc_percent=10.0,
|
||||
batt_terminal_value_chf_kwh=0.0,
|
||||
)
|
||||
self.assertTrue(all(plan["battery"][timestamp] >= -1e-6 for timestamp in index))
|
||||
self.assertTrue(all(abs(plan["grid"][timestamp] - 1000.0) <= 1e-5 for timestamp in index))
|
||||
self.assertAlmostEqual(plan["economic_min_soc_percent"], 5.0)
|
||||
|
||||
def test_profitable_export_discharge_remains_allowed_above_reserve(self):
|
||||
index, plan = self.plan(
|
||||
[100.0, 100.0],
|
||||
[1000.0, 1000.0],
|
||||
import_prices=[0.20, 0.20],
|
||||
export_prices=[0.80, 0.80],
|
||||
batt_soc_percent=100.0,
|
||||
batt_economic_reserve_soc_percent=10.0,
|
||||
batt_degradation_chf_kwh=0.03,
|
||||
batt_terminal_value_chf_kwh=0.0,
|
||||
)
|
||||
self.assertTrue(any(plan["battery"][timestamp] < -1e-6 for timestamp in index))
|
||||
self.assertTrue(all(plan["grid"][timestamp] <= -899.0 for timestamp in index))
|
||||
self.assertAlmostEqual(plan["economic_min_soc_percent"], 10.0)
|
||||
|
||||
def test_power_and_soc_limits_hold(self):
|
||||
load = [0.0] * 12 + [1000.0] * 12
|
||||
pv = [1000.0] * 12 + [0.0] * 12
|
||||
index, plan = self.plan(
|
||||
load,
|
||||
pv,
|
||||
batt_capacity_kwh=1.0,
|
||||
batt_power_kw=0.5,
|
||||
batt_min_soc=20.0,
|
||||
batt_max_soc=80.0,
|
||||
batt_soc_percent=20.0,
|
||||
)
|
||||
soc = 0.2
|
||||
for timestamp in index:
|
||||
target = plan["battery"][timestamp]
|
||||
self.assertLessEqual(abs(target), 500.0 + 1e-6)
|
||||
if target >= 0.0:
|
||||
soc += target * DT_H / 1000.0
|
||||
else:
|
||||
soc += target * DT_H / 1000.0
|
||||
self.assertGreaterEqual(soc, 0.2 - 1e-8)
|
||||
self.assertLessEqual(soc, 0.8 + 1e-8)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,66 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from methods.var_2 import predict
|
||||
from methods.var_11 import predict as predict_repeat
|
||||
|
||||
|
||||
class LoadProfileForecastTest(unittest.TestCase):
|
||||
def test_profile_keeps_daily_shape_across_48_hours(self):
|
||||
history_index = pd.date_range("2026-09-21", periods=8 * 288, freq="5min")
|
||||
phase = 2 * np.pi * (
|
||||
history_index.hour.to_numpy() + history_index.minute.to_numpy() / 60.0
|
||||
) / 24.0
|
||||
history = pd.DataFrame(
|
||||
{"Hausverbrauch": 1800.0 + 900.0 * np.cos(phase - np.pi)},
|
||||
index=history_index,
|
||||
)
|
||||
future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=2 * 288, freq="5min")
|
||||
future_phase = 2 * np.pi * (
|
||||
future_index.hour.to_numpy() + future_index.minute.to_numpy() / 60.0
|
||||
) / 24.0
|
||||
future = pd.DataFrame(
|
||||
{
|
||||
"temp_c": 15.0,
|
||||
"cloud": 20.0,
|
||||
"hour_sin": np.sin(future_phase),
|
||||
"hour_cos": np.cos(future_phase),
|
||||
"weekday": future_index.weekday,
|
||||
"is_weekday": (future_index.weekday < 5).astype(int),
|
||||
},
|
||||
index=future_index,
|
||||
)
|
||||
data = {
|
||||
"config": {"anlagen_id": "test"},
|
||||
"df_hist": history,
|
||||
"df_load_training": history,
|
||||
"df_recent_raw": history.iloc[-2 * 288 :],
|
||||
"df_fut": future,
|
||||
}
|
||||
|
||||
with patch("methods.var_2.load_model", return_value=None):
|
||||
values = np.array(list(predict(data).values()))
|
||||
|
||||
self.assertGreater(float(values.max() - values.min()), 1200.0)
|
||||
np.testing.assert_allclose(values[:288], values[288:], rtol=0.0, atol=1e-6)
|
||||
|
||||
def test_repeat_profile_remains_available_on_second_day(self):
|
||||
history_index = pd.date_range("2026-09-29", periods=288, freq="5min")
|
||||
daily_values = np.arange(288, dtype=float) + 1000.0
|
||||
history = pd.DataFrame({"Hausverbrauch": daily_values}, index=history_index)
|
||||
future_index = pd.date_range(history_index[-1] + pd.Timedelta(minutes=5), periods=576, freq="5min")
|
||||
|
||||
values = np.array(list(predict_repeat({
|
||||
"df_hist": history,
|
||||
"df_fut": pd.DataFrame(index=future_index),
|
||||
}).values()))
|
||||
|
||||
np.testing.assert_allclose(values[:288], daily_values)
|
||||
np.testing.assert_allclose(values[288:], daily_values)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
import ast
|
||||
import contextlib
|
||||
import datetime
|
||||
import io
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import Mock
|
||||
from model_isolation import collect_predictions
|
||||
|
||||
|
||||
class ModelIsolationTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.index = [datetime.datetime(2026,10,2,6,0) + datetime.timedelta(minutes=5*i) for i in range(3)]
|
||||
self.data = {'df_fut': SimpleNamespace(index=self.index)}
|
||||
self.good = {t: 1000.0 for t in self.index}
|
||||
def model(self, values=None):
|
||||
return SimpleNamespace(predict=Mock(return_value=self.good if values is None else values))
|
||||
def call(self, models, enabled=lambda c,n:True):
|
||||
return collect_predictions(self.data, {}, models, enabled)
|
||||
def test_one_failed_model_does_not_remove_other_families(self):
|
||||
bad=self.model();bad.predict.side_effect=ValueError('private path not logged')
|
||||
forecasts,errors=self.call({1:self.model(),10:bad,21:self.model()})
|
||||
self.assertEqual(forecasts[1],self.good);self.assertEqual(forecasts[21],self.good)
|
||||
self.assertEqual(forecasts[10],{});self.assertEqual(errors[10]['errorType'],'ValueError')
|
||||
self.assertNotIn('private',str(errors))
|
||||
def test_missing_timestamp_does_not_get_filled(self):
|
||||
forecasts,errors=self.call({1:self.model(),10:self.model({self.index[0]:20.0})})
|
||||
self.assertFalse(forecasts[10]);self.assertIn(10,errors)
|
||||
def test_nonfinite_negative_and_boolean_rejected(self):
|
||||
for value in (float('nan'),float('inf'),-1.0,True,'2'):
|
||||
with self.subTest(value=value):
|
||||
result,errors=self.call({1:self.model(),10:self.model({t:value for t in self.index})})
|
||||
self.assertFalse(result[10]);self.assertIn(10,errors)
|
||||
def test_explicit_zero_forecast_is_not_imputed(self):
|
||||
result,errors=self.call({10:self.model({t:0.0 for t in self.index})})
|
||||
self.assertEqual(errors,{});self.assertEqual(sum(result[10].values()),0.)
|
||||
def test_all_requested_models_fail_closed(self):
|
||||
with self.assertRaisesRegex(ValueError,'All requested'):
|
||||
self.call({10:self.model({})})
|
||||
def test_disabled_models_not_called(self):
|
||||
model=self.model();result,errors=self.call({10:model},lambda c,n:False)
|
||||
model.predict.assert_not_called();self.assertEqual(result,{10:{}});self.assertEqual(errors,{})
|
||||
def test_input_time_order_preserved(self):
|
||||
result,_=self.call({1:self.model(dict(reversed(list(self.good.items()))))})
|
||||
self.assertEqual(list(result[1]),self.index)
|
||||
def test_run_forecast_publishes_valid_pairs_after_repeat_failure(self):
|
||||
root=Path(__file__).resolve().parents[1]
|
||||
tree=ast.parse((root/'main.py').read_text())
|
||||
fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run_forecast')
|
||||
modules={n:self.model() for n in (1,2,3,10,11,13,21,22,23)}
|
||||
modules[10].predict.side_effect=ValueError('profile gap')
|
||||
client=Mock();published=Mock()
|
||||
data={**self.data,'current_soc_perc':20.,'current_soc_source':'telemetry'}
|
||||
ns={'datetime':datetime,'LOCAL_TZ':datetime.timezone.utc,'traceback':Mock(),
|
||||
'get_configs':lambda:[{'anlagen_id':'test','batt_capacity_kwh':10.}],
|
||||
'InfluxDBClient':Mock(return_value=client),'SYNCHRONOUS':object(),
|
||||
'INFLUX_URL':'offline','INFLUX_TOKEN':'synthetic','INFLUX_ORG':'offline',
|
||||
'INFLUX_BUCKET':'offline','INFLUX_TIMEOUT_MS':1,
|
||||
'build_data_object':lambda *a,**k:data,'active':lambda c,n:True,
|
||||
'require_recent_telemetry':lambda *a,**k:{},'collect_predictions':collect_predictions,
|
||||
'_v4_publish_forecasts':published,'battery_soc_points':lambda *a:[],
|
||||
'_forecast_point':lambda *a:a,'_snapshot_point':lambda *a:a,
|
||||
'write_quality_metrics':Mock(),**{'v'+str(n):m for n,m in modules.items()}}
|
||||
exec(compile(ast.Module(body=[fn],type_ignores=[]),'source-run-forecast','exec'),ns)
|
||||
with contextlib.redirect_stdout(io.StringIO()):r=ns['run_forecast']()
|
||||
self.assertEqual(r['completed'],['test'])
|
||||
families=published.call_args.args[1]
|
||||
self.assertTrue(families[0][1] and families[0][2]);self.assertFalse(families[1][1])
|
||||
self.assertTrue(families[2][1] and families[2][2]);modules[13].predict.assert_not_called()
|
||||
client.write_api.return_value.write.assert_called_once()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Offline regressions against the exact reviewed orchestration source.
|
||||
Only selected function definitions are compiled; main is NOT imported, and there
|
||||
is no database, network, model-file, prediction-publication or device access.
|
||||
"""
|
||||
import ast
|
||||
import contextlib
|
||||
import datetime
|
||||
import io
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from telemetry_quality import (TelemetryUnavailable, require_recent_telemetry,
|
||||
sanitize_measured_frame, repeat_daily_profile, profile_source_value)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
AT = pd.Timestamp('2026-10-01T20:00:00')
|
||||
|
||||
|
||||
def functions(*names, **extra):
|
||||
tree = ast.parse((ROOT/'main.py').read_text())
|
||||
nodes = [n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name in names]
|
||||
if {n.name for n in nodes} != set(names):
|
||||
raise AssertionError('Reviewed function missing')
|
||||
ns = {'pd':pd,'np':np,'datetime':datetime,
|
||||
'sanitize_measured_frame':sanitize_measured_frame,
|
||||
'require_recent_telemetry':require_recent_telemetry}
|
||||
ns.update(extra)
|
||||
exec(compile(ast.Module(body=nodes,type_ignores=[]),str(ROOT/'main.py'),'exec'),ns)
|
||||
return ns
|
||||
|
||||
|
||||
def raw_frame(start=None, periods=288):
|
||||
index=pd.date_range(start if start is not None else AT-pd.Timedelta(days=1),periods=periods,freq='5min')
|
||||
return pd.DataFrame({'PV':200.,'Hausverbrauch':3200.,'SOC':25.,'Netzleistung':3000.},index=index)
|
||||
|
||||
|
||||
class MeasuredTelemetryTest(unittest.TestCase):
|
||||
def test_empty_history_is_not_zero_consumption(self):
|
||||
ns=functions('_fill_defaults','_add_time_features')
|
||||
frame=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=3,freq='5min')),True)
|
||||
self.assertTrue(frame[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all())
|
||||
self.assertEqual(frame['temp_c'].tolist(),[15.]*3)
|
||||
|
||||
def test_recorded_zero_is_preserved(self):
|
||||
frame=raw_frame(periods=3);frame[:]=0.
|
||||
ns=functions('_fill_defaults','_add_time_features')
|
||||
out=ns['_fill_defaults'](frame,True)
|
||||
self.assertEqual(out['Hausverbrauch'].tolist(),[0.]*3)
|
||||
self.assertEqual(out['PV'].tolist(),[0.]*3)
|
||||
|
||||
def test_internal_and_trailing_measurement_gaps_remain_missing(self):
|
||||
frame=raw_frame(periods=7)
|
||||
frame.loc[frame.index[[0,2,3,6]],['PV','Hausverbrauch','SOC','Netzleistung']]=np.nan
|
||||
ns=functions('_fill_defaults','_add_time_features')
|
||||
out=ns['_fill_defaults'](frame,True)
|
||||
self.assertEqual(int(out['Hausverbrauch'].isna().sum()),4)
|
||||
self.assertEqual(int(out['SOC'].isna().sum()),4)
|
||||
self.assertEqual(int(out['Netzleistung'].isna().sum()),4)
|
||||
|
||||
def test_bad_values_not_real_measurements(self):
|
||||
frame=pd.DataFrame({'Hausverbrauch':[np.inf,-1.,True,0.,250.], 'Netzleistung':[-100.,np.nan,0.,1.,2.], 'SOC':[101.,-1.,np.inf,0.,100.]})
|
||||
out=sanitize_measured_frame(frame)
|
||||
self.assertTrue(out['Hausverbrauch'].iloc[:3].isna().all())
|
||||
self.assertEqual(out['Hausverbrauch'].iloc[3],0.)
|
||||
self.assertEqual(out['Netzleistung'].iloc[0],-100.)
|
||||
self.assertTrue(out['SOC'].iloc[:3].isna().all())
|
||||
|
||||
def test_recent_recorded_zero_passes(self):
|
||||
frame=raw_frame();frame[['PV','Hausverbrauch']]=0.
|
||||
report=require_recent_telemetry({'now':AT,'df_recent_raw':frame})
|
||||
self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.)
|
||||
|
||||
def test_missing_raw_cannot_be_hidden_by_filled_feature_grid(self):
|
||||
with self.assertRaises(TelemetryUnavailable):
|
||||
require_recent_telemetry({'now':AT,'df_recent_raw':pd.DataFrame(),'df_hist':raw_frame()})
|
||||
|
||||
def test_stale_values_fail(self):
|
||||
with self.assertRaises(TelemetryUnavailable):
|
||||
require_recent_telemetry({'now':AT,'df_recent_raw':raw_frame(AT-pd.Timedelta(days=2))})
|
||||
|
||||
def test_freshness_is_checked_per_field(self):
|
||||
frame=raw_frame();frame.loc[frame.index[-12:],'Hausverbrauch']=np.nan
|
||||
with self.assertRaisesRegex(TelemetryUnavailable,'Hausverbrauch'):
|
||||
require_recent_telemetry({'now':AT,'df_recent_raw':frame})
|
||||
|
||||
def test_future_measurements_do_not_rescue_freshness(self):
|
||||
frame=raw_frame(AT,periods=3)
|
||||
with self.assertRaises(TelemetryUnavailable):
|
||||
require_recent_telemetry({'now':AT,'df_recent_raw':frame})
|
||||
|
||||
def test_aware_timestamps_normalized_to_utc(self):
|
||||
frame=raw_frame();frame.index=frame.index.tz_localize('UTC').tz_convert('Europe/Zurich')
|
||||
report=require_recent_telemetry({'now':AT.tz_localize('UTC'),'df_recent_raw':frame})
|
||||
self.assertEqual(report['Hausverbrauch']['ageMinutes'],5.)
|
||||
|
||||
def test_soc_staleness_cannot_be_hidden(self):
|
||||
frame=raw_frame();frame.loc[frame.index[-24:],'SOC']=np.nan
|
||||
with self.assertRaisesRegex(TelemetryUnavailable,'SOC'):
|
||||
require_recent_telemetry({'now':AT,'df_recent_raw':frame},['SOC'])
|
||||
|
||||
def test_only_exogenous_future_defaults_are_filled(self):
|
||||
ns=functions('_fill_defaults','_add_time_features')
|
||||
out=ns['_fill_defaults'](pd.DataFrame(index=pd.date_range(AT,periods=6,freq='5min')),False)
|
||||
self.assertTrue(out[['PV','Hausverbrauch','SOC','Netzleistung']].isna().all().all())
|
||||
|
||||
def test_input_unchanged(self):
|
||||
frame=raw_frame();before=frame.copy(deep=True)
|
||||
sanitize_measured_frame(frame);require_recent_telemetry({'now':AT,'df_recent_raw':frame})
|
||||
pd.testing.assert_frame_equal(frame,before)
|
||||
|
||||
|
||||
class HistoryOrchestrationTest(unittest.TestCase):
|
||||
def data(self,telemetry):
|
||||
class FrozenDateTime(datetime.datetime):
|
||||
@classmethod
|
||||
def utcnow(cls):return AT.to_pydatetime()
|
||||
import types
|
||||
dates=types.SimpleNamespace(datetime=FrozenDateTime,timedelta=datetime.timedelta)
|
||||
weather=pd.DataFrame({'temp_c':15.,'cloud':20.},index=pd.date_range(AT-pd.Timedelta(days=14),AT+pd.Timedelta(days=2),freq='5min'))
|
||||
fetch=Mock(return_value=(telemetry,weather,pd.DataFrame()))
|
||||
ns=functions('build_data_object','_fill_defaults','_add_time_features','_longest_consistent_segment',datetime=dates,fetch_influx_frames=fetch,FORECAST_HORIZON_HOURS=48)
|
||||
return ns['build_data_object']({'anlagen_id':'offline'},False)
|
||||
|
||||
def test_weather_tail_does_not_erase_historical_load(self):
|
||||
telemetry=raw_frame(AT-pd.Timedelta(days=2),periods=288)
|
||||
data=self.data(telemetry)
|
||||
self.assertEqual(data['df_hist']['Hausverbrauch'].count(),len(telemetry))
|
||||
self.assertEqual(data['df_recent_raw']['Hausverbrauch'].count(),len(telemetry))
|
||||
self.assertEqual(data['df_recent_raw'].index.max(),telemetry.index.max())
|
||||
self.assertTrue(data['df_hist']['Hausverbrauch'].iloc[-24:].isna().all())
|
||||
with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data)
|
||||
|
||||
def test_no_measurements_preserves_all_missing(self):
|
||||
data=self.data(pd.DataFrame())
|
||||
self.assertTrue(data['df_hist']['Hausverbrauch'].isna().all())
|
||||
with self.assertRaises(TelemetryUnavailable):require_recent_telemetry(data)
|
||||
|
||||
def test_reconnected_short_tail_does_not_delete_past_profile(self):
|
||||
old=raw_frame(AT-pd.Timedelta(days=2),periods=288)
|
||||
new=raw_frame(AT-pd.Timedelta(minutes=10),periods=2)
|
||||
data=self.data(pd.concat([old,new]))
|
||||
self.assertEqual(data['df_hist']['Hausverbrauch'].count(),290)
|
||||
self.assertEqual(require_recent_telemetry(data)['Hausverbrauch']['ageMinutes'],5.)
|
||||
|
||||
def test_stale_forecast_never_predicts_or_publishes(self):
|
||||
import types
|
||||
models={n:types.SimpleNamespace(predict=Mock()) for n in (1,2,3,10,11,13,21,22,23)}
|
||||
client=Mock();write=Mock();client.write_api.return_value=write
|
||||
publish=Mock();trace=Mock()
|
||||
ns=functions('run_forecast',get_configs=lambda:[{'anlagen_id':'offline','batt_capacity_kwh':10}],
|
||||
InfluxDBClient=Mock(return_value=client),SYNCHRONOUS=object(),
|
||||
INFLUX_URL='offline',INFLUX_TOKEN='synthetic',INFLUX_ORG='offline',INFLUX_BUCKET='offline',INFLUX_TIMEOUT_MS=1,
|
||||
build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},active=lambda *args:True,
|
||||
_v4_publish_forecasts=publish,traceback=trace,**{'v'+str(k):v for k,v in models.items()})
|
||||
with contextlib.redirect_stdout(io.StringIO()),self.assertRaises(RuntimeError):ns['run_forecast']()
|
||||
for model in models.values():model.predict.assert_not_called()
|
||||
publish.assert_not_called();write.write.assert_not_called();client.close.assert_called_once()
|
||||
|
||||
def test_stale_training_does_not_overwrite_model(self):
|
||||
import types
|
||||
train=Mock();mod=types.SimpleNamespace(train=train)
|
||||
ns=functions('run_training',get_configs=lambda:[{'anlagen_id':'offline'}],
|
||||
MODEL_MODULES={2:mod},QUALITY_TARGETS={2:'Hausverbrauch'},active=lambda *a:True,
|
||||
build_data_object=lambda *a,**k:{'now':AT,'df_recent_raw':pd.DataFrame()},traceback=Mock())
|
||||
with contextlib.redirect_stdout(io.StringIO()):ns['run_training']()
|
||||
train.assert_not_called()
|
||||
|
||||
def test_queries_exclude_forecasts_and_align_interval_starts(self):
|
||||
source=(ROOT/'main.py').read_text();ns=functions('fetch_influx_frames',
|
||||
HISTORY_START='1970-01-01T00:00:00Z',FORECAST_HORIZON_HOURS=48,INFLUX_BUCKET='offline',
|
||||
_query_df=Mock(return_value=pd.DataFrame()),_pivot_frame=lambda df,fields:df,
|
||||
_tariff_frame=lambda df,cfg:df,_time_literal=lambda t:t.isoformat())
|
||||
ns['fetch_influx_frames']({'anlagen_id':'offline'},False)
|
||||
query=ns['_query_df'].call_args_list[0].args[0]
|
||||
self.assertIn('timeSrc: "_start"',query)
|
||||
self.assertIn('!= "forecast_snapshot"',query)
|
||||
self.assertIn('!= "forecast"',query)
|
||||
|
||||
|
||||
class RepeatProfileIntegrityTest(unittest.TestCase):
|
||||
def test_pv_repeats_on_second_day(self):
|
||||
history=raw_frame();history['PV']=np.maximum(0.,np.sin(np.arange(288)*2*np.pi/288))*12000.
|
||||
idx=pd.date_range(AT,periods=576,freq='5min')
|
||||
actual=np.array(list(repeat_daily_profile(history,idx,'PV').values()))
|
||||
np.testing.assert_allclose(actual[:288],history['PV'])
|
||||
np.testing.assert_allclose(actual[288:],history['PV'])
|
||||
|
||||
def test_missing_yesterday_uses_older_finite_day(self):
|
||||
t=AT
|
||||
history=pd.DataFrame({'Hausverbrauch':[3500.,np.nan]},index=[t-pd.Timedelta(days=7),t-pd.Timedelta(days=1)])
|
||||
self.assertEqual(profile_source_value(history,t,'Hausverbrauch'),3500.)
|
||||
|
||||
def test_missing_profile_is_not_zero(self):
|
||||
history=pd.DataFrame({'Hausverbrauch':[np.nan]},index=[AT-pd.Timedelta(days=1)])
|
||||
with self.assertRaises(TelemetryUnavailable):repeat_daily_profile(history,[AT],'Hausverbrauch')
|
||||
|
||||
def test_recorded_profile_zero_is_valid(self):
|
||||
history=pd.DataFrame({'PV':[0.]},index=[AT-pd.Timedelta(days=1)])
|
||||
self.assertEqual(profile_source_value(history,AT,'PV'),0.)
|
||||
|
||||
def test_negative_profile_cannot_be_silently_clamped(self):
|
||||
history=pd.DataFrame({'Hausverbrauch':[-100.]},index=[AT-pd.Timedelta(days=1)])
|
||||
with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch')
|
||||
|
||||
def test_future_value_is_never_used_as_history(self):
|
||||
history=pd.DataFrame({'Hausverbrauch':[3300.]},index=[AT+pd.Timedelta(days=1)])
|
||||
with self.assertRaises(TelemetryUnavailable):profile_source_value(history,AT,'Hausverbrauch')
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a",
|
||||
"libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9",
|
||||
"libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7",
|
||||
"tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6",
|
||||
"tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2",
|
||||
"tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c",
|
||||
"tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
// OFFLINE ONLY: no Symcon kernel, HTTP requests, timers or connected devices.
|
||||
if (PHP_SAPI !== 'cli' || function_exists('IPS_GetVariable')) {
|
||||
fwrite(STDERR, "This is an isolated CLI test, not a Symcon runtime script.\n");
|
||||
exit(2);
|
||||
}
|
||||
set_error_handler(static function (int $severity, string $message, string $file, int $line): bool {
|
||||
throw new ErrorException($message, 0, $severity, $file, $line);
|
||||
});
|
||||
try {
|
||||
$manifest = json_decode(file_get_contents(__DIR__ . '/SOURCE_MANIFEST.json'), true, 512, JSON_THROW_ON_ERROR);
|
||||
foreach ($manifest as $relative => $expected) {
|
||||
$path = __DIR__ . '/' . $relative;
|
||||
if (!is_file($path) || hash_file('sha256', $path) !== $expected) {
|
||||
throw new RuntimeException('Source manifest mismatch: ' . $relative);
|
||||
}
|
||||
$command = escapeshellarg(PHP_BINARY) . ' -l ' . escapeshellarg($path);
|
||||
$lines = [];
|
||||
exec($command, $lines, $code);
|
||||
if ($code !== 0) {
|
||||
throw new RuntimeException('PHP syntax failure: ' . $relative);
|
||||
}
|
||||
}
|
||||
require __DIR__ . '/libs/NetzfahrplanV4Bezugszaehler.php';
|
||||
require __DIR__ . '/libs/NetzfahrplanV4Betriebsdaten.php';
|
||||
require __DIR__ . '/libs/ManagerNetzfahrplanV4Trait.php';
|
||||
require __DIR__ . '/tests/fixtures/NativeV4Scenarios.php';
|
||||
require __DIR__ . '/tests/fixtures/V4BezugszaehlerScenarios.php';
|
||||
$native = \Belevo\EnelixEMS\Tests\NativeV4Scenarios::run();
|
||||
$meters = \Belevo\EnelixEMS\Tests\V4BezugszaehlerScenarios::run();
|
||||
if (count($native) !== 24 || count($meters) !== 20) {
|
||||
throw new RuntimeException('Expected 24 native and 20 meter scenarios.');
|
||||
}
|
||||
foreach (array_merge($native, $meters) as $name) {
|
||||
echo 'PASS ', $name, "\n";
|
||||
}
|
||||
echo 'PHP ', PHP_VERSION, ': 44 offline conversion scenarios passed.', "\n";
|
||||
echo "NOT a full EMS suite and NOT an IP-Symcon runtime/physical test.\n";
|
||||
} catch (Throwable $error) {
|
||||
fwrite(STDERR, 'FAIL: ' . $error->getMessage() . "\n");
|
||||
exit(1);
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS;
|
||||
|
||||
use RuntimeException;
|
||||
use Throwable;
|
||||
|
||||
/** Shadow telemetry only. Does not read/apply a V4 schedule or write actuators. */
|
||||
trait ManagerNetzfahrplanV4Trait
|
||||
{
|
||||
private function registriereNetzfahrplanV4(): void
|
||||
{
|
||||
$this->RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false);
|
||||
$this->RegisterPropertyBoolean('NetzfahrplanV4NetzladenErlaubt', false);
|
||||
$this->RegisterPropertyString('NetzfahrplanV4BatterieOptionen', '{}');
|
||||
$this->RegisterPropertyInteger('NetzfahrplanV4MessnachweisVariableID', 0);
|
||||
$this->RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]');
|
||||
$this->RegisterAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}');
|
||||
$this->RegisterTimer('NetzfahrplanV4Senden', 0,
|
||||
"IPS_RequestAction(\$_IPS['TARGET'], 'NetzfahrplanV4Senden', true);");
|
||||
}
|
||||
|
||||
public function GetNetzfahrplanV4Diagnose(): string
|
||||
{
|
||||
try {
|
||||
$manager = [];
|
||||
foreach (['NetzleistungVariableID', 'MesswertMaxAlter', 'VerbraucherTimeout',
|
||||
'Lastspitzenmodus'] as $key) {
|
||||
$manager[$key] = $this->ReadPropertyInteger($key);
|
||||
}
|
||||
foreach (['Netzleistungsfaktor', 'Lastspitzengrenze', 'Einspeisegrenze'] as $key) {
|
||||
$manager[$key] = $this->ReadPropertyFloat($key);
|
||||
}
|
||||
$manager['V4ControlPlanSource'] = $this->leseJsonAttribut('Netzfahrplan')['sourceModel'] ?? 'legacy-unspecified';
|
||||
$manager['EinspeisebegrenzungAktiv'] = $this->ReadPropertyBoolean('EinspeisebegrenzungAktiv');
|
||||
$manager['NetzladenErlaubt'] = $this->ReadPropertyBoolean('NetzfahrplanV4NetzladenErlaubt');
|
||||
$manager['Monatsgrenzen'] = json_decode($this->ReadPropertyString('Monatsgrenzen'), true, 512, JSON_THROW_ON_ERROR);
|
||||
$manager['BatterieOptionen'] = json_decode($this->ReadPropertyString('NetzfahrplanV4BatterieOptionen'), true, 512, JSON_THROW_ON_ERROR);
|
||||
$manager['V4BezugszaehlerQuellen'] = json_decode(
|
||||
$this->ReadPropertyString('NetzfahrplanV4BezugszaehlerQuellen'), true, 512, JSON_THROW_ON_ERROR
|
||||
);
|
||||
$assets = json_decode($this->ReadPropertyString('AnlagenBatterien'), true, 512, JSON_THROW_ON_ERROR);
|
||||
$cache = $this->leseJsonAttribut('VerbraucherCache');
|
||||
$controllers = [];
|
||||
foreach ($this->aktiveVerbraucherIDs() as $id) {
|
||||
if (!IPS_InstanceExists($id)
|
||||
|| IPS_GetInstance($id)['ModuleInfo']['ModuleID'] !== '{437FB683-517F-4FEC-8CCB-FE6B0A62B69E}') {
|
||||
continue;
|
||||
}
|
||||
$c = ['InstanzID' => $id];
|
||||
foreach (['LadezustandVariableID', 'MaxLadeleistungVariableID', 'MaxEntladeleistungVariableID',
|
||||
'IstleistungVariableID', 'MindestLadezustand', 'ReserveLadezustand', 'LadezustandHysterese', 'MesswertMaxAlter', 'Batteriemanagement'] as $key) {
|
||||
$c[$key] = IPS_GetProperty($id, $key);
|
||||
}
|
||||
$entry = $cache[(string) $id] ?? [];
|
||||
$c['EmpfangenAm'] = $entry['EmpfangenAm'] ?? 0;
|
||||
$c['Verfuegbar'] = $entry['Daten']['Verfuegbar'] ?? false;
|
||||
foreach (($entry['Daten']['Zustand'] ?? []) as $state) {
|
||||
if (($state['Kennung'] ?? '') === 'HystereseAktiv') {
|
||||
$c['HystereseAktiv'] = $state['Wert'];
|
||||
}
|
||||
}
|
||||
$controllers[] = $c;
|
||||
}
|
||||
$evidence = [];
|
||||
$evidenceID = $this->ReadPropertyInteger('NetzfahrplanV4MessnachweisVariableID');
|
||||
if ($evidenceID > 0) {
|
||||
if (!IPS_VariableExists($evidenceID) || IPS_GetVariable($evidenceID)['VariableType'] !== 3) {
|
||||
throw new RuntimeException('Der konfigurierte Messnachweis ist keine JSON-Stringvariable.');
|
||||
}
|
||||
$evidence = json_decode(GetValue($evidenceID), true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!is_array($evidence)) {
|
||||
throw new RuntimeException('Ungueltiger Messnachweis.');
|
||||
}
|
||||
}
|
||||
$read = static function (int $id): array {
|
||||
if (!IPS_VariableExists($id)) {
|
||||
throw new RuntimeException('Messvariable ' . $id . ' fehlt.');
|
||||
}
|
||||
$before = IPS_GetVariable($id);
|
||||
$value = GetValue($id);
|
||||
$after = IPS_GetVariable($id);
|
||||
if ($before['VariableUpdated'] !== $after['VariableUpdated']) {
|
||||
throw new RuntimeException('Messwert hat sich beim Lesen geaendert; naechsten Durchlauf abwarten.');
|
||||
}
|
||||
$object = IPS_GetObject($id);
|
||||
return ['value' => $value, 'updated' => (int) $after['VariableUpdated'],
|
||||
'ident' => $object['ObjectIdent'], 'parentID' => (int) $object['ParentID']];
|
||||
};
|
||||
return json_encode(NetzfahrplanV4Betriebsdaten::erstellen($manager, $assets, $controllers,
|
||||
$read, $evidence, time()), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES);
|
||||
} catch (Throwable $error) {
|
||||
return json_encode(['status' => 'invalid_inputs', 'reason' => substr($error->getMessage(), 0, 300)], JSON_THROW_ON_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
public function GetNetzfahrplanV4Sendestatus(): string
|
||||
{
|
||||
return $this->ReadAttributeString('NetzfahrplanV4Sendestatus');
|
||||
}
|
||||
|
||||
private function sendeNetzfahrplanV4(): void
|
||||
{
|
||||
if (!$this->ReadPropertyBoolean('NetzfahrplanV4SchattenAktiv')) {
|
||||
$this->WriteAttributeString('NetzfahrplanV4Sendestatus', '{"status":"disabled"}');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (!$this->berechtigungLizenziert(Lizenzpruefung::NETZFAHRPLAN)) {
|
||||
throw new RuntimeException('Netzfahrplanberechtigung fehlt.');
|
||||
}
|
||||
$snapshot = $this->GetNetzfahrplanV4Diagnose();
|
||||
$data = json_decode($snapshot, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!isset($data['operation'])) {
|
||||
throw new RuntimeException($data['reason'] ?? 'Betriebsdaten unvollstaendig.');
|
||||
}
|
||||
// Decoding as objects preserves empty JSON dictionaries in the API contract.
|
||||
$payload = json_decode($snapshot, false, 512, JSON_THROW_ON_ERROR);
|
||||
if (!isset($payload->operation)) {
|
||||
throw new RuntimeException('Keine konsistenten Betriebsdaten.');
|
||||
}
|
||||
$id = $this->ReadAttributeString('LizenzInstallationID');
|
||||
$token = $this->ReadAttributeString('PrognoseInstallationsToken');
|
||||
if (!preg_match('/^[0-9a-f-]{36}$/i', $id) || $token === '') {
|
||||
throw new RuntimeException('Installations-ID oder bestehender Geraetezugang fehlt.');
|
||||
}
|
||||
$url = 'https://license.enelix.ch/api/v1/installations/' . rawurlencode($id) . '/prognosis/planner-v4/operation';
|
||||
$handle = curl_init($url);
|
||||
if ($handle === false) {
|
||||
throw new RuntimeException('V4-Verbindung konnte nicht vorbereitet werden.');
|
||||
}
|
||||
try {
|
||||
curl_setopt_array($handle, [CURLOPT_POST => true, CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_FOLLOWLOCATION => false, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_TIMEOUT => 5,
|
||||
CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2,
|
||||
CURLOPT_POSTFIELDS => json_encode($payload->operation, JSON_THROW_ON_ERROR),
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json', 'Accept: application/json', 'Authorization: Bearer ' . $token]]);
|
||||
$answer = curl_exec($handle);
|
||||
$status = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE);
|
||||
if ($answer === false || $status !== 200) {
|
||||
throw new RuntimeException('V4-Schattenanbindung HTTP ' . $status . '; bestehende Regelung unveraendert.');
|
||||
}
|
||||
if (strlen($answer) > 65536) {
|
||||
throw new RuntimeException('V4-Antwort ist zu gross.');
|
||||
}
|
||||
$ack = json_decode($answer, true, 512, JSON_THROW_ON_ERROR);
|
||||
if (!in_array($ack['status'] ?? '', ['stored', 'duplicate', 'archived_older'], true)) {
|
||||
throw new RuntimeException('Unerwartete V4-Annahmebestaetigung.');
|
||||
}
|
||||
} finally {
|
||||
curl_close($handle);
|
||||
}
|
||||
$this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([
|
||||
'status' => 'sent_shadow', 'capturedAt' => $payload->operation->observedAt,
|
||||
'acceptance' => $ack['status'], 'warnings' => $data['warnings'], 'liveEnabled' => false], JSON_THROW_ON_ERROR));
|
||||
} catch (Throwable $error) {
|
||||
// Only this sender's diagnosis changes; never SetStatus, live plan or actuator commands.
|
||||
$this->WriteAttributeString('NetzfahrplanV4Sendestatus', json_encode([
|
||||
'status' => 'error', 'reason' => substr($error->getMessage(), 0, 300), 'liveEnabled' => false], JSON_THROW_ON_ERROR));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS;
|
||||
|
||||
use DateTimeImmutable;
|
||||
use DateTimeZone;
|
||||
use InvalidArgumentException;
|
||||
|
||||
require_once __DIR__ . '/NetzfahrplanV4Bezugszaehler.php';
|
||||
|
||||
/** Read-only conversion. Meter evidence is never derived from a configured limit. */
|
||||
final class NetzfahrplanV4Betriebsdaten
|
||||
{
|
||||
private static function number($value, string $label, float $min = -1.0e9, float $max = 1.0e9): float
|
||||
{
|
||||
if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value)
|
||||
|| $value < $min || $value > $max) {
|
||||
throw new InvalidArgumentException($label . ': ungueltiger Zahlenwert.');
|
||||
}
|
||||
return (float) $value;
|
||||
}
|
||||
|
||||
private static function timestamp($value): int
|
||||
{
|
||||
if (!is_string($value) || !preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) {
|
||||
throw new InvalidArgumentException('Messnachweis benoetigt einen Zeitpunkt mit Zeitzone.');
|
||||
}
|
||||
return (new DateTimeImmutable($value))->getTimestamp();
|
||||
}
|
||||
|
||||
/** $read returns ['value' => int|float, 'updated' => int], without changing a device. */
|
||||
private static function measurement(callable $read, int $id, int $now, int $age, string $label): array
|
||||
{
|
||||
if ($id <= 0) {
|
||||
throw new InvalidArgumentException($label . ': Messquelle fehlt.');
|
||||
}
|
||||
$m = $read($id);
|
||||
if (!isset($m['updated']) || !is_int($m['updated']) || $m['updated'] > $now
|
||||
|| $now - $m['updated'] > $age) {
|
||||
throw new InvalidArgumentException($label . ': Messwert fehlt, ist veraltet oder liegt in der Zukunft.');
|
||||
}
|
||||
return ['value' => self::number($m['value'] ?? null, $label), 'updated' => $m['updated']];
|
||||
}
|
||||
|
||||
/** No inference of paid peak or elapsed energy from power, reserve or forecast. */
|
||||
private static function evidence(array $input, int $now, string $meterID, array &$warnings): array
|
||||
{
|
||||
$peaks = [];
|
||||
$past = null;
|
||||
if ($input === []) {
|
||||
$warnings[] = 'Autoritativer Monatspeak und laufende Viertelstundenenergie fehlen.';
|
||||
return [(object) [], null];
|
||||
}
|
||||
if ($meterID === '' || ($input['version'] ?? null) !== 1 || ($input['meterId'] ?? null) !== $meterID) {
|
||||
throw new InvalidArgumentException('Messnachweis: falsche Version oder Bezugszaehler-Zuordnung.');
|
||||
}
|
||||
$measured = self::timestamp($input['measuredAt'] ?? null);
|
||||
if ($measured > $now || $now - $measured > 120) {
|
||||
throw new InvalidArgumentException('Messnachweis ist nicht aktuell.');
|
||||
}
|
||||
$month = (new DateTimeImmutable('@' . $now))->setTimezone(new DateTimeZone('Europe/Zurich'))->format('Y-m');
|
||||
foreach (($input['measuredPeaks'] ?? []) as $key => $p) {
|
||||
if (!is_string($key) || !preg_match('/^\d{4}-(0[1-9]|1[0-2])$/', $key) || $key > $month
|
||||
|| !in_array($p['source'] ?? '', ['meter_month_register', 'verified_month_history', 'verified_new_month'], true)) {
|
||||
throw new InvalidArgumentException('Messnachweis: Monatspeak ist kein gueltiger Messnachweis.');
|
||||
}
|
||||
$peaks[$key] = ['kw' => self::number($p['kw'] ?? null, 'Monatspeak', 0.0), 'source' => $p['source']];
|
||||
}
|
||||
if (!array_key_exists($month, $peaks)) {
|
||||
$warnings[] = 'Vollstaendiger Monatspeak fuer ' . $month . ' fehlt; Managergrenze ist kein Ersatz.';
|
||||
}
|
||||
$q = intdiv($now, 900) * 900;
|
||||
if (isset($input['quarterPast'])) {
|
||||
$p = $input['quarterPast'];
|
||||
// Stale quarter evidence remains absent, not extrapolated to the decision time.
|
||||
if ($measured === $now && self::timestamp($p['start'] ?? null) === $q
|
||||
&& ($p['measuredSeconds'] ?? null) === $now - $q) {
|
||||
$past = ['start' => gmdate('c', $q),
|
||||
'measuredSeconds' => $now - $q,
|
||||
'importKwh' => self::number($p['importKwh'] ?? null, 'Viertelstundenenergie', 0.0)];
|
||||
} else {
|
||||
$warnings[] = 'Viertelstundenenergie passt nicht exakt zum Entscheidungszeitpunkt; nicht verwendet.';
|
||||
}
|
||||
}
|
||||
if ($now !== $q && $past === null) {
|
||||
$warnings[] = 'Bisherige Energie der laufenden Viertelstunde fehlt; keine Hochrechnung als Messung.';
|
||||
}
|
||||
return [(object) $peaks, $past];
|
||||
}
|
||||
|
||||
/** Inputs are explicit non-secret configuration/cache fields, not full instance settings. */
|
||||
public static function erstellen(array $manager, array $assets, array $controllers, callable $read, array $evidence, int $now): array
|
||||
{
|
||||
$warnings = [];
|
||||
$meter = NetzfahrplanV4Bezugszaehler::lesen(
|
||||
$manager['V4BezugszaehlerQuellen'] ?? [], $read, $now
|
||||
);
|
||||
$grid = self::measurement($read, (int) ($manager['NetzleistungVariableID'] ?? 0), $now,
|
||||
(int) ($manager['MesswertMaxAlter'] ?? 60), 'Netzleistung');
|
||||
$gridW = $grid['value'] * self::number($manager['Netzleistungsfaktor'] ?? 1.0, 'Netzleistungsfaktor');
|
||||
$mode = $manager['Lastspitzenmodus'] ?? 0;
|
||||
$import = null;
|
||||
$months = [];
|
||||
if ($mode === 1) {
|
||||
$import = self::number($manager['Lastspitzengrenze'] ?? null, 'Managergrenze', 0.0);
|
||||
} elseif ($mode === 2) {
|
||||
foreach (($manager['Monatsgrenzen'] ?? []) as $row) {
|
||||
$m = $row['MonatIndex'] ?? null;
|
||||
if (!is_int($m) || $m < 1 || $m > 12 || isset($months[(string) $m])) {
|
||||
throw new InvalidArgumentException('Monatsgrenzen fehlen oder sind doppelt.');
|
||||
}
|
||||
$months[(string) $m] = self::number($row['Grenze_W'] ?? null, 'Monatsgrenze', 0.0);
|
||||
}
|
||||
if (count($months) !== 12) {
|
||||
throw new InvalidArgumentException('Alle zwoelf Monatsgrenzen werden benoetigt.');
|
||||
}
|
||||
} elseif ($mode !== 0) {
|
||||
throw new InvalidArgumentException('Unbekannter Lastspitzenmodus.');
|
||||
}
|
||||
$export = !empty($manager['EinspeisebegrenzungAktiv'])
|
||||
? self::number($manager['Einspeisegrenze'] ?? null, 'Einspeisegrenze', 0.0) : null;
|
||||
$result = [];
|
||||
$seen = [];
|
||||
$used = [];
|
||||
foreach ($assets as $asset) {
|
||||
$id = $asset['ID'] ?? '';
|
||||
if (!is_string($id) || !preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/', $id) || isset($seen[$id])) {
|
||||
throw new InvalidArgumentException('Batterie-ID fehlt oder ist doppelt.');
|
||||
}
|
||||
$seen[$id] = true;
|
||||
$socID = (int) ($asset['SOCVariableID'] ?? 0);
|
||||
$matches = array_values(array_filter($controllers, static function (array $c) use ($socID): bool {
|
||||
return $socID > 0 && ($c['LadezustandVariableID'] ?? 0) === $socID;
|
||||
}));
|
||||
if (count($matches) !== 1) {
|
||||
throw new InvalidArgumentException($id . ': keine eindeutige aktive Batterieinstanz zur SOC-Quelle.');
|
||||
}
|
||||
$c = $matches[0];
|
||||
$controllerID = $c['InstanzID'];
|
||||
if (isset($used[$controllerID])) {
|
||||
throw new InvalidArgumentException('Eine Batterieinstanz darf nicht doppelt bilanziert werden.');
|
||||
}
|
||||
$used[$controllerID] = true;
|
||||
if (($asset['LeistungVariableID'] ?? 0) !== ($c['IstleistungVariableID'] ?? 0)) {
|
||||
throw new InvalidArgumentException($id . ': Topologie und Batterieinstanz verwenden verschiedene Leistungsmessungen.');
|
||||
}
|
||||
$opts = $manager['BatterieOptionen'][$id] ?? [];
|
||||
$nominal = self::number($asset['Nennkapazitaet_kWh'] ?? null, 'Nennkapazitaet', 0.001);
|
||||
$usable = self::number($asset['Nutzkapazitaet_kWh'] ?? null, 'Nutzkapazitaet', 0.001, $nominal);
|
||||
if (abs($nominal - $usable) > 1.0e-6 && !array_key_exists('SOCKapazitaet_kWh', $opts)) {
|
||||
throw new InvalidArgumentException($id . ': SOC-Kapazitaetsbasis bei verschiedener Nenn-/Nutzkapazitaet bestaetigen.');
|
||||
}
|
||||
$capacity = self::number($opts['SOCKapazitaet_kWh'] ?? $nominal, 'SOC-Kapazitaet', 0.001, $nominal);
|
||||
$age = (int) ($c['MesswertMaxAlter'] ?? 30);
|
||||
$soc = self::measurement($read, $socID, $now, $age, $id . ' SOC');
|
||||
$charge = self::measurement($read, (int) $c['MaxLadeleistungVariableID'], $now, $age, $id . ' max. Laden');
|
||||
$discharge = self::measurement($read, (int) $c['MaxEntladeleistungVariableID'], $now, $age, $id . ' max. Entladen');
|
||||
$min = max(self::number($c['MindestLadezustand'], 'BMS-Minimum', 0, 100),
|
||||
self::number($c['ReserveLadezustand'], 'Betriebsreserve', 0, 100));
|
||||
$max = self::number($opts['MaxSOC_Prozent'] ?? 100.0, 'Maximal-SOC', $min, 100);
|
||||
$socValue = self::number($soc['value'], 'SOC', 0, 100);
|
||||
$physicalMin = self::number($c['MindestLadezustand'], 'Technisches Minimum', 0, $min);
|
||||
if ($socValue < $physicalMin || $socValue > $max) {
|
||||
throw new InvalidArgumentException($id . ': SOC ausserhalb des Planungsbereichs; kein kuenstliches Anheben.');
|
||||
}
|
||||
if (($c['EmpfangenAm'] ?? 0) > $now || $now - ($c['EmpfangenAm'] ?? 0) > (int) ($manager['VerbraucherTimeout'] ?? 60)) {
|
||||
throw new InvalidArgumentException($id . ': Verbraucher-Rueckmeldung veraltet.');
|
||||
}
|
||||
$maxCharge = min(self::number($charge['value'], 'Ladeleistung', 0), 1000 * self::number($asset['MaxLadeleistung_kW'], 'Nennladeleistung', 0));
|
||||
$maxDischarge = min(self::number($discharge['value'], 'Entladeleistung', 0), 1000 * self::number($asset['MaxEntladeleistung_kW'], 'Nennentladeleistung', 0));
|
||||
if (($c['Verfuegbar'] ?? false) !== true || ($c['Batteriemanagement'] ?? 0) !== 2) {
|
||||
$maxCharge = $maxDischarge = 0.0;
|
||||
$warnings[] = $id . ': nicht fuer Managerregelung verfuegbar; keine Batterieleistung eingeplant.';
|
||||
}
|
||||
$rearm = min($max, $min + self::number($c['LadezustandHysterese'] ?? 0.0, 'Entladehysterese', 0, 100));
|
||||
$blocked = !empty($c['HystereseAktiv']) || $socValue <= $min;
|
||||
if ($blocked) {
|
||||
$warnings[] = $id . ': Entladesperre im Modell aktiv bis zur Wiederfreigabe nach vorheriger Ladung.';
|
||||
}
|
||||
if ($socValue < $min) {
|
||||
$warnings[] = $id . ': unter Betriebsreserve; realen SOC behalten und nur zulassige Wiederaufladung planen.';
|
||||
}
|
||||
$result[] = ['id' => $id, 'capacityKwh' => $capacity, 'socPercent' => $socValue,
|
||||
'minSocPercent' => $min, 'maxSocPercent' => $max, 'maxChargeW' => $maxCharge,
|
||||
'maxDischargeW' => $maxDischarge, 'measuredAt' => gmdate('c', min($soc['updated'], $charge['updated'], $discharge['updated'])),
|
||||
'gridCharging' => ($manager['NetzladenErlaubt'] ?? false) === true,
|
||||
'physicalMinSocPercent' => $physicalMin, 'recoveryAllowed' => true,
|
||||
'dischargeBlocked' => $blocked, 'rearmSocPercent' => $rearm];
|
||||
}
|
||||
if ($result === []) {
|
||||
throw new InvalidArgumentException('Keine eindeutige steuerbare Batterie konfiguriert.');
|
||||
}
|
||||
[$peaks, $past] = self::evidence($evidence, $now, $meter['meterId'], $warnings);
|
||||
$payload = ['version' => 1, 'observedAt' => gmdate('c', $now), 'gridW' => $gridW,
|
||||
'meteringBoundary' => 'common_pcc', 'batteries' => $result,
|
||||
'limits' => ['importW' => $import, 'exportW' => $export, 'managerMonthLimitsW' => (object) $months],
|
||||
'measuredPeaks' => $peaks, 'quarterPast' => $past];
|
||||
// A fresh acquisition sample is an operational estimate. It does NOT
|
||||
// claim a calibrated billing-period boundary or full historical coverage.
|
||||
$policy = [
|
||||
'adapterVersion' => 'v4-native-estimated-meter-1',
|
||||
'sourcePlan' => $manager['V4ControlPlanSource'] ?? 'legacy-unspecified',
|
||||
'mode' => $mode, 'importW' => $import, 'exportW' => $export,
|
||||
'months' => $months,
|
||||
'batteryPolicy' => array_map(static function (array $b): array {
|
||||
return array_intersect_key($b, array_flip(['id', 'capacityKwh', 'minSocPercent', 'maxSocPercent', 'physicalMinSocPercent', 'rearmSocPercent']));
|
||||
}, $result),
|
||||
];
|
||||
$payload['meterObservation'] = [
|
||||
'meterId' => $meter['meterId'], 'sampleAt' => gmdate('c', $now),
|
||||
'powerW' => $gridW, 'totalImportKwh' => $meter['totalKwh'],
|
||||
'controlPolicyId' => hash('sha256', json_encode($policy, JSON_THROW_ON_ERROR)),
|
||||
];
|
||||
$payload['eventId'] = 'symcon-operation-' . hash('sha256', json_encode($payload, JSON_THROW_ON_ERROR));
|
||||
return ['status' => $warnings === [] ? 'ready_shadow' : 'incomplete_shadow', 'warnings' => $warnings, 'bezugszaehler' => $meter, 'operation' => $payload];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS;
|
||||
|
||||
use InvalidArgumentException;
|
||||
|
||||
/** Read-only active-import counter sum, independent of the legacy energy counter.
|
||||
* Observation timestamps are NOT proof of a billing-quarter boundary or history.
|
||||
*/
|
||||
final class NetzfahrplanV4Bezugszaehler
|
||||
{
|
||||
public static function quellen(array $sources): array
|
||||
{
|
||||
if ($sources === [] || count($sources) > 8
|
||||
|| array_keys($sources) !== range(0, count($sources) - 1)) {
|
||||
throw new InvalidArgumentException('V4 benoetigt 1 bis 8 explizite Wirkenergie-Bezugsquellen.');
|
||||
}
|
||||
$result = [];
|
||||
$parent = null;
|
||||
foreach ($sources as $source) {
|
||||
if (!is_array($source) || count($source) !== 5
|
||||
|| ($source['Messgroesse'] ?? null) !== 'WirkenergieBezug') {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Messgroesse oder Quellenschema ungueltig.');
|
||||
}
|
||||
$id = $source['VariableID'] ?? null;
|
||||
$pid = $source['ElternID'] ?? null;
|
||||
$ident = $source['Ident'] ?? null;
|
||||
$factor = $source['FaktorZuKWh'] ?? null;
|
||||
if (!is_int($id) || $id <= 0 || !is_int($pid) || $pid <= 0
|
||||
|| !is_string($ident) || !preg_match('/^[A-Za-z][A-Za-z0-9_]{0,63}$/D', $ident)
|
||||
|| (!is_int($factor) && !is_float($factor)) || !is_finite((float) $factor)
|
||||
|| $factor <= 0 || $factor > 1.0e6 || isset($result[$id])) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: ID, Ident, Einheit oder doppelte Quelle ungueltig.');
|
||||
}
|
||||
if ($parent !== null && $parent !== $pid) {
|
||||
throw new InvalidArgumentException('V4 T1/T2 muessen zum selben physischen Bezugszaehler gehoeren.');
|
||||
}
|
||||
$parent = $pid;
|
||||
foreach ($result as $existing) {
|
||||
if ($existing['Ident'] === $ident) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Register doppelt angegeben.');
|
||||
}
|
||||
}
|
||||
$result[$id] = ['VariableID' => $id, 'ElternID' => $pid, 'Ident' => $ident,
|
||||
'FaktorZuKWh' => (float) $factor, 'Messgroesse' => 'WirkenergieBezug'];
|
||||
}
|
||||
ksort($result, SORT_NUMERIC);
|
||||
return array_values($result);
|
||||
}
|
||||
|
||||
public static function identitaet(array $sources): string
|
||||
{
|
||||
// Source/factor changes invalidate old measurement evidence; names do not matter.
|
||||
return 'symcon-active-import:' . hash('sha256', json_encode(self::quellen($sources),
|
||||
JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION));
|
||||
}
|
||||
|
||||
private static function probe(array $source, callable $read, int $now, int $maxAge): array
|
||||
{
|
||||
$m = $read($source['VariableID']);
|
||||
if (!is_array($m) || ($m['parentID'] ?? null) !== $source['ElternID']
|
||||
|| ($m['ident'] ?? null) !== $source['Ident']) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Variable passt nicht zum konfigurierten Register.');
|
||||
}
|
||||
$value = $m['value'] ?? null;
|
||||
$updated = $m['updated'] ?? null;
|
||||
if ((!is_int($value) && !is_float($value)) || !is_finite((float) $value) || $value < 0
|
||||
|| !is_int($updated) || $updated <= 0 || $updated > $now || $now - $updated > $maxAge) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Teilwert fehlt, ist veraltet oder ungueltig.');
|
||||
}
|
||||
$kwh = (float) $value * $source['FaktorZuKWh'];
|
||||
if (!is_finite($kwh) || $kwh > 1.0e12) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Energie ausserhalb des Messbereichs.');
|
||||
}
|
||||
return ['variableId' => $source['VariableID'], 'rawValue' => (float) $value,
|
||||
'totalKwh' => $kwh, 'updatedAt' => $updated];
|
||||
}
|
||||
|
||||
/** $read must return only value, updated, ident, parentID. Never uses an archive fallback. */
|
||||
public static function lesen(array $sources, callable $read, int $now, int $maxAge = 60, int $maxSkew = 2): array
|
||||
{
|
||||
if ($now <= 0 || $maxAge < 1 || $maxAge > 300 || $maxSkew < 0 || $maxSkew > 5) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: Zeitfenster ungueltig.');
|
||||
}
|
||||
$sources = self::quellen($sources);
|
||||
$samples = [];
|
||||
foreach ($sources as $source) {
|
||||
$samples[] = self::probe($source, $read, $now, $maxAge);
|
||||
}
|
||||
// Reread all channels to reject concurrent updates, including changes in the same second.
|
||||
foreach ($sources as $i => $source) {
|
||||
if ($samples[$i] !== self::probe($source, $read, $now, $maxAge)) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler wurde waehrend des Lesens aktualisiert.');
|
||||
}
|
||||
}
|
||||
$times = array_column($samples, 'updatedAt');
|
||||
if (max($times) - min($times) > $maxSkew) {
|
||||
throw new InvalidArgumentException('V4 Bezugszaehler: T1/T2-Zeitpunkte liegen zu weit auseinander.');
|
||||
}
|
||||
return ['meterId' => self::identitaet($sources), 'quantity' => 'active_import', 'unit' => 'kWh',
|
||||
'totalKwh' => array_sum(array_column($samples, 'totalKwh')),
|
||||
'observedAt' => gmdate('c', $now), 'sourceObservationFrom' => gmdate('c', min($times)),
|
||||
'sourceObservationUntil' => gmdate('c', max($times)), 'components' => $samples,
|
||||
'billingEvidence' => false, 'historyComplete' => false];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS\Tests;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
require_once __DIR__ . '/../libs/NetzfahrplanV4Betriebsdaten.php';
|
||||
require_once __DIR__ . '/fixtures/NativeV4Scenarios.php';
|
||||
|
||||
final class NetzfahrplanV4BetriebsdatenTest extends TestCase
|
||||
{
|
||||
public function testOfflineInputScenarios(): void
|
||||
{
|
||||
self::assertCount(24, NativeV4Scenarios::run());
|
||||
}
|
||||
|
||||
public function testSenderCannotChangeLivePlanOrActuators(): void
|
||||
{
|
||||
$source = file_get_contents(__DIR__ . '/../libs/ManagerNetzfahrplanV4Trait.php');
|
||||
foreach (['sendeManagerdaten(', 'aktualisiereNetzfahrplan(', '->regeln(', '->SetStatus(', "WriteAttributeString('Netzfahrplan',"] as $forbidden) {
|
||||
self::assertStringNotContainsString($forbidden, $source);
|
||||
}
|
||||
self::assertStringContainsString("RegisterPropertyBoolean('NetzfahrplanV4SchattenAktiv', false)", $source);
|
||||
self::assertStringContainsString('/prognosis/planner-v4/operation', $source);
|
||||
self::assertStringNotContainsString('/prognosis/schedule', $source);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS\Tests;
|
||||
|
||||
use PHPUnit\Framework\TestCase;
|
||||
|
||||
require_once __DIR__ . '/../libs/NetzfahrplanV4Bezugszaehler.php';
|
||||
require_once __DIR__ . '/fixtures/V4BezugszaehlerScenarios.php';
|
||||
|
||||
final class NetzfahrplanV4BezugszaehlerTest extends TestCase
|
||||
{
|
||||
public function testReadOnlyCounterSources(): void
|
||||
{
|
||||
self::assertCount(20, V4BezugszaehlerScenarios::run());
|
||||
}
|
||||
|
||||
public function testNoLegacyMeterOrActuatorFallback(): void
|
||||
{
|
||||
$source = file_get_contents(__DIR__ . '/../libs/NetzfahrplanV4Bezugszaehler.php');
|
||||
foreach (['SetValue(', 'IPS_SetProperty(', 'RequestAction(', 'AC_Set', '53476'] as $forbidden) {
|
||||
self::assertStringNotContainsString($forbidden, $source);
|
||||
}
|
||||
$trait = file_get_contents(__DIR__ . '/../libs/ManagerNetzfahrplanV4Trait.php');
|
||||
self::assertStringNotContainsString("'NetzbezugEnergieVariableID'", $trait);
|
||||
self::assertStringContainsString("RegisterPropertyString('NetzfahrplanV4BezugszaehlerQuellen', '[]')", $trait);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS\Tests;
|
||||
|
||||
use Belevo\EnelixEMS\NetzfahrplanV4Betriebsdaten;
|
||||
use RuntimeException;
|
||||
use InvalidArgumentException;
|
||||
|
||||
/** Runs offline. No IPS API, network, physical devices or production settings. */
|
||||
final class NativeV4Scenarios
|
||||
{
|
||||
public static function run(): array
|
||||
{
|
||||
$now = strtotime('2026-10-01T12:05:00Z');
|
||||
$m = ['NetzleistungVariableID' => 1, 'NetzbezugEnergieVariableID' => 9,
|
||||
'Netzleistungsfaktor' => 1.0, 'MesswertMaxAlter' => 60, 'Lastspitzenmodus' => 1,
|
||||
'Lastspitzengrenze' => 25000.0, 'EinspeisebegrenzungAktiv' => true, 'Einspeisegrenze' => 25000.0];
|
||||
$m['V4BezugszaehlerQuellen'] = [
|
||||
['VariableID' => 6, 'ElternID' => 99, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'],
|
||||
['VariableID' => 7, 'ElternID' => 99, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'],
|
||||
];
|
||||
$a = [['ID' => 'ev', 'Nennkapazitaet_kWh' => 161.44, 'Nutzkapazitaet_kWh' => 161.44,
|
||||
'MaxLadeleistung_kW' => 39.0, 'MaxEntladeleistung_kW' => 30.0, 'SOCVariableID' => 2, 'LeistungVariableID' => 5]];
|
||||
$c = [['InstanzID' => 42, 'LadezustandVariableID' => 2, 'MaxLadeleistungVariableID' => 3,
|
||||
'MaxEntladeleistungVariableID' => 4, 'IstleistungVariableID' => 5,
|
||||
'ReserveLadezustand' => 15.0, 'MindestLadezustand' => 3.0, 'MesswertMaxAlter' => 30,
|
||||
'Verfuegbar' => true, 'Batteriemanagement' => 2, 'EmpfangenAm' => $now]];
|
||||
$read = static function (int $id) use ($now): array {
|
||||
return ['value' => [1 => -5332.0, 2 => 19.0, 3 => 39000.0, 4 => 35000.0, 6 => 3992.203, 7 => 0.0][$id],
|
||||
'updated' => $now, 'parentID' => 99, 'ident' => [6 => 'Energy_0', 7 => 'Energy_1'][$id] ?? 'other'];
|
||||
};
|
||||
$check = static function (bool $ok): void { if (!$ok) { throw new RuntimeException('Assertion failed'); } };
|
||||
$fail = static function (callable $fn): void {
|
||||
try { $fn(); } catch (InvalidArgumentException $e) { return; }
|
||||
throw new RuntimeException('Expected invalid-input rejection');
|
||||
};
|
||||
$cases = [];
|
||||
$build = static function ($mm = null, $aa = null, $cc = null, $r = null, $e = []) use ($m, $a, $c, $read, $now): array {
|
||||
return NetzfahrplanV4Betriebsdaten::erstellen($mm ?? $m, $aa ?? $a, $cc ?? $c, $r ?? $read, $e, $now);
|
||||
};
|
||||
$cases['native_mapping_and_asymmetric_limits'] = static function () use ($build, $check): void {
|
||||
$x = $build()['operation']; $b = $x['batteries'][0];
|
||||
$check($x['gridW'] === -5332.0 && $b['capacityKwh'] === 161.44 && $b['minSocPercent'] === 15.0
|
||||
&& $b['maxChargeW'] === 39000.0 && $b['maxDischargeW'] === 30000.0);
|
||||
};
|
||||
$cases['manager_cap_not_paid_peak'] = static function () use ($build, $check): void {
|
||||
$x = $build()['operation']; $check($x['limits']['importW'] === 25000.0 && (array) $x['measuredPeaks'] === [] && $x['quarterPast'] === null);
|
||||
};
|
||||
$cases['empty_maps_are_objects'] = static function () use ($build, $check): void {
|
||||
$x = json_decode(json_encode($build()['operation'])); $check(is_object($x->measuredPeaks) && is_object($x->limits->managerMonthLimitsW));
|
||||
};
|
||||
$cases['zero_limits_not_unlimited'] = static function () use ($build, $m, $check): void {
|
||||
$m['Einspeisegrenze'] = 0.0; $m['Lastspitzengrenze'] = 0.0; $x = $build($m)['operation'];
|
||||
$check($x['limits']['importW'] === 0.0 && $x['limits']['exportW'] === 0.0);
|
||||
};
|
||||
$cases['disabled_limits_are_null'] = static function () use ($build, $m, $check): void {
|
||||
$m['EinspeisebegrenzungAktiv'] = false; $m['Lastspitzenmodus'] = 0; $x = $build($m)['operation'];
|
||||
$check($x['limits']['importW'] === null && $x['limits']['exportW'] === null);
|
||||
};
|
||||
$cases['all_monthly_limits'] = static function () use ($build, $m, $check, $fail): void {
|
||||
$m['Lastspitzenmodus'] = 2; $m['Monatsgrenzen'] = [];
|
||||
for ($i = 1; $i <= 12; $i++) { $m['Monatsgrenzen'][] = ['MonatIndex' => $i, 'Grenze_W' => $i * 1000]; }
|
||||
$x = $build($m)['operation']; $check($x['limits']['managerMonthLimitsW']->{'10'} === 10000.0);
|
||||
array_pop($m['Monatsgrenzen']); $fail(static fn() => $build($m));
|
||||
};
|
||||
$cases['no_unconfirmed_soc_capacity'] = static function () use ($build, $a, $m, $fail, $check): void {
|
||||
$a[0]['Nutzkapazitaet_kWh'] = 140.; $fail(static fn() => $build(null, $a));
|
||||
$m['BatterieOptionen'] = ['ev' => ['SOCKapazitaet_kWh' => 161.44]];
|
||||
$check($build($m, $a)['operation']['batteries'][0]['capacityKwh'] === 161.44);
|
||||
};
|
||||
$cases['no_duplicate_controller'] = static function () use ($build, $a, $fail): void {
|
||||
$a[] = array_replace($a[0], ['ID' => 'other']); $fail(static fn() => $build(null, $a));
|
||||
};
|
||||
$cases['no_ambiguous_mapping'] = static function () use ($build, $c, $fail): void {
|
||||
$c[] = array_replace($c[0], ['InstanzID' => 43]); $fail(static fn() => $build(null, null, $c));
|
||||
};
|
||||
$cases['mismatched_power_source'] = static function () use ($build, $c, $fail): void {
|
||||
$c[0]['IstleistungVariableID'] = 100; $fail(static fn() => $build(null, null, $c));
|
||||
};
|
||||
$cases['stale_measurements_fail'] = static function () use ($build, $read, $now, $fail): void {
|
||||
$r = static function ($id) use ($read, $now) { $v = $read($id); $v['updated'] = $now - 61; return $v; };
|
||||
$fail(static fn() => $build(null, null, null, $r));
|
||||
};
|
||||
$cases['stale_consumer_cache_fails'] = static function () use ($build, $c, $now, $fail): void {
|
||||
$c[0]['EmpfangenAm'] = $now - 61; $fail(static fn() => $build(null, null, $c));
|
||||
};
|
||||
$cases['soc_below_reserve_is_not_fabricated'] = static function () use ($build, $read, $check): void {
|
||||
$r = static function ($id) use ($read) { $v = $read($id); if ($id === 2) { $v['value'] = 5.; } return $v; };
|
||||
$b = $build(null, null, null, $r)['operation']['batteries'][0];
|
||||
$check($b['socPercent'] === 5.0 && $b['minSocPercent'] === 15.0 && $b['physicalMinSocPercent'] === 3.0 && $b['recoveryAllowed'] === true && $b['dischargeBlocked'] === true);
|
||||
};
|
||||
$cases['unavailable_asset_no_power'] = static function () use ($build, $c, $check): void {
|
||||
$c[0]['Verfuegbar'] = false; $x = $build(null, null, $c)['operation']['batteries'][0];
|
||||
$check($x['maxChargeW'] === 0.0 && $x['maxDischargeW'] === 0.0);
|
||||
};
|
||||
$cases['hysteresis_not_silently_ignored'] = static function () use ($build, $c, $check): void {
|
||||
$c[0]['HystereseAktiv'] = true; $x = $build(null, null, $c);
|
||||
$check($x['operation']['batteries'][0]['dischargeBlocked'] === true && $x['operation']['batteries'][0]['maxDischargeW'] === 30000.0 && count($x['warnings']) > 1);
|
||||
};
|
||||
$cases['grid_charge_explicit_opt_in'] = static function () use ($build, $m, $check): void {
|
||||
$check($build()['operation']['batteries'][0]['gridCharging'] === false);
|
||||
$m['NetzladenErlaubt'] = true; $check($build($m)['operation']['batteries'][0]['gridCharging'] === true);
|
||||
};
|
||||
$ev = ['version' => 1, 'meterId' => \Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler::identitaet($m['V4BezugszaehlerQuellen']), 'measuredAt' => gmdate('c', $now),
|
||||
'measuredPeaks' => ['2026-10' => ['kw' => 18.4, 'source' => 'meter_month_register']],
|
||||
'quarterPast' => ['start' => '2026-10-01T12:00:00Z', 'measuredSeconds' => 300, 'importKwh' => 0.5]];
|
||||
$cases['actual_meter_evidence_used'] = static function () use ($build, $ev, $check): void {
|
||||
$x = $build(null, null, null, null, $ev);
|
||||
$check($x['status'] === 'ready_shadow' && $x['operation']['measuredPeaks']->{'2026-10'}['kw'] === 18.4 && $x['operation']['quarterPast']['importKwh'] === 0.5);
|
||||
};
|
||||
$cases['stale_quarter_not_extrapolated'] = static function () use ($build, $ev, $now, $check): void {
|
||||
$ev['measuredAt'] = gmdate('c', $now - 1); $x = $build(null, null, null, null, $ev);
|
||||
$check($x['operation']['quarterPast'] === null && count($x['warnings']) > 0);
|
||||
};
|
||||
$cases['wrong_meter_evidence_fails'] = static function () use ($build, $ev, $fail): void {
|
||||
$ev['meterId'] = 'symcon:100'; $fail(static fn() => $build(null, null, null, null, $ev));
|
||||
};
|
||||
$cases['planned_peak_not_evidence'] = static function () use ($build, $ev, $fail): void {
|
||||
$ev['measuredPeaks']['2026-10']['source'] = 'manager_cap'; $fail(static fn() => $build(null, null, null, null, $ev));
|
||||
};
|
||||
$cases['deterministic_retry_event_id'] = static function () use ($build, $check): void {
|
||||
$check($build()['operation']['eventId'] === $build()['operation']['eventId']);
|
||||
};
|
||||
$cases['legacy_meter_identity_not_accepted'] = static function () use ($build, $ev, $fail): void {
|
||||
$ev['meterId'] = 'symcon:9'; $fail(static fn() => $build(null, null, null, null, $ev));
|
||||
};
|
||||
$cases['missing_v4_sources_never_use_legacy'] = static function () use ($build, $m, $fail): void {
|
||||
unset($m['V4BezugszaehlerQuellen']); $fail(static fn() => $build($m));
|
||||
};
|
||||
$cases['counter_snapshot_is_not_billing_evidence'] = static function () use ($build, $check): void {
|
||||
$v = $build(); $check($v['bezugszaehler']['totalKwh'] === 3992.203
|
||||
&& !$v['bezugszaehler']['billingEvidence'] && (array) $v['operation']['measuredPeaks'] === []);
|
||||
};
|
||||
$passed = [];
|
||||
foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; }
|
||||
return $passed;
|
||||
}
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Belevo\EnelixEMS\Tests;
|
||||
|
||||
use Belevo\EnelixEMS\NetzfahrplanV4Bezugszaehler as Meter;
|
||||
use InvalidArgumentException;
|
||||
use RuntimeException;
|
||||
|
||||
/** Offline only: injected readings, no Symcon, archive, network or actuator access. */
|
||||
final class V4BezugszaehlerScenarios
|
||||
{
|
||||
public static function run(): array
|
||||
{
|
||||
$now = 1790874000;
|
||||
$sources = [
|
||||
['VariableID' => 59607, 'ElternID' => 11490, 'Ident' => 'Energy_0', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'],
|
||||
['VariableID' => 26620, 'ElternID' => 11490, 'Ident' => 'Energy_1', 'FaktorZuKWh' => 1.0, 'Messgroesse' => 'WirkenergieBezug'],
|
||||
];
|
||||
$samples = [
|
||||
59607 => ['value' => 3992.203, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_0'],
|
||||
26620 => ['value' => 0.0, 'updated' => $now, 'parentID' => 11490, 'ident' => 'Energy_1'],
|
||||
];
|
||||
$check = static function (bool $value): void { if (!$value) { throw new RuntimeException('Assertion failed'); } };
|
||||
$reject = static function (callable $fn): void {
|
||||
try { $fn(); } catch (InvalidArgumentException $e) { return; }
|
||||
throw new RuntimeException('Expected invalid source/sample rejection');
|
||||
};
|
||||
$snapshot = static function ($ss = null, $mm = null) use ($sources, $samples, $now): array {
|
||||
$mm = $mm ?? $samples;
|
||||
return Meter::lesen($ss ?? $sources, static fn(int $id) => $mm[$id] ?? null, $now);
|
||||
};
|
||||
$cases = [];
|
||||
$cases['zero_t2_is_valid_but_not_history'] = static function () use ($snapshot, $check): void {
|
||||
$v = $snapshot();
|
||||
$check(abs($v['totalKwh'] - 3992.203) < 1e-9 && !$v['historyComplete'] && !$v['billingEvidence']);
|
||||
$check(!isset($v['measuredPeaks']) && !isset($v['quarterPast']));
|
||||
};
|
||||
$cases['both_tariffs_are_summed'] = static function () use ($snapshot, $samples, $check): void {
|
||||
$samples[26620]['value'] = 27.25;
|
||||
$check(abs($snapshot(null, $samples)['totalKwh'] - 4019.453) < 1e-9);
|
||||
};
|
||||
$cases['zero_total_not_replaced'] = static function () use ($snapshot, $samples, $check): void {
|
||||
$samples[59607]['value'] = 0;
|
||||
$check($snapshot(null, $samples)['totalKwh'] === 0.0);
|
||||
};
|
||||
$cases['source_order_does_not_change_identity'] = static function () use ($sources, $check): void {
|
||||
$check(Meter::identitaet($sources) === Meter::identitaet(array_reverse($sources)));
|
||||
};
|
||||
$cases['changed_factor_invalidates_identity'] = static function () use ($sources, $check): void {
|
||||
$old = Meter::identitaet($sources); $sources[0]['FaktorZuKWh'] = 0.001;
|
||||
$check($old !== Meter::identitaet($sources));
|
||||
};
|
||||
$cases['changed_variable_invalidates_identity'] = static function () use ($sources, $check): void {
|
||||
$old = Meter::identitaet($sources); $sources[0]['VariableID'] = 12345;
|
||||
$check($old !== Meter::identitaet($sources));
|
||||
};
|
||||
$cases['no_fallback_to_legacy_source'] = static function () use ($snapshot, $reject): void { $reject(static fn() => $snapshot([])); };
|
||||
$cases['duplicate_variable_rejected'] = static function () use ($sources, $snapshot, $reject): void {
|
||||
$sources[1] = $sources[0]; $reject(static fn() => $snapshot($sources));
|
||||
};
|
||||
$cases['different_meter_boundary_rejected'] = static function () use ($sources, $snapshot, $reject): void {
|
||||
$sources[1]['ElternID'] = 48065; $reject(static fn() => $snapshot($sources));
|
||||
};
|
||||
$cases['reactive_quantity_rejected'] = static function () use ($sources, $snapshot, $reject): void {
|
||||
$sources[0]['Messgroesse'] = 'Blindenergie'; $reject(static fn() => $snapshot($sources));
|
||||
};
|
||||
$cases['explicit_conversion_required'] = static function () use ($sources, $snapshot, $reject): void {
|
||||
unset($sources[0]['FaktorZuKWh']); $reject(static fn() => $snapshot($sources));
|
||||
};
|
||||
$cases['missing_tariff_not_zero'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
unset($samples[26620]); $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['stale_zero_tariff_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
$samples[26620]['updated'] -= 61; $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['future_reading_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
$samples[26620]['updated']++; $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['wrong_ident_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
$samples[59607]['ident'] = 'Energy_6'; $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['wrong_parent_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
$samples[59607]['parentID'] = 48065; $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['invalid_numbers_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
foreach ([false, '3992.203', NAN, INF, -1.0] as $bad) {
|
||||
$samples[59607]['value'] = $bad; $reject(static fn() => $snapshot(null, $samples));
|
||||
}
|
||||
};
|
||||
$cases['skewed_tariff_observations_rejected'] = static function () use ($snapshot, $samples, $reject): void {
|
||||
$samples[59607]['updated'] -= 3; $reject(static fn() => $snapshot(null, $samples));
|
||||
};
|
||||
$cases['concurrent_update_in_same_second_rejected'] = static function () use ($sources, $samples, $now, $reject): void {
|
||||
$calls = 0;
|
||||
$read = static function (int $id) use ($samples, &$calls): array {
|
||||
$v = $samples[$id]; if (++$calls > 2) { $v['value'] += 0.1; } return $v;
|
||||
};
|
||||
$reject(static fn() => Meter::lesen($sources, $read, $now));
|
||||
};
|
||||
$cases['observation_not_fake_exact_billing_time'] = static function () use ($snapshot, $samples, $now, $check): void {
|
||||
$samples[59607]['updated'] -= 1;
|
||||
$v = $snapshot(null, $samples);
|
||||
$check($v['sourceObservationFrom'] === gmdate('c', $now - 1));
|
||||
$check($v['sourceObservationUntil'] === gmdate('c', $now) && $v['observedAt'] === gmdate('c', $now));
|
||||
$check(!$v['billingEvidence']);
|
||||
};
|
||||
$passed = [];
|
||||
foreach ($cases as $name => $fn) { $fn(); $passed[] = $name; }
|
||||
return $passed;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Read-only probe executed INSIDE the existing forecast-engine container.
|
||||
|
||||
No get_configs() (it may migrate SQL), no training/prediction, no forecast writing,
|
||||
no manual /run_now request, and no user credentials in output. Reads one plant's
|
||||
stored raw 5-minute forecast values and the input frames used by the engine.
|
||||
"""
|
||||
import contextlib
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
class Discard(io.TextIOBase):
|
||||
def write(self,text):return len(text)
|
||||
|
||||
def clean_time(value):
|
||||
if hasattr(value,'to_pydatetime'):value=value.to_pydatetime()
|
||||
if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat()
|
||||
|
||||
def run():
|
||||
aid=str(UUID(os.environ['ENELIX_ACCEPTANCE_PLANT']))
|
||||
os.environ['FORECAST_INFLUX_TIMEOUT_MS']='20000'
|
||||
# These assignments affect only this diagnostic process, not the service.
|
||||
with contextlib.redirect_stdout(Discard()),contextlib.redirect_stderr(Discard()):
|
||||
import main
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
path=Path(main.SQLITE_DB_PATH)
|
||||
if not path.is_file():raise RuntimeError('Existing configuration database missing')
|
||||
con=sqlite3.connect('file:'+quote(str(path))+'?mode=ro',uri=True)
|
||||
con.row_factory=sqlite3.Row
|
||||
try:row=con.execute('SELECT * FROM anlagen_meta WHERE anlagen_id=?',(aid,)).fetchone()
|
||||
finally:con.close()
|
||||
if row is None:raise RuntimeError('Installation not found')
|
||||
cfg=dict(row)
|
||||
cfg['daecher']=json.loads(cfg.get('daecher') or '[]')
|
||||
for key,default in [('ac_leistung',10.),('batt_capacity_kwh',0.),('batt_power_kw',0.),('tarif_bezug_fest',.3),('tarif_einspeisung_fest',.1),('tarif_peak_fest',5.)]:
|
||||
value=cfg.get(key);cfg[key]=float(default if value is None or value=='' else value)
|
||||
data=main.build_data_object(cfg,training=False)
|
||||
frames={}
|
||||
for key in ('df_hist','df_recent_raw','df_load_training','df_fut'):
|
||||
frame=data.get(key)
|
||||
if frame is None:frames[key]={'available':False};continue
|
||||
item={'rows':len(frame),'from':clean_time(frame.index.min()) if len(frame) else None,'until':clean_time(frame.index.max()) if len(frame) else None,'columns':{}}
|
||||
for col in ('Hausverbrauch','PV','Netzleistung','SOC'):
|
||||
if col not in frame.columns:continue
|
||||
values=pd.to_numeric(frame[col],errors='coerce');valid=values[np.isfinite(values)]
|
||||
item['columns'][col]={'finite':len(valid),'zeros':int((valid==0).sum()),'median':float(valid.median()) if len(valid) else None,'maximum':float(valid.max()) if len(valid) else None,'lastFiniteAt':clean_time(valid.index[-1]) if len(valid) else None,'recentValues':[{'time':clean_time(t),'value':float(v)} for t,v in valid.tail(12).items()]}
|
||||
frames[key]=item
|
||||
start=datetime.now(timezone.utc).replace(second=0,microsecond=0)
|
||||
from datetime import timedelta
|
||||
end=start+timedelta(hours=48)
|
||||
fields=('prog_var_1','prog_var_2','prog_var_10','prog_var_11','prog_var_21','prog_var_22')
|
||||
field_filter=' or '.join('r["_field"] == '+json.dumps(f) for f in fields)
|
||||
query='''from(bucket: %s)
|
||||
|> range(start: %s, stop: %s)
|
||||
|> filter(fn: (r) => r["_measurement"] == "api_telemetry")
|
||||
|> filter(fn: (r) => r["anlagen_id"] == %s)
|
||||
|> filter(fn: (r) => r["data_type"] == "forecast")
|
||||
|> filter(fn: (r) => %s)
|
||||
|> keep(columns: ["_time", "_field", "_value"])
|
||||
''' % (json.dumps(main.INFLUX_BUCKET),start.isoformat(),end.isoformat(),json.dumps(aid),field_filter)
|
||||
client=main.InfluxDBClient(url=main.INFLUX_URL,token=main.INFLUX_TOKEN,org=main.INFLUX_ORG,timeout=20000)
|
||||
series={field:[] for field in fields}
|
||||
try:
|
||||
tables=client.query_api().query(org=main.INFLUX_ORG,query=query)
|
||||
for table in tables:
|
||||
for record in table.records:
|
||||
value=record.get_value();field=record.get_field()
|
||||
if field in series and isinstance(value,(int,float)) and math.isfinite(value):
|
||||
series[field].append({'time':clean_time(record.get_time()),'value':float(value)})
|
||||
finally:client.close()
|
||||
for field in series:series[field].sort(key=lambda p:p['time'])
|
||||
versions={}
|
||||
for relative in ('main.py','methods/var_1.py','methods/var_2.py','methods/var_10.py','methods/var_11.py','methods/var_21.py','methods/var_22.py'):
|
||||
source=Path('/app')/relative
|
||||
if source.is_file():versions[relative]=hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
summary={}
|
||||
for field,points in series.items():
|
||||
values=[p['value'] for p in points]
|
||||
summary[field]={'points':len(values),'allZero':bool(values) and max(abs(v) for v in values)==0,'maximumW':max(values) if values else None,'from':points[0]['time'] if points else None,'until':points[-1]['time'] if points else None}
|
||||
native=True
|
||||
for points in series.values():
|
||||
stamps=[datetime.fromisoformat(p['time']).timestamp() for p in points]
|
||||
if not stamps or any(t%300 for t in stamps) or any(b-a!=300 for a,b in zip(stamps,stamps[1:])):native=False
|
||||
return {'status':'read_only_acquired','installationId':aid,'observedAt':datetime.now(timezone.utc).isoformat(),'queryResolution':'raw_no_chart_resampling','nativeFiveMinuteForecast':native,'generationTimeVerified':False,'inputFrames':frames,'forecastSummary':summary,'forecastSeries':series,'runningSourceHashes':versions,'modelWrite':False,'liveControlChanged':False}
|
||||
|
||||
try:
|
||||
result=run()
|
||||
except Exception as error:
|
||||
# Never echo exception details containing SQL payloads, credentials or URLs.
|
||||
result={'status':'read_only_probe_failed','errorType':type(error).__name__,'liveControlChanged':False}
|
||||
print(json.dumps(result,allow_nan=False))
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"license/public/netplan-v4.js": [
|
||||
"e859e91a52dd1d6c96f411d16caed17375f6925bf867d09992ab32ce93bff5d7"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
# Integrated application data/model path - 2026-10-02
|
||||
|
||||
User request: finish application implementation, stop asking for repeated hardware
|
||||
confirmations and adding standalone diagnostic samplers. No actuator permission is
|
||||
implied by that development request. Confirmed EV allocation remains 161.44 kWh /
|
||||
39 kW each direction, with SDL reserve already excluded.
|
||||
|
||||
## Implemented in the existing application
|
||||
|
||||
- Native ManagerNetzfahrplanV4DatenTrait: 30-second acquisition of configured numeric
|
||||
sources; durable private app-owned outbox; original timestamps, no new Modbus
|
||||
polls; acknowledged per-plant batches with retries/backoff. Cursor advances only
|
||||
after exact dataset/acceptedThrough acknowledgement. Batches bounded below the
|
||||
existing portal's 1 MiB body limit. No credentials in data payload or reports.
|
||||
- Existing device proxy now forwards /planner-v4/measurements through the existing
|
||||
authentication, plant binding, licence check and separate V4 rate limiter.
|
||||
Device route cannot modify the dataset mapping or issue a trial grant.
|
||||
- Existing V4 database stores immutable versioned source mappings and observations.
|
||||
Same-time conflicts reject the whole batch; repeated identical data is idempotent.
|
||||
No writes to portal/users/legacy measurement databases.
|
||||
- Existing V4 worker aggregates physical load using original source times. Virtual
|
||||
EV/SDL filter outputs are not load inputs. Solar terminal variant counts signed
|
||||
terminal power once. Small unobserved parts remain quantified, never zero-filled.
|
||||
This configured formula remains an estimate, not independent AC/DC certification.
|
||||
- Application profile training actually runs daily/weekly. Models are immutable,
|
||||
bootstrap/holdout status explicit. Incumbent replacement requires a causal
|
||||
holdout when available. Existing data and model versions are not relabelled.
|
||||
- Three corrected load variants (daily robust profile, last-day profile,
|
||||
weekday/weekend profile) are paired with the existing PV families 3/13/23.
|
||||
They are tagged physical-profile-v1; they are NOT claimed to be the old trained
|
||||
load algorithms under a new data name. Automatic future family registry work
|
||||
and economic replay are still distinct from this load-profile implementation.
|
||||
- Selecting corrected_profile in the existing GUI/setting now actually changes
|
||||
the load entering the existing optimizer. Missing trained data cannot silently
|
||||
fall back to the contaminated legacy household series.
|
||||
- A current SDL request is an explicitly labelled PERSISTENCE SCENARIO for the
|
||||
shadow economic calculation, not a published future schedule. Missing/stale SDL
|
||||
is not zero. This is not a robust/full SDL production-dispatch policy.
|
||||
- GUI now exposes dataset/source selection, records, usable equivalent hours,
|
||||
model version, validation and real training cadence. Existing normal portal
|
||||
app.js/index files are not changed.
|
||||
|
||||
## Deployment package
|
||||
|
||||
Server: commissioning/deploy_application.py --plant <approved UUID> --apply
|
||||
Rebuild/test V4 with Python 3.11, portal routing tests, consistent SQLite backup,
|
||||
replace ONLY V4 and portal, configure the new per-plant dataset. No automatic
|
||||
forecast-source switch and no control grant. Atomic file-bind replacement requires
|
||||
portal recreation, not a simple restart. Additive DB rollback retains new data.
|
||||
Default invocation without --apply validates source manifests only.
|
||||
|
||||
Test host: /srv/agent/netplan-v4-application-build/install.php inside Symcon.
|
||||
Installs a DATA-ONLY patch of the currently installed passive manager (not the
|
||||
unreleased controller-trial candidate), reloads EMS, configures acquisition and
|
||||
imports up to 48h of the existing 23-source observer as a delivery backlog.
|
||||
Original observer and old raw journals remain intact. Existing separate observers
|
||||
are NOT automatically stopped by this initial installer; consolidation should
|
||||
follow confirmed native batch acceptance. No new root diagnostic category.
|
||||
An asynchronous module registration may require one repeat, handled explicitly.
|
||||
|
||||
Dataset lihrenmoos-physical-v1: minimum coverage 95%, longest unsupported portion
|
||||
10s, at least 24 equivalent usable hours, 28-day profile history. The 95% rule is
|
||||
an explicit modelling policy, not a statement that missing energy was measured.
|
||||
Insufficient data remains collecting; no promised time to good forecasts.
|
||||
All original acquisition thresholds remain unchanged.
|
||||
|
||||
## Tests and honest limits
|
||||
|
||||
Host tests use existing isolated QA dependencies; native tests mock IPS/HTTP and
|
||||
use temporary files. Real container and kernel tests occur on the user-run deploy
|
||||
commands, not during preparation. No services/modules/settings/devices were modified
|
||||
by this development session.
|
||||
|
||||
Still NOT a complete production commissioning: the corrected profile-to-local
|
||||
feedback/dispatch integration, full automatic cost replay, independent long-term
|
||||
outbox/server retention and staged live/failure acceptance remain open. The trial
|
||||
code from previous commits remains gated and is not included in the data-only
|
||||
native installation. No accountingEvidenceId/device watchdog proof is fabricated.
|
||||
This release is an integrated application build, NOT a claim that the full original
|
||||
multi-plant production scope is done. No main/beta release is authorized by tests alone.
|
||||
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"scope": "integrated_measurement_application",
|
||||
"installationId": "e3a08f9e-af12-4695-99bd-8b51c0520021",
|
||||
"sourceHashes": {
|
||||
".dockerignore": "fab6861d98f34e54646ae966b237e792fc0a0b4f95f1df3b62a1f062cbb8f790",
|
||||
"Dockerfile": "655c600a0364e91d47bfc80faaf27e26362bfc2683c8e57b653d913133865713",
|
||||
"acceptance/Dockerfile.php": "715a2d232d7f901bd6ca1f2e453b7b7f48fc1d7f49bff8944a7d277a8dc30062",
|
||||
"acceptance/check_forecast.py": "ccee58ec1078767a580f15f895506eceeb15e13b54e8eb9b4905cc03ba14ccd5",
|
||||
"acceptance/forecast-src/SOURCE_MANIFEST.json": "8103775396c58d82921e2e2a1513ee185ae2431200763717544ff9b1da181fe5",
|
||||
"acceptance/forecast-src/main.py": "4060564a4a33400ef6b8547633fc4c97caa3f674494d8cfc53a7aae0ed011b6c",
|
||||
"acceptance/forecast-src/methods/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"acceptance/forecast-src/methods/battery_optimizer.py": "27e7404d3bf4a2511e022232c2f6877adc0db014bdeb132a2a13cd4949d4d5e6",
|
||||
"acceptance/forecast-src/methods/common.py": "0fe5c9bc3fa8b6d40f0f9db36843623c6e469c150ed25899e3176355d5bb1db8",
|
||||
"acceptance/forecast-src/methods/var_1.py": "6a7fc3aaf904442aba44bd211d89e4ba54f10485f54441a1239de19e91fa5fe4",
|
||||
"acceptance/forecast-src/methods/var_10.py": "a3caf21387620687646775e6b0bf85c97af8bc6d0a48e3d779e9684d310333b7",
|
||||
"acceptance/forecast-src/methods/var_11.py": "1f783e57fee22761e8e5439caef48e275876a7ebf8380467d9b71ad2aa1b8edc",
|
||||
"acceptance/forecast-src/methods/var_13.py": "f12407cd056a1f28f47ab93b1262c62627c80d4765e75dd495e2c19e8f8e2ad9",
|
||||
"acceptance/forecast-src/methods/var_2.py": "5bdfd61bb108368890ff1b920f60499eabf6363c6380637100380eb0e28f0c6a",
|
||||
"acceptance/forecast-src/methods/var_21.py": "e2e3361d57fae8379dcce89cd98595a0d56d23decd1073d94474302b53ff8e15",
|
||||
"acceptance/forecast-src/methods/var_22.py": "cd51104bf98c686360c037cb74ff0a40bb748f24e52575eb3a3ef9932fac8cd6",
|
||||
"acceptance/forecast-src/methods/var_23.py": "d356d4078723a59e6cfad5ade631883cf22e7abd9caf38cec046e26b0580c678",
|
||||
"acceptance/forecast-src/methods/var_3.py": "87662e491ba33e170f3bcfdbd8dd2e54630073fcf4cb6e2e8ab768f9a6d95984",
|
||||
"acceptance/forecast-src/model_isolation.py": "db33ee8e9583cf006a5224a9f9efeed874ce04144d74f1b1bb25852c614468c5",
|
||||
"acceptance/forecast-src/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161",
|
||||
"acceptance/forecast-src/shared_utils.py": "271489e491305d97706e0a4b5c8745bcb01e19628a0cee71da15512ee2d85e57",
|
||||
"acceptance/forecast-src/soc_diagnostics.py": "06ce7c55aa69875df94471a9fe47ff3d95da372192b737b9a0ff6493503ac15a",
|
||||
"acceptance/forecast-src/telemetry_quality.py": "bb959d08d2d50d5597dc10b47a35d510e43ba8bfa755db236652071b57ad1810",
|
||||
"acceptance/forecast-src/tests/test_battery_optimizer.py": "5b8fa3185672530c072a8cfe506ac8d4878846efbb2ee818b93ee04d57ca7cc8",
|
||||
"acceptance/forecast-src/tests/test_load_forecast.py": "000903a3691297dd7cfc160b3702825a6f04d53ae9d745dc08f7bfadec06465c",
|
||||
"acceptance/forecast-src/tests/test_model_isolation.py": "4dadcc7541181a57badc337fa33100c0ba2b9fd20b7dd36b87326eeeb285cf30",
|
||||
"acceptance/forecast-src/tests/test_telemetry_integrity.py": "8e6a200d6a108d309b8c5ceba15fdb1653644789875648c4284b75170d56b5ce",
|
||||
"acceptance/php-src/SOURCE_MANIFEST.json": "3f8add37ac99ebbb0b3e77093ee586e5deedce14643190c673e7a3dd942d2a50",
|
||||
"acceptance/php-src/check.php": "92599dc10d0b8b0bb97cab3c8ae8fbefd084c8cd65992f0e848e8042385cf473",
|
||||
"acceptance/php-src/libs/ManagerNetzfahrplanV4Trait.php": "13d2867d2b4fe7f8846a08d9b4b81269320db6373d44c5f731a09c21c4912ab7",
|
||||
"acceptance/php-src/libs/NetzfahrplanV4Betriebsdaten.php": "6ff7d5710995778e7f941020a6f18555307ef51f16867f13efc204915e6dc9d9",
|
||||
"acceptance/php-src/libs/NetzfahrplanV4Bezugszaehler.php": "7aa01ce83a343eb767a889575fa04cece7f1c65cda347723e24dd68da40cea9a",
|
||||
"acceptance/php-src/tests/NetzfahrplanV4BetriebsdatenTest.php": "4a4f5af4cd86797fc40a4a36a7103a26fcf1e59ab381bafa8f67d35b34419dbc",
|
||||
"acceptance/php-src/tests/NetzfahrplanV4BezugszaehlerTest.php": "65b2daa769773198859ab40d2b230b1f3c43f1c618df0c4b90a494d31efb786c",
|
||||
"acceptance/php-src/tests/fixtures/NativeV4Scenarios.php": "56a5a0df03cb53ea69f6e199c6d2405041a329c7df540ea8bacc08bfaa8766c6",
|
||||
"acceptance/php-src/tests/fixtures/V4BezugszaehlerScenarios.php": "7820bab98d1249aac3fee9f015f8da500744c12bfb5b198fcc735cadf3167ec2",
|
||||
"acceptance/read_native_forecasts.py": "1092413b70714c2e12e3af2697a8d3eb7a70efb5965295c20efc92eaf647c77f",
|
||||
"approved_previous_assets.json": "0fd70ccba10d2970a187dca1be3690461aa143eb29d7e5c244a97951e275cfb0",
|
||||
"commissioning/APPLICATION_STATUS.md": "efec16d79ccd8cb7531c3c135bca5bb38381123666a45cae30d9631183071abc",
|
||||
"commissioning/application-source/server-dataset.json": "844c7b76370f451af172c79a876b8d298236e7dd25b613fc41e0b05338565f6d",
|
||||
"commissioning/deploy_application.py": "8fabbbbf41e00be677a6f109690758035bbe098f9161d9c44865b28ab7f1bd4e",
|
||||
"compose.portal-bridge.yaml": "4299d9de0e8707777052589e4097744c712698ff254a8bdc886041d2c4925197",
|
||||
"compose.yaml": "aae681e18be81633589926db93b27043fca983e255f2f62108ad38df42e1d607",
|
||||
"deploy_integrated_shadow.py": "ec8b324dd5041f89ee84849937f27e9feffb80c6301efe48e9504b326e1f7898",
|
||||
"forecast_acceptance.py": "cc018773c65b61e37f13dfafe2c53b71eec87311d9f3abb514e78cd5917251f9",
|
||||
"gui/netplan-v4.css": "216959a2d90f346a167a4ac809e6cf96c00461abd9853f5c5b5052d6e34d7efd",
|
||||
"gui/netplan-v4.html": "296a247a7a9724dbe8c873372b1d5536ec02ea6eede823c1545e1d8749d64755",
|
||||
"gui/netplan-v4.js": "25c3f80e7a134efaff170e0fd8e16823e1418466bc6e23b11c6b4d1ffa3c1cc5",
|
||||
"install_hooks.py": "1c16586f970742c994cd0cf9ffb41e921ad34f64fa6043fef89d6bdd686799f3",
|
||||
"integrations/NetzfahrplanV4.php": "8ecb7311c6db5998db5fa1b03bd70536bf1fb29f4fea45d257824c033358baf3",
|
||||
"integrations/netplan-v4-bridge.mjs": "f67c28a148b9233da44be817940425265ca98278dc5e04cd36942432febf8dc7",
|
||||
"integrations/netplan_v4_publisher.py": "cb8efe10d2799215b347985c53c96a6420e5461fff4c4e232b6f918d8bff2161",
|
||||
"netplan_v4/__init__.py": "8fe1793927dcdd2d15d0ce1bd3f53b759c0622d6754b5b8efdbb9764e76fde48",
|
||||
"netplan_v4/battery_model.py": "2142f9173e872da8718ce3f5fa6a0062bd4f2f8ed75d42889bc41e2bd00ef332",
|
||||
"netplan_v4/controlled_trial.py": "4d4bd8ed97050b3bf5872c839e4bcc2fc9dcb9f2e0a8c66a4611bbd50bf9e1c4",
|
||||
"netplan_v4/domain.py": "03aadfb6d69f349041c872e105b3ae31e880b507a630f82bdd774948a618b39b",
|
||||
"netplan_v4/forecast_quality.py": "b656868538cb822dc1dec97c7726bcc47d78744a1db3ce02a11d259ba9c2a840",
|
||||
"netplan_v4/measurement_pipeline.py": "caab05a69ac28c086f06b10b20460330b4c3721a6467417542d7666ce82ac743",
|
||||
"netplan_v4/meter_runtime.py": "43275072a212351fa35d34ed2f4932a259112d516d594fad19a7e2b3fd44c547",
|
||||
"netplan_v4/metering.py": "9ed4d3747de76f646d7be603cbdf37a3fc971109605705017c6644c0baa80987",
|
||||
"netplan_v4/optimizer.py": "bf1ad3dcadf10c84e76f6758525fc1309b9f665853c660a1107c1347f1ce9063",
|
||||
"netplan_v4/peak_policy.py": "5ac706655041efb964d4217b5c66a5454aa10348f45d29a98f1a736f3a860576",
|
||||
"netplan_v4/receiver_contract.py": "a43bec2bc2ad8d621f6e85594013b6da3bee1a6ff8ef523f25dafa04672f401c",
|
||||
"netplan_v4/selection.py": "8a2fd034b7a74c9d00d12e12cd76da113c29c3458541c89831c25874c98298ad",
|
||||
"netplan_v4/service.py": "310498c22f235ddaf87e42a2b03e20da4b89d58709964043cc3142417a0f46c4",
|
||||
"netplan_v4/store.py": "7d8ae265dcc3cc4261289b6e38c4f4877246ef088a78f1150f6764d1c97e2e99",
|
||||
"release_preflight.py": "85755aea15daa4709b29838fa25b96cf4aef0166114a8b8474ef0d04422927d5",
|
||||
"requirements.txt": "0b6febdec6a430645b1a17068c19799f9bc451b0b5b174ea119f026063f92464",
|
||||
"run_tests.py": "17192e693fb8a97be1f0a2f166568d84e86056d4a7ff97f94dcbe1f4391719a4",
|
||||
"runtime_preflight.py": "69b0fa8925c00cbd2399375301c63acc60f6dd5fe1438ad4458eaa07f9d087a1",
|
||||
"tests/manager_protocol.php": "043c763f578d176b09224170690c1fb0a204c7e821b3687fbcf33895355e6f93",
|
||||
"tests/portal.test.mjs": "4e08acda7cbf5eac5b9e3d8d032ca1403250a993ac022e94f440d54f65f90f0a",
|
||||
"tests/test_application_deployment.py": "9fa667e08f9003dd942c6bbb1e8b77321f88b8d4ec2c042f91450e7d34bdeee5",
|
||||
"tests/test_battery_recovery.py": "5185416073b4144f00c643acba48dc93be29d03163de10b1f29a8fd84c191ca3",
|
||||
"tests/test_container_access.py": "181aa47e5c09235ae45a25e3221fc6871bbf89c49ce541599e9b8f7da8570df9",
|
||||
"tests/test_controlled_trial.py": "8988396ddc709dc8d6bb039f0c4d1ab50efb47e455975f35de5b0a305bdbfe64",
|
||||
"tests/test_controlled_trial_pipeline.py": "df3d8d6d816c77cdcb2f15d4caaf61fb52e919e56c1c842902b0808f54788e0d",
|
||||
"tests/test_delivery.py": "ca36fbb6fccc7e89ffbf7147bb8f2b6ed2a5614f270ac87499c6360ccd95af09",
|
||||
"tests/test_forecast_acceptance.py": "d2305dee8ccd8e633ca5b44b497dde519b715369f5a5572b864919fa2bdddd76",
|
||||
"tests/test_forecast_quality.py": "461750d0b91d5fe21ec5b0a7d97a83e1bfe76fd6893950281be4753dac3baf82",
|
||||
"tests/test_integrated_deployment.py": "ac4b1cf56a4222b423fa78d1870ccb8cf02a07a3ad2119a950dd009ea157e883",
|
||||
"tests/test_measurement_pipeline.py": "761887486922762dd12a1b3beb00f41e0000ec3cbfe3b101484d0fe8f2364eb4",
|
||||
"tests/test_metering.py": "6c7af6e624c93cde4b6cca00e66fc8fec0b72a047ac77e16778dc1926194a69d",
|
||||
"tests/test_peak_release.py": "223f4374111cfab99ef352a71a7b91a7f71abc4ed126f2c180b138d1bd390d3d",
|
||||
"tests/test_receiver_contract.py": "21daccb358fa62a983d2292d5de2b9b3c7300a08e314e850749316a24d3d254e",
|
||||
"tests/test_release_preflight.py": "64104923bea0890e5010471466de87c289a97bca7dd29d56734d0009e79f24d8",
|
||||
"tests/test_v4.py": "c35db22a864aa0afc4d0abc357ae854f038e86de8e3001760c0e63ce036a7763"
|
||||
},
|
||||
"portalBefore": {
|
||||
"netplan-v4-bridge.mjs": "f58c652b04cd2ac20f2bec11e1161c44b72c26267b4dfb42096a1fad36a3c9c2",
|
||||
"public/netplan-v4.js": "6cd1d887bb08c73a815c92eb684a0992e8cb092f748770cc94debcfd9705a628"
|
||||
},
|
||||
"createdAt": "2026-10-02T20:57:00.387145+00:00",
|
||||
"controlEnabled": false
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{"datasetId":"lihrenmoos-physical-v1","mappingSha256":"517d1907631c7f152096e21759bb3452cdd0b4b1b73f91f1c7cd3bd62d8aaa8b","inventorySha256":"0a9dde190d81bc263da222ad8bfb6268295ea03aec239f6c1fcd249b3d34c1bc","sources":[{"key":"grid","variableId":40348,"factorToW":1000,"maxAgeSeconds":60,"role":"grid"},{"key":"pv_goodwe1","variableId":48459,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_goodwe2","variableId":53802,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"pv_solaredge","variableId":20335,"factorToW":1,"maxAgeSeconds":60,"role":"pv"},{"key":"physical_goodwe1","variableId":47725,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_goodwe2","variableId":35724,"factorToW":-1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"physical_solaredge","variableId":21447,"factorToW":1,"maxAgeSeconds":60,"role":"physical_storage"},{"key":"ev_account","variableId":52020,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"sdl_account","variableId":25085,"factorToW":1,"maxAgeSeconds":60,"role":"reference"},{"key":"grid_display","variableId":49301,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe1","variableId":27361,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_goodwe2","variableId":23109,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_solaredge","variableId":51938,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_ev","variableId":32871,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"soc_sdl","variableId":23879,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_charge","variableId":50230,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_available_discharge","variableId":43899,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t1","variableId":59607,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"energy_t2","variableId":26620,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"ev_requested","variableId":19651,"role":"reference","factorToW":1,"maxAgeSeconds":60},{"key":"sdl_requested","variableId":38943,"role":"sdl_request","factorToW":1,"maxAgeSeconds":120},{"key":"solar_ac_value","variableId":37975,"role":"solar_raw","factorToW":1,"maxAgeSeconds":60},{"key":"solar_ac_scale","variableId":41853,"role":"solar_scale","factorToW":1,"maxAgeSeconds":60}],"formula":"solar_terminal_v1","solarReference":{"pvKey":"pv_solaredge","batteryKey":"physical_solaredge","rawKey":"solar_ac_value","scaleKey":"solar_ac_scale"},"minimumCoverage":0.95,"maximumGapSeconds":10,"minimumTrainingHours":24,"historyDays":28}
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Deploy the integrated measurement/training application, NOT actuator permission.
|
||||
|
||||
Only V4 and the portal are recreated. Existing legacy forecasts, tariffs, Symcon and
|
||||
battery dispatch are unchanged. Source-only checks are the default; --apply is explicit.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
import argparse,hashlib,json,os,sqlite3,subprocess,tempfile
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
PORTAL=ROOT.parent/'license'
|
||||
MANIFEST=Path(__file__).with_name('application-source')/'RELEASE.json'
|
||||
DATASET=Path(__file__).with_name('application-source')/'server-dataset.json'
|
||||
PORTAL_FILES={'integrations/netplan-v4-bridge.mjs':'netplan-v4-bridge.mjs','gui/netplan-v4.js':'public/netplan-v4.js'}
|
||||
|
||||
def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
def verify():
|
||||
m=json.loads(MANIFEST.read_text())
|
||||
if m.get('scope')!='integrated_measurement_application' or not m.get('sourceHashes'):
|
||||
raise ValueError('Unexpected application release manifest')
|
||||
for n,h in m['sourceHashes'].items():
|
||||
p=ROOT/n
|
||||
if Path(n).is_absolute() or '..' in Path(n).parts or p.is_symlink() or not p.is_file() or digest(p)!=h:
|
||||
raise ValueError('Source drift: '+n)
|
||||
for n,old in m['portalBefore'].items():
|
||||
if n not in PORTAL_FILES.values():raise ValueError('Unexpected portal target')
|
||||
p=PORTAL/n
|
||||
if p.is_symlink() or not p.is_file():raise ValueError('Portal target changed')
|
||||
source=next(s for s,t in PORTAL_FILES.items() if t==n)
|
||||
if digest(p) not in (old,m['sourceHashes'][source]):raise ValueError('Concurrent portal change; not overwritten')
|
||||
return m
|
||||
|
||||
def atomic(path,data,mode=0o644):
|
||||
if path.is_symlink():raise ValueError('Symlink refused')
|
||||
fd,name=tempfile.mkstemp(prefix='.v4-app-',dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno())
|
||||
os.chmod(name,mode);os.replace(name,path)
|
||||
finally:
|
||||
if os.path.exists(name):os.unlink(name)
|
||||
|
||||
def backup_database(source,destination):
|
||||
if not source.is_file() or source.is_symlink() or destination.exists():raise ValueError('Explicit existing database and new backup path required')
|
||||
with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as a,sqlite3.connect(destination) as b:
|
||||
a.backup(b)
|
||||
if b.execute('PRAGMA integrity_check').fetchone()[0]!='ok':raise ValueError('Backup failed')
|
||||
destination.chmod(0o600)
|
||||
|
||||
BOOTSTRAP='''import json,os,urllib.request,sys
|
||||
p=json.load(sys.stdin)
|
||||
base='http://127.0.0.1:9100/internal/v2/prognosis/'+p['plant']+'/planner'
|
||||
h={'X-Enelix-Service-Token':os.environ['PROGNOSIS_SERVICE_TOKEN'],'Content-Type':'application/json'}
|
||||
req=urllib.request.Request(base+'/datasets/'+p['dataset']['datasetId'],data=json.dumps(p['dataset']).encode(),headers=h,method='PUT')
|
||||
receipt=json.load(urllib.request.urlopen(req,timeout=10))
|
||||
state=json.load(urllib.request.urlopen(urllib.request.Request(base,headers=h),timeout=10))
|
||||
assert state['liveEnabled'] is False
|
||||
print(json.dumps({'dataset':receipt,'existingForecastSource':state['settings']['forecastSource'],'liveEnabled':False}))
|
||||
'''
|
||||
|
||||
def run(plant,apply=False):
|
||||
m=verify()
|
||||
if plant!=m['installationId']:raise ValueError('Use the prepared installation-specific mapping')
|
||||
if not apply:
|
||||
print('APPLICATION SOURCE CHECK PASSED:',len(m['sourceHashes']),'files. No deployment.');return
|
||||
if os.geteuid()!=0:raise ValueError('Run as root; do not widen Docker permissions')
|
||||
folder=ROOT/'application-releases'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S.%fZ')
|
||||
folder.mkdir(parents=True,mode=0o700)
|
||||
env={**os.environ,'NETPLAN_V4_PLANTS':plant}
|
||||
compose=['docker','compose','-f',str(ROOT/'compose.yaml')]
|
||||
portal=['docker','compose','-f',str(PORTAL/'compose.yaml'),'-f',str(ROOT/'compose.portal-bridge.yaml')]
|
||||
result={'scope':'integrated_measurement_application','startedAt':datetime.now(timezone.utc).isoformat(),
|
||||
'liveEnabled':False,'productionCommissioned':False,'steps':[],'sourceHashes':m['sourceHashes']}
|
||||
changed={};old_image=None;v4_replaced=False;portal_replaced=False
|
||||
def cmd(args,timeout=180,capture=False,input=None):
|
||||
return subprocess.run(args,cwd=ROOT,env=env,check=True,timeout=timeout,text=True,input=input,
|
||||
stdout=subprocess.PIPE if capture else None,stderr=subprocess.PIPE if capture else None)
|
||||
try:
|
||||
ids=cmd(compose+['ps','-q','netplan-v4'],capture=True).stdout.split()
|
||||
if len(ids)!=1:raise ValueError('Expected existing V4 service')
|
||||
old_image=cmd(['docker','inspect','--format','{{.Image}}',ids[0]],capture=True).stdout.strip()
|
||||
result['previousV4Image']=old_image
|
||||
cmd(compose+['build','netplan-v4'],timeout=900)
|
||||
cmd(compose+['run','--rm','--no-deps','--entrypoint','python','netplan-v4','/app/run_tests.py'],timeout=240)
|
||||
cmd(['node','--test',str(ROOT/'tests/portal.test.mjs')])
|
||||
cmd(['node','--check',str(ROOT/'gui/netplan-v4.js')])
|
||||
result['steps'].append('target_python_and_portal_tests_passed');verify()
|
||||
backup_database(ROOT/'data/netplan-v4.sqlite',folder/'before.sqlite')
|
||||
result['steps'].append('consistent_database_backup')
|
||||
for source,target in PORTAL_FILES.items():
|
||||
p=PORTAL/target;old=p.read_bytes();new=(ROOT/source).read_bytes()
|
||||
if old==new:continue
|
||||
if hashlib.sha256(old).hexdigest()!=m['portalBefore'][target]:raise ValueError('Concurrent portal change')
|
||||
dest=folder/'portal-before'/target;dest.parent.mkdir(parents=True,exist_ok=True);dest.write_bytes(old)
|
||||
atomic(p,new);changed[target]=(old,new)
|
||||
v4_replaced=True;cmd(compose+['up','-d','--no-deps','--no-build','--wait','netplan-v4'])
|
||||
# Recreate rather than restart: atomic replacement of a file bind mount needs a new mount.
|
||||
portal_replaced=True;cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal'])
|
||||
payload=json.dumps({'plant':plant,'dataset':json.loads(DATASET.read_text())})
|
||||
receipt=cmd(compose+['exec','-T','netplan-v4','python','-c',BOOTSTRAP],capture=True,input=payload)
|
||||
result['application']=json.loads(receipt.stdout)
|
||||
result['status']='application_deployed_no_actuator_permission'
|
||||
result['steps'].append('configured_dataset_and_health_verified')
|
||||
except Exception as e:
|
||||
result['status']='needs_review';result['errorType']=type(e).__name__
|
||||
restored=[]
|
||||
for target,(old,new) in changed.items():
|
||||
p=PORTAL/target
|
||||
if p.read_bytes()==new:atomic(p,old);restored.append(target)
|
||||
result['restoredPortalFiles']=restored
|
||||
try:
|
||||
if v4_replaced and old_image:
|
||||
rollback=folder/'rollback.yaml';rollback.write_text('services:\n netplan-v4:\n image: '+old_image+'\n')
|
||||
cmd(compose+['-f',str(rollback),'up','-d','--no-deps','--no-build','--pull','never','--wait','netplan-v4'])
|
||||
if portal_replaced:cmd(portal+['up','-d','--no-deps','--no-build','--force-recreate','--wait','license-portal'])
|
||||
result['rollback']='previous_runtime_restored_additive_data_retained'
|
||||
except Exception:result['rollback']='manual_review_required'
|
||||
raise
|
||||
finally:
|
||||
result['finishedAt']=datetime.now(timezone.utc).isoformat()
|
||||
report=folder/'REPORT.json';report.write_text(json.dumps(result,indent=2)+'\n')
|
||||
uid=ROOT.stat().st_uid;gid=ROOT.stat().st_gid
|
||||
os.chown(folder,uid,gid);os.chown(folder.parent,uid,gid);os.chown(report,uid,gid);report.chmod(0o640)
|
||||
print('APPLICATION RELEASE REPORT:',report)
|
||||
print('V4 data/model application and portal updated. Install manager data integration separately. No V4 actuation enabled.')
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--plant',required=True,type=lambda v:str(UUID(v)));p.add_argument('--apply',action='store_true');a=p.parse_args()
|
||||
try:run(a.plant,a.apply)
|
||||
except Exception as e:raise SystemExit('Stopped: '+type(e).__name__+'. See the application release report.')
|
||||
@@ -0,0 +1,6 @@
|
||||
services:
|
||||
license-portal:
|
||||
environment:
|
||||
NETPLAN_V4_URL: http://netplan-v4:9100
|
||||
volumes:
|
||||
- /home/agent/services/license/netplan-v4-bridge.mjs:/app/netplan-v4-bridge.mjs:ro
|
||||
@@ -0,0 +1,37 @@
|
||||
name: enelix-netplan-v4-shadow
|
||||
services:
|
||||
netplan-v4:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
user: "1000:1000"
|
||||
read_only: true
|
||||
env_file:
|
||||
- /home/agent/services/prognosis-integration.env
|
||||
environment:
|
||||
NETPLAN_V4_DB: /data/netplan-v4.sqlite
|
||||
NETPLAN_V4_PLANTS: ${NETPLAN_V4_PLANTS:?Explicit installation ID required}
|
||||
NETPLAN_V4_TEST_REPORT_DIR: /tmp/test-results
|
||||
volumes:
|
||||
- ./data:/data
|
||||
tmpfs:
|
||||
- /tmp:size=64m,mode=1777,noexec,nosuid
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
cap_drop:
|
||||
- ALL
|
||||
networks:
|
||||
prognosis_integration:
|
||||
aliases:
|
||||
- netplan-v4
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9100/health', timeout=3).read()"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
# No public port, no Docker socket, no access to legacy databases.
|
||||
networks:
|
||||
prognosis_integration:
|
||||
external: true
|
||||
name: enelix-platform-internal
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Install the reviewed server-side SHADOW integration; no Symcon/device writes.
|
||||
|
||||
Default: verify source and print scope only. --apply requires root. Build and test
|
||||
all changed code before recreating services. Save old image/config references and
|
||||
an integrity-checked online backup of the V4 DB. Does not enable native telemetry
|
||||
or dispatch: operation ingress may still be awaiting installation on Symcon.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
SERVICES = ROOT.parent
|
||||
PROJECT = SERVICES / 'prognosis-manager-enelix2'
|
||||
PORTAL = SERVICES / 'license'
|
||||
PREFLIGHT = ROOT / 'acceptance-reports/20261002T043021Z/REPORT.json'
|
||||
MANIFEST = ROOT / 'INTEGRATED_SHADOW_SOURCE.json'
|
||||
|
||||
|
||||
def verify_sources():
|
||||
report = json.loads(PREFLIGHT.read_text())
|
||||
if report.get('preflightChecksPassed') is not True:
|
||||
raise ValueError('Expected successful earlier runtime preflight is missing')
|
||||
expected = json.loads(MANIFEST.read_text())['files']
|
||||
for relative, digest in expected.items():
|
||||
path = SERVICES / relative
|
||||
if not path.resolve().is_relative_to(SERVICES) or path.is_symlink() or not path.is_file():
|
||||
raise ValueError('Invalid release source path: ' + relative)
|
||||
if hashlib.sha256(path.read_bytes()).hexdigest() != digest:
|
||||
raise ValueError('Source changed since preparation: ' + relative)
|
||||
from forecast_acceptance import verify_forecast_bundle
|
||||
verify_forecast_bundle(ROOT/'acceptance/forecast-src', PROJECT/'forecast_engine')
|
||||
from release_preflight import verify_native_bundle
|
||||
verify_native_bundle()
|
||||
return expected
|
||||
|
||||
|
||||
def compose_groups():
|
||||
return {
|
||||
'v4': (ROOT, [ROOT/'compose.yaml'], ['netplan-v4']),
|
||||
'forecast': (PROJECT, [PROJECT/'compose.yaml', ROOT/'compose.forecast-bridge.yaml'], ['api','forecast-engine','tariff-importer']),
|
||||
'portal': (PORTAL, [PORTAL/'compose.yaml', ROOT/'compose.portal-bridge.yaml'], ['license-portal']),
|
||||
}
|
||||
|
||||
|
||||
def compose_args(files):
|
||||
result=['docker','compose']
|
||||
for path in files:result.extend(['-f',str(path)])
|
||||
return result
|
||||
|
||||
|
||||
def backup_database(source, destination):
|
||||
if not source.exists():return {'exists':False}
|
||||
src=sqlite3.connect(source.resolve().as_uri()+'?mode=ro',uri=True,timeout=15)
|
||||
dst=sqlite3.connect(destination,timeout=15)
|
||||
try:
|
||||
src.backup(dst)
|
||||
if dst.execute('PRAGMA integrity_check').fetchone()[0]!='ok':
|
||||
raise ValueError('V4 database backup failed integrity check')
|
||||
finally:dst.close();src.close()
|
||||
os.chmod(destination,0o640)
|
||||
if os.geteuid()==0:os.chown(destination,1000,1000)
|
||||
return {'exists':True,'path':str(destination),'sha256':hashlib.sha256(destination.read_bytes()).hexdigest(),'integrity':'ok'}
|
||||
|
||||
|
||||
def save_json(path, value):
|
||||
path.write_text(json.dumps(value,indent=2,allow_nan=False)+'\n')
|
||||
os.chmod(path,0o640)
|
||||
if os.geteuid()==0:os.chown(path,1000,1000)
|
||||
|
||||
|
||||
def run(plant):
|
||||
sources=verify_sources()
|
||||
env=dict(os.environ,NETPLAN_V4_PLANTS=plant)
|
||||
stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
folder=ROOT/'deployment-reports'/stamp
|
||||
folder.mkdir(parents=True,exist_ok=False)
|
||||
if os.geteuid()==0:
|
||||
os.chown(folder.parent,1000,1000);os.chown(folder,1000,1000)
|
||||
report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant,
|
||||
'scope':'server_shadow_only','liveEnabled':False,'productionReady':False,
|
||||
'sourceHashes':sources,'steps':[],'before':{},'existingServiceRecreated':False}
|
||||
def command(args,cwd=ROOT,capture=False,stdin=None,timeout=900):
|
||||
result=subprocess.run(args,cwd=cwd,env=env,input=stdin,text=True,
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.PIPE if capture else None,timeout=timeout,check=False)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Step failed (exit '+str(result.returncode)+'): '+args[0])
|
||||
return result.stdout if capture else ''
|
||||
def step(name,args,cwd=ROOT,timeout=900):
|
||||
print('\n=== '+name+' ===',flush=True)
|
||||
command(args,cwd,timeout=timeout)
|
||||
report['steps'].append(name);save_json(folder/'DEPLOYMENT.json',report)
|
||||
try:
|
||||
# Capture only selected nonsecret Docker fields, never the environment.
|
||||
for group,(cwd,files,services) in compose_groups().items():
|
||||
oldfiles=[files[0]]
|
||||
items=[]
|
||||
for service in services:
|
||||
cid=command(compose_args(oldfiles)+['ps','-q',service],cwd,True).strip()
|
||||
if not cid or '\n' in cid:raise ValueError('Expected exactly one existing container: '+service)
|
||||
image=command(['docker','inspect','--format','{{.Image}}',cid],cwd,True).strip()
|
||||
configs=command(['docker','inspect','--format','{{index .Config.Labels "com.docker.compose.project.config_files"}}',cid],cwd,True).strip()
|
||||
items.append({'service':service,'containerId':cid,'imageId':image,'previousComposeFiles':configs})
|
||||
report['before'][group]=items
|
||||
save_json(folder/'DEPLOYMENT.json',report)
|
||||
step('review optional source installer',[sys.executable,str(ROOT/'install_hooks.py')])
|
||||
# All new forecasting tests run offline in a disposable Python 3.11 image.
|
||||
step('build forecast acceptance image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')])
|
||||
step('forecast candidate offline tests',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local'])
|
||||
step('V4 and bridge target tests',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only'])
|
||||
verify_sources()
|
||||
from install_hooks import plan,apply
|
||||
receipt=apply(plan())
|
||||
report['sourceInstallReceipt']=str(receipt) if receipt else None
|
||||
step('portal syntax',['node','--check',str(PORTAL/'server.mjs')])
|
||||
# Build every production-shaped candidate before changing a running container.
|
||||
cwd,files,svcs=compose_groups()['forecast']
|
||||
step('build server-side images',compose_args(files)+['build']+svcs,cwd)
|
||||
report['v4DatabaseBackup']=backup_database(ROOT/'data/netplan-v4.sqlite',folder/'netplan-v4-before.sqlite')
|
||||
report['existingServiceRecreated']=True
|
||||
save_json(folder/'DEPLOYMENT.json',report)
|
||||
for group,(cwd,files,svcs) in compose_groups().items():
|
||||
step('start shadow integration '+group,compose_args(files)+['up','-d','--no-deps','--no-build','--wait','--wait-timeout','180']+svcs,cwd,timeout=300)
|
||||
cwd,files,_=compose_groups()['v4']
|
||||
# Check shadow guard and report incoming types without disclosing payloads.
|
||||
probe='''import json,os,sqlite3,urllib.request
|
||||
from datetime import datetime,timezone
|
||||
plant=os.environ['NETPLAN_V4_PLANTS']
|
||||
health=json.loads(urllib.request.urlopen('http://127.0.0.1:9100/health',timeout=5).read())
|
||||
assert health.get('mode')=='shadow' and health.get('liveEnabled') is False
|
||||
con=sqlite3.connect('file:/data/netplan-v4.sqlite?mode=ro',uri=True)
|
||||
rows=con.execute('SELECT kind,count(*) FROM planner_inputs WHERE plant=? GROUP BY kind',(plant,)).fetchall()
|
||||
status=con.execute('SELECT status FROM planner_run_status WHERE plant=?',(plant,)).fetchone()
|
||||
con.close()
|
||||
print(json.dumps({'checkedAt':datetime.now(timezone.utc).isoformat(),'health':health,'inputEventsByKind':dict(rows),'lastRunStatus':status[0] if status else None,'nativeSenderInstalledByThisCommand':False}))
|
||||
'''
|
||||
report['readiness']=json.loads(command(compose_args(files)+['exec','-T','netplan-v4','python','-B','-'],cwd,True,probe,60))
|
||||
report['status']='server_shadow_installed'
|
||||
report['remaining']='Native operation sender, plant-level initialization and real shadow-plan acceptance still required; no live dispatch installed'
|
||||
except Exception as error:
|
||||
report['status']='failed';report['errorType']=type(error).__name__;report['error']=str(error)[:500]
|
||||
report['remaining']='Inspect completed steps and before-image references. No automatic database restore or device changes.'
|
||||
raise
|
||||
finally:
|
||||
save_json(folder/'DEPLOYMENT.json',report)
|
||||
print('\nDEPLOYMENT REPORT: '+str(folder/'DEPLOYMENT.json'),flush=True)
|
||||
print('SERVER SHADOW ONLY. Symcon, device permissions and actuators were not modified.',flush=True)
|
||||
print(json.dumps(report['readiness'],indent=2))
|
||||
print('Native operating-state input may still be missing; this is NOT production commissioning.')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--plant',required=True,type=lambda p:str(UUID(p)))
|
||||
parser.add_argument('--apply',action='store_true')
|
||||
args=parser.parse_args()
|
||||
if args.plant!='e3a08f9e-af12-4695-99bd-8b51c0520021':
|
||||
raise SystemExit('This first integration rollout is allowlisted for Lihrenmoos only.')
|
||||
if not args.apply:
|
||||
files=verify_sources()
|
||||
print('CHECK ONLY:',len(files),'reviewed source files. No running service changed.')
|
||||
for group,(_,_,svcs) in compose_groups().items():print(group,', '.join(svcs))
|
||||
else:
|
||||
if os.geteuid()!=0:raise SystemExit('Run as root; do not change Docker socket access.')
|
||||
try:run(args.plant)
|
||||
except (OSError,ValueError,RuntimeError,subprocess.TimeoutExpired) as error:
|
||||
raise SystemExit('Deployment stopped: '+str(error))
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Verify the separately staged forecast repair before target-runtime testing.
|
||||
Only explicit Python source and requirements are copied. No models, credentials,
|
||||
live databases or telemetry are needed by this offline test image.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
|
||||
def source_paths(root):
|
||||
root = Path(root)
|
||||
paths = [root / 'requirements.txt'] + list(root.glob('*.py'))
|
||||
for folder in ('methods', 'tests'):
|
||||
paths.extend((root / folder).glob('*.py'))
|
||||
if not (root / 'telemetry_quality.py').is_file():
|
||||
raise ValueError('Forecast telemetry repair missing')
|
||||
for path in paths:
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()):
|
||||
raise ValueError('External forecast source is not allowed')
|
||||
return sorted(paths)
|
||||
|
||||
|
||||
def verify_forecast_bundle(bundle, source):
|
||||
bundle, source = Path(bundle), Path(source)
|
||||
manifest_path = bundle / 'SOURCE_MANIFEST.json'
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
expected_paths = {str(p.relative_to(source)) for p in source_paths(source)}
|
||||
if not isinstance(manifest, dict) or not manifest or set(manifest) != expected_paths:
|
||||
raise ValueError('Forecast source set changed; rebuild reviewed test bundle')
|
||||
if {str(p.relative_to(bundle)) for p in source_paths(bundle)} != expected_paths:
|
||||
raise ValueError('Staged forecast source set does not match development source')
|
||||
for relative, expected in manifest.items():
|
||||
if Path(relative).is_absolute() or '..' in Path(relative).parts:
|
||||
raise ValueError('Unsafe forecast manifest path')
|
||||
for path in (source / relative, bundle / relative):
|
||||
if not path.is_file() or path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != expected:
|
||||
raise ValueError('Forecast source changed after staging; review before test')
|
||||
return dict(manifest)
|
||||
@@ -0,0 +1 @@
|
||||
body{font:16px system-ui,sans-serif;background:#eef4f4;color:#243c42}main{max-width:1100px;margin:2rem auto;padding:1.5rem;background:white}a{color:#006b83}label{display:block;margin:.8rem 0}select,button{font:inherit;padding:.6rem;margin:.3rem;border:1px solid #bacaca;border-radius:.25rem}button{background:#006b83;color:white;cursor:pointer}.notice{background:#e9f7fa;padding:1rem;border-left:4px solid #006b83}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#eef4f4;padding:1rem}svg{width:100%;height:auto}.curve{fill:none;stroke-width:2}.baseline{stroke:#7d888c}.optimized{stroke:#006b83}.axis{stroke:#b7c8cc}button:disabled{opacity:.5}
|
||||
@@ -0,0 +1 @@
|
||||
<!doctype html><html lang="de"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>ENELIX Netzfahrplan V4</title><link rel="stylesheet" href="/netplan-v4.css"></head><body><main><a href="/">Zurueck zum Portal</a><h1>Netzfahrplan V4</h1><p class="notice">Schattenbetrieb. Diese Ansicht steuert keine Verbraucher. Die bestehende Regelung bleibt unveraendert.</p><label>Anlage <select id="plants"></select></label><p id="message" role="status"></p><section id="panel"></section></main><script type="module" src="/netplan-v4.js"></script></body></html>
|
||||
@@ -0,0 +1,107 @@
|
||||
const message = document.getElementById('message');
|
||||
const panel = document.getElementById('panel');
|
||||
const plants = document.getElementById('plants');
|
||||
let session, generation = 0;
|
||||
const element = (tag, text) => { const e = document.createElement(tag); if (text !== undefined) e.textContent = text; return e; };
|
||||
const time = value => value ? new Date(value).toLocaleString('de-CH', { timeZone: 'Europe/Zurich' }) : 'nicht vorhanden';
|
||||
const num = (v, digits = 2) => typeof v === 'number' && Number.isFinite(v) ? v.toLocaleString('de-CH', { maximumFractionDigits: digits }) : 'unbekannt';
|
||||
|
||||
async function request(url, method = 'GET', body) {
|
||||
const headers = {};
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
if (method !== 'GET') headers['X-CSRF-Token'] = session.csrfToken;
|
||||
const response = await fetch(url, { method, headers, credentials: 'same-origin', body: body === undefined ? undefined : JSON.stringify(body) });
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || data.detail || 'Anfrage fehlgeschlagen');
|
||||
return data;
|
||||
}
|
||||
function select(label, choices, selected) {
|
||||
const wrap = element('label', label + ' '), field = element('select');
|
||||
for (const [value, caption] of choices) { const option = element('option', caption); option.value = value; field.append(option); }
|
||||
field.value = selected; wrap.append(field); return { wrap, field };
|
||||
}
|
||||
function table(title, rows) {
|
||||
panel.append(element('h3', title)); const grid = element('table');
|
||||
for (const [name, value] of rows) { const row = element('tr'); row.append(element('th', name), element('td', String(value))); grid.append(row); }
|
||||
panel.append(grid);
|
||||
}
|
||||
function curve(title, points, series) {
|
||||
panel.append(element('h3', title));
|
||||
if (!points.length) { panel.append(element('p', 'Keine Daten.')); return; }
|
||||
const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
|
||||
svg.setAttribute('viewBox', '0 0 1000 210'); svg.setAttribute('role', 'img'); svg.setAttribute('aria-label', title);
|
||||
const values = points.flatMap(p => series.map(s => s.value(p))).filter(v => typeof v === 'number' && Number.isFinite(v));
|
||||
if (!values.length) { panel.append(element('p', 'Keine gueltigen Werte.')); return; }
|
||||
const low = Math.min(0, ...values), high = Math.max(1, ...values);
|
||||
const start = Date.parse(points[0].time), end = Date.parse(points[points.length - 1].validUntil || points[points.length - 1].time);
|
||||
for (const [i, source] of series.entries()) {
|
||||
let section = [];
|
||||
const flush = () => { if (!section.length) return; const line = document.createElementNS(svg.namespaceURI, 'polyline'); line.setAttribute('class', 'curve ' + (i ? 'optimized' : 'baseline')); line.setAttribute('points', section.join(' ')); svg.append(line); section = []; };
|
||||
for (const p of points) {
|
||||
const value = source.value(p);
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) { flush(); continue; }
|
||||
const x = 10 + 980 * (Date.parse(p.time) - start) / Math.max(1, end - start);
|
||||
const y = 190 - (value - low) / (high - low) * 180; section.push(`${x},${y}`);
|
||||
}
|
||||
flush();
|
||||
}
|
||||
panel.append(svg, element('p', `${num(low)} bis ${num(high)}. ${series.map((s, i) => (i ? 'Blau: ' : 'Grau: ') + s.label).join('; ')}. ${time(points[0].time)} – ${time(points[points.length - 1].validUntil)}.`));
|
||||
}
|
||||
async function render() {
|
||||
const ticket = ++generation, id = plants.value; if (!id) return;
|
||||
const base = `/api/plants/${encodeURIComponent(id)}/prognosis/planner-v4`;
|
||||
message.textContent = 'Lade Planungsstand ...';
|
||||
try {
|
||||
const state = await request(base); if (ticket !== generation) return; panel.replaceChildren();
|
||||
panel.append(element('p', 'SCHATTENBETRIEB: Die angezeigten Vorgaben steuern keine Geraete.'));
|
||||
const form = element('form');
|
||||
const family = select('Modellfamilie', [['auto', 'Automatisch – historischer Kostenvergleich'], ...state.families.map(f => [f.key, f.label])], state.settings.family);
|
||||
const measurement = select('Messdatenbasis', [['verified_only', 'Nur verifizierte Peak-/Viertelstundenwerte'], ['allow_estimates', 'Gekennzeichnete Schaetzungen fuer die Planung zulassen']], state.settings.measurementPolicy || 'verified_only');
|
||||
const outlook = select('Peakbewertung', [['full_incremental', 'Voller zusaetzlicher Monatstarif'], ['empirical_if_available', 'Restmonats-Szenarien bei ausreichender Historie']], state.settings.peakOutlookPolicy || 'full_incremental');
|
||||
const dataSource = select('Verbrauchsprognose', [['legacy', 'Bisherige Datenquelle'], ['corrected_profile', 'Bereinigte physische Lastprofile']], state.settings.forecastSource || 'legacy');
|
||||
const datasets = state.dataPipeline?.datasets || [];
|
||||
const dataset = select('Messdatensatz', [['', 'Datensatz auswaehlen'], ...datasets.map(d => [d.datasetId, d.datasetId + ' (' + d.status + ')'])], state.settings.measurementDataset || '');
|
||||
const cadence = select('Nachtraining', [['daily', 'Taeglich'], ['weekly', 'Woechentlich']], state.settings.trainingCadence);
|
||||
const relianceWrap = element('label', 'Gewicht der Restmonatsprognose (0 bis 100 %) '), reliance = element('input');
|
||||
reliance.type = 'number'; reliance.min = '0'; reliance.max = '100'; reliance.step = '1'; reliance.value = String(100 * (state.settings.peakOutlookReliance ?? .5)); relianceWrap.append(reliance);
|
||||
const save = element('button', 'Speichern und neu berechnen'), replan = element('button', 'Neu berechnen'), refresh = element('button', 'Status aktualisieren');
|
||||
save.type = 'submit'; replan.type = refresh.type = 'button';
|
||||
form.append(family.wrap, dataSource.wrap, dataset.wrap, measurement.wrap, outlook.wrap, relianceWrap, cadence.wrap, save, replan, refresh); panel.append(form);
|
||||
let revision = state.settings.revision;
|
||||
form.addEventListener('submit', async event => {
|
||||
event.preventDefault(); save.disabled = true;
|
||||
try {
|
||||
const r = await request(base + '/settings', 'PUT', { expectedRevision: revision, changes: { family: family.field.value, forecastSource: dataSource.field.value, measurementDataset: dataset.field.value, measurementPolicy: measurement.field.value, peakOutlookPolicy: outlook.field.value, peakOutlookReliance: Number(reliance.value) / 100, trainingCadence: cadence.field.value } });
|
||||
revision = r.revision; message.textContent = `Revision ${revision} gespeichert. Neuberechnung angefordert, noch nicht abgeschlossen.`;
|
||||
} catch (error) { message.textContent = error.message; } finally { save.disabled = false; }
|
||||
});
|
||||
replan.addEventListener('click', async () => { try { await request(base + '/replan', 'POST', {}); message.textContent = 'Neuberechnung angefordert.'; } catch (error) { message.textContent = error.message; } });
|
||||
refresh.addEventListener('click', render);
|
||||
panel.append(element('p', 'Die Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe. Ohne ausreichende vergleichbare Historie gilt der volle zusaetzliche Monatstarif. Technische und konfigurierte Managergrenzen bleiben verbindlich.'));
|
||||
panel.append(element('p', 'Fuer bereinigte Lastprofile steuert die Trainingsauswahl den automatischen Modellaufbau. Historische Prognosen bleiben unveraendert. Die wirtschaftliche Modellautomatik wartet weiterhin auf vergleichbare Kosten-Replays, nicht bloss auf gute R2-Werte.'));
|
||||
for (const d of datasets) table('Daten und Training: ' + d.datasetId, [['Zustand', d.status], ['Angenommene Aufnahmen', d.records], ['Letzte Aufnahme', time(d.lastCapture)], ['Nutzbare 5-Minutenwerte', d.detail?.usableWindows ?? 0], ['Datenbasis in Stunden', num(d.detail?.usableEquivalentHours)], ['Modellstand', d.detail?.modelId || 'noch in Datensammlung'], ['Letztes Training', time(d.detail?.trainedAt)], ['Validierung', d.detail?.validation?.status || 'noch ausstehend'], ['Messgrundlage', 'Konfigurierte physische Schaetzung; kein unabhaengiger Messnachweis']] );
|
||||
panel.append(element('p', state.lastRun ? `Rechenstatus: ${state.lastRun.status} ${state.lastRun.detail?.reason || ''}` : 'Noch kein Rechenlauf.'));
|
||||
if (state.pending) panel.append(element('p', 'Neuberechnung vorgemerkt oder aktiv.'));
|
||||
for (const [month, basis] of Object.entries(state.peakPlanningBases || {})) table(`Vorlaeufige Peakbasis ${month}`, [['Wert', num(basis.kw) + ' kW'], ['Qualitaet', basis.quality], ['Quelle', basis.source], ['Beobachtet', time(basis.observedAt)], ['Hinweis', basis.notes || 'Kein Abrechnungsnachweis']]);
|
||||
const plan = state.plan;
|
||||
if (plan) {
|
||||
if (!state.fresh) panel.append(element('p', 'ACHTUNG: Plan veraltet oder Neuberechnung ausstehend.'));
|
||||
table('Plan und Kosten', [['Plan-ID', plan.planId], ['Modellfamilie', plan.sourceFamily], ['Revision', plan.configRevision], ['Prognose bis', time(plan.forecastUntil)], ['Bekannte Preise bis', time(plan.pricesKnownUntil)], ['Energiekosten', num(plan.energyCostChf) + ' CHF'], ['Zusaetzliche Peakkosten zum vollen Tarif', num(plan.additionalPeakCostChf) + ' CHF'], ['Peakbewertung unter Restmonats-Szenarien', num(plan.planningPeakCostChf) + ' CHF'], ['Planung mit geschaetzter Peakbasis', plan.peakCostIsEstimate ? 'Ja – kein Abrechnungsnachweis' : 'Nein'], ['V4 steuert die Anlage', 'Nein']]);
|
||||
for (const warning of plan.warnings || []) panel.append(element('p', warning));
|
||||
for (const [month, basis] of Object.entries(plan.peakBasis || {})) table(`Peak ${month}`, [['Ausgangsbasis', num(basis.kw) + ' kW (' + basis.quality + ')'], ['Geplant', num(plan.plannedPeaksKw?.[month]) + ' kW'], ['Bewertung', plan.peakOutlook]]);
|
||||
curve('Netzleistung (W)', plan.points, [{ label: 'ohne Optimierung', value: p => p.baselineGridW }, { label: 'optimiert', value: p => p.gridTargetW }]);
|
||||
curve('Batterieleistung (W; positiv = Laden)', plan.points, [{ label: 'Batterieplan', value: p => p.batteryTargetW }]);
|
||||
for (const asset of Object.keys(plan.points[0]?.socEndPercent || {})) curve(`SOC ${asset} (%)`, plan.points, [{ label: 'geplanter SOC', value: p => p.socEndPercent?.[asset] }]);
|
||||
curve('Kumulierte Zahlungen inklusive vollem Peakpreis (CHF)', plan.points, [{ label: 'ohne Optimierung', value: p => p.cumulativeBaselineCashCostChf }, { label: 'optimiert', value: p => p.cumulativeCashCostChf }]);
|
||||
panel.append(element('p', 'Kostenverlaeufe sind Prognosen. Restwert der Batterie und Restmonatsbewertung sind keine bereits erzielten Erloese.'));
|
||||
}
|
||||
message.textContent = '';
|
||||
} catch (error) { if (ticket === generation) { panel.replaceChildren(); message.textContent = error.message; } }
|
||||
}
|
||||
try {
|
||||
session = await request('/api/session'); if (!session.authenticated) throw new Error('Bitte zuerst im Portal anmelden.');
|
||||
const data = await request('/api/plants');
|
||||
for (const plant of data.plants || []) { if (!plant.installation_id) continue; const option = element('option', plant.name); option.value = plant.id; plants.append(option); }
|
||||
if (!plants.options.length) throw new Error('Keine verknuepfte Anlage vorhanden.');
|
||||
plants.addEventListener('change', render); await render();
|
||||
} catch (error) { message.textContent = error.message; }
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Count-checked optional bridge installation. Default CHECK ONLY, no service action.
|
||||
Only dedicated V4 GUI files are added. app.js/index.html/styles.css stay untouched.
|
||||
Every changed legacy source is backed up with hashes. Concurrent changes abort.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from datetime import datetime,timezone
|
||||
import argparse,ast,hashlib,json,os,tempfile
|
||||
ROOT=Path(__file__).resolve().parent
|
||||
SERVICES=Path('/home/agent/services')
|
||||
|
||||
def once(text,old,new):
|
||||
if text.count(old)!=1:raise ValueError('Source anchor changed/ambiguous; review before updating')
|
||||
return text.replace(old,new,1)
|
||||
|
||||
def patch_forecast(text):
|
||||
if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text
|
||||
text='from netplan_v4_publisher import publish_forecasts as _v4_publish_forecasts\n'+text
|
||||
anchor=' p_3 = v3.predict(data_obj, p_1, p_2)'
|
||||
text=once(text,anchor,' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_forecasts(config, [(3,p_1,p_2),(13,p_10,p_11),(23,p_21,p_22)])\n\n'+anchor)
|
||||
text=once(text,' d[key] = float(d.get(key) or default)',' raw_value = d.get(key)\n d[key] = float(default if raw_value is None or raw_value == "" else raw_value)')
|
||||
ast.parse(text);return text
|
||||
|
||||
def patch_api(text):
|
||||
if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text
|
||||
text='from netplan_v4_publisher import publish_tariffs as _v4_publish_tariffs\n'+text
|
||||
anchor=' _portal_store_configuration(anlagen_id, configuration)\n'
|
||||
text=once(text,anchor,anchor+' # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_tariffs(anlagen_id, configuration)\n')
|
||||
ast.parse(text);return text
|
||||
|
||||
def patch_tariff(text):
|
||||
if '# ENELIX_V4_SHADOW_BRIDGE' in text:return text
|
||||
text='from netplan_v4_publisher import publish_ckw as _v4_publish_ckw\n'+text
|
||||
anchor=' return candidate_points\n'
|
||||
text=once(text,anchor," # ENELIX_V4_SHADOW_BRIDGE\n _v4_publish_ckw(source['label'], rows, data.get('publication_timestamp') if isinstance(data, dict) else None, tariff_type)\n"+anchor)
|
||||
for name in ('row','integrated'):
|
||||
old=f'{name}.get("value") or {name}.get("price") or {name}.get("amount")'
|
||||
if old in text:text=once(text,old,f'next(({name}[key] for key in ("value", "price", "amount") if {name}.get(key) is not None), None)')
|
||||
ast.parse(text);return text
|
||||
|
||||
def patch_portal(text):
|
||||
if '// ENELIX_V4_SHADOW_BRIDGE' in text:return text
|
||||
anchor='const server = createServer(async (req, res) => {'
|
||||
extra="""// ENELIX_V4_SHADOW_BRIDGE
|
||||
// No dependency or changed route while NETPLAN_V4_URL is absent.
|
||||
const plannerV4Bridge = process.env.NETPLAN_V4_URL
|
||||
? (await import('./netplan-v4-bridge.mjs')).createPlannerV4Bridge({
|
||||
configuredPrognosisPlant, roleAllowed, bodyJson, json, deviceActivation,
|
||||
checkDeviceRate, ownedLicenseState, serviceToken: prognosisServiceToken,
|
||||
upstreamUrl: process.env.NETPLAN_V4_URL
|
||||
})
|
||||
: null;
|
||||
|
||||
"""
|
||||
text=once(text,anchor,extra+anchor)
|
||||
anchor=' if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {'
|
||||
return once(text,anchor,' if (plannerV4Bridge && await plannerV4Bridge(req, res, url)) return;\n'+anchor)
|
||||
|
||||
def plan(root=SERVICES, approved_assets=None):
|
||||
root=Path(root).resolve();project=root/'prognosis-manager-enelix2';result={}
|
||||
if approved_assets is None:
|
||||
manifest=ROOT/'approved_previous_assets.json'
|
||||
approved_assets=json.loads(manifest.read_text()) if manifest.is_file() else {}
|
||||
for path,patch in [(project/'api/main.py',patch_api),(project/'forecast_engine/main.py',patch_forecast),(project/'tariff_importer/main.py',patch_tariff),(root/'license/server.mjs',patch_portal)]:
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused')
|
||||
old=path.read_bytes();new=patch(old.decode()).encode()
|
||||
if old!=new:result[path]=(old,new)
|
||||
files={project/service/'netplan_v4_publisher.py':ROOT/'integrations/netplan_v4_publisher.py' for service in ('api','forecast_engine','tariff_importer')}
|
||||
files[root/'license/netplan-v4-bridge.mjs']=ROOT/'integrations/netplan-v4-bridge.mjs'
|
||||
for name in ('netplan-v4.html','netplan-v4.js','netplan-v4.css'):files[root/'license/public'/name]=ROOT/'gui'/name
|
||||
for path,source in files.items():
|
||||
if path.is_symlink() or not path.resolve().is_relative_to(root):raise ValueError('External/symlink target refused')
|
||||
old=path.read_bytes() if path.exists() else None;new=source.read_bytes()
|
||||
if old is not None and old!=new:
|
||||
approved=approved_assets.get(str(path.relative_to(root)),[])
|
||||
if hashlib.sha256(old).hexdigest() not in approved:
|
||||
raise ValueError('Existing V4 file differs from the reviewed previous release; explicit merge required: '+str(path))
|
||||
if old!=new:result[path]=(old,new)
|
||||
return result
|
||||
|
||||
def atomic(path,data,mode=0o644):
|
||||
path.parent.mkdir(parents=True,exist_ok=True);fd,tmp=tempfile.mkstemp(prefix=path.name+'.v4-',dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd,'wb') as out:out.write(data);out.flush();os.fsync(out.fileno())
|
||||
os.chmod(tmp,mode);os.replace(tmp,path)
|
||||
finally:
|
||||
if os.path.exists(tmp):os.unlink(tmp)
|
||||
|
||||
def apply(entries,root=SERVICES):
|
||||
root=Path(root).resolve()
|
||||
for path,(old,new) in entries.items():
|
||||
if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent change: no files modified')
|
||||
if not entries:return None
|
||||
backup=root/'change-backups'/'netplan-v4'/datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')
|
||||
backup.mkdir(parents=True);receipt={'root':str(root),'files':[]};written=[]
|
||||
try:
|
||||
for path,(old,new) in entries.items():
|
||||
if (path.read_bytes() if path.exists() else None)!=old:raise ValueError('Concurrent source change during install')
|
||||
rel=path.relative_to(root);mode=path.stat().st_mode&0o777 if path.exists() else 0o644
|
||||
if old is not None:
|
||||
original=backup/'originals'/rel;original.parent.mkdir(parents=True,exist_ok=True);original.write_bytes(old)
|
||||
atomic(path,new,mode);written.append(path)
|
||||
receipt['files'].append({'path':str(rel),'before':hashlib.sha256(old).hexdigest() if old is not None else None,'after':hashlib.sha256(new).hexdigest(),'mode':mode})
|
||||
atomic(backup/'receipt.json',json.dumps(receipt,indent=2).encode());return backup/'receipt.json'
|
||||
except Exception:
|
||||
for path in reversed(written):
|
||||
old,new=entries[path]
|
||||
if path.read_bytes()==new:
|
||||
if old is None:path.unlink()
|
||||
else:atomic(path,old)
|
||||
raise
|
||||
|
||||
def rollback(receipt_path):
|
||||
path=Path(receipt_path).resolve();data=json.loads(path.read_text());root=Path(data['root']).resolve()
|
||||
for item in data['files']:
|
||||
target=(root/item['path']).resolve()
|
||||
if not target.is_relative_to(root) or hashlib.sha256(target.read_bytes()).hexdigest()!=item['after']:raise ValueError('Later change detected; automatic rollback refused')
|
||||
for item in reversed(data['files']):
|
||||
target=root/item['path']
|
||||
if item['before'] is None:target.unlink()
|
||||
else:
|
||||
original=(path.parent/'originals'/item['path']).read_bytes()
|
||||
if hashlib.sha256(original).hexdigest()!=item['before']:raise ValueError('Backup checksum mismatch')
|
||||
atomic(target,original,item['mode'])
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--apply',action='store_true');parser.add_argument('--rollback');args=parser.parse_args()
|
||||
if args.rollback:rollback(args.rollback);print('Source rollback completed. No service restart.')
|
||||
else:
|
||||
entries=plan()
|
||||
for path in entries:print(path)
|
||||
print('Receipt:',apply(entries)) if args.apply else print('CHECK ONLY:',len(entries),'files. No source/database/service changed.')
|
||||
@@ -0,0 +1,71 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
namespace Belevo\EnelixEMS;
|
||||
use InvalidArgumentException;
|
||||
use DateTimeImmutable;
|
||||
|
||||
/** Pure protocol/control helper. No IPS calls, no device output. */
|
||||
final class NetzfahrplanV4
|
||||
{
|
||||
private static function stamp(string $value): int
|
||||
{
|
||||
if (!preg_match('/(?:Z|[+-]\d{2}:\d{2})$/', $value)) {
|
||||
throw new InvalidArgumentException('Explicit timezone required');
|
||||
}
|
||||
return (new DateTimeImmutable($value))->getTimestamp();
|
||||
}
|
||||
private static function number(mixed $value): float
|
||||
{
|
||||
if ((!is_float($value) && !is_int($value)) || !is_finite((float)$value)) {
|
||||
throw new InvalidArgumentException('Finite numeric value required');
|
||||
}
|
||||
return (float)$value;
|
||||
}
|
||||
public static function currentPoint(array $plan, int $now, int $revision, bool $shadowInspection=false): ?array
|
||||
{
|
||||
if (($plan['schemaVersion']??null)!==2 || ($plan['configRevision']??null)!==$revision
|
||||
|| ($plan['executable']??false)!==true || !is_array($plan['points']??null)
|
||||
|| !in_array($plan['status']??'', ['optimal','feasible_time_limit'], true)
|
||||
|| !is_string($plan['planId']??null) || $plan['planId']==='') return null;
|
||||
if ($shadowInspection ? ($plan['runMode']??'')!=='shadow'
|
||||
: (($plan['runMode']??'')!=='live' || ($plan['liveEnabled']??false)!==true)) return null;
|
||||
try {
|
||||
$age=$now-self::stamp($plan['generatedAt']);
|
||||
if ($age < -30 || $age>900 || self::stamp($plan['validFrom'])>$now || self::stamp($plan['validUntil'])<=$now) return null;
|
||||
$selected=null; $previousEnd=null;
|
||||
foreach ($plan['points'] as $p) {
|
||||
$start=self::stamp($p['time']); $end=self::stamp($p['validUntil']);
|
||||
self::number($p['gridTargetW']);
|
||||
if ($end<=$start || $end-$start>300 || ($previousEnd!==null && $previousEnd!==$start)) return null;
|
||||
$previousEnd=$end;
|
||||
if ($start<=$now && $now<$end) $selected=$p;
|
||||
}
|
||||
return $selected;
|
||||
} catch (\Throwable $e) { return null; }
|
||||
}
|
||||
public static function correction(array $point, float $grid, float $battery, float $chargeAvailable, float $dischargeAvailable, ?float $hardImportLimit=null): array
|
||||
{
|
||||
foreach ([$grid,$battery,$chargeAvailable,$dischargeAvailable] as $v) self::number($v);
|
||||
if ($chargeAvailable<0 || $dischargeAvailable<0) throw new InvalidArgumentException('Invalid available power');
|
||||
$target=self::number($point['gridTargetW']??null);
|
||||
if ($hardImportLimit!==null) {
|
||||
if (self::number($hardImportLimit)<0) throw new InvalidArgumentException('Negative hard limit');
|
||||
$target=min($target,$hardImportLimit);
|
||||
}
|
||||
$raw=$battery+$target-$grid;
|
||||
if (abs($target)>1) {
|
||||
$raw=match ($point['intent']??'') {
|
||||
'gridCharge'=>max(0.,$raw),
|
||||
'pvCharge'=>max(0.,min($battery-$grid,$raw)),
|
||||
'discharge','export'=>min(0.,$raw),
|
||||
'hold'=>0.,
|
||||
default=>throw new InvalidArgumentException('Unknown plan intent'),
|
||||
};
|
||||
}
|
||||
if ($hardImportLimit!==null) $raw=min($raw,$battery+$hardImportLimit-$grid);
|
||||
$wanted=min($chargeAvailable,max(-$dischargeAvailable,$raw));
|
||||
$expectedGrid=$grid+$wanted-$battery;
|
||||
return ['batteryTargetW'=>$wanted,'gridTargetW'=>$target,'limited'=>abs($wanted-$raw)>1,
|
||||
'expectedGridW'=>$expectedGrid,'trackingErrorW'=>$expectedGrid-$target];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/** Scoped opt-in proxy. Never intercept the existing V1 schedule. */
|
||||
export function createPlannerV4Bridge({configuredPrognosisPlant,roleAllowed,bodyJson,json,
|
||||
deviceActivation,checkDeviceRate,ownedLicenseState,serviceToken,
|
||||
upstreamUrl='http://netplan-v4:9100',fetchImpl=fetch}){
|
||||
const base=new URL(upstreamUrl);
|
||||
if(!['http:','https:'].includes(base.protocol)||base.username||base.password)throw new Error('Invalid V4 URL');
|
||||
async function forward(req,res,installation,suffix,method){
|
||||
if(!serviceToken){json(res,503,{error:'V4 ist nicht angebunden.'});return;}
|
||||
const url=new URL(`/internal/v2/prognosis/${encodeURIComponent(installation)}/planner${suffix}`,base);
|
||||
let response;
|
||||
const payload=method==='GET'?undefined:await bodyJson(req);
|
||||
try{response=await fetchImpl(url,{method,headers:{'X-Enelix-Service-Token':serviceToken,'Content-Type':'application/json'},body:payload===undefined?undefined:JSON.stringify(payload),signal:AbortSignal.timeout(10000),redirect:'error'});}
|
||||
catch{json(res,503,{error:'V4 nicht erreichbar. Bestehende Regelung bleibt unveraendert.'});return;}
|
||||
const data=await response.json().catch(()=>null);
|
||||
if(!response.ok){const code=[400,401,403,404,409,413].includes(response.status)?response.status:503;json(res,code,{error:code===409?'Versionskonflikt: Ansicht neu laden.':'V4-Anfrage abgelehnt oder Dienst nicht verfuegbar.'});return;}
|
||||
if(!data||typeof data!=='object'){json(res,503,{error:'Ungueltige V4-Antwort.'});return;}
|
||||
json(res,200,data);
|
||||
}
|
||||
return async function(req,res,url){
|
||||
const c=url.pathname.match(/^\/api\/plants\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(settings|replan))?$/i);
|
||||
if(c){
|
||||
const action=c[2]||'',method={'':'GET',settings:'PUT',replan:'POST'}[action];
|
||||
if(req.method!==method){json(res,405,{error:'Methode nicht erlaubt.'});return true;}
|
||||
const access=configuredPrognosisPlant(req,res,c[1],method!=='GET');if(!access)return true;
|
||||
if(Number(access.license.quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;}
|
||||
if(method!=='GET'&&!roleAllowed(access.session,['owner','admin','operator'])){json(res,403,{error:'Keine Aenderungsberechtigung.'});return true;}
|
||||
await forward(req,res,access.plant.installation_id,action?`/${action}`:'',method);return true;
|
||||
}
|
||||
const d=url.pathname.match(/^\/api\/v1\/installations\/([0-9a-f-]{36})\/prognosis\/planner-v4(?:\/(operation|ack|measurements))?$/i);
|
||||
if(d){
|
||||
const action=d[2]||'';if(req.method!==(action?'POST':'GET')){json(res,405,{error:'Methode nicht erlaubt.'});return true;}
|
||||
if(!checkDeviceRate(req,d[1],'planner-v4')){json(res,429,{error:'Zu viele Anfragen.'});return true;}
|
||||
const activation=deviceActivation(req,res,d[1]);if(!activation)return true;
|
||||
if(Number(ownedLicenseState(activation.plant_id).quantities.grid_schedule||0)<1){json(res,403,{error:'Netzfahrplan nicht lizenziert.'});return true;}
|
||||
await forward(req,res,d[1],action==='operation'?'/inputs/operation':action==='measurements'?'/measurements':action?'/ack':'',req.method);return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Stdlib-only, explicit opt-in publisher for the existing ENELIX services.
|
||||
No credentials are printed. Existing V1 computation survives shadow-service errors.
|
||||
Legacy UTC-naive forecast indices are explicitly interpreted as UTC here.
|
||||
"""
|
||||
from datetime import datetime,timezone
|
||||
from hashlib import sha256
|
||||
from uuid import uuid4
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request,urlopen
|
||||
import json,logging,math,os
|
||||
|
||||
def timestamp(value):
|
||||
if hasattr(value,'to_pydatetime'):value=value.to_pydatetime()
|
||||
if isinstance(value,str):value=datetime.fromisoformat(value.replace('Z','+00:00'))
|
||||
if value.tzinfo is None:value=value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat()
|
||||
|
||||
def tariff_id(label):
|
||||
if not isinstance(label,str) or not label.strip():raise ValueError('Exact tariff label required')
|
||||
return 'legacy:'+sha256(label.encode()).hexdigest()[:32]
|
||||
|
||||
def envelope(at=None):return {'version':1,'eventId':str(uuid4()),'observedAt':timestamp(at or datetime.now(timezone.utc))}
|
||||
|
||||
def tariff_payload(config,at=None):
|
||||
result=envelope(at)
|
||||
for side,prefix in (('import','tarif_bezug'),('export','tarif_einspeisung')):
|
||||
mode={'statisch':'static','dynamisch':'dynamic','static':'static','dynamic':'dynamic'}.get(config.get(prefix+'_modus'))
|
||||
if mode is None:raise ValueError('Explicit tariff mode required; not inferred from name')
|
||||
item={'mode':mode,'tariffId':tariff_id(config[prefix])}
|
||||
if mode=='static':
|
||||
value=config.get(prefix+'_fest')
|
||||
if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Static price missing/invalid')
|
||||
item['staticChfKwh']=float(value)
|
||||
result[side]=item
|
||||
peak=config.get('tarif_peak_fest')
|
||||
if peak is None or isinstance(peak,bool) or not math.isfinite(float(peak)) or float(peak)<0:raise ValueError('Explicit peak price required')
|
||||
result['peakChfKwMonth']=float(peak)
|
||||
return result
|
||||
|
||||
def forecast_payload(forecasts,at=None,load_basis='house_total',trained_until=None):
|
||||
result=envelope(at);result['families']={}
|
||||
for key,pv,load in forecasts:
|
||||
if not pv or not load:continue
|
||||
if set(pv)!=set(load):raise ValueError('PV/load timestamps differ')
|
||||
result['families'][str(key)]={'loadBasis':load_basis,'trainedUntil':trained_until,
|
||||
'points':[{'time':timestamp(t),'pvW':float(pv[t]),'loadW':float(load[t])} for t in sorted(pv)]}
|
||||
if not result['families']:raise ValueError('No forecast families supplied')
|
||||
return result
|
||||
|
||||
def ckw_payload(label,rows,publication_timestamp=None,at=None):
|
||||
"""Provider-delimited integrated price intervals only. No scalar price replication."""
|
||||
result=envelope(at);periods=[]
|
||||
units={'CHF_kWh':'CHF/kWh','CHF/kWh':'CHF/kWh','Rp/kWh':'Rp/kWh','CHF/MWh':'CHF/MWh'}
|
||||
for row in rows:
|
||||
if not row.get('start_timestamp') or not row.get('end_timestamp'):raise ValueError('Explicit delivery start/end required')
|
||||
integrated=row.get('integrated')
|
||||
if isinstance(integrated,list):
|
||||
if len(integrated)!=1:raise ValueError('Ambiguous integrated price components')
|
||||
integrated=integrated[0]
|
||||
if not isinstance(integrated,dict) or integrated.get('unit') not in units:raise ValueError('Provider-declared unit required')
|
||||
value=integrated.get('value')
|
||||
if value is None or isinstance(value,bool) or not math.isfinite(float(value)):raise ValueError('Finite integrated price required')
|
||||
item={'tariffId':tariff_id(label),'side':'import','start':timestamp(row['start_timestamp']),
|
||||
'end':timestamp(row['end_timestamp']),'value':float(value),'unit':units[integrated['unit']],
|
||||
'observedAt':result['observedAt'],'sourceKind':'published_interval'}
|
||||
if publication_timestamp:
|
||||
published=timestamp(publication_timestamp)
|
||||
if datetime.fromisoformat(published)>datetime.fromisoformat(result['observedAt']):raise ValueError('Future publication')
|
||||
item['publishedAt']=published
|
||||
periods.append(item)
|
||||
result['periods']=periods
|
||||
return result
|
||||
|
||||
def enabled(plant):return bool(os.getenv('NETPLAN_V4_URL') and plant in {p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()})
|
||||
|
||||
def send(plant,kind,payload):
|
||||
if not enabled(plant):return {'status':'disabled'}
|
||||
token=os.getenv('PROGNOSIS_SERVICE_TOKEN','');base=os.environ['NETPLAN_V4_URL'].rstrip('/');url=urlparse(base)
|
||||
if not token or url.scheme not in ('http','https') or url.username or url.password:raise ValueError('Private V4 transport not configured')
|
||||
UUID=__import__('uuid').UUID;UUID(plant)
|
||||
if kind not in ('forecast','tariffs','prices'):raise ValueError('Unsupported publisher input kind')
|
||||
req=Request(base+'/internal/v2/prognosis/'+plant+'/planner/inputs/'+kind,
|
||||
json.dumps(payload,allow_nan=False).encode(),{'Content-Type':'application/json','X-Enelix-Service-Token':token},method='POST')
|
||||
with urlopen(req,timeout=5) as response:
|
||||
data=response.read(65537)
|
||||
if len(data)>65536:raise ValueError('Oversized service response')
|
||||
return json.loads(data)
|
||||
|
||||
def publish_forecasts(config,forecasts):
|
||||
plant=config['anlagen_id']
|
||||
if not enabled(plant):return
|
||||
try:
|
||||
send(plant,'tariffs',tariff_payload(config));send(plant,'forecast',forecast_payload(forecasts))
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 shadow forecast for %s: %s',plant,type(exc).__name__)
|
||||
|
||||
def publish_tariffs(plant,config):
|
||||
if not enabled(plant):return
|
||||
try:send(plant,'tariffs',tariff_payload(config))
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 shadow tariffs for %s: %s',plant,type(exc).__name__)
|
||||
|
||||
def publish_ckw(label,rows,publication_timestamp=None,tariff_type='integrated'):
|
||||
plants=[p.strip() for p in os.getenv('NETPLAN_V4_PLANTS','').split(',') if p.strip()]
|
||||
if not os.getenv('NETPLAN_V4_URL') or not plants:return
|
||||
try:
|
||||
if tariff_type!='integrated':raise ValueError('Full integrated tariff required')
|
||||
payload=ckw_payload(label,rows,publication_timestamp)
|
||||
for plant in plants:send(plant,'prices',payload)
|
||||
except Exception as exc:logging.getLogger(__name__).warning('V4 CKW price provenance: %s',type(exc).__name__)
|
||||
@@ -0,0 +1 @@
|
||||
"""ENELIX V4 shadow planner. No live actuator interface."""
|
||||
@@ -0,0 +1,44 @@
|
||||
"""SOC-aware battery model, including reserve recovery and hysteresis rearming."""
|
||||
from math import sqrt
|
||||
|
||||
|
||||
class BatteryModel:
|
||||
@staticmethod
|
||||
def build(model,battery,steps,direction):
|
||||
cap=battery.capacity_kwh;eta=sqrt(battery.roundtrip_efficiency)
|
||||
reserve=cap*battery.min_soc_percent/100
|
||||
initial=cap*battery.soc_percent/100
|
||||
lower=min(initial,reserve) if battery.recovery_allowed else reserve
|
||||
upper=cap*battery.max_soc_percent/100
|
||||
energies=[model.variable(lower,upper) for _ in range(len(steps)+1)]
|
||||
model.constraint({energies[0]:1},initial,initial)
|
||||
terminal=battery.terminal_soc_min_percent
|
||||
if terminal is None:terminal=max(battery.soc_percent,battery.min_soc_percent)
|
||||
model.constraint({energies[-1]:1},cap*terminal/100)
|
||||
model.cost[energies[-1]]=-battery.terminal_value_chf_kwh
|
||||
rearm=cap*(battery.rearm_soc_percent if battery.rearm_soc_percent is not None else battery.min_soc_percent)/100
|
||||
enabled=[model.variable(0,1,integer=True) for _ in steps]
|
||||
# Only a numerical threshold, not an undisclosed extra operating reserve.
|
||||
epsilon=min(1e-5,max(0.,upper-reserve)/1000)
|
||||
initially_enabled=not battery.discharge_blocked and initial>=reserve+max(epsilon,1e-8)
|
||||
model.constraint({enabled[0]:1},int(initially_enabled),int(initially_enabled))
|
||||
big=max(upper-lower+epsilon,1.)
|
||||
charge,discharge=[],[]
|
||||
for i,step in enumerate(steps):
|
||||
dt=step.seconds/3600;cmax=battery.max_charge_w/1000;dmax=battery.max_discharge_w/1000
|
||||
c=model.variable(0,cmax,battery.throughput_chf_kwh*dt)
|
||||
d=model.variable(0,dmax,battery.throughput_chf_kwh*dt)
|
||||
charge.append(c);discharge.append(d)
|
||||
model.constraint({c:1,direction[i]:-cmax},upper=0)
|
||||
model.constraint({d:1,direction[i]:dmax},upper=dmax)
|
||||
model.constraint({d:1,enabled[i]:-dmax},upper=0)
|
||||
model.constraint({energies[i]:1,enabled[i]:-big},lower=reserve+epsilon-big)
|
||||
# Once enabled, discharge may consume only energy above reserve.
|
||||
model.constraint({energies[i+1]:1,enabled[i]:-big},lower=reserve-big)
|
||||
if i:
|
||||
# A disabled battery can rearm only AFTER prior charging has
|
||||
# actually reached the configured hysteresis threshold.
|
||||
model.constraint({energies[i]:1,enabled[i]:-big,enabled[i-1]:big},lower=rearm-big)
|
||||
model.constraint({energies[i+1]:1,energies[i]:-1,c:-eta*dt,d:dt/eta},0,0)
|
||||
return {'charge':charge,'discharge':discharge,'energy':energies,
|
||||
'terminal_min_percent':terminal,'discharge_enabled':enabled}
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Explicit, time-limited commissioning authority; NEVER enables shadow execution.
|
||||
|
||||
Only a reviewed internal operator call can arm a trial, and only for a separate
|
||||
allowlist (empty by default). The manager and battery must additionally consent
|
||||
locally. Existing shadow plans/acknowledgements keep their original meaning.
|
||||
"""
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
import json
|
||||
|
||||
MAX_SECONDS = 1800
|
||||
MAX_POWER_W = 5000
|
||||
AUTHORITY_TTL_SECONDS = 90
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.execute('''CREATE TABLE IF NOT EXISTS planner_controlled_trials(
|
||||
plant TEXT PRIMARY KEY, session_id TEXT NOT NULL UNIQUE,
|
||||
value TEXT NOT NULL, revoked_at TEXT)''')
|
||||
|
||||
|
||||
def timestamp(v):
|
||||
if not isinstance(v, str):
|
||||
raise ValueError('Explicit timestamp required')
|
||||
t = datetime.fromisoformat(v.replace('Z', '+00:00'))
|
||||
if t.tzinfo is None:
|
||||
raise ValueError('Timezone required')
|
||||
return t.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def uuid(v):
|
||||
if not isinstance(v, str) or str(UUID(v)) != v:
|
||||
raise ValueError('Canonical UUID required')
|
||||
return v
|
||||
|
||||
|
||||
def power(v):
|
||||
if type(v) not in (int, float) or not 0 < v <= MAX_POWER_W:
|
||||
raise ValueError('Pilot limit must be explicit and at most 5000 W')
|
||||
return float(v)
|
||||
|
||||
|
||||
def accounting(plan):
|
||||
# A human checkbox alone must not relabel aggregate house consumption.
|
||||
quality = plan.get('inputQuality', {})
|
||||
if quality.get('loadBasis') != 'base_load':
|
||||
raise ValueError('Verified base-load/SDL adapter required before a control trial')
|
||||
evidence = quality.get('accountingEvidenceId')
|
||||
if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160:
|
||||
raise ValueError('Plan lacks traceable base-load accounting evidence')
|
||||
return evidence
|
||||
|
||||
|
||||
def eligibility(view, plant):
|
||||
if view.get('installationId') != plant or view.get('liveEnabled') is not False:
|
||||
raise ValueError('Wrong plant or service mode')
|
||||
p = view.get('plan')
|
||||
if view.get('fresh') is not True or not isinstance(p, dict):
|
||||
raise ValueError('Fresh independently validated planning input required')
|
||||
if (p.get('installationId') != plant or p.get('runMode') != 'shadow'
|
||||
or p.get('liveEnabled') is not False or p.get('executable') is not True):
|
||||
raise ValueError('Wrong source plan identity or mode')
|
||||
settings = view.get('settings', {})
|
||||
if settings.get('family') == 'auto' or p.get('sourceFamily') != settings.get('family'):
|
||||
raise ValueError('First controlled trial requires a fixed model family')
|
||||
if p.get('configRevision') != settings.get('revision'):
|
||||
raise ValueError('Configuration revision changed')
|
||||
if len(p.get('controlContext', {}).get('batteries', {})) != 1:
|
||||
raise ValueError('First controlled trial supports exactly one battery')
|
||||
accounting(p)
|
||||
return p
|
||||
|
||||
|
||||
def arm(store, plant, request, view, now, allowed_plants):
|
||||
"""Caller is authenticated internal operator; not exposed via device proxy."""
|
||||
if plant not in allowed_plants:
|
||||
raise ValueError('Controlled trial disabled by server allowlist')
|
||||
fields = {'sessionId', 'expectedPlanId', 'expectedRevision', 'durationSeconds',
|
||||
'maxChargeW', 'maxDischargeW', 'assetId', 'managerId', 'batteryInstanceId',
|
||||
'acceptEstimatedPeak', 'actuatorWatchdogEvidenceId', 'confirmation'}
|
||||
if set(request) != fields or request['confirmation'] != 'ARM_BOUNDED_CONTROL_TRIAL':
|
||||
raise ValueError('Explicit reviewed commissioning request required')
|
||||
session = uuid(request['sessionId']); p = eligibility(view, plant)
|
||||
if type(request['expectedRevision']) is not int or request['expectedRevision'] < 0:
|
||||
raise ValueError('Expected revision must be an integer')
|
||||
if request['expectedPlanId'] != p['planId'] or request['expectedRevision'] != p['configRevision']:
|
||||
raise ValueError('Source plan/revision changed; review again')
|
||||
seconds = request['durationSeconds']
|
||||
if type(seconds) is not int or not 30 <= seconds <= MAX_SECONDS:
|
||||
raise ValueError('Trial duration must be 30..1800 seconds')
|
||||
for k in ('managerId', 'batteryInstanceId'):
|
||||
if type(request[k]) is not int or not 1 <= request[k] <= 99999:
|
||||
raise ValueError('Explicit local instance binding required')
|
||||
if request['managerId'] == request['batteryInstanceId']:
|
||||
raise ValueError('Manager and battery instance must differ')
|
||||
if request['assetId'] not in p['controlContext']['batteries']:
|
||||
raise ValueError('Wrong trial battery')
|
||||
if type(request['acceptEstimatedPeak']) is not bool:
|
||||
raise ValueError('Explicit estimated-peak policy required')
|
||||
if p.get('peakCostIsEstimate') and not request['acceptEstimatedPeak']:
|
||||
raise ValueError('Estimated peak has not been accepted for this trial')
|
||||
evidence = request['actuatorWatchdogEvidenceId']
|
||||
if not isinstance(evidence, str) or not 8 <= len(evidence) <= 160:
|
||||
raise ValueError('Device-side command-loss watchdog proof required')
|
||||
value = {'kind': 'controlled_trial_grant', 'version': 1, 'sessionId': session,
|
||||
'installationId': plant, 'issuedAt': now.isoformat(),
|
||||
'expiresAt': (now + timedelta(seconds=seconds)).isoformat(),
|
||||
'revision': p['configRevision'], 'family': p['sourceFamily'],
|
||||
'assetId': request['assetId'], 'managerId': request['managerId'],
|
||||
'batteryInstanceId': request['batteryInstanceId'],
|
||||
'maxChargeW': power(request['maxChargeW']),
|
||||
'maxDischargeW': power(request['maxDischargeW']),
|
||||
'acceptEstimatedPeak': request['acceptEstimatedPeak'],
|
||||
'accountingEvidenceId': accounting(p),
|
||||
'actuatorWatchdogEvidenceId': evidence,
|
||||
'controlContext': deepcopy(p['controlContext'])}
|
||||
with store.con:
|
||||
existing = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone()
|
||||
if existing and existing[1] is None and timestamp(json.loads(existing[0])['expiresAt']) > now:
|
||||
raise ValueError('Existing trial must first end; implicit extension forbidden')
|
||||
# Reusing an expired/revoked session ID must never resurrect it.
|
||||
used = store.con.execute("SELECT 1 FROM planner_audit WHERE plant=? AND kind='trial_arm' AND detail=?", (plant, session)).fetchone()
|
||||
if used:
|
||||
raise ValueError('Session ID already used')
|
||||
store.con.execute('INSERT INTO planner_controlled_trials VALUES(?,?,?,NULL) ON CONFLICT(plant) DO UPDATE SET session_id=excluded.session_id,value=excluded.value,revoked_at=NULL',
|
||||
(plant, session, json.dumps(value, sort_keys=True, allow_nan=False)))
|
||||
store.con.execute("INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,'trial_arm',?)", (plant, now.isoformat(), session))
|
||||
return value
|
||||
|
||||
|
||||
def revoke(store, plant, session, now):
|
||||
uuid(session)
|
||||
with store.con:
|
||||
store.con.execute('UPDATE planner_controlled_trials SET revoked_at=? WHERE plant=? AND session_id=?', (now.isoformat(), plant, session))
|
||||
return {'status': 'revoked', 'sessionId': session, 'remoteRevocationMaxSeconds': AUTHORITY_TTL_SECONDS}
|
||||
|
||||
|
||||
def authority(store, plant, view, now, allowed_plants):
|
||||
"""Separate authorization envelope, not a mutation of the shadow plan."""
|
||||
if plant not in allowed_plants:
|
||||
return None
|
||||
row = store.con.execute('SELECT value,revoked_at FROM planner_controlled_trials WHERE plant=?', (plant,)).fetchone()
|
||||
if not row or row[1] is not None:
|
||||
return None
|
||||
grant = json.loads(row[0])
|
||||
try:
|
||||
p = eligibility(view, plant)
|
||||
if not timestamp(grant['issuedAt']) <= now < timestamp(grant['expiresAt']):
|
||||
return None
|
||||
if (p['configRevision'] != grant['revision'] or p['sourceFamily'] != grant['family']
|
||||
or p['controlContext'] != grant['controlContext']
|
||||
or accounting(p) != grant['accountingEvidenceId']
|
||||
or p.get('peakCostIsEstimate') and not grant['acceptEstimatedPeak']):
|
||||
return None
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
until = min(timestamp(grant['expiresAt']), timestamp(p['validUntil']),
|
||||
now + timedelta(seconds=AUTHORITY_TTL_SECONDS))
|
||||
return {**grant, 'kind': 'controlled_trial_authority',
|
||||
'sourceShadowPlanId': p['planId'], 'checkedAt': now.isoformat(),
|
||||
'validUntil': until.isoformat(), 'sourcePlanRemainsShadow': True}
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from math import isfinite
|
||||
from typing import Mapping
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
UTC = timezone.utc
|
||||
ZURICH = ZoneInfo('Europe/Zurich')
|
||||
|
||||
def utc(value):
|
||||
if isinstance(value, str):
|
||||
value = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError('Timezone required')
|
||||
return value.astimezone(UTC)
|
||||
|
||||
def number(value, name, minimum=None, maximum=None):
|
||||
if isinstance(value, bool) or not isinstance(value, (float, int)):
|
||||
raise ValueError(f'{name}: finite number required')
|
||||
value = float(value)
|
||||
if not isfinite(value) or minimum is not None and value < minimum or maximum is not None and value > maximum:
|
||||
raise ValueError(f'{name}: outside permitted range')
|
||||
return value
|
||||
|
||||
def quarter_start(value):
|
||||
value = utc(value)
|
||||
return value.replace(minute=value.minute // 15 * 15, second=0, microsecond=0)
|
||||
|
||||
def month_key(value):
|
||||
return utc(value).astimezone(ZURICH).strftime('%Y-%m')
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Family:
|
||||
key: str
|
||||
pv: str
|
||||
load: str
|
||||
schedule: str
|
||||
label: str
|
||||
|
||||
class FamilyRegistry:
|
||||
def __init__(self, families=()):
|
||||
self._families = {}
|
||||
for family in families:
|
||||
self.register(family)
|
||||
def register(self, family):
|
||||
if family.key in self._families or any(f.schedule == family.schedule for f in self._families.values()):
|
||||
raise ValueError('Duplicate family or schedule identifier')
|
||||
self._families[family.key] = family
|
||||
def get(self, key):
|
||||
if key not in self._families:
|
||||
raise ValueError(f'Unknown model family: {key}')
|
||||
return self._families[key]
|
||||
def entries(self):
|
||||
return tuple(self._families.values())
|
||||
|
||||
def default_registry():
|
||||
return FamilyRegistry([
|
||||
Family('3','prog_var_1','prog_var_2','prog_var_3','Variante 1 (1 / 2 / 3)'),
|
||||
Family('13','prog_var_10','prog_var_11','prog_var_13','Variante 2 (10 / 11 / 13)'),
|
||||
Family('23','prog_var_21','prog_var_22','prog_var_23','Variante 3 (21 / 22 / 23)'),
|
||||
])
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Price:
|
||||
chf_kwh: float
|
||||
published_at: datetime | None = None
|
||||
mode: str = 'static'
|
||||
def known_at(self, at):
|
||||
number(self.chf_kwh, 'energy price')
|
||||
if self.mode not in ('static','dynamic'):
|
||||
raise ValueError('Explicit static/dynamic price mode required')
|
||||
return self.mode == 'static' or self.published_at is not None and utc(self.published_at) <= utc(at)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Step:
|
||||
start: datetime
|
||||
base_load_w: float
|
||||
pv_w: float
|
||||
import_price: Price | None
|
||||
export_price: Price | None
|
||||
external_w: float = 0.0
|
||||
seconds: int = 300
|
||||
@property
|
||||
def end(self):
|
||||
return utc(self.start) + timedelta(seconds=self.seconds)
|
||||
@property
|
||||
def residual_w(self):
|
||||
return self.base_load_w + self.external_w - self.pv_w
|
||||
|
||||
def split_base_load(measured_house_w, flexible_w, external_w=0.0, external_already_removed=False):
|
||||
measured = number(measured_house_w, 'measured_house_w')
|
||||
flex = sum(number(v, 'flexible measurement') for v in flexible_w)
|
||||
external = number(external_w, 'external measurement')
|
||||
base = measured - flex - (0.0 if external_already_removed else external)
|
||||
if base < -1.0:
|
||||
raise ValueError('Negative base load: inconsistent measurement boundary or double subtraction')
|
||||
return max(0.0, base)
|
||||
|
||||
def priced_prefix(steps, at):
|
||||
result = []
|
||||
for step in steps:
|
||||
if not step.import_price or not step.export_price:
|
||||
break
|
||||
if not step.import_price.known_at(at) or not step.export_price.known_at(at):
|
||||
break
|
||||
result.append(step)
|
||||
while result and result[-1].end != quarter_start(result[-1].end):
|
||||
result.pop()
|
||||
return result
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Battery:
|
||||
asset_id: str
|
||||
capacity_kwh: float
|
||||
soc_percent: float
|
||||
min_soc_percent: float
|
||||
max_soc_percent: float
|
||||
max_charge_w: float
|
||||
max_discharge_w: float
|
||||
measured_at: datetime
|
||||
roundtrip_efficiency: float = 0.90
|
||||
grid_charging: bool = False
|
||||
throughput_chf_kwh: float = 0.0
|
||||
terminal_soc_min_percent: float | None = None
|
||||
terminal_value_chf_kwh: float = 0.0
|
||||
recovery_allowed: bool = False
|
||||
physical_min_soc_percent: float = 0.0
|
||||
discharge_blocked: bool = False
|
||||
rearm_soc_percent: float | None = None
|
||||
def validate(self, at):
|
||||
if not self.asset_id:
|
||||
raise ValueError('Battery asset_id required')
|
||||
number(self.capacity_kwh, 'capacity_kwh', 0.001)
|
||||
lo = number(self.min_soc_percent, 'min_soc_percent', 0, 100)
|
||||
hi = number(self.max_soc_percent, 'max_soc_percent', lo, 100)
|
||||
physical=number(self.physical_min_soc_percent,'physical minimum SOC',0,lo)
|
||||
if type(self.recovery_allowed) is not bool or type(self.discharge_blocked) is not bool:
|
||||
raise ValueError('Explicit battery recovery and hysteresis flags required')
|
||||
number(self.soc_percent, 'SOC', physical if self.recovery_allowed else lo, hi)
|
||||
if self.rearm_soc_percent is not None:number(self.rearm_soc_percent,'hysteresis rearm SOC',lo,hi)
|
||||
number(self.max_charge_w, 'max_charge_w', 0)
|
||||
number(self.max_discharge_w, 'max_discharge_w', 0)
|
||||
number(self.roundtrip_efficiency, 'roundtrip_efficiency', 0.01, 1)
|
||||
number(self.throughput_chf_kwh, 'throughput_chf_kwh', 0)
|
||||
number(self.terminal_value_chf_kwh, 'terminal_value_chf_kwh', 0)
|
||||
if self.terminal_soc_min_percent is not None:
|
||||
number(self.terminal_soc_min_percent, 'terminal_soc_min_percent', lo, hi)
|
||||
age = (utc(at) - utc(self.measured_at)).total_seconds()
|
||||
if age < -30 or age > 1800:
|
||||
raise ValueError('SOC is stale or from the future')
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuarterPast:
|
||||
import_kwh: float
|
||||
measured_seconds: int
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Limits:
|
||||
export_w: float | None = None
|
||||
import_w: float | None = None
|
||||
manager_month_limits_w: Mapping[int,float] = field(default_factory=dict)
|
||||
def import_limit(self, timestamp):
|
||||
values = []
|
||||
if self.import_w is not None:
|
||||
values.append(number(self.import_w, 'import limit', 0))
|
||||
m = utc(timestamp).astimezone(ZURICH).month
|
||||
if m in self.manager_month_limits_w:
|
||||
values.append(number(self.manager_month_limits_w[m], 'monthly manager limit', 0))
|
||||
return min(values) if values else None
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Minimal validity gates, not a claim of forecast accuracy or model ranking."""
|
||||
from math import isfinite
|
||||
|
||||
|
||||
def assess_family(family, *, minimum_steps=3):
|
||||
points=family.get('points',[])
|
||||
if len(points)<minimum_steps:
|
||||
return {'valid':False,'reason':'Forecast has too few intervals'}
|
||||
loads=[]
|
||||
for point in points:
|
||||
for key in ('loadW','pvW'):
|
||||
value=point.get(key)
|
||||
if isinstance(value,bool) or not isinstance(value,(int,float)) or not isfinite(value) or value<0:
|
||||
return {'valid':False,'reason':'Forecast contains missing or invalid power'}
|
||||
loads.append(point['loadW'])
|
||||
# A PV-only plant or confirmed shutdown may legitimately forecast zero load.
|
||||
# It must be explicit; missing observations filled with zero are not savings.
|
||||
if max(loads)==0 and family.get('zeroLoadConfirmed') is not True:
|
||||
return {'valid':False,'reason':'Unconfirmed all-zero load forecast; likely missing data, not zero electricity costs'}
|
||||
return {'valid':True,'reason':None,'loadMaximumW':max(loads),
|
||||
'loadZeroFraction':sum(v==0 for v in loads)/len(loads)}
|
||||
@@ -0,0 +1,449 @@
|
||||
"""Application data path: versioned numeric observations -> physical load -> trained profiles.
|
||||
|
||||
Lives in the existing planner service/database; no separate diagnostic service.
|
||||
The device may append only to an operator-configured dataset. Original observations,
|
||||
model revisions and prediction vintages are preserved. Output is never an actuator grant.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from bisect import bisect_right
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
from math import isfinite
|
||||
from statistics import median
|
||||
from zoneinfo import ZoneInfo
|
||||
import json
|
||||
|
||||
UTC = timezone.utc
|
||||
LOCAL = ZoneInfo('Europe/Zurich')
|
||||
FAMILIES = ('3', '13', '23')
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
|
||||
|
||||
|
||||
def epoch(value):
|
||||
if not isinstance(value, str):
|
||||
raise ValueError('UTC timestamp required')
|
||||
t = datetime.fromisoformat(value.replace('Z', '+00:00'))
|
||||
if t.tzinfo is None or t.utcoffset().total_seconds() != 0 or t.microsecond:
|
||||
raise ValueError('Explicit whole-second UTC timestamp required')
|
||||
return int(t.timestamp())
|
||||
|
||||
|
||||
def iso(t):
|
||||
return datetime.fromtimestamp(t, UTC).isoformat()
|
||||
|
||||
|
||||
def numeric(value, bound=1e12):
|
||||
return type(value) in (int, float) and isfinite(value) and abs(value) <= bound
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_data_sets(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, config TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL, PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_observations(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, captured_at INTEGER NOT NULL,
|
||||
received_at INTEGER NOT NULL, fingerprint TEXT NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,captured_at));
|
||||
CREATE TABLE IF NOT EXISTS planner_load_windows(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, start INTEGER NOT NULL,
|
||||
available_at INTEGER NOT NULL, coverage REAL NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,start));
|
||||
CREATE TABLE IF NOT EXISTS planner_load_models(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
trained_at INTEGER NOT NULL, trained_through INTEGER NOT NULL, value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,model_id));
|
||||
CREATE TABLE IF NOT EXISTS planner_model_current(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_pipeline_state(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, tick INTEGER NOT NULL,
|
||||
status TEXT NOT NULL, detail TEXT NOT NULL, PRIMARY KEY(plant,dataset));
|
||||
CREATE TABLE IF NOT EXISTS planner_prediction_vintages(
|
||||
plant TEXT NOT NULL, dataset TEXT NOT NULL, issued_at INTEGER NOT NULL,
|
||||
target INTEGER NOT NULL, family TEXT NOT NULL, model_id TEXT NOT NULL,
|
||||
load_w REAL NOT NULL, pv_w REAL NOT NULL,
|
||||
PRIMARY KEY(plant,dataset,issued_at,target,family));
|
||||
CREATE INDEX IF NOT EXISTS planner_observation_window
|
||||
ON planner_observations(plant,dataset,captured_at);
|
||||
CREATE INDEX IF NOT EXISTS planner_prediction_target
|
||||
ON planner_prediction_vintages(plant,dataset,target);
|
||||
''')
|
||||
|
||||
|
||||
def validate_config(c):
|
||||
fields = {'datasetId', 'mappingSha256', 'inventorySha256', 'sources',
|
||||
'formula', 'solarReference', 'minimumCoverage', 'maximumGapSeconds',
|
||||
'minimumTrainingHours', 'historyDays'}
|
||||
if not isinstance(c, dict) or set(c) != fields:
|
||||
raise ValueError('Explicit dataset configuration required')
|
||||
name = c['datasetId']
|
||||
if not isinstance(name, str) or not 1 <= len(name) <= 80 or any(x not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' for x in name):
|
||||
raise ValueError('Invalid dataset ID')
|
||||
for field in ('mappingSha256', 'inventorySha256'):
|
||||
h = c[field]
|
||||
if not isinstance(h, str) or len(h) != 64 or any(x not in '0123456789abcdef' for x in h):
|
||||
raise ValueError('Explicit mapping/inventory fingerprint required')
|
||||
if c['formula'] not in ('physical_sum_v1', 'solar_terminal_v1'):
|
||||
raise ValueError('Unknown physical formula')
|
||||
if not numeric(c['minimumCoverage']) or not .90 <= c['minimumCoverage'] <= 1:
|
||||
raise ValueError('Coverage must be .90..1; recorded gaps remain visible')
|
||||
for field, lo, hi in (('maximumGapSeconds', 1, 10), ('minimumTrainingHours', 1, 168), ('historyDays', 2, 90)):
|
||||
if type(c[field]) is not int or not lo <= c[field] <= hi:
|
||||
raise ValueError('Invalid '+field)
|
||||
sources = c['sources']
|
||||
if not isinstance(sources, list) or not 3 <= len(sources) <= 80:
|
||||
raise ValueError('Source list required')
|
||||
seen, ids = set(), set()
|
||||
roles = {'grid', 'pv', 'physical_storage', 'flexible_load', 'reference', 'sdl_request', 'solar_raw', 'solar_scale'}
|
||||
for s in sources:
|
||||
if set(s) != {'key', 'variableId', 'role', 'factorToW', 'maxAgeSeconds'}:
|
||||
raise ValueError('Explicit source definition required')
|
||||
k = s['key']
|
||||
if not isinstance(k, str) or not 1 <= len(k) <= 64 or k in seen or type(s['variableId']) is not int or not 1 <= s['variableId'] <= 99999 or s['variableId'] in ids:
|
||||
raise ValueError('Duplicate/invalid source')
|
||||
if s['role'] not in roles or not numeric(s['factorToW'], 1e6) or s['factorToW'] == 0:
|
||||
raise ValueError('Source role/factor invalid')
|
||||
if type(s['maxAgeSeconds']) is not int or not 1 <= s['maxAgeSeconds'] <= 300:
|
||||
raise ValueError('Source lifetime invalid')
|
||||
seen.add(k); ids.add(s['variableId'])
|
||||
if sum(s['role'] == 'grid' for s in sources) != 1 or not any(s['role'] == 'pv' for s in sources):
|
||||
raise ValueError('Grid and PV measurement sources required')
|
||||
sr = c['solarReference']
|
||||
if c['formula'] == 'solar_terminal_v1':
|
||||
if not isinstance(sr, dict) or set(sr) != {'pvKey', 'batteryKey', 'rawKey', 'scaleKey'}:
|
||||
raise ValueError('Solar terminal sources required')
|
||||
bykey = {s['key']: s['role'] for s in sources}
|
||||
if any(bykey.get(sr[k]) != role for k, role in (('pvKey','pv'),('batteryKey','physical_storage'),('rawKey','solar_raw'),('scaleKey','solar_scale'))):
|
||||
raise ValueError('Solar origin roles mismatch')
|
||||
elif sr is not None:
|
||||
raise ValueError('No unused solar mapping allowed')
|
||||
canonical(c)
|
||||
return c
|
||||
|
||||
|
||||
def register_dataset(con, plant, c, now):
|
||||
"""Operator endpoint only; device append endpoint cannot change units or limits."""
|
||||
validate_config(c)
|
||||
value = canonical(c)
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
old = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,c['datasetId'])).fetchone()
|
||||
if old and old[0] != value:
|
||||
raise ValueError('Dataset is immutable; use a new datasetId for changed measurement meaning')
|
||||
con.execute('INSERT OR IGNORE INTO planner_data_sets VALUES(?,?,?,?)', (plant,c['datasetId'],value,now))
|
||||
con.commit()
|
||||
except Exception:
|
||||
con.rollback(); raise
|
||||
return {'status':'configured', 'datasetId':c['datasetId'], 'mappingSha256':c['mappingSha256'], 'controlEnabled':False}
|
||||
|
||||
|
||||
def configuration(con, plant, dataset):
|
||||
row = con.execute('SELECT config FROM planner_data_sets WHERE plant=? AND dataset=?', (plant,dataset)).fetchone()
|
||||
if row is None:
|
||||
raise ValueError('Dataset not configured for this installation')
|
||||
return json.loads(row[0])
|
||||
|
||||
|
||||
def project(record, c, plant, now):
|
||||
if not isinstance(record, dict) or type(record.get('schemaVersion')) is not int or record.get('schemaVersion') != 1 or record.get('kind') != 'raw_accounting_capture' or record.get('installationId') != plant:
|
||||
raise ValueError('Wrong capture identity')
|
||||
if record.get('mappingSha256') != c['mappingSha256'] or record.get('reportedInventorySha256') != c['inventorySha256']:
|
||||
raise ValueError('Wrong capture mapping or inventory')
|
||||
t = epoch(record.get('capturedAt')); start = epoch(record.get('captureStartedAt'))
|
||||
if start > t or t > now+30 or t < now-90*86400:
|
||||
raise ValueError('Capture timestamp outside permitted range')
|
||||
if not isinstance(record.get('raw'), dict):
|
||||
raise ValueError('Numeric raw observations required')
|
||||
out = {}; issues = []
|
||||
for s in c['sources']:
|
||||
r = record['raw'].get(s['key'], {})
|
||||
if not isinstance(r, dict):
|
||||
r = {}
|
||||
v, at = r.get('value'), r.get('sourceUpdatedAt')
|
||||
good = r.get('variableId') == s['variableId'] and numeric(v) and type(at) is int and 0 < at <= t and r.get('issues') == []
|
||||
if not good:
|
||||
v = at = None
|
||||
issues.append(s['key'])
|
||||
# Unknown/free-text fields, credentials, client quality claims never persisted.
|
||||
out[s['key']] = {'value':v, 'sourceUpdatedAt':at, 'valid':bool(good)}
|
||||
return {'capturedAt':t, 'captureDurationSeconds':t-start, 'raw':out, 'invalidSources':issues}
|
||||
|
||||
|
||||
def ingest_batch(con, plant, payload, now):
|
||||
if not isinstance(payload,dict) or set(payload) != {'version','datasetId','records'} or type(payload.get('version')) is not int or payload['version'] != 1:
|
||||
raise ValueError('Measurement batch version/fields invalid')
|
||||
c = configuration(con,plant,payload['datasetId'])
|
||||
records = payload['records']
|
||||
if not isinstance(records,list) or not 1 <= len(records) <= 120:
|
||||
raise ValueError('Batch requires 1..120 captures')
|
||||
rows = [project(r,c,plant,now) for r in records]
|
||||
if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(rows,rows[1:])):
|
||||
raise ValueError('Batch must be in increasing capture order')
|
||||
stored = duplicate = 0
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
for r in rows:
|
||||
value = canonical(r); digest = sha256(value.encode()).hexdigest()
|
||||
old = con.execute('SELECT fingerprint FROM planner_observations WHERE plant=? AND dataset=? AND captured_at=?', (plant,c['datasetId'],r['capturedAt'])).fetchone()
|
||||
if old:
|
||||
if old[0] != digest:
|
||||
raise ValueError('Conflicting immutable observation')
|
||||
duplicate += 1
|
||||
else:
|
||||
con.execute('INSERT INTO planner_observations VALUES(?,?,?,?,?,?)',(plant,c['datasetId'],r['capturedAt'],now,digest,value)); stored += 1
|
||||
con.commit()
|
||||
except Exception:
|
||||
con.rollback(); raise
|
||||
return {'status':'stored' if stored else 'duplicate','stored':stored,'duplicates':duplicate,
|
||||
'acceptedThrough':iso(rows[-1]['capturedAt']),'datasetId':c['datasetId'],'controlEnabled':False}
|
||||
|
||||
|
||||
def physical_value(values, c):
|
||||
"""Same explicit sign convention as configured acquisition. No virtual power in load."""
|
||||
total = defaultdict(float)
|
||||
sr = c['solarReference']
|
||||
for s in c['sources']:
|
||||
if s['role'] not in ('grid','pv','physical_storage','flexible_load'):
|
||||
continue
|
||||
if sr and s['key'] in (sr['pvKey'],sr['batteryKey']):
|
||||
continue
|
||||
val = values[s['key']]*s['factorToW']
|
||||
if not numeric(val,1e9) or (s['role'] in ('pv','flexible_load') and val < 0):
|
||||
raise ValueError('Invalid physical power')
|
||||
total[s['role']] += val
|
||||
solar = 0.
|
||||
if sr:
|
||||
raw, sf = values[sr['rawKey']], values[sr['scaleKey']]
|
||||
if int(raw) != raw or not -32768 < raw <= 32767 or int(sf) != sf or not -6 <= sf <= 6:
|
||||
raise ValueError('Invalid solar power/scaling sentinel')
|
||||
solar = raw*10**int(sf)
|
||||
load = total['grid']+total['pv']-total['physical_storage']-total['flexible_load']+solar
|
||||
if not numeric(load,1e9) or load < 0:
|
||||
raise ValueError('Negative/nonfinite physical load')
|
||||
return load
|
||||
|
||||
|
||||
def reconstruct(records, c):
|
||||
"""Bounded retrospective estimation, never a real-time feedback signal.
|
||||
|
||||
Missing observations split support. Source timestamps are not refreshed. Small
|
||||
uncovered portions remain quantified and are never filled with zero.
|
||||
"""
|
||||
if len(records) < 2:
|
||||
return []
|
||||
if any(a['capturedAt'] >= b['capturedAt'] for a,b in zip(records,records[1:])):
|
||||
raise ValueError('Capture sequence not ordered')
|
||||
sr = c['solarReference']
|
||||
primary = {s['key']:s for s in c['sources'] if s['role'] in ('grid','pv','physical_storage','flexible_load')}
|
||||
if sr:
|
||||
primary.pop(sr['pvKey']); primary.pop(sr['batteryKey'])
|
||||
for s in c['sources']:
|
||||
if s['key'] in (sr['rawKey'],sr['scaleKey']): primary[s['key']] = s
|
||||
first,last = records[0]['capturedAt'],records[-1]['capturedAt']
|
||||
series = {k:{} for k in primary}; blocks = {k:[] for k in primary}; gaps = []
|
||||
pending = {k:None for k in primary}; high = {k:0 for k in primary}
|
||||
edges = {first,last}
|
||||
for a,b in zip(records,records[1:]):
|
||||
if b['capturedAt']-a['capturedAt'] > 45:
|
||||
gaps.append((a['capturedAt'],b['capturedAt']));edges.update(gaps[-1])
|
||||
for r in records:
|
||||
at = r['capturedAt']
|
||||
for k in primary:
|
||||
v = r['raw'].get(k,{})
|
||||
t = v.get('sourceUpdatedAt')
|
||||
if not v.get('valid') or not numeric(v.get('value')) or type(t) is not int or t > at or t < high[k] or r['captureDurationSeconds'] > 5:
|
||||
if pending[k] is None: pending[k] = at
|
||||
continue
|
||||
high[k] = max(high[k],t)
|
||||
if pending[k] is not None:
|
||||
blocks[k].append((pending[k],at));edges.update(blocks[k][-1]);pending[k] = None
|
||||
if t in series[k] and series[k][t] != v['value']:
|
||||
series[k][t] = None
|
||||
else:
|
||||
series[k].setdefault(t,v['value'])
|
||||
for k,s in primary.items():
|
||||
if pending[k] is not None:
|
||||
blocks[k].append((pending[k],last));edges.update(blocks[k][-1])
|
||||
for t in series[k]: edges.update((t,t+s['maxAgeSeconds']))
|
||||
edges.update(range(first//300*300+300,last,300))
|
||||
edges = sorted(x for x in edges if first <= x <= last)
|
||||
knots = {k:sorted(v) for k,v in series.items()}
|
||||
bins = {}
|
||||
for a,b in zip(edges,edges[1:]):
|
||||
start = a//300*300; item = bins.setdefault(start,{'start':start,'seconds':0,'wattSeconds':0.,'maxGapSeconds':0,'currentGap':0})
|
||||
vals = {}; usable = not any(x <= a < y for x,y in gaps)
|
||||
for k,s in primary.items():
|
||||
pos = bisect_right(knots[k],a)-1
|
||||
t = knots[k][pos] if pos >= 0 else None
|
||||
if t is None or a >= t+s['maxAgeSeconds'] or series[k][t] is None or any(x <= a < y for x,y in blocks[k]):
|
||||
usable = False
|
||||
else: vals[k] = series[k][t]
|
||||
load = None
|
||||
if usable:
|
||||
try: load = physical_value(vals,c)
|
||||
except ValueError: usable = False
|
||||
if usable:
|
||||
item['seconds'] += b-a; item['wattSeconds'] += load*(b-a);item['currentGap'] = 0
|
||||
else:
|
||||
item['currentGap'] += b-a; item['maxGapSeconds'] = max(item['maxGapSeconds'],item['currentGap'])
|
||||
out=[]
|
||||
for t,item in sorted(bins.items()):
|
||||
# Partial beginning/end bins remain diagnostic and cannot train.
|
||||
complete_extent = first <= t and last >= t+300
|
||||
coverage = item['seconds']/300
|
||||
eligible = complete_extent and coverage >= c['minimumCoverage'] and item['maxGapSeconds'] <= c['maximumGapSeconds']
|
||||
out.append({'start':t,'coverage':coverage,'coveredSeconds':item['seconds'],'maxGapSeconds':item['maxGapSeconds'],
|
||||
'loadW':item['wattSeconds']/item['seconds'] if item['seconds'] else None,
|
||||
'profileUsable':eligible,'estimated':True,'fullPhysicalIntervalMeasured':False,
|
||||
'meterBoundaryVerified':False,'method':c['formula']})
|
||||
return out
|
||||
|
||||
|
||||
def slot(t):
|
||||
local = datetime.fromtimestamp(t,UTC).astimezone(LOCAL)
|
||||
return local.hour*12+local.minute//5
|
||||
|
||||
|
||||
def build_profiles(rows):
|
||||
samples=defaultdict(list); recent=defaultdict(list); weekend={False:defaultdict(list),True:defaultdict(list)}
|
||||
anchor=max(r['start'] for r in rows)
|
||||
for r in rows:
|
||||
i=slot(r['start']); v=r['loadW']; samples[i].append(v)
|
||||
if anchor-r['start'] < 86400: recent[i].append(v)
|
||||
weekend[datetime.fromtimestamp(r['start'],UTC).astimezone(LOCAL).weekday()>=5][i].append(v)
|
||||
overall = median([r['loadW'] for r in rows])
|
||||
def profile(values):
|
||||
# Missing calendar slots are a model estimate, not invented historical measurements.
|
||||
result=[]
|
||||
for i in range(288):
|
||||
local=values.get(i,[])
|
||||
if not local:
|
||||
local=[v for j in ((i-2)%288,(i-1)%288,(i+1)%288,(i+2)%288) for v in values.get(j,[])]
|
||||
result.append(float(median(local)) if local else float(overall))
|
||||
return result
|
||||
return {'3':profile(samples),'13':profile(recent),'23':{'weekday':profile(weekend[False] or samples),'weekend':profile(weekend[True] or samples)},
|
||||
'slotCoverage':len(samples)/288}
|
||||
|
||||
|
||||
def predict(model, family, t):
|
||||
p=model['profiles'][family]
|
||||
if family=='23': p=p['weekend' if datetime.fromtimestamp(t,UTC).astimezone(LOCAL).weekday()>=5 else 'weekday']
|
||||
return p[slot(t)]
|
||||
|
||||
|
||||
def advance(con, plant, dataset, settings, now):
|
||||
"""Called by the existing worker; bounded data/model update once per five-minute tick."""
|
||||
c=configuration(con,plant,dataset); tick=now//300
|
||||
old=con.execute('SELECT tick FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()
|
||||
if old and old[0]==tick: return
|
||||
fetched=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at>=? AND captured_at<=? AND received_at<=? ORDER BY captured_at',
|
||||
(plant,dataset,now-172800-300,now,now)).fetchall()
|
||||
records=[json.loads(r[0]) for r in fetched]
|
||||
windows=reconstruct(records,c)
|
||||
with con:
|
||||
for w in windows:
|
||||
if w['start']+300 > now-30: continue
|
||||
con.execute('INSERT INTO planner_load_windows VALUES(?,?,?,?,?,?) ON CONFLICT(plant,dataset,start) DO UPDATE SET available_at=excluded.available_at,coverage=excluded.coverage,value=excluded.value',
|
||||
(plant,dataset,w['start'],now,w['coverage'],canonical(w)))
|
||||
rows=[json.loads(r[0]) for r in con.execute('SELECT value FROM planner_load_windows WHERE plant=? AND dataset=? AND start>=? AND start+300<=? ORDER BY start',(plant,dataset,now-c['historyDays']*86400,now))]
|
||||
good=[r for r in rows if r['profileUsable']]
|
||||
active=current_model(con,plant,dataset,now)
|
||||
cadence=86400 if settings['trainingCadence']=='daily' else 604800
|
||||
detail={'observationsInLast48h':len(records),'usableWindows':len(good),'requiredEquivalentHours':c['minimumTrainingHours'],
|
||||
'usableEquivalentHours':sum(r['coverage'] for r in good)/12,'datasetId':dataset,'trainingCadence':settings['trainingCadence'],
|
||||
'automaticTrainingConnected':True,'liveEnabled':False,'sourceIsConfiguredEstimate':True}
|
||||
state='collecting'
|
||||
if sum(r['coverage'] for r in good) >= c['minimumTrainingHours']*12:
|
||||
state='model_ready' if active else 'training'
|
||||
attempted=con.execute('SELECT MAX(trained_at) FROM planner_load_models WHERE plant=? AND dataset=?',(plant,dataset)).fetchone()[0]
|
||||
if attempted is None or now-attempted>=cadence:
|
||||
profiles=build_profiles(good)
|
||||
candidate={'profiles':profiles,'trainedAt':now,'trainedThrough':max(r['start']+300 for r in good),
|
||||
'trainingWindowFrom':good[0]['start'],'sourceDataset':dataset,'formula':c['formula'],
|
||||
'methodVersion':'physical-profile-v1','validation':{'status':'bootstrap_insufficient_holdout'},
|
||||
'measurementBoundaryVerified':False}
|
||||
# Causal held-out validation: build validation profiles without the final day.
|
||||
split=good[-1]['start']-86400
|
||||
train=[r for r in good if r['start']+300<=split]; test=[r for r in good if r['start']>=split]
|
||||
if len(train)>=288 and len(test)>=240:
|
||||
val={'profiles':build_profiles(train)}
|
||||
errors={f:sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for r in test)/sum(r['coverage'] for r in test) for f in FAMILIES}
|
||||
candidate['validation']={'status':'causal_holdout','holdoutFrom':split,'holdoutWindows':len(test),'loadMaeWByFamily':errors}
|
||||
# Initial model is labelled bootstrap, never a production measurement proof.
|
||||
# Existing model can be replaced only with held-out evidence and no aggregate regression.
|
||||
promote=active is None
|
||||
if active and candidate['validation']['status']=='causal_holdout':
|
||||
past_model_eligible=active['trainedThrough']<=split
|
||||
if past_model_eligible:
|
||||
incumbent=sum(abs(predict(active,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test)
|
||||
challenger=sum(abs(predict(val,f,r['start'])-r['loadW'])*r['coverage'] for f in FAMILIES for r in test)
|
||||
promote=challenger<=incumbent
|
||||
candidate['validation']['incumbentCompared']=True
|
||||
else:
|
||||
candidate['validation']['status']='holdout_overlaps_active_training'
|
||||
ident=sha256(canonical(candidate).encode()).hexdigest()
|
||||
with con:
|
||||
con.execute('INSERT OR IGNORE INTO planner_load_models VALUES(?,?,?,?,?,?)',(plant,dataset,ident,now,candidate['trainedThrough'],canonical(candidate)))
|
||||
if promote:
|
||||
con.execute('INSERT INTO planner_model_current VALUES(?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET model_id=excluded.model_id',(plant,dataset,ident))
|
||||
detail['candidateModelId']=ident;detail['candidatePromoted']=promote
|
||||
state='model_ready' if promote or active else 'candidate_pending'
|
||||
active=current_model(con,plant,dataset,now)
|
||||
if active: detail.update({'modelId':active['modelId'],'trainedAt':iso(active['trainedAt']),'trainedThrough':iso(active['trainedThrough']),'validation':active['validation']})
|
||||
with con:
|
||||
con.execute('INSERT INTO planner_pipeline_state VALUES(?,?,?,?,?) ON CONFLICT(plant,dataset) DO UPDATE SET tick=excluded.tick,status=excluded.status,detail=excluded.detail',
|
||||
(plant,dataset,tick,state,canonical(detail)))
|
||||
|
||||
|
||||
def current_model(con,plant,dataset,at):
|
||||
row=con.execute('SELECT m.model_id,m.value FROM planner_load_models m JOIN planner_model_current c ON m.plant=c.plant AND m.dataset=c.dataset AND m.model_id=c.model_id WHERE m.plant=? AND m.dataset=? AND m.trained_at<=?',(plant,dataset,at)).fetchone()
|
||||
return {**json.loads(row[1]),'modelId':row[0]} if row else None
|
||||
|
||||
|
||||
def apply_load_forecast(con,plant,dataset,forecast,decision):
|
||||
model=current_model(con,plant,dataset,decision)
|
||||
if not model:
|
||||
raise ValueError('Corrected profile is collecting data; legacy household forecast is not silently reused')
|
||||
last=con.execute('SELECT value FROM planner_observations WHERE plant=? AND dataset=? AND captured_at<=? AND received_at<=? ORDER BY captured_at DESC LIMIT 1',(plant,dataset,decision,decision)).fetchone()
|
||||
if not last: raise ValueError('No recent corrected observation')
|
||||
last=json.loads(last[0]); c=configuration(con,plant,dataset)
|
||||
if decision-last['capturedAt']>120: raise ValueError('Corrected measurements older than 120 seconds')
|
||||
sdl_sources=[s for s in c['sources'] if s['role']=='sdl_request']
|
||||
if len(sdl_sources)!=1: raise ValueError('Explicit SDL request channel needed for the labelled persistence scenario')
|
||||
s=sdl_sources[0];r=last['raw'][s['key']]
|
||||
if not r['valid'] or decision-r['sourceUpdatedAt']>s['maxAgeSeconds']:
|
||||
raise ValueError('No current external SDL request for the persistence scenario')
|
||||
sdl=r['value']*s['factorToW']
|
||||
result=json.loads(canonical(forecast)); result['families']={}
|
||||
result['observedAt']=iso(max(epoch(forecast['observedAt']),model['trainedAt'],last['capturedAt']))
|
||||
for family,old in forecast['families'].items():
|
||||
if family not in FAMILIES: continue
|
||||
points=[]
|
||||
for p in old['points']:
|
||||
t=epoch(p['time'])
|
||||
points.append({**p,'loadW':predict(model,family,t),'externalW':sdl})
|
||||
result['families'][family]={'loadBasis':'base_load','trainedUntil':iso(model['trainedThrough']),
|
||||
'points':points,'dataPipeline':{'datasetId':dataset,'modelId':model['modelId'],
|
||||
'loadMethodVersion':model['methodVersion'],'loadVariant':family,
|
||||
'loadModelTrainedAt':iso(model['trainedAt']),'pvForecastEventId':forecast.get('eventId'),
|
||||
'measurementBasis':'configured_physical_estimate','measurementBoundaryVerified':False,
|
||||
'externalPolicy':'last_sdl_request_persistence_estimate','externalObservedAt':iso(r['sourceUpdatedAt']),
|
||||
'externalPowerW':sdl,'futureSdlPublished':False,'validation':model['validation']}}
|
||||
return result
|
||||
|
||||
|
||||
def pipeline_status(con,plant):
|
||||
out=[]
|
||||
for row in con.execute('SELECT dataset,config FROM planner_data_sets WHERE plant=? ORDER BY dataset',(plant,)):
|
||||
ds=row[0]; c=json.loads(row[1]); state=con.execute('SELECT status,detail FROM planner_pipeline_state WHERE plant=? AND dataset=?',(plant,ds)).fetchone()
|
||||
count=con.execute('SELECT COUNT(*),MIN(captured_at),MAX(captured_at) FROM planner_observations WHERE plant=? AND dataset=?',(plant,ds)).fetchone()
|
||||
out.append({'datasetId':ds,'formula':c['formula'],'mappingSha256':c['mappingSha256'],'records':count[0],
|
||||
'firstCapture':iso(count[1]) if count[1] else None,'lastCapture':iso(count[2]) if count[2] else None,
|
||||
'status':state[0] if state else 'awaiting_measurements','detail':json.loads(state[1]) if state else {},
|
||||
'minimumCoverage':c['minimumCoverage'],'maximumGapSeconds':c['maximumGapSeconds']})
|
||||
return {'datasets':out,'liveEnabled':False,'legacyHistoryModified':False}
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Persistent, explicitly estimated operational demand tracking.
|
||||
|
||||
No inference from a configured cap. Samples are device-reception observations;
|
||||
last-value integration is a labelled control estimate, not settlement metering.
|
||||
No reset, stale sample or long communication gap is bridged silently.
|
||||
"""
|
||||
from datetime import datetime,timedelta,timezone
|
||||
import json
|
||||
from .domain import number,utc,month_key,quarter_start,ZURICH
|
||||
from .peak_policy import basis_record
|
||||
|
||||
|
||||
def schema(con):
|
||||
con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_peak_assumptions(
|
||||
plant TEXT NOT NULL,month TEXT NOT NULL,kw REAL NOT NULL,value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,month));
|
||||
CREATE TABLE IF NOT EXISTS planner_runtime_samples(
|
||||
plant TEXT NOT NULL,meter_id TEXT NOT NULL,at INTEGER NOT NULL,
|
||||
power_w REAL NOT NULL,total_kwh REAL,policy_id TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,meter_id,at));
|
||||
CREATE TABLE IF NOT EXISTS planner_runtime_quarters(
|
||||
plant TEXT NOT NULL,meter_id TEXT NOT NULL,start INTEGER NOT NULL,
|
||||
import_kwh REAL NOT NULL,policy_id TEXT NOT NULL,value TEXT NOT NULL,
|
||||
PRIMARY KEY(plant,meter_id,start));
|
||||
CREATE INDEX IF NOT EXISTS idx_runtime_samples_time ON planner_runtime_samples(plant,at);
|
||||
''')
|
||||
|
||||
|
||||
def validate_observation(obs, observed_at):
|
||||
if not isinstance(obs,dict) or set(obs)!={'meterId','sampleAt','powerW','totalImportKwh','controlPolicyId'}:
|
||||
raise ValueError('Explicit meter observation schema required')
|
||||
import re
|
||||
if not isinstance(obs['meterId'],str) or not re.fullmatch(r'symcon-active-import:[0-9a-f]{64}',obs['meterId']):
|
||||
raise ValueError('Active import identity required')
|
||||
if not isinstance(obs['controlPolicyId'],str) or not 1<=len(obs['controlPolicyId'])<=160:
|
||||
raise ValueError('Control policy identity required')
|
||||
t=utc(obs['sampleAt'])
|
||||
if t.microsecond or not 0<=(utc(observed_at)-t).total_seconds()<=60:
|
||||
raise ValueError('Fresh whole-second acquisition timestamp required')
|
||||
number(obs['powerW'],'meter power',-1e9,1e9)
|
||||
number(obs['totalImportKwh'],'active import total',0,1e12)
|
||||
|
||||
|
||||
def integrate_power(samples,start,end,max_gap=120):
|
||||
"""samples sorted tuples (epoch,power,total,policy). Return None on gaps/reset.
|
||||
|
||||
A source can keep the same value while receiving fresh telemetry. The caller
|
||||
records all acquisitions, not only changes. A quarter is never labelled exact.
|
||||
"""
|
||||
if end<=start:return None
|
||||
rows=sorted(samples,key=lambda r:r[0])
|
||||
if len(rows)<2:return None
|
||||
covered=energy=0.;policies=set();largest_gap=0
|
||||
previous=None
|
||||
for row in rows:
|
||||
if previous is not None:
|
||||
ta,pa,ea,pola=previous;tb,pb,eb,polb=row
|
||||
if tb<=ta:return None
|
||||
left=max(start,ta);right=min(end,tb)
|
||||
if right>left:
|
||||
if tb-ta>max_gap or pola!=polb:return None
|
||||
if ea is not None and eb is not None and eb<ea-1e-8:return None
|
||||
seconds=right-left;covered+=seconds;energy+=max(0.,pa)*seconds/3600000
|
||||
policies.add(pola);largest_gap=max(largest_gap,tb-ta)
|
||||
previous=row
|
||||
if covered!=end-start or len(policies)!=1:return None
|
||||
return {'importKwh':energy,'measuredSeconds':end-start,'quality':'estimated',
|
||||
'source':'sampled_power_estimate','method':'positive_power_left_hold',
|
||||
'maxSampleGapSeconds':largest_gap,'controlPolicyId':policies.pop(),
|
||||
'billingEvidence':False}
|
||||
|
||||
|
||||
def save_assumption(con,plant,month,record,at):
|
||||
validated=basis_record(record,month,at,allow_estimates=True)
|
||||
if validated['quality']!='estimated':raise ValueError('Only estimates in assumption storage')
|
||||
current=con.execute('SELECT value FROM planner_peak_assumptions WHERE plant=? AND month=?',(plant,month)).fetchone()
|
||||
# Approximate historical maximum is monotonic within this quality channel.
|
||||
# An authoritative corrected source is stored separately and has precedence.
|
||||
if current:
|
||||
old=json.loads(current[0])
|
||||
if old.get('meterId') and validated.get('meterId') and old['meterId']!=validated['meterId']:
|
||||
raise ValueError('Peak assumption belongs to a different physical meter')
|
||||
if old['kw']>=validated['kw']:return old
|
||||
con.execute('INSERT INTO planner_peak_assumptions VALUES(?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET kw=excluded.kw,value=excluded.value',
|
||||
(plant,month,validated['kw'],json.dumps(validated,sort_keys=True,allow_nan=False)))
|
||||
return validated
|
||||
|
||||
|
||||
def assumptions(con,plant):
|
||||
return {r[0]:json.loads(r[1]) for r in con.execute('SELECT month,value FROM planner_peak_assumptions WHERE plant=?',(plant,))}
|
||||
|
||||
|
||||
def observe(con,plant,obs,at):
|
||||
validate_observation(obs,at)
|
||||
mid=obs['meterId'];stamp=int(utc(obs['sampleAt']).timestamp())
|
||||
existing=con.execute('SELECT power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at=?',(plant,mid,stamp)).fetchone()
|
||||
values=(float(obs['powerW']),float(obs['totalImportKwh']),obs['controlPolicyId'])
|
||||
if existing and tuple(existing)!=values:raise ValueError('Conflicting meter acquisition at same time')
|
||||
con.execute('INSERT OR IGNORE INTO planner_runtime_samples VALUES(?,?,?,?,?,?)',(plant,mid,stamp,*values))
|
||||
# Only a just-completed quarter is finalised; late acquisition never promotes
|
||||
# a historical gap to complete data without all supporting samples.
|
||||
current=stamp//900*900
|
||||
rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at',
|
||||
(plant,mid,current-900-120,stamp))]
|
||||
report=integrate_power(rows,current-900,current)
|
||||
if report:
|
||||
quarter=current-900
|
||||
previous=con.execute('SELECT import_kwh FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start=?',(plant,mid,quarter)).fetchone()
|
||||
if previous is None:
|
||||
con.execute('INSERT INTO planner_runtime_quarters VALUES(?,?,?,?,?,?)',(plant,mid,quarter,report['importKwh'],report['controlPolicyId'],json.dumps(report)))
|
||||
m=month_key(datetime.fromtimestamp(quarter,timezone.utc))
|
||||
save_assumption(con,plant,m,{'kw':report['importKwh']/.25,'quality':'estimated','source':'sampled_power_estimate',
|
||||
'observedAt':utc(at).isoformat(),'meterId':mid,'notes':'Maximum of available sampled quarters; earlier month may be incomplete'},at)
|
||||
# Samples are retained for reproducibility; production retention job required.
|
||||
return report
|
||||
|
||||
|
||||
def current_quarter(con,plant,obs,decision):
|
||||
q=quarter_start(decision);start=int(q.timestamp());end=int(utc(decision).timestamp())
|
||||
if start==end:return None
|
||||
rows=[tuple(r) for r in con.execute('SELECT at,power_w,total_kwh,policy_id FROM planner_runtime_samples WHERE plant=? AND meter_id=? AND at>=? AND at<=? ORDER BY at',
|
||||
(plant,obs['meterId'],start-120,end))]
|
||||
report=integrate_power(rows,start,end)
|
||||
if not report:return None
|
||||
return {'start':q.isoformat(),'measuredSeconds':end-start,'importKwh':report['importKwh'],
|
||||
'quality':'estimated','source':'sampled_power_estimate','coverage':1.0,'notes':'Acquisition power estimate, not an exact billing counter boundary'}
|
||||
|
||||
|
||||
def daily_peaks(con,plant,meter_id,now):
|
||||
rows=con.execute('SELECT start,import_kwh,policy_id FROM planner_runtime_quarters WHERE plant=? AND meter_id=? AND start>=? ORDER BY start',
|
||||
(plant,meter_id,int((utc(now)-timedelta(days=91)).timestamp())))
|
||||
grouped={}
|
||||
for start,energy,policy in rows:
|
||||
t=datetime.fromtimestamp(start,timezone.utc).astimezone(ZURICH)
|
||||
grouped.setdefault((t.strftime('%Y-%m-%d'),policy),[]).append((start,energy))
|
||||
result=[]
|
||||
for (day,policy),items in grouped.items():
|
||||
begin=datetime.strptime(day,'%Y-%m-%d').replace(tzinfo=ZURICH);end=begin+timedelta(days=1)
|
||||
if utc(end)>utc(now):continue
|
||||
expected=set(range(int(begin.timestamp()),int(end.timestamp()),900))
|
||||
complete={t for t,e in items}==expected
|
||||
result.append({'day':day,'peakKw':max(e/.25 for t,e in items),'controlPolicyId':policy,'complete':complete,
|
||||
'observedAt':utc(end).isoformat(),'quality':'estimated'})
|
||||
return result
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Auditable import-energy evidence. Pure calculations; no device/database writes.
|
||||
|
||||
Raw change archives are NOT proof of uninterrupted meter communication. Boundaries
|
||||
without an exact counter record yield intervals, never silently interpolated
|
||||
billing facts. Only a separately verified source and chronology can be promoted
|
||||
into the existing strict planner evidence contract.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from bisect import bisect_left, bisect_right
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
import hashlib
|
||||
import json
|
||||
from math import isfinite
|
||||
from .domain import utc, month_key, quarter_start, ZURICH
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Reading:
|
||||
at: datetime
|
||||
kwh: float
|
||||
|
||||
|
||||
class CounterSeries:
|
||||
def __init__(self, source_id: str, rows, *, max_bracket_seconds=180,
|
||||
quantity='active_import', unit='kWh', timestamp_verified=False,
|
||||
identity_verified=False, chronology_verified=False):
|
||||
if quantity != 'active_import' or unit != 'kWh':
|
||||
raise ValueError('Active import energy in kWh required')
|
||||
if not isinstance(source_id, str) or not source_id:
|
||||
raise ValueError('Nonempty stable source identity required')
|
||||
if type(max_bracket_seconds) is not int or max_bracket_seconds < 1:
|
||||
raise ValueError('Positive bracket age required')
|
||||
self.source_id = source_id
|
||||
self.max_gap = max_bracket_seconds
|
||||
self.verified = all(x is True for x in
|
||||
(timestamp_verified, identity_verified, chronology_verified))
|
||||
values = {}
|
||||
for row in rows:
|
||||
at = utc(row.at)
|
||||
value = row.kwh
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value) or value < 0:
|
||||
raise ValueError('Invalid cumulative active import value')
|
||||
if at in values and abs(values[at] - value) > 1e-9:
|
||||
raise ValueError('Conflicting duplicate timestamp')
|
||||
values[at] = float(value)
|
||||
self.times = sorted(values)
|
||||
self.values = [values[t] for t in self.times]
|
||||
self.resets = [self.times[i] for i in range(1,len(self.times))
|
||||
if self.values[i] < self.values[i-1] - 1e-9]
|
||||
|
||||
def boundary(self, at):
|
||||
at = utc(at)
|
||||
pos = bisect_left(self.times, at)
|
||||
if pos < len(self.times) and self.times[pos] == at:
|
||||
value = self.values[pos]
|
||||
return {'lowerKwh':value, 'upperKwh':value, 'exactRecord':True,
|
||||
'before':at.isoformat(), 'after':at.isoformat()}
|
||||
if pos == 0 or pos == len(self.times):
|
||||
return None
|
||||
before, after = self.times[pos-1], self.times[pos]
|
||||
if (after-before).total_seconds() > self.max_gap:
|
||||
return None
|
||||
if self.values[pos] < self.values[pos-1] - 1e-9:
|
||||
return None
|
||||
return {'lowerKwh':self.values[pos-1], 'upperKwh':self.values[pos],
|
||||
'exactRecord':False, 'before':before.isoformat(), 'after':after.isoformat()}
|
||||
|
||||
def energy(self, start, end):
|
||||
start, end = utc(start), utc(end)
|
||||
if end <= start:
|
||||
raise ValueError('Positive interval required')
|
||||
left, right = self.boundary(start), self.boundary(end)
|
||||
if not left or not right:
|
||||
return None
|
||||
support_start = utc(left['before'])
|
||||
support_end = utc(right['after'])
|
||||
if any(support_start < reset <= support_end for reset in self.resets):
|
||||
return None
|
||||
lower = max(0., right['lowerKwh'] - left['upperKwh'])
|
||||
upper = right['upperKwh'] - left['lowerKwh']
|
||||
if upper < lower - 1e-9:
|
||||
return None
|
||||
exact = bool(left['exactRecord'] and right['exactRecord'])
|
||||
return {'lowerKwh':lower, 'upperKwh':max(lower, upper),
|
||||
'exactRecords':exact, 'billingEvidence':exact and self.verified}
|
||||
|
||||
|
||||
def native_meter_id(sources):
|
||||
"""Same ordered JSON and SHA256 as NetzfahrplanV4Bezugszaehler::identitaet."""
|
||||
import re
|
||||
if not isinstance(sources,list) or not 1<=len(sources)<=8:
|
||||
raise ValueError('Explicit native import sources required')
|
||||
normalized=[];ids=set();idents=set();parents=set()
|
||||
for source in sources:
|
||||
if set(source)!={'VariableID','ElternID','Ident','FaktorZuKWh','Messgroesse'}:
|
||||
raise ValueError('Unknown native source fields')
|
||||
sid,pid,ident,factor=(source[k] for k in ('VariableID','ElternID','Ident','FaktorZuKWh'))
|
||||
if (type(sid) is not int or sid<=0 or type(pid) is not int or pid<=0
|
||||
or sid in ids or ident in idents or not isinstance(ident,str)
|
||||
or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}',ident)
|
||||
or isinstance(factor,bool) or not isinstance(factor,(float,int))
|
||||
or not isfinite(factor) or not 0<factor<=1e6
|
||||
or source['Messgroesse']!='WirkenergieBezug'):
|
||||
raise ValueError('Invalid or duplicate native source')
|
||||
ids.add(sid);idents.add(ident);parents.add(pid)
|
||||
normalized.append({'VariableID':sid,'ElternID':pid,'Ident':ident,
|
||||
'FaktorZuKWh':float(factor),'Messgroesse':'WirkenergieBezug'})
|
||||
if len(parents)!=1:raise ValueError('T1 and T2 must share physical meter')
|
||||
normalized.sort(key=lambda s:s['VariableID'])
|
||||
text=json.dumps(normalized,separators=(',',':'),ensure_ascii=True,allow_nan=False)
|
||||
# PHP preserves .0 on scientific floats and omits zero-padding in exponents.
|
||||
def exponent(match):
|
||||
mantissa=match.group(1)
|
||||
if '.' not in mantissa:mantissa+='.0'
|
||||
power=int(match.group(2))
|
||||
return '"FaktorZuKWh":'+mantissa+'e'+('+' if power>=0 else '')+str(power)
|
||||
text=re.sub(r'"FaktorZuKWh":([0-9]+(?:\.[0-9]+)?)e([+-]?[0-9]+)',exponent,text)
|
||||
return 'symcon-active-import:'+hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
class MeterEvidence:
|
||||
def __init__(self, series, *, native_sources=None):
|
||||
self.series = list(series)
|
||||
if not self.series or len({s.source_id for s in self.series}) != len(self.series):
|
||||
raise ValueError('Unique, nonempty set of counter sources required')
|
||||
self.native_bound = native_sources is not None
|
||||
if self.native_bound:
|
||||
self.meter_id=native_meter_id(native_sources)
|
||||
if {s.source_id for s in self.series}!={'symcon:'+str(s['VariableID']) for s in native_sources}:
|
||||
raise ValueError('Evidence stream does not match native source set')
|
||||
else:
|
||||
signature = json.dumps(sorted(s.source_id for s in self.series),separators=(',',':'))
|
||||
self.meter_id = 'audit-only:' + hashlib.sha256(signature.encode()).hexdigest()
|
||||
|
||||
def interval(self, start, end):
|
||||
start, end = utc(start), utc(end)
|
||||
parts = [s.energy(start,end) for s in self.series]
|
||||
missing = [s.source_id for s,p in zip(self.series,parts) if p is None]
|
||||
if missing:
|
||||
return {'status':'missing', 'start':start.isoformat(),'end':end.isoformat(),
|
||||
'missingSources':missing, 'billingEvidence':False}
|
||||
lo = sum(p['lowerKwh'] for p in parts)
|
||||
hi = sum(p['upperKwh'] for p in parts)
|
||||
seconds = (end-start).total_seconds()
|
||||
exact = all(p['exactRecords'] for p in parts)
|
||||
return {'status':'exact_records' if exact else 'bounded_records',
|
||||
'start':start.isoformat(),'end':end.isoformat(),'lowerKwh':lo,'upperKwh':hi,
|
||||
'lowerAverageKw':lo*3600/seconds,'upperAverageKw':hi*3600/seconds,
|
||||
'billingEvidence':all(p['billingEvidence'] for p in parts)}
|
||||
|
||||
def month(self, at):
|
||||
at = utc(at)
|
||||
local = at.astimezone(ZURICH)
|
||||
begin = utc(local.replace(day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
stop = quarter_start(at)
|
||||
count = int((stop-begin).total_seconds())//900
|
||||
covered = exact = verified = 0
|
||||
lo = hi = 0.
|
||||
gaps = []
|
||||
quarters = []
|
||||
for n in range(count):
|
||||
start = begin + timedelta(minutes=15*n)
|
||||
result = self.interval(start,start+timedelta(minutes=15))
|
||||
quarters.append(result)
|
||||
if result['status'] == 'missing':
|
||||
if len(gaps)<8:gaps.append(start.isoformat())
|
||||
continue
|
||||
covered += 1
|
||||
exact += result['status']=='exact_records'
|
||||
verified += result['billingEvidence']
|
||||
lo = max(lo,result['lowerAverageKw'])
|
||||
hi = max(hi,result['upperAverageKw'])
|
||||
complete = count>0 and covered==count
|
||||
certified = count>0 and verified==count
|
||||
return {'meterId':self.meter_id,'month':month_key(at),'completedQuarters':count,
|
||||
'coveredQuarters':covered,'exactRecordQuarters':exact,'verifiedQuarters':verified,
|
||||
'historyComplete':complete,'billingEvidence':certified,
|
||||
'observedPeakLowerKw':lo if covered else None,
|
||||
'observedPeakUpperKw':hi if covered else None,
|
||||
'monthPeakUpperKw':hi if complete else None,
|
||||
'verifiedMonthPeakKw':hi if certified else None,
|
||||
'firstMissingQuarters':gaps,'quarters':quarters}
|
||||
|
||||
def strict_contract(self, at):
|
||||
"""Fail closed; do not promote archive gaps, estimated boundaries or a cap."""
|
||||
at = utc(at)
|
||||
if not self.native_bound:
|
||||
raise ValueError('Explicit native counter identity required')
|
||||
result = self.month(at)
|
||||
if not result['billingEvidence']:
|
||||
raise ValueError('Full verified month history missing')
|
||||
q = quarter_start(at)
|
||||
past = None
|
||||
if at != q:
|
||||
partial = self.interval(q,at)
|
||||
if not partial['billingEvidence']:
|
||||
raise ValueError('Verified elapsed-quarter energy missing')
|
||||
past = {'start':q.isoformat(),'measuredSeconds':int((at-q).total_seconds()),
|
||||
'importKwh':partial['upperKwh']}
|
||||
return {'version':1,'meterId':self.meter_id,'measuredAt':at.isoformat(),
|
||||
'measuredPeaks':{month_key(at):{'kw':result['verifiedMonthPeakKw'],
|
||||
'source':'verified_month_history'}},
|
||||
'quarterPast':past}
|
||||
|
||||
|
||||
def audit_capture(payload):
|
||||
"""Read-only report from the dedicated Symcon export, never planner ingestion."""
|
||||
if payload.get('schemaVersion') != 1 or payload.get('kind') != 'v4_meter_capture':
|
||||
raise ValueError('Unknown capture contract')
|
||||
now = utc(payload['capturedAt'])
|
||||
channels = payload['channels']
|
||||
series = []
|
||||
summaries = []
|
||||
for variable in ('59607','26620'):
|
||||
channel = channels[variable]
|
||||
rows = channel.get('history',[])
|
||||
readings = [Reading(utc(datetime.fromtimestamp(r['TimeStamp'],now.tzinfo)),r['Value']) for r in rows]
|
||||
# An explicitly captured current value is usable from its capture, not
|
||||
# from an earlier last-change timestamp, and never before logging began.
|
||||
current = channel.get('snapshot')
|
||||
if current and channel.get('snapshotConsistent'):
|
||||
readings.append(Reading(utc(current['capturedAt']),current['value']))
|
||||
s = CounterSeries('symcon:'+variable, readings, timestamp_verified=False,
|
||||
identity_verified=False, chronology_verified=False)
|
||||
series.append(s)
|
||||
summaries.append({'variableId':int(variable),'ident':channel['ident'],
|
||||
'logging':channel.get('logging'), 'rows':len(rows),
|
||||
'queryComplete':channel.get('queryComplete',False),
|
||||
'firstRecord':s.times[0].isoformat() if s.times else None,
|
||||
'lastRecord':s.times[-1].isoformat() if s.times else None,
|
||||
'counterDecreases':[t.isoformat() for t in s.resets]})
|
||||
meter = MeterEvidence(series)
|
||||
month = meter.month(now)
|
||||
current = None
|
||||
q = quarter_start(now)
|
||||
if now>q:current=meter.interval(q,now)
|
||||
blockers=['Meter register identity/unit and acquisition timestamps still require verification.',
|
||||
'Change-only archive does not prove uninterrupted meter acquisition.']
|
||||
if any(not s['queryComplete'] for s in summaries):blockers.append('Archive export incomplete; do not infer complete history.')
|
||||
if not month['historyComplete']:blockers.append('At least one completed billing quarter lacks bounded readings for every tariff.')
|
||||
return {'capturedAt':now.isoformat(),'status':'audit_only','sourceSummary':summaries,
|
||||
'month':{k:v for k,v in month.items() if k!='quarters'},
|
||||
'recentQuarters':month['quarters'][-8:], 'currentQuarter':current,
|
||||
'billingEvidence':False, 'liveEnabled':False,'blockers':blockers}
|
||||
@@ -0,0 +1,199 @@
|
||||
from __future__ import annotations
|
||||
from math import sqrt
|
||||
from uuid import uuid4
|
||||
import numpy as np
|
||||
from scipy.optimize import Bounds, LinearConstraint, milp
|
||||
from scipy.sparse import coo_matrix
|
||||
from .domain import Battery, Limits, Step, month_key, number, quarter_start, utc
|
||||
from .peak_policy import basis_record, RestMonthOutlook
|
||||
|
||||
class Model:
|
||||
def __init__(self):
|
||||
self.lower,self.upper,self.cost,self.integer=[],[],[],[]
|
||||
self.rows,self.row_lo,self.row_hi=[],[],[]
|
||||
def variable(self, lower=0., upper=np.inf, cost=0., integer=False):
|
||||
index=len(self.lower)
|
||||
self.lower.append(lower);self.upper.append(upper);self.cost.append(cost);self.integer.append(int(integer))
|
||||
return index
|
||||
def constraint(self, coefficients, lower=-np.inf, upper=np.inf):
|
||||
self.rows.append(coefficients);self.row_lo.append(lower);self.row_hi.append(upper)
|
||||
def matrices(self):
|
||||
rr,cc,vv=[],[],[]
|
||||
for row,coefficients in enumerate(self.rows):
|
||||
for col,value in coefficients.items():
|
||||
rr.append(row);cc.append(col);vv.append(value)
|
||||
matrix=coo_matrix((vv,(rr,cc)),shape=(len(self.rows),len(self.lower))).tocsr()
|
||||
return matrix,np.array(self.row_lo),np.array(self.row_hi)
|
||||
|
||||
from .battery_model import BatteryModel
|
||||
|
||||
class AssetRegistry:
|
||||
"""Reviewed asset classes only; future EV/thermal models add their own constraints."""
|
||||
def __init__(self):self.models={Battery:BatteryModel}
|
||||
def register(self,asset_type,implementation):
|
||||
if asset_type in self.models:raise ValueError('Asset model already registered')
|
||||
self.models[asset_type]=implementation
|
||||
def build(self,model,asset,steps,direction):
|
||||
if type(asset) not in self.models:raise ValueError('Unsupported asset model')
|
||||
return self.models[type(asset)].build(model,asset,steps,direction)
|
||||
|
||||
def _error(reason,status='invalid_inputs'):
|
||||
return {'schemaVersion':2,'status':status,'executable':False,'points':[],'reason':str(reason)}
|
||||
|
||||
def _validate(steps,assets,at,past,observed_peaks,peak_prices,limits):
|
||||
if not steps or len(steps)>576:raise ValueError('Need 1..576 steps')
|
||||
for i,step in enumerate(steps):
|
||||
start=utc(step.start)
|
||||
if type(step.seconds) is not int or not 1<=step.seconds<=300:raise ValueError('Interval duration must be 1..300 seconds')
|
||||
if start.microsecond or step.end.second or step.end.microsecond or step.end.minute%5:raise ValueError('Intervals must end on a 5-minute boundary')
|
||||
if i and (step.seconds!=300 or start.second or start.minute%5):raise ValueError('Only first interval may be shortened')
|
||||
if i and start!=steps[i-1].end:raise ValueError('Missing/duplicated/overlapping interval')
|
||||
number(step.base_load_w,'base load',0);number(step.pv_w,'PV',0);number(step.external_w,'external flow')
|
||||
if not step.import_price or not step.export_price or not step.import_price.known_at(at) or not step.export_price.known_at(at):raise ValueError('Unknown or unpublished prices')
|
||||
limits.import_limit(start)
|
||||
if utc(at)!=utc(steps[0].start):raise ValueError('First interval must start at decision time')
|
||||
if steps[-1].end!=quarter_start(steps[-1].end):raise ValueError('End horizon on complete billing quarter')
|
||||
if limits.export_w is not None:number(limits.export_w,'export limit',0)
|
||||
ids=set()
|
||||
for asset in assets:
|
||||
asset.validate(at)
|
||||
if asset.asset_id in ids:raise ValueError('Duplicate asset_id')
|
||||
ids.add(asset.asset_id)
|
||||
q=quarter_start(steps[0].start);elapsed=int((utc(steps[0].start)-q).total_seconds())
|
||||
if any(key!=q for key in past):raise ValueError('Only elapsed energy of first quarter permitted')
|
||||
if not elapsed and q in past and (past[q].import_kwh!=0 or past[q].measured_seconds!=0):raise ValueError('No elapsed energy at quarter boundary')
|
||||
if elapsed:
|
||||
if q not in past or past[q].measured_seconds!=elapsed:raise ValueError('Actual elapsed quarter import energy missing')
|
||||
number(past[q].import_kwh,'quarter import energy',0)
|
||||
for month in {month_key(s.start) for s in steps}:
|
||||
if month not in observed_peaks or month not in peak_prices:raise ValueError(f'Measured peak state or tariff missing for {month}')
|
||||
number(observed_peaks[month],'measured peak',0);number(peak_prices[month],'peak tariff',0)
|
||||
|
||||
def optimize(steps,batteries,*,at,limits=None,observed_peaks=None,peak_prices=None,quarter_history=None,config_revision=1,family='3',timeout_seconds=30.,asset_registry=None,peak_context=None,peak_outlooks=None):
|
||||
"""Pure MILP. Peak state is measured, never a configured cap. No device calls."""
|
||||
limits=limits or Limits();observed_peaks=observed_peaks or {};peak_prices=peak_prices or {}
|
||||
past={utc(k):v for k,v in (quarter_history or {}).items()}
|
||||
try:
|
||||
_validate(steps,batteries,at,past,observed_peaks,peak_prices,limits)
|
||||
number(timeout_seconds,'solver timeout',.01,600)
|
||||
contexts={}
|
||||
for month in {month_key(s.start) for s in steps}:
|
||||
if peak_context is None:
|
||||
contexts[month]={'kw':observed_peaks[month],'quality':'verified','source':'legacy_verified_contract','observedAt':utc(at).isoformat()}
|
||||
else:
|
||||
contexts[month]=basis_record(peak_context[month],month,at,allow_estimates=True)
|
||||
if abs(contexts[month]['kw']-observed_peaks[month])>1e-8:raise ValueError('Peak context differs from numerical basis')
|
||||
outlooks=peak_outlooks or {}
|
||||
if set(outlooks)-set(contexts):raise ValueError('Outlook for month outside horizon')
|
||||
for month,outlook in outlooks.items():
|
||||
if not isinstance(outlook,RestMonthOutlook) or outlook.month!=month:raise ValueError('Invalid peak outlook')
|
||||
outlook.validate(at,steps[-1].end)
|
||||
except (ValueError,TypeError,KeyError,AttributeError) as exc:return _error(exc)
|
||||
model=Model();direction=[model.variable(0,1,integer=True) for _ in steps];registry=asset_registry or AssetRegistry()
|
||||
try:handles={b.asset_id:registry.build(model,b,steps,direction) for b in batteries}
|
||||
except ValueError as exc:return _error(exc)
|
||||
total_charge=sum(b.max_charge_w for b in batteries)/1000
|
||||
total_discharge=sum(b.max_discharge_w for b in batteries)/1000
|
||||
imp,exp,curtail=[],[],[];quarters={}
|
||||
for i,step in enumerate(steps):
|
||||
residual=step.residual_w/1000;dt=step.seconds/3600
|
||||
imax=max(0.,(step.base_load_w+step.external_w)/1000)+total_charge
|
||||
emax=max(0.,-residual)+total_discharge;limit=limits.import_limit(step.start)
|
||||
if limit is not None:imax=min(imax,limit/1000)
|
||||
if limits.export_w is not None:emax=min(emax,limits.export_w/1000)
|
||||
pi=model.variable(0,imax,step.import_price.chf_kwh*dt)
|
||||
pe=model.variable(0,emax,-step.export_price.chf_kwh*dt)
|
||||
pc=model.variable(0,step.pv_w/1000)
|
||||
imp.append(pi);exp.append(pe);curtail.append(pc)
|
||||
gm=model.variable(0,1,integer=True)
|
||||
model.constraint({pi:1,gm:-imax},upper=0);model.constraint({pe:1,gm:emax},upper=emax)
|
||||
balance={pi:1,pe:-1,pc:-1};pv_only={}
|
||||
for b in batteries:
|
||||
h=handles[b.asset_id];balance[h['charge'][i]]=-1;balance[h['discharge'][i]]=1
|
||||
if not b.grid_charging:pv_only[h['charge'][i]]=1
|
||||
model.constraint(balance,residual,residual)
|
||||
if pv_only:
|
||||
model.constraint(pv_only,upper=max(0.,-residual))
|
||||
no_grid_max=sum(b.max_charge_w for b in batteries if not b.grid_charging)/1000
|
||||
model.constraint({**pv_only,gm:no_grid_max},upper=no_grid_max)
|
||||
quarters.setdefault(quarter_start(step.start),[]).append(i)
|
||||
months=sorted({month_key(s.start) for s in steps})
|
||||
peaks={m:model.variable(observed_peaks[m]) for m in months}
|
||||
for m in months:
|
||||
if m in outlooks:outlooks[m].add_to_model(model,peaks[m],observed_peaks[m],peak_prices[m])
|
||||
else:model.cost[peaks[m]]=peak_prices[m]
|
||||
for quarter,positions in quarters.items():
|
||||
coefficients={imp[i]:steps[i].seconds/900 for i in positions};coefficients[peaks[month_key(quarter)]]=-1
|
||||
used=past[quarter].import_kwh if quarter in past else 0.
|
||||
model.constraint(coefficients,upper=-used/.25)
|
||||
matrix,row_lo,row_hi=model.matrices()
|
||||
try:
|
||||
result=milp(np.array(model.cost),integrality=np.array(model.integer),bounds=Bounds(model.lower,model.upper),constraints=LinearConstraint(matrix,row_lo,row_hi),options={'time_limit':float(timeout_seconds),'mip_rel_gap':1e-4})
|
||||
except Exception as exc:return _error(f'Solver exception: {type(exc).__name__}','solver_error')
|
||||
x=result.x
|
||||
if result.status not in (0,1) or x is None:return _error(result.message,'no_feasible_plan')
|
||||
if not np.isfinite(x).all():return _error('Non-finite solver result','validation_failed')
|
||||
ax=matrix@x;tol=1e-6
|
||||
if (np.any(x<np.array(model.lower)-tol) or np.any(x>np.array(model.upper)+tol) or np.any(ax<row_lo-tol) or np.any(ax>row_hi+tol) or any(abs(x[i]-round(x[i]))>tol for i,flag in enumerate(model.integer) if flag)):
|
||||
return _error('Solver incumbent violates constraints','validation_failed')
|
||||
running_peaks=dict(observed_peaks);baseline_peaks=dict(observed_peaks);peak_deltas={};baseline_peak_deltas={}
|
||||
for quarter,positions in quarters.items():
|
||||
used=past[quarter].import_kwh if quarter in past else 0.;m=month_key(quarter)
|
||||
demand=(used+sum(x[imp[i]]*steps[i].seconds/3600 for i in positions))/.25
|
||||
baseline=(used+sum(max(0.,steps[i].residual_w)*steps[i].seconds/3600000 for i in positions))/.25
|
||||
before=running_peaks[m];running_peaks[m]=max(before,demand);peak_deltas[positions[-1]]=(running_peaks[m]-before)*peak_prices[m]
|
||||
before=baseline_peaks[m];baseline_peaks[m]=max(before,baseline);baseline_peak_deltas[positions[-1]]=(baseline_peaks[m]-before)*peak_prices[m]
|
||||
points=[];cumulative_energy=cumulative_cash=cumulative_baseline_energy=cumulative_baseline_cash=cumulative_throughput=0.
|
||||
for i,step in enumerate(steps):
|
||||
dt=step.seconds/3600;p_import=max(0.,float(x[imp[i]]));p_export=max(0.,float(x[exp[i]]))
|
||||
energy_cost=(p_import*step.import_price.chf_kwh-p_export*step.export_price.chf_kwh)*dt
|
||||
baseline_import=max(0.,step.residual_w)/1000;baseline_export=max(0.,-step.residual_w)/1000
|
||||
if limits.export_w is not None:baseline_export=min(baseline_export,limits.export_w/1000)
|
||||
baseline_cost=(baseline_import*step.import_price.chf_kwh-baseline_export*step.export_price.chf_kwh)*dt
|
||||
targets,soc_end={},{};throughput_cost=0.
|
||||
for b in batteries:
|
||||
h=handles[b.asset_id];targets[b.asset_id]=float((x[h['charge'][i]]-x[h['discharge'][i]])*1000)
|
||||
soc_end[b.asset_id]=float(100*x[h['energy'][i+1]]/b.capacity_kwh)
|
||||
throughput_cost+=float((x[h['charge'][i]]+x[h['discharge'][i]])*dt*b.throughput_chf_kwh)
|
||||
cumulative_energy+=energy_cost;cumulative_cash+=energy_cost+peak_deltas.get(i,0.)
|
||||
cumulative_baseline_energy+=baseline_cost;cumulative_baseline_cash+=baseline_cost+baseline_peak_deltas.get(i,0.)
|
||||
cumulative_throughput+=throughput_cost;battery_w=sum(targets.values())
|
||||
if battery_w>1:intent='gridCharge' if p_import>.001 else 'pvCharge'
|
||||
elif battery_w< -1:intent='export' if p_export>.001 else 'discharge'
|
||||
else:intent='hold'
|
||||
points.append({'time':utc(step.start).isoformat(),'validUntil':step.end.isoformat(),
|
||||
'baselineGridW':float(step.residual_w),'gridTargetW':(p_import-p_export)*1000,
|
||||
'batteryTargetW':battery_w,'assetTargetsW':targets,'socEndPercent':soc_end,
|
||||
'pvCurtailmentW':float(x[curtail[i]]*1000),'intent':intent,
|
||||
'importLimitW':limits.import_limit(step.start),'exportLimitW':limits.export_w,
|
||||
'importPriceChfKwh':step.import_price.chf_kwh,'exportPriceChfKwh':step.export_price.chf_kwh,
|
||||
'energyCostChf':energy_cost,'additionalPeakCostChf':peak_deltas.get(i,0.),
|
||||
'throughputCostChf':throughput_cost,'cumulativeEnergyCostChf':cumulative_energy,
|
||||
'cumulativeCashCostChf':cumulative_cash,'cumulativeBaselineEnergyCostChf':cumulative_baseline_energy,
|
||||
'cumulativeBaselineCashCostChf':cumulative_baseline_cash,'baselineAdditionalPeakCostChf':baseline_peak_deltas.get(i,0.)})
|
||||
end_value=sum(float(x[handles[b.asset_id]['energy'][-1]])*b.terminal_value_chf_kwh for b in batteries)
|
||||
def planning_cost(chosen):
|
||||
return sum(outlooks[m].incremental_cost(observed_peaks[m],chosen[m],peak_prices[m]) if m in outlooks
|
||||
else max(0.,chosen[m]-observed_peaks[m])*peak_prices[m] for m in months)
|
||||
planning_peak=planning_cost(running_peaks)
|
||||
full_peak=cumulative_cash-cumulative_energy
|
||||
estimated=any(c['quality']=='estimated' for c in contexts.values())
|
||||
return {'schemaVersion':2,'planId':str(uuid4()),'configRevision':config_revision,'sourceFamily':family,
|
||||
'status':'optimal' if result.status==0 else 'feasible_time_limit','executable':True,
|
||||
'generatedAt':utc(at).isoformat(),'validFrom':points[0]['time'],'validUntil':points[-1]['validUntil'],
|
||||
'intervalMinutes':5,'points':points,'solverGap':float(result.mip_gap) if getattr(result,'mip_gap',None) is not None else None,
|
||||
'measuredPeaksKw':{m:observed_peaks[m] for m in months if contexts[m]['quality']=='verified'},
|
||||
'peakBasisKw':{m:observed_peaks[m] for m in months},'peakBasis':contexts,
|
||||
'peakCostIsEstimate':estimated,'planningPeakCostChf':planning_peak,
|
||||
'restMonthAdjustmentChf':planning_peak-full_peak,
|
||||
'peakScenarioOutlook':{m:o.as_dict() for m,o in outlooks.items()},
|
||||
'plannedPeaksKw':{m:running_peaks[m] for m in months},
|
||||
'additionalPeakCostChf':cumulative_cash-cumulative_energy,'energyCostChf':cumulative_energy,'cashCostChf':cumulative_cash,
|
||||
'baselineEnergyCostChf':cumulative_baseline_energy,'baselineCashCostChf':cumulative_baseline_cash,
|
||||
'baselineAdditionalPeakCostChf':sum(baseline_peak_deltas.values()),'baselinePeaksKw':baseline_peaks,
|
||||
'throughputCostChf':cumulative_throughput,'terminalValueChf':end_value,
|
||||
'objectiveChf':cumulative_energy+planning_peak+cumulative_throughput-end_value,
|
||||
'baselinePlanningPeakCostChf':planning_cost(baseline_peaks),
|
||||
'cashCostMeaning':'Projected horizon energy plus full incremental monthly tariff, relative to the labelled peak basis; not an invoice',
|
||||
'terminalMinSocPercent':{b.asset_id:handles[b.asset_id]['terminal_min_percent'] for b in batteries},
|
||||
'peakOutlook':'rest_month_scenarios' if outlooks else 'full_incremental_tariff'}
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Economic peak policy. Planning assumptions NEVER become metering facts.
|
||||
|
||||
Monthly marginal cost is convex. A rest-month scenario is a peak expected after
|
||||
this horizon under a comparable control policy, not a configured limit, free
|
||||
allowance, or a promise of savings. With insufficient evidence use full tariff.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from math import isclose
|
||||
from .domain import number, utc, month_key, ZURICH
|
||||
|
||||
VERIFIED_SOURCES = frozenset({'meter_month_register','verified_month_history','verified_new_month'})
|
||||
ESTIMATE_SOURCES = frozenset({'power_history_estimate','sampled_power_estimate','counter_interval_estimate','operator_estimate','new_month'})
|
||||
|
||||
|
||||
def valid_month(value):
|
||||
if not isinstance(value,str) or len(value)!=7:
|
||||
raise ValueError('Calendar month YYYY-MM required')
|
||||
if datetime.strptime(value,'%Y-%m').strftime('%Y-%m') != value:
|
||||
raise ValueError('Invalid calendar month')
|
||||
return value
|
||||
|
||||
|
||||
def basis_record(value, month, at, allow_estimates=False):
|
||||
"""Strict provenance; missing stays missing. Caller decides explicit opt-in."""
|
||||
valid_month(month)
|
||||
if not isinstance(value,dict) or set(value)-{'kw','quality','source','observedAt','notes','coverage','meterId'}:
|
||||
raise ValueError('Invalid peak-basis schema')
|
||||
kw=number(value.get('kw'),'peak basis kW',0,1e6)
|
||||
quality=value.get('quality')
|
||||
source=value.get('source')
|
||||
if quality=='verified':
|
||||
if source not in VERIFIED_SOURCES:raise ValueError('Not a verified peak source')
|
||||
elif quality=='estimated':
|
||||
if not allow_estimates or source not in ESTIMATE_SOURCES:
|
||||
raise ValueError('Estimated planning basis not permitted or source invalid')
|
||||
elif quality=='new_month':
|
||||
if source!='new_month' or kw!=0 or month<=month_key(at):
|
||||
raise ValueError('Zero future-month state is not a past measured peak')
|
||||
else:raise ValueError('Peak quality must be explicit')
|
||||
stamp=utc(value.get('observedAt'))
|
||||
if stamp>utc(at):raise ValueError('Peak evidence from the future')
|
||||
if quality!='new_month' and month>month_key(at):
|
||||
raise ValueError('Future peak is an outlook, not historical evidence')
|
||||
notes=value.get('notes','')
|
||||
if not isinstance(notes,str) or len(notes)>500:raise ValueError('Invalid peak notes')
|
||||
coverage=value.get('coverage')
|
||||
if coverage is not None:number(coverage,'peak coverage',0,1)
|
||||
mid=value.get('meterId')
|
||||
if mid is not None and (not isinstance(mid,str) or not 1<=len(mid)<=160):raise ValueError('Invalid meter identity')
|
||||
return {**value,'kw':kw,'quality':quality,'source':source,'observedAt':stamp.isoformat()}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PeakScenario:
|
||||
future_peak_kw: float
|
||||
probability: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RestMonthOutlook:
|
||||
month: str
|
||||
scenarios: tuple[PeakScenario, ...]
|
||||
issued_at: datetime
|
||||
future_from: datetime
|
||||
history_until: datetime
|
||||
control_policy_id: str
|
||||
method: str
|
||||
evidence_id: str
|
||||
reliance: float = 0.5
|
||||
|
||||
def validate(self, at, horizon_end):
|
||||
valid_month(self.month)
|
||||
at=utc(at);end=utc(horizon_end)
|
||||
if utc(self.issued_at)>at or utc(self.history_until)>utc(self.issued_at):
|
||||
raise ValueError('Rest-month outlook contains future information')
|
||||
if at-utc(self.issued_at)>timedelta(days=2):raise ValueError('Stale rest-month outlook')
|
||||
if utc(self.future_from)<end:
|
||||
raise ValueError('Rest-month outlook overlaps optimized horizon')
|
||||
if month_key(self.future_from)!=self.month:
|
||||
raise ValueError('Rest-month outlook outside target calendar month')
|
||||
if not self.control_policy_id or not self.evidence_id or not self.method:
|
||||
raise ValueError('Comparable control policy and historical evidence required')
|
||||
if not 1<=len(self.scenarios)<=100:raise ValueError('Need 1..100 scenarios')
|
||||
for item in self.scenarios:
|
||||
number(item.future_peak_kw,'future scenario peak',0,1e6)
|
||||
number(item.probability,'scenario probability',0,1)
|
||||
if not isclose(sum(s.probability for s in self.scenarios),1.,abs_tol=1e-8):
|
||||
raise ValueError('Scenario probabilities must sum to one')
|
||||
number(self.reliance,'outlook reliance',0,1)
|
||||
|
||||
def incremental_cost(self, basis_kw, planned_kw, tariff):
|
||||
"""Separate planning value, never an already-paid or guaranteed saving."""
|
||||
full=max(0.,planned_kw-basis_kw)*tariff
|
||||
expected=tariff*sum(s.probability*(max(planned_kw,basis_kw,s.future_peak_kw)-max(basis_kw,s.future_peak_kw)) for s in self.scenarios)
|
||||
return (1-self.reliance)*full+self.reliance*expected
|
||||
|
||||
def add_to_model(self, model, peak_variable, basis_kw, tariff):
|
||||
model.cost[peak_variable]+=(1-self.reliance)*tariff
|
||||
for s in self.scenarios:
|
||||
if s.probability<=0:continue
|
||||
end_peak=model.variable(max(basis_kw,s.future_peak_kw),cost=self.reliance*tariff*s.probability)
|
||||
model.constraint({end_peak:1.,peak_variable:-1.},lower=0.)
|
||||
|
||||
def as_dict(self):
|
||||
return {'month':self.month,'issuedAt':utc(self.issued_at).isoformat(),'futureFrom':utc(self.future_from).isoformat(),
|
||||
'historyUntil':utc(self.history_until).isoformat(),'controlPolicyId':self.control_policy_id,'method':self.method,
|
||||
'evidenceId':self.evidence_id,'reliance':self.reliance,
|
||||
'scenarios':[{'peakKw':s.future_peak_kw,'probability':s.probability} for s in self.scenarios]}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value):
|
||||
if set(value)!={'month','issuedAt','futureFrom','historyUntil','controlPolicyId','method','evidenceId','reliance','scenarios'}:
|
||||
raise ValueError('Unexpected rest-month outlook fields')
|
||||
if any(set(s)!={'peakKw','probability'} for s in value['scenarios']):raise ValueError('Unexpected scenario fields')
|
||||
return cls(value['month'],tuple(PeakScenario(s['peakKw'],s['probability']) for s in value['scenarios']),
|
||||
utc(value['issuedAt']),utc(value['futureFrom']),utc(value['historyUntil']),
|
||||
value['controlPolicyId'],value['method'],value['evidenceId'],value['reliance'])
|
||||
|
||||
|
||||
def next_month_start(at):
|
||||
local=utc(at).astimezone(ZURICH)
|
||||
if local.month==12:return utc(local.replace(year=local.year+1,month=1,day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
return utc(local.replace(month=local.month+1,day=1,hour=0,minute=0,second=0,microsecond=0))
|
||||
|
||||
|
||||
def empirical_rest_month(daily_records, *, month, at, horizon_end, control_policy_id, minimum_days=14, reliance=.5):
|
||||
"""Deterministic circular block bootstrap over comparable completed daily peaks.
|
||||
|
||||
Returns None without sufficient historical evidence; never fills with a cap.
|
||||
Input requires valid whole-day coverage and recorded policy identity. Historical
|
||||
measured daily peaks are only a planning proxy for future comparable operation.
|
||||
No fabricated future energy prices are needed for this peak-only outlook.
|
||||
"""
|
||||
valid_month(month)
|
||||
start=max(utc(horizon_end),utc(datetime.strptime(month,'%Y-%m').replace(tzinfo=ZURICH)))
|
||||
end=next_month_start(start)
|
||||
if month_key(start)!=month or start>=end:return None
|
||||
good={}
|
||||
for r in daily_records:
|
||||
if r.get('controlPolicyId')!=control_policy_id or r.get('complete') is not True:continue
|
||||
if r.get('quality') not in ('verified','estimated'):continue
|
||||
day=datetime.strptime(r['day'],'%Y-%m-%d').replace(tzinfo=ZURICH)
|
||||
finished=utc(day+timedelta(days=1))
|
||||
observed=utc(r['observedAt'])
|
||||
if finished>utc(at) or observed>utc(at) or observed<finished:continue
|
||||
if utc(at)-finished>timedelta(days=90):continue
|
||||
kw=number(r['peakKw'],'historical daily peak',0,1e6)
|
||||
if r['day'] in good and good[r['day']]!=kw:raise ValueError('Conflicting daily peak evidence')
|
||||
good[r['day']]=kw
|
||||
if len(good)<minimum_days:return None
|
||||
days=sorted(good)
|
||||
# Use a continuous segment. Missing days cannot be disguised as complete coverage.
|
||||
longest=[];segment=[]
|
||||
for day in days:
|
||||
if segment and datetime.strptime(day,'%Y-%m-%d')-datetime.strptime(segment[-1],'%Y-%m-%d')!=timedelta(days=1):
|
||||
if len(segment)>len(longest):longest=segment
|
||||
segment=[]
|
||||
segment.append(day)
|
||||
if len(segment)>len(longest):longest=segment
|
||||
if len(longest)<minimum_days:return None
|
||||
future_days=max(1,(end.astimezone(ZURICH).date()-start.astimezone(ZURICH).date()).days)
|
||||
values=[good[d] for d in longest]
|
||||
maxima=[max(values[(offset+n)%len(values)] for n in range(future_days)) for offset in range(len(values))]
|
||||
weights={}
|
||||
for value in maxima:weights[value]=weights.get(value,0)+1
|
||||
weights={value:count/len(maxima) for value,count in weights.items()}
|
||||
import hashlib,json
|
||||
evidence=hashlib.sha256(json.dumps({'history':[(d,good[d]) for d in longest],'policy':control_policy_id},sort_keys=True).encode()).hexdigest()
|
||||
return RestMonthOutlook(month,tuple(PeakScenario(k,v) for k,v in sorted(weights.items())),utc(at),start,
|
||||
utc(datetime.strptime(longest[-1],'%Y-%m-%d').replace(tzinfo=ZURICH)+timedelta(days=1)),
|
||||
control_policy_id,'comparable_daily_peak_block_bootstrap',evidence,reliance)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Explicit identity/provenance for the OBSERVATION-ONLY Symcon receiver.
|
||||
This is not a dispatch permit and does not make a shadow plan executable locally.
|
||||
"""
|
||||
from copy import deepcopy
|
||||
|
||||
|
||||
def control_context(operation):
|
||||
assets = {}
|
||||
for b in operation['batteries']:
|
||||
rearm = b.get('rearmSocPercent')
|
||||
assets[b['id']] = {
|
||||
'capacityKwh': b['capacityKwh'],
|
||||
'minSocPercent': b['minSocPercent'],
|
||||
'maxSocPercent': b['maxSocPercent'],
|
||||
'physicalMinSocPercent': b.get('physicalMinSocPercent', 0.0),
|
||||
'rearmSocPercent': b['minSocPercent'] if rearm is None else rearm,
|
||||
'gridCharging': b['gridCharging'],
|
||||
}
|
||||
return {'batteries': assets, 'limits': deepcopy(operation['limits'])}
|
||||
|
||||
|
||||
def provenance(inputs):
|
||||
# Capture BEFORE optimization. Never attach a newer operation to an older plan.
|
||||
return {'inputRefs': {kind: inputs[kind]['eventId'] for kind in ('operation', 'forecast', 'tariffs')},
|
||||
'controlContext': control_context(inputs['operation'])}
|
||||
@@ -0,0 +1,53 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from .domain import default_registry,number,utc,ZURICH
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReplayScore:
|
||||
family:str
|
||||
comparison_key:str
|
||||
first_decision:object
|
||||
last_decision:object
|
||||
forecast_issued_at:object
|
||||
actual_available_at:object
|
||||
cost_chf:float
|
||||
coverage:float
|
||||
days:int
|
||||
constraint_breaches:int=0
|
||||
terminal_normalized:bool=True
|
||||
def valid(self):
|
||||
number(self.cost_chf,'replay cost');number(self.coverage,'coverage',0,1)
|
||||
return (utc(self.forecast_issued_at)<=utc(self.first_decision) and utc(self.actual_available_at)>=utc(self.last_decision) and self.terminal_normalized and self.constraint_breaches==0)
|
||||
|
||||
def choose_family(setting,current,scores,*,registry=None,lookback_days=14,minimum_days=7,minimum_coverage=.9,margin_chf=1.,now):
|
||||
registry=registry or default_registry()
|
||||
if setting!='auto':
|
||||
registry.get(setting)
|
||||
return {'family':setting,'mode':'configured','reason':'Explicit installation setting'}
|
||||
registry.get(current);number(margin_chf,'margin',0);valid=[]
|
||||
for score in scores:
|
||||
registry.get(score.family)
|
||||
if (score.valid() and score.coverage>=minimum_coverage and score.days>=minimum_days
|
||||
and utc(score.first_decision)>=utc(now)-timedelta(days=lookback_days)
|
||||
and utc(score.last_decision)<=utc(now) and utc(score.actual_available_at)<=utc(now)):
|
||||
valid.append(score)
|
||||
groups={}
|
||||
for score in valid:
|
||||
key=(score.comparison_key,utc(score.first_decision),utc(score.last_decision),score.days)
|
||||
groups.setdefault(key,{})[score.family]=score
|
||||
complete=[g for g in groups.values() if len(g)==len(registry.entries())]
|
||||
if not complete:return {'family':current,'mode':'collecting','reason':'Insufficient comparable out-of-sample replay evidence'}
|
||||
group=max(complete,key=lambda g:utc(next(iter(g.values())).last_decision))
|
||||
best=min(group,key=lambda f:(group[f].cost_chf,f));improvement=group[current].cost_chf-group[best].cost_chf
|
||||
chosen=best if improvement>margin_chf else current
|
||||
return {'family':chosen,'mode':'economic_replay','improvementChf':improvement,'reason':'Matched historical cost replay; switching margin applied','costByFamilyChf':{k:v.cost_chf for k,v in group.items()}}
|
||||
|
||||
def training_due(last_trained_at,now,cadence='daily'):
|
||||
if cadence not in ('daily','weekly'):raise ValueError('Unknown training cadence')
|
||||
if last_trained_at is None:return True
|
||||
now,last=utc(now).astimezone(ZURICH),utc(last_trained_at).astimezone(ZURICH)
|
||||
return (now.date()-last.date()).days >= (1 if cadence=='daily' else 7)
|
||||
|
||||
def promote_candidate(*,active_cost,candidate_cost,valid_coverage,no_data_leakage,constraints_passed):
|
||||
number(active_cost,'active cost');number(candidate_cost,'candidate cost')
|
||||
return bool(valid_coverage and no_data_leakage and constraints_passed and candidate_cost<active_cost)
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Isolated V4 service: immutable inputs, coalesced replan queue, SHADOW publication.
|
||||
No external actuator endpoint, no reads of users.db, no legacy schedule changes.
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import asdict,replace
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
from threading import Event,Thread
|
||||
from uuid import UUID
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import logging
|
||||
from fastapi import FastAPI,Header,HTTPException
|
||||
from .domain import Battery,Limits,Price,QuarterPast,Step,month_key,quarter_start,utc,number,priced_prefix
|
||||
from .store import PlannerStore,canonical
|
||||
from .selection import choose_family
|
||||
from .optimizer import optimize
|
||||
from . import meter_runtime, controlled_trial, measurement_pipeline
|
||||
from .forecast_quality import assess_family
|
||||
from .receiver_contract import provenance
|
||||
from .peak_policy import basis_record, RestMonthOutlook, empirical_rest_month
|
||||
|
||||
KINDS={'forecast','operation','tariffs','prices','planning_basis','peak_outlook'}
|
||||
|
||||
def latest(store,plant,kind):
|
||||
row=store.con.execute('''SELECT value FROM planner_inputs i JOIN planner_input_current c
|
||||
ON i.plant=c.plant AND i.kind=c.kind AND i.event_id=c.event_id WHERE i.plant=? AND i.kind=?''',(plant,kind)).fetchone()
|
||||
return json.loads(row[0]) if row else None
|
||||
|
||||
def batteries(value):
|
||||
result=[]
|
||||
for b in value:
|
||||
result.append(Battery(asset_id=b['id'],capacity_kwh=b['capacityKwh'],soc_percent=b['socPercent'],
|
||||
min_soc_percent=b['minSocPercent'],max_soc_percent=b['maxSocPercent'],
|
||||
max_charge_w=b['maxChargeW'],max_discharge_w=b['maxDischargeW'],measured_at=utc(b['measuredAt']),
|
||||
grid_charging=b['gridCharging'],throughput_chf_kwh=b.get('throughputChfKwh',0.),
|
||||
terminal_soc_min_percent=b.get('terminalMinSocPercent'),terminal_value_chf_kwh=b.get('terminalValueChfKwh',0.),
|
||||
recovery_allowed=b.get('recoveryAllowed',False),physical_min_soc_percent=b.get('physicalMinSocPercent',0.),
|
||||
discharge_blocked=b.get('dischargeBlocked',False),rearm_soc_percent=b.get('rearmSocPercent')))
|
||||
if type(b['gridCharging']) is not bool:raise ValueError('Explicit boolean grid-charging permission required')
|
||||
if len(result)>20 or len({b.asset_id for b in result})!=len(result):raise ValueError('Duplicate/too many batteries')
|
||||
return result
|
||||
|
||||
def validate(kind,value,now,registry):
|
||||
if kind not in KINDS or type(value.get('version')) is not int or value['version']!=1:raise ValueError('Unsupported event version/kind')
|
||||
identifier=value['eventId']
|
||||
if not isinstance(identifier,str) or not 1<=len(identifier)<=160:raise ValueError('Event ID required')
|
||||
observed=utc(value['observedAt'])
|
||||
if observed>now+timedelta(seconds=30):raise ValueError('Observation from future')
|
||||
fields={'version','eventId','observedAt'}
|
||||
if kind=='forecast':
|
||||
fields|={'families','modelVersions'}
|
||||
if not value['families']:raise ValueError('No forecast families')
|
||||
for key,family in value['families'].items():
|
||||
registry.get(key)
|
||||
if family['loadBasis'] not in ('base_load','house_total'):raise ValueError('Explicit metering basis required')
|
||||
evidence=family.get('accountingEvidenceId')
|
||||
if evidence is not None and (family['loadBasis']!='base_load' or not isinstance(evidence,str) or not 8<=len(evidence)<=160):raise ValueError('Explicit base-load accounting evidence required')
|
||||
if family.get('trainedUntil') and utc(family['trainedUntil'])>observed:raise ValueError('Training leakage')
|
||||
if not 1<=len(family['points'])<=576:raise ValueError('Need 1..576 forecast intervals')
|
||||
previous=None
|
||||
for p in family['points']:
|
||||
t=utc(p['time'])
|
||||
if t.second or t.microsecond or t.minute%5:raise ValueError('Forecast interval alignment')
|
||||
if previous and t-previous!=timedelta(minutes=5):raise ValueError('Forecast gap/overlap')
|
||||
previous=t;number(p['pvW'],'PV',0,1e9);number(p['loadW'],'load',0,1e9);number(p.get('externalW',0.),'external',-1e9,1e9)
|
||||
elif kind=='operation':
|
||||
fields|={'gridW','meteringBoundary','batteries','limits','measuredPeaks','quarterPast','planningPeaks','quarterEstimate','meterObservation'}
|
||||
if value['meteringBoundary']!='common_pcc':raise ValueError('Common metering boundary required')
|
||||
number(value['gridW'],'grid W',-1e9,1e9)
|
||||
for b in batteries(value['batteries']):b.validate(observed)
|
||||
limits=value['limits']
|
||||
if set(limits)!={'importW','exportW','managerMonthLimitsW'}:raise ValueError('Explicit limits required; null unlimited, zero zero')
|
||||
for k in ('importW','exportW'):
|
||||
if limits[k] is not None:number(limits[k],k,0,1e9)
|
||||
for k,v in limits['managerMonthLimitsW'].items():
|
||||
if str(int(k))!=k or not 1<=int(k)<=12:raise ValueError('Invalid manager month')
|
||||
number(v,'manager limit',0,1e9)
|
||||
for m,p in value.get('measuredPeaks',{}).items():
|
||||
datetime.strptime(m,'%Y-%m');number(p['kw'],'peak kW',0)
|
||||
if m>month_key(observed) or p['source'] not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Measured peak source invalid; cap is not paid peak')
|
||||
past=value.get('quarterPast')
|
||||
if past:
|
||||
q=quarter_start(observed)
|
||||
if utc(past['start'])!=q or type(past['measuredSeconds']) is not int or past['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Quarter measurement timestamp mismatch')
|
||||
number(past['importKwh'],'quarter energy',0)
|
||||
for m,p in value.get('planningPeaks',{}).items():
|
||||
record=basis_record(p,m,observed,allow_estimates=True)
|
||||
if record['quality']!='estimated':raise ValueError('planningPeaks contains estimates only')
|
||||
estimate=value.get('quarterEstimate')
|
||||
if estimate:
|
||||
if set(estimate)-{'start','measuredSeconds','importKwh','quality','source','coverage','notes'}:raise ValueError('Unknown quarter estimate field')
|
||||
if estimate.get('quality')!='estimated' or estimate.get('source') not in ('power_history_estimate','sampled_power_estimate','counter_interval_estimate'):raise ValueError('Explicit quarter estimate provenance required')
|
||||
q=quarter_start(observed)
|
||||
if utc(estimate['start'])!=q or type(estimate['measuredSeconds']) is not int or estimate['measuredSeconds']!=int((observed-q).total_seconds()):raise ValueError('Estimated quarter timing mismatch')
|
||||
number(estimate['importKwh'],'estimated quarter energy',0)
|
||||
if number(estimate.get('coverage'),'quarter estimate coverage',0,1)<1.:raise ValueError('Missing current-quarter coverage')
|
||||
if value.get('meterObservation') is not None:meter_runtime.validate_observation(value['meterObservation'],observed)
|
||||
elif kind=='planning_basis':
|
||||
fields|={'peaks'}
|
||||
if not isinstance(value.get('peaks'),dict) or not value['peaks']:raise ValueError('Explicit peak estimates required')
|
||||
for m,p in value['peaks'].items():
|
||||
if basis_record(p,m,observed,allow_estimates=True)['quality']!='estimated':raise ValueError('Planning basis is not a metering import')
|
||||
elif kind=='peak_outlook':
|
||||
fields|={'outlooks'}
|
||||
for m,v in value['outlooks'].items():
|
||||
outlook=RestMonthOutlook.from_dict(v)
|
||||
if m!=outlook.month:raise ValueError('Outlook month mismatch')
|
||||
outlook.validate(observed,observed)
|
||||
elif kind=='tariffs':
|
||||
fields|={'import','export','peakChfKwMonth'}
|
||||
for side in ('import','export'):
|
||||
p=value[side]
|
||||
if p['mode'] not in ('static','dynamic') or not p['tariffId']:raise ValueError('Explicit price mode/id required')
|
||||
if p['mode']=='static':number(p['staticChfKwh'],'static price')
|
||||
peaks=value['peakChfKwMonth']
|
||||
if isinstance(peaks,dict):
|
||||
for m,v in peaks.items():datetime.strptime(m,'%Y-%m');number(v,'peak tariff',0)
|
||||
else:number(peaks,'peak tariff',0)
|
||||
else:
|
||||
fields|={'periods'}
|
||||
if len(value['periods'])>3000:raise ValueError('Too many price intervals')
|
||||
for p in value['periods']:
|
||||
if p['unit'] not in ('CHF/kWh','CHF_kWh','Rp/kWh','CHF/MWh') or p['side'] not in ('import','export'):raise ValueError('Explicit price unit/direction required')
|
||||
if p['sourceKind'] not in ('published_interval','estimate','carried_forward'):raise ValueError('Explicit price provenance required')
|
||||
number(p['value'],'price')
|
||||
if utc(p['end'])<=utc(p['start']) or utc(p['observedAt'])>observed:raise ValueError('Invalid price interval or observation')
|
||||
if p.get('publishedAt') and utc(p['publishedAt'])>utc(p['observedAt']):raise ValueError('Price not published when observed')
|
||||
if set(value)-fields:raise ValueError('Unknown fields: extra device data/credentials must not be submitted')
|
||||
canonical(value)
|
||||
|
||||
def ingest(store,plant,kind,value,now):
|
||||
validate(kind,value,now,store.registry);data=canonical(value);con=store.con;con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
old=con.execute('SELECT value FROM planner_inputs WHERE plant=? AND kind=? AND event_id=?',(plant,kind,value['eventId'])).fetchone()
|
||||
if old:
|
||||
if old[0]!=data:raise ValueError('Immutable event conflict')
|
||||
con.commit();return {'status':'duplicate','queued':False}
|
||||
current=latest(store,plant,kind)
|
||||
if current and utc(current['observedAt'])==utc(value['observedAt']):
|
||||
left,right=dict(current),dict(value);left.pop('eventId');right.pop('eventId')
|
||||
if canonical(left)!=canonical(right):raise ValueError('Conflicting simultaneous observations')
|
||||
con.execute('INSERT INTO planner_inputs VALUES(?,?,?,?,?)',(plant,kind,value['eventId'],utc(value['observedAt']).isoformat(timespec='microseconds'),data))
|
||||
newer=not current or utc(current['observedAt'])<=utc(value['observedAt'])
|
||||
if newer:
|
||||
if kind=='operation':
|
||||
known=store.peaks(plant)
|
||||
for m,p in value.get('measuredPeaks',{}).items():
|
||||
if m in known and p['kw']<known[m]-1e-9:raise ValueError('Measured peak decreased')
|
||||
con.execute('''INSERT INTO planner_month_peaks VALUES(?,?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET peak_kw=excluded.peak_kw,source=excluded.source,updated_at=excluded.updated_at''',(plant,m,p['kw'],p['source'],utc(now).isoformat()))
|
||||
if kind=='operation':
|
||||
for m,p in value.get('planningPeaks',{}).items():meter_runtime.save_assumption(con,plant,m,p,now)
|
||||
if value.get('meterObservation'):meter_runtime.observe(con,plant,value['meterObservation'],utc(value['observedAt']))
|
||||
elif kind=='planning_basis':
|
||||
for m,p in value['peaks'].items():meter_runtime.save_assumption(con,plant,m,p,now)
|
||||
con.execute('INSERT INTO planner_input_current VALUES(?,?,?) ON CONFLICT(plant,kind) DO UPDATE SET event_id=excluded.event_id',(plant,kind,value['eventId']))
|
||||
store._request(plant,store.settings(plant)['revision'],kind+'_changed',now)
|
||||
con.commit();return {'status':'stored' if newer else 'archived_older','queued':newer}
|
||||
except Exception:con.rollback();raise
|
||||
|
||||
def price_at(store,plant,side,config,start,end,now):
|
||||
if config['mode']=='static':return Price(config['staticChfKwh'])
|
||||
rows=store.con.execute('''SELECT value FROM planner_inputs WHERE plant=? AND kind='prices' AND observed_at<=? ORDER BY observed_at DESC''',(plant,utc(now).isoformat(timespec='microseconds')))
|
||||
candidates=[]
|
||||
for row in rows:
|
||||
for p in json.loads(row[0])['periods']:
|
||||
if (p['tariffId']==config['tariffId'] and p['side']==side and p['sourceKind']=='published_interval'
|
||||
and utc(p['start'])<=start and utc(p['end'])>=end and utc(p['observedAt'])<=now):
|
||||
factor={'CHF/kWh':1.,'CHF_kWh':1.,'Rp/kWh':.01,'CHF/MWh':.001}[p['unit']]
|
||||
candidates.append((utc(p['observedAt']),p['value']*factor))
|
||||
if not candidates:return None
|
||||
last=max(t for t,v in candidates);values={v for t,v in candidates if t==last}
|
||||
if len(values)!=1:raise ValueError('Conflicting published price intervals')
|
||||
return Price(values.pop(),last,'dynamic')
|
||||
|
||||
class AwaitingInput(ValueError):pass
|
||||
|
||||
def assemble(store,plant,family,now):
|
||||
values={k:latest(store,plant,k) for k in ('operation','forecast','tariffs')}
|
||||
for k,v in values.items():
|
||||
if not v:raise AwaitingInput('Missing '+k+' input')
|
||||
op,forecast,tariffs=(values[k] for k in ('operation','forecast','tariffs'))
|
||||
decision=utc(op['observedAt']).replace(microsecond=0)
|
||||
settings=store.settings(plant);allow_estimates=settings['measurementPolicy']=='allow_estimates'
|
||||
input_quality={'quarter':'verified','warnings':[]}
|
||||
if not 0<=(now-decision).total_seconds()<=120:raise AwaitingInput('Fresh manager observation required (120s maximum)')
|
||||
if (now-utc(forecast['observedAt'])).total_seconds()>5400:raise AwaitingInput('Forecast older than 90 minutes')
|
||||
if utc(forecast['observedAt'])>decision or utc(tariffs['observedAt'])>decision:raise AwaitingInput('New data awaiting fresh manager observation')
|
||||
if settings['forecastSource']=='corrected_profile':
|
||||
try:forecast=measurement_pipeline.apply_load_forecast(store.con,plant,settings['measurementDataset'],forecast,int(decision.timestamp()))
|
||||
except ValueError as exc:raise AwaitingInput(str(exc)) from exc
|
||||
if family not in forecast['families']:raise AwaitingInput('Chosen family is unavailable; no silent switch')
|
||||
source=forecast['families'][family];steps=[]
|
||||
future_source={**source,'points':[p for p in source['points'] if utc(p['time'])+timedelta(minutes=5)>decision]}
|
||||
assessment=assess_family(future_source)
|
||||
if not assessment['valid']:raise AwaitingInput(assessment['reason'])
|
||||
input_quality['forecastAssessment']=assessment
|
||||
if source.get('dataPipeline'):
|
||||
input_quality['dataPipeline']=source['dataPipeline']
|
||||
input_quality['warnings'].append('Corrected physical-load profile uses configured measurement mapping; external SDL is an explicitly labelled last-request persistence scenario, not a published future SDL schedule')
|
||||
for p in source['points']:
|
||||
start=utc(p['time']);end=start+timedelta(minutes=5)
|
||||
if end<=decision:continue
|
||||
start=max(start,decision)
|
||||
external=p.get('externalW',0.) if source['loadBasis']=='base_load' else 0.
|
||||
steps.append(Step(start,p['loadW'],p['pvW'],price_at(store,plant,'import',tariffs['import'],start,end,decision),price_at(store,plant,'export',tariffs['export'],start,end,decision),external,int((end-start).total_seconds())))
|
||||
if not steps or steps[0].start!=decision:raise AwaitingInput('Forecast has no current interval')
|
||||
full_end=steps[-1].end;steps=priced_prefix(steps,decision)
|
||||
if not steps:raise AwaitingInput('No complete published-price billing quarter')
|
||||
q=quarter_start(decision);elapsed=int((decision-q).total_seconds());past={}
|
||||
if elapsed:
|
||||
p=op.get('quarterPast')
|
||||
if not p and allow_estimates:
|
||||
p=op.get('quarterEstimate')
|
||||
if not p and op.get('meterObservation'):p=meter_runtime.current_quarter(store.con,plant,op['meterObservation'],decision)
|
||||
if p:
|
||||
input_quality['quarter']='estimated'
|
||||
input_quality['warnings'].append('Current quarter uses an explicitly estimated energy value, not a billing measurement')
|
||||
if not p or utc(p['start'])!=q or p['measuredSeconds']!=elapsed:raise AwaitingInput('Current-quarter energy missing (measured or explicitly permitted estimate)')
|
||||
past[q]=QuarterPast(p['importKwh'],elapsed)
|
||||
peaks=store.peaks(plant);months={month_key(s.start) for s in steps};contexts={}
|
||||
estimates=meter_runtime.assumptions(store.con,plant)
|
||||
for m in months:
|
||||
if m>month_key(decision):
|
||||
peaks[m]=0.;contexts[m]={'kw':0.,'quality':'new_month','source':'new_month','observedAt':decision.isoformat()}
|
||||
elif m in peaks:
|
||||
r=store.con.execute('SELECT source,updated_at FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone()
|
||||
known_at=op['observedAt'] if m in op.get('measuredPeaks',{}) else r['updated_at']
|
||||
contexts[m]={'kw':peaks[m],'quality':'verified','source':r['source'],'observedAt':known_at}
|
||||
# A later acquired larger quarter may raise an older verified baseline,
|
||||
# but the resulting combined planning basis must then say estimated.
|
||||
if allow_estimates and m in estimates and estimates[m]['kw']>peaks[m] and utc(estimates[m]['observedAt'])>utc(known_at):
|
||||
contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True)
|
||||
peaks[m]=contexts[m]['kw']
|
||||
input_quality['warnings'].append('A newer sampled quarter increased the earlier verified peak baseline; current planning maximum is estimated')
|
||||
elif allow_estimates and m in estimates:
|
||||
contexts[m]=basis_record(estimates[m],m,decision,allow_estimates=True);peaks[m]=contexts[m]['kw']
|
||||
input_quality['warnings'].append('Monthly peak '+m+' is a planning estimate, not an authoritative billing maximum')
|
||||
else:raise AwaitingInput('Peak basis missing; supply a verified maximum or explicitly permit a labelled estimate')
|
||||
peak_prices=tariffs['peakChfKwMonth']
|
||||
if not isinstance(peak_prices,dict):peak_prices={m:peak_prices for m in months}
|
||||
lim=op['limits'];limits=Limits(lim['exportW'],lim['importW'],{int(k):v for k,v in lim['managerMonthLimitsW'].items()})
|
||||
outlooks={};horizon_end=steps[-1].end
|
||||
if settings['peakOutlookPolicy']=='empirical_if_available':
|
||||
explicit=latest(store,plant,'peak_outlook')
|
||||
for m in months:
|
||||
if explicit and m in explicit['outlooks']:
|
||||
candidate=RestMonthOutlook.from_dict(explicit['outlooks'][m])
|
||||
try:
|
||||
candidate.validate(decision,horizon_end)
|
||||
policy=op.get('meterObservation',{}).get('controlPolicyId')
|
||||
if not policy or candidate.control_policy_id!=policy:raise ValueError('Different or unknown control policy')
|
||||
except ValueError:input_quality['warnings'].append('Stale, overlapping or incomparable rest-month outlook ignored; full incremental tariff used')
|
||||
else:outlooks[m]=replace(candidate,reliance=min(candidate.reliance,settings['peakOutlookReliance']))
|
||||
elif op.get('meterObservation'):
|
||||
observation=op['meterObservation']
|
||||
candidate=empirical_rest_month(meter_runtime.daily_peaks(store.con,plant,observation['meterId'],decision),
|
||||
month=m,at=decision,horizon_end=horizon_end,control_policy_id=observation['controlPolicyId'],
|
||||
reliance=settings['peakOutlookReliance'])
|
||||
if candidate is not None:outlooks[m]=candidate
|
||||
if not outlooks:input_quality['warnings'].append('Insufficient comparable rest-month history; full incremental peak tariff used, no arbitrary free peak allowance')
|
||||
data={'steps':steps,'batteries':batteries(op['batteries']),'limits':limits,'observed_peaks':peaks,'peak_prices':peak_prices,
|
||||
'quarter_history':past,'at':decision,'peak_context':contexts,'peak_outlooks':outlooks}
|
||||
if source['loadBasis']=='base_load' and source.get('accountingEvidenceId'):
|
||||
input_quality['accountingEvidenceId']=source['accountingEvidenceId']
|
||||
return data,full_end,{'loadBasis':source['loadBasis'],**input_quality,**provenance(values)}
|
||||
|
||||
def run_once(store,now):
|
||||
stamp=int(now.timestamp())//300
|
||||
plants=[r[0] for r in store.con.execute('SELECT plant FROM planner_settings UNION SELECT DISTINCT plant FROM planner_input_current UNION SELECT plant FROM planner_data_sets')]
|
||||
for plant in plants:
|
||||
config=store.settings(plant)
|
||||
datasets=[r[0] for r in store.con.execute('SELECT dataset FROM planner_data_sets WHERE plant=?',(plant,))]
|
||||
for dataset in datasets:
|
||||
try:measurement_pipeline.advance(store.con,plant,dataset,config,int(now.timestamp()))
|
||||
except ValueError as exc:logging.getLogger(__name__).warning('Data pipeline unavailable for configured dataset: %s',type(exc).__name__)
|
||||
with store.con:
|
||||
old=store.con.execute('SELECT tick FROM planner_ticks WHERE plant=?',(plant,)).fetchone()
|
||||
if not old or old[0]!=stamp:
|
||||
store._request(plant,store.settings(plant)['revision'],'five_minute_tick',now)
|
||||
store.con.execute('INSERT INTO planner_ticks VALUES(?,?) ON CONFLICT(plant) DO UPDATE SET tick=excluded.tick',(plant,stamp))
|
||||
claim=store.claim(now)
|
||||
if not claim:return {'status':'idle'}
|
||||
plant=claim['plant'];result={'status':'internal_error','executable':False,'points':[]}
|
||||
try:
|
||||
settings=store.settings(plant);previous=store.current(plant)
|
||||
current=previous['sourceFamily'] if previous else store.registry.entries()[0].key
|
||||
# Productive replay ingestion is intentionally not fabricated from R2 metrics.
|
||||
selection=choose_family(settings['family'],current,(),registry=store.registry,now=now)
|
||||
data,full_end,quality=assemble(store,plant,selection['family'],now)
|
||||
data['batteries']=[replace(b,roundtrip_efficiency=settings['roundtripEfficiency']) for b in data['batteries']]
|
||||
result=optimize(**data,config_revision=settings['revision'],family=selection['family'])
|
||||
if result['executable']:
|
||||
result.update({'installationId':plant,'inputRefs':quality.pop('inputRefs'),'controlContext':quality.pop('controlContext'),'runMode':'shadow','liveEnabled':False,'sourceSelection':selection,'forecastUntil':full_end.isoformat(),'pricesKnownUntil':result['validUntil'],'inputQuality':quality,'warnings':quality['warnings']+([] if quality['loadBasis']=='base_load' else ['Aggregate house forecast: base-load/SDL separation not verified; shadow only'])})
|
||||
store.publish_shadow(plant,result,settings['revision'],now,claim['sequence'],claim['lease_token'])
|
||||
return result
|
||||
except (ValueError,TypeError,KeyError) as exc:
|
||||
result={'status':'awaiting_inputs' if isinstance(exc,AwaitingInput) else 'invalid_inputs','reason':str(exc)[:300],'executable':False,'points':[]}
|
||||
return result
|
||||
finally:
|
||||
detail={k:v for k,v in result.items() if k in ('status','reason','planId','configRevision','sourceFamily')}
|
||||
with store.con:store.con.execute('INSERT INTO planner_run_status VALUES(?,?,?,?) ON CONFLICT(plant) DO UPDATE SET updated_at=excluded.updated_at,status=excluded.status,detail=excluded.detail',(plant,now.isoformat(),result['status'],canonical(detail)))
|
||||
store.finish(claim)
|
||||
|
||||
def status(store,plant,now):
|
||||
plan=store.current(plant);settings=store.settings(plant)
|
||||
row=store.con.execute('SELECT * FROM planner_run_status WHERE plant=?',(plant,)).fetchone()
|
||||
pending=store.con.execute('SELECT reasons,requested_at FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
ack=store.con.execute('SELECT * FROM planner_ack WHERE plant=?',(plant,)).fetchone()
|
||||
fresh=bool(plan and plan['configRevision']==settings['revision'] and utc(plan['validUntil'])>now and 0<=(now-utc(plan['generatedAt'])).total_seconds()<=900 and not pending and row and row['status'] in ('optimal','feasible_time_limit'))
|
||||
return {'receiverProtocolVersion':1,'installationId':plant,'checkedAt':utc(now).isoformat(),'settings':settings,'peakPlanningBases':meter_runtime.assumptions(store.con,plant),'families':[asdict(f) for f in store.registry.entries()],'plan':plan,'fresh':fresh,'pending':dict(pending) if pending else None,'lastRun':{**dict(row),'detail':json.loads(row['detail'])} if row else None,'acknowledgement':dict(ack) if ack else None,'liveEnabled':False,'dataPipeline':measurement_pipeline.pipeline_status(store.con,plant)}
|
||||
|
||||
def create_app(db_path,service_token,plants,*,start_worker=True,controlled_trial_plants=()):
|
||||
allowed={str(UUID(p)) for p in plants}
|
||||
trial_allowed={str(UUID(p)) for p in controlled_trial_plants}
|
||||
if not trial_allowed <= allowed:raise ValueError('Trial allowlist must be a subset of plant allowlist')
|
||||
if not allowed or not service_token or len(service_token)<24:raise ValueError('Private service token and explicit plant allowlist required')
|
||||
path=Path(db_path).resolve()
|
||||
if path.name in ('users.db','portal.sqlite','settings.json'):raise ValueError('Dedicated planner database required')
|
||||
path.parent.mkdir(parents=True,exist_ok=True);stop=Event()
|
||||
def factory():return PlannerStore(str(path))
|
||||
def loop():
|
||||
while not stop.is_set():
|
||||
s=factory()
|
||||
try:run_once(s,datetime.now(timezone.utc).replace(microsecond=0))
|
||||
except Exception as exc:logging.getLogger(__name__).error('V4 worker error: %s',type(exc).__name__)
|
||||
finally:s.close()
|
||||
stop.wait(1.)
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
thread=Thread(target=loop,name='v4-shadow',daemon=True)
|
||||
if start_worker:thread.start()
|
||||
yield
|
||||
stop.set()
|
||||
if start_worker:thread.join(35)
|
||||
app=FastAPI(title='ENELIX V4 - Schattenbetrieb',lifespan=lifespan)
|
||||
app.state.store_factory=factory
|
||||
def authorize(plant,token):
|
||||
if not secrets.compare_digest(token or '',service_token):raise HTTPException(401,'Unauthorized')
|
||||
try:plant=str(UUID(plant))
|
||||
except ValueError:raise HTTPException(400,'Invalid installation ID')
|
||||
if plant not in allowed:raise HTTPException(403,'Installation not enabled for shadow trial')
|
||||
return factory()
|
||||
@app.get('/health')
|
||||
def health():return {'status':'ok','mode':'shadow','liveEnabled':False,'receiverProtocolVersion':1}
|
||||
@app.get('/internal/v2/prognosis/{plant}/planner')
|
||||
def read(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc);view=status(s,plant,now)
|
||||
view['controlledTrial']=controlled_trial.authority(s,plant,view,now,trial_allowed)
|
||||
view['controlledTrialAuthorized']=view['controlledTrial'] is not None
|
||||
return view
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/trial/arm')
|
||||
def arm_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc)
|
||||
return controlled_trial.arm(s,plant,payload,status(s,plant,now),now,trial_allowed)
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/trial/revoke')
|
||||
def revoke_trial(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
if set(payload)!={'sessionId'}:raise ValueError('Session ID only')
|
||||
return controlled_trial.revoke(s,plant,payload['sessionId'],datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.put('/internal/v2/prognosis/{plant}/planner/settings')
|
||||
def save(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:return s.save_settings(plant,payload['changes'],payload['expectedRevision'],datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(409 if 'Revision conflict' in str(exc) else 400,str(exc))
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/replan')
|
||||
def replan(plant:str,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:s.request(plant,'manual',datetime.now(timezone.utc));return {'status':'queued','liveEnabled':False}
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/inputs/{kind}')
|
||||
def input_event(plant:str,kind:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:return ingest(s,plant,kind,payload,datetime.now(timezone.utc))
|
||||
except (ValueError,KeyError,TypeError,AttributeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
@app.put('/internal/v2/prognosis/{plant}/planner/datasets/{dataset}')
|
||||
def configure_dataset(plant:str,dataset:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
if payload.get('datasetId')!=dataset:raise ValueError('Dataset path/payload mismatch')
|
||||
return measurement_pipeline.register_dataset(s.con,plant,payload,int(datetime.now(timezone.utc).timestamp()))
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/measurements')
|
||||
def measurement_batch(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:
|
||||
now=datetime.now(timezone.utc)
|
||||
result=measurement_pipeline.ingest_batch(s.con,plant,payload,int(now.timestamp()))
|
||||
# Existing five-minute worker handles rollup/training; no per-record optimizer flood.
|
||||
return result
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:200])
|
||||
finally:s.close()
|
||||
@app.post('/internal/v2/prognosis/{plant}/planner/ack')
|
||||
def ack(plant:str,payload:dict,token:str=Header(default='',alias='X-Enelix-Service-Token')):
|
||||
s=authorize(plant,token)
|
||||
try:s.acknowledge(plant,payload['planId'],payload['revision'],datetime.now(timezone.utc),payload.get('step'),payload.get('status','shadow_seen'));return {'status':'recorded'}
|
||||
except (ValueError,KeyError,TypeError) as exc:raise HTTPException(400,str(exc)[:300])
|
||||
finally:s.close()
|
||||
from starlette.responses import JSONResponse
|
||||
class BodyLimit:
|
||||
def __init__(self,app):self.app=app
|
||||
async def __call__(self,scope,receive,send):
|
||||
if scope['type']!='http' or scope['method'] not in ('POST','PUT'):return await self.app(scope,receive,send)
|
||||
chunks=[];total=0
|
||||
while True:
|
||||
msg=await receive()
|
||||
if msg['type']=='http.disconnect':return
|
||||
total+=len(msg.get('body',b''))
|
||||
if total>2000000:return await JSONResponse({'detail':'Request too large'},status_code=413)(scope,receive,send)
|
||||
chunks.append(msg)
|
||||
if not msg.get('more_body',False):break
|
||||
async def replay():return chunks.pop(0) if chunks else await receive()
|
||||
return await self.app(scope,replay,send)
|
||||
app.add_middleware(BodyLimit)
|
||||
return app
|
||||
|
||||
def from_environment():
|
||||
return create_app(os.environ.get('NETPLAN_V4_DB','/data/netplan-v4.sqlite'),os.environ.get('PROGNOSIS_SERVICE_TOKEN',''),[p.strip() for p in os.environ.get('NETPLAN_V4_PLANTS','').split(',') if p.strip()], controlled_trial_plants=[p.strip() for p in os.environ.get('NETPLAN_V4_CONTROL_TRIAL_PLANTS','').split(',') if p.strip()])
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime,timedelta
|
||||
from uuid import uuid4
|
||||
from .domain import default_registry,month_key,number,quarter_start,utc
|
||||
from . import meter_runtime, controlled_trial, measurement_pipeline
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False)
|
||||
|
||||
DEFAULT_SETTINGS={'family':'3','autoLookbackDays':14,'autoMinimumDays':7,'autoMinimumCoverage':.9,'autoSwitchMarginChf':1.,'tariffPolicy':'published_only','trainingCadence':'daily','trainingPromotion':'validated_only','runMode':'shadow','roundtripEfficiency':.90,'measurementPolicy':'verified_only','peakOutlookPolicy':'empirical_if_available','peakOutlookReliance':.5,'forecastSource':'legacy','measurementDataset':''}
|
||||
|
||||
class PlannerStore:
|
||||
"""Own SQLite file, no mutation of legacy application databases."""
|
||||
def __init__(self,path,registry=None):
|
||||
self.registry=registry or default_registry();self.con=sqlite3.connect(path,timeout=10)
|
||||
self.con.row_factory=sqlite3.Row;self.con.execute('PRAGMA foreign_keys=ON')
|
||||
self.con.executescript('''
|
||||
CREATE TABLE IF NOT EXISTS planner_settings(plant TEXT PRIMARY KEY,revision INTEGER NOT NULL,value TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_audit(id INTEGER PRIMARY KEY,plant TEXT NOT NULL,at TEXT NOT NULL,kind TEXT NOT NULL,detail TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_work(plant TEXT PRIMARY KEY,sequence INTEGER NOT NULL,revision INTEGER NOT NULL,reasons TEXT NOT NULL,requested_at TEXT NOT NULL,lease_until TEXT,lease_token TEXT);
|
||||
CREATE TABLE IF NOT EXISTS planner_measurements(plant TEXT NOT NULL,start TEXT NOT NULL,import_kwh REAL NOT NULL,PRIMARY KEY(plant,start));
|
||||
CREATE TABLE IF NOT EXISTS planner_month_peaks(plant TEXT NOT NULL,month TEXT NOT NULL,peak_kw REAL NOT NULL,source TEXT NOT NULL,updated_at TEXT NOT NULL,PRIMARY KEY(plant,month));
|
||||
CREATE TABLE IF NOT EXISTS planner_snapshots(id TEXT PRIMARY KEY,plant TEXT NOT NULL,issued_at TEXT NOT NULL,value TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_plans(plan_id TEXT PRIMARY KEY,plant TEXT NOT NULL,revision INTEGER NOT NULL,mode TEXT NOT NULL,value TEXT NOT NULL,created_at TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_current(plant TEXT NOT NULL,mode TEXT NOT NULL,plan_id TEXT NOT NULL REFERENCES planner_plans(plan_id),PRIMARY KEY(plant,mode));
|
||||
CREATE TABLE IF NOT EXISTS planner_ack(plant TEXT PRIMARY KEY,plan_id TEXT NOT NULL,revision INTEGER NOT NULL,received_at TEXT NOT NULL,applied_step TEXT,status TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_inputs(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,observed_at TEXT NOT NULL,value TEXT NOT NULL,PRIMARY KEY(plant,kind,event_id));
|
||||
CREATE TABLE IF NOT EXISTS planner_input_current(plant TEXT NOT NULL,kind TEXT NOT NULL,event_id TEXT NOT NULL,PRIMARY KEY(plant,kind));
|
||||
CREATE TABLE IF NOT EXISTS planner_run_status(plant TEXT PRIMARY KEY,updated_at TEXT NOT NULL,status TEXT NOT NULL,detail TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS planner_ticks(plant TEXT PRIMARY KEY,tick INTEGER NOT NULL);
|
||||
''')
|
||||
meter_runtime.schema(self.con)
|
||||
controlled_trial.schema(self.con)
|
||||
measurement_pipeline.schema(self.con)
|
||||
def close(self):self.con.close()
|
||||
def settings(self,plant):
|
||||
row=self.con.execute('SELECT revision,value FROM planner_settings WHERE plant=?',(plant,)).fetchone()
|
||||
return {'revision':row['revision'],**DEFAULT_SETTINGS,**json.loads(row['value'])} if row else {'revision':0,**DEFAULT_SETTINGS}
|
||||
def _validate_settings(self,value):
|
||||
if set(value)!=set(DEFAULT_SETTINGS):raise ValueError('Unknown or missing setting')
|
||||
if value['family']!='auto':self.registry.get(value['family'])
|
||||
for key in ('autoLookbackDays','autoMinimumDays'):
|
||||
if type(value[key]) is not int:raise ValueError('Days must be integers')
|
||||
number(value['autoLookbackDays'],'lookback',7,90);number(value['autoMinimumDays'],'minimum days',1,value['autoLookbackDays'])
|
||||
number(value['autoMinimumCoverage'],'coverage',.5,1);number(value['autoSwitchMarginChf'],'margin',0)
|
||||
if value['tariffPolicy']!='published_only':raise ValueError('Only published-price policy implemented')
|
||||
if value['trainingCadence'] not in ('daily','weekly') or value['trainingPromotion']!='validated_only':raise ValueError('Training must use validated promotion')
|
||||
if value['runMode']!='shadow':raise ValueError('Shadow-only: live release requires separate validation')
|
||||
number(value['roundtripEfficiency'],'roundtrip efficiency',.01,1)
|
||||
if value['measurementPolicy'] not in ('verified_only','allow_estimates'):raise ValueError('Invalid measurement policy')
|
||||
if value['peakOutlookPolicy'] not in ('full_incremental','empirical_if_available'):raise ValueError('Invalid peak outlook policy')
|
||||
number(value['peakOutlookReliance'],'peak outlook reliance',0,1)
|
||||
if value['forecastSource'] not in ('legacy','corrected_profile'):raise ValueError('Unknown forecast source')
|
||||
if not isinstance(value['measurementDataset'],str) or len(value['measurementDataset'])>80:raise ValueError('Invalid measurement dataset')
|
||||
if value['forecastSource']=='corrected_profile' and not value['measurementDataset']:raise ValueError('Corrected forecast requires an explicit dataset')
|
||||
def _request(self,plant,revision,reason,now):
|
||||
row=self.con.execute('SELECT * FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
reasons=set(json.loads(row['reasons'])) if row else set();reasons.add(reason)
|
||||
seq=row['sequence']+1 if row else 1
|
||||
self.con.execute('''INSERT INTO planner_work(plant,sequence,revision,reasons,requested_at) VALUES(?,?,?,?,?)
|
||||
ON CONFLICT(plant) DO UPDATE SET sequence=excluded.sequence,revision=excluded.revision,reasons=excluded.reasons,requested_at=excluded.requested_at''',(plant,seq,revision,canonical(sorted(reasons)),utc(now).isoformat()))
|
||||
def save_settings(self,plant,changes,expected_revision,now):
|
||||
if type(expected_revision) is not int or expected_revision<0:raise ValueError('Invalid expected revision')
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
current=self.settings(plant)
|
||||
if current.pop('revision')!=expected_revision:raise ValueError('Revision conflict; reload before saving')
|
||||
current.update(changes);self._validate_settings(current);revision=expected_revision+1
|
||||
self.con.execute('''INSERT INTO planner_settings VALUES(?,?,?) ON CONFLICT(plant) DO UPDATE SET revision=excluded.revision,value=excluded.value''',(plant,revision,canonical(current)))
|
||||
self._request(plant,revision,'configuration_changed',now)
|
||||
self.con.execute('INSERT INTO planner_audit(plant,at,kind,detail) VALUES(?,?,?,?)',(plant,utc(now).isoformat(),'settings',canonical({'revision':revision,'changes':changes})))
|
||||
self.con.commit();return {'revision':revision,**current}
|
||||
except Exception:self.con.rollback();raise
|
||||
def request(self,plant,reason,now):
|
||||
if reason not in ('prices_changed','telemetry_changed','five_minute_tick','manual','model_promoted','forecast_changed','operation_changed','tariffs_changed'):raise ValueError('Unknown trigger')
|
||||
with self.con:self._request(plant,self.settings(plant)['revision'],reason,now)
|
||||
def claim(self,now,lease_seconds=120):
|
||||
at=utc(now);self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
row=self.con.execute('SELECT * FROM planner_work WHERE lease_until IS NULL OR lease_until < ? ORDER BY requested_at LIMIT 1',(at.isoformat(),)).fetchone()
|
||||
if not row:self.con.commit();return None
|
||||
token=str(uuid4());self.con.execute('UPDATE planner_work SET lease_until=?,lease_token=? WHERE plant=?',((at+timedelta(seconds=lease_seconds)).isoformat(),token,row['plant']))
|
||||
self.con.commit();return {**dict(row),'lease_token':token}
|
||||
except Exception:self.con.rollback();raise
|
||||
def finish(self,claim):
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
row=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(claim['plant'],)).fetchone()
|
||||
if not row or row['lease_token']!=claim['lease_token']:self.con.commit();return False
|
||||
if row['sequence']==claim['sequence']:self.con.execute('DELETE FROM planner_work WHERE plant=?',(claim['plant'],))
|
||||
else:self.con.execute('UPDATE planner_work SET lease_until=NULL,lease_token=NULL WHERE plant=?',(claim['plant'],))
|
||||
self.con.commit();return True
|
||||
except Exception:self.con.rollback();raise
|
||||
def initialize_peak(self,plant,month,peak_kw,source,now):
|
||||
number(peak_kw,'authoritative measured peak',0)
|
||||
if source not in ('meter_month_register','verified_month_history','verified_new_month'):raise ValueError('Configured cap is NOT measured peak')
|
||||
datetime.strptime(month,'%Y-%m')
|
||||
if month>month_key(now):raise ValueError('Future month cannot have a measured peak')
|
||||
with self.con:
|
||||
old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,month)).fetchone()
|
||||
if old and peak_kw<old[0]:raise ValueError('Cannot lower measured peak silently')
|
||||
self.con.execute('''INSERT INTO planner_month_peaks VALUES(?,?,?,?,?) ON CONFLICT(plant,month) DO UPDATE SET peak_kw=excluded.peak_kw,source=excluded.source,updated_at=excluded.updated_at''',(plant,month,peak_kw,source,utc(now).isoformat()))
|
||||
def record_import_interval(self,plant,start,import_kwh,received_at):
|
||||
start,received_at=utc(start),utc(received_at);number(import_kwh,'metered import energy',0)
|
||||
if start.minute%5 or start.second or start.microsecond or start+timedelta(minutes=5)>received_at:raise ValueError('Completed aligned intervals required')
|
||||
with self.con:
|
||||
old=self.con.execute('SELECT import_kwh FROM planner_measurements WHERE plant=? AND start=?',(plant,start.isoformat())).fetchone()
|
||||
if old and abs(old[0]-import_kwh)>1e-9:raise ValueError('Conflicting metering fact')
|
||||
self.con.execute('INSERT OR IGNORE INTO planner_measurements VALUES(?,?,?)',(plant,start.isoformat(),import_kwh))
|
||||
q=quarter_start(start);rows=self.con.execute('SELECT start,import_kwh FROM planner_measurements WHERE plant=? AND start>=? AND start<? ORDER BY start',(plant,q.isoformat(),(q+timedelta(minutes=15)).isoformat())).fetchall()
|
||||
if len(rows)!=3:return None
|
||||
peak=sum(r['import_kwh'] for r in rows)/.25;m=month_key(q)
|
||||
old=self.con.execute('SELECT peak_kw FROM planner_month_peaks WHERE plant=? AND month=?',(plant,m)).fetchone()
|
||||
if not old:return {'quarterPeakKw':peak,'monthState':'needs_initialization'}
|
||||
self.con.execute('UPDATE planner_month_peaks SET peak_kw=MAX(peak_kw,?),updated_at=? WHERE plant=? AND month=?',(peak,received_at.isoformat(),plant,m))
|
||||
return {'quarterPeakKw':peak,'monthState':'measured'}
|
||||
def peaks(self,plant):return {r['month']:r['peak_kw'] for r in self.con.execute('SELECT month,peak_kw FROM planner_month_peaks WHERE plant=?',(plant,))}
|
||||
def snapshot(self,plant,issued_at,value,snapshot_id=None):
|
||||
identifier=snapshot_id or str(uuid4())
|
||||
with self.con:self.con.execute('INSERT INTO planner_snapshots VALUES(?,?,?,?)',(identifier,plant,utc(issued_at).isoformat(),canonical(value)))
|
||||
return identifier
|
||||
def publish_shadow(self,plant,plan,expected_revision,now,work_sequence=None,work_token=None):
|
||||
if not plan.get('executable') or plan.get('configRevision')!=expected_revision:raise ValueError('Only validated plans for exact revision')
|
||||
if plan.get('runMode','shadow')!='shadow':raise ValueError('Only shadow publication permitted')
|
||||
self.con.execute('BEGIN IMMEDIATE')
|
||||
try:
|
||||
if self.settings(plant)['revision']!=expected_revision:raise ValueError('Configuration changed while computing')
|
||||
if work_sequence is not None:
|
||||
work=self.con.execute('SELECT sequence,lease_token FROM planner_work WHERE plant=?',(plant,)).fetchone()
|
||||
if not work or work[0]!=work_sequence or work_token is not None and work[1]!=work_token:raise ValueError('Newer request arrived while computing')
|
||||
identifier=plan['planId']
|
||||
self.con.execute('INSERT INTO planner_plans VALUES(?,?,?,?,?,?)',(identifier,plant,expected_revision,'shadow',canonical(plan),utc(now).isoformat()))
|
||||
self.con.execute('INSERT INTO planner_current VALUES(?,?,?) ON CONFLICT(plant,mode) DO UPDATE SET plan_id=excluded.plan_id',(plant,'shadow',identifier))
|
||||
self.con.commit()
|
||||
except Exception:self.con.rollback();raise
|
||||
def current(self,plant,mode='shadow'):
|
||||
row=self.con.execute('SELECT value FROM planner_plans JOIN planner_current USING(plan_id) WHERE planner_current.plant=? AND planner_current.mode=?',(plant,mode)).fetchone()
|
||||
return json.loads(row[0]) if row else None
|
||||
def acknowledge(self,plant,plan_id,revision,now,step=None,status='received'):
|
||||
if status not in ('received','applied','rejected','shadow_seen'):raise ValueError('Unknown acknowledgement')
|
||||
row=self.con.execute('SELECT mode,revision,value FROM planner_plans WHERE plan_id=? AND plant=?',(plan_id,plant)).fetchone()
|
||||
if not row or row['revision']!=revision:raise ValueError('Unknown plan/revision')
|
||||
if status=='applied' and row['mode']!='live':raise ValueError('Shadow plan must never be applied')
|
||||
if step is not None and step not in {p['time'] for p in json.loads(row['value'])['points']}:raise ValueError('Step does not belong to plan')
|
||||
with self.con:self.con.execute('''INSERT INTO planner_ack VALUES(?,?,?,?,?,?) ON CONFLICT(plant) DO UPDATE SET plan_id=excluded.plan_id,revision=excluded.revision,received_at=excluded.received_at,applied_step=excluded.applied_step,status=excluded.status''',(plant,plan_id,revision,utc(now).isoformat(),step,status))
|
||||
@@ -0,0 +1,128 @@
|
||||
"""One consolidated ROOT preflight; tests and read-only acquisition, NEVER deploy.
|
||||
|
||||
Builds disposable test images, executes Python and native-PHP conversion tests,
|
||||
then reads actual raw forecast/input diagnostics using the existing container.
|
||||
No docker up/restart, no live-setting change, no training or actuator call.
|
||||
A passing preflight is NOT a production release or full plant acceptance.
|
||||
"""
|
||||
from datetime import datetime,timezone
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from uuid import UUID
|
||||
from forecast_acceptance import verify_forecast_bundle
|
||||
|
||||
ROOT=Path(__file__).resolve().parent
|
||||
REPO=Path('/srv/agent/repos/Enelix-EMS')
|
||||
PROJECT=Path('/home/agent/services/prognosis-manager-enelix2')
|
||||
|
||||
|
||||
def command_list(plant):
|
||||
return [
|
||||
('python311_and_portal',[sys.executable,str(ROOT/'deploy_shadow.py'),'--plant',plant,'--test-only'],ROOT,900),
|
||||
('php_image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.php'),'-t','enelix-netplan-v4-php-check:local',str(ROOT/'acceptance')],ROOT,900),
|
||||
('php83_offline',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=32m','enelix-netplan-v4-php-check:local'],ROOT,120),
|
||||
('forecast_candidate_image',['docker','build','-f',str(ROOT/'acceptance/Dockerfile.forecast'),'-t','enelix-forecast-candidate-check:local',str(ROOT/'acceptance')],ROOT,900),
|
||||
('forecast_candidate_python311',['docker','run','--rm','--network','none','--read-only','--user','1000:1000','--cap-drop','ALL','--security-opt','no-new-privileges:true','--tmpfs','/tmp:rw,noexec,nosuid,size=64m','enelix-forecast-candidate-check:local'],ROOT,180),
|
||||
('gui_syntax',['node','--check',str(ROOT/'gui/netplan-v4.js')],ROOT,30),
|
||||
]
|
||||
|
||||
|
||||
def source_manifest():
|
||||
selected=list((ROOT/'netplan_v4').glob('*.py'))+list((ROOT/'tests').glob('*.py'))+[ROOT/'gui/netplan-v4.js',ROOT/'acceptance/read_native_forecasts.py',ROOT/'forecast_acceptance.py',ROOT/'acceptance/check_forecast.py',ROOT/'acceptance/Dockerfile.forecast',ROOT/'acceptance/forecast-src/SOURCE_MANIFEST.json']
|
||||
selected += [ROOT/'release_preflight.py', ROOT/'Dockerfile', ROOT/'acceptance/Dockerfile.php', ROOT/'acceptance/php-src/SOURCE_MANIFEST.json']
|
||||
return {str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(selected)}
|
||||
|
||||
|
||||
def verify_native_bundle():
|
||||
bundle=ROOT/'acceptance/php-src'
|
||||
manifest=json.loads((bundle/'SOURCE_MANIFEST.json').read_text())
|
||||
for relative,expected in manifest.items():
|
||||
if Path(relative).is_absolute() or '..' in Path(relative).parts:raise ValueError('Unsafe source manifest')
|
||||
for p in (bundle/relative,REPO/relative):
|
||||
if not p.is_file() or p.is_symlink() or hashlib.sha256(p.read_bytes()).hexdigest()!=expected:
|
||||
raise ValueError('Native source changed after staging; review and refresh acceptance bundle')
|
||||
|
||||
|
||||
def save(path,value):
|
||||
raw=json.dumps(value,indent=2,allow_nan=False)+'\n'
|
||||
with path.open('w') as output:output.write(raw)
|
||||
os.chmod(path,0o640)
|
||||
if os.geteuid()==0:os.chown(path,1000,1000)
|
||||
|
||||
|
||||
def run(plant):
|
||||
verify_native_bundle()
|
||||
forecast_hashes=verify_forecast_bundle(ROOT/'acceptance/forecast-src',PROJECT/'forecast_engine')
|
||||
stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
reports=ROOT/'acceptance-reports'/stamp;reports.mkdir(parents=True,exist_ok=False)
|
||||
if os.geteuid()==0:
|
||||
os.chown(reports.parent,1000,1000);os.chown(reports,1000,1000)
|
||||
report={'createdAt':datetime.now(timezone.utc).isoformat(),'installationId':plant,'productionReady':False,'deploymentPerformed':False,'tests':{},'sourceHashes':source_manifest(),'remainingWork':['Native source installation and complete Symcon runtime test','Restore/validate real forecast telemetry and nonzero load profiles','Wire productive historical family replay and model promotion','Verify base-load/SDL accounting and physical meter metadata','Implement and test live V4 plan execution and fallback before activation']}
|
||||
report['forecastCandidateSourceHashes']=forecast_hashes
|
||||
report['forecastCandidateRuntimeTestIsOffline']=True
|
||||
env=dict(os.environ,NETPLAN_V4_PLANTS=plant)
|
||||
success=True
|
||||
for name,command,cwd,timeout in command_list(plant):
|
||||
print('\n=== '+name+' ===',flush=True)
|
||||
try:
|
||||
completed=subprocess.run(command,cwd=cwd,env=env,timeout=timeout,check=False)
|
||||
code=completed.returncode
|
||||
except subprocess.TimeoutExpired:code=124
|
||||
report['tests'][name]={'exitCode':code,'passed':code==0}
|
||||
if code:
|
||||
success=False;break
|
||||
if success:
|
||||
print('\n=== Existing forecast engine: READ ONLY ===',flush=True)
|
||||
command=['docker','compose','-f',str(PROJECT/'compose.yaml'),'exec','-T','-e','ENELIX_ACCEPTANCE_PLANT='+plant,'forecast-engine','python','-B','-']
|
||||
try:
|
||||
result=subprocess.run(command,cwd=PROJECT,env=env,input=(ROOT/'acceptance/read_native_forecasts.py').read_text(),text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,timeout=120,check=False)
|
||||
if result.returncode or len(result.stdout)>8000000:raise RuntimeError('Probe execution failed or oversized')
|
||||
probe=json.loads(result.stdout)
|
||||
save(reports/'native-inputs.json',probe)
|
||||
report['nativeInputStatus']=probe.get('status')
|
||||
report['forecastSummary']=probe.get('forecastSummary',{})
|
||||
report['dataBlockers']=[]
|
||||
for key,value in report['forecastSummary'].items():
|
||||
if key in ('prog_var_2','prog_var_11','prog_var_22') and (not value.get('points') or value.get('allZero')):
|
||||
report['dataBlockers'].append(key+': missing or unconfirmed all-zero load forecast')
|
||||
recent=probe.get('inputFrames',{}).get('df_recent_raw',{}).get('columns',{}).get('Hausverbrauch',{}).get('lastFiniteAt')
|
||||
if not recent or (datetime.now(timezone.utc)-datetime.fromisoformat(recent)).total_seconds()>1800:
|
||||
report['dataBlockers'].append('Recent raw load telemetry missing or older than 30 minutes')
|
||||
if probe.get('status')!='read_only_acquired' or report['dataBlockers']:success=False
|
||||
except (ValueError,RuntimeError,subprocess.TimeoutExpired) as error:
|
||||
report['nativeInputStatus']='failed';report['nativeInputErrorType']=type(error).__name__;success=False
|
||||
try:
|
||||
verify_native_bundle()
|
||||
report['sourceStableDuringTests']=(source_manifest()==report['sourceHashes'] and verify_forecast_bundle(ROOT/'acceptance/forecast-src',PROJECT/'forecast_engine')==forecast_hashes)
|
||||
except (OSError,ValueError):
|
||||
report['sourceStableDuringTests']=False
|
||||
if not report['sourceStableDuringTests']:
|
||||
report.setdefault('dataBlockers',[]).append('Source changed during checks; results cannot certify current candidate')
|
||||
success=False
|
||||
report['preflightChecksPassed']=success
|
||||
save(reports/'REPORT.json',report)
|
||||
save(ROOT/'acceptance-reports'/'LATEST.json',{'report':str(reports/'REPORT.json')})
|
||||
print('\nPRECHECK '+('PASSED' if success else 'NEEDS REVIEW')+'; NOT a production release.')
|
||||
print('REPORT: '+str(reports/'REPORT.json'))
|
||||
for key,value in report.get('forecastSummary',{}).items():print(key,json.dumps(value))
|
||||
for blocker in report.get('dataBlockers',[]):print('DATA BLOCKER:',blocker)
|
||||
print('Existing containers, Symcon settings, timers and actuators unchanged.')
|
||||
return 0 if success else 1
|
||||
|
||||
if __name__=='__main__':
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--plant',required=True,type=lambda v:str(UUID(v)))
|
||||
parser.add_argument('--show-commands',action='store_true')
|
||||
args=parser.parse_args()
|
||||
if args.show_commands:
|
||||
for name,command,cwd,timeout in command_list(args.plant):print(name,json.dumps(command))
|
||||
print('Native input probe: read-only docker compose exec forecast-engine python with reviewed stdin script')
|
||||
else:
|
||||
if os.geteuid()!=0:raise SystemExit('Run as root; do not alter Docker socket permissions.')
|
||||
try:raise SystemExit(run(args.plant))
|
||||
except (OSError,ValueError) as error:raise SystemExit('Preflight stopped safely: '+str(error))
|
||||
@@ -0,0 +1,5 @@
|
||||
numpy>=2.2,<3
|
||||
scipy>=1.15,<1.18
|
||||
fastapi>=0.115,<0.129
|
||||
httpx>=0.27,<0.29
|
||||
uvicorn>=0.30,<0.41
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Run against the exact server working tree; no live data or credentials used."""
|
||||
from pathlib import Path
|
||||
import hashlib,json,os,sys,unittest
|
||||
ROOT=Path(__file__).resolve().parent
|
||||
DEPS=Path('/home/agent/services/qa/netplan-v4-candidate-20261001/python-dependencies')
|
||||
if DEPS.exists():sys.path.insert(0,str(DEPS))
|
||||
sys.path.insert(0,str(ROOT))
|
||||
if __name__=='__main__':
|
||||
import numpy,scipy,fastapi,httpx
|
||||
versions={'python':sys.version.split()[0],**{m.__name__:m.__version__ for m in (numpy,scipy,fastapi,httpx)}}
|
||||
print('Runtime',json.dumps(versions),flush=True)
|
||||
reports=Path(os.getenv('NETPLAN_V4_TEST_REPORT_DIR',str(ROOT)))
|
||||
reports.mkdir(parents=True,exist_ok=True)
|
||||
suite=unittest.defaultTestLoader.discover(str(ROOT/'tests'))
|
||||
with (reports/'TEST_RESULTS.txt').open('w') as output:
|
||||
output.write('Actual server/target tests; synthetic data only.\n'+json.dumps(versions)+'\n\n')
|
||||
result=unittest.TextTestRunner(stream=output,verbosity=2).run(suite)
|
||||
print((reports/'TEST_RESULTS.txt').read_text())
|
||||
hashes={str(p.relative_to(ROOT)):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(ROOT.rglob('*.py')) if '__pycache__' not in p.parts}
|
||||
(reports/'TESTED_SOURCE_HASHES.json').write_text(json.dumps(hashes,indent=2)+'\n')
|
||||
raise SystemExit(0 if result.wasSuccessful() else 1)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Check packaged code readability and writable temporary test output as non-root."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
SOURCE_DIRS = ("netplan_v4", "tests", "integrations", "gui")
|
||||
SOURCE_FILES = ("run_tests.py", "install_hooks.py", "runtime_preflight.py", "requirements.txt")
|
||||
|
||||
|
||||
def verify_access(root, reports):
|
||||
root, reports = Path(root).resolve(), Path(reports).resolve()
|
||||
if reports == root or root in reports.parents:
|
||||
raise ValueError("Test reports must be outside the packaged application tree")
|
||||
if not root.is_dir() or not os.access(root, os.R_OK | os.X_OK):
|
||||
raise PermissionError(f"Application directory is not accessible: {root}")
|
||||
paths = [root / name for name in SOURCE_FILES]
|
||||
for name in SOURCE_DIRS:
|
||||
directory = root / name
|
||||
if not directory.is_dir():
|
||||
raise FileNotFoundError(f"Missing packaged directory: {directory}")
|
||||
def on_error(error):
|
||||
raise error
|
||||
for parent, directories, files in os.walk(directory, onerror=on_error):
|
||||
base = Path(parent)
|
||||
if not os.access(base, os.R_OK | os.X_OK):
|
||||
raise PermissionError(f"Packaged directory is not accessible: {base}")
|
||||
if any((base / child).is_symlink() for child in directories + files):
|
||||
raise ValueError(f"Unexpected symlink in packaged source: {base}")
|
||||
paths.extend(base / name for name in files)
|
||||
for path in paths:
|
||||
with path.open("rb") as handle:
|
||||
handle.read(1)
|
||||
reports.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryFile(dir=reports) as probe:
|
||||
probe.write(b"permission check")
|
||||
probe.flush()
|
||||
return len(paths)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.geteuid() == 0:
|
||||
raise SystemExit("Runtime preflight must run as the unprivileged container user")
|
||||
reports = Path(os.getenv("NETPLAN_V4_TEST_REPORT_DIR", "/tmp/test-results"))
|
||||
count = verify_access(Path(__file__).resolve().parent, reports)
|
||||
import numpy, scipy, fastapi, httpx
|
||||
print(f"Non-root runtime check OK: uid={os.geteuid()}, readable_files={count}, reports={reports}")
|
||||
@@ -0,0 +1,24 @@
|
||||
<?php
|
||||
declare(strict_types=1);
|
||||
require_once dirname(__DIR__).'/integrations/NetzfahrplanV4.php';
|
||||
use Belevo\EnelixEMS\NetzfahrplanV4;
|
||||
$count=0;
|
||||
function check(bool $ok,string $message):void {global $count;$count++;if(!$ok)throw new RuntimeException($message);}
|
||||
$point=['time'=>'2026-10-01T12:00:00Z','validUntil'=>'2026-10-01T12:05:00Z','gridTargetW'=>0.,'intent'=>'pvCharge'];
|
||||
$plan=['schemaVersion'=>2,'planId'=>'test','configRevision'=>7,'status'=>'optimal','executable'=>true,
|
||||
'runMode'=>'shadow','liveEnabled'=>false,'generatedAt'=>$point['time'],'validFrom'=>$point['time'],'validUntil'=>'2026-10-01T12:10:00Z',
|
||||
'points'=>[$point,['time'=>'2026-10-01T12:05:00Z','validUntil'=>'2026-10-01T12:10:00Z','gridTargetW'=>9000.,'intent'=>'gridCharge']]];
|
||||
$now=strtotime('2026-10-01T12:03:00Z');
|
||||
check(NetzfahrplanV4::currentPoint($plan,$now,7)===null,'Shadow never live');
|
||||
check(NetzfahrplanV4::currentPoint($plan,$now,7,true)['gridTargetW']===0.,'Current interval, not nearest future point');
|
||||
check(NetzfahrplanV4::currentPoint($plan,$now,8,true)===null,'Revision mismatch');
|
||||
check(NetzfahrplanV4::currentPoint($plan,$now+3600,7,true)===null,'Expired plan');
|
||||
$bad=$plan;$bad['points'][1]['time']=$point['time'];check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Overlapping intervals');
|
||||
$bad=$plan;$bad['points'][0]['gridTargetW']=NAN;check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'NaN rejected');
|
||||
$bad=$plan;$bad['generatedAt']='2026-10-01T12:00:00';check(NetzfahrplanV4::currentPoint($bad,$now,7,true)===null,'Timezone required');
|
||||
$r=NetzfahrplanV4::correction($point,-5332.,0.,39000.,0.,15000.);check($r['batteryTargetW']===5332.,'Surplus charges');check($r['gridTargetW']===0.,'Peak cap never replaces zero target');check($r['expectedGridW']===0.,'Expected grid balanced');
|
||||
$r=NetzfahrplanV4::correction($point,-5332.,0.,3000.,0.);check($r['batteryTargetW']===3000.&&$r['limited'],'Charging availability enforced');
|
||||
$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],1000.,0.,39000.,0.);check($r['batteryTargetW']===0.,'No inadvertent grid charge from load forecast');
|
||||
$r=NetzfahrplanV4::correction(['gridTargetW'=>5000.,'intent'=>'hold'],7000.,0.,39000.,1000.,5000.);check($r['batteryTargetW']===-1000.&&$r['limited'],'Safety override still respects available discharge');
|
||||
$r=NetzfahrplanV4::correction(['gridTargetW'=>-40000.,'intent'=>'pvCharge'],-10000.,0.,39000.,10000.);check($r['batteryTargetW']===0.,'No unintended discharge in PV charge intent');check($r['trackingErrorW']===30000.,'Intent saturation reports unmet target');
|
||||
echo "Netzfahrplan V4: {$count} assertions passed. No IPS or device calls.\n";
|
||||
@@ -0,0 +1,32 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createPlannerV4Bridge} from '../integrations/netplan-v4-bridge.mjs';
|
||||
const PLANT='00000000-0000-4000-8000-000000000001',INSTALL='e3a08f9e-af12-4695-99bd-8b51c0520021';
|
||||
function fixture(options={}){
|
||||
const sent=[],auth=[],upstream=[];
|
||||
const bridge=createPlannerV4Bridge({
|
||||
configuredPrognosisPlant:(req,res,id,csrf)=>{auth.push({id,csrf});return options.denied?null:{plant:{installation_id:INSTALL},license:{quantities:{grid_schedule:options.unlicensed?0:1}},session:{}};},
|
||||
roleAllowed:()=>!options.viewer,bodyJson:async()=>({expectedRevision:0,changes:{family:'23'}}),
|
||||
json:(res,status,payload)=>sent.push({status,payload}),deviceActivation:()=>options.deviceDenied?null:{plant_id:PLANT},
|
||||
checkDeviceRate:()=>!options.ratelimited,ownedLicenseState:()=>({quantities:{grid_schedule:1}}),serviceToken:'synthetic-test-only',
|
||||
fetchImpl:async(url,args)=>{upstream.push({url:String(url),args});if(options.down)throw new Error('secret-host-detail');return {ok:!options.conflict,status:options.conflict?409:200,json:async()=>options.conflict?{detail:'internal-field'}:{liveEnabled:false,plan:{planId:'p1'}}};}
|
||||
});return {bridge,sent,auth,upstream};
|
||||
}
|
||||
const url=suffix=>new URL(`https://portal.test/api/plants/${PLANT}/prognosis/planner-v4${suffix}`);
|
||||
test('read scoped by customer plant, upstream uses installation id',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},url('')),true);assert.equal(f.auth[0].csrf,false);assert.match(f.upstream[0].url,new RegExp(INSTALL));assert.equal(f.sent[0].status,200);});
|
||||
test('save requires existing CSRF checks',async()=>{const f=fixture();await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.auth[0].csrf,true);assert.equal(f.upstream[0].args.method,'PUT');});
|
||||
test('viewer cannot mutate',async()=>{const f=fixture({viewer:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,403);assert.equal(f.upstream.length,0);});
|
||||
test('unowned plant blocked',async()=>{const f=fixture({denied:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.upstream.length,0);});
|
||||
test('license required',async()=>{const f=fixture({unlicensed:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,403);});
|
||||
test('revision conflict preserved without internal error disclosure',async()=>{const f=fixture({conflict:true});await f.bridge({method:'PUT'},{},url('/settings'));assert.equal(f.sent[0].status,409);assert.doesNotMatch(JSON.stringify(f.sent),/internal-field/);});
|
||||
test('service failure neither exposes details nor changes live plan',async()=>{const f=fixture({down:true});await f.bridge({method:'GET'},{},url(''));assert.equal(f.sent[0].status,503);assert.doesNotMatch(JSON.stringify(f.sent),/secret-host-detail|synthetic-test-only/);});
|
||||
test('V1 live schedule is NOT intercepted',async()=>{const f=fixture();assert.equal(await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/schedule`)),false);assert.equal(f.upstream.length,0);});
|
||||
test('device telemetry mapped only to operation ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/operation`));assert.match(f.upstream[0].url,/\/inputs\/operation$/);});
|
||||
test('unauthenticated device blocked',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.upstream.length,0);});
|
||||
test('device rate limit reused',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'GET'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);});
|
||||
test('unexpected method rejected',async()=>{const f=fixture();await f.bridge({method:'DELETE'},{},url(''));assert.equal(f.sent[0].status,405);assert.equal(f.upstream.length,0);});
|
||||
|
||||
test('measurement batches use authenticated dedicated data ingress',async()=>{const f=fixture();await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.match(f.upstream[0].url,/\/planner\/measurements$/);assert.equal(f.upstream[0].args.method,'POST');});
|
||||
test('unauthenticated measurement upload cannot reach storage',async()=>{const f=fixture({deviceDenied:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.upstream.length,0);});
|
||||
test('device cannot change dataset mapping through public proxy',async()=>{const f=fixture();assert.equal(await f.bridge({method:'PUT'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/datasets/physical-v1`)),false);assert.equal(f.upstream.length,0);});
|
||||
test('measurement upload keeps device rate protection',async()=>{const f=fixture({ratelimited:true});await f.bridge({method:'POST'},{},new URL(`https://portal.test/api/v1/installations/${INSTALL}/prognosis/planner-v4/measurements`));assert.equal(f.sent[0].status,429);assert.equal(f.upstream.length,0);});
|
||||
@@ -0,0 +1,39 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import sqlite3,tempfile,unittest
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
spec=importlib.util.spec_from_file_location('app_deploy',ROOT/'commissioning/deploy_application.py');d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d)
|
||||
class ApplicationDeploymentTest(unittest.TestCase):
|
||||
def test_backup_consistent_and_original_not_changed(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src=Path(tmp)/'source.sqlite';out=Path(tmp)/'backup.sqlite'
|
||||
with sqlite3.connect(src) as c:c.execute('CREATE TABLE sample(value)');c.execute('INSERT INTO sample VALUES(3)')
|
||||
before=src.read_bytes();d.backup_database(src,out)
|
||||
with sqlite3.connect(out) as c:self.assertEqual(c.execute('SELECT value FROM sample').fetchone()[0],3)
|
||||
self.assertEqual(before,src.read_bytes())
|
||||
def test_backup_cannot_invent_missing_database(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with self.assertRaises(ValueError):d.backup_database(Path(tmp)/'missing.sqlite',Path(tmp)/'new.sqlite')
|
||||
def test_atomic_refuses_symlink(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
a=Path(tmp)/'a';b=Path(tmp)/'b';a.write_text('original');b.symlink_to(a)
|
||||
with self.assertRaises(ValueError):d.atomic(b,b'changed')
|
||||
self.assertEqual(a.read_text(),'original')
|
||||
def test_only_two_reviewed_portal_files(self):
|
||||
self.assertEqual(set(d.PORTAL_FILES.values()),{'netplan-v4-bridge.mjs','public/netplan-v4.js'})
|
||||
def test_build_tests_backup_before_runtime_replacement(self):
|
||||
s=(ROOT/'commissioning/deploy_application.py').read_text()
|
||||
self.assertLess(s.index("'target_python_and_portal_tests_passed'"),s.index('v4_replaced=True'))
|
||||
self.assertLess(s.index("'consistent_database_backup'"),s.index('v4_replaced=True'))
|
||||
def test_does_not_change_optimizer_or_device_permissions(self):
|
||||
self.assertNotIn('/settings',d.BOOTSTRAP)
|
||||
self.assertNotIn('trial/arm',d.BOOTSTRAP)
|
||||
self.assertNotIn('IPS_',d.BOOTSTRAP)
|
||||
self.assertIn("state['liveEnabled'] is False",d.BOOTSTRAP)
|
||||
def test_image_contains_application_installer_imported_by_tests(self):
|
||||
self.assertIn('COPY commissioning/deploy_application.py ./commissioning/deploy_application.py',(ROOT/'Dockerfile').read_text())
|
||||
def test_file_bind_mount_recreated_not_only_restarted(self):
|
||||
s=(ROOT/'commissioning/deploy_application.py').read_text()
|
||||
self.assertIn("'--force-recreate','--wait','license-portal'",s)
|
||||
self.assertIn('additive_data_retained',s)
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,43 @@
|
||||
from dataclasses import replace
|
||||
import unittest
|
||||
from netplan_v4.optimizer import optimize
|
||||
from netplan_v4.domain import Limits
|
||||
from test_v4 import AT,seq,batt
|
||||
|
||||
class BatteryRecoveryTest(unittest.TestCase):
|
||||
def plan(self,b,steps=None):
|
||||
steps=steps or seq([0.]*3+[1000.]*3,pv=[1000.]*3+[0.]*3,sell=[0.]*6)
|
||||
return optimize(steps,[b],at=AT,observed_peaks={'2026-10':1.},peak_prices={'2026-10':0.})
|
||||
def test_reserve_recovery_preserves_real_initial_energy(self):
|
||||
b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.)
|
||||
p=self.plan(b);self.assertTrue(p['executable'],p)
|
||||
energy=.05
|
||||
for x in p['points']:
|
||||
target=x['batteryTargetW'];energy+=target/1000/12*(.9**.5 if target>=0 else 1/(.9**.5))
|
||||
self.assertAlmostEqual(100*energy,x['socEndPercent']['b'],places=5)
|
||||
self.assertGreaterEqual(energy,.05-1e-6)
|
||||
self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],15.-1e-5)
|
||||
def test_below_hardware_min_not_fabricated(self):
|
||||
b=batt(soc_percent=1.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True)
|
||||
self.assertFalse(self.plan(b)['executable'])
|
||||
def test_disabled_can_rearm_after_charging(self):
|
||||
b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False,max_charge_w=1000.,max_discharge_w=1000.)
|
||||
p=self.plan(b);self.assertTrue(p['executable'],p)
|
||||
self.assertAlmostEqual(min(0.,p['points'][0]['batteryTargetW']),0.)
|
||||
self.assertTrue(any(x['batteryTargetW']<-1. for x in p['points'][3:]))
|
||||
first=next(i for i,x in enumerate(p['points']) if x['batteryTargetW']<-1.)
|
||||
self.assertGreaterEqual(p['points'][first-1]['socEndPercent']['b'],17.-1e-5)
|
||||
def test_cannot_discharge_below_rearm_while_blocked(self):
|
||||
b=batt(capacity_kwh=10.,soc_percent=16.,min_soc_percent=15.,rearm_soc_percent=17.,discharge_blocked=True,grid_charging=False)
|
||||
p=self.plan(b,seq([1000.]*6));self.assertTrue(p['executable'])
|
||||
self.assertTrue(all(x['batteryTargetW']>=-1e-5 for x in p['points']))
|
||||
def test_unavailable_power_does_not_get_invented(self):
|
||||
b=batt(capacity_kwh=1.,soc_percent=5.,min_soc_percent=15.,physical_min_soc_percent=3.,recovery_allowed=True,max_charge_w=0.,max_discharge_w=0.)
|
||||
p=self.plan(b);self.assertFalse(p['executable']);self.assertEqual(p['status'],'no_feasible_plan')
|
||||
def test_invalid_hysteresis_range_rejected(self):
|
||||
b=batt(rearm_soc_percent=95.)
|
||||
self.assertFalse(self.plan(b)['executable'])
|
||||
def test_no_recovery_permission_keeps_strict_behavior(self):
|
||||
self.assertFalse(self.plan(batt(soc_percent=5.))['executable'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,60 @@
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import os
|
||||
import unittest
|
||||
from runtime_preflight import SOURCE_DIRS, SOURCE_FILES, verify_access
|
||||
|
||||
|
||||
class ContainerAccessTest(unittest.TestCase):
|
||||
def fixture(self, tmp):
|
||||
root = Path(tmp) / "app"
|
||||
root.mkdir()
|
||||
for name in SOURCE_DIRS:
|
||||
(root / name).mkdir()
|
||||
(root / name / "sample.txt").write_text("readable")
|
||||
for name in SOURCE_FILES:
|
||||
(root / name).write_text("readable")
|
||||
return root, Path(tmp) / "reports"
|
||||
|
||||
def test_code_readable_and_reports_outside_readonly_tree(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root, reports = self.fixture(tmp)
|
||||
self.assertEqual(verify_access(root, reports), 8)
|
||||
self.assertEqual(list(reports.iterdir()), [])
|
||||
|
||||
def test_missing_runner_is_reported(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root, reports = self.fixture(tmp)
|
||||
(root / "run_tests.py").unlink()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
verify_access(root, reports)
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user")
|
||||
def test_unreadable_source_fails(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root, reports = self.fixture(tmp)
|
||||
path = root / "netplan_v4" / "sample.txt"
|
||||
path.chmod(0)
|
||||
try:
|
||||
with self.assertRaises(PermissionError):
|
||||
verify_access(root, reports)
|
||||
finally:
|
||||
path.chmod(0o600)
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "Permission semantics require an unprivileged user")
|
||||
def test_untraversable_directory_is_not_silently_skipped(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root, reports = self.fixture(tmp)
|
||||
path = root / "tests"
|
||||
path.chmod(0)
|
||||
try:
|
||||
with self.assertRaises(PermissionError):
|
||||
verify_access(root, reports)
|
||||
finally:
|
||||
path.chmod(0o700)
|
||||
|
||||
def test_reports_cannot_target_application_tree(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root, _ = self.fixture(tmp)
|
||||
with self.assertRaises(ValueError):
|
||||
verify_access(root, root / "test-results")
|
||||
@@ -0,0 +1,120 @@
|
||||
import copy
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from fastapi.testclient import TestClient
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.service import create_app
|
||||
from netplan_v4 import controlled_trial as t
|
||||
|
||||
AT=datetime(2026,10,2,12,tzinfo=timezone.utc)
|
||||
PLANT='00000000-0000-4000-8000-000000000001'
|
||||
SESSION='00000000-0000-4000-8000-000000000002'
|
||||
PLAN='00000000-0000-4000-8000-000000000003'
|
||||
TOKEN='synthetic-only-control-test-token'
|
||||
|
||||
def fixture():
|
||||
context={'batteries':{'b':{'capacityKwh':10.}},'limits':{'importW':10000.,'exportW':None,'managerMonthLimitsW':{}}}
|
||||
plan={'planId':PLAN,'installationId':PLANT,'sourceFamily':'3','configRevision':1,
|
||||
'runMode':'shadow','liveEnabled':False,'executable':True,'validUntil':(AT+timedelta(hours=1)).isoformat(),
|
||||
'controlContext':context,'inputQuality':{'loadBasis':'base_load','accountingEvidenceId':'synthetic-meter-boundary-v1'},
|
||||
'peakCostIsEstimate':True}
|
||||
view={'installationId':PLANT,'liveEnabled':False,'fresh':True,'plan':plan,'settings':{'family':'3','revision':1}}
|
||||
request={'sessionId':SESSION,'expectedPlanId':PLAN,'expectedRevision':1,'durationSeconds':300,
|
||||
'maxChargeW':5000.,'maxDischargeW':5000.,'assetId':'b','managerId':11111,'batteryInstanceId':22222,
|
||||
'acceptEstimatedPeak':True,'actuatorWatchdogEvidenceId':'synthetic-hardware-watchdog',
|
||||
'confirmation':'ARM_BOUNDED_CONTROL_TRIAL'}
|
||||
return view,request
|
||||
|
||||
class ControlledTrialTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.s=PlannerStore(':memory:');self.v,self.r=fixture()
|
||||
def tearDown(self): self.s.close()
|
||||
def arm(self,**kw):
|
||||
return t.arm(self.s,PLANT,kw.get('request',self.r),kw.get('view',self.v),kw.get('now',AT),kw.get('allowed',{PLANT}))
|
||||
def test_default_has_no_authority(self):
|
||||
self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,set()))
|
||||
with self.assertRaises(ValueError):self.arm(allowed=set())
|
||||
def test_shadow_plan_never_becomes_live(self):
|
||||
before=copy.deepcopy(self.v);self.arm()
|
||||
a=t.authority(self.s,PLANT,self.v,AT,{PLANT})
|
||||
self.assertEqual(self.v,before);self.assertEqual(a['kind'],'controlled_trial_authority')
|
||||
self.assertTrue(a['sourcePlanRemainsShadow']);self.assertFalse(self.v['plan']['liveEnabled'])
|
||||
def test_short_authority_lease(self):
|
||||
self.arm();a=t.authority(self.s,PLANT,self.v,AT,{PLANT})
|
||||
self.assertEqual(t.timestamp(a['validUntil'])-AT,timedelta(seconds=90))
|
||||
def test_session_expiry(self):
|
||||
self.arm();self.assertIsNone(t.authority(self.s,PLANT,self.v,AT+timedelta(seconds=300),{PLANT}))
|
||||
def test_revoke(self):
|
||||
self.arm();t.revoke(self.s,PLANT,SESSION,AT);self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT}))
|
||||
def test_expired_or_revoked_session_not_resurrected(self):
|
||||
self.arm();t.revoke(self.s,PLANT,SESSION,AT)
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_active_session_cannot_be_extended(self):
|
||||
self.arm()
|
||||
with self.assertRaises(ValueError):self.arm(now=AT+timedelta(seconds=30))
|
||||
def test_other_plant(self):
|
||||
self.arm();self.assertIsNone(t.authority(self.s,SESSION,self.v,AT,{SESSION}))
|
||||
def test_revocation_mismatched_id_leaves_real_session(self):
|
||||
self.arm();t.revoke(self.s,PLANT,PLAN,AT);self.assertIsNotNone(t.authority(self.s,PLANT,self.v,AT,{PLANT}))
|
||||
def test_stale_source_plan_cannot_arm(self):
|
||||
self.v['fresh']=False
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_stale_source_removes_authority(self):
|
||||
self.arm();self.v['fresh']=False;self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT}))
|
||||
def test_revision_bool_not_an_integer(self):
|
||||
self.r['expectedRevision']=True
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_wrong_revision(self):
|
||||
self.r['expectedRevision']=2
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_revision_change_invalidates_authority(self):
|
||||
self.arm();self.v['settings']['revision']=2;self.v['plan']['configRevision']=2
|
||||
self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT}))
|
||||
def test_policy_change_invalidates_authority(self):
|
||||
self.arm();self.v['plan']['controlContext']['limits']['importW']=12000.
|
||||
self.assertIsNone(t.authority(self.s,PLANT,self.v,AT,{PLANT}))
|
||||
def test_newer_plan_same_policy_supported(self):
|
||||
self.arm();self.v['plan']['planId']=SESSION
|
||||
self.assertEqual(t.authority(self.s,PLANT,self.v,AT,{PLANT})['sourceShadowPlanId'],SESSION)
|
||||
def test_auto_family_not_enabled_for_first_trial(self):
|
||||
self.v['settings']['family']='auto'
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_aggregate_house_refused(self):
|
||||
self.v['plan']['inputQuality']['loadBasis']='house_total'
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_no_accounting_evidence_no_trial(self):
|
||||
del self.v['plan']['inputQuality']['accountingEvidenceId']
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_no_watchdog_evidence_no_trial(self):
|
||||
self.r['actuatorWatchdogEvidenceId']=''
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_no_implicit_estimate_acceptance(self):
|
||||
self.r['acceptEstimatedPeak']=False
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_power_cap_and_nonfinite(self):
|
||||
for v in (5001, float('nan'),float('inf'),True,0,'5000'):
|
||||
with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'maxChargeW':v})
|
||||
def test_duration_bound(self):
|
||||
for v in (0,1801,True,30.5):
|
||||
with self.subTest(v=v),self.assertRaises(ValueError):self.arm(request={**self.r,'durationSeconds':v})
|
||||
def test_cannot_replace_asset(self):
|
||||
self.r['assetId']='other'
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_two_assets_not_supported(self):
|
||||
self.v['plan']['controlContext']['batteries']['other']={}
|
||||
with self.assertRaises(ValueError):self.arm()
|
||||
def test_operator_endpoint_requires_auth(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
app=create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False)
|
||||
with TestClient(app) as c:
|
||||
self.assertEqual(c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',json=self.r).status_code,401)
|
||||
r=c.post(f'/internal/v2/prognosis/{PLANT}/planner/trial/arm',headers={'X-Enelix-Service-Token':TOKEN},json=self.r)
|
||||
self.assertEqual(r.status_code,409);self.assertIn('allowlist',r.text)
|
||||
self.assertIsNone(c.get(f'/internal/v2/prognosis/{PLANT}/planner',headers={'X-Enelix-Service-Token':TOKEN}).json()['controlledTrial'])
|
||||
def test_trial_allowlist_cannot_expand_installation_access(self):
|
||||
with tempfile.TemporaryDirectory() as td,self.assertRaises(ValueError):
|
||||
create_app(Path(td)/'test.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[SESSION])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import copy
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from fastapi.testclient import TestClient
|
||||
from netplan_v4.service import create_app, ingest, run_once
|
||||
from test_v4 import inputs, AT
|
||||
from test_controlled_trial import fixture, PLANT, SESSION, TOKEN
|
||||
|
||||
class ControlledTrialPipelineTest(unittest.TestCase):
|
||||
def test_real_planner_to_internal_trial_api_and_revoke(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
app=create_app(Path(td)/'trial.sqlite',TOKEN,[PLANT],start_worker=False,controlled_trial_plants=[PLANT])
|
||||
store=app.state.store_factory()
|
||||
op,fc,tar=inputs()
|
||||
for f in fc['families'].values():f['accountingEvidenceId']='synthetic-accounting-review'
|
||||
for k,v in [('operation',op),('forecast',fc),('tariffs',tar)]:ingest(store,PLANT,k,v,AT)
|
||||
plan=run_once(store,AT);self.assertEqual(plan['status'],'optimal');store.close()
|
||||
headers={'X-Enelix-Service-Token':TOKEN};prefix=f'/internal/v2/prognosis/{PLANT}/planner'
|
||||
with patch('netplan_v4.service.datetime',wraps=datetime) as clock,TestClient(app) as c:
|
||||
clock.now.return_value=AT
|
||||
view=c.get(prefix,headers=headers).json()
|
||||
self.assertIsNone(view['controlledTrial']);self.assertTrue(view['fresh'])
|
||||
_,request=fixture();request.update(expectedPlanId=plan['planId'],expectedRevision=0,acceptEstimatedPeak=False)
|
||||
grant=c.post(prefix+'/trial/arm',headers=headers,json=request)
|
||||
self.assertEqual(grant.status_code,200,grant.text)
|
||||
view=c.get(prefix,headers=headers).json()
|
||||
self.assertEqual(view['controlledTrial']['sourceShadowPlanId'],plan['planId'])
|
||||
self.assertFalse(view['liveEnabled']);self.assertFalse(view['plan']['liveEnabled'])
|
||||
self.assertEqual(view['plan']['runMode'],'shadow')
|
||||
self.assertEqual(c.post(prefix+'/ack',headers=headers,json={'planId':plan['planId'],'revision':0,'status':'applied'}).status_code,400)
|
||||
self.assertEqual(c.post(prefix+'/trial/revoke',headers=headers,json={'sessionId':SESSION}).status_code,200)
|
||||
self.assertIsNone(c.get(prefix,headers=headers).json()['controlledTrial'])
|
||||
self.assertEqual(c.post(prefix+'/trial/arm',headers=headers,json=request).status_code,409)
|
||||
def test_accounting_marker_cannot_label_aggregate_load_verified(self):
|
||||
from netplan_v4.store import PlannerStore
|
||||
s=PlannerStore(':memory:');op,fc,tar=inputs()
|
||||
f=fc['families']['3'];f['loadBasis']='house_total';f['accountingEvidenceId']='synthetic-incorrect-label'
|
||||
with self.assertRaises(ValueError):ingest(s,PLANT,'forecast',fc,AT)
|
||||
s.close()
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from datetime import datetime,timedelta,timezone
|
||||
import unittest
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def module(name,path):
|
||||
spec=importlib.util.spec_from_file_location(name,path);value=importlib.util.module_from_spec(spec);spec.loader.exec_module(value);return value
|
||||
publisher=module('publisher',ROOT/'integrations/netplan_v4_publisher.py')
|
||||
hooks=module('hooks',ROOT/'install_hooks.py')
|
||||
AT=datetime(2026,10,1,12,0,tzinfo=timezone.utc)
|
||||
|
||||
class PublisherTest(unittest.TestCase):
|
||||
def test_tariff_null_preserved_and_mode_explicit(self):
|
||||
cfg={'tarif_bezug_modus':'statisch','tarif_bezug':'custom','tarif_bezug_fest':0.,'tarif_einspeisung_modus':'dynamisch','tarif_einspeisung':'dynamic','tarif_peak_fest':0.}
|
||||
p=publisher.tariff_payload(cfg,AT);self.assertEqual(p['import']['staticChfKwh'],0.);self.assertEqual(p['peakChfKwMonth'],0.)
|
||||
del cfg['tarif_bezug_modus']
|
||||
with self.assertRaises(ValueError):publisher.tariff_payload(cfg,AT)
|
||||
def test_native_ckw_unit_zero_and_delivery_range(self):
|
||||
rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'CHF_kWh','value':0.}]}]
|
||||
p=publisher.ckw_payload('CKW Dynamisch Home',rows,AT-timedelta(hours=1),AT)
|
||||
self.assertEqual(p['periods'][0]['value'],0.);self.assertEqual(p['periods'][0]['unit'],'CHF/kWh');self.assertEqual(p['periods'][0]['sourceKind'],'published_interval')
|
||||
del rows[0]['end_timestamp']
|
||||
with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT)
|
||||
def test_unknown_unit_and_future_publication_rejected(self):
|
||||
rows=[{'start_timestamp':AT.isoformat(),'end_timestamp':(AT+timedelta(minutes=15)).isoformat(),'integrated':[{'unit':'unknown','value':100.}]}]
|
||||
with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,at=AT)
|
||||
rows[0]['integrated'][0]['unit']='CHF_kWh'
|
||||
with self.assertRaises(ValueError):publisher.ckw_payload('CKW',rows,AT+timedelta(minutes=1),AT)
|
||||
def test_forecast_not_relabelled_as_base_load(self):
|
||||
p=publisher.forecast_payload([(23,{AT:5000.},{AT:2000.})],AT)
|
||||
self.assertEqual(p['families']['23']['loadBasis'],'house_total');self.assertIsNone(p['families']['23']['trainedUntil'])
|
||||
|
||||
class InstallerTest(unittest.TestCase):
|
||||
def fixtures(self,root):
|
||||
files={'prognosis-manager-enelix2/api/main.py':'def f():\n _portal_store_configuration(anlagen_id, configuration)\n',
|
||||
'prognosis-manager-enelix2/forecast_engine/main.py':'def f():\n for a in []:\n for key, default in []:\n d[key] = float(d.get(key) or default)\n for c in []:\n p_3 = v3.predict(data_obj, p_1, p_2)\n',
|
||||
'prognosis-manager-enelix2/tariff_importer/main.py':'def f():\n for a in []:\n for b in []:\n for c in []:\n if True:\n return candidate_points\n',
|
||||
'license/server.mjs':'const server = createServer(async (req, res) => {\n if (url.pathname === "/healthz" && req.method === "GET") return json(res, 200, {status:"ok"});\n});',
|
||||
'license/public/app.js':'// Parallel GUI work. Must remain byte-identical.'}
|
||||
for name,data in files.items():p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(data)
|
||||
return files
|
||||
def test_install_idempotent_and_verified_rollback(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);originals=self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root)
|
||||
self.assertFalse(hooks.plan(root));self.assertEqual((root/'license/public/app.js').read_text(),originals['license/public/app.js'])
|
||||
hooks.rollback(receipt)
|
||||
for name,data in originals.items():self.assertEqual((root/name).read_text(),data)
|
||||
self.assertFalse((root/'license/public/netplan-v4.html').exists())
|
||||
def test_concurrent_changes_never_overwritten(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.fixtures(root);entries=hooks.plan(root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Concurrent')
|
||||
with self.assertRaises(ValueError):hooks.apply(entries,root)
|
||||
self.assertTrue(p.read_text().endswith('// Concurrent'))
|
||||
def test_later_changes_block_rollback(self):
|
||||
with TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.fixtures(root);receipt=hooks.apply(hooks.plan(root),root);p=root/'license/server.mjs';p.write_text(p.read_text()+'// Later')
|
||||
with self.assertRaises(ValueError):hooks.rollback(receipt)
|
||||
self.assertTrue(p.read_text().endswith('// Later'))
|
||||
def test_changed_or_ambiguous_anchor_fails_closed(self):
|
||||
with self.assertRaises(ValueError):hooks.once('x x','x','y')
|
||||
with self.assertRaises(ValueError):hooks.patch_api('def f():pass')
|
||||
|
||||
class HorizonValidationTest(unittest.TestCase):
|
||||
def test_synthetic_48h_energy_soc_limits(self):
|
||||
import math
|
||||
from netplan_v4.domain import Battery,Step,Price,Limits
|
||||
from netplan_v4.optimizer import optimize
|
||||
steps=[]
|
||||
for i in range(576):
|
||||
solar=max(0.,math.sin(math.pi*((i%288)/12-6)/12))*10000.
|
||||
steps.append(Step(AT+timedelta(minutes=5*i),3000.,solar,Price(.15 if i%288<72 else .35),Price(.1)))
|
||||
b=Battery('synthetic',40.,20.,10.,90.,10000.,8000.,AT,grid_charging=True)
|
||||
p=optimize(steps,[b],at=AT,limits=Limits(import_w=12000.,export_w=10000.),observed_peaks={'2026-10':8.},peak_prices={'2026-10':5.})
|
||||
self.assertTrue(p['executable']);self.assertEqual(len(p['points']),576)
|
||||
for s,x in zip(steps,p['points']):
|
||||
self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=3)
|
||||
self.assertLessEqual(x['gridTargetW'],12000.001);self.assertGreaterEqual(x['gridTargetW'],-10000.001)
|
||||
self.assertTrue(9.999<=x['socEndPercent']['synthetic']<=90.001)
|
||||
self.assertGreaterEqual(p['points'][-1]['socEndPercent']['synthetic'],19.999)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,80 @@
|
||||
import ast
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
from forecast_acceptance import source_paths, verify_forecast_bundle
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location('forecast_release_test', ROOT / 'release_preflight.py')
|
||||
preflight = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(preflight)
|
||||
|
||||
|
||||
class ForecastAcceptanceTest(unittest.TestCase):
|
||||
def fixture(self, root):
|
||||
src, bundle = root / 'source', root / 'bundle'
|
||||
for path in (src, bundle):
|
||||
path.mkdir()
|
||||
for name, value in {'main.py': 'x=1\n', 'telemetry_quality.py': 'x=2\n', 'requirements.txt': 'pandas\n'}.items():
|
||||
(path / name).write_text(value)
|
||||
manifest = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in source_paths(src)}
|
||||
(bundle / 'SOURCE_MANIFEST.json').write_text(json.dumps(manifest))
|
||||
return src, bundle
|
||||
|
||||
def test_exact_source_is_verified(self):
|
||||
with TemporaryDirectory() as d:
|
||||
src, bundle = self.fixture(Path(d))
|
||||
self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3)
|
||||
|
||||
def test_modified_development_source_blocks_test(self):
|
||||
with TemporaryDirectory() as d:
|
||||
src, bundle = self.fixture(Path(d))
|
||||
(src / 'main.py').write_text('x=3\n')
|
||||
with self.assertRaises(ValueError):
|
||||
verify_forecast_bundle(bundle, src)
|
||||
|
||||
def test_extra_staged_python_file_blocks_test(self):
|
||||
with TemporaryDirectory() as d:
|
||||
src, bundle = self.fixture(Path(d))
|
||||
(bundle / 'surprise.py').write_text('x=3\n')
|
||||
with self.assertRaises(ValueError):
|
||||
verify_forecast_bundle(bundle, src)
|
||||
|
||||
def test_symlink_is_rejected(self):
|
||||
with TemporaryDirectory() as d:
|
||||
src, bundle = self.fixture(Path(d))
|
||||
(bundle / 'main.py').unlink()
|
||||
(bundle / 'main.py').symlink_to(src / 'main.py')
|
||||
with self.assertRaises(ValueError):
|
||||
verify_forecast_bundle(bundle, src)
|
||||
|
||||
def test_models_secrets_and_databases_not_staged(self):
|
||||
with TemporaryDirectory() as d:
|
||||
src, bundle = self.fixture(Path(d))
|
||||
for name in ('.env', 'model.pkl', 'users.db'):
|
||||
(src / name).write_text('not a source file')
|
||||
self.assertEqual(len(verify_forecast_bundle(bundle, src)), 3)
|
||||
|
||||
def test_forecast_checks_have_no_network_or_live_mounts(self):
|
||||
args = next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='forecast_candidate_python311')
|
||||
self.assertEqual(args[args.index('--network')+1], 'none')
|
||||
self.assertEqual(args[args.index('--user')+1], '1000:1000')
|
||||
self.assertIn('--read-only', args)
|
||||
for forbidden in ('--volume', '-v', '--env-file', '--privileged', 'up', 'restart'):
|
||||
self.assertNotIn(forbidden, args)
|
||||
|
||||
def test_runner_does_not_import_application_main(self):
|
||||
source = (ROOT / 'acceptance/check_forecast.py').read_text()
|
||||
tree = ast.parse(source)
|
||||
imports = {a.name for node in ast.walk(tree) if isinstance(node, ast.Import) for a in node.names}
|
||||
self.assertNotIn('main', imports)
|
||||
self.assertNotIn('run_forecast(', source)
|
||||
self.assertNotIn('get_configs(', source)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,31 @@
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
from netplan_v4.forecast_quality import assess_family
|
||||
from netplan_v4.service import ingest,run_once
|
||||
from netplan_v4.store import PlannerStore
|
||||
from test_v4 import inputs,AT,AID
|
||||
|
||||
class ForecastQualityTest(unittest.TestCase):
|
||||
def test_zero_load_not_free_energy_assumption(self):
|
||||
f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)]}
|
||||
self.assertFalse(assess_family(f)['valid'])
|
||||
def test_confirmed_pv_only_plant_is_supported(self):
|
||||
f={'points':[{'loadW':0.,'pvW':1000.} for _ in range(3)],'zeroLoadConfirmed':True}
|
||||
self.assertTrue(assess_family(f)['valid'])
|
||||
def test_boolean_and_nan_are_not_forecast_powers(self):
|
||||
for value in (float('nan'),True,-1.,None):
|
||||
f={'points':[{'loadW':value,'pvW':1000.} for _ in range(3)]}
|
||||
self.assertFalse(assess_family(f)['valid'])
|
||||
def test_selected_invalid_family_not_silently_replaced(self):
|
||||
with TemporaryDirectory() as d:
|
||||
s=PlannerStore(str(Path(d)/'p.sqlite'));op,fc,tar=inputs()
|
||||
for p in fc['families']['3']['points']:p['loadW']=0.
|
||||
for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(s,AID,k,v,AT)
|
||||
plan=run_once(s,AT)
|
||||
self.assertEqual(plan['status'],'awaiting_inputs');self.assertIn('all-zero',plan['reason']);self.assertIsNone(s.current(AID));s.close()
|
||||
def test_nonzero_family_remains_usable(self):
|
||||
f={'points':[{'loadW':50.,'pvW':0.} for _ in range(3)]}
|
||||
self.assertTrue(assess_family(f)['valid'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
import ast
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import contextlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
spec=importlib.util.spec_from_file_location('integrated_deployment_test',ROOT/'deploy_integrated_shadow.py')
|
||||
d=importlib.util.module_from_spec(spec);spec.loader.exec_module(d)
|
||||
|
||||
|
||||
class IntegratedDeploymentTest(unittest.TestCase):
|
||||
def test_exact_server_scope_and_no_symcon(self):
|
||||
scope=d.compose_groups()
|
||||
self.assertEqual(set(scope),{'v4','forecast','portal'})
|
||||
self.assertEqual(scope['forecast'][2],['api','forecast-engine','tariff-importer'])
|
||||
self.assertNotIn('license-admin',scope['portal'][2])
|
||||
self.assertNotIn('/var/lib/symcon',(ROOT/'deploy_integrated_shadow.py').read_text())
|
||||
def test_sqlite_backup_is_consistent_and_source_unchanged(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
source=Path(temp)/'before.sqlite';dest=Path(temp)/'backup.sqlite'
|
||||
con=sqlite3.connect(source);con.execute('CREATE TABLE test(value INTEGER)')
|
||||
con.execute('INSERT INTO test VALUES (37)');con.commit();con.close()
|
||||
before=source.read_bytes()
|
||||
with patch.object(d.os,'geteuid',return_value=1000):result=d.backup_database(source,dest)
|
||||
self.assertEqual(result['integrity'],'ok');self.assertEqual(before,source.read_bytes())
|
||||
check=sqlite3.connect(dest);self.assertEqual(check.execute('SELECT value FROM test').fetchone()[0],37);check.close()
|
||||
self.assertEqual(result['sha256'],hashlib.sha256(dest.read_bytes()).hexdigest())
|
||||
def test_missing_database_not_fabricated(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
dst=Path(temp)/'backup'
|
||||
self.assertEqual(d.backup_database(Path(temp)/'missing',dst),{'exists':False})
|
||||
self.assertFalse(dst.exists())
|
||||
def test_scope_uses_no_environment_dump_or_control_activation(self):
|
||||
text=(ROOT/'deploy_integrated_shadow.py').read_text()
|
||||
self.assertNotIn('.Config.Env',text)
|
||||
self.assertNotIn('NetzfahrplanAktiv',text)
|
||||
self.assertNotIn('IPS_RequestAction',text)
|
||||
self.assertNotIn('PROGNOSIS_SERVICE_TOKEN',text)
|
||||
ast.parse(text)
|
||||
def test_build_and_tests_precede_all_service_recreation(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root=Path(temp)/'netplan';root.mkdir();project=Path(temp)/'forecast';project.mkdir()
|
||||
portal=Path(temp)/'portal';portal.mkdir();(root/'acceptance').mkdir()
|
||||
commands=[]
|
||||
def fake(args,**kwargs):
|
||||
commands.append(list(args));out=''
|
||||
if 'ps' in args:out='0123456789abcdef\n'
|
||||
elif 'inspect' in args:
|
||||
out='sha256:'+('a'*64) if '{{.Image}}' in args else '/some/compose.yaml'
|
||||
elif 'exec' in args:
|
||||
out=json.dumps({'health':{'mode':'shadow','liveEnabled':False},'inputEventsByKind':{},'lastRunStatus':None})
|
||||
return SimpleNamespace(returncode=0,stdout=out)
|
||||
with patch.object(d,'ROOT',root),patch.object(d,'PROJECT',project),patch.object(d,'PORTAL',portal),patch.object(d,'verify_sources',return_value={}),patch.object(d.os,'geteuid',return_value=1000),patch.object(d.subprocess,'run',side_effect=fake),patch('install_hooks.plan',return_value={}),patch('install_hooks.apply',return_value=None),contextlib.redirect_stdout(io.StringIO()):
|
||||
d.run('e3a08f9e-af12-4695-99bd-8b51c0520021')
|
||||
ups=[i for i,c in enumerate(commands) if 'up' in c]
|
||||
self.assertEqual(len(ups),3)
|
||||
test_at=next(i for i,c in enumerate(commands) if '--test-only' in c)
|
||||
build_at=max(i for i,c in enumerate(commands) if 'build' in c)
|
||||
self.assertLess(test_at,min(ups));self.assertLess(build_at,min(ups))
|
||||
self.assertTrue(all('--no-build' in commands[i] and '--no-deps' in commands[i] for i in ups))
|
||||
report=json.loads(next((root/'deployment-reports').glob('*/DEPLOYMENT.json')).read_text())
|
||||
self.assertEqual(report['status'],'server_shadow_installed')
|
||||
self.assertFalse(report['liveEnabled']);self.assertFalse(report['productionReady'])
|
||||
@@ -0,0 +1,161 @@
|
||||
import copy
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from fastapi.testclient import TestClient
|
||||
from netplan_v4 import measurement_pipeline as m
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.service import create_app, ingest, run_once, status
|
||||
from test_v4 import inputs, AID, TOKEN
|
||||
|
||||
NOW=int(datetime(2026,10,2,12,tzinfo=timezone.utc).timestamp())
|
||||
|
||||
def config(**changes):
|
||||
c={'datasetId':'physical-v1','mappingSha256':'a'*64,'inventorySha256':'b'*64,
|
||||
'formula':'physical_sum_v1','solarReference':None,'minimumCoverage':.95,
|
||||
'maximumGapSeconds':10,'minimumTrainingHours':1,'historyDays':14,'sources':[]}
|
||||
for i,(k,role) in enumerate((('grid','grid'),('pv','pv'),('battery','physical_storage'),('sdl','sdl_request'))):
|
||||
c['sources'].append({'key':k,'role':role,'variableId':101+i,'factorToW':1,'maxAgeSeconds':60})
|
||||
c.update(changes);return c
|
||||
|
||||
def record(t, c=None, **values):
|
||||
c=c or config(); v={'grid':3000.,'pv':5000.,'battery':2000.,'sdl':0.,**values}
|
||||
return {'schemaVersion':1,'kind':'raw_accounting_capture','installationId':AID,
|
||||
'capturedAt':m.iso(t),'captureStartedAt':m.iso(t),'mappingSha256':c['mappingSha256'],
|
||||
'reportedInventorySha256':c['inventorySha256'],'raw':{s['key']:{'variableId':s['variableId'],'value':v.get(s['key'],0),
|
||||
'sourceUpdatedAt':t,'issues':[]} for s in c['sources']}}
|
||||
|
||||
class MeasurementPipelineTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.path=str(Path(self.tmp.name)/'pipeline.sqlite')
|
||||
self.s=PlannerStore(self.path);self.c=config();m.register_dataset(self.s.con,AID,self.c,NOW)
|
||||
def tearDown(self):self.s.close();self.tmp.cleanup()
|
||||
def batch(self,rows,now=NOW,plant=AID):return m.ingest_batch(self.s.con,plant,{'version':1,'datasetId':'physical-v1','records':rows},now)
|
||||
def history(self,hours=2):
|
||||
rows=[record(t) for t in range(NOW-hours*3600,NOW+1,30)]
|
||||
for i in range(0,len(rows),120):self.batch(rows[i:i+120])
|
||||
def test_append_ack_idempotence(self):
|
||||
r=record(NOW);self.assertEqual(self.batch([r])['stored'],1)
|
||||
a=self.batch([r]);self.assertEqual(a['duplicates'],1);self.assertEqual(a['acceptedThrough'],m.iso(NOW))
|
||||
def test_conflict_rolls_back_whole_batch(self):
|
||||
self.batch([record(NOW)])
|
||||
with self.assertRaises(ValueError):self.batch([record(NOW-30),record(NOW,grid=9000)])
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_observations').fetchone()[0],1)
|
||||
def test_registration_immutable(self):
|
||||
self.assertEqual(m.register_dataset(self.s.con,AID,self.c,NOW)['status'],'configured')
|
||||
with self.assertRaises(ValueError):m.register_dataset(self.s.con,AID,{**self.c,'minimumCoverage':1.},NOW)
|
||||
def test_plants_isolated(self):
|
||||
with self.assertRaises(ValueError):self.batch([record(NOW)],plant='30509683-4569-49e4-848f-4905e4cc813a')
|
||||
def test_mapping_is_not_device_supplied(self):
|
||||
r=record(NOW);r['mappingSha256']='c'*64
|
||||
with self.assertRaises(ValueError):self.batch([r])
|
||||
def test_no_unknown_text_persisted(self):
|
||||
r=record(NOW);r['unknown']='do-not-store';r['raw']['grid']['value']='do-not-store';r['assessment']={'status':'verified'}
|
||||
self.batch([r]);data=self.s.con.execute('SELECT value FROM planner_observations').fetchone()[0]
|
||||
self.assertNotIn('do-not-store',data);self.assertIn('null',data);self.assertNotIn('verified',data)
|
||||
def test_missing_zero_not_invented(self):
|
||||
r=record(NOW);r['raw'].pop('grid');p=m.project(r,self.c,AID,NOW)
|
||||
self.assertIsNone(p['raw']['grid']['value']);self.assertFalse(p['raw']['grid']['valid'])
|
||||
def test_boolean_not_numeric(self):
|
||||
p=m.project(record(NOW,grid=True),self.c,AID,NOW);self.assertFalse(p['raw']['grid']['valid'])
|
||||
def test_batch_order_required(self):
|
||||
with self.assertRaises(ValueError):self.batch([record(NOW),record(NOW-30)])
|
||||
def test_batch_size_limit(self):
|
||||
with self.assertRaises(ValueError):self.batch([record(NOW)]*121)
|
||||
def test_no_naive_time(self):
|
||||
r=record(NOW);r['capturedAt']='2026-10-02T12:00:00'
|
||||
with self.assertRaises(ValueError):self.batch([r])
|
||||
def test_future_source_not_rejuvenated(self):
|
||||
r=record(NOW);r['raw']['grid']['sourceUpdatedAt']=NOW+1
|
||||
self.assertFalse(m.project(r,self.c,AID,NOW)['raw']['grid']['valid'])
|
||||
def test_profile_uses_physical_not_virtual(self):
|
||||
r=record(NOW);r['raw']['virtual_ev']={'value':-90000}
|
||||
self.assertEqual(m.physical_value({'grid':3000,'pv':5000,'battery':2000},self.c),6000)
|
||||
def test_zero_numeric_valid(self):
|
||||
p=m.project(record(NOW,grid=0,pv=0,battery=0),self.c,AID,NOW);self.assertTrue(p['raw']['grid']['valid'])
|
||||
def projected(self,rows):return [m.project(r,self.c,AID,NOW) for r in rows]
|
||||
def test_rollup_time_weighted(self):
|
||||
rows=[record(t,grid=3000 if t<NOW-150 else 6000) for t in range(NOW-300,NOW+1,30)]
|
||||
w=m.reconstruct(self.projected(rows),self.c)[0]
|
||||
self.assertEqual(w['coveredSeconds'],300);self.assertAlmostEqual(w['loadW'],7500)
|
||||
def test_gap_remains_explicit(self):
|
||||
rows=[record(t) for t in range(NOW-300,NOW+1,30) if t!=NOW-120]
|
||||
w=m.reconstruct(self.projected(rows),self.c)[0]
|
||||
self.assertLess(w['coverage'],1);self.assertFalse(w['profileUsable'])
|
||||
def test_stale_source_expires(self):
|
||||
rows=[record(t) for t in range(NOW-300,NOW+1,30)]
|
||||
for r in rows:r['raw']['pv']['sourceUpdatedAt']=NOW-300
|
||||
w=m.reconstruct(self.projected(rows),self.c)[0]
|
||||
self.assertEqual(w['coveredSeconds'],60)
|
||||
def test_virtual_missing_not_veto_physical(self):
|
||||
rows=[record(t,sdl=None) for t in range(NOW-300,NOW+1,30)]
|
||||
w=m.reconstruct(self.projected(rows),self.c)[0];self.assertTrue(w['profileUsable'])
|
||||
def test_last_bin_no_extrapolation(self):
|
||||
rows=[record(t) for t in range(NOW-300,NOW,30)]
|
||||
w=m.reconstruct(self.projected(rows),self.c)[0];self.assertEqual(w['coveredSeconds'],270);self.assertFalse(w['profileUsable'])
|
||||
def test_short_history_collects(self):
|
||||
self.batch([record(NOW-30),record(NOW)])
|
||||
m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
|
||||
self.assertIsNone(m.current_model(self.s.con,AID,'physical-v1',NOW))
|
||||
def test_trained_model_persistent_and_versioned(self):
|
||||
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
|
||||
model=m.current_model(self.s.con,AID,'physical-v1',NOW)
|
||||
self.assertIsNotNone(model);self.assertEqual(m.predict(model,'3',NOW),6000)
|
||||
self.assertEqual(model['validation']['status'],'bootstrap_insufficient_holdout')
|
||||
self.s.close();self.s=PlannerStore(self.path)
|
||||
self.assertEqual(m.current_model(self.s.con,AID,'physical-v1',NOW)['modelId'],model['modelId'])
|
||||
def test_daily_training_not_every_sample(self):
|
||||
self.history();settings=self.s.settings(AID);m.advance(self.s.con,AID,'physical-v1',settings,NOW)
|
||||
m.advance(self.s.con,AID,'physical-v1',settings,NOW+300)
|
||||
self.assertEqual(self.s.con.execute('SELECT COUNT(*) FROM planner_load_models').fetchone()[0],1)
|
||||
def test_corrected_source_never_falls_back(self):
|
||||
op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc))
|
||||
with self.assertRaises(ValueError):m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW)
|
||||
def test_external_is_labelled_persistence_not_published(self):
|
||||
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
|
||||
op,fc,tar=inputs(datetime.fromtimestamp(NOW,timezone.utc));out=m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW)
|
||||
self.assertEqual(out['families']['3']['points'][0]['loadW'],6000)
|
||||
self.assertFalse(out['families']['3']['dataPipeline']['futureSdlPublished'])
|
||||
self.assertNotIn('accountingEvidenceId',out['families']['3'])
|
||||
self.assertEqual(fc['families']['3']['points'][0]['loadW'],2000)
|
||||
def test_unknown_external_no_fake_zero(self):
|
||||
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
|
||||
r=record(NOW+30,sdl=None);self.batch([r],NOW+30)
|
||||
op,fc,tar=inputs(datetime.fromtimestamp(NOW+30,timezone.utc))
|
||||
with self.assertRaises(ValueError):m.apply_load_forecast(self.s.con,AID,'physical-v1',fc,NOW+30)
|
||||
def test_native_data_to_model_to_existing_optimizer(self):
|
||||
self.history();now=datetime.fromtimestamp(NOW,timezone.utc)
|
||||
self.s.save_settings(AID,{'forecastSource':'corrected_profile','measurementDataset':'physical-v1'},0,now)
|
||||
for kind,v in zip(('operation','forecast','tariffs'),inputs(now)):ingest(self.s,AID,kind,v,now)
|
||||
p=run_once(self.s,now)
|
||||
self.assertTrue(p.get('executable'),p);self.assertFalse(p['liveEnabled'])
|
||||
self.assertEqual(p['inputQuality']['dataPipeline']['datasetId'],'physical-v1')
|
||||
self.assertTrue(status(self.s,AID,now)['dataPipeline']['datasets'][0]['records'])
|
||||
def test_model_not_available_before_training_time(self):
|
||||
self.history();m.advance(self.s.con,AID,'physical-v1',self.s.settings(AID),NOW)
|
||||
self.assertIsNone(m.current_model(self.s.con,AID,'physical-v1',NOW-1))
|
||||
def test_dataset_setting_explicit(self):
|
||||
with self.assertRaises(ValueError):self.s.save_settings(AID,{'forecastSource':'corrected_profile'},0,datetime.fromtimestamp(NOW,timezone.utc))
|
||||
def test_http_scoped_ingest_and_configuration(self):
|
||||
app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False)
|
||||
path=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN}
|
||||
with TestClient(app) as client:
|
||||
self.assertEqual(client.put(path+'/datasets/physical-v1',json=self.c).status_code,401)
|
||||
self.assertEqual(client.put(path+'/datasets/physical-v1',json=self.c,headers=h).status_code,200)
|
||||
now=int(datetime.now(timezone.utc).timestamp());payload={'version':1,'datasetId':'physical-v1','records':[record(now)]}
|
||||
self.assertEqual(client.post(path+'/measurements',json=payload,headers=h).status_code,200)
|
||||
s=client.get(path,headers=h).json();self.assertEqual(s['dataPipeline']['datasets'][0]['records'],1)
|
||||
self.assertFalse(s['liveEnabled'])
|
||||
def test_solar_terminal_not_battery_twice(self):
|
||||
c=copy.deepcopy(self.c);c['formula']='solar_terminal_v1'
|
||||
c['solarReference']={'pvKey':'pv','batteryKey':'battery','rawKey':'ac','scaleKey':'sf'}
|
||||
c['sources'] += [{'key':k,'role':role,'variableId':i,'factorToW':1,'maxAgeSeconds':60} for k,role,i in [('ac','solar_raw',105),('sf','solar_scale',106)]]
|
||||
m.validate_config(c)
|
||||
self.assertEqual(m.physical_value({'grid':3000,'ac':-4000,'sf':-2},c),2960)
|
||||
def test_solar_invalid_sentinel(self):
|
||||
c=copy.deepcopy(self.c);c['formula']='solar_terminal_v1';c['solarReference']={'pvKey':'pv','batteryKey':'battery','rawKey':'ac','scaleKey':'sf'}
|
||||
with self.assertRaises(ValueError):m.physical_value({'grid':3000,'ac':-32768,'sf':0},c)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,134 @@
|
||||
from datetime import datetime,timedelta,timezone
|
||||
import json
|
||||
import unittest
|
||||
from netplan_v4.metering import CounterSeries,Reading,MeterEvidence,audit_capture,native_meter_id
|
||||
|
||||
UTC=timezone.utc
|
||||
# October begins at September 30 22:00 UTC in Europe/Zurich.
|
||||
BEGIN=datetime(2026,9,30,22,0,tzinfo=UTC)
|
||||
END=BEGIN+timedelta(minutes=15)
|
||||
SOURCES=[{'VariableID':59607,'ElternID':11490,'Ident':'Energy_0','FaktorZuKWh':1.,'Messgroesse':'WirkenergieBezug'},
|
||||
{'VariableID':26620,'ElternID':11490,'Ident':'Energy_1','FaktorZuKWh':1.,'Messgroesse':'WirkenergieBezug'}]
|
||||
|
||||
def stream(name,pairs,verified=False,**kwargs):
|
||||
return CounterSeries(name,[Reading(BEGIN+timedelta(seconds=t),v) for t,v in pairs],
|
||||
timestamp_verified=verified,identity_verified=verified,chronology_verified=verified,**kwargs)
|
||||
|
||||
def meter(t1=None,t2=None,verified=False):
|
||||
return MeterEvidence([stream('symcon:59607',t1 or [(0,100.),(900,102.)],verified),
|
||||
stream('symcon:26620',t2 or [(0,0.),(900,0.)],verified)],native_sources=SOURCES)
|
||||
|
||||
class MeterEvidenceTest(unittest.TestCase):
|
||||
def test_t1_t2_sum_and_average(self):
|
||||
result=meter(t2=[(0,10.),(900,11.)],verified=True).interval(BEGIN,END)
|
||||
self.assertAlmostEqual(result['lowerKwh'],3.)
|
||||
self.assertAlmostEqual(result['upperAverageKw'],12.)
|
||||
self.assertTrue(result['billingEvidence'])
|
||||
def test_known_zero_is_valid_not_missing(self):
|
||||
self.assertEqual(meter(verified=True).strict_contract(END)['measuredPeaks']['2026-10']['kw'],8.)
|
||||
def test_missing_tariff_not_assumed_zero(self):
|
||||
m=MeterEvidence([stream('a',[(0,1.),(900,2.)]),stream('b',[])])
|
||||
self.assertEqual(m.interval(BEGIN,END)['status'],'missing')
|
||||
def test_current_zero_does_not_prove_past(self):
|
||||
m=MeterEvidence([stream('a',[(0,1.),(900,2.)]),stream('b',[(900,0.)])])
|
||||
self.assertFalse(m.month(END)['historyComplete'])
|
||||
def test_missing_month_peak_not_zero(self):
|
||||
result=meter(t2=[(900,0.)]).month(END)
|
||||
self.assertIsNone(result['verifiedMonthPeakKw'])
|
||||
self.assertIsNone(result['monthPeakUpperKw'])
|
||||
def test_first_quarter_not_already_paid_peak(self):
|
||||
result=meter(verified=True).month(BEGIN+timedelta(minutes=5))
|
||||
self.assertEqual(result['completedQuarters'],0)
|
||||
self.assertFalse(result['billingEvidence'])
|
||||
self.assertIsNone(result['verifiedMonthPeakKw'])
|
||||
def test_reset_quarantines_interval(self):
|
||||
result=meter(t1=[(0,100.),(400,0.),(900,2.)]).interval(BEGIN,END)
|
||||
self.assertEqual(result['status'],'missing')
|
||||
def test_unverified_archive_never_becomes_billing_evidence(self):
|
||||
m=meter()
|
||||
self.assertTrue(m.month(END)['historyComplete'])
|
||||
with self.assertRaises(ValueError):m.strict_contract(END)
|
||||
def test_boundaries_not_interpolated_into_fact(self):
|
||||
s=stream('a',[(-2,100.),(2,100.1),(898,101.9),(902,102.)],verified=True)
|
||||
e=MeterEvidence([s]).interval(BEGIN,END)
|
||||
self.assertEqual(e['status'],'bounded_records')
|
||||
self.assertAlmostEqual(e['lowerKwh'],1.8)
|
||||
self.assertAlmostEqual(e['upperKwh'],2.)
|
||||
self.assertFalse(e['billingEvidence'])
|
||||
def test_wide_boundary_gap_is_unknown(self):
|
||||
s=stream('a',[(-200,100.),(200,100.1),(900,101.)])
|
||||
self.assertIsNone(s.energy(BEGIN,END))
|
||||
def test_no_extrapolation(self):
|
||||
self.assertIsNone(stream('a',[(0,1.),(900,2.)]).energy(BEGIN,END+timedelta(seconds=1)))
|
||||
def test_duplicate_identical_records_are_idempotent(self):
|
||||
self.assertEqual(len(stream('a',[(0,1.),(0,1.),(900,2.)]).times),2)
|
||||
def test_duplicate_conflicting_records_fail(self):
|
||||
with self.assertRaises(ValueError):stream('a',[(0,1.),(0,1.1)])
|
||||
def test_invalid_counter_values_fail(self):
|
||||
for value in (float('nan'),float('inf'),True,-1.,'2'):
|
||||
with self.subTest(value=value),self.assertRaises(ValueError):stream('a',[(0,value)])
|
||||
def test_duplicate_tariffs_fail(self):
|
||||
with self.assertRaises(ValueError):MeterEvidence([stream('a',[]),stream('a',[])])
|
||||
def test_native_identity_ignores_order_not_factor(self):
|
||||
self.assertEqual(native_meter_id(SOURCES),native_meter_id(list(reversed(SOURCES))))
|
||||
changed=[dict(s) for s in SOURCES];changed[0]['FaktorZuKWh']=.001
|
||||
self.assertNotEqual(native_meter_id(SOURCES),native_meter_id(changed))
|
||||
def test_identity_matches_php_encoder(self):
|
||||
# Captured from PHP 8.3 on the test host, not calculated by this Python code.
|
||||
cases={1.:'29f33f47e5c3a312e9c3c85735f95ea16d237860d76c0e3720c59e567d77ec8c',
|
||||
.001:'5600bff9a5d82aa6230196f90f836c69c966374a1458864e770083656ed65f4c',
|
||||
.000001:'477aca4ce6eb1079b58114911125675137a5f7d0ec8b0b3a2025e3323cb269a4'}
|
||||
for factor,digest in cases.items():
|
||||
sources=[dict(s) for s in SOURCES];sources[0]['FaktorZuKWh']=factor
|
||||
with self.subTest(factor=factor):self.assertEqual(native_meter_id(sources),'symcon-active-import:'+digest)
|
||||
def test_native_source_set_must_match(self):
|
||||
with self.assertRaises(ValueError):MeterEvidence([stream('symcon:53476',[])],native_sources=SOURCES)
|
||||
def test_no_legacy_evidence_identity(self):
|
||||
m=MeterEvidence([stream('a',[(0,1.),(900,2.)],verified=True)])
|
||||
self.assertTrue(m.meter_id.startswith('audit-only:'))
|
||||
with self.assertRaises(ValueError):m.strict_contract(END)
|
||||
def test_native_other_meter_or_quantity_rejected(self):
|
||||
for field,value in [('ElternID',1),('Messgroesse','Blindenergie')]:
|
||||
sources=[dict(s) for s in SOURCES];sources[0][field]=value
|
||||
with self.subTest(field=field),self.assertRaises(ValueError):native_meter_id(sources)
|
||||
def test_partial_quarter_must_be_verified(self):
|
||||
at=END+timedelta(minutes=5)
|
||||
m=meter(t1=[(0,100.),(900,102.),(1200,103.)],t2=[(0,0.),(900,0.),(1200,0.)],verified=True)
|
||||
contract=m.strict_contract(at)
|
||||
self.assertEqual(contract['quarterPast']['measuredSeconds'],300)
|
||||
self.assertEqual(contract['quarterPast']['importKwh'],1.)
|
||||
def test_partial_quarter_not_extrapolated(self):
|
||||
with self.assertRaises(ValueError):meter(verified=True).strict_contract(END+timedelta(seconds=1))
|
||||
def test_zurich_month_not_utc_month(self):
|
||||
self.assertEqual(meter().month(END)['month'],'2026-10')
|
||||
def test_dst_repeated_hour_not_merged(self):
|
||||
from zoneinfo import ZoneInfo
|
||||
zone=ZoneInfo('Europe/Zurich')
|
||||
a=datetime(2026,10,25,2,0,tzinfo=zone,fold=0)
|
||||
b=datetime(2026,10,25,2,0,tzinfo=zone,fold=1)
|
||||
s=CounterSeries('a',[Reading(a,1.),Reading(b,2.)])
|
||||
self.assertEqual(len(s.times),2)
|
||||
self.assertEqual((s.times[1]-s.times[0]).total_seconds(),3600)
|
||||
def test_naive_timestamps_rejected(self):
|
||||
with self.assertRaises(ValueError):CounterSeries('a',[Reading(datetime(2026,10,1),1.)])
|
||||
def test_capture_is_only_an_audit(self):
|
||||
channels={}
|
||||
for var,ident,value in [('59607','Energy_0',2.),('26620','Energy_1',0.)]:
|
||||
channels[var]={'ident':ident,'logging':True,'queryComplete':True,
|
||||
'history':[{'TimeStamp':int(BEGIN.timestamp()),'Value':0.},
|
||||
{'TimeStamp':int(END.timestamp()),'Value':value}],
|
||||
'snapshotConsistent':True,'snapshot':{'capturedAt':END.isoformat(),'value':value}}
|
||||
result=audit_capture({'schemaVersion':1,'kind':'v4_meter_capture',
|
||||
'capturedAt':END.isoformat(),'channels':channels})
|
||||
self.assertTrue(result['month']['historyComplete'])
|
||||
self.assertFalse(result['billingEvidence'])
|
||||
self.assertFalse(result['liveEnabled'])
|
||||
def test_missing_logging_survives_without_fake_1970(self):
|
||||
channels={str(var):{'ident':ident,'logging':False,'queryComplete':False,
|
||||
'history':[],'snapshotConsistent':True,'snapshot':{'capturedAt':END.isoformat(),'value':0.}}
|
||||
for var,ident in [(59607,'Energy_0'),(26620,'Energy_1')]}
|
||||
result=audit_capture({'schemaVersion':1,'kind':'v4_meter_capture','capturedAt':END.isoformat(),'channels':channels})
|
||||
self.assertFalse(result['month']['historyComplete'])
|
||||
self.assertNotIn('1970-',json.dumps(result))
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,155 @@
|
||||
import copy
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
from netplan_v4.domain import Battery,Limits,Price,Step,ZURICH
|
||||
from netplan_v4.peak_policy import basis_record,PeakScenario,RestMonthOutlook,empirical_rest_month
|
||||
from netplan_v4.meter_runtime import integrate_power,assumptions,observe,daily_peaks
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.service import ingest,run_once,status,create_app
|
||||
from netplan_v4.optimizer import optimize
|
||||
from test_v4 import inputs,seq,batt,AT,AID,TOKEN
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def estimate(kw=5.,at=AT,source='power_history_estimate'):
|
||||
return {'kw':kw,'quality':'estimated','source':source,'observedAt':at.isoformat(),'coverage':.8,'notes':'Incomplete historical archive'}
|
||||
|
||||
def outlook(reliance=1.,scenarios=((5.,1.),),at=AT):
|
||||
return RestMonthOutlook('2026-10',tuple(PeakScenario(*s) for s in scenarios),at,at+timedelta(hours=1),at-timedelta(days=1),'same-policy','synthetic-test','evidence-test',reliance)
|
||||
|
||||
|
||||
class PeakEconomicsTest(unittest.TestCase):
|
||||
def solve(self,**kw):
|
||||
kw.setdefault('steps',seq([1000.]*6,buy=[.01]*3+[1.]*3))
|
||||
kw.setdefault('batteries',[batt(soc_percent=10.)]);kw.setdefault('at',AT)
|
||||
kw.setdefault('observed_peaks',{'2026-10':1.});kw.setdefault('peak_prices',{'2026-10':5.})
|
||||
return optimize(**kw)
|
||||
def test_full_tariff_still_default(self):
|
||||
p=self.solve();self.assertTrue(p['executable']);self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001)
|
||||
self.assertEqual(p['peakOutlook'],'full_incremental_tariff')
|
||||
def test_increase_allowed_when_future_month_peak_expected(self):
|
||||
p=self.solve(peak_outlooks={'2026-10':outlook()})
|
||||
self.assertTrue(p['executable']);self.assertGreater(p['plannedPeaksKw']['2026-10'],1.1)
|
||||
self.assertGreater(p['additionalPeakCostChf'],0.)
|
||||
self.assertAlmostEqual(p['planningPeakCostChf'],0.)
|
||||
self.assertEqual(p['measuredPeaksKw']['2026-10'],1.)
|
||||
def test_zero_reliance_equals_full_tariff(self):
|
||||
p=self.solve(peak_outlooks={'2026-10':outlook(reliance=0.)})
|
||||
self.assertAlmostEqual(p['planningPeakCostChf'],p['additionalPeakCostChf'])
|
||||
def test_marginal_price_non_decreasing_and_full_above_all_scenarios(self):
|
||||
o=outlook(.8,((2.,.5),(4.,.5)))
|
||||
costs=[o.incremental_cost(1.,x,5.) for x in range(1,8)]
|
||||
slopes=[b-a for a,b in zip(costs,costs[1:])]
|
||||
self.assertTrue(all(b>=a-1e-10 for a,b in zip(slopes,slopes[1:])));self.assertAlmostEqual(slopes[-1],5.)
|
||||
self.assertAlmostEqual(o.incremental_cost(1.,.8,5.),0.)
|
||||
def test_hard_cap_beats_outlook(self):
|
||||
p=self.solve(limits=Limits(import_w=1000.),peak_outlooks={'2026-10':outlook()})
|
||||
self.assertLessEqual(max(x['gridTargetW'] for x in p['points']),1000.001)
|
||||
def test_no_fixed_cap_is_supported(self):
|
||||
p=self.solve(limits=Limits(),peak_outlooks={'2026-10':outlook()})
|
||||
self.assertTrue(p['executable']);self.assertTrue(all(x['importLimitW'] is None for x in p['points']))
|
||||
def test_monthly_cap_beats_outlook(self):
|
||||
p=self.solve(limits=Limits(manager_month_limits_w={10:1000.}),peak_outlooks={'2026-10':outlook()})
|
||||
self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001)
|
||||
def test_estimated_basis_not_in_measured_map(self):
|
||||
p=self.solve(peak_context={'2026-10':estimate(1.)})
|
||||
self.assertEqual(p['measuredPeaksKw'],{});self.assertEqual(p['peakBasisKw']['2026-10'],1.)
|
||||
self.assertTrue(p['peakCostIsEstimate'])
|
||||
def test_context_cannot_lie_about_numerical_basis(self):
|
||||
self.assertFalse(self.solve(peak_context={'2026-10':estimate(3.)})['executable'])
|
||||
def test_no_free_cap_source(self):
|
||||
with self.assertRaises(ValueError):basis_record(estimate(source='manager_cap'),'2026-10',AT,True)
|
||||
def test_overlap_rejected(self):
|
||||
self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),future_from=AT)})['executable'])
|
||||
def test_future_observation_rejected(self):
|
||||
self.assertFalse(self.solve(peak_outlooks={'2026-10':replace(outlook(),issued_at=AT+timedelta(seconds=1))})['executable'])
|
||||
def test_wrong_month_rejected(self):
|
||||
self.assertFalse(self.solve(peak_outlooks={'2026-11':outlook()})['executable'])
|
||||
def test_invalid_probabilities_rejected(self):
|
||||
self.assertFalse(self.solve(peak_outlooks={'2026-10':outlook(scenarios=((5.,.8),))})['executable'])
|
||||
def test_no_history_no_outlook_not_fake_allowance(self):
|
||||
self.assertIsNone(empirical_rest_month([],month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p'))
|
||||
def test_empirical_requires_comparable_complete_history(self):
|
||||
records=[]
|
||||
for n in range(20):
|
||||
d=AT.astimezone(ZURICH).replace(hour=0,minute=0)-timedelta(days=n+1)
|
||||
records.append({'day':d.strftime('%Y-%m-%d'),'peakKw':3.+n%4,'observedAt':(d+timedelta(days=1)).isoformat(),'quality':'estimated','complete':True,'controlPolicyId':'p'})
|
||||
args=dict(month='2026-10',at=AT,horizon_end=AT+timedelta(days=1),control_policy_id='p')
|
||||
o=empirical_rest_month(records,**args);self.assertIsNotNone(o);o.validate(AT,args['horizon_end'])
|
||||
for r in records:r['controlPolicyId']='different'
|
||||
self.assertIsNone(empirical_rest_month(records,**args))
|
||||
|
||||
|
||||
class EstimatedBasisServiceTest(unittest.TestCase):
|
||||
def setUp(self):self.tmp=TemporaryDirectory();self.path=str(Path(self.tmp.name)/'planner.sqlite');self.s=PlannerStore(self.path)
|
||||
def tearDown(self):self.s.close();self.tmp.cleanup()
|
||||
def load(self,when=AT,quarter=False):
|
||||
op,fc,tar=inputs(when);op['measuredPeaks']={};op['planningPeaks']={'2026-10':estimate(at=when)}
|
||||
if quarter:
|
||||
op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.2,'measuredSeconds':int((when-AT).total_seconds()),'quality':'estimated','source':'power_history_estimate','coverage':1.}
|
||||
for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,when)
|
||||
return op
|
||||
def allow(self):self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},self.s.settings(AID)['revision'],AT)
|
||||
def test_opt_in_required(self):
|
||||
self.load();self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs')
|
||||
self.allow();p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertTrue(p['peakCostIsEstimate'])
|
||||
self.assertEqual(self.s.peaks(AID),{})
|
||||
def test_no_peak_not_invented_even_when_opted_in(self):
|
||||
op,fc,tar=inputs();op['measuredPeaks']={};self.allow()
|
||||
for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT)
|
||||
self.assertEqual(run_once(self.s,AT)['status'],'awaiting_inputs')
|
||||
def test_historical_seed_independent_from_live_operation(self):
|
||||
self.allow();op,fc,tar=inputs();op['measuredPeaks']={}
|
||||
for k,v in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,k,v,AT)
|
||||
ingest(self.s,AID,'planning_basis',{'version':1,'eventId':'seed','observedAt':AT.isoformat(),'peaks':{'2026-10':estimate()}},AT)
|
||||
p=run_once(self.s,AT);self.assertTrue(p['executable']);self.assertEqual(p['measuredPeaksKw'],{})
|
||||
def test_estimates_persist_but_plans_do_not_increase_them(self):
|
||||
self.load();self.allow();p=run_once(self.s,AT)
|
||||
self.assertEqual(assumptions(self.s.con,AID)['2026-10']['kw'],5.)
|
||||
self.s.close();self.s=PlannerStore(self.path)
|
||||
self.assertEqual(assumptions(self.s.con,AID)['2026-10']['quality'],'estimated')
|
||||
def test_verified_source_has_precedence_even_if_lower_than_estimate(self):
|
||||
self.load();self.allow();self.s.initialize_peak(AID,'2026-10',4.,'verified_month_history',AT)
|
||||
p=run_once(self.s,AT);self.assertEqual(p['measuredPeaksKw'],{'2026-10':4.});self.assertFalse(p['peakCostIsEstimate'])
|
||||
def test_quarter_estimate_never_disguised(self):
|
||||
self.load(AT+timedelta(minutes=5),True);self.allow();p=run_once(self.s,AT+timedelta(minutes=5))
|
||||
self.assertTrue(p['executable']);self.assertEqual(p['inputQuality']['quarter'],'estimated')
|
||||
def test_incomplete_quarter_refused(self):
|
||||
op,_,_=inputs();op['quarterEstimate']={'start':AT.isoformat(),'importKwh':.1,'measuredSeconds':0,'quality':'estimated','source':'power_history_estimate','coverage':.8}
|
||||
with self.assertRaises(ValueError):ingest(self.s,AID,'operation',op,AT)
|
||||
def test_new_settings_load_old_rows(self):
|
||||
old=self.s.settings(AID);old.pop('revision');old.pop('measurementPolicy');old.pop('peakOutlookPolicy');old.pop('peakOutlookReliance')
|
||||
with self.s.con:self.s.con.execute('INSERT INTO planner_settings VALUES(?,?,?)',(AID,1,json.dumps(old)))
|
||||
self.assertEqual(self.s.settings(AID)['measurementPolicy'],'verified_only')
|
||||
self.s.save_settings(AID,{'measurementPolicy':'allow_estimates'},1,AT)
|
||||
def test_http_explicit_estimate_optin_and_no_live(self):
|
||||
app=create_app(str(Path(self.tmp.name)/'api.sqlite'),TOKEN,[AID],start_worker=False)
|
||||
with TestClient(app) as c:
|
||||
url=f'/internal/v2/prognosis/{AID}/planner';h={'X-Enelix-Service-Token':TOKEN}
|
||||
self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':0,'changes':{'measurementPolicy':'allow_estimates'}}).status_code,200)
|
||||
self.assertEqual(c.put(url+'/settings',headers=h,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400)
|
||||
|
||||
|
||||
class RuntimeMeterTest(unittest.TestCase):
|
||||
def test_exact_samples_still_estimates(self):
|
||||
r=integrate_power([(0,1000.,0.,'p'),(60,1000.,.02,'p'),(120,1000.,.04,'p')],0,120)
|
||||
self.assertAlmostEqual(r['importKwh'],1/30);self.assertFalse(r['billingEvidence'])
|
||||
def test_gap_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,0.,'p'),(180,1000.,.1,'p')],0,180))
|
||||
def test_reset_not_bridged(self):self.assertIsNone(integrate_power([(0,1000.,1.,'p'),(60,1000.,0.,'p')],0,60))
|
||||
def test_control_policy_change_not_comparable(self):self.assertIsNone(integrate_power([(0,1000.,0.,'a'),(60,1000.,1.,'b')],0,60))
|
||||
def test_only_completed_quarters_update_peak(self):
|
||||
with TemporaryDirectory() as d:
|
||||
s=PlannerStore(str(Path(d)/'p.sqlite'));mid='symcon-active-import:'+'a'*64
|
||||
for i in range(16):
|
||||
at=AT+timedelta(minutes=i)
|
||||
obs={'meterId':mid,'sampleAt':at.isoformat(),'powerW':2000.,'totalImportKwh':10+i/30,'controlPolicyId':'p'}
|
||||
with s.con:observe(s.con,AID,obs,at)
|
||||
if i<15:self.assertEqual(assumptions(s.con,AID),{})
|
||||
self.assertAlmostEqual(assumptions(s.con,AID)['2026-10']['kw'],2.)
|
||||
self.assertEqual(s.peaks(AID),{});s.close()
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import copy
|
||||
from datetime import timedelta
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from netplan_v4.receiver_contract import control_context, provenance
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.service import ingest, run_once, status
|
||||
from test_v4 import inputs, AT, AID
|
||||
|
||||
class ReceiverContractTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.s=PlannerStore(str(Path(self.tmp.name)/'qa.sqlite'))
|
||||
op,fc,tar=inputs()
|
||||
for kind,value in zip(('operation','forecast','tariffs'),(op,fc,tar)):ingest(self.s,AID,kind,value,AT)
|
||||
def tearDown(self):self.s.close();self.tmp.cleanup()
|
||||
def test_published_plan_and_response_bound_to_same_plant(self):
|
||||
p=run_once(self.s,AT);r=status(self.s,AID,AT)
|
||||
self.assertEqual(p['installationId'],AID);self.assertEqual(r['installationId'],AID)
|
||||
self.assertEqual(r['receiverProtocolVersion'],1);self.assertTrue(r['fresh'])
|
||||
self.assertFalse(r['liveEnabled']);self.assertFalse(p['liveEnabled'])
|
||||
self.assertEqual(p['inputRefs'],{'operation':'op1','forecast':'fc1','tariffs':'tar1'})
|
||||
def test_control_context_tracks_stable_policy_not_volatile_soc(self):
|
||||
op=inputs()[0];a=control_context(op);op['batteries'][0]['socPercent']=80.;op['gridW']=1000.
|
||||
self.assertEqual(a,control_context(op));op['batteries'][0]['minSocPercent']=15.
|
||||
self.assertNotEqual(a,control_context(op))
|
||||
def test_context_is_snapshot_not_reference_to_future_changes(self):
|
||||
op,fc,tar=inputs();p=provenance({'operation':op,'forecast':fc,'tariffs':tar})
|
||||
op['limits']['importW']=100.;self.assertEqual(p['controlContext']['limits']['importW'],10000.)
|
||||
def test_no_plan_for_another_installation(self):
|
||||
run_once(self.s,AT);r=status(self.s,'00000000-0000-4000-8000-000000000002',AT)
|
||||
self.assertIsNone(r['plan']);self.assertFalse(r['fresh'])
|
||||
def test_shadow_ack_is_not_execution(self):
|
||||
p=run_once(self.s,AT)
|
||||
self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,step=p['points'][0]['time'],status='shadow_seen')
|
||||
self.assertEqual(status(self.s,AID,AT)['acknowledgement']['status'],'shadow_seen')
|
||||
with self.assertRaises(ValueError):self.s.acknowledge(AID,p['planId'],p['configRevision'],AT,status='applied')
|
||||
def test_new_configuration_invalidates_fresh_flag(self):
|
||||
run_once(self.s,AT);self.s.save_settings(AID,{'family':'23'},0,AT+timedelta(seconds=1))
|
||||
self.assertFalse(status(self.s,AID,AT+timedelta(seconds=2))['fresh'])
|
||||
def test_rearm_default_matches_php_contract(self):
|
||||
op=inputs()[0];op['batteries'][0]['rearmSocPercent']=None
|
||||
self.assertEqual(control_context(op)['batteries']['b']['rearmSocPercent'],10.)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,50 @@
|
||||
import ast
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
import unittest
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
spec=importlib.util.spec_from_file_location('release_preflight_test',ROOT/'release_preflight.py')
|
||||
preflight=importlib.util.module_from_spec(spec);spec.loader.exec_module(preflight)
|
||||
|
||||
class ReleasePreflightTest(unittest.TestCase):
|
||||
def test_preflight_never_restarts_existing_services(self):
|
||||
for name,args,cwd,timeout in preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021'):
|
||||
self.assertNotIn('up',args);self.assertNotIn('restart',args);self.assertNotIn('--with-bridges',args)
|
||||
self.assertGreater(timeout,0)
|
||||
first=preflight.command_list('e3a08f9e-af12-4695-99bd-8b51c0520021')[0][1]
|
||||
self.assertIn('--test-only',first)
|
||||
def test_php_runs_without_network_or_writeable_root(self):
|
||||
args=next(args for name,args,cwd,timeout in preflight.command_list('p') if name=='php83_offline')
|
||||
self.assertIn('--read-only',args);self.assertEqual(args[args.index('--network')+1],'none')
|
||||
self.assertEqual(args[args.index('--user')+1],'1000:1000')
|
||||
def test_native_bundle_requires_same_development_source(self):
|
||||
with TemporaryDirectory() as d:
|
||||
root=Path(d)/'root';repo=Path(d)/'repo';out=root/'acceptance/php-src'
|
||||
out.mkdir(parents=True);repo.mkdir()
|
||||
source=b'<?php // test';(out/'a.php').write_bytes(source);(repo/'a.php').write_bytes(source)
|
||||
(out/'SOURCE_MANIFEST.json').write_text(json.dumps({'a.php':hashlib.sha256(source).hexdigest()}))
|
||||
with patch.object(preflight,'ROOT',root),patch.object(preflight,'REPO',repo):
|
||||
preflight.verify_native_bundle()
|
||||
(repo/'a.php').write_text('<?php // changed')
|
||||
with self.assertRaises(ValueError):preflight.verify_native_bundle()
|
||||
def test_native_probe_has_no_training_or_publish_calls(self):
|
||||
tree=ast.parse((ROOT/'acceptance/read_native_forecasts.py').read_text())
|
||||
calls={n.func.attr for n in ast.walk(tree) if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute)}
|
||||
self.assertFalse(calls & {'train','run_forecast','run_forecast_isolated','get_configs','publish_forecasts','write_api'})
|
||||
text=(ROOT/'acceptance/read_native_forecasts.py').read_text()
|
||||
self.assertIn('?mode=ro',text);self.assertNotIn('ALTER TABLE',text)
|
||||
def test_gui_labels_estimates_and_incomplete_features(self):
|
||||
text=(ROOT/'gui/netplan-v4.js').read_text()
|
||||
self.assertIn('SCHATTENBETRIEB',text)
|
||||
self.assertIn('measurementPolicy',text)
|
||||
self.assertIn('wartet weiterhin auf vergleichbare Kosten-Replays',text)
|
||||
self.assertIn('forecastSource',text)
|
||||
self.assertIn('trainingCadence',text)
|
||||
self.assertIn('Restmonatsbewertung ist keine bereits bezahlte Peakfreigabe',text)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,184 @@
|
||||
import copy
|
||||
from dataclasses import replace
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from netplan_v4.domain import Battery,Family,Limits,Price,QuarterPast,Step,default_registry,split_base_load,priced_prefix
|
||||
from netplan_v4.optimizer import optimize
|
||||
from netplan_v4.selection import ReplayScore,choose_family,training_due,promote_candidate
|
||||
from netplan_v4.store import PlannerStore
|
||||
from netplan_v4.service import ingest,latest,run_once,status,create_app
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
AT=datetime(2026,10,1,12,0,tzinfo=timezone.utc)
|
||||
AID='e3a08f9e-af12-4695-99bd-8b51c0520021'
|
||||
TOKEN='synthetic-test-token-not-real-credentials'
|
||||
|
||||
def seq(load,pv=None,buy=None,sell=None,at=AT):
|
||||
n=len(load);pv=pv or [0.]*n;buy=buy or [.3]*n;sell=sell or [.1]*n
|
||||
return [Step(at+timedelta(minutes=5*i),load[i],pv[i],Price(buy[i]),Price(sell[i])) for i in range(n)]
|
||||
def batt(**kw):
|
||||
values=dict(asset_id='b',capacity_kwh=10.,soc_percent=20.,min_soc_percent=10.,max_soc_percent=90.,max_charge_w=5000.,max_discharge_w=4000.,measured_at=AT,grid_charging=True)
|
||||
values.update(kw);return Battery(**values)
|
||||
def solve(steps,assets=None,**kw):
|
||||
kw.setdefault('observed_peaks',{s.start.astimezone(__import__('zoneinfo').ZoneInfo('Europe/Zurich')).strftime('%Y-%m'):5. for s in steps})
|
||||
kw.setdefault('peak_prices',{m:0. for m in kw['observed_peaks']})
|
||||
return optimize(steps,assets or [],at=steps[0].start,**kw)
|
||||
|
||||
def inputs(at=AT):
|
||||
env=lambda identifier:{'version':1,'eventId':identifier,'observedAt':at.isoformat()}
|
||||
op={**env('op1'),'gridW':-3000.,'meteringBoundary':'common_pcc','batteries':[{'id':'b','capacityKwh':10.,'socPercent':20.,'minSocPercent':10.,'maxSocPercent':90.,'maxChargeW':5000.,'maxDischargeW':4000.,'measuredAt':at.isoformat(),'gridCharging':True}],
|
||||
'limits':{'importW':10000.,'exportW':None,'managerMonthLimitsW':{}},'measuredPeaks':{'2026-10':{'kw':5.,'source':'meter_month_register'}},'quarterPast':None}
|
||||
points=[{'time':(at+timedelta(minutes=5*i)).isoformat(),'loadW':2000.,'pvW':5000. if i<3 else 0.} for i in range(6)]
|
||||
forecast={**env('fc1'),'families':{k:{'loadBasis':'base_load','trainedUntil':(at-timedelta(days=1)).isoformat(),'points':copy.deepcopy(points)} for k in ('3','13','23')}}
|
||||
tariff={**env('tar1'),'import':{'mode':'static','tariffId':'buy','staticChfKwh':.3},'export':{'mode':'static','tariffId':'sell','staticChfKwh':.1},'peakChfKwMonth':5.}
|
||||
return op,forecast,tariff
|
||||
|
||||
class OptimizerTest(unittest.TestCase):
|
||||
def test_grid_price_arbitrage(self):
|
||||
p=solve(seq([1000.]*6,buy=[.01]*3+[.7]*3),[batt()]);self.assertTrue(p['executable']);self.assertGreater(p['points'][0]['batteryTargetW'],0);self.assertLess(p['points'][-1]['batteryTargetW'],0)
|
||||
def test_no_future_energy(self):
|
||||
p=solve(seq([3000.]*3+[0.]*3,pv=[0.]*3+[5000.]*3),[batt(soc_percent=10.,grid_charging=False)])
|
||||
self.assertTrue(all(x['batteryTargetW']>=-1e-5 for x in p['points'][:3]))
|
||||
def test_roundtrip_90_percent(self):
|
||||
p=solve(seq([0.]*3+[5000.]*3,pv=[5000.]*3+[0.]*3,buy=[.3]*6,sell=[0.]*6),[batt(soc_percent=10.,grid_charging=False,max_discharge_w=5000.)])
|
||||
c=sum(max(0,x['batteryTargetW']) for x in p['points']);d=sum(max(0,-x['batteryTargetW']) for x in p['points']);self.assertAlmostEqual(d/c,.9,places=5)
|
||||
def test_nulleinspeisung(self):
|
||||
p=solve(seq([0.]*3,pv=[10000.]*3),limits=Limits(export_w=0.));self.assertTrue(p['executable']);self.assertTrue(all(abs(x['gridTargetW'])<1e-4 for x in p['points']))
|
||||
def test_nulltariffs(self):
|
||||
p=solve(seq([1000.]*3,buy=[0.]*3,sell=[0.]*3));self.assertEqual(p['cashCostChf'],0.)
|
||||
def test_negative_prices(self):
|
||||
p=solve(seq([1000.]*3,buy=[-.1]*3));self.assertLess(p['energyCostChf'],0.)
|
||||
def test_asymmetric_limits_and_balance(self):
|
||||
steps=seq([0.]*3+[3000.]*3,pv=[5000.]*3+[0.]*3)
|
||||
p=solve(steps,[batt(max_charge_w=700.,max_discharge_w=300.)]);self.assertTrue(p['executable'])
|
||||
for s,x in zip(steps,p['points']):
|
||||
self.assertLessEqual(x['batteryTargetW'],700.001);self.assertGreaterEqual(x['batteryTargetW'],-300.001)
|
||||
self.assertAlmostEqual(x['gridTargetW'],s.residual_w+x['batteryTargetW']+x['pvCurtailmentW'],places=4)
|
||||
self.assertTrue(9.999<=x['socEndPercent']['b']<=90.001)
|
||||
def test_stale_soc_rejected(self):self.assertFalse(solve(seq([0.]*3),[batt(measured_at=AT-timedelta(hours=1))])['executable'])
|
||||
def test_soc_not_fabricated(self):self.assertFalse(solve(seq([0.]*3),[batt(soc_percent=5.)])['executable'])
|
||||
def test_peak_increment_only_and_baseline(self):
|
||||
p=solve(seq([2000.]*3),observed_peaks={'2026-10':1.5},peak_prices={'2026-10':5.})
|
||||
self.assertAlmostEqual(p['additionalPeakCostChf'],2.5);self.assertAlmostEqual(p['baselineCashCostChf'],2.65)
|
||||
def test_peak_blocks_unprofitable_charging(self):
|
||||
p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.)],observed_peaks={'2026-10':1.},peak_prices={'2026-10':20.})
|
||||
self.assertLessEqual(p['plannedPeaksKw']['2026-10'],1.0001)
|
||||
def test_month_switch(self):
|
||||
at=datetime(2026,9,30,21,45,tzinfo=timezone.utc)
|
||||
p=solve(seq([2000.]*6,at=at),observed_peaks={'2026-09':2.,'2026-10':.5},peak_prices={'2026-09':5.,'2026-10':5.})
|
||||
self.assertAlmostEqual(p['additionalPeakCostChf'],7.5)
|
||||
def test_manager_cap_is_not_measured_peak(self):
|
||||
p=solve(seq([1000.]*3),limits=Limits(manager_month_limits_w={10:1500.}),observed_peaks={'2026-10':0.},peak_prices={'2026-10':5.})
|
||||
self.assertEqual(p['measuredPeaksKw']['2026-10'],0.);self.assertAlmostEqual(p['additionalPeakCostChf'],5.)
|
||||
def test_infeasible_not_fake_residual(self):
|
||||
p=solve(seq([10000.]*3),limits=Limits(import_w=1000.));self.assertFalse(p['executable']);self.assertEqual(p['points'],[])
|
||||
def test_elapsed_quarter_required(self):
|
||||
s=seq([1000.],at=AT+timedelta(minutes=10));self.assertFalse(solve(s)['executable'])
|
||||
p=solve(s,quarter_history={AT:QuarterPast(.5,600)},peak_prices={'2026-10':5.},observed_peaks={'2026-10':0.})
|
||||
self.assertAlmostEqual(p['plannedPeaksKw']['2026-10'],7/3)
|
||||
def test_partial_first_step(self):
|
||||
s=seq([1000.]*3);s[0]=replace(s[0],start=AT+timedelta(seconds=10),seconds=290)
|
||||
p=solve(s,quarter_history={AT:QuarterPast(.02,10)});self.assertTrue(p['executable']);self.assertEqual(p['validFrom'],s[0].start.isoformat())
|
||||
def test_terminal_guard(self):
|
||||
p=solve(seq([0.]*3,sell=[10.]*3),[batt(soc_percent=50.,grid_charging=False)]);self.assertGreaterEqual(p['points'][-1]['socEndPercent']['b'],49.999)
|
||||
def test_grid_charging_opt_in(self):
|
||||
p=solve(seq([1000.]*6,buy=[.01]*3+[.5]*3),[batt(soc_percent=10.,grid_charging=False)]);self.assertTrue(all(x['batteryTargetW']<=1e-5 for x in p['points']))
|
||||
|
||||
class StoreAndSelectionTest(unittest.TestCase):
|
||||
def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'))
|
||||
def tearDown(self):self.store.close();self.temp.cleanup()
|
||||
def test_settings_revision_and_shadow_only(self):
|
||||
s=self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(s['revision'],1)
|
||||
with self.assertRaises(ValueError):self.store.save_settings(AID,{'family':'3'},0,AT)
|
||||
with self.assertRaises(ValueError):self.store.save_settings(AID,{'runMode':'live'},1,AT)
|
||||
def test_request_during_calculation_survives(self):
|
||||
self.store.request(AID,'manual',AT);claim=self.store.claim(AT);self.store.request(AID,'prices_changed',AT)
|
||||
p=solve(seq([0.]*3),config_revision=0)
|
||||
with self.assertRaises(ValueError):self.store.publish_shadow(AID,p,0,AT,claim['sequence'],claim['lease_token'])
|
||||
self.store.finish(claim);self.assertIsNotNone(self.store.claim(AT))
|
||||
def test_month_peak_persistent_and_monotonic(self):
|
||||
self.store.initialize_peak(AID,'2026-10',20.,'meter_month_register',AT)
|
||||
with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',15.,'meter_month_register',AT)
|
||||
self.store.close();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'));self.assertEqual(self.store.peaks(AID)['2026-10'],20.)
|
||||
def test_cap_is_not_valid_peak_source(self):
|
||||
with self.assertRaises(ValueError):self.store.initialize_peak(AID,'2026-10',20.,'configured_limit',AT)
|
||||
def test_meter_peak_requires_complete_quarter(self):
|
||||
self.store.initialize_peak(AID,'2026-10',0.,'verified_new_month',AT)
|
||||
for i in range(3):r=self.store.record_import_interval(AID,AT+timedelta(minutes=5*i),.5,AT+timedelta(hours=1))
|
||||
self.assertEqual(r['quarterPeakKw'],6.);self.assertEqual(self.store.peaks(AID)['2026-10'],6.)
|
||||
def test_shadow_ack_cannot_claim_applied(self):
|
||||
p=solve(seq([0.]*3),config_revision=0);self.store.publish_shadow(AID,p,0,AT)
|
||||
with self.assertRaises(ValueError):self.store.acknowledge(AID,p['planId'],0,AT,status='applied')
|
||||
self.store.acknowledge(AID,p['planId'],0,AT,status='shadow_seen')
|
||||
def test_family_registry_extensible(self):
|
||||
r=default_registry();r.register(Family('42','p42','l42','g42','Future'));self.assertEqual(r.get('42').pv,'p42')
|
||||
def test_sdl_removed_once(self):
|
||||
self.assertEqual(split_base_load(15000,[3000],2000),10000);self.assertEqual(split_base_load(13000,[3000],2000,True),10000)
|
||||
with self.assertRaises(ValueError):split_base_load(100,[1000],0)
|
||||
def test_auto_uses_comparable_cost_and_margin(self):
|
||||
scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT,c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]]
|
||||
self.assertEqual(choose_family('auto','3',scores,now=AT)['family'],'23');self.assertEqual(choose_family('auto','3',scores,now=AT,margin_chf=6.)['family'],'3')
|
||||
def test_auto_no_future_outcomes(self):
|
||||
scores=[ReplayScore(k,'same',AT-timedelta(days=14),AT,AT-timedelta(days=14),AT+timedelta(hours=1),c,1.,14) for k,c in [('3',10.),('13',9.5),('23',5.)]]
|
||||
self.assertEqual(choose_family('auto','3',scores,now=AT)['mode'],'collecting')
|
||||
def test_training_schedule_and_validation(self):
|
||||
self.assertTrue(training_due(AT-timedelta(days=1),AT));self.assertFalse(training_due(AT-timedelta(days=1),AT,'weekly'))
|
||||
self.assertFalse(promote_candidate(active_cost=10,candidate_cost=1,valid_coverage=True,no_data_leakage=False,constraints_passed=True))
|
||||
|
||||
class ServiceIntegrationTest(unittest.TestCase):
|
||||
def setUp(self):self.temp=tempfile.TemporaryDirectory();self.store=PlannerStore(str(Path(self.temp.name)/'planner.sqlite'))
|
||||
def tearDown(self):self.store.close();self.temp.cleanup()
|
||||
def load(self,values=None,now=AT):
|
||||
values=values or inputs()
|
||||
for kind,value in zip(('operation','forecast','tariffs'),values):ingest(self.store,AID,kind,value,now)
|
||||
def test_pipeline_shadow_plan(self):
|
||||
self.load();p=run_once(self.store,AT);self.assertTrue(p['executable']);self.assertEqual(p['runMode'],'shadow');self.assertTrue(status(self.store,AID,AT)['fresh']);self.assertEqual(self.store.current(AID)['planId'],p['planId'])
|
||||
def test_exact_model_selection(self):
|
||||
self.load();self.store.save_settings(AID,{'family':'23'},0,AT);self.assertEqual(run_once(self.store,AT)['sourceFamily'],'23')
|
||||
def test_one_tick_not_every_poll(self):
|
||||
self.load();run_once(self.store,AT);self.assertEqual(run_once(self.store,AT)['status'],'idle')
|
||||
def test_stale_operation_visible(self):
|
||||
self.load();p=run_once(self.store,AT+timedelta(minutes=5));self.assertEqual(p['status'],'awaiting_inputs');self.assertIsNone(self.store.current(AID))
|
||||
def test_missing_peak_not_invented(self):
|
||||
op,fc,tar=inputs();op['measuredPeaks']={};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs')
|
||||
def test_static_zero_roundtrip(self):
|
||||
op,fc,tar=inputs();tar['import']['staticChfKwh']=0.;tar['export']['staticChfKwh']=0.;tar['peakChfKwMonth']=0.;self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['cashCostChf'],0.)
|
||||
def test_immutable_id_conflict(self):
|
||||
op,fc,tar=inputs();self.load((op,fc,tar));self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'duplicate');op['gridW']=123.
|
||||
with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT)
|
||||
def test_older_input_never_overwrites_current(self):
|
||||
op,fc,tar=inputs();self.load((op,fc,tar));op['observedAt']=(AT-timedelta(seconds=1)).isoformat();op['eventId']='old'
|
||||
self.assertEqual(ingest(self.store,AID,'operation',op,AT)['status'],'archived_older');self.assertEqual(latest(self.store,AID,'operation')['eventId'],'op1')
|
||||
def test_unpublished_dynamic_tail_not_executed(self):
|
||||
op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar));self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs')
|
||||
def test_published_negative_price_and_horizon(self):
|
||||
op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar))
|
||||
price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(minutes=15)).isoformat(),'value':-2.,'unit':'Rp/kWh','observedAt':AT.isoformat(),'sourceKind':'published_interval'}]}
|
||||
ingest(self.store,AID,'prices',price,AT);p=run_once(self.store,AT);self.assertEqual(len(p['points']),3);self.assertEqual(p['points'][0]['importPriceChfKwh'],-.02)
|
||||
def test_carried_price_does_not_extend_horizon(self):
|
||||
op,fc,tar=inputs();tar['import']={'mode':'dynamic','tariffId':'buy'};self.load((op,fc,tar))
|
||||
price={'version':1,'eventId':'price1','observedAt':AT.isoformat(),'periods':[{'tariffId':'buy','side':'import','start':AT.isoformat(),'end':(AT+timedelta(hours=48)).isoformat(),'value':.1,'unit':'CHF/kWh','observedAt':AT.isoformat(),'sourceKind':'carried_forward'}]}
|
||||
ingest(self.store,AID,'prices',price,AT);self.assertEqual(run_once(self.store,AT)['status'],'awaiting_inputs')
|
||||
def test_aggregate_load_warning(self):
|
||||
op,fc,tar=inputs();fc['families']['3']['loadBasis']='house_total';self.load((op,fc,tar));self.assertTrue(run_once(self.store,AT)['warnings'])
|
||||
def test_extra_credentials_not_stored(self):
|
||||
op,fc,tar=inputs();op['password']='do-not-store'
|
||||
with self.assertRaises(ValueError):ingest(self.store,AID,'operation',op,AT)
|
||||
def test_http_auth_allowlist_and_revision_conflict(self):
|
||||
app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False)
|
||||
with TestClient(app) as client:
|
||||
path=f'/internal/v2/prognosis/{AID}/planner';headers={'X-Enelix-Service-Token':TOKEN}
|
||||
self.assertEqual(client.get(path).status_code,401);self.assertEqual(client.get(path,headers=headers).status_code,200)
|
||||
other='30509683-4569-49e4-848f-4905e4cc813a';self.assertEqual(client.get(path.replace(AID,other),headers=headers).status_code,403)
|
||||
self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'23'}}).status_code,200)
|
||||
self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':0,'changes':{'family':'3'}}).status_code,409)
|
||||
self.assertEqual(client.put(path+'/settings',headers=headers,json={'expectedRevision':1,'changes':{'runMode':'live'}}).status_code,400)
|
||||
def test_body_limit_and_no_legacy_db(self):
|
||||
with self.assertRaises(ValueError):create_app(str(Path(self.temp.name)/'users.db'),TOKEN,[AID])
|
||||
app=create_app(str(Path(self.temp.name)/'http.sqlite'),TOKEN,[AID],start_worker=False)
|
||||
with TestClient(app) as client:
|
||||
r=client.post(f'/internal/v2/prognosis/{AID}/planner/inputs/forecast',json={'x':'a'*2000001});self.assertEqual(r.status_code,413)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user