change file structure
This commit is contained in:
dhafli01
114
Influxclient.py
114
Influxclient.py
@@ -1,114 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Mon Oct 17 13:45:38 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
import math
|
|
||||||
import requests, json
|
|
||||||
from datetime import datetime, date, time
|
|
||||||
|
|
||||||
# Convert a date and time to to complex numbers
|
|
||||||
# [year_real, year_imag, day_real, day_imag]
|
|
||||||
def Convert_Date(atime):
|
|
||||||
|
|
||||||
y_r = math.sin(atime.timetuple().tm_yday/365*2*math.pi)
|
|
||||||
y_i = math.cos(atime.timetuple().tm_yday/365*2*math.pi)
|
|
||||||
d_r = math.sin((atime.hour*3600+atime.minute*60+atime.second)/((24*3600))*2*math.pi)
|
|
||||||
d_i = math.cos((atime.hour*3600+atime.minute*60+atime.second)/((24*3600))*2*math.pi)
|
|
||||||
return [y_r, y_i, d_r, d_i]
|
|
||||||
|
|
||||||
|
|
||||||
# Return a list of datas of a specified measurement of an influxDB
|
|
||||||
def Get_Value(client, bucket, atime, measurement, datapoint):
|
|
||||||
|
|
||||||
query_api = client.query_api()
|
|
||||||
query = 'from(bucket:"'+bucket+'")\
|
|
||||||
|> range(start: -'+atime+')\
|
|
||||||
|> filter(fn:(r) => r._field == "'+datapoint+'" )\
|
|
||||||
|> filter(fn:(r) => r._measurement == "'+measurement+'" )'
|
|
||||||
result = query_api.query(org=client.org, query=query)
|
|
||||||
thelist = []
|
|
||||||
for table in result:
|
|
||||||
for record in table.records:
|
|
||||||
thelist.append((record.get_value()))
|
|
||||||
return thelist
|
|
||||||
|
|
||||||
|
|
||||||
# Return the time of datas of a specified measurement of an influxDB
|
|
||||||
def Get_Time(client, bucket, atime, measurement, datapoint):
|
|
||||||
|
|
||||||
query_api = client.query_api()
|
|
||||||
query = 'from(bucket:"'+bucket+'")\
|
|
||||||
|> range(start: -'+atime+')\
|
|
||||||
|> filter(fn:(r) => r._field == "'+datapoint+'" )\
|
|
||||||
|> filter(fn:(r) => r._measurement == "'+measurement+'" )'
|
|
||||||
result = query_api.query(org=client.org, query=query)
|
|
||||||
thelist = []
|
|
||||||
for table in result:
|
|
||||||
for record in table.records:
|
|
||||||
thelist.append((record.get_time()))
|
|
||||||
|
|
||||||
|
|
||||||
a=[]
|
|
||||||
b=[]
|
|
||||||
c=[]
|
|
||||||
d=[]
|
|
||||||
|
|
||||||
for i in thelist:
|
|
||||||
x = Convert_Date(i)
|
|
||||||
a.append(x[0])
|
|
||||||
b.append(x[1])
|
|
||||||
c.append(x[2])
|
|
||||||
d.append(x[3])
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
return [a,b,c,d,thelist]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Need duration in hours, interval in minutes
|
|
||||||
# Retruns array with complex time, timestamp and prediction
|
|
||||||
# Works only for max 3 days due api returns only values until then
|
|
||||||
def GetPrediction(duration, intervall, datapoint):
|
|
||||||
|
|
||||||
xfile = pd.read_xml('config.xml')
|
|
||||||
|
|
||||||
apikey = (xfile.loc[xfile['category'] == 'influx'].weatherapitoken[0])
|
|
||||||
place = (xfile.loc[xfile['category'] == 'influx'].place[0])
|
|
||||||
dats = requests.get("https://api.weatherapi.com/v1/forecast.json?key="+apikey+"&q="+place+"&days=3&aqi=yes&alerts=no")
|
|
||||||
dats=json.loads(dats.content)
|
|
||||||
|
|
||||||
a=[]
|
|
||||||
b=[]
|
|
||||||
c=[]
|
|
||||||
d=[]
|
|
||||||
e=[]
|
|
||||||
f=[]
|
|
||||||
now = datetime.now()
|
|
||||||
daycounter = 0
|
|
||||||
lasthour=now.hour
|
|
||||||
for i in range(0, int(duration*60/intervall)):
|
|
||||||
|
|
||||||
now = datetime.timestamp(now)+(intervall*60)
|
|
||||||
now = datetime.fromtimestamp(now)
|
|
||||||
|
|
||||||
x = Convert_Date(now)
|
|
||||||
a.append(x[0])
|
|
||||||
b.append(x[1])
|
|
||||||
c.append(x[2])
|
|
||||||
d.append(x[3])
|
|
||||||
e.append(now)
|
|
||||||
bc = dats["forecast"]["forecastday"][daycounter]["hour"][now.hour][datapoint]
|
|
||||||
|
|
||||||
f.append(bc)
|
|
||||||
if lasthour>now.hour:
|
|
||||||
daycounter+=1
|
|
||||||
|
|
||||||
lasthour=now.hour
|
|
||||||
|
|
||||||
return[a,b,c,d,e,f]
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Fri Dec 2 07:46:09 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
"""
|
|
||||||
import pandas as pd
|
|
||||||
import Influxclient as I
|
|
||||||
import influxdb_client
|
|
||||||
|
|
||||||
|
|
||||||
# Returns values of an influxDB as they where specified in the config.xml
|
|
||||||
# as a Pandas Dataframe
|
|
||||||
def Get_Df():
|
|
||||||
|
|
||||||
|
|
||||||
xfile = pd.read_xml('config.xml')
|
|
||||||
org = (xfile.loc[xfile['category'] == 'influx'].organisation[0])
|
|
||||||
token = (xfile.loc[xfile['category'] == 'influx'].influxtoken[0])
|
|
||||||
url = (xfile.loc[xfile['category'] == 'influx'].url[0])
|
|
||||||
|
|
||||||
client = influxdb_client.InfluxDBClient(
|
|
||||||
url=url,
|
|
||||||
token=token,
|
|
||||||
org=org
|
|
||||||
)
|
|
||||||
|
|
||||||
bucket = (xfile.loc[xfile['category'] == 'influx'].bucket[0])
|
|
||||||
measurement = (xfile.loc[xfile['category'] == 'influx'].measurement[0])
|
|
||||||
|
|
||||||
interval = (xfile.loc[xfile['category'] == 'influx'].interval[0])
|
|
||||||
valuessince = (xfile.loc[xfile['category'] == 'influx'].time[0])
|
|
||||||
|
|
||||||
Input = xfile.loc[xfile['category'] == 'feature'].f[1]
|
|
||||||
#numberofFeature=(xfile.loc[xfile['category']=='feature'].features[1])
|
|
||||||
|
|
||||||
feature = Input.split()
|
|
||||||
pred = xfile.loc[xfile['category'] == 'pred'].p1[2]
|
|
||||||
|
|
||||||
|
|
||||||
data = {
|
|
||||||
|
|
||||||
"y_r": I.Get_Time(client, bucket, valuessince, measurement, feature[0])[0],
|
|
||||||
"y_i": I.Get_Time(client, bucket, valuessince, measurement, feature[0])[1],
|
|
||||||
"d_r": I.Get_Time(client, bucket, valuessince, measurement, feature[0])[2],
|
|
||||||
"d_i": I.Get_Time(client, bucket, valuessince, measurement, feature[0])[3],
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
for i in range(len(feature)):
|
|
||||||
newd = {(feature[i]): I.Get_Value(client, bucket,
|
|
||||||
valuessince, measurement, feature[i])}
|
|
||||||
data.update(newd)
|
|
||||||
|
|
||||||
newd = {(pred): I.Get_Value(client, bucket,
|
|
||||||
valuessince, measurement, pred)}
|
|
||||||
data.update(newd)
|
|
||||||
|
|
||||||
return pd.DataFrame(data)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Shifts the target variable and adds [shift_number] colums with the shiftet
|
|
||||||
# values, then return the new Dataframe
|
|
||||||
def ShiftTarget(df, shift_number, targetvariable):
|
|
||||||
dfziel = df[[targetvariable]].copy()
|
|
||||||
df = df.rename(columns={targetvariable:(targetvariable+str(0))})
|
|
||||||
dfziel = dfziel.rename(columns={targetvariable:(targetvariable+str(0))})
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
for i in range(shift_number):
|
|
||||||
dfziel = dfziel.rename(columns={(targetvariable+str(i)):targetvariable+str(i+1)})
|
|
||||||
dfziel = dfziel.shift(1)
|
|
||||||
df = pd.concat([df, dfziel], axis=1)
|
|
||||||
|
|
||||||
return df
|
|
||||||
30
config.xml
30
config.xml
@@ -1,30 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<config>
|
|
||||||
|
|
||||||
<variable category="influx">
|
|
||||||
<organisation>enter influx organisation name</organisation>
|
|
||||||
<url>enter influx url</url>
|
|
||||||
<bucket>enter influx bucket name</bucket>
|
|
||||||
<measurement>enter influx measurement name</measurement>
|
|
||||||
<influxtoken>enter a influx-read token for specific bucket</influxtoken>
|
|
||||||
<weatherapitoken>enter a weatherapitoken for weatherapi.com </weatherapitoken>
|
|
||||||
<place>enter place where weatherapi should predict</place>
|
|
||||||
<interval>5M</interval>
|
|
||||||
<time>10d</time>
|
|
||||||
</variable>
|
|
||||||
|
|
||||||
<!-- Enter features here as a array, separated with space -->
|
|
||||||
<!-- E. g. <f>feautre1 feautre2 feature3</f> -->
|
|
||||||
<variable category="feature">
|
|
||||||
<f>P_AT_5M_0 </f>
|
|
||||||
</variable>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Enter target variable here as a array, separated with space -->
|
|
||||||
<!-- Attention! Currently just one target varibale is supported! -->
|
|
||||||
<variable category="pred">
|
|
||||||
<p1>W_AT_5M_0</p1>
|
|
||||||
</variable>
|
|
||||||
|
|
||||||
|
|
||||||
</config>
|
|
||||||
100
example.py
100
example.py
@@ -1,100 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Fri Dec 9 08:51:35 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
|
|
||||||
This file shows an example of how to use the Influxclient and Influxdataframe.
|
|
||||||
The example try to fit an prediction of a temperature of a specific measure
|
|
||||||
with time and a given prediction of the temperature as features.
|
|
||||||
|
|
||||||
There is no data cleaning ect. in this example, its just to show how the lib
|
|
||||||
works...
|
|
||||||
|
|
||||||
Be aware that you need to add a influx- and weatherapi-token and aviable datas
|
|
||||||
to the config file. Otherwise the example, as the lib to, wont work.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Fri Dec 2 08:51:19 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
"""
|
|
||||||
import pandas as pd
|
|
||||||
import Influxdataframe as Idf
|
|
||||||
import Influxclient as I
|
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
from sklearn.preprocessing import MinMaxScaler
|
|
||||||
from sklearn.metrics import r2_score
|
|
||||||
from sklearn.neural_network import MLPRegressor
|
|
||||||
|
|
||||||
|
|
||||||
# Get a dataframe from the influxDB with the specified values in the config.xml
|
|
||||||
df = Idf.Get_Df()
|
|
||||||
|
|
||||||
|
|
||||||
# Do a train test split and scale the datas
|
|
||||||
train_df, test_df = train_test_split(df, test_size=0.3, random_state=42)
|
|
||||||
|
|
||||||
scaler = MinMaxScaler()
|
|
||||||
|
|
||||||
train_df = pd.DataFrame(scaler.fit_transform(
|
|
||||||
train_df), columns=train_df.columns, index=train_df.index)
|
|
||||||
test_df = pd.DataFrame(scaler.transform(
|
|
||||||
test_df), columns=test_df.columns, index=test_df.index)
|
|
||||||
|
|
||||||
X = test_df[["y_r", "y_i", "d_r", "d_i", "P_AT_5M_0"]]
|
|
||||||
y = test_df[["W_AT_5M_0"]]
|
|
||||||
|
|
||||||
X_train, X_test, y_train, y_test = train_test_split(
|
|
||||||
X, y, test_size=0.2, random_state=42)
|
|
||||||
|
|
||||||
|
|
||||||
# Train data and print r2
|
|
||||||
dt = MLPRegressor(activation='relu')
|
|
||||||
dt.fit(X_train, y_train)
|
|
||||||
print(r2_score(y_test, dt.predict(X_test)))
|
|
||||||
|
|
||||||
|
|
||||||
# Get a df for the current prediciton with values from weather api
|
|
||||||
pred = I.GetPrediction(48, 5, "temp_c")
|
|
||||||
data = {
|
|
||||||
|
|
||||||
"y_r": pred[0],
|
|
||||||
"y_i": pred[1],
|
|
||||||
"d_r": pred[2],
|
|
||||||
"d_i": pred[3],
|
|
||||||
"P_AT_5M_0": pred[5],
|
|
||||||
"W_AT_5M_0": pred[5],
|
|
||||||
}
|
|
||||||
preddf = pd.DataFrame((data))
|
|
||||||
|
|
||||||
|
|
||||||
preddf = pd.DataFrame(scaler.transform(
|
|
||||||
preddf), columns=preddf.columns, index=preddf.index)
|
|
||||||
|
|
||||||
preddf = preddf[["y_r", "y_i", "d_r", "d_i", "P_AT_5M_0"]]
|
|
||||||
|
|
||||||
|
|
||||||
# Predict temprature for the values returned by the api
|
|
||||||
ATpred = dt.predict(preddf)
|
|
||||||
Prog24 = pd.DataFrame(ATpred)
|
|
||||||
|
|
||||||
df_all_cols = pd.concat([preddf, Prog24], axis=1)
|
|
||||||
|
|
||||||
Prog24 = scaler.inverse_transform(df_all_cols)
|
|
||||||
|
|
||||||
# Rewrite datas to get it into a df
|
|
||||||
data = {
|
|
||||||
|
|
||||||
"W_AT_5M_0": Prog24[:, 5].tolist(),
|
|
||||||
"Timestamp": pred[4],
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
prognose = pd.DataFrame(data)
|
|
||||||
prognose.plot(x='Timestamp', y='W_AT_5M_0')
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Fri Dec 9 09:31:31 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
|
|
||||||
|
|
||||||
This file shows an example of how to use the Influxclient and Influxdataframe.
|
|
||||||
The example try to fit an prediction of a temperature of a specific measure
|
|
||||||
with time and a given prediction of the temperature as features with time series
|
|
||||||
forecasting depending ond them.
|
|
||||||
|
|
||||||
There is no data cleaning ect. in this example, its just to show how the lib
|
|
||||||
works...
|
|
||||||
|
|
||||||
Be aware that you need to add a influx- and weatherapi-token and aviable datas
|
|
||||||
to the config file. Otherwise the example, as the lib to, wont work.
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
|
||||||
import pandas as pd
|
|
||||||
import Influxdataframe as Idf
|
|
||||||
import Influxclient as I
|
|
||||||
import seaborn as sns
|
|
||||||
from sklearn.model_selection import train_test_split
|
|
||||||
from sklearn.preprocessing import MinMaxScaler
|
|
||||||
from sklearn.metrics import r2_score
|
|
||||||
from sklearn.neural_network import MLPRegressor
|
|
||||||
|
|
||||||
|
|
||||||
target_variable = "W_AT_5M_0"
|
|
||||||
interval = 60
|
|
||||||
valuesince = 10
|
|
||||||
prediction_hours = 24
|
|
||||||
|
|
||||||
shift_number=4
|
|
||||||
|
|
||||||
df = Idf.Get_Df()
|
|
||||||
|
|
||||||
df = df[::12]
|
|
||||||
|
|
||||||
pred =I.GetPrediction(prediction_hours, interval, "temp_c")
|
|
||||||
|
|
||||||
|
|
||||||
data = {
|
|
||||||
|
|
||||||
"y_r": pred[0],
|
|
||||||
"y_i": pred[1],
|
|
||||||
"d_r": pred[2],
|
|
||||||
"d_i": pred[3],
|
|
||||||
|
|
||||||
"P_AT_5M_0": pred[5],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
preddf = pd.DataFrame(data)
|
|
||||||
|
|
||||||
df = pd.concat([df, preddf], axis=0).reset_index(drop=True)
|
|
||||||
|
|
||||||
|
|
||||||
df = Idf.ShiftTarget(df, shift_number, target_variable)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
df_predict = df.dropna()
|
|
||||||
|
|
||||||
train_df, test_df = train_test_split(df_predict, test_size=0.3, random_state=42)
|
|
||||||
|
|
||||||
scaler = MinMaxScaler()
|
|
||||||
|
|
||||||
train_df = pd.DataFrame(scaler.fit_transform(
|
|
||||||
train_df), columns=train_df.columns, index=train_df.index)
|
|
||||||
test_df = pd.DataFrame(scaler.transform(
|
|
||||||
test_df), columns=test_df.columns, index=test_df.index)
|
|
||||||
|
|
||||||
y = train_df[[target_variable+str(0)]]
|
|
||||||
|
|
||||||
X = train_df
|
|
||||||
X.pop(target_variable+str(0))
|
|
||||||
|
|
||||||
X_train, X_test, y_train, y_test = train_test_split(
|
|
||||||
X, y, test_size=0.2, random_state=42)
|
|
||||||
|
|
||||||
dt = MLPRegressor(activation='relu')
|
|
||||||
dt.fit(X_train, y_train)
|
|
||||||
|
|
||||||
print(r2_score(y_test, dt.predict(X_test)))
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
for i in range(int(prediction_hours*60/interval)):
|
|
||||||
preddf = pd.DataFrame(df.iloc[(int(df_predict.shape[0]+shift_number)-1+i)]).transpose()
|
|
||||||
preddf = pd.DataFrame(scaler.transform(
|
|
||||||
preddf), columns=preddf.columns, index=preddf.index)
|
|
||||||
preddf.pop(target_variable+str(0))
|
|
||||||
at = dt.predict(preddf)
|
|
||||||
preddf[target_variable+str(0)]=at
|
|
||||||
newval = pd.DataFrame(scaler.inverse_transform(preddf))
|
|
||||||
df.at[(int(df_predict.shape[0]+shift_number)-1+i), target_variable+str(0)] = newval.at[0, 5]
|
|
||||||
|
|
||||||
for j in range(shift_number):
|
|
||||||
df.at[int(df_predict.shape[0]+shift_number)+i, target_variable+str(j+1)] = df.at[int(df_predict.shape[0]+shift_number)-1+i, target_variable+str(j)]
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
sns.lineplot(df.index, df[target_variable+str(0)])
|
|
||||||
|
|
||||||
35
xmlconifg.py
35
xmlconifg.py
@@ -1,35 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Created on Sun Nov 13 19:33:47 2022
|
|
||||||
|
|
||||||
@author: Daniel Häfliger
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
def GetXmlConfig(xmlfile):
|
|
||||||
|
|
||||||
return
|
|
||||||
|
|
||||||
df = pd.read_xml('config.xml')
|
|
||||||
print(df.loc[df['category']=='influx'].organisation[0])
|
|
||||||
|
|
||||||
print(df.loc[df['category']=='influx'].url[0])
|
|
||||||
print(df.loc[df['category']=='influx'].bucket[0])
|
|
||||||
print(df.loc[df['category']=='influx'].measurement[0])
|
|
||||||
print(df.loc[df['category']=='influx'].Influxtoken[0])
|
|
||||||
print(df.loc[df['category']=='influx'].Weatherapitoken[0])
|
|
||||||
print(df.loc[df['category']=='influx'].place[0])
|
|
||||||
print(df.loc[df['category']=='influx'].interval[0])
|
|
||||||
print(df.loc[df['category']=='influx'].time[0])
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
a=df.loc[df['category']=='feature'].name
|
|
||||||
print(df.loc[df['category']=='feature'].name[1])
|
|
||||||
print(df.loc[df['category']=='pred'].name[2])
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
print(df.loc[df['category']=='pred'].name[2])
|
|
||||||
|
|
||||||
Reference in New Issue
Block a user