Related
My program is optimizing the charging and decharging of a home battery to minimize the cost of electricity at the end of the year. In this case there also is a PV, which means that sometimes you're injecting electricity into the grid and receive money. The net offtake is the result of the usage of the home and the PV installation. So these are the possible situations:
Net offtake > 0 => usage home > PV => discharge from battery or take from grid
Net offtake < 0 => usage home < PV => charge battery or injection into grid
The electricity usage of homes is measured each 15 minutes, so I have 96 measurement point in 1 day. I want to optimilize the charging and decharging of the battery for 2 days, so that day 1 takes the usage of day 2 into account.
I wrote a controller that reads the data and gives each time the input values for 2 days to the optimization. With a rolling principle, it goes to the next 2 days and so on. Below you can see the code from my controller.
from gekko import GEKKO
from simulationModel2_2d_1 import getSimulation2
from exportModel2 import exportToExcelModel2
import numpy as np
#import matplotlib.pyplot as plt
import pandas as pd
import time
import math
# ------------------------ Import and read input data ------------------------
file = r'Data Sim 2.xlsx'
data = pd.read_excel(file, sheet_name='Input', na_values='NaN')
dataRead = pd.DataFrame(data, columns= ['Timestep','Verbruik woning (kWh)','Netto afname (kWh)','Prijs afname (€/kWh)',
'Prijs injectie (€/kWh)','Capaciteit batterij (kW)',
'Capaciteit batterij (kWh)','Rendement (%)',
'Verbruikersprofiel','Capaciteit PV (kWp)','Aantal dagen'])
timestep = dataRead['Timestep'].to_numpy()
usage_home = dataRead['Verbruik woning (kWh)'].to_numpy()
net_offtake = dataRead['Netto afname (kWh)'].to_numpy()
price_offtake = dataRead['Prijs afname (€/kWh)'].to_numpy()
price_injection = dataRead['Prijs injectie (€/kWh)'].to_numpy()
cap_batt_kW = dataRead['Capaciteit batterij (kW)'].iloc[0]
cap_batt_kWh = dataRead['Capaciteit batterij (kWh)'].iloc[0]
efficiency = dataRead['Rendement (%)'].iloc[0]
usersprofile = dataRead['Verbruikersprofiel'].iloc[0]
days = dataRead['Aantal dagen'].iloc[0]
pv = dataRead['Capaciteit PV (kWp)'].iloc[0]
# ------------- Optimization model & Rolling principle (2 days) --------------
# Initialise model
m = GEKKO()
# Output data
ts = []
charging = [] # Amount to charge/decharge batterij
e_batt = [] # Amount of energy in the battery
usage_net = [] # Usage after home, battery and pv
p_paid = [] # Price paid for energy of 15min
# Energy in battery to pass
energy = 0
# Iterate each day for one year
for d in range(int(days)-1):
d1_timestep = []
d1_net_offtake = []
d1_price_offtake = []
d1_price_injection = []
d2_timestep = []
d2_net_offtake = []
d2_price_offtake = []
d2_price_injection = []
# Iterate timesteps
for i in range(96):
d1_timestep.append(timestep[d*96+i])
d2_timestep.append(timestep[d*96+i+96])
d1_net_offtake.append(net_offtake[d*96+i])
d2_net_offtake.append(net_offtake[d*96+i+96])
d1_price_offtake.append(price_offtake[d*96+i])
d2_price_offtake.append(price_offtake[d*96+i+96])
d1_price_injection.append(price_injection[d*96+i])
d2_price_injection.append(price_injection[d*96+i+96])
# Input data simulation of 2 days
ts_temp = np.concatenate((d1_timestep, d2_timestep))
net_offtake_temp = np.concatenate((d1_net_offtake, d2_net_offtake))
price_offtake_temp = np.concatenate((d1_price_offtake, d2_price_offtake))
price_injection_temp = np.concatenate((d1_price_injection, d2_price_injection))
if(d == 7):
print(ts_temp)
print(energy)
# Simulatie uitvoeren
charging_temp, e_batt_temp, usage_net_temp, p_paid_temp, energy_temp = getSimulation2(ts_temp, net_offtake_temp, price_offtake_temp, price_injection_temp, cap_batt_kW, cap_batt_kWh, efficiency, energy, pv)
# Take over output first day, unless last 2 days
energy = energy_temp
if(d == (days-2)):
for t in range(1,len(ts_temp)):
ts.append(ts_temp[t])
charging.append(charging_temp[t])
e_batt.append(e_batt_temp[t])
usage_net.append(usage_net_temp[t])
p_paid.append(p_paid_temp[t])
elif(d == 0):
for t in range(int(len(ts_temp)/2)+1):
ts.append(ts_temp[t])
charging.append(charging_temp[t])
e_batt.append(e_batt_temp[t])
usage_net.append(usage_net_temp[t])
p_paid.append(p_paid_temp[t])
else:
for t in range(1,int(len(ts_temp)/2)+1):
ts.append(ts_temp[t])
charging.append(charging_temp[t])
e_batt.append(e_batt_temp[t])
usage_net.append(usage_net_temp[t])
p_paid.append(p_paid_temp[t])
print('Simulation day '+str(d+1)+' complete.')
# ------------------------ Export output data to Excel -----------------------
a = exportToExcelModel2(ts, usage_home, net_offtake, price_offtake, price_injection, charging, e_batt, usage_net, p_paid, cap_batt_kW, cap_batt_kWh, efficiency, usersprofile, pv)
print(a)
The optimization with Gekko happens in the following code:
from gekko import GEKKO
def getSimulation2(timestep, net_offtake, price_offtake, price_injection,
cap_batt_kW, cap_batt_kWh, efficiency, start_energy, pv):
# ---------------------------- Optimization model ----------------------------
# Initialise model
m = GEKKO(remote = False)
# Global options
m.options.SOLVER = 1
m.options.IMODE = 6
# Constants
speed_charging = cap_batt_kW/4
m.time = timestep
max_cap_batt = m.Const(value = cap_batt_kWh)
min_cap_batt = m.Const(value = 0)
max_charge = m.Const(value = speed_charging) # max battery can charge in 15min
max_decharge = m.Const(value = -speed_charging) # max battery can decharge in 15min
# Parameters
usage_home = m.Param(net_offtake)
price_offtake = m.Param(price_offtake)
price_injection = m.Param(price_injection)
# Variables
e_batt = m.Var(value=start_energy, lb = min_cap_batt, ub = max_cap_batt) # energy in battery
price_paid = m.Var() # price paid each 15min
charging = m.Var(lb = max_decharge, ub = max_charge) # amount charge/decharge each 15min
usage_net = m.Var(lb=min_cap_batt)
# Equations
m.Equation(e_batt==(m.integral(charging)+start_energy)*efficiency)
m.Equation(-charging <= e_batt)
m.Equation(usage_net==usage_home + charging)
price = m.Intermediate(m.if2(usage_net*1e6, price_injection, price_offtake))
price_paid = m.Intermediate(usage_net * price / 100)
# Objective
m.Minimize(price_paid)
# Solve problem
m.options.COLDSTART=2
m.solve()
m.options.TIME_SHIFT=0
m.options.COLDSTART=0
m.solve()
# Energy to pass
energy_left = e_batt[95]
#m.cleanup()
return charging, e_batt, usage_net, price_paid, energy_left
The data you need for input can be found in this Excel document:
https://docs.google.com/spreadsheets/d/1S40Ut9-eN_PrftPCNPoWl8WDDQtu54f0/edit?usp=sharing&ouid=104786612700360067470&rtpof=true&sd=true
With this code, it always ends at day 17 with the "Solution Not Found" Error.
I already tried extending the default iteration limit to 500 but it didn't work.
I also tried with other solvers but also no improvement.
By presolving with COLDSTART it already reached day 17, without this it ends at day 8.
I solved the days were my optimization ends individually and then the solution was always found immediately with the same code.
Someone who can explain this to me and maybe find a solution? Thanks in advance!
This is kind of big to troubleshoot, but here are some general ideas that might help. This assumes, as you said, that the model solves fine for day 1-2, and day 3-4, and day 5-6, etc. And that those results pass inspection (aka the basic model is working as you say).
Then something is (obviously) amiss around day 17. Some things to look at and try:
Start the model at day 16-17, see if it works in isolation
gather your results as you go and do a time series plot of the key variables, maybe one of them is on an obvious bad trend towards a limit causing an infeasibility... Perhaps the e_batt variable is slowly declining because not enough PV energy is available and hits minimum on Day 17
Radically change the upper/lower bounds on your variables to test them to see if they might be involved in the infeasibility (assuming there is one)
Make a different (fake) dataset where the solution is assured and kind of obvious... all constants or a pattern that you know will produce some known result and test the model outputs against it.
It might also be useful to pursue the tips in this excellent post on troubleshooting gekko, and edit your question with the results of that effort.
edit: couple ideas from your comment...
You didn't say what the result was from troubleshooting and whether the error is infeasible or max iterations, or ???. But...
If the model seems to crunch after about 15 days, I'm betting that it is becoming infeasible. Did you plot the battery level over course of days?
Also, I'm suspicious of your equation for the e_batt level... Why are you multiplying the prior battery state by the efficiency factor? That seems incorrect. That is charge that is already in the battery. Odds are you are (incorrectly) hitting the battery charge level every day with the efficiency tax and that the max charge level isn't sufficient to keep up with demand.
In addition to tips above try:
fix your efficiency formula to not multiply the efficiency times the previous state
change the efficiency to 100%
make the upper limit on charge huge
As an aside: I don't really see the connection to PV energy available here. What you are basically modeling is some "mystery battery" that you can charge and discharge anytime you want. I would get this debugged first and then look at the energy available by time of day...you aren't going to be generating charge at midnight. :).
I want to make Action Scheduler to run in every 30 seconds, but it only has 1 minute as the lowest possible selection for its interval.
How can I customize/change it so that it can run in seconds?
This is what I end up doing, which I need to test it for a few days before confirming if it gets the job done.
Inherited ir.cron model to add seconds selection.
Added seconds to _intervalTypes to make it compatible with seconds selection.
Copied the original _process_job to replace in here so that the modified _intervalTypes with seconds property exists for the method.
Installed this customized module.
Here is the code.
from odoo import models, fields, api
from datetime import datetime
from dateutil.relativedelta import relativedelta
import pytz
_intervalTypes = {
'days': lambda interval: relativedelta(days=interval),
'hours': lambda interval: relativedelta(hours=interval),
'weeks': lambda interval: relativedelta(days=7*interval),
'months': lambda interval: relativedelta(months=interval),
'minutes': lambda interval: relativedelta(minutes=interval),
'seconds': lambda interval: relativedelta(seconds=interval),
}
class IrCronInherit(models.Model):
_inherit = 'ir.cron'
interval_type = fields.Selection([
('seconds', 'Seconds'),
('minutes', 'Minutes'),
('hours', 'Hours'),
('days', 'Days'),
('weeks', 'Weeks'),
('months', 'Months')], string='Interval Unit', default='months')
#classmethod
def _process_job(cls, job_cr, job, cron_cr):
""" Run a given job taking care of the repetition.
:param job_cr: cursor to use to execute the job, safe to commit/rollback
:param job: job to be run (as a dictionary).
:param cron_cr: cursor holding lock on the cron job row, to use to update the next exec date,
must not be committed/rolled back!
"""
with api.Environment.manage():
try:
cron = api.Environment(job_cr, job['user_id'], {
'lastcall': fields.Datetime.from_string(job['lastcall'])
})[cls._name]
# Use the user's timezone to compare and compute datetimes,
# otherwise unexpected results may appear. For instance, adding
# 1 month in UTC to July 1st at midnight in GMT+2 gives July 30
# instead of August 1st!
now = fields.Datetime.context_timestamp(cron, datetime.now())
nextcall = fields.Datetime.context_timestamp(cron, fields.Datetime.from_string(job['nextcall']))
numbercall = job['numbercall']
ok = False
while nextcall < now and numbercall:
if numbercall > 0:
numbercall -= 1
if not ok or job['doall']:
cron._callback(job['cron_name'], job['ir_actions_server_id'], job['id'])
if numbercall:
nextcall += _intervalTypes[job['interval_type']](job['interval_number'])
ok = True
addsql = ''
if not numbercall:
addsql = ', active=False'
cron_cr.execute("UPDATE ir_cron SET nextcall=%s, numbercall=%s, lastcall=%s"+addsql+" WHERE id=%s",(
fields.Datetime.to_string(nextcall.astimezone(pytz.UTC)),
numbercall,
fields.Datetime.to_string(now.astimezone(pytz.UTC)),
job['id']
))
cron.flush()
cron.invalidate_cache()
finally:
job_cr.commit()
cron_cr.commit()
Right now, it runs under a minute but not constantly at 30 seconds when I actually set it to 30 seconds, but it will do for now.
I want to be able to visualise and analyse data for a specific period of time. The data is in different files so i read each file and appended them into an array. The data I have is in Julien date format so i used a function to convert it to datetime so that i am able to plot the data. However the function is now giving an error:
File "D:\DATA\TEC DATA\2018\amco\RES\CMN_append_df_stack_merge.py", line 162, in jd_to_date
jd = jd + 0.5
TypeError: can only concatenate str (not "float") to str
It was not giving this error before however the plot had lines crisscrossing everywhere which i suspect is because of the datetime not being formatted correctly.
Here is the code i used
*- coding: utf-8 -*-
"""
Created on Mon Mar 9 17:51:41 2020
#author: user
"""
import glob as glob
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from datetime import datetime
"""
Functions for converting dates to/from JD and MJD. Assumes dates are historical
dates, including the transition from the Julian calendar to the Gregorian
calendar in 1582. No support for proleptic Gregorian/Julian calendars.
:Author: Matt Davis
:Website: http://github.com/jiffyclub
"""
import math
import datetime as dt
# Note: The Python datetime module assumes an infinitely valid Gregorian calendar.
# The Gregorian calendar took effect after 10-15-1582 and the dates 10-05 through
# 10-14-1582 never occurred. Python datetime objects will produce incorrect
# time deltas if one date is from before 10-15-1582.
def mjd_to_jd(mjd):
"""
Convert Modified Julian Day to Julian Day.
Parameters
----------
mjd : float
Modified Julian Day
Returns
-------
jd : float
Julian Day
"""
return mjd + 2400000.5
def jd_to_mjd(jd):
"""
Convert Julian Day to Modified Julian Day
Parameters
----------
jd : float
Julian Day
Returns
-------
mjd : float
Modified Julian Day
"""
return jd - 2400000.5
def date_to_jd(year,month,day):
"""
Convert a date to Julian Day.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
Parameters
----------
year : int
Year as integer. Years preceding 1 A.D. should be 0 or negative.
The year before 1 A.D. is 0, 10 B.C. is year -9.
month : int
Month as integer, Jan = 1, Feb. = 2, etc.
day : float
Day, may contain fractional part.
Returns
-------
jd : float
Julian Day
Examples
--------
Convert 6 a.m., February 17, 1985 to Julian Day
>>> date_to_jd(1985,2,17.25)
2446113.75
"""
if month == 1 or month == 2:
yearp = year - 1
monthp = month + 12
else:
yearp = year
monthp = month
# this checks where we are in relation to October 15, 1582, the beginning
# of the Gregorian calendar.
if ((year < 1582) or
(year == 1582 and month < 10) or
(year == 1582 and month == 10 and day < 15)):
# before start of Gregorian calendar
B = 0
else:
# after start of Gregorian calendar
A = math.trunc(yearp / 100.)
B = 2 - A + math.trunc(A / 4.)
if yearp < 0:
C = math.trunc((365.25 * yearp) - 0.75)
else:
C = math.trunc(365.25 * yearp)
D = math.trunc(30.6001 * (monthp + 1))
jd = B + C + D + day + 1720994.5
return jd
def jd_to_date(jd):
"""
Convert Julian Day to date.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
Parameters
----------
jd : float
Julian Day
Returns
-------
year : int
Year as integer. Years preceding 1 A.D. should be 0 or negative.
The year before 1 A.D. is 0, 10 B.C. is year -9.
month : int
Month as integer, Jan = 1, Feb. = 2, etc.
day : float
Day, may contain fractional part.
Examples
--------
Convert Julian Day 2446113.75 to year, month, and day.
>>> jd_to_date(2446113.75)
(1985, 2, 17.25)
"""
jd = jd + 0.5
F, I = math.modf(jd)
I = int(I)
A = math.trunc((I - 1867216.25)/36524.25)
if I > 2299160:
B = I + 1 + A - math.trunc(A / 4.)
else:
B = I
C = B + 1524
D = math.trunc((C - 122.1) / 365.25)
E = math.trunc(365.25 * D)
G = math.trunc((C - E) / 30.6001)
day = C - E + F - math.trunc(30.6001 * G)
if G < 13.5:
month = G - 1
else:
month = G - 13
if month > 2.5:
year = D - 4716
else:
year = D - 4715
return year, month, day
def hmsm_to_days(hour=0,min=0,sec=0,micro=0):
"""
Convert hours, minutes, seconds, and microseconds to fractional days.
Parameters
----------
hour : int, optional
Hour number. Defaults to 0.
min : int, optional
Minute number. Defaults to 0.
sec : int, optional
Second number. Defaults to 0.
micro : int, optional
Microsecond number. Defaults to 0.
Returns
-------
days : float
Fractional days.
Examples
--------
>>> hmsm_to_days(hour=6)
0.25
"""
days = sec + (micro / 1.e6)
days = min + (days / 60.)
days = hour + (days / 60.)
return days / 24.
def days_to_hmsm(days):
"""
Convert fractional days to hours, minutes, seconds, and microseconds.
Precision beyond microseconds is rounded to the nearest microsecond.
Parameters
----------
days : float
A fractional number of days. Must be less than 1.
Returns
-------
hour : int
Hour number.
min : int
Minute number.
sec : int
Second number.
micro : int
Microsecond number.
Raises
------
ValueError
If `days` is >= 1.
Examples
--------
>>> days_to_hmsm(0.1)
(2, 24, 0, 0)
"""
hours = days * 24.
hours, hour = math.modf(hours)
mins = hours * 60.
mins, min = math.modf(mins)
secs = mins * 60.
secs, sec = math.modf(secs)
micro = round(secs * 1.e6)
return int(hour), int(min), int(sec), int(micro)
def datetime_to_jd(date):
"""
Convert a `datetime.datetime` object to Julian Day.
Parameters
----------
date : `datetime.datetime` instance
Returns
-------
jd : float
Julian day.
Examples
--------
>>> d = datetime.datetime(1985,2,17,6)
>>> d
datetime.datetime(1985, 2, 17, 6, 0)
>>> jdutil.datetime_to_jd(d)
2446113.75
"""
days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond)
return date_to_jd(date.year,date.month,days)
def jd_to_datetime(jd):
"""
Convert a Julian Day to an `jdutil.datetime` object.
Parameters
----------
jd : float
Julian day.
Returns
-------
dt : `jdutil.datetime` object
`jdutil.datetime` equivalent of Julian day.
--------------------------------------------------------------------------------------------------------------------------------------------------------
--------
>>> jd_to_datetime(2446113.75)
datetime(1985, 2, 17, 6, 0)
"""
year, month, day = jd_to_date(jd)
frac_days,day = math.modf(day)
day = int(day)
hour,min,sec,micro = days_to_hmsm(frac_days)
return datetime(year,month,day,hour,min,sec,micro)
'---------------------------------------------------------------------------------------------------------------------------------------------------------'
#"""
#JULIEN DAY CONVERTOR
def timedelta_to_days(td):
"""
Convert a `datetime.timedelta` object to a total number of days.
Parameters
----------
td : `datetime.timedelta` instance
Returns
-------
days : float
Total number of days in the `datetime.timedelta` object.
Examples
--------
>>> td = datetime.timedelta(4.5)
>>> td
datetime.timedelta(4, 43200)
>>> timedelta_to_days(td)
4.5
"""
seconds_in_day = 24. * 3600.
days = td.days + (td.seconds + (td.microseconds * 10.e6)) / seconds_in_day
return days
"""
'Start of main Code---------------------------------------------------------------------------------------------------'
"""
#########
files = glob.glob('*.Cmn')
files = files[:3]
np_array_values = []
for file in files:
df = pd.read_csv(file, delimiter="\t", sep ='\t', skiprows = 5, names = ["Jdate" ,'Time' ,'PRN' ,'Az','Ele','Lat', 'Lon' ,'Stec', 'Vtec', 'S4'])
#df.set_index('Jdatet')
#Appending Each data frame to the Giant Array np-array_values then Stacking them
np_array_values.append (df)
merge_values = np.vstack(np_array_values)
#CONVERTING ARRAY BACK TO DATA FRAME
TEC_data =pd.DataFrame(merge_values)
Vtec = TEC_data.loc[:,7]
jdate = TEC_data.loc[:,0]
Time = TEC_data.loc[:,1]
month_name = file[:-7]
STATION_NAME = file[:4]
df.replace('-99.000', np.nan)
pos= np.where(jdate)[0]
pos1=np.where(Vtec[pos]<-20.0)[0]
Vtec[pos1]='nan'
fulldate = []
for i in jdate:
a = jd_to_datetime(i)
fulldate.append(a)
#plt.show()
plt.plot(fulldate, Vtec)
#plt.xlim(0, 24)
#plt.xticks(np.arange(0, 26, 2))
plt.ylabel('TECU')
plt.xlabel('Date')
#plt.grid(axis='both')
plt.title("Station : " + STATION_NAME.upper())t**
Here is a sample of the data. The columns are:
nknown_station, "E:\GPS DATA\rbmc2\2018\amco\amco3630.18o"
-4.87199 294.66602 75.87480
Jdatet Time PRN Az Ele Lat Lon Stec Vtec S4
2458481.500000 -24.000000 1 198.34 23.37 -10.70 292.70 18.62 13.70 -99.000
2458481.500347 0.008333 1 198.21 23.53 -10.67 292.73 18.65 13.74 -99.000
2458481.500694 0.016667 1 198.07 23.69 -10.64 292.75 18.76 13.84 -99.000
2458481.501042 0.025000 1 197.94 23.85 -10.61 292.78 18.68 13.83 -99.000
2458481.501389 0.033333 1 197.81 24.01 -10.58 292.80 18.60 13.83 -99.000
2458481.501736 0.041667 1 197.68 24.17 -10.55 292.83 18.53 13.83 -99.000
2458481.502083 0.050000 1 197.54 24.33 -10.52 292.85 18.53 13.86 -99.000
2458481.502431 0.058333 1 197.41 24.49 -10.49 292.88 18.51 13.88 -99.000
2458481.502778 0.066667 1 197.28 24.65 -10.46 292.90 18.66 14.00 -99.000
2458481.503125 0.075000 1 197.15 24.81 -10.43 292.92 18.78 14.09 0.238
2458481.503472 0.083333 1 197.01 24.98 -10.40 292.95 18.55 14.01 -99.000
2458481.503819 0.091667 1 196.88 25.14 -10.37 292.97 18.39 13.96 -99.000
2458481.504167 0.100000 1 196.75 25.30 -10.34 292.99 18.33 13.97 -99.000
2458481.504514 0.108333 1 196.62 25.47 -10.31 293.02 18.20 13.94 -99.000
2458481.504861 0.116667 1 196.49 25.63 -10.28 293.04 17.61 13.67 -99.000
2458481.505208 0.125000 1 196.36 25.80 -10.25 293.06 16.74 13.25 -99.000
2458481.505556 0.133333 1 196.23 25.96 -10.22 293.09 16.06 12.92 -99.000
2458481.505903 0.141667 1 196.10 26.13 -10.19 293.11 15.46 12.64 -99.000
2458481.506250 0.150000 1 195.97 26.30 -10.16 293.13 14.77 12.31 -99.000
2458481.506597 0.158333 1 195.84 26.46 -10.13 293.15 14.42 12.15 0.127
It seems like your variable jd is a string instead of a float. Can you show some sample data? This would probably explain why this is happening.
Some additional points about your code:
While looping over all files, you want to create one dataframe. This can be done easier:
dfMerged = pd.DataFrame()
for file in files:
dfTemp = pd.read_csv(...)
dfMerged = dfMerged.append(dfTemp, ignore_index=True)
This would lead to the additional benefit that you can use your column headings like they are supposed to be used, e.g., Vtec = dfMerged['Vtec'].
Your month_name and STATION_NAME are overwritten in each iteration of the loop. If they are different for each file, you want to save them as a list or as a separate column in the dataframe, e.g., dfTemp['Station'] = STATION_NAME. Keep in mind to do this before you merge the dataframes.
Also, the replace can be substituted by adding '-99.000' to the na_values argument in read_csv.
Selecting your data can be done with pandas. dfMerged.loc[(dfMerged['Jdate'].notna()) & (dfMerged['Vtec'] < -20), 'Vtec'] = pd.NA.
I added the list containing the datetime object as an index to the Dataframe.
Apparently to recognise the dates you need to use plt.plot_date instead of using the usual plt.plot. Datetime index will take care of the rest.
I need to write a function below that can compute the moving average of time series using a sliding window over an array. This function should take an array of date strings (say arr_date), an array of numbers (say arr_record), and a sliding window (default value 50). It should:
Return a list of dictionaries for all windows.
Each dictionary should include the date, average value, min, max, standard deviation at each window.
Able to handle missing data in time series by replacing missing data with the most recent available data.
(b) Download SPY daily data (Dec. 31, 2017 to Dec. 31, 2018) from Yahoo! as your test data in a .csv file. Read reading .csv file example and write a test programming for calling your function.
Does anyone have any thoughts? Extremely new to python and struggling.
So something following this logic should probably be a good starting point. Hope this is a helpful start, and welcome to the cs community.
def sliding_window( dates, numbers, sliding_window_value):
# list of dictionaries
return_dicts =[{}]
# if window size is greater than length of dates, there's only one window
if sliding_window_value >= len(dates):
return_dicts += [create_window(dates, numbers)]
return return_dicts
# gather all our windows into one list
for i in range (0, len(dates) - sliding_window_value ):
# get our window subsets
dates_subset = dates[i:(sliding_window_value+1)]
numbers_subset = numbers[i:(sliding_window_value+1)]
# get our window stats dictionary
window_stats = create_window(dates_subset,numbers_subset)
# add these stats to our return list
return_dicts += [window_stats]
return return_dicts
def create_window(dates_subset, numbers_subset):
window_min = 1000000 # some high minimum to start
window_max = -1000000 # some low maximuim to start
window_total = 0
for i in range ( 0, len(dates_subset)):
# calculate total
window_total += numbers_subset[i]
# calculate max
if numbers_subset[i] > window_max:
window_max = numbers_subset[i]
# calculate min
if numbers_subset[i] < window_min:
window_min = numbers_subset[i]
# other calculations....
return_dict = {
"min" : window_min,
"max" : window_max,
"average" : window_total / len(dates_subset),
# other calculations....
}
return return_dict
Good luck bud, the work is worth it.
Currently I am getting time with the keyword Get time epoch , which is returning time in seconds. But I need time in milliseconds , So that I can get time span for a particular event.
or is there any other way to get the time span for a particular event or a testsceanrio?
Check the new test library DateTime, which contains keyword Get Current Date, which also returns milliseconds. It also has keyword Subtract Dates to calculate difference between two timestamps.
One of the more powerful features of robot is that you can directly call python code from a test script using the Evaluate keyword. For example, you can call the time.time() function, and do a little math:
*** Test cases
| Example getting the time in milliseconds
| | ${ms}= | Evaluate | int(round(time.time() * 1000)) | time
| | log | time in ms: ${ms}
Note that even though time.time returns a floating point value, not all systems will return a value more precise than one second.
Using the DateTime library, as suggested by janne:
*** Settings ***
Library DateTime
*** Test Cases ***
Performance Test
${timeAvgMs} = Test wall clock time 100 MyKeywordToPerformanceTest and optional arguments
Should be true ${timeAvgMs} < 50
*** Keywords ***
MyKeywordToPerformanceTest
# Do something here
Test wall clock time
[Arguments] ${iterations} #{commandAndArgs}
${timeBefore} = Get Current Date
:FOR ${it} IN RANGE ${iterations}
\ #{commandAndArgs}
${timeAfter} = Get Current Date
${timeTotalMs} = Subtract Date From Date ${timeAfter} ${timeBefore} result_format=number
${timeAvgMs} = Evaluate int(${timeTotalMs} / ${iterations} * 1000)
Return from keyword ${timeAvgMs}
In the report, for each suite, test and keyword, you have the information about start, end and length with millisecond details. Something like:
Start / End / Elapsed: 20140602 10:57:15.948 / 20140602 10:57:16.985 / 00:00:01.037
I don't see a way to do it using Builtin, look:
def get_time(format='timestamp', time_=None):
"""Return the given or current time in requested format.
If time is not given, current time is used. How time is returned is
is deternined based on the given 'format' string as follows. Note that all
checks are case insensitive.
- If 'format' contains word 'epoch' the time is returned in seconds after
the unix epoch.
- If 'format' contains any of the words 'year', 'month', 'day', 'hour',
'min' or 'sec' only selected parts are returned. The order of the returned
parts is always the one in previous sentence and order of words in
'format' is not significant. Parts are returned as zero padded strings
(e.g. May -> '05').
- Otherwise (and by default) the time is returned as a timestamp string in
format '2006-02-24 15:08:31'
"""
time_ = int(time_ or time.time())
format = format.lower()
# 1) Return time in seconds since epoc
if 'epoch' in format:
return time_
timetuple = time.localtime(time_)
parts = []
for i, match in enumerate('year month day hour min sec'.split()):
if match in format:
parts.append('%.2d' % timetuple[i])
# 2) Return time as timestamp
if not parts:
return format_time(timetuple, daysep='-')
# Return requested parts of the time
elif len(parts) == 1:
return parts[0]
else:
return parts
You have to write your own module, you need something like:
import time
def get_time_in_millies():
time_millies = lambda: int(round(time.time() * 1000))
return time_millies
Then import this library in Ride for the suite and you can use the method name like keyword, in my case it would be Get Time In Millies. More info here.