Financial time series: python Matplotlib "specgram" y-axis displaying Period instead of Frequency - matplotlib

python Matplotlib's "specgram" display of a heatmap showing frequency (y-axis) vs. time (x-axis) is useful for time series analysis, but I would like to have the y-axis displayed in terms of Period (= 1/frequency), rather than frequency. I am still asking if anyone has a complete working solution to achieve this?
The immediately following python code generates the author's original plot using "specgram" and (currently commented out) a comparison with the suggested solution that was offered using "mlab.specgram". This suggested solution succeeds with the easy conversion from frequency to period = 1/frequency, but does not generate a viable plot for the authors example.
from __future__ import division
from datetime import datetime
import numpy as np
from pandas import DataFrame, Series
import pandas.io.data as web
import pandas as pd
from pylab import plot,show,subplot,specgram
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
################################################
# obtain data:
ticker = "SPY"
source = "google"
start_date = datetime(1999,1,1)
end_date = datetime(2012,1,1)
qt = web.DataReader(ticker, source, start_date, end_date)
qtC = qt.Close
################################################
data = qtC
fs = 1 # 1 sample / day
nfft = 128
# display the time-series data
fig = plt.figure()
ax1 = fig.add_subplot(311)
ax1.plot(range(len(data)),data)
#----------------
# Original version
##################
# specgram (NOT mlab.specgram) --> gives direct plot, but in Frequency space (want plot in Period, not freq).
ax2 = fig.add_subplot(212)
spec, freq, t = specgram(data, NFFT=nfft, Fs=fs, noverlap=0)
#----------------
"""
# StackOverflow version (with minor changes to axis titles)
########################
# calcuate the spectrogram
spec, freq, t = mlab.specgram(data, NFFT=nfft, Fs=fs, noverlap=0)
# calculate the bin limits in time (x dir)
# note that there are n+1 fence posts
dt = t[1] - t[0]
t_edge = np.empty(len(t) + 1)
t_edge[:-1] = t - dt / 2.
# however, due to the way the spectrogram is calculates, the first and last bins
# a bit different:
t_edge[0] = 0
t_edge[-1] = t_edge[0] + len(data) / fs
# calculate the frequency bin limits:
df = freq[1] - freq[0]
freq_edge = np.empty(len(freq) + 1)
freq_edge[:-1] = freq - df / 2.
freq_edge[-1] = freq_edge[-2] + df
# calculate the period bin limits, omit the zero frequency bin
p_edge = 1. / freq_edge[1:]
# we'll plot both
ax2 = fig.add_subplot(312)
ax2.pcolormesh(t_edge, freq_edge, spec)
ax2.set_ylim(0, fs/2)
ax2.set_ylabel('freq.[day^-1]')
ax3 = fig.add_subplot(313)
# note that the period has to be inverted both in the vector and the spectrum,
# as pcolormesh wants to have a positive difference between samples
ax3.pcolormesh(t_edge, p_edge[::-1], spec[:0:-1])
#ax3.set_ylim(0, 100/fs)
ax3.set_ylim(0, nfft)
ax3.set_xlabel('t [days]')
ax3.set_ylabel('period [days]')
"""

If you are only asking how to display the spectrogram differently, then it is actually rather straightforward.
One thing to note is that there are two functions called specgram: matplotlib.pyplot.specgram and matplotlib.mlab.specgram. The difference between these two is that the former draws a spectrogram wheras the latter only calculates one (and that's what we want).
The only slightly tricky thing is to calculate the colour mesh rectangle edge positions. We get the following from the specgram:
t: centerpoints in time
freq: frequency centers of the bins
For the time dimension it is easy to calculate the bin limits by the centers:
t_edge[n] = t[0] + (n - .5) * dt, where dt is the time difference of two consecutive bins
It would be similarly simple for frequencies:
f_edge[n] = freq[0] + (n - .5) * df
but we want to use the period instead of frequency. This makes the first bin unusable, and we'll have to toss the DC component away.
A bit of code:
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import numpy as np
# create some data: (fs = sampling frequency)
fs = 2000.
ts = np.arange(10000) / fs
sig = np.sin(500 * np.pi * ts)
sig[5000:8000] += np.sin(200 * np.pi * (ts[5000:8000] + 0.0005 * np.random.random(3000)))
# calcuate the spectrogram
spec, freq, t = mlab.specgram(sig, Fs=fs)
# calculate the bin limits in time (x dir)
# note that there are n+1 fence posts
dt = t[1] - t[0]
t_edge = np.empty(len(t) + 1)
t_edge[:-1] = t - dt / 2.
# however, due to the way the spectrogram is calculates, the first and last bins
# a bit different:
t_edge[0] = 0
t_edge[-1] = t_edge[0] + len(sig) / fs
# calculate the frequency bin limits:
df = freq[1] - freq[0]
freq_edge = np.empty(len(freq) + 1)
freq_edge[:-1] = freq - df / 2.
freq_edge[-1] = freq_edge[-2] + df
# calculate the period bin limits, omit the zero frequency bin
p_edge = 1. / freq_edge[1:]
# we'll plot both
fig = plt.figure()
ax1 = fig.add_subplot(211)
ax1.pcolormesh(t_edge, freq_edge, spec)
ax1.set_ylim(0, fs/2)
ax1.set_ylabel('frequency [Hz]')
ax2 = fig.add_subplot(212)
# note that the period has to be inverted both in the vector and the spectrum,
# as pcolormesh wants to have a positive difference between samples
ax2.pcolormesh(t_edge, p_edge[::-1], spec[:0:-1])
ax2.set_ylim(0, 100/fs)
ax2.set_xlabel('t [s]')
ax2.set_ylabel('period [s]')
This gives:

Related

Pandas get max delta in a timeseries for a specified period

Given a dataframe with a non-regular time series as an index, I'd like to find the max delta between the values for a period of 10 secs. Here is some code that does the same thing:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(0)
xs = np.cumsum(np.random.rand(200))
# This function is to create a general situation where the max is not aways at the end or beginning
ys = xs**1.2 + 10 * np.sin(xs)
plt.plot(xs, ys, '+-')
threshold = 10
xs_thresh_ind = np.zeros_like(xs, dtype=int)
deltas = np.zeros_like(ys)
for i, x in enumerate(xs):
# Find indices that lie within the time threshold
period_end_ind = np.argmax(xs > x + threshold)
# Only operate when the window is wide enough (this can be treated differently)
if period_end_ind > 0:
xs_thresh_ind[i] = period_end_ind
# Find extrema in the period
period_min = np.min(ys[i:period_end_ind + 1])
period_max = np.max(ys[i:period_end_ind + 1])
deltas[i] = period_max - period_min
max_ind_low = np.argmax(deltas)
max_ind_high = xs_thresh_ind[max_ind_low]
max_delta = deltas[max_ind_low]
print(
'Max delta {:.2f} is in period x[{}]={:.2f},{:.2f} and x[{}]={:.2f},{:.2f}'
.format(max_delta, max_ind_low, xs[max_ind_low], ys[max_ind_low],
max_ind_high, xs[max_ind_high], ys[max_ind_high]))
df = pd.DataFrame(ys, index=xs)
OUTPUT:
Max delta 48.76 is in period x[167]=86.10,200.32 and x[189]=96.14,249.09
Is there an efficient pandaic way to achieve something similar?
Create a Series from ys values, indexed by xs - but convert xs to be actual timedelta elements, rather than the float equivalent.
ts = pd.Series(ys, index=pd.to_timedelta(xs, unit="s"))
We want to apply a leading, 10 second window in which we calculate the difference between max and min. Because we want it to be leading, we'll sort the Series in descending order and apply a trailing window.
deltas = ts.sort_index(ascending=False).rolling("10s").agg(lambda s: s.max() - s.min())
Find the maximum delta with deltas[deltas == deltas.max()], which gives
0 days 00:01:26.104797298 48.354851
meaning a delta of 48.35 was found in the interval [86.1, 96.1)

How to change a seaborn histogram plot to work for hours of the day?

I have a pandas dataframe with lots of time intervals of varying start times and lengths. I am interested in the distribution of start times over 24hours. I therefore have another column entitled Hour with just that in. I have plotted a histogram using seaborn to look at the distribution but obviously the x axis starts at 0 and runs to 24. I wonder if there is a way to change so it runs from 8 to 8 and loops over at 23 to 0 so it provides a better visualisation of my data from a time perspective. Thanks in advance.
sns.distplot(df2['Hour'], bins = 24, kde = False).set(xlim=(0,23))
If you want to have a custom order of x-values on your bar plot, I'd suggest using matplotlib directly and plot your histogram simply as a bar plot with width=1 to get rid of padding between bars.
import pandas as pd
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt
# prepare sample data
dates = pd.date_range(
start=datetime(2020, 1, 1),
end=datetime(2020, 1, 7),
freq="H")
random_dates = np.random.choice(dates, 1000)
df = pd.DataFrame(data={"date":random_dates})
df["hour"] = df["date"].dt.hour
# set your preferred order of hours
hour_order = [8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,0,1,2,3,4,5,6,7]
# calculate frequencies of each hour and sort them
plot_df = (
df["hour"]
.value_counts()
.rename_axis("hour", axis=0)
.reset_index(name="freq")
.set_index("hour")
.loc[hour_order]
.reset_index())
# day / night colour split
day_mask = ((8 <= plot_df["hour"]) & (plot_df["hour"] <= 20))
plot_df["color"] = np.where(day_mask, "skyblue", "midnightblue")
# actual plotting - note that you have to cast hours as strings
fig = plt.figure(figsize=(8,4))
ax = fig.add_subplot(111)
ax.bar(
x=plot_df["hour"].astype(str),
height=plot_df["freq"],
color=plot_df["color"], width=1)
ax.set_xlabel('Hour')
ax.set_ylabel('Frequency')
plt.show()

adding regression line in python using matplotlib

I have a question about drawing a regression line and determining the slope of that line. I am doing research for water heights of inland lakes in Tibet with the help of satellite date. I have the data for one year of one lake in this script.
However I want to determine the annual rise of the lake for as well the reference height as for the total beams. Is there some one that could help me?
This is the link towards the excel file: https://drive.google.com/file/d/12wD2ByQC6ObNCWq_yIhkXiNsV3KfDpit/view?usp=sharing
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
# Graph in chronological order
heights = pd.read_excel ('Qinghai_dates_heights.xlsx')
dates = (heights.loc[:,'Date'])
strong_beams = (heights.loc[:,'Strong total'])
weak_beams = (heights.loc[:,'Weak total'])
total_beams = (heights.loc[:,'Total'])
# setting the reference data from Hydrolabs
reference_dates = (heights.loc[:,'Date.1'])
reference_heights = (heights.loc[:,'Hydrolabs'])
# Set the locator
locator = mdates.MonthLocator() # every month
# Specify the format - %b gives us Jan, Feb...
fmt = mdates.DateFormatter('%b')
#plt.plot(dates,strong_beams, label='Strong Beams', marker="o")
#plt.plot(dates,weak_beams, label='Weak Beams', marker="o")
plt.plot(dates, total_beams, label='Total Beams', marker="o")
plt.plot(reference_dates, reference_heights, label='Reference height (Hydrolabs)', marker="o")
X = plt.gca().xaxis
X.set_major_locator(locator)
# Specify formatter
X.set_major_formatter(fmt)
plt.xlabel('Date [months]')
plt.ylabel('elevation [m]')
plt.title("Water-Height Qinghai from November 2018 - November 2019 ")
plt.legend()
plt.show()
Does this help ? I usually use sklearn for this.
import numpy as np
from matplotlib import pyplot as plt
from sklearn import linear_model, datasets
Generate a set of data
X = np.linspace(0, 10)
line_X = X[:, np.newaxis]
Y = X + 0.2*np.random.normal(size=50)
Choose your regression model (there are plenty more, depending on your needs)
lr = linear_model.LinearRegression()
Here you really do the fit
lr.fit(line_X, Y)
Here u extract the parameters, since you seems to need it ;)
slope = lr.coef_[0]
intercept = lr.intercept_
And then you plot
plt.plot(X, slope*X + intercept, ls='-', marker=' ')
plt.plot(X, Y)

fft find low frequencies in short time history

I have 1 time unit of signal history. My dominant frequency is 1/100 time units. When I use numpy's fft function, I am limited in resolution by the extent of the signal history. How can I increase the resolution of my frequency comb without corrupting my signal?
import numpy as np
import matplotlib.pyplot as plt
'''
I need to caputre a low-frequency oscillation with only 1 time unit of data.
So far, I have not been able to find a way to make the fft resolution < 1.
'''
timeResolution = 10000
mytimes = np.linspace(0, 1, timeResolution)
mypressures = np.sin(2 * np.pi * mytimes / 100)
fft = np.fft.fft(mypressures[:])
T = mytimes[1] - mytimes[0]
N = mypressures.size
# fft of original signal is limitted by the maximum time
f = np.linspace(0, 1 / T, N)
filteredidx = f > 0.001
freq = f[filteredidx][np.argmax(np.abs(fft[filteredidx][:N//2]))]
print('freq bin is is ', f[1] - f[0]) # 1.0
print('frequency is ', freq) # 1.0
print('(real frequency is 0.01)')
I thought that I could artificially increase the time history length (and thus decrease the width of the frequency comb) by pasting the signal end-to-end and doing the fft. That didn't work for me for some reason I don't understand:
import numpy as np
import matplotlib.pyplot as plt
timeResolution = 10000
mytimes = np.linspace(0, 1, timeResolution)
mypressures = np.sin(2 * np.pi * mytimes / 100)
# glue data to itself to make signal articicially longer
timesby = 1000
newtimes = np.concatenate([mytimes * ii for ii in range(1, timesby + 1)])
newpressures = np.concatenate([mypressures] * timesby)
fft = np.fft.fft(newpressures[:])
T = newtimes[1] - newtimes[0]
N = newpressures.size
# fft of original signal is limitted by the maximum time
f = np.linspace(0, 1 / T, N)
filteredidx = f > 0.001
freq = f[filteredidx][np.argmax(np.abs(fft[filteredidx][:N//2]))]
print('freq bin is is ', f[1] - f[0]) # 0.001
print('frequency is ', freq) # 1.0
print('(real frequency is 0.01)')
Your goal, recovering spectral information from a "too short" , i.e. << sample_rate / frequency_of_interest, window seems ambitious.
Even in the most simple case (clean sine wave, your example) the data look pretty much like a straight line (left panel below). Only after detrending we can see a tiny bit of curvature (right panel below, note the very small y-values) and that is all any hypothetical algorithm can go by. In particular, FT---as far as I can see---will not work.
If we are very lucky there is one way out: comparing derivatives.
If you have a sinosoidal signal with an offset---like f = c + sin(om * t´---then the 1st and 3rd derivatives will be om * cos(om * t) and -om^3 * cos(om * t)´´.
If the signal is simple and clean enough this together with robust numerical differentiation can be used to recover the frequency omega.
In the demo code below I use a SavGol filter to obtain the derivatives while getting rid of some high frequency noise (blue curve below) that had been added to the signal (orange curve). Other (better) methods of numerical differentiation may exist.
Sample run:
Estimated freq clean signal: 0.009998
Estimated freq noisy signal: 0.009871
We can see that in this very simple case the frequency is recovered ok.
It may be possible to recover multiple frequencies using more derivatives and some linear decomposition voodoo, but I'm not going to explore this here.
Code:
import numpy as np
import matplotlib.pyplot as plt
'''
I need to caputre a low-frequency oscillation with only 1 time unit of data.
So far, I have not been able to find a way to make the fft resolution < 1.
'''
timeResolution = 10000
mytimes = np.linspace(0, 1, timeResolution)
mypressures = np.sin(2 * np.pi * mytimes / 100)
fft = np.fft.fft(mypressures[:])
T = mytimes[1] - mytimes[0]
N = mypressures.size
# fft of original signal is limitted by the maximum time
f = np.linspace(0, 1 / T, N)
filteredidx = f > 0.001
freq = f[filteredidx][np.argmax(np.abs(fft[filteredidx][:N//2]))]
print('freq bin is is ', f[1] - f[0]) # 1.0
print('frequency is ', freq) # 1.0
print('(real frequency is 0.01)')
import scipy.signal as ss
plt.figure(1)
plt.subplot(121)
plt.plot(mytimes, mypressures)
plt.subplot(122)
plt.plot(mytimes, ss.detrend(mypressures))
plt.figure(2)
mycorrupted = mypressures + 0.00001 * np.random.normal(size=mypressures.shape)
plt.plot(mytimes, ss.detrend(mycorrupted))
plt.plot(mytimes, ss.detrend(mypressures))
width, order = 8999, 3
hw = (width+3) // 2
dsdt = ss.savgol_filter(mypressures, width, order, 1, 1/timeResolution)[hw:-hw]
d3sdt3 = ss.savgol_filter(mypressures, width, order, 3, 1/timeResolution)[hw:-hw]
est_freq_clean = np.nanmean(np.sqrt(-d3sdt3/dsdt) / (2 * np.pi))
dsdt = ss.savgol_filter(mycorrupted, width, order, 1, 1/timeResolution)[hw:-hw]
d3sdt3 = ss.savgol_filter(mycorrupted, width, order, 3, 1/timeResolution)[hw:-hw]
est_freq_noisy = np.nanmean(np.sqrt(-d3sdt3/dsdt) / (2 * np.pi))
print(f"Estimated freq clean signal: {est_freq_clean:10.6f}")
print(f"Estimated freq noisy signal: {est_freq_noisy:10.6f}")

Pandas bar plot changes date format

I have a simple stacked line plot that has exactly the date format I want magically set when using the following code.
df_ts = df.resample("W", how='max')
df_ts.plot(figsize=(12,8), stacked=True)
However, the dates mysteriously transform themselves to an ugly and unreadable format when plotting the same data as a bar plot.
df_ts = df.resample("W", how='max')
df_ts.plot(kind='bar', figsize=(12,8), stacked=True)
The original data was transformed a bit to have the weekly max. Why is this radical change in automatically set dates happening? How can I have the nicely formatted dates as above?
Here is some dummy data
start = pd.to_datetime("1-1-2012")
idx = pd.date_range(start, periods= 365).tolist()
df=pd.DataFrame({'A':np.random.random(365), 'B':np.random.random(365)})
df.index = idx
df_ts = df.resample('W', how= 'max')
df_ts.plot(kind='bar', stacked=True)
The plotting code assumes that each bar in a bar plot deserves its own label.
You could override this assumption by specifying your own formatter:
ax.xaxis.set_major_formatter(formatter)
The pandas.tseries.converter.TimeSeries_DateFormatter that Pandas uses to format the dates in the "good" plot works well with line plots when the x-values are dates. However, with a bar plot the x-values (at least those received by TimeSeries_DateFormatter.__call__) are merely integers starting at zero. If you try to use TimeSeries_DateFormatter with a bar plot, all the labels thus start at the Epoch, 1970-1-1 UTC, since this is the date which corresponds to zero. So the formatter used for line plots is unfortunately useless for bar plots (at least as far as I can see).
The easiest way I see to produce the desired formatting is to generate and set the labels explicitly:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import matplotlib.ticker as ticker
start = pd.to_datetime("5-1-2012")
idx = pd.date_range(start, periods=365)
df = pd.DataFrame({'A': np.random.random(365), 'B': np.random.random(365)})
df.index = idx
df_ts = df.resample('W').max()
ax = df_ts.plot(kind='bar', stacked=True)
# Make most of the ticklabels empty so the labels don't get too crowded
ticklabels = ['']*len(df_ts.index)
# Every 4th ticklable shows the month and day
ticklabels[::4] = [item.strftime('%b %d') for item in df_ts.index[::4]]
# Every 12th ticklabel includes the year
ticklabels[::12] = [item.strftime('%b %d\n%Y') for item in df_ts.index[::12]]
ax.xaxis.set_major_formatter(ticker.FixedFormatter(ticklabels))
plt.gcf().autofmt_xdate()
plt.show()
yields
For those looking for a simple example of a bar plot with dates:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
dates = pd.date_range('2012-1-1', '2017-1-1', freq='M')
df = pd.DataFrame({'A':np.random.random(len(dates)), 'Date':dates})
fig, ax = plt.subplots()
df.plot.bar(x='Date', y='A', ax=ax)
ticklabels = ['']*len(df)
skip = len(df)//12
ticklabels[::skip] = df['Date'].iloc[::skip].dt.strftime('%Y-%m-%d')
ax.xaxis.set_major_formatter(mticker.FixedFormatter(ticklabels))
fig.autofmt_xdate()
# fixes the tracker
# https://matplotlib.org/users/recipes.html
def fmt(x, pos=0, max_i=len(ticklabels)-1):
i = int(x)
i = 0 if i < 0 else max_i if i > max_i else i
return dates[i]
ax.fmt_xdata = fmt
plt.show()
I've struggled with this problem too, and after reading several posts came up with the following solution, which seems to me slightly clearer than matplotlib.dates approach.
Labels without modification:
# Use DatetimeIndex instead of date_range for pandas earlier than 1.0.0 version
timeline = pd.date_range(start='2018, November', freq='M', periods=15)
df = pd.DataFrame({'date': timeline, 'value': np.random.randn(15)})
df.set_index('date', inplace=True)
df.plot(kind='bar', figsize=(12, 8), color='#2ecc71')
Labels with modification:
def line_format(label):
"""
Convert time label to the format of pandas line plot
"""
month = label.month_name()[:3]
if month == 'Jan':
month += f'\n{label.year}'
return month
# Note that we specify rot here
ax = df.plot(kind='bar', figsize=(12, 8), color='#2ecc71', rot=0)
ax.set_xticklabels(map(line_format, df.index))
This approach will add year to the label only if it is January
Here's an easy approach with pandas plot() and without using matplotlib dates:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
# generate sample data
start = pd.to_datetime("1-1-2012")
index = pd.date_range(start, periods= 365)
df = pd.DataFrame({'A' : np.random.random(365), 'B' : np.random.random(365)}, index=index)
# resample to any timeframe you need, e.g. months
df_months = df.resample("M").sum()
# plot
fig, ax = plt.subplots()
df_months.plot(kind="bar", figsize=(16,5), stacked=True, ax=ax)
# format xtick-labels with list comprehension
ax.set_xticklabels([x.strftime("%Y-%m") for x in df_months.index], rotation=45)
plt.show()
How to get nicely formatted dates like the pandas line plot
The issue is that the pandas bar plot processes the date variable as a categorical variable where each date is considered to be a unique category, so the x-axis units are set to integers starting at 0 (like the default DataFrame index when none is assigned) and the full string of each date is shown without any automatic formatting.
Here are two solutions to format the date tick labels of a pandas (stacked) bar chart of a time series:
The first is a variation of the answer by unutbu and is made to better fit the data shown in the question;
The second is a generalized solution that lets you use matplotlib date tick locators and formatters which produces appropriate date labels for time series of any type of frequency.
But first, let's see what the nicely formatted tick labels look like when the sample data is plotted with a pandas line plot.
Default pandas line plot date formatting
import numpy as np # v 1.19.2
import pandas as pd # v 1.1.3
import matplotlib.dates as mdates # v 3.3.2
# Create sample dataset with a daily frequency and resample it to a weekly frequency
rng = np.random.default_rng(seed=123) # random number generator
idx = pd.date_range(start='2012-01-01', end='2013-12-31', freq='D')
df_raw = pd.DataFrame(rng.random(size=(idx.size, 3)),
index=idx, columns=list('ABC'))
df = df_raw.resample('W').sum() # default is 'W-SUN'
# Create pandas stacked line plot
ax = df.plot(stacked=True, figsize=(10,5))
Because the data is grouped by week with timestamps for Sundays (frequency W-SUN), the monthly tick labels are not necessarily placed on the first day of the month and there can be 3 or 4 weeks between each first week of the month so the minor ticks are unevenly spaced (noticeable if you look closely). Here are the exact dates of the major ticks:
# Convert major x ticks to date labels
np.array([mdates.num2date(tick*7-4).strftime('%Y-%b-%d') for tick in ax.get_xticks()])
"""
array(['2012-Jan-01', '2012-Apr-01', '2012-Jul-01', '2012-Oct-07',
'2013-Jan-06', '2013-Apr-07', '2013-Jul-07', '2013-Oct-06',
'2014-Jan-05'], dtype='<U11')
"""
The challenge lies in selecting the ticks for each first week of the month seeing as they are unequally spaced. Other answers have provided simple solutions based on a fixed tick frequency which produces oddly spaced labels in terms of dates where the months can be sometimes repeated (for example the month of July in unutbu's answer). Or they have provided solutions based on a monthly time series instead of a weekly time series, which is simpler to format seeing as there are always 12 months per year. So here is a solution that gives nicely formatted tick labels like in the pandas line plot and that works for any frequency of data.
Solution 1: pandas bar plot with tick labels based on the DatetimeIndex
# Create pandas stacked bar chart
ax = df.plot.bar(stacked=True, figsize=(10,5))
# Create list of monthly timestamps by selecting the first weekly timestamp of each
# month (in this example, the first Sunday of each month)
monthly_timestamps = [timestamp for idx, timestamp in enumerate(df.index)
if (timestamp.month != df.index[idx-1].month) | (idx == 0)]
# Automatically select appropriate number of timestamps so that x-axis does
# not get overcrowded with tick labels
step = 1
while len(monthly_timestamps[::step]) > 10: # increase number if time range >3 years
step += 1
timestamps = monthly_timestamps[::step]
# Create tick labels from timestamps
labels = [ts.strftime('%b\n%Y') if ts.year != timestamps[idx-1].year
else ts.strftime('%b') for idx, ts in enumerate(timestamps)]
# Set major ticks and labels
ax.set_xticks([df.index.get_loc(ts) for ts in timestamps])
ax.set_xticklabels(labels)
# Set minor ticks without labels
ax.set_xticks([df.index.get_loc(ts) for ts in monthly_timestamps], minor=True)
# Rotate and center labels
ax.figure.autofmt_xdate(rotation=0, ha='center')
To my knowledge, there is no way of getting this exact label formatting with the matplotlib.dates (mdates) tick locators and formatters. Nevertheless, combining mdates functionalities with a pandas stacked bar plot can come in handy if you prefer using tick locators/formatters or if you want to have dynamic ticks when using the interactive interface of matplotlib (to pan/zoom in and out).
At this point, it may be useful to consider creating the stacked bar plot in matplotlib directly, where you need to loop through the variables to create the stacked bar. The pandas-based solution shown below works by looping through the patches of the bars to relocate them according to matplotlib date units. So it is basically one loop instead of another, up to you to see which is more convenient.
Solution 2: pandas bar plot with matplotlib tick locators and formatters
This generalized solution uses the mdates AutoDateLocator which places ticks at the beginning of months/years. If you generate data and timestamps with pd.date_range in pandas (like in this example), you should keep in mind that the commonly used 'M' and 'Y' frequencies produce timestamps for the end date of the periods. The code given in the following example aligns monthly/yearly tick marks with 'MS' and 'YS' frequencies.
If you import a dataset using end-of-period dates (or some other type of pandas frequency not aligned with AutoDateLocator ticks), I am not aware of any convenient way to shift the AutoDateLocator accordingly so that the labels become correctly aligned with the bars. I see two options: i) resample the data using df.resample('MS').sum() if that does not cause any issue regarding the meaning of the underlying data; ii) or else use another date locator.
This issue causes no problem in the following example seeing as the data has a week end frequency 'W-SUN' so the monthly/yearly labels placed at a month/year start frequency are fine.
# Create pandas stacked bar chart with the default bar width = 0.5
ax = df.plot.bar(stacked=True, figsize=(10,5))
# Compute width of bars in matplotlib date units, 'md' (in days) and adjust it if
# the bar width in df.plot.bar has been set to something else than the default 0.5
bar_width_md_default, = np.diff(mdates.date2num(df.index[:2]))/2
bar_width = ax.patches[0].get_width()
bar_width_md = bar_width*bar_width_md_default/0.5
# Compute new x values in matplotlib date units for the patches (rectangles) that
# make up the stacked bars, adjusting the positions according to the bar width:
# if the frequency is in months (or years), the bars may not always be perfectly
# centered over the tick marks depending on the number of days difference between
# the months (or years) given by df.index[0] and [1] used to compute the bar
# width, this should not be noticeable if the bars are wide enough.
x_bars_md = mdates.date2num(df.index) - bar_width_md/2
nvar = len(ax.get_legend_handles_labels()[1])
x_patches_md = np.ravel(nvar*[x_bars_md])
# Set bars to new x positions and adjust width: this loop works fine with NaN
# values as well because in bar plot NaNs are drawn with a rectangle of 0 height
# located at the foot of the bar, you can verify this with patch.get_bbox()
for patch, x_md in zip(ax.patches, x_patches_md):
patch.set_x(x_md)
patch.set_width(bar_width_md)
# Set major ticks
maj_loc = mdates.AutoDateLocator()
ax.xaxis.set_major_locator(maj_loc)
# Show minor tick under each bar (instead of each month) to highlight
# discrepancy between major tick locator and bar positions seeing as no tick
# locator is available for first-week-of-the-month frequency
ax.set_xticks(x_bars_md + bar_width_md/2, minor=True)
# Set major tick formatter
zfmts = ['', '%b\n%Y', '%b', '%b-%d', '%H:%M', '%H:%M']
fmt = mdates.ConciseDateFormatter(maj_loc, zero_formats=zfmts, show_offset=False)
ax.xaxis.set_major_formatter(fmt)
# Shift the plot frame to where the bars are now located
xmin = min(x_bars_md) - bar_width_md
xmax = max(x_bars_md) + 2*bar_width_md
ax.set_xlim(xmin, xmax)
# Adjust tick label format last, else it may sometimes not be applied correctly
ax.figure.autofmt_xdate(rotation=0, ha='center')
Minor ticks a displayed under each bar to highlight the fact that the timestamps of the bars often do not coincide with a month/year start marked by the labels of the AutoDateLocator ticks. I am not aware of any date locator that can be used to select ticks for the first week of each month and reproduce exactly the result shown in solution 1.
Documentation: date format codes, mdates.ConciseDateFormatter
Here's a possibly easier approach using mdates, though requires you to loop over your columns, calling bar plot from matplotlib. Here's an example where I plot just one column and use mdates for customized ticks and labels (EDIT Added looping function to plot all columns stacked):
import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
def format_x_date_month_day(ax):
# Standard date x-axis formatting block, labels each month and ticks each day
days = mdates.DayLocator()
months = mdates.MonthLocator() # every month
dayFmt = mdates.DateFormatter('%D')
monthFmt = mdates.DateFormatter('%Y-%m')
ax.figure.autofmt_xdate()
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(monthFmt)
ax.xaxis.set_minor_locator(days)
def df_stacked_bar_formattable(df, ax, **kwargs):
P = []
lastBar = None
for col in df.columns:
X = df.index
Y = df[col]
if lastBar is not None:
P.append(ax.bar(X, Y, bottom=lastBar, **kwargs))
else:
P.append(ax.bar(X, Y, **kwargs))
lastBar = Y
plt.legend([p[0] for p in P], df.columns)
span_days = 90
start = pd.to_datetime("1-1-2012")
idx = pd.date_range(start, periods=span_days).tolist()
df=pd.DataFrame(index=idx, data={'A':np.random.random(span_days), 'B':np.random.random(span_days)})
plt.close('all')
fig, ax = plt.subplots(1)
df_stacked_bar_formattable(df, ax)
format_x_date_month_day(ax)
plt.show()
(Referencing matplotlib.org for example of looping to create a stacked bar plot.) This gives us
Another approach that should work and be much easier is to use df.plot.bar(ax=ax, stacked=True), however it does not admit date axis formatting with mdates and is the subject of my question.
Maybe not the most elegant, but hopefully easy way:
fig = plt.figure()
ax = fig.add_subplot(111)
df_ts.plot(kind='bar', figsize=(12,8), stacked=True,ax=ax)
ax.set_xticklabels(''*len(df_ts.index))
df_ts.plot(linewidth=0, ax=ax) # This sets the nice x_ticks automatically
[EDIT]: ax=ax neede in df_ts.plot()