def plot_slippage_sweep(returns, positions, transactions,
                        slippage_params=(3, 8, 10, 12, 15, 20, 50),
                        ax=None, **kwargs):
    """
    Plots equity curves at different per-dollar slippage assumptions.

    Parameters
    ----------
    returns : pd.Series
        Timeseries of portfolio returns to be adjusted for various
        degrees of slippage.
    positions : pd.DataFrame
        Daily net position values.
         - See full explanation in tears.create_full_tear_sheet.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
         - See full explanation in tears.create_full_tear_sheet.
    slippage_params: tuple
        Slippage pameters to apply to the return time series (in
        basis points).
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to seaborn plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.gca()

    slippage_sweep = pd.DataFrame()
    for bps in slippage_params:
        adj_returns = txn.adjust_returns_for_slippage(returns, positions,
                                                      transactions, bps)
        label = str(bps) + " bps"
        slippage_sweep[label] = ep.cum_returns(adj_returns, 1)

    slippage_sweep.plot(alpha=1.0, lw=0.5, ax=ax)

    ax.set_title('Cumulative returns given additional per-dollar slippage')
    ax.set_ylabel('')

    ax.legend(loc='center left', frameon=True, framealpha=0.5)

    return ax

Plots equity curves at different per - dollar slippage assumptions .


def plot_slippage_sensitivity(returns, positions, transactions,
                              ax=None, **kwargs):
    """
    Plots curve relating per-dollar slippage to average annual returns.

    Parameters
    ----------
    returns : pd.Series
        Timeseries of portfolio returns to be adjusted for various
        degrees of slippage.
    positions : pd.DataFrame
        Daily net position values.
         - See full explanation in tears.create_full_tear_sheet.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
         - See full explanation in tears.create_full_tear_sheet.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to seaborn plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.gca()

    avg_returns_given_slippage = pd.Series()
    for bps in range(1, 100):
        adj_returns = txn.adjust_returns_for_slippage(returns, positions,
                                                      transactions, bps)
        avg_returns = ep.annual_return(adj_returns)
        avg_returns_given_slippage.loc[bps] = avg_returns

    avg_returns_given_slippage.plot(alpha=1.0, lw=2, ax=ax)

    ax.set_title('Average annual returns given additional per-dollar slippage')
    ax.set_xticks(np.arange(0, 100, 10))
    ax.set_ylabel('Average annual return')
    ax.set_xlabel('Per-dollar slippage (bps)')

    return ax

Plots curve relating per - dollar slippage to average annual returns .


def plot_daily_turnover_hist(transactions, positions,
                             ax=None, **kwargs):
    """
    Plots a histogram of daily turnover rates.

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
         - See full explanation in tears.create_full_tear_sheet.
    positions : pd.DataFrame
        Daily net position values.
         - See full explanation in tears.create_full_tear_sheet.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to seaborn plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.gca()
    turnover = txn.get_turnover(positions, transactions)
    sns.distplot(turnover, ax=ax, **kwargs)
    ax.set_title('Distribution of daily turnover rates')
    ax.set_xlabel('Turnover rate')
    return ax

def
plot_daily_turnover_hist
(
transactions
,
positions
,
ax
=
,
**
kwargs
)
:
if
ax
is
:
ax
=
plt
.
gca
(
)
turnover
=
txn
.
get_turnover
(
positions
,
transactions
)
sns
.
distplot
(
turnover
,
ax
=
ax
,
**
kwargs
)
ax
.
set_title
(
'Distribution of daily turnover rates'
)
ax
.
set_xlabel
(
'Turnover rate'
)
return
ax

Plots a histogram of daily turnover rates .


def plot_daily_volume(returns, transactions, ax=None, **kwargs):
    """
    Plots trading volume per day vs. date.

    Also displays all-time daily average.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
         - See full explanation in tears.create_full_tear_sheet.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.gca()
    daily_txn = txn.get_txn_vol(transactions)
    daily_txn.txn_shares.plot(alpha=1.0, lw=0.5, ax=ax, **kwargs)
    ax.axhline(daily_txn.txn_shares.mean(), color='steelblue',
               linestyle='--', lw=3, alpha=1.0)
    ax.set_title('Daily trading volume')
    ax.set_xlim((returns.index[0], returns.index[-1]))
    ax.set_ylabel('Amount of shares traded')
    ax.set_xlabel('')
    return ax

def
plot_daily_volume
(
returns
,
transactions
,
ax
=
,
**
kwargs
)
:
if
ax
is
:
ax
=
plt
.
gca
(
)
daily_txn
=
txn
.
get_txn_vol
(
transactions
)
daily_txn
.
txn_shares
.
plot
(
alpha
=
1.0
,
lw
=
0.5
,
ax
=
ax
,
**
kwargs
)
ax
.
axhline
(
daily_txn
.
txn_shares
.
mean
(
)
,
color
=
'steelblue'
,
linestyle
=
'--'
,
lw
=
3
,
alpha
=
1.0
)
ax
.
set_title
(
'Daily trading volume'
)
ax
.
set_xlim
(
(
returns
.
index
[
0
]
,
returns
.
index
[
1
]
)
)
ax
.
set_ylabel
(
'Amount of shares traded'
)
ax
.
set_xlabel
(
''
)
return
ax

Plots trading volume per day vs . date .


def plot_txn_time_hist(transactions, bin_minutes=5, tz='America/New_York',
                       ax=None, **kwargs):
    """
    Plots a histogram of transaction times, binning the times into
    buckets of a given duration.

    Parameters
    ----------
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
         - See full explanation in tears.create_full_tear_sheet.
    bin_minutes : float, optional
        Sizes of the bins in minutes, defaults to 5 minutes.
    tz : str, optional
        Time zone to plot against. Note that if the specified
        zone does not apply daylight savings, the distribution
        may be partially offset.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.gca()

    txn_time = transactions.copy()

    txn_time.index = txn_time.index.tz_convert(pytz.timezone(tz))
    txn_time.index = txn_time.index.map(lambda x: x.hour * 60 + x.minute)
    txn_time['trade_value'] = (txn_time.amount * txn_time.price).abs()
    txn_time = txn_time.groupby(level=0).sum().reindex(index=range(570, 961))
    txn_time.index = (txn_time.index / bin_minutes).astype(int) * bin_minutes
    txn_time = txn_time.groupby(level=0).sum()

    txn_time['time_str'] = txn_time.index.map(lambda x:
                                              str(datetime.time(int(x / 60),
                                                                x % 60))[:-3])

    trade_value_sum = txn_time.trade_value.sum()
    txn_time.trade_value = txn_time.trade_value.fillna(0) / trade_value_sum

    ax.bar(txn_time.index, txn_time.trade_value, width=bin_minutes, **kwargs)

    ax.set_xlim(570, 960)
    ax.set_xticks(txn_time.index[::int(30 / bin_minutes)])
    ax.set_xticklabels(txn_time.time_str[::int(30 / bin_minutes)])
    ax.set_title('Transaction time distribution')
    ax.set_ylabel('Proportion')
    ax.set_xlabel('')
    return ax

def
plot_txn_time_hist
(
transactions
,
bin_minutes
=
5
,
tz
=
'America/New_York'
,
ax
=
,
**
kwargs
)
:
if
ax
is
:
ax
=
plt
.
gca
(
)
txn_time
=
transactions
.
copy
(
)
txn_time
.
index
=
txn_time
.
index
.
tz_convert
(
pytz
.
timezone
(
tz
)
)
txn_time
.
index
=
txn_time
.
index
.
map
(
lambda
x
:
x
.
hour
*
60
+
x
.
minute
)
txn_time
[
'trade_value'
]
=
(
txn_time
.
amount
*
txn_time
.
price
)
.
abs
(
)
txn_time
=
txn_time
.
groupby
(
level
=
0
)
.
sum
(
)
.
reindex
(
index
=
range
(
570
,
961
)
)
txn_time
.
index
=
(
txn_time
.
index
/
bin_minutes
)
.
astype
(
int
)
*
bin_minutes
txn_time
=
txn_time
.
groupby
(
level
=
0
)
.
sum
(
)
txn_time
[
'time_str'
]
=
txn_time
.
index
.
map
(
lambda
x
:
str
(
datetime
.
time
(
int
(
x
/
60
)
,
x
%
60
)
)
[
:
3
]
)
trade_value_sum
=
txn_time
.
trade_value
.
sum
(
)
txn_time
.
trade_value
=
txn_time
.
trade_value
.
fillna
(
0
)
/
trade_value_sum
ax
.
bar
(
txn_time
.
index
,
txn_time
.
trade_value
,
width
=
bin_minutes
,
**
kwargs
)
ax
.
set_xlim
(
570
,
960
)
ax
.
set_xticks
(
txn_time
.
index
[
:
:
int
(
30
/
bin_minutes
)
]
)
ax
.
set_xticklabels
(
txn_time
.
time_str
[
:
:
int
(
30
/
bin_minutes
)
]
)
ax
.
set_title
(
'Transaction time distribution'
)
ax
.
set_ylabel
(
'Proportion'
)
ax
.
set_xlabel
(
''
)
return
ax

Plots a histogram of transaction times binning the times into buckets of a given duration .


def show_worst_drawdown_periods(returns, top=5):
    """
    Prints information about the worst drawdown periods.

    Prints peak dates, valley dates, recovery dates, and net
    drawdowns.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    top : int, optional
        Amount of top drawdowns periods to plot (default 5).
    """

    drawdown_df = timeseries.gen_drawdown_table(returns, top=top)
    utils.print_table(
        drawdown_df.sort_values('Net drawdown in %', ascending=False),
        name='Worst drawdown periods',
        float_format='{0:.2f}'.format,
    )

def
show_worst_drawdown_periods
(
returns
,
top
=
5
)
:
drawdown_df
=
timeseries
.
gen_drawdown_table
(
returns
,
top
=
top
)
utils
.
print_table
(
drawdown_df
.
sort_values
(
'Net drawdown in %'
,
ascending
=
False
)
,
name
=
'Worst drawdown periods'
,
float_format
=
'{0:.2f}'
.
format
,
)

Prints information about the worst drawdown periods .


def plot_monthly_returns_timeseries(returns, ax=None, **kwargs):
    """
    Plots monthly returns as a timeseries.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    **kwargs, optional
        Passed to seaborn plotting function.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    def cumulate_returns(x):
        return ep.cum_returns(x)[-1]

    if ax is None:
        ax = plt.gca()

    monthly_rets = returns.resample('M').apply(lambda x: cumulate_returns(x))
    monthly_rets = monthly_rets.to_period()

    sns.barplot(x=monthly_rets.index,
                y=monthly_rets.values,
                color='steelblue')

    locs, labels = plt.xticks()
    plt.setp(labels, rotation=90)

    # only show x-labels on year boundary
    xticks_coord = []
    xticks_label = []
    count = 0
    for i in monthly_rets.index:
        if i.month == 1:
            xticks_label.append(i)
            xticks_coord.append(count)
            # plot yearly boundary line
            ax.axvline(count, color='gray', ls='--', alpha=0.3)

        count += 1

    ax.axhline(0.0, color='darkgray', ls='-')
    ax.set_xticks(xticks_coord)
    ax.set_xticklabels(xticks_label)

    return ax

def plot_monthly_returns_timeseries ( returns , ax = None , ** kwargs ) : def cumulate_returns ( x ) : return ep . cum_returns ( x ) [ - 1 ] if ax is None : ax = plt . gca ( ) monthly_rets = returns . resample ( 'M' ) . apply ( lambda x : cumulate_returns ( x ) ) monthly_rets = monthly_rets . to_period ( ) sns . barplot ( x = monthly_rets . index , y = monthly_rets . values , color = 'steelblue' ) locs , labels = plt . xticks ( ) plt . setp ( labels , rotation = 90 ) xticks_coord = [ ] xticks_label = [ ] count = 0 for i in monthly_rets . index : if i . month == 1 : xticks_label . append ( i ) xticks_coord . append ( count ) ax . axvline ( count , color = 'gray' , ls = '--' , alpha = 0.3 ) count += 1 ax . axhline ( 0.0 , color = 'darkgray' , ls = '-' ) ax . set_xticks ( xticks_coord ) ax . set_xticklabels ( xticks_label ) return ax


def plot_round_trip_lifetimes(round_trips, disp_amount=16, lsize=18, ax=None):
    """
    Plots timespans and directions of a sample of round trip trades.

    Parameters
    ----------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip trade.
        - See full explanation in round_trips.extract_round_trips
    ax : matplotlib.Axes, optional
        Axes upon which to plot.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    if ax is None:
        ax = plt.subplot()

    symbols_sample = round_trips.symbol.unique()
    np.random.seed(1)
    sample = np.random.choice(round_trips.symbol.unique(), replace=False,
                              size=min(disp_amount, len(symbols_sample)))
    sample_round_trips = round_trips[round_trips.symbol.isin(sample)]

    symbol_idx = pd.Series(np.arange(len(sample)), index=sample)

    for symbol, sym_round_trips in sample_round_trips.groupby('symbol'):
        for _, row in sym_round_trips.iterrows():
            c = 'b' if row.long else 'r'
            y_ix = symbol_idx[symbol] + 0.05
            ax.plot([row['open_dt'], row['close_dt']],
                    [y_ix, y_ix], color=c,
                    linewidth=lsize, solid_capstyle='butt')

    ax.set_yticks(range(disp_amount))
    ax.set_yticklabels([utils.format_asset(s) for s in sample])

    ax.set_ylim((-0.5, min(len(sample), disp_amount) - 0.5))
    blue = patches.Rectangle([0, 0], 1, 1, color='b', label='Long')
    red = patches.Rectangle([0, 0], 1, 1, color='r', label='Short')
    leg = ax.legend(handles=[blue, red], loc='lower left',
                    frameon=True, framealpha=0.5)
    leg.get_frame().set_edgecolor('black')
    ax.grid(False)

    return ax

def
plot_round_trip_lifetimes
(
round_trips
,
disp_amount
=
16
,
lsize
=
18
,
ax
=
)
:
if
ax
is
:
ax
=
plt
.
subplot
(
)
symbols_sample
=
round_trips
.
symbol
.
unique
(
)
np
.
random
.
seed
(
1
)
sample
=
np
.
random
.
choice
(
round_trips
.
symbol
.
unique
(
)
,
replace
=
False
,
size
=
min
(
disp_amount
,
len
(
symbols_sample
)
)
)
sample_round_trips
=
round_trips
[
round_trips
.
symbol
.
isin
(
sample
)
]
symbol_idx
=
pd
.
Series
(
np
.
arange
(
len
(
sample
)
)
,
index
=
sample
)
for
symbol
,
sym_round_trips
in
sample_round_trips
.
groupby
(
'symbol'
)
:
for
_
,
row
in
sym_round_trips
.
iterrows
(
)
:
c
=
'b'
if
row
.
long
else
'r'
y_ix
=
symbol_idx
[
symbol
]
+
0.05
ax
.
plot
(
[
row
[
'open_dt'
]
,
row
[
'close_dt'
]
]
,
[
y_ix
,
y_ix
]
,
color
=
c
,
linewidth
=
lsize
,
solid_capstyle
=
'butt'
)
ax
.
set_yticks
(
range
(
disp_amount
)
)
ax
.
set_yticklabels
(
[
utils
.
format_asset
(
s
)
for
s
in
sample
]
)
ax
.
set_ylim
(
(
0.5
,
min
(
len
(
sample
)
,
disp_amount
)
0.5
)
)
blue
=
patches
.
Rectangle
(
[
0
,
0
]
,
1
,
1
,
color
=
'b'
,
label
=
'Long'
)
red
=
patches
.
Rectangle
(
[
0
,
0
]
,
1
,
1
,
color
=
'r'
,
label
=
'Short'
)
leg
=
ax
.
legend
(
handles
=
[
blue
,
red
]
,
loc
=
'lower left'
,
frameon
=
True
,
framealpha
=
0.5
)
leg
.
get_frame
(
)
.
set_edgecolor
(
'black'
)
ax
.
grid
(
False
)
return
ax

Plots timespans and directions of a sample of round trip trades .


def show_profit_attribution(round_trips):
    """
    Prints the share of total PnL contributed by each
    traded name.

    Parameters
    ----------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip trade.
        - See full explanation in round_trips.extract_round_trips
    ax : matplotlib.Axes, optional
        Axes upon which to plot.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    total_pnl = round_trips['pnl'].sum()
    pnl_attribution = round_trips.groupby('symbol')['pnl'].sum() / total_pnl
    pnl_attribution.name = ''

    pnl_attribution.index = pnl_attribution.index.map(utils.format_asset)
    utils.print_table(
        pnl_attribution.sort_values(
            inplace=False,
            ascending=False,
        ),
        name='Profitability (PnL / PnL total) per name',
        float_format='{:.2%}'.format,
    )

Prints the share of total PnL contributed by each traded name .


def plot_prob_profit_trade(round_trips, ax=None):
    """
    Plots a probability distribution for the event of making
    a profitable trade.

    Parameters
    ----------
    round_trips : pd.DataFrame
        DataFrame with one row per round trip trade.
        - See full explanation in round_trips.extract_round_trips
    ax : matplotlib.Axes, optional
        Axes upon which to plot.

    Returns
    -------
    ax : matplotlib.Axes
        The axes that were plotted on.
    """

    x = np.linspace(0, 1., 500)

    round_trips['profitable'] = round_trips.pnl > 0

    dist = sp.stats.beta(round_trips.profitable.sum(),
                         (~round_trips.profitable).sum())
    y = dist.pdf(x)
    lower_perc = dist.ppf(.025)
    upper_perc = dist.ppf(.975)

    lower_plot = dist.ppf(.001)
    upper_plot = dist.ppf(.999)

    if ax is None:
        ax = plt.subplot()

    ax.plot(x, y)
    ax.axvline(lower_perc, color='0.5')
    ax.axvline(upper_perc, color='0.5')

    ax.set_xlabel('Probability of making a profitable decision')
    ax.set_ylabel('Belief')
    ax.set_xlim(lower_plot, upper_plot)
    ax.set_ylim((0, y.max() + 1.))

    return ax

Plots a probability distribution for the event of making a profitable trade .


def plot_cones(name, bounds, oos_returns, num_samples=1000, ax=None,
               cone_std=(1., 1.5, 2.), random_seed=None, num_strikes=3):
    """
    Plots the upper and lower bounds of an n standard deviation
    cone of forecasted cumulative returns. Redraws a new cone when
    cumulative returns fall outside of last cone drawn.

    Parameters
    ----------
    name : str
        Account name to be used as figure title.
    bounds : pandas.core.frame.DataFrame
        Contains upper and lower cone boundaries. Column names are
        strings corresponding to the number of standard devations
        above (positive) or below (negative) the projected mean
        cumulative returns.
    oos_returns : pandas.core.frame.DataFrame
        Non-cumulative out-of-sample returns.
    num_samples : int
        Number of samples to draw from the in-sample daily returns.
        Each sample will be an array with length num_days.
        A higher number of samples will generate a more accurate
        bootstrap cone.
    ax : matplotlib.Axes, optional
        Axes upon which to plot.
    cone_std : list of int/float
        Number of standard devations to use in the boundaries of
        the cone. If multiple values are passed, cone bounds will
        be generated for each value.
    random_seed : int
        Seed for the pseudorandom number generator used by the pandas
        sample method.
    num_strikes : int
        Upper limit for number of cones drawn. Can be anything from 0 to 3.

    Returns
    -------
    Returns are either an ax or fig option, but not both. If a
    matplotlib.Axes instance is passed in as ax, then it will be modified
    and returned. This allows for users to plot interactively in jupyter
    notebook. When no ax object is passed in, a matplotlib.figure instance
    is generated and returned. This figure can then be used to save
    the plot as an image without viewing it.

    ax : matplotlib.Axes
        The axes that were plotted on.
    fig : matplotlib.figure
        The figure instance which contains all the plot elements.
    """

    if ax is None:
        fig = figure.Figure(figsize=(10, 8))
        FigureCanvasAgg(fig)
        axes = fig.add_subplot(111)
    else:
        axes = ax

    returns = ep.cum_returns(oos_returns, starting_value=1.)
    bounds_tmp = bounds.copy()
    returns_tmp = returns.copy()
    cone_start = returns.index[0]
    colors = ["green", "orange", "orangered", "darkred"]

    for c in range(num_strikes + 1):
        if c > 0:
            tmp = returns.loc[cone_start:]
            bounds_tmp = bounds_tmp.iloc[0:len(tmp)]
            bounds_tmp = bounds_tmp.set_index(tmp.index)
            crossing = (tmp < bounds_tmp[float(-2.)].iloc[:len(tmp)])
            if crossing.sum() <= 0:
                break
            cone_start = crossing.loc[crossing].index[0]
            returns_tmp = returns.loc[cone_start:]
            bounds_tmp = (bounds - (1 - returns.loc[cone_start]))
        for std in cone_std:
            x = returns_tmp.index
            y1 = bounds_tmp[float(std)].iloc[:len(returns_tmp)]
            y2 = bounds_tmp[float(-std)].iloc[:len(returns_tmp)]
            axes.fill_between(x, y1, y2, color=colors[c], alpha=0.5)

    # Plot returns line graph
    label = 'Cumulative returns = {:.2f}%'.format((returns.iloc[-1] - 1) * 100)
    axes.plot(returns.index, returns.values, color='black', lw=3.,
              label=label)

    if name is not None:
        axes.set_title(name)
    axes.axhline(1, color='black', alpha=0.2)
    axes.legend(frameon=True, framealpha=0.5)

    if ax is None:
        return fig
    else:
        return axes

def
plot_cones
(
name
,
bounds
,
oos_returns
,
num_samples
=
1000
,
ax
=
,
cone_std
=
(
1.
,
1.5
,
2.
)
,
random_seed
=
,
num_strikes
=
3
)
:
if
ax
is
:
fig
=
figure
.
Figure
(
figsize
=
(
10
,
8
)
)
FigureCanvasAgg
(
fig
)
axes
=
fig
.
add_subplot
(
111
)
else
:
axes
=
ax
returns
=
ep
.
cum_returns
(
oos_returns
,
starting_value
=
1.
)
bounds_tmp
=
bounds
.
copy
(
)
returns_tmp
=
returns
.
copy
(
)
cone_start
=
returns
.
index
[
0
]
colors
=
[
"green"
,
"orange"
,
"orangered"
,
"darkred"
]
for
c
in
range
(
num_strikes
+
1
)
:
if
c
>
0
:
tmp
=
returns
.
loc
[
cone_start
:
]
bounds_tmp
=
bounds_tmp
.
iloc
[
0
:
len
(
tmp
)
]
bounds_tmp
=
bounds_tmp
.
set_index
(
tmp
.
index
)
crossing
=
(
tmp
<
bounds_tmp
[
float
(
2.
)
]
.
iloc
[
:
len
(
tmp
)
]
)
if
crossing
.
sum
(
)
<=
0
:
break
cone_start
=
crossing
.
loc
[
crossing
]
.
index
[
0
]
returns_tmp
=
returns
.
loc
[
cone_start
:
]
bounds_tmp
=
(
bounds
(
1
returns
.
loc
[
cone_start
]
)
)
for
std
in
cone_std
:
x
=
returns_tmp
.
index
y1
=
bounds_tmp
[
float
(
std
)
]
.
iloc
[
:
len
(
returns_tmp
)
]
y2
=
bounds_tmp
[
float
(
std
)
]
.
iloc
[
:
len
(
returns_tmp
)
]
axes
.
fill_between
(
x
,
y1
,
y2
,
color
=
colors
[
c
]
,
alpha
=
0.5
)
label
=
'Cumulative returns = {:.2f}%'
.
format
(
(
returns
.
iloc
[
1
]
1
)
*
100
)
axes
.
plot
(
returns
.
index
,
returns
.
values
,
color
=
'black'
,
lw
=
3.
,
label
=
label
)
if
name
is
not
:
axes
.
set_title
(
name
)
axes
.
axhline
(
1
,
color
=
'black'
,
alpha
=
0.2
)
axes
.
legend
(
frameon
=
True
,
framealpha
=
0.5
)
if
ax
is
:
return
fig
else
:
return
axes

Plots the upper and lower bounds of an n standard deviation cone of forecasted cumulative returns . Redraws a new cone when cumulative returns fall outside of last cone drawn .


def var_cov_var_normal(P, c, mu=0, sigma=1):
    """
    Variance-covariance calculation of daily Value-at-Risk in a
    portfolio.

    Parameters
    ----------
    P : float
        Portfolio value.
    c : float
        Confidence level.
    mu : float, optional
        Mean.

    Returns
    -------
    float
        Variance-covariance.
    """

    alpha = sp.stats.norm.ppf(1 - c, mu, sigma)
    return P - P * (alpha + 1)

def var_cov_var_normal ( P , c , mu = 0 , sigma = 1 ) : alpha = sp . stats . norm . ppf ( 1 - c , mu , sigma ) return P - P * ( alpha + 1 )

Variance - covariance calculation of daily Value - at - Risk in a portfolio .


def sortino_ratio(returns, required_return=0, period=DAILY):
    """
    Determines the Sortino ratio of a strategy.

    Parameters
    ----------
    returns : pd.Series or pd.DataFrame
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
    required_return: float / series
        minimum acceptable return
    period : str, optional
        Defines the periodicity of the 'returns' data for purposes of
        annualizing. Can be 'monthly', 'weekly', or 'daily'.
        - Defaults to 'daily'.

    Returns
    -------
    depends on input type
    series ==> float
    DataFrame ==> np.array

        Annualized Sortino ratio.
    """

    return ep.sortino_ratio(returns, required_return=required_return)

def sortino_ratio ( returns , required_return = 0 , period = DAILY ) : return ep . sortino_ratio ( returns , required_return = required_return )

Determines the Sortino ratio of a strategy .


def downside_risk(returns, required_return=0, period=DAILY):
    """
    Determines the downside deviation below a threshold

    Parameters
    ----------
    returns : pd.Series or pd.DataFrame
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
    required_return: float / series
        minimum acceptable return
    period : str, optional
        Defines the periodicity of the 'returns' data for purposes of
        annualizing. Can be 'monthly', 'weekly', or 'daily'.
        - Defaults to 'daily'.

    Returns
    -------
    depends on input type
    series ==> float
    DataFrame ==> np.array

        Annualized downside deviation
    """

    return ep.downside_risk(returns,
                            required_return=required_return,
                            period=period)

def downside_risk ( returns , required_return = 0 , period = DAILY ) : return ep . downside_risk ( returns , required_return = required_return , period = period )


def sharpe_ratio(returns, risk_free=0, period=DAILY):
    """
    Determines the Sharpe ratio of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.
    risk_free : int, float
        Constant risk-free return throughout the period.
    period : str, optional
        Defines the periodicity of the 'returns' data for purposes of
        annualizing. Can be 'monthly', 'weekly', or 'daily'.
        - Defaults to 'daily'.

    Returns
    -------
    float
        Sharpe ratio.
    np.nan
        If insufficient length of returns or if if adjusted returns are 0.

    Note
    -----
    See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.
    """

    return ep.sharpe_ratio(returns, risk_free=risk_free, period=period)

def sharpe_ratio ( returns , risk_free = 0 , period = DAILY ) : return ep . sharpe_ratio ( returns , risk_free = risk_free , period = period )

Determines the Sharpe ratio of a strategy .


def rolling_beta(returns, factor_returns,
                 rolling_window=APPROX_BDAYS_PER_MONTH * 6):
    """
    Determines the rolling beta of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series or pd.DataFrame
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - If DataFrame is passed, computes rolling beta for each column.
         - This is in the same style as returns.
    rolling_window : int, optional
        The size of the rolling window, in days, over which to compute
        beta (default 6 months).

    Returns
    -------
    pd.Series
        Rolling beta.

    Note
    -----
    See https://en.wikipedia.org/wiki/Beta_(finance) for more details.
    """

    if factor_returns.ndim > 1:
        # Apply column-wise
        return factor_returns.apply(partial(rolling_beta, returns),
                                    rolling_window=rolling_window)
    else:
        out = pd.Series(index=returns.index)
        for beg, end in zip(returns.index[0:-rolling_window],
                            returns.index[rolling_window:]):
            out.loc[end] = ep.beta(
                returns.loc[beg:end],
                factor_returns.loc[beg:end])

        return out

def rolling_beta ( returns , factor_returns , rolling_window = APPROX_BDAYS_PER_MONTH * 6 ) : if factor_returns . ndim > 1 : return factor_returns . apply ( partial ( rolling_beta , returns ) , rolling_window = rolling_window ) else : out = pd . Series ( index = returns . index ) for beg , end in zip ( returns . index [ 0 : - rolling_window ] , returns . index [ rolling_window : ] ) : out . loc [ end ] = ep . beta ( returns . loc [ beg : end ] , factor_returns . loc [ beg : end ] ) return out

Determines the rolling beta of a strategy .


def gross_lev(positions):
    """
    Calculates the gross leverage of a strategy.

    Parameters
    ----------
    positions : pd.DataFrame
        Daily net position values.
         - See full explanation in tears.create_full_tear_sheet.

    Returns
    -------
    pd.Series
        Gross leverage.
    """

    exposure = positions.drop('cash', axis=1).abs().sum(axis=1)
    return exposure / positions.sum(axis=1)

def gross_lev ( positions ) : exposure = positions . drop ( 'cash' , axis = 1 ) . abs ( ) . sum ( axis = 1 ) return exposure / positions . sum ( axis = 1 )

Calculates the gross leverage of a strategy .


def perf_stats(returns, factor_returns=None, positions=None,
               transactions=None, turnover_denom='AGB'):
    """
    Calculates various performance metrics of a strategy, for use in
    plotting.show_perf_stats.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
         - If None, do not compute alpha, beta, and information ratio.
    positions : pd.DataFrame
        Daily net position values.
         - See full explanation in tears.create_full_tear_sheet.
    transactions : pd.DataFrame
        Prices and amounts of executed trades. One row per trade.
        - See full explanation in tears.create_full_tear_sheet.
    turnover_denom : str
        Either AGB or portfolio_value, default AGB.
        - See full explanation in txn.get_turnover.

    Returns
    -------
    pd.Series
        Performance metrics.
    """

    stats = pd.Series()
    for stat_func in SIMPLE_STAT_FUNCS:
        stats[STAT_FUNC_NAMES[stat_func.__name__]] = stat_func(returns)

    if positions is not None:
        stats['Gross leverage'] = gross_lev(positions).mean()
        if transactions is not None:
            stats['Daily turnover'] = get_turnover(positions,
                                                   transactions,
                                                   turnover_denom).mean()
    if factor_returns is not None:
        for stat_func in FACTOR_STAT_FUNCS:
            res = stat_func(returns, factor_returns)
            stats[STAT_FUNC_NAMES[stat_func.__name__]] = res

    return stats

def
perf_stats
(
returns
,
factor_returns
=
,
positions
=
,
transactions
=
,
turnover_denom
=
'AGB'
)
:
stats
=
pd
.
Series
(
)
for
stat_func
in
SIMPLE_STAT_FUNCS
:
stats
[
STAT_FUNC_NAMES
[
stat_func
.
__name__
]
]
=
stat_func
(
returns
)
if
positions
is
not
:
stats
[
'Gross leverage'
]
=
gross_lev
(
positions
)
.
mean
(
)
if
transactions
is
not
:
stats
[
'Daily turnover'
]
=
get_turnover
(
positions
,
transactions
,
turnover_denom
)
.
mean
(
)
if
factor_returns
is
not
:
for
stat_func
in
FACTOR_STAT_FUNCS
:
res
=
stat_func
(
returns
,
factor_returns
)
stats
[
STAT_FUNC_NAMES
[
stat_func
.
__name__
]
]
=
res
return
stats

Calculates various performance metrics of a strategy for use in plotting . show_perf_stats .


def perf_stats_bootstrap(returns, factor_returns=None, return_stats=True,
                         **kwargs):
    """Calculates various bootstrapped performance metrics of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
         - If None, do not compute alpha, beta, and information ratio.
    return_stats : boolean (optional)
        If True, returns a DataFrame of mean, median, 5 and 95 percentiles
        for each perf metric.
        If False, returns a DataFrame with the bootstrap samples for
        each perf metric.

    Returns
    -------
    pd.DataFrame
        if return_stats is True:
        - Distributional statistics of bootstrapped sampling
        distribution of performance metrics.
        if return_stats is False:
        - Bootstrap samples for each performance metric.
    """

    bootstrap_values = OrderedDict()

    for stat_func in SIMPLE_STAT_FUNCS:
        stat_name = STAT_FUNC_NAMES[stat_func.__name__]
        bootstrap_values[stat_name] = calc_bootstrap(stat_func,
                                                     returns)

    if factor_returns is not None:
        for stat_func in FACTOR_STAT_FUNCS:
            stat_name = STAT_FUNC_NAMES[stat_func.__name__]
            bootstrap_values[stat_name] = calc_bootstrap(
                stat_func,
                returns,
                factor_returns=factor_returns)

    bootstrap_values = pd.DataFrame(bootstrap_values)

    if return_stats:
        stats = bootstrap_values.apply(calc_distribution_stats)
        return stats.T[['mean', 'median', '5%', '95%']]
    else:
        return bootstrap_values

def perf_stats_bootstrap ( returns , factor_returns = None , return_stats = True , ** kwargs ) : bootstrap_values = OrderedDict ( ) for stat_func in SIMPLE_STAT_FUNCS : stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ] bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns ) if factor_returns is not None : for stat_func in FACTOR_STAT_FUNCS : stat_name = STAT_FUNC_NAMES [ stat_func . __name__ ] bootstrap_values [ stat_name ] = calc_bootstrap ( stat_func , returns , factor_returns = factor_returns ) bootstrap_values = pd . DataFrame ( bootstrap_values ) if return_stats : stats = bootstrap_values . apply ( calc_distribution_stats ) return stats . T [ [ 'mean' , 'median' , '5%' , '95%' ] ] else : return bootstrap_values

Calculates various bootstrapped performance metrics of a strategy .


def calc_bootstrap(func, returns, *args, **kwargs):
    """Performs a bootstrap analysis on a user-defined function returning
    a summary statistic.

    Parameters
    ----------
    func : function
        Function that either takes a single array (commonly returns)
        or two arrays (commonly returns and factor returns) and
        returns a single value (commonly a summary
        statistic). Additional args and kwargs are passed as well.
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    factor_returns : pd.Series, optional
        Daily noncumulative returns of the benchmark factor to which betas are
        computed. Usually a benchmark such as market returns.
         - This is in the same style as returns.
    n_samples : int, optional
        Number of bootstrap samples to draw. Default is 1000.
        Increasing this will lead to more stable / accurate estimates.

    Returns
    -------
    numpy.ndarray
        Bootstrapped sampling distribution of passed in func.
    """

    n_samples = kwargs.pop('n_samples', 1000)
    out = np.empty(n_samples)

    factor_returns = kwargs.pop('factor_returns', None)

    for i in range(n_samples):
        idx = np.random.randint(len(returns), size=len(returns))
        returns_i = returns.iloc[idx].reset_index(drop=True)
        if factor_returns is not None:
            factor_returns_i = factor_returns.iloc[idx].reset_index(drop=True)
            out[i] = func(returns_i, factor_returns_i,
                          *args, **kwargs)
        else:
            out[i] = func(returns_i,
                          *args, **kwargs)

    return out

def calc_bootstrap ( func , returns , * args , ** kwargs ) : n_samples = kwargs . pop ( 'n_samples' , 1000 ) out = np . empty ( n_samples ) factor_returns = kwargs . pop ( 'factor_returns' , None ) for i in range ( n_samples ) : idx = np . random . randint ( len ( returns ) , size = len ( returns ) ) returns_i = returns . iloc [ idx ] . reset_index ( drop = True ) if factor_returns is not None : factor_returns_i = factor_returns . iloc [ idx ] . reset_index ( drop = True ) out [ i ] = func ( returns_i , factor_returns_i , * args , ** kwargs ) else : out [ i ] = func ( returns_i , * args , ** kwargs ) return out

Performs a bootstrap analysis on a user - defined function returning a summary statistic .


def calc_distribution_stats(x):
    """Calculate various summary statistics of data.

    Parameters
    ----------
    x : numpy.ndarray or pandas.Series
        Array to compute summary statistics for.

    Returns
    -------
    pandas.Series
        Series containing mean, median, std, as well as 5, 25, 75 and
        95 percentiles of passed in values.
    """

    return pd.Series({'mean': np.mean(x),
                      'median': np.median(x),
                      'std': np.std(x),
                      '5%': np.percentile(x, 5),
                      '25%': np.percentile(x, 25),
                      '75%': np.percentile(x, 75),
                      '95%': np.percentile(x, 95),
                      'IQR': np.subtract.reduce(
                          np.percentile(x, [75, 25])),
                      })

def calc_distribution_stats ( x ) : return pd . Series ( { 'mean' : np . mean ( x ) , 'median' : np . median ( x ) , 'std' : np . std ( x ) , '5%' : np . percentile ( x , 5 ) , '25%' : np . percentile ( x , 25 ) , '75%' : np . percentile ( x , 75 ) , '95%' : np . percentile ( x , 95 ) , 'IQR' : np . subtract . reduce ( np . percentile ( x , [ 75 , 25 ] ) ) , } )

Calculate various summary statistics of data .


def get_max_drawdown_underwater(underwater):
    """
    Determines peak, valley, and recovery dates given an 'underwater'
    DataFrame.

    An underwater DataFrame is a DataFrame that has precomputed
    rolling drawdown.

    Parameters
    ----------
    underwater : pd.Series
       Underwater returns (rolling drawdown) of a strategy.

    Returns
    -------
    peak : datetime
        The maximum drawdown's peak.
    valley : datetime
        The maximum drawdown's valley.
    recovery : datetime
        The maximum drawdown's recovery.
    """

    valley = np.argmin(underwater)  # end of the period
    # Find first 0
    peak = underwater[:valley][underwater[:valley] == 0].index[-1]
    # Find last 0
    try:
        recovery = underwater[valley:][underwater[valley:] == 0].index[0]
    except IndexError:
        recovery = np.nan  # drawdown not recovered
    return peak, valley, recovery

def get_max_drawdown_underwater ( underwater ) : valley = np . argmin ( underwater ) peak = underwater [ : valley ] [ underwater [ : valley ] == 0 ] . index [ - 1 ] try : recovery = underwater [ valley : ] [ underwater [ valley : ] == 0 ] . index [ 0 ] except IndexError : recovery = np . nan return peak , valley , recovery

Determines peak valley and recovery dates given an underwater DataFrame .


def get_max_drawdown(returns):
    """
    Determines the maximum drawdown of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
        - See full explanation in :func:`~pyfolio.timeseries.cum_returns`.

    Returns
    -------
    float
        Maximum drawdown.

    Note
    -----
    See https://en.wikipedia.org/wiki/Drawdown_(economics) for more details.
    """

    returns = returns.copy()
    df_cum = cum_returns(returns, 1.0)
    running_max = np.maximum.accumulate(df_cum)
    underwater = df_cum / running_max - 1
    return get_max_drawdown_underwater(underwater)

def get_max_drawdown ( returns ) : returns = returns . copy ( ) df_cum = cum_returns ( returns , 1.0 ) running_max = np . maximum . accumulate ( df_cum ) underwater = df_cum / running_max - 1 return get_max_drawdown_underwater ( underwater )

Determines the maximum drawdown of a strategy .


def get_top_drawdowns(returns, top=10):
    """
    Finds top drawdowns, sorted by drawdown amount.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    top : int, optional
        The amount of top drawdowns to find (default 10).

    Returns
    -------
    drawdowns : list
        List of drawdown peaks, valleys, and recoveries. See get_max_drawdown.
    """

    returns = returns.copy()
    df_cum = ep.cum_returns(returns, 1.0)
    running_max = np.maximum.accumulate(df_cum)
    underwater = df_cum / running_max - 1

    drawdowns = []
    for t in range(top):
        peak, valley, recovery = get_max_drawdown_underwater(underwater)
        # Slice out draw-down period
        if not pd.isnull(recovery):
            underwater.drop(underwater[peak: recovery].index[1:-1],
                            inplace=True)
        else:
            # drawdown has not ended yet
            underwater = underwater.loc[:peak]

        drawdowns.append((peak, valley, recovery))
        if (len(returns) == 0) or (len(underwater) == 0):
            break

    return drawdowns

def get_top_drawdowns ( returns , top = 10 ) : returns = returns . copy ( ) df_cum = ep . cum_returns ( returns , 1.0 ) running_max = np . maximum . accumulate ( df_cum ) underwater = df_cum / running_max - 1 drawdowns = [ ] for t in range ( top ) : peak , valley , recovery = get_max_drawdown_underwater ( underwater ) if not pd . isnull ( recovery ) : underwater . drop ( underwater [ peak : recovery ] . index [ 1 : - 1 ] , inplace = True ) else : underwater = underwater . loc [ : peak ] drawdowns . append ( ( peak , valley , recovery ) ) if ( len ( returns ) == 0 ) or ( len ( underwater ) == 0 ) : break return drawdowns

Finds top drawdowns sorted by drawdown amount .


def gen_drawdown_table(returns, top=10):
    """
    Places top drawdowns in a table.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    top : int, optional
        The amount of top drawdowns to find (default 10).

    Returns
    -------
    df_drawdowns : pd.DataFrame
        Information about top drawdowns.
    """

    df_cum = ep.cum_returns(returns, 1.0)
    drawdown_periods = get_top_drawdowns(returns, top=top)
    df_drawdowns = pd.DataFrame(index=list(range(top)),
                                columns=['Net drawdown in %',
                                         'Peak date',
                                         'Valley date',
                                         'Recovery date',
                                         'Duration'])

    for i, (peak, valley, recovery) in enumerate(drawdown_periods):
        if pd.isnull(recovery):
            df_drawdowns.loc[i, 'Duration'] = np.nan
        else:
            df_drawdowns.loc[i, 'Duration'] = len(pd.date_range(peak,
                                                                recovery,
                                                                freq='B'))
        df_drawdowns.loc[i, 'Peak date'] = (peak.to_pydatetime()
                                            .strftime('%Y-%m-%d'))
        df_drawdowns.loc[i, 'Valley date'] = (valley.to_pydatetime()
                                              .strftime('%Y-%m-%d'))
        if isinstance(recovery, float):
            df_drawdowns.loc[i, 'Recovery date'] = recovery
        else:
            df_drawdowns.loc[i, 'Recovery date'] = (recovery.to_pydatetime()
                                                    .strftime('%Y-%m-%d'))
        df_drawdowns.loc[i, 'Net drawdown in %'] = (
            (df_cum.loc[peak] - df_cum.loc[valley]) / df_cum.loc[peak]) * 100

    df_drawdowns['Peak date'] = pd.to_datetime(df_drawdowns['Peak date'])
    df_drawdowns['Valley date'] = pd.to_datetime(df_drawdowns['Valley date'])
    df_drawdowns['Recovery date'] = pd.to_datetime(
        df_drawdowns['Recovery date'])

    return df_drawdowns

def
gen_drawdown_table
(
returns
,
top
=
10
)
:
df_cum
=
ep
.
cum_returns
(
returns
,
1.0
)
drawdown_periods
=
get_top_drawdowns
(
returns
,
top
=
top
)
df_drawdowns
=
pd
.
DataFrame
(
index
=
list
(
range
(
top
)
)
,
columns
=
[
'Net drawdown in %'
,
'Peak date'
,
'Valley date'
,
'Recovery date'
,
'Duration'
]
)
for
i
,
(
peak
,
valley
,
recovery
)
in
enumerate
(
drawdown_periods
)
:
if
pd
.
isnull
(
recovery
)
:
df_drawdowns
.
loc
[
i
,
'Duration'
]
=
np
.
else
:
df_drawdowns
.
loc
[
i
,
'Duration'
]
=
len
(
pd
.
date_range
(
peak
,
recovery
,
freq
=
'B'
)
)
df_drawdowns
.
loc
[
i
,
'Peak date'
]
=
(
peak
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
df_drawdowns
.
loc
[
i
,
'Valley date'
]
=
(
valley
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
if
isinstance
(
recovery
,
float
)
:
df_drawdowns
.
loc
[
i
,
'Recovery date'
]
=
recovery
else
:
df_drawdowns
.
loc
[
i
,
'Recovery date'
]
=
(
recovery
.
to_pydatetime
(
)
.
strftime
(
'%Y-%m-%d'
)
)
df_drawdowns
.
loc
[
i
,
'Net drawdown in %'
]
=
(
(
df_cum
.
loc
[
peak
]
df_cum
.
loc
[
valley
]
)
/
df_cum
.
loc
[
peak
]
)
*
100
df_drawdowns
[
'Peak date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Peak date'
]
)
df_drawdowns
[
'Valley date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Valley date'
]
)
df_drawdowns
[
'Recovery date'
]
=
pd
.
to_datetime
(
df_drawdowns
[
'Recovery date'
]
)
return
df_drawdowns


def rolling_volatility(returns, rolling_vol_window):
    """
    Determines the rolling volatility of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    rolling_vol_window : int
        Length of rolling window, in days, over which to compute.

    Returns
    -------
    pd.Series
        Rolling volatility.
    """

    return returns.rolling(rolling_vol_window).std() \
        * np.sqrt(APPROX_BDAYS_PER_YEAR)

def rolling_volatility ( returns , rolling_vol_window ) : return returns . rolling ( rolling_vol_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )

Determines the rolling volatility of a strategy .


def rolling_sharpe(returns, rolling_sharpe_window):
    """
    Determines the rolling Sharpe ratio of a strategy.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.
    rolling_sharpe_window : int
        Length of rolling window, in days, over which to compute.

    Returns
    -------
    pd.Series
        Rolling Sharpe ratio.

    Note
    -----
    See https://en.wikipedia.org/wiki/Sharpe_ratio for more details.
    """

    return returns.rolling(rolling_sharpe_window).mean() \
        / returns.rolling(rolling_sharpe_window).std() \
        * np.sqrt(APPROX_BDAYS_PER_YEAR)

def rolling_sharpe ( returns , rolling_sharpe_window ) : return returns . rolling ( rolling_sharpe_window ) . mean ( ) / returns . rolling ( rolling_sharpe_window ) . std ( ) * np . sqrt ( APPROX_BDAYS_PER_YEAR )

Determines the rolling Sharpe ratio of a strategy .


def simulate_paths(is_returns, num_days,
                   starting_value=1, num_samples=1000, random_seed=None):
    """
    Gnerate alternate paths using available values from in-sample returns.

    Parameters
    ----------
    is_returns : pandas.core.frame.DataFrame
        Non-cumulative in-sample returns.
    num_days : int
        Number of days to project the probability cone forward.
    starting_value : int or float
        Starting value of the out of sample period.
    num_samples : int
        Number of samples to draw from the in-sample daily returns.
        Each sample will be an array with length num_days.
        A higher number of samples will generate a more accurate
        bootstrap cone.
    random_seed : int
        Seed for the pseudorandom number generator used by the pandas
        sample method.

    Returns
    -------
    samples : numpy.ndarray
    """

    samples = np.empty((num_samples, num_days))
    seed = np.random.RandomState(seed=random_seed)
    for i in range(num_samples):
        samples[i, :] = is_returns.sample(num_days, replace=True,
                                          random_state=seed)

    return samples

def simulate_paths ( is_returns , num_days , starting_value = 1 , num_samples = 1000 , random_seed = None ) : samples = np . empty ( ( num_samples , num_days ) ) seed = np . random . RandomState ( seed = random_seed ) for i in range ( num_samples ) : samples [ i , : ] = is_returns . sample ( num_days , replace = True , random_state = seed ) return samples

Gnerate alternate paths using available values from in - sample returns .


def summarize_paths(samples, cone_std=(1., 1.5, 2.), starting_value=1.):
    """
    Gnerate the upper and lower bounds of an n standard deviation
    cone of forecasted cumulative returns.

    Parameters
    ----------
    samples : numpy.ndarray
        Alternative paths, or series of possible outcomes.
    cone_std : list of int/float
        Number of standard devations to use in the boundaries of
        the cone. If multiple values are passed, cone bounds will
        be generated for each value.

    Returns
    -------
    samples : pandas.core.frame.DataFrame
    """

    cum_samples = ep.cum_returns(samples.T,
                                 starting_value=starting_value).T

    cum_mean = cum_samples.mean(axis=0)
    cum_std = cum_samples.std(axis=0)

    if isinstance(cone_std, (float, int)):
        cone_std = [cone_std]

    cone_bounds = pd.DataFrame(columns=pd.Float64Index([]))
    for num_std in cone_std:
        cone_bounds.loc[:, float(num_std)] = cum_mean + cum_std * num_std
        cone_bounds.loc[:, float(-num_std)] = cum_mean - cum_std * num_std

    return cone_bounds

def summarize_paths ( samples , cone_std = ( 1. , 1.5 , 2. ) , starting_value = 1. ) : cum_samples = ep . cum_returns ( samples . T , starting_value = starting_value ) . T cum_mean = cum_samples . mean ( axis = 0 ) cum_std = cum_samples . std ( axis = 0 ) if isinstance ( cone_std , ( float , int ) ) : cone_std = [ cone_std ] cone_bounds = pd . DataFrame ( columns = pd . Float64Index ( [ ] ) ) for num_std in cone_std : cone_bounds . loc [ : , float ( num_std ) ] = cum_mean + cum_std * num_std cone_bounds . loc [ : , float ( - num_std ) ] = cum_mean - cum_std * num_std return cone_bounds

Gnerate the upper and lower bounds of an n standard deviation cone of forecasted cumulative returns .


def extract_interesting_date_ranges(returns):
    """
    Extracts returns based on interesting events. See
    gen_date_range_interesting.

    Parameters
    ----------
    returns : pd.Series
        Daily returns of the strategy, noncumulative.
         - See full explanation in tears.create_full_tear_sheet.

    Returns
    -------
    ranges : OrderedDict
        Date ranges, with returns, of all valid events.
    """

    returns_dupe = returns.copy()
    returns_dupe.index = returns_dupe.index.map(pd.Timestamp)
    ranges = OrderedDict()
    for name, (start, end) in PERIODS.items():
        try:
            period = returns_dupe.loc[start:end]
            if len(period) == 0:
                continue
            ranges[name] = period
        except BaseException:
            continue

    return ranges

def extract_interesting_date_ranges ( returns ) : returns_dupe = returns . copy ( ) returns_dupe . index = returns_dupe . index . map ( pd . Timestamp ) ranges = OrderedDict ( ) for name , ( start , end ) in PERIODS . items ( ) : try : period = returns_dupe . loc [ start : end ] if len ( period ) == 0 : continue ranges [ name ] = period except BaseException : continue return ranges

Extracts returns based on interesting events . See gen_date_range_interesting .


def model_returns_t_alpha_beta(data, bmark, samples=2000, progressbar=True):
    """
    Run Bayesian alpha-beta-model with T distributed returns.

    This model estimates intercept (alpha) and slope (beta) of two
    return sets. Usually, these will be algorithm returns and
    benchmark returns (e.g. S&P500). The data is assumed to be T
    distributed and thus is robust to outliers and takes tail events
    into account.  If a pandas.DataFrame is passed as a benchmark, then
    multiple linear regression is used to estimate alpha and beta.

    Parameters
    ----------
    returns : pandas.Series
        Series of simple returns of an algorithm or stock.
    bmark : pandas.DataFrame
        DataFrame of benchmark returns (e.g., S&P500) or risk factors (e.g.,
        Fama-French SMB, HML, and UMD).
        If bmark has more recent returns than returns_train, these dates
        will be treated as missing values and predictions will be
        generated for them taking market correlations into account.
    samples : int (optional)
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    data_bmark = pd.concat([data, bmark], axis=1).dropna()

    with pm.Model() as model:
        sigma = pm.HalfCauchy(
            'sigma',
            beta=1)
        nu = pm.Exponential('nu_minus_two', 1. / 10.)

        # alpha and beta
        X = data_bmark.iloc[:, 1]
        y = data_bmark.iloc[:, 0]

        alpha_reg = pm.Normal('alpha', mu=0, sd=.1)
        beta_reg = pm.Normal('beta', mu=0, sd=1)

        mu_reg = alpha_reg + beta_reg * X
        pm.StudentT('returns',
                    nu=nu + 2,
                    mu=mu_reg,
                    sd=sigma,
                    observed=y)
        trace = pm.sample(samples, progressbar=progressbar)

    return model, trace

def model_returns_t_alpha_beta ( data , bmark , samples = 2000 , progressbar = True ) : data_bmark = pd . concat ( [ data , bmark ] , axis = 1 ) . dropna ( ) with pm . Model ( ) as model : sigma = pm . HalfCauchy ( 'sigma' , beta = 1 ) nu = pm . Exponential ( 'nu_minus_two' , 1. / 10. ) X = data_bmark . iloc [ : , 1 ] y = data_bmark . iloc [ : , 0 ] alpha_reg = pm . Normal ( 'alpha' , mu = 0 , sd = .1 ) beta_reg = pm . Normal ( 'beta' , mu = 0 , sd = 1 ) mu_reg = alpha_reg + beta_reg * X pm . StudentT ( 'returns' , nu = nu + 2 , mu = mu_reg , sd = sigma , observed = y ) trace = pm . sample ( samples , progressbar = progressbar ) return model , trace

Run Bayesian alpha - beta - model with T distributed returns .


def model_returns_normal(data, samples=500, progressbar=True):
    """
    Run Bayesian model assuming returns are normally distributed.

    Parameters
    ----------
    returns : pandas.Series
        Series of simple returns of an algorithm or stock.
    samples : int (optional)
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    with pm.Model() as model:
        mu = pm.Normal('mean returns', mu=0, sd=.01, testval=data.mean())
        sigma = pm.HalfCauchy('volatility', beta=1, testval=data.std())
        returns = pm.Normal('returns', mu=mu, sd=sigma, observed=data)
        pm.Deterministic(
            'annual volatility',
            returns.distribution.variance**.5 *
            np.sqrt(252))
        pm.Deterministic(
            'sharpe',
            returns.distribution.mean /
            returns.distribution.variance**.5 *
            np.sqrt(252))

        trace = pm.sample(samples, progressbar=progressbar)
    return model, trace

def
model_returns_normal
(
data
,
samples
=
500
,
progressbar
=
True
)
:
with
pm
.
Model
(
)
as
model
:
mu
=
pm
.
Normal
(
'mean returns'
,
mu
=
0
,
sd
=
.01
,
testval
=
data
.
mean
(
)
)
sigma
=
pm
.
HalfCauchy
(
'volatility'
,
beta
=
1
,
testval
=
data
.
std
(
)
)
returns
=
pm
.
Normal
(
'returns'
,
mu
=
mu
,
sd
=
sigma
,
observed
=
data
)
pm
.
Deterministic
(
'annual volatility'
,
returns
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'sharpe'
,
returns
.
distribution
.
mean
/
returns
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
trace
=
pm
.
sample
(
samples
,
progressbar
=
progressbar
)
return
model
,
trace

Run Bayesian model assuming returns are normally distributed .


def model_best(y1, y2, samples=1000, progressbar=True):
    """
    Bayesian Estimation Supersedes the T-Test

    This model runs a Bayesian hypothesis comparing if y1 and y2 come
    from the same distribution. Returns are assumed to be T-distributed.

    In addition, computes annual volatility and Sharpe of in and
    out-of-sample periods.

    This model replicates the example used in:
    Kruschke, John. (2012) Bayesian estimation supersedes the t
    test. Journal of Experimental Psychology: General.

    Parameters
    ----------
    y1 : array-like
        Array of returns (e.g. in-sample)
    y2 : array-like
        Array of returns (e.g. out-of-sample)
    samples : int, optional
        Number of posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    See Also
    --------
    plot_stoch_vol : plotting of tochastic volatility model
    """

    y = np.concatenate((y1, y2))

    mu_m = np.mean(y)
    mu_p = 0.000001 * 1 / np.std(y)**2

    sigma_low = np.std(y) / 1000
    sigma_high = np.std(y) * 1000
    with pm.Model() as model:
        group1_mean = pm.Normal('group1_mean', mu=mu_m, tau=mu_p,
                                testval=y1.mean())
        group2_mean = pm.Normal('group2_mean', mu=mu_m, tau=mu_p,
                                testval=y2.mean())
        group1_std = pm.Uniform('group1_std', lower=sigma_low,
                                upper=sigma_high, testval=y1.std())
        group2_std = pm.Uniform('group2_std', lower=sigma_low,
                                upper=sigma_high, testval=y2.std())
        nu = pm.Exponential('nu_minus_two', 1 / 29., testval=4.) + 2.

        returns_group1 = pm.StudentT('group1', nu=nu, mu=group1_mean,
                                     lam=group1_std**-2, observed=y1)
        returns_group2 = pm.StudentT('group2', nu=nu, mu=group2_mean,
                                     lam=group2_std**-2, observed=y2)

        diff_of_means = pm.Deterministic('difference of means',
                                         group2_mean - group1_mean)
        pm.Deterministic('difference of stds',
                         group2_std - group1_std)
        pm.Deterministic('effect size', diff_of_means /
                         pm.math.sqrt((group1_std**2 +
                                       group2_std**2) / 2))

        pm.Deterministic('group1_annual_volatility',
                         returns_group1.distribution.variance**.5 *
                         np.sqrt(252))
        pm.Deterministic('group2_annual_volatility',
                         returns_group2.distribution.variance**.5 *
                         np.sqrt(252))

        pm.Deterministic('group1_sharpe', returns_group1.distribution.mean /
                         returns_group1.distribution.variance**.5 *
                         np.sqrt(252))
        pm.Deterministic('group2_sharpe', returns_group2.distribution.mean /
                         returns_group2.distribution.variance**.5 *
                         np.sqrt(252))

        trace = pm.sample(samples, progressbar=progressbar)
    return model, trace

def
model_best
(
y1
,
y2
,
samples
=
1000
,
progressbar
=
True
)
:
y
=
np
.
concatenate
(
(
y1
,
y2
)
)
mu_m
=
np
.
mean
(
y
)
mu_p
=
0.000001
*
1
/
np
.
std
(
y
)
**
2
sigma_low
=
np
.
std
(
y
)
/
1000
sigma_high
=
np
.
std
(
y
)
*
1000
with
pm
.
Model
(
)
as
model
:
group1_mean
=
pm
.
Normal
(
'group1_mean'
,
mu
=
mu_m
,
tau
=
mu_p
,
testval
=
y1
.
mean
(
)
)
group2_mean
=
pm
.
Normal
(
'group2_mean'
,
mu
=
mu_m
,
tau
=
mu_p
,
testval
=
y2
.
mean
(
)
)
group1_std
=
pm
.
Uniform
(
'group1_std'
,
lower
=
sigma_low
,
upper
=
sigma_high
,
testval
=
y1
.
std
(
)
)
group2_std
=
pm
.
Uniform
(
'group2_std'
,
lower
=
sigma_low
,
upper
=
sigma_high
,
testval
=
y2
.
std
(
)
)
nu
=
pm
.
Exponential
(
'nu_minus_two'
,
1
/
29.
,
testval
=
4.
)
+
2.
returns_group1
=
pm
.
StudentT
(
'group1'
,
nu
=
nu
,
mu
=
group1_mean
,
lam
=
group1_std
**
2
,
observed
=
y1
)
returns_group2
=
pm
.
StudentT
(
'group2'
,
nu
=
nu
,
mu
=
group2_mean
,
lam
=
group2_std
**
2
,
observed
=
y2
)
diff_of_means
=
pm
.
Deterministic
(
'difference of means'
,
group2_mean
group1_mean
)
pm
.
Deterministic
(
'difference of stds'
,
group2_std
group1_std
)
pm
.
Deterministic
(
'effect size'
,
diff_of_means
/
pm
.
math
.
sqrt
(
(
group1_std
**
2
+
group2_std
**
2
)
/
2
)
)
pm
.
Deterministic
(
'group1_annual_volatility'
,
returns_group1
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group2_annual_volatility'
,
returns_group2
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group1_sharpe'
,
returns_group1
.
distribution
.
mean
/
returns_group1
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
pm
.
Deterministic
(
'group2_sharpe'
,
returns_group2
.
distribution
.
mean
/
returns_group2
.
distribution
.
variance
**
.5
*
np
.
sqrt
(
252
)
)
trace
=
pm
.
sample
(
samples
,
progressbar
=
progressbar
)
return
model
,
trace

Bayesian Estimation Supersedes the T - Test


def model_stoch_vol(data, samples=2000, progressbar=True):
    """
    Run stochastic volatility model.

    This model estimates the volatility of a returns series over time.
    Returns are assumed to be T-distributed. lambda (width of
    T-distributed) is assumed to follow a random-walk.

    Parameters
    ----------
    data : pandas.Series
        Return series to model.
    samples : int, optional
        Posterior samples to draw.

    Returns
    -------
    model : pymc.Model object
        PyMC3 model containing all random variables.
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    See Also
    --------
    plot_stoch_vol : plotting of tochastic volatility model
    """

    from pymc3.distributions.timeseries import GaussianRandomWalk

    with pm.Model() as model:
        nu = pm.Exponential('nu', 1. / 10, testval=5.)
        sigma = pm.Exponential('sigma', 1. / .02, testval=.1)
        s = GaussianRandomWalk('s', sigma**-2, shape=len(data))
        volatility_process = pm.Deterministic('volatility_process',
                                              pm.math.exp(-2 * s))
        pm.StudentT('r', nu, lam=volatility_process, observed=data)

        trace = pm.sample(samples, progressbar=progressbar)

    return model, trace

def model_stoch_vol ( data , samples = 2000 , progressbar = True ) : from pymc3 . distributions . timeseries import GaussianRandomWalk with pm . Model ( ) as model : nu = pm . Exponential ( 'nu' , 1. / 10 , testval = 5. ) sigma = pm . Exponential ( 'sigma' , 1. / .02 , testval = .1 ) s = GaussianRandomWalk ( 's' , sigma ** - 2 , shape = len ( data ) ) volatility_process = pm . Deterministic ( 'volatility_process' , pm . math . exp ( - 2 * s ) ) pm . StudentT ( 'r' , nu , lam = volatility_process , observed = data ) trace = pm . sample ( samples , progressbar = progressbar ) return model , trace


def plot_stoch_vol(data, trace=None, ax=None):
    """
    Generate plot for stochastic volatility model.

    Parameters
    ----------
    data : pandas.Series
        Returns to model.
    trace : pymc3.sampling.BaseTrace object, optional
        trace as returned by model_stoch_vol
        If not passed, sample from model.
    ax : matplotlib.axes object, optional
        Plot into axes object

    Returns
    -------
    ax object

    See Also
    --------
    model_stoch_vol : run stochastic volatility model
    """

    if trace is None:
        trace = model_stoch_vol(data)

    if ax is None:
        fig, ax = plt.subplots(figsize=(15, 8))

    data.abs().plot(ax=ax)
    ax.plot(data.index, np.exp(trace['s', ::30].T), 'r', alpha=.03)
    ax.set(title='Stochastic volatility', xlabel='Time', ylabel='Volatility')
    ax.legend(['Abs returns', 'Stochastic volatility process'],
              frameon=True, framealpha=0.5)

    return ax

def
plot_stoch_vol
(
data
,
trace
=
,
ax
=
)
:
if
trace
is
:
trace
=
model_stoch_vol
(
data
)
if
ax
is
:
fig
,
ax
=
plt
.
subplots
(
figsize
=
(
15
,
8
)
)
data
.
abs
(
)
.
plot
(
ax
=
ax
)
ax
.
plot
(
data
.
index
,
np
.
exp
(
trace
[
's'
,
:
:
30
]
.
T
)
,
'r'
,
alpha
=
.03
)
ax
.
set
(
title
=
'Stochastic volatility'
,
xlabel
=
'Time'
,
ylabel
=
'Volatility'
)
ax
.
legend
(
[
'Abs returns'
,
'Stochastic volatility process'
]
,
frameon
=
True
,
framealpha
=
0.5
)
return
ax

Generate plot for stochastic volatility model .


def compute_bayes_cone(preds, starting_value=1.):
    """
    Compute 5, 25, 75 and 95 percentiles of cumulative returns, used
    for the Bayesian cone.

    Parameters
    ----------
    preds : numpy.array
        Multiple (simulated) cumulative returns.
    starting_value : int (optional)
        Have cumulative returns start around this value.
        Default = 1.

    Returns
    -------
    dict of percentiles over time
        Dictionary mapping percentiles (5, 25, 75, 95) to a
        timeseries.
    """

    def scoreatpercentile(cum_preds, p):
        return [stats.scoreatpercentile(
            c, p) for c in cum_preds.T]

    cum_preds = np.cumprod(preds + 1, 1) * starting_value
    perc = {p: scoreatpercentile(cum_preds, p) for p in (5, 25, 75, 95)}

    return perc

def compute_bayes_cone ( preds , starting_value = 1. ) : def scoreatpercentile ( cum_preds , p ) : return [ stats . scoreatpercentile ( c , p ) for c in cum_preds . T ] cum_preds = np . cumprod ( preds + 1 , 1 ) * starting_value perc = { p : scoreatpercentile ( cum_preds , p ) for p in ( 5 , 25 , 75 , 95 ) } return perc

Compute 5 25 75 and 95 percentiles of cumulative returns used for the Bayesian cone .


def compute_consistency_score(returns_test, preds):
    """
    Compute Bayesian consistency score.

    Parameters
    ----------
    returns_test : pd.Series
        Observed cumulative returns.
    preds : numpy.array
        Multiple (simulated) cumulative returns.

    Returns
    -------
    Consistency score
        Score from 100 (returns_test perfectly on the median line of the
        Bayesian cone spanned by preds) to 0 (returns_test completely
        outside of Bayesian cone.)
    """

    returns_test_cum = cum_returns(returns_test, starting_value=1.)
    cum_preds = np.cumprod(preds + 1, 1)

    q = [sp.stats.percentileofscore(cum_preds[:, i],
                                    returns_test_cum.iloc[i],
                                    kind='weak')
         for i in range(len(returns_test_cum))]
    # normalize to be from 100 (perfect median line) to 0 (completely outside
    # of cone)
    return 100 - np.abs(50 - np.mean(q)) / .5

def compute_consistency_score ( returns_test , preds ) : returns_test_cum = cum_returns ( returns_test , starting_value = 1. ) cum_preds = np . cumprod ( preds + 1 , 1 ) q = [ sp . stats . percentileofscore ( cum_preds [ : , i ] , returns_test_cum . iloc [ i ] , kind = 'weak' ) for i in range ( len ( returns_test_cum ) ) ] return 100 - np . abs ( 50 - np . mean ( q ) ) / .5


def run_model(model, returns_train, returns_test=None,
              bmark=None, samples=500, ppc=False, progressbar=True):
    """
    Run one of the Bayesian models.

    Parameters
    ----------
    model : {'alpha_beta', 't', 'normal', 'best'}
        Which model to run
    returns_train : pd.Series
        Timeseries of simple returns
    returns_test : pd.Series (optional)
        Out-of-sample returns. Datetimes in returns_test will be added to
        returns_train as missing values and predictions will be generated
        for them.
    bmark : pd.Series or pd.DataFrame (optional)
        Only used for alpha_beta to estimate regression coefficients.
        If bmark has more recent returns than returns_train, these dates
        will be treated as missing values and predictions will be
        generated for them taking market correlations into account.
    samples : int (optional)
        Number of posterior samples to draw.
    ppc : boolean (optional)
        Whether to run a posterior predictive check. Will generate
        samples of length returns_test.  Returns a second argument
        that contains the PPC of shape samples x len(returns_test).

    Returns
    -------
    trace : pymc3.sampling.BaseTrace object
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.

    ppc : numpy.array (if ppc==True)
       PPC of shape samples x len(returns_test).
    """

    if model == 'alpha_beta':
        model, trace = model_returns_t_alpha_beta(returns_train,
                                                  bmark, samples,
                                                  progressbar=progressbar)
    elif model == 't':
        model, trace = model_returns_t(returns_train, samples,
                                       progressbar=progressbar)
    elif model == 'normal':
        model, trace = model_returns_normal(returns_train, samples,
                                            progressbar=progressbar)
    elif model == 'best':
        model, trace = model_best(returns_train, returns_test,
                                  samples=samples,
                                  progressbar=progressbar)
    else:
        raise NotImplementedError(
            'Model {} not found.'
            'Use alpha_beta, t, normal, or best.'.format(model))

    if ppc:
        ppc_samples = pm.sample_ppc(trace, samples=samples,
                                    model=model, size=len(returns_test),
                                    progressbar=progressbar)
        return trace, ppc_samples['returns']

    return trace

def
run_model
(
model
,
returns_train
,
returns_test
=
,
bmark
=
,
samples
=
500
,
ppc
=
False
,
progressbar
=
True
)
:
if
model
==
'alpha_beta'
:
model
,
trace
=
model_returns_t_alpha_beta
(
returns_train
,
bmark
,
samples
,
progressbar
=
progressbar
)
elif
model
==
't'
:
model
,
trace
=
model_returns_t
(
returns_train
,
samples
,
progressbar
=
progressbar
)
elif
model
==
'normal'
:
model
,
trace
=
model_returns_normal
(
returns_train
,
samples
,
progressbar
=
progressbar
)
elif
model
==
'best'
:
model
,
trace
=
model_best
(
returns_train
,
returns_test
,
samples
=
samples
,
progressbar
=
progressbar
)
else
:
raise
NotImplementedError
(
'Model {} not found.'
'Use alpha_beta, t, normal, or best.'
.
format
(
model
)
)
if
ppc
:
ppc_samples
=
pm
.
sample_ppc
(
trace
,
samples
=
samples
,
model
=
model
,
size
=
len
(
returns_test
)
,
progressbar
=
progressbar
)
return
trace
,
ppc_samples
[
'returns'
]
return
trace


def plot_bayes_cone(returns_train, returns_test, ppc,
                    plot_train_len=50, ax=None):
    """
    Generate cumulative returns plot with Bayesian cone.

    Parameters
    ----------
    returns_train : pd.Series
        Timeseries of simple returns
    returns_test : pd.Series
        Out-of-sample returns. Datetimes in returns_test will be added to
        returns_train as missing values and predictions will be generated
        for them.
    ppc : np.array
        Posterior predictive samples of shape samples x
        len(returns_test).
    plot_train_len : int (optional)
        How many data points to plot of returns_train. Useful to zoom in on
        the prediction if there is a long backtest period.
    ax : matplotlib.Axis (optional)
        Axes upon which to plot.

    Returns
    -------
    score : float
        Consistency score (see compute_consistency_score)
    trace : pymc3.sampling.BaseTrace
        A PyMC3 trace object that contains samples for each parameter
        of the posterior.
    """

    score = compute_consistency_score(returns_test,
                                      ppc)

    ax = _plot_bayes_cone(
        returns_train,
        returns_test,
        ppc,
        plot_train_len=plot_train_len,
        ax=ax)
    ax.text(
        0.40,
        0.90,
        'Consistency score: %.1f' %
        score,
        verticalalignment='bottom',
        horizontalalignment='right',
        transform=ax.transAxes,
    )

    ax.set_ylabel('Cumulative returns')
    return score

def
plot_bayes_cone
(
returns_train
,
returns_test
,
ppc
,
plot_train_len
=
50
,
ax
=
)
:
score
=
compute_consistency_score
(
returns_test
,
ppc
)
ax
=
_plot_bayes_cone
(
returns_train
,
returns_test
,
ppc
,
plot_train_len
=
plot_train_len
,
ax
=
ax
)
ax
.
text
(
0.40
,
0.90
,
'Consistency score: %.1f'
%
score
,
verticalalignment
=
'bottom'
,
horizontalalignment
=
'right'
,
transform
=
ax
.
transAxes
,
)
ax
.
set_ylabel
(
'Cumulative returns'
)
return
score

Generate cumulative returns plot with Bayesian cone .


def _GetNextLogCountPerToken(token):
    """Wrapper for _log_counter_per_token.

    Args:
    token: The token for which to look up the count.

    Returns:
    The number of times this function has been called with
    *token* as an argument (starting at 0)
    """
    global _log_counter_per_token  # pylint: disable=global-variable-not-assigned
    _log_counter_per_token[token] = 1 + _log_counter_per_token.get(token, -1)
    return _log_counter_per_token[token]

def _GetNextLogCountPerToken ( token ) : global _log_counter_per_token _log_counter_per_token [ token ] = 1 + _log_counter_per_token . get ( token , - 1 ) return _log_counter_per_token [ token ]


def log_every_n(level, msg, n, *args):
    """Log 'msg % args' at level 'level' once per 'n' times.

    Logs the 1st call, (N+1)st call, (2N+1)st call,  etc.
    Not threadsafe.

    Args:
    level: The level at which to log.
    msg: The message to be logged.
    n: The number of times this should be called before it is logged.
    *args: The args to be substituted into the msg.
    """
    count = _GetNextLogCountPerToken(_GetFileAndLine())
    log_if(level, msg, not (count % n), *args)

def log_every_n ( level , msg , n , * args ) : count = _GetNextLogCountPerToken ( _GetFileAndLine ( ) ) log_if ( level , msg , not ( count % n ) , * args )

Log msg % args at level level once per n times .


def log_if(level, msg, condition, *args):
    """Log 'msg % args' at level 'level' only if condition is fulfilled."""
    if condition:
        vlog(level, msg, *args)

def log_if ( level , msg , condition , * args ) : if condition : vlog ( level , msg , * args )

Log msg % args at level level only if condition is fulfilled .


def google2_log_prefix(level, timestamp=None, file_and_line=None):
    """Assemble a logline prefix using the google2 format."""
    # pylint: disable=global-variable-not-assigned
    global _level_names
    # pylint: enable=global-variable-not-assigned

    # Record current time
    now = timestamp or _time.time()
    now_tuple = _time.localtime(now)
    now_microsecond = int(1e6 * (now % 1.0))

    (filename, line) = file_and_line or _GetFileAndLine()
    basename = _os.path.basename(filename)

    # Severity string
    severity = 'I'
    if level in _level_names:
        severity = _level_names[level][0]

    s = '%c%02d%02d %02d: %02d: %02d.%06d %5d %s: %d] ' % (
        severity,
        now_tuple[1],  # month
        now_tuple[2],  # day
        now_tuple[3],  # hour
        now_tuple[4],  # min
        now_tuple[5],  # sec
        now_microsecond,
        _get_thread_id(),
        basename,
        line
    )

    return s

Assemble a logline prefix using the google2 format .


def create_distributed_session(
        task_spec=None, checkpoint_dir=None, scaffold=None, hooks=None, chief_only_hooks=None, save_checkpoint_secs=600,
        save_summaries_steps=object(), save_summaries_secs=object(), config=None, stop_grace_period_secs=120,
        log_step_count_steps=100
):
    """Creates a distributed session.

    It calls `MonitoredTrainingSession` to create a :class:`MonitoredSession` for distributed training.

    Parameters
    ----------
    task_spec : :class:`TaskSpecDef`.
        The task spec definition from create_task_spec_def()
    checkpoint_dir : str.
        Optional path to a directory where to restore variables.
    scaffold : ``Scaffold``
        A `Scaffold` used for gathering or building supportive ops.
        If not specified, a default one is created. It's used to finalize the graph.
    hooks : list of ``SessionRunHook`` objects.
        Optional
    chief_only_hooks : list of ``SessionRunHook`` objects.
        Activate these hooks if `is_chief==True`, ignore otherwise.
    save_checkpoint_secs : int
        The frequency, in seconds, that a checkpoint is saved
        using a default checkpoint saver. If `save_checkpoint_secs` is set to
        `None`, then the default checkpoint saver isn't used.
    save_summaries_steps : int
        The frequency, in number of global steps, that the
        summaries are written to disk using a default summary saver. If both
        `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
        the default summary saver isn't used. Default 100.
    save_summaries_secs : int
        The frequency, in secs, that the summaries are written
        to disk using a default summary saver.  If both `save_summaries_steps` and
        `save_summaries_secs` are set to `None`, then the default summary saver
        isn't used. Default not enabled.
    config : ``tf.ConfigProto``
        an instance of `tf.ConfigProto` proto used to configure the session.
        It's the `config` argument of constructor of `tf.Session`.
    stop_grace_period_secs : int
        Number of seconds given to threads to stop after
        `close()` has been called.
    log_step_count_steps : int
        The frequency, in number of global steps, that the
        global step/sec is logged.

    Examples
    --------
    A simple example for distributed training where all the workers use the same dataset:

    >>> task_spec = TaskSpec()
    >>> with tf.device(task_spec.device_fn()):
    >>>      tensors = create_graph()
    >>> with tl.DistributedSession(task_spec=task_spec,
    ...                            checkpoint_dir='/tmp/ckpt') as session:
    >>>      while not session.should_stop():
    >>>           session.run(tensors)

    An example where the dataset is shared among the workers
    (see https://www.tensorflow.org/programmers_guide/datasets):

    >>> task_spec = TaskSpec()
    >>> # dataset is a :class:`tf.data.Dataset` with the raw data
    >>> dataset = create_dataset()
    >>> if task_spec is not None:
    >>>     dataset = dataset.shard(task_spec.num_workers, task_spec.shard_index)
    >>> # shuffle or apply a map function to the new sharded dataset, for example:
    >>> dataset = dataset.shuffle(buffer_size=10000)
    >>> dataset = dataset.batch(batch_size)
    >>> dataset = dataset.repeat(num_epochs)
    >>> # create the iterator for the dataset and the input tensor
    >>> iterator = dataset.make_one_shot_iterator()
    >>> next_element = iterator.get_next()
    >>> with tf.device(task_spec.device_fn()):
    >>>      # next_element is the input for the graph
    >>>      tensors = create_graph(next_element)
    >>> with tl.DistributedSession(task_spec=task_spec,
    ...                            checkpoint_dir='/tmp/ckpt') as session:
    >>>      while not session.should_stop():
    >>>           session.run(tensors)

    References
    ----------
    - `MonitoredTrainingSession <https://www.tensorflow.org/api_docs/python/tf/train/MonitoredTrainingSession>`__

    """
    target = task_spec.target() if task_spec is not None else None
    is_chief = task_spec.is_master() if task_spec is not None else True
    return tf.train.MonitoredTrainingSession(
        master=target, is_chief=is_chief, checkpoint_dir=checkpoint_dir, scaffold=scaffold,
        save_checkpoint_secs=save_checkpoint_secs, save_summaries_steps=save_summaries_steps,
        save_summaries_secs=save_summaries_secs, log_step_count_steps=log_step_count_steps,
        stop_grace_period_secs=stop_grace_period_secs, config=config, hooks=hooks, chief_only_hooks=chief_only_hooks
    )

def create_distributed_session ( task_spec = None , checkpoint_dir = None , scaffold = None , hooks = None , chief_only_hooks = None , save_checkpoint_secs = 600 , save_summaries_steps = object ( ) , save_summaries_secs = object ( ) , config = None , stop_grace_period_secs = 120 , log_step_count_steps = 100 ) : target = task_spec . target ( ) if task_spec is not None else None is_chief = task_spec . is_master ( ) if task_spec is not None else True return tf . train . MonitoredTrainingSession ( master = target , is_chief = is_chief , checkpoint_dir = checkpoint_dir , scaffold = scaffold , save_checkpoint_secs = save_checkpoint_secs , save_summaries_steps = save_summaries_steps , save_summaries_secs = save_summaries_secs , log_step_count_steps = log_step_count_steps , stop_grace_period_secs = stop_grace_period_secs , config = config , hooks = hooks , chief_only_hooks = chief_only_hooks )


def validation_metrics(self):
        """A helper function to compute validation related metrics"""

        if (self._validation_iterator is None) or (self._validation_metrics is None):
            raise AttributeError('Validation is not setup.')

        n = 0.0
        metric_sums = [0.0] * len(self._validation_metrics)
        self._sess.run(self._validation_iterator.initializer)
        while True:
            try:
                metrics = self._sess.run(self._validation_metrics)
                for i, m in enumerate(metrics):
                    metric_sums[i] += m
                n += 1.0
            except tf.errors.OutOfRangeError:
                break
        for i, m in enumerate(metric_sums):
            metric_sums[i] = metric_sums[i] / n
        return zip(self._validation_metrics, metric_sums)

def
validation_metrics
(
self
)
:
if
(
self
.
_validation_iterator
is
)
or
(
self
.
_validation_metrics
is
)
:
raise
AttributeError
(
'Validation is not setup.'
)
n
=
0.0
metric_sums
=
[
0.0
]
*
len
(
self
.
_validation_metrics
)
self
.
_sess
.
run
(
self
.
_validation_iterator
.
initializer
)
while
True
:
try
:
metrics
=
self
.
_sess
.
run
(
self
.
_validation_metrics
)
for
i
,
m
in
enumerate
(
metrics
)
:
metric_sums
[
i
]
+=
m
n
+=
1.0
except
tf
.
errors
.
OutOfRangeError
:
break
for
i
,
m
in
enumerate
(
metric_sums
)
:
metric_sums
[
i
]
=
metric_sums
[
i
]
/
n
return
zip
(
self
.
_validation_metrics
,
metric_sums
)


def train_and_validate_to_end(self, validate_step_size=50):
        """A helper function that shows how to train and validate a model at the same time.

        Parameters
        ----------
        validate_step_size : int
            Validate the training network every N steps.

        """
        while not self._sess.should_stop():
            self.train_on_batch()  # Run a training step synchronously.
            if self.global_step % validate_step_size == 0:
                # logging.info("Average loss for validation dataset: %s" % self.get_validation_metrics())
                log_str = 'step: %d, ' % self.global_step
                for n, m in self.validation_metrics:
                    log_str += '%s: %f, ' % (n.name, m)
                logging.info(log_str)

def
train_and_validate_to_end
(
self
,
validate_step_size
=
50
)
:
while
not
self
.
_sess
.
should_stop
(
)
:
self
.
train_on_batch
(
)
if
self
.
global_step
%
validate_step_size
==
0
:
log_str
=
'step: %d, '
%
self
.
global_step
for
n
,
m
in
self
.
validation_metrics
:
log_str
+=
'%s: %f, '
%
(
n
.
name
,
m
)
logging
.
info
(
log_str
)

A helper function that shows how to train and validate a model at the same time .


def _load_mnist_dataset(shape, path, name='mnist', url='http://yann.lecun.com/exdb/mnist/'):
    """A generic function to load mnist-like dataset.

    Parameters:
    ----------
    shape : tuple
        The shape of digit images.
    path : str
        The path that the data is downloaded to.
    name : str
        The dataset name you want to use(the default is 'mnist').
    url : str
        The url of dataset(the default is 'http://yann.lecun.com/exdb/mnist/').
    """
    path = os.path.join(path, name)

    # Define functions for loading mnist-like data's images and labels.
    # For convenience, they also download the requested files if needed.
    def load_mnist_images(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)

        logging.info(filepath)
        # Read the inputs in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=16)
        # The inputs are vectors now, we reshape them to monochrome 2D images,
        # following the shape convention: (examples, channels, rows, columns)
        data = data.reshape(shape)
        # The inputs come as bytes, we convert them to float32 in range [0,1].
        # (Actually to range [0, 255/256], for compatibility to the version
        # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
        return data / np.float32(256)

    def load_mnist_labels(path, filename):
        filepath = maybe_download_and_extract(filename, path, url)
        # Read the labels in Yann LeCun's binary format.
        with gzip.open(filepath, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=8)
        # The labels are vectors of integers now, that's exactly what we want.
        return data

    # Download and read the training and test set images and labels.
    logging.info("Load or Download {0} > {1}".format(name.upper(), path))
    X_train = load_mnist_images(path, 'train-images-idx3-ubyte.gz')
    y_train = load_mnist_labels(path, 'train-labels-idx1-ubyte.gz')
    X_test = load_mnist_images(path, 't10k-images-idx3-ubyte.gz')
    y_test = load_mnist_labels(path, 't10k-labels-idx1-ubyte.gz')

    # We reserve the last 10000 training examples for validation.
    X_train, X_val = X_train[:-10000], X_train[-10000:]
    y_train, y_val = y_train[:-10000], y_train[-10000:]

    # We just return all the arrays in order, as expected in main().
    # (It doesn't matter how we do this as long as we can read them again.)
    X_train = np.asarray(X_train, dtype=np.float32)
    y_train = np.asarray(y_train, dtype=np.int32)
    X_val = np.asarray(X_val, dtype=np.float32)
    y_val = np.asarray(y_val, dtype=np.int32)
    X_test = np.asarray(X_test, dtype=np.float32)
    y_test = np.asarray(y_test, dtype=np.int32)
    return X_train, y_train, X_val, y_val, X_test, y_test

def
_load_mnist_dataset
(
shape
,
path
,
name
=
'mnist'
,
url
=
'http://yann.lecun.com/exdb/mnist/'
)
:
path
=
os
.
path
.
join
(
path
,
name
)
def
load_mnist_images
(
path
,
filename
)
:
filepath
=
maybe_download_and_extract
(
filename
,
path
,
url
)
logging
.
info
(
filepath
)
with
gzip
.
open
(
filepath
,
'rb'
)
as
f
:
data
=
np
.
frombuffer
(
f
.
read
(
)
,
np
.
uint8
,
offset
=
16
)
data
=
data
.
reshape
(
shape
)
return
data
/
np
.
float32
(
256
)
def
load_mnist_labels
(
path
,
filename
)
:
filepath
=
maybe_download_and_extract
(
filename
,
path
,
url
)
with
gzip
.
open
(
filepath
,
'rb'
)
as
f
:
data
=
np
.
frombuffer
(
f
.
read
(
)
,
np
.
uint8
,
offset
=
8
)
return
data
logging
.
info
(
"Load or Download {0} > {1}"
.
format
(
name
.
upper
(
)
,
path
)
)
X_train
=
load_mnist_images
(
path
,
'train-images-idx3-ubyte.gz'
)
y_train
=
load_mnist_labels
(
path
,
'train-labels-idx1-ubyte.gz'
)
X_test
=
load_mnist_images
(
path
,
't10k-images-idx3-ubyte.gz'
)
y_test
=
load_mnist_labels
(
path
,
't10k-labels-idx1-ubyte.gz'
)
X_train
,
X_val
=
X_train
[
:
10000
]
,
X_train
[
10000
:
]
y_train
,
y_val
=
y_train
[
:
10000
]
,
y_train
[
10000
:
]
X_train
=
np
.
asarray
(
X_train
,
dtype
=
np
.
float32
)
y_train
=
np
.
asarray
(
y_train
,
dtype
=
np
.
int32
)
X_val
=
np
.
asarray
(
X_val
,
dtype
=
np
.
float32
)
y_val
=
np
.
asarray
(
y_val
,
dtype
=
np
.
int32
)
X_test
=
np
.
asarray
(
X_test
,
dtype
=
np
.
float32
)
y_test
=
np
.
asarray
(
y_test
,
dtype
=
np
.
int32
)
return
X_train
,
y_train
,
X_val
,
y_val
,
X_test
,
y_test

A generic function to load mnist - like dataset .


def load_matt_mahoney_text8_dataset(path='data'):
    """Load Matt Mahoney's dataset.

    Download a text file from Matt Mahoney's website
    if not present, and make sure it's the right size.
    Extract the first file enclosed in a zip file as a list of words.
    This dataset can be used for Word Embedding.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/mm_test8/``.

    Returns
    --------
    list of str
        The raw text data e.g. [.... 'their', 'families', 'who', 'were', 'expelled', 'from', 'jerusalem', ...]

    Examples
    --------
    >>> words = tl.files.load_matt_mahoney_text8_dataset()
    >>> print('Data size', len(words))

    """
    path = os.path.join(path, 'mm_test8')
    logging.info("Load or Download matt_mahoney_text8 Dataset> {}".format(path))

    filename = 'text8.zip'
    url = 'http://mattmahoney.net/dc/'
    maybe_download_and_extract(filename, path, url, expected_bytes=31344016)

    with zipfile.ZipFile(os.path.join(path, filename)) as f:
        word_list = f.read(f.namelist()[0]).split()
        for idx, _ in enumerate(word_list):
            word_list[idx] = word_list[idx].decode()
    return word_list


def load_imdb_dataset(
        path='data', nb_words=None, skip_top=0, maxlen=None, test_split=0.2, seed=113, start_char=1, oov_char=2,
        index_from=3
):
    """Load IMDB dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/imdb/``.
    nb_words : int
        Number of words to get.
    skip_top : int
        Top most frequent words to ignore (they will appear as oov_char value in the sequence data).
    maxlen : int
        Maximum sequence length. Any longer sequence will be truncated.
    seed : int
        Seed for reproducible data shuffling.
    start_char : int
        The start of a sequence will be marked with this character. Set to 1 because 0 is usually the padding character.
    oov_char : int
        Words that were cut out because of the num_words or skip_top limit will be replaced with this character.
    index_from : int
        Index actual words with this index and higher.

    Examples
    --------
    >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
    ...                                 nb_words=20000, test_split=0.2)
    >>> print('X_train.shape', X_train.shape)
    (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
    >>> print('y_train.shape', y_train.shape)
    (20000,)  [1 0 0 ..., 1 0 1]

    References
    -----------
    - `Modified from keras. <https://github.com/fchollet/keras/blob/master/keras/datasets/imdb.py>`__

    """
    path = os.path.join(path, 'imdb')

    filename = "imdb.pkl"
    url = 'https://s3.amazonaws.com/text-datasets/'
    maybe_download_and_extract(filename, path, url)

    if filename.endswith(".gz"):
        f = gzip.open(os.path.join(path, filename), 'rb')
    else:
        f = open(os.path.join(path, filename), 'rb')

    X, labels = cPickle.load(f)
    f.close()

    np.random.seed(seed)
    np.random.shuffle(X)
    np.random.seed(seed)
    np.random.shuffle(labels)

    if start_char is not None:
        X = [[start_char] + [w + index_from for w in x] for x in X]
    elif index_from:
        X = [[w + index_from for w in x] for x in X]

    if maxlen:
        new_X = []
        new_labels = []
        for x, y in zip(X, labels):
            if len(x) < maxlen:
                new_X.append(x)
                new_labels.append(y)
        X = new_X
        labels = new_labels
    if not X:
        raise Exception(
            'After filtering for sequences shorter than maxlen=' + str(maxlen) + ', no sequence was kept. '
            'Increase maxlen.'
        )
    if not nb_words:
        nb_words = max([max(x) for x in X])

    # by convention, use 2 as OOV word
    # reserve 'index_from' (=3 by default) characters: 0 (padding), 1 (start), 2 (OOV)
    if oov_char is not None:
        X = [[oov_char if (w >= nb_words or w < skip_top) else w for w in x] for x in X]
    else:
        nX = []
        for x in X:
            nx = []
            for w in x:
                if (w >= nb_words or w < skip_top):
                    nx.append(w)
            nX.append(nx)
        X = nX

    X_train = np.array(X[:int(len(X) * (1 - test_split))])
    y_train = np.array(labels[:int(len(X) * (1 - test_split))])

    X_test = np.array(X[int(len(X) * (1 - test_split)):])
    y_test = np.array(labels[int(len(X) * (1 - test_split)):])

    return X_train, y_train, X_test, y_test


def load_nietzsche_dataset(path='data'):
    """Load Nietzsche dataset.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/nietzsche/``.

    Returns
    --------
    str
        The content.

    Examples
    --------
    >>> see tutorial_generate_text.py
    >>> words = tl.files.load_nietzsche_dataset()
    >>> words = basic_clean_str(words)
    >>> words = words.split()

    """
    logging.info("Load or Download nietzsche dataset > {}".format(path))
    path = os.path.join(path, 'nietzsche')

    filename = "nietzsche.txt"
    url = 'https://s3.amazonaws.com/text-datasets/'
    filepath = maybe_download_and_extract(filename, path, url)

    with open(filepath, "r") as f:
        words = f.read()
        return words


def load_wmt_en_fr_dataset(path='data'):
    """Load WMT'15 English-to-French translation dataset.

    It will download the data from the WMT'15 Website (10^9-French-English corpus), and the 2013 news test from the same site as development set.
    Returns the directories of training data and test data.

    Parameters
    ----------
    path : str
        The path that the data is downloaded to, defaults is ``data/wmt_en_fr/``.

    References
    ----------
    - Code modified from /tensorflow/models/rnn/translation/data_utils.py

    Notes
    -----
    Usually, it will take a long time to download this dataset.

    """
    path = os.path.join(path, 'wmt_en_fr')
    # URLs for WMT data.
    _WMT_ENFR_TRAIN_URL = "http://www.statmt.org/wmt10/"
    _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/"

    def gunzip_file(gz_path, new_path):
        """Unzips from gz_path into new_path."""
        logging.info("Unpacking %s to %s" % (gz_path, new_path))
        with gzip.open(gz_path, "rb") as gz_file:
            with open(new_path, "wb") as new_file:
                for line in gz_file:
                    new_file.write(line)

    def get_wmt_enfr_train_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "training-giga-fren.tar"
        maybe_download_and_extract(filename, path, _WMT_ENFR_TRAIN_URL, extract=True)
        train_path = os.path.join(path, "giga-fren.release2.fixed")
        gunzip_file(train_path + ".fr.gz", train_path + ".fr")
        gunzip_file(train_path + ".en.gz", train_path + ".en")
        return train_path

    def get_wmt_enfr_dev_set(path):
        """Download the WMT en-fr training corpus to directory unless it's there."""
        filename = "dev-v2.tgz"
        dev_file = maybe_download_and_extract(filename, path, _WMT_ENFR_DEV_URL, extract=False)
        dev_name = "newstest2013"
        dev_path = os.path.join(path, "newstest2013")
        if not (gfile.Exists(dev_path + ".fr") and gfile.Exists(dev_path + ".en")):
            logging.info("Extracting tgz file %s" % dev_file)
            with tarfile.open(dev_file, "r:gz") as dev_tar:
                fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
                en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
                fr_dev_file.name = dev_name + ".fr"  # Extract without "dev/" prefix.
                en_dev_file.name = dev_name + ".en"
                dev_tar.extract(fr_dev_file, path)
                dev_tar.extract(en_dev_file, path)
        return dev_path

    logging.info("Load or Download WMT English-to-French translation > {}".format(path))

    train_path = get_wmt_enfr_train_set(path)
    dev_path = get_wmt_enfr_dev_set(path)

    return train_path, dev_path

Load WMT 15 English - to - French translation dataset .