CNN / TCN Models

Convolutional and temporal convolutional network architectures for time series.

CNN

CNN(input_size: int, output_size: int, hl_depth: int = 1, hl_width: int = 10, act: type[Module] = Mish, bn: bool = False)

Bases: Module

Simple stacked 1D CNN with a pointwise output projection.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
hl_depth	int	Number of hidden convolutional layers.	1
hl_width	int	Number of channels in each hidden layer.	10
act	type[Module]	Activation function class.	Mish
bn	bool	Whether to apply batch normalization.	False

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_size: int,
    output_size: int,
    hl_depth: int = 1,
    hl_width: int = 10,
    act: type[nn.Module] = Mish,
    bn: bool = False,
):
    super().__init__()

    conv_layers = [
        Conv1D(input_size if i == 0 else hl_width, hl_width, bn=bn, activation=act, padding=1)
        for i in range(hl_depth)
    ]
    self.conv_layers = nn.Sequential(*conv_layers)
    self.final = nn.Conv1d(hl_width, output_size, kernel_size=1)

CausalConv1d

CausalConv1d(in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, dilation: int = 1, groups: int = 1, bias: bool = True)

Bases: Conv1d

Causal 1D convolution that pads only on the left to prevent future leakage.

Parameters:

Name	Type	Description	Default
in_channels	int	Number of input channels.	required
out_channels	int	Number of output channels.	required
kernel_size	int	Size of the convolving kernel.	required
stride	int	Stride of the convolution.	1
dilation	int	Dilation factor for the kernel.	1
groups	int	Number of blocked connections from input to output channels.	1
bias	bool	Whether to add a learnable bias.	True

Source code in tsfast/models/cnn.py

def __init__(
    self,
    in_channels: int,
    out_channels: int,
    kernel_size: int,
    stride: int = 1,
    dilation: int = 1,
    groups: int = 1,
    bias: bool = True,
):
    super().__init__(
        in_channels,
        out_channels,
        kernel_size=kernel_size,
        stride=stride,
        padding=0,
        dilation=dilation,
        groups=groups,
        bias=bias,
    )
    self.__init_size = (kernel_size - 1) * dilation

TCN_Block

TCN_Block(input_size: int, output_size: int, num_layers: int = 1, activation: type[Module] | None = Mish, wn: bool = True, bn: bool = False, **kwargs)

Bases: Module

Single TCN residual block with stacked causal convolutions.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
num_layers	int	Number of causal convolution layers in the block.	1
activation	type[Module] | None	Activation function class, or None to disable.	Mish
wn	bool	Whether to apply weight normalization.	True
bn	bool	Whether to apply batch normalization.	False
**kwargs		Additional arguments passed to CausalConv1d.	{}

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_size: int,
    output_size: int,
    num_layers: int = 1,
    activation: type[nn.Module] | None = Mish,
    wn: bool = True,
    bn: bool = False,
    **kwargs,
):
    super().__init__()

    layers = []
    for _ in range(num_layers):
        conv = CausalConv1d(input_size, output_size, 2, **kwargs)
        if wn:
            conv = weight_norm(conv)
        act = activation() if activation is not None else None
        bn = nn.BatchNorm1d(input_size) if bn else None
        layers += [m for m in [bn, conv, act] if m is not None]

    self.layers = nn.Sequential(*layers)

    self.residual = nn.Conv1d(input_size, output_size, kernel_size=1) if output_size != input_size else None

TCN

TCN(input_size: int, output_size: int, hl_depth: int = 1, hl_width: int = 10, act: type[Module] = Mish, bn: bool = False)

Bases: Module

Temporal Convolutional Network with exponentially increasing dilation.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
hl_depth	int	Number of TCN blocks (each with doubled dilation).	1
hl_width	int	Number of channels in each hidden TCN block.	10
act	type[Module]	Activation function class.	Mish
bn	bool	Whether to apply batch normalization.	False

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_size: int,
    output_size: int,
    hl_depth: int = 1,
    hl_width: int = 10,
    act: type[nn.Module] = Mish,
    bn: bool = False,
):
    super().__init__()

    conv_layers = [
        TCN_Block(
            input_size if i == 0 else hl_width,
            hl_width,
            dilation=2 ** (i),
            bn=bn,
            activation=act,
        )
        for i in range(hl_depth)
    ]
    self.conv_layers = nn.Sequential(*conv_layers)
    self.final = nn.Conv1d(hl_width, output_size, kernel_size=1)

SeperateTCN

SeperateTCN(input_list: list[int], output_size: int, hl_depth: int = 1, hl_width: int = 10, act: type[Module] = Mish, bn: bool = False, final_layer: int = 3)

Bases: Module

TCN with separate convolutional branches per input group, merged by a linear head.

Parameters:

Name	Type	Description	Default
input_list	list[int]	List of channel counts, one per input group.	required
output_size	int	Number of output channels.	required
hl_depth	int	Number of TCN blocks per branch.	1
hl_width	int	Total hidden width split evenly across branches.	10
act	type[Module]	Activation function class.	Mish
bn	bool	Whether to apply batch normalization.	False
final_layer	int	Number of hidden layers in the final linear head.	3

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_list: list[int],
    output_size: int,
    hl_depth: int = 1,
    hl_width: int = 10,
    act: type[nn.Module] = Mish,
    bn: bool = False,
    final_layer: int = 3,
):
    super().__init__()
    self.input_list = np.cumsum([0] + input_list)

    tcn_width = hl_width // len(input_list)
    layers = [
        [
            TCN_Block(n if i == 0 else tcn_width, tcn_width, dilation=2 ** (i), bn=bn, activation=act)
            for i in range(hl_depth)
        ]
        for n in input_list
    ]
    self.layers = nn.ModuleList([nn.Sequential(*layer) for layer in layers])

    self.final = SeqLinear(tcn_width * len(input_list), output_size, hidden_size=hl_width, hidden_layer=final_layer)

CRNN

CRNN(input_size: int, output_size: int, num_ft: int = 10, num_cnn_layers: int = 4, num_rnn_layers: int = 2, hs_cnn: int = 10, hs_rnn: int = 10, hidden_p: float = 0, input_p: float = 0, weight_p: float = 0, rnn_type: str = 'gru')

Bases: Module

Convolutional-Recurrent Neural Network combining a TCN front-end with an RNN back-end.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
num_ft	int	Number of intermediate features between the CNN and RNN stages.	10
num_cnn_layers	int	Number of TCN blocks in the convolutional stage.	4
num_rnn_layers	int	Number of stacked RNN layers.	2
hs_cnn	int	Hidden channel width of the TCN stage.	10
hs_rnn	int	Hidden size of the RNN stage.	10
hidden_p	float	Dropout probability on RNN hidden-to-hidden connections.	0
input_p	float	Dropout probability on RNN inputs.	0
weight_p	float	Weight dropout probability for RNN parameters.	0
rnn_type	str	RNN cell type ("gru" or "lstm").	'gru'

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_size: int,
    output_size: int,
    num_ft: int = 10,
    num_cnn_layers: int = 4,
    num_rnn_layers: int = 2,
    hs_cnn: int = 10,
    hs_rnn: int = 10,
    hidden_p: float = 0,
    input_p: float = 0,
    weight_p: float = 0,
    rnn_type: str = "gru",
):
    super().__init__()
    self.cnn = TCN(input_size, num_ft, num_cnn_layers, hs_cnn, act=nn.ReLU)
    self.rnn = SimpleRNN(
        num_ft,
        output_size,
        num_layers=num_rnn_layers,
        hidden_size=hs_rnn,
        hidden_p=hidden_p,
        input_p=input_p,
        weight_p=weight_p,
        rnn_type=rnn_type,
    )

SeperateCRNN

SeperateCRNN(input_list: list[int], output_size: int, num_ft: int = 10, num_cnn_layers: int = 4, num_rnn_layers: int = 2, hs_cnn: int = 10, hs_rnn: int = 10, hidden_p: float = 0, input_p: float = 0, weight_p: float = 0, rnn_type: str = 'gru')

Bases: Module

CRNN with separate TCN branches per input group, merged before the RNN stage.

Parameters:

Name	Type	Description	Default
input_list	list[int]	List of channel counts, one per input group.	required
output_size	int	Number of output channels.	required
num_ft	int	Number of intermediate features between the CNN and RNN stages.	10
num_cnn_layers	int	Number of TCN blocks per branch in the convolutional stage.	4
num_rnn_layers	int	Number of stacked RNN layers.	2
hs_cnn	int	Hidden channel width of the TCN branches.	10
hs_rnn	int	Hidden size of the RNN stage.	10
hidden_p	float	Dropout probability on RNN hidden-to-hidden connections.	0
input_p	float	Dropout probability on RNN inputs.	0
weight_p	float	Weight dropout probability for RNN parameters.	0
rnn_type	str	RNN cell type ("gru" or "lstm").	'gru'

Source code in tsfast/models/cnn.py

def __init__(
    self,
    input_list: list[int],
    output_size: int,
    num_ft: int = 10,
    num_cnn_layers: int = 4,
    num_rnn_layers: int = 2,
    hs_cnn: int = 10,
    hs_rnn: int = 10,
    hidden_p: float = 0,
    input_p: float = 0,
    weight_p: float = 0,
    rnn_type: str = "gru",
):
    super().__init__()
    self.cnn = SeperateTCN(input_list, num_ft, num_cnn_layers, hs_cnn, act=nn.ReLU, final_layer=0)
    self.rnn = SimpleRNN(
        num_ft,
        output_size,
        num_layers=num_rnn_layers,
        hidden_size=hs_rnn,
        hidden_p=hidden_p,
        input_p=input_p,
        weight_p=weight_p,
        rnn_type=rnn_type,
    )

Conv1D

Conv1D(input_size: int, output_size: int, kernel_size: int = 3, activation: type[Module] | None = Mish, wn: bool = True, bn: bool = False, **kwargs) -> nn.Sequential

Create a 1D convolutional block with optional activation and batch norm.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
kernel_size	int	Size of the convolving kernel.	3
activation	type[Module] | None	Activation function class, or None to disable.	Mish
wn	bool	Whether to apply weight normalization.	True
bn	bool	Whether to apply batch normalization before the convolution.	False
**kwargs		Additional arguments passed to nn.Conv1d.	{}

Source code in tsfast/models/cnn.py

def Conv1D(
    input_size: int,
    output_size: int,
    kernel_size: int = 3,
    activation: type[nn.Module] | None = Mish,
    wn: bool = True,
    bn: bool = False,
    **kwargs,
) -> nn.Sequential:
    """Create a 1D convolutional block with optional activation and batch norm.

    Args:
        input_size: Number of input channels.
        output_size: Number of output channels.
        kernel_size: Size of the convolving kernel.
        activation: Activation function class, or None to disable.
        wn: Whether to apply weight normalization.
        bn: Whether to apply batch normalization before the convolution.
        **kwargs: Additional arguments passed to ``nn.Conv1d``.
    """
    conv = nn.Conv1d(input_size, output_size, kernel_size, **kwargs)
    act = activation() if activation is not None else None
    bn = nn.BatchNorm1d(input_size) if bn else None
    m = [m for m in [bn, conv, act] if m is not None]
    return nn.Sequential(*m)

CConv1D

CConv1D(input_size: int, output_size: int, kernel_size: int = 2, activation: type[Module] | None = Mish, wn: bool = True, bn: bool = False, **kwargs) -> nn.Sequential

Create a causal 1D convolutional block with optional weight norm and batch norm.

Parameters:

Name	Type	Description	Default
input_size	int	Number of input channels.	required
output_size	int	Number of output channels.	required
kernel_size	int	Size of the convolving kernel.	2
activation	type[Module] | None	Activation function class, or None to disable.	Mish
wn	bool	Whether to apply weight normalization.	True
bn	bool	Whether to apply batch normalization before the convolution.	False
**kwargs		Additional arguments passed to CausalConv1d.	{}

Source code in tsfast/models/cnn.py

def CConv1D(
    input_size: int,
    output_size: int,
    kernel_size: int = 2,
    activation: type[nn.Module] | None = Mish,
    wn: bool = True,
    bn: bool = False,
    **kwargs,
) -> nn.Sequential:
    """Create a causal 1D convolutional block with optional weight norm and batch norm.

    Args:
        input_size: Number of input channels.
        output_size: Number of output channels.
        kernel_size: Size of the convolving kernel.
        activation: Activation function class, or None to disable.
        wn: Whether to apply weight normalization.
        bn: Whether to apply batch normalization before the convolution.
        **kwargs: Additional arguments passed to ``CausalConv1d``.
    """
    conv = CausalConv1d(input_size, output_size, kernel_size, **kwargs)
    if wn:
        conv = weight_norm(conv)
    act = activation() if activation is not None else None
    bn = nn.BatchNorm1d(input_size) if bn else None
    m = [m for m in [bn, conv, act] if m is not None]
    return nn.Sequential(*m)

TCNLearner

TCNLearner(dls, num_layers: int = 3, hidden_size: int = 100, loss_func: Module = nn.L1Loss(), metrics: list | None = None, n_skip: int | None = None, opt_func: type = torch.optim.Adam, input_norm: type[Scaler] | None = StandardScaler, output_norm: type[Scaler] | None = None, **kwargs) -> Learner

Create a Learner with a TCN model.

Parameters:

Name	Type	Description	Default
dls		DataLoaders providing training and validation data.	required
num_layers	int	Number of TCN hidden layers (sets receptive field to 2**num_layers).	3
hidden_size	int	Number of channels in hidden TCN layers.	100
loss_func	Module	Loss function instance.	L1Loss()
metrics	list | None	List of metric functions.	None
n_skip	int | None	Number of initial time steps to skip in the loss (defaults to 2**num_layers).	None
opt_func	type	Optimizer constructor.	Adam
input_norm	type[Scaler] | None	Input normalization scaler class, or None to disable.	StandardScaler
output_norm	type[Scaler] | None	Output denormalization scaler class, or None to disable.	None
**kwargs		Additional arguments passed to TCN.	{}

Source code in tsfast/models/cnn.py

def TCNLearner(
    dls,
    num_layers: int = 3,
    hidden_size: int = 100,
    loss_func: nn.Module = nn.L1Loss(),
    metrics: list | None = None,
    n_skip: int | None = None,
    opt_func: type = torch.optim.Adam,
    input_norm: type[Scaler] | None = StandardScaler,
    output_norm: type[Scaler] | None = None,
    **kwargs,
) -> Learner:
    """Create a Learner with a TCN model.

    Args:
        dls: DataLoaders providing training and validation data.
        num_layers: Number of TCN hidden layers (sets receptive field to 2**num_layers).
        hidden_size: Number of channels in hidden TCN layers.
        loss_func: Loss function instance.
        metrics: List of metric functions.
        n_skip: Number of initial time steps to skip in the loss (defaults to 2**num_layers).
        opt_func: Optimizer constructor.
        input_norm: Input normalization scaler class, or None to disable.
        output_norm: Output denormalization scaler class, or None to disable.
        **kwargs: Additional arguments passed to ``TCN``.
    """
    if metrics is None:
        metrics = [fun_rmse]

    inp, out = get_io_size(dls)
    n_skip = 2**num_layers if n_skip is None else n_skip
    model = TCN(inp, out, num_layers, hidden_size, **kwargs)
    model = ScaledModel.from_dls(model, dls, input_norm, output_norm)

    return Learner(model, dls, loss_func=loss_func, opt_func=opt_func, metrics=metrics, n_skip=n_skip, lr=3e-3)

CRNNLearner

CRNNLearner(dls, loss_func: Module = nn.L1Loss(), metrics: list | None = None, n_skip: int = 0, opt_func: type = torch.optim.Adam, input_norm: type[Scaler] | None = StandardScaler, output_norm: type[Scaler] | None = None, **kwargs) -> Learner

Create a Learner with a CRNN model.

Parameters:

Name	Type	Description	Default
dls		DataLoaders providing training and validation data.	required
loss_func	Module	Loss function instance.	L1Loss()
metrics	list | None	List of metric functions.	None
n_skip	int	Number of initial time steps to skip in the loss.	0
opt_func	type	Optimizer constructor.	Adam
input_norm	type[Scaler] | None	Input normalization scaler class, or None to disable.	StandardScaler
output_norm	type[Scaler] | None	Output denormalization scaler class, or None to disable.	None
**kwargs		Additional arguments passed to CRNN.	{}

Source code in tsfast/models/cnn.py

def CRNNLearner(
    dls,
    loss_func: nn.Module = nn.L1Loss(),
    metrics: list | None = None,
    n_skip: int = 0,
    opt_func: type = torch.optim.Adam,
    input_norm: type[Scaler] | None = StandardScaler,
    output_norm: type[Scaler] | None = None,
    **kwargs,
) -> Learner:
    """Create a Learner with a CRNN model.

    Args:
        dls: DataLoaders providing training and validation data.
        loss_func: Loss function instance.
        metrics: List of metric functions.
        n_skip: Number of initial time steps to skip in the loss.
        opt_func: Optimizer constructor.
        input_norm: Input normalization scaler class, or None to disable.
        output_norm: Output denormalization scaler class, or None to disable.
        **kwargs: Additional arguments passed to ``CRNN``.
    """
    if metrics is None:
        metrics = [fun_rmse]

    inp, out = get_io_size(dls)
    model = CRNN(inp, out, **kwargs)
    model = ScaledModel.from_dls(model, dls, input_norm, output_norm)

    return Learner(model, dls, loss_func=loss_func, opt_func=opt_func, metrics=metrics, n_skip=n_skip, lr=3e-3)

AR_TCNLearner

AR_TCNLearner(dls, hl_depth: int = 3, alpha: float = 1, beta: float = 1, metrics: list | None = None, n_skip: int | None = None, opt_func: type = torch.optim.Adam, input_norm: type[Scaler] | None = StandardScaler, **kwargs) -> Learner

Create a Learner with an autoregressive TCN model.

Parameters:

Name	Type	Description	Default
dls		DataLoaders providing training and validation data.	required
hl_depth	int	Number of TCN hidden layers.	3
alpha	float	Regularization weight for smoothness penalty.	1
beta	float	Regularization weight for sparsity penalty.	1
metrics	list | None	Metric functions (defaults to RMSE).	None
n_skip	int | None	Number of initial time steps to skip in the loss (defaults to 2**hl_depth).	None
opt_func	type	Optimizer constructor.	Adam
input_norm	type[Scaler] | None	Input normalization scaler class, or None to disable.	StandardScaler
**kwargs		Additional arguments passed to TCN.	{}

Source code in tsfast/models/cnn.py

def AR_TCNLearner(
    dls,
    hl_depth: int = 3,
    alpha: float = 1,
    beta: float = 1,
    metrics: list | None = None,
    n_skip: int | None = None,
    opt_func: type = torch.optim.Adam,
    input_norm: type[Scaler] | None = StandardScaler,
    **kwargs,
) -> Learner:
    """Create a Learner with an autoregressive TCN model.

    Args:
        dls: DataLoaders providing training and validation data.
        hl_depth: Number of TCN hidden layers.
        alpha: Regularization weight for smoothness penalty.
        beta: Regularization weight for sparsity penalty.
        metrics: Metric functions (defaults to RMSE).
        n_skip: Number of initial time steps to skip in the loss (defaults to 2**hl_depth).
        opt_func: Optimizer constructor.
        input_norm: Input normalization scaler class, or None to disable.
        **kwargs: Additional arguments passed to ``TCN``.
    """
    if metrics is None:
        metrics = [fun_rmse]
    n_skip = 2**hl_depth if n_skip is None else n_skip

    inp, out = get_io_size(dls)
    ar_model = AR_Model(TCN(inp + out, out, hl_depth, **kwargs), ar=False)
    conv_module = ar_model.model.conv_layers[-1]

    model = ScaledModel.from_dls(ar_model, dls, input_norm, autoregressive=True)

    return Learner(
        model,
        dls,
        loss_func=nn.L1Loss(),
        opt_func=opt_func,
        metrics=metrics,
        n_skip=n_skip,
        lr=3e-3,
        transforms=[prediction_concat(t_offset=0)],
        aux_losses=[
            ActivationRegularizer(modules=[conv_module], alpha=alpha),
            TemporalActivationRegularizer(modules=[conv_module], beta=beta),
        ],
    )