Skip to content

Length Adaptor

LengthAdaptor

Bases: Module

DEPRECATED: The LengthAdaptor module is used to adjust the duration of phonemes. It contains a dedicated duration predictor and methods to upsample the input features to match predicted durations.

Parameters:

Name Type Description Default
model_config AcousticModelConfigType

The model configuration object containing model parameters.

 required
Source code in models/tts/delightful_tts/acoustic_model/length_adaptor.py 
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
class LengthAdaptor(Module):
    r"""DEPRECATED: The LengthAdaptor module is used to adjust the duration of phonemes.
    It contains a dedicated duration predictor and methods to upsample the input features to match predicted durations.

    Args:
        model_config (AcousticModelConfigType): The model configuration object containing model parameters.
    """

    def __init__(
        self,
        model_config: AcousticModelConfigType,
    ):
        super().__init__()
        # Initialize the duration predictor
        self.duration_predictor = VariancePredictor(
            channels_in=model_config.encoder.n_hidden,
            channels=model_config.variance_adaptor.n_hidden,
            channels_out=1,
            kernel_size=model_config.variance_adaptor.kernel_size,
            p_dropout=model_config.variance_adaptor.p_dropout,
        )

    def length_regulate(
        self,
        x: torch.Tensor,
        duration: torch.Tensor,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        r"""Regulates the length of the input tensor using the duration tensor.

        Args:
            x (torch.Tensor): The input tensor.
            duration (torch.Tensor): The tensor containing duration for each time step in x.

        Returns:
            Tuple[torch.Tensor, torch.Tensor]: The regulated output tensor and the tensor containing the length of each sequence in the batch.
        """
        output = torch.jit.annotate(List[torch.Tensor], [])
        mel_len = torch.jit.annotate(List[int], [])
        max_len = 0
        for batch, expand_target in zip(x, duration):
            expanded = self.expand(batch, expand_target)
            if expanded.shape[0] > max_len:
                max_len = expanded.shape[0]
            output.append(expanded)
            mel_len.append(expanded.shape[0])
        output = tools.pad(output, max_len)
        return output, torch.tensor(mel_len, dtype=torch.int64)

    def expand(self, batch: torch.Tensor, predicted: torch.Tensor) -> torch.Tensor:
        r"""Expands the input tensor based on the predicted values.

        Args:
            batch (torch.Tensor): The input tensor.
            predicted (torch.Tensor): The tensor containing predicted expansion factors.

        Returns:
            torch.Tensor: The expanded tensor.
        """
        out = torch.jit.annotate(List[torch.Tensor], [])
        for i, vec in enumerate(batch):
            expand_size = predicted[i].item()
            out.append(vec.expand(max(int(expand_size), 0), -1))
        return torch.cat(out, 0)

    def upsample_train(
        self,
        x: torch.Tensor,
        x_res: torch.Tensor,
        duration_target: torch.Tensor,
        embeddings: torch.Tensor,
        src_mask: torch.Tensor,
    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        r"""Upsamples the input tensor during training using ground truth durations.

        Args:
            x (torch.Tensor): The input tensor.
            x_res (torch.Tensor): Another input tensor for duration prediction.
            duration_target (torch.Tensor): The ground truth durations tensor.
            embeddings (torch.Tensor): The tensor containing phoneme embeddings.
            src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res.

        Returns:
            Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, log duration prediction, and upsampled embeddings.
        """
        x_res = x_res.detach()
        log_duration_prediction = self.duration_predictor(
            x_res,
            src_mask,
        )  # type: torch.Tensor
        x, _ = self.length_regulate(x, duration_target)
        embeddings, _ = self.length_regulate(embeddings, duration_target)
        return x, log_duration_prediction, embeddings

    def upsample(
        self,
        x: torch.Tensor,
        x_res: torch.Tensor,
        src_mask: torch.Tensor,
        embeddings: torch.Tensor,
        control: float,
    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        r"""Upsamples the input tensor during inference.

        Args:
            x (torch.Tensor): The input tensor.
            x_res (torch.Tensor): Another input tensor for duration prediction.
            src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res.
            embeddings (torch.Tensor): The tensor containing phoneme embeddings.
            control (float): A control parameter for pitch regulation.

        Returns:
            Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, approximated duration, and upsampled embeddings.
        """
        log_duration_prediction = self.duration_predictor(
            x_res,
            src_mask,
        )
        duration_rounded = torch.clamp(
            (torch.round(torch.exp(log_duration_prediction) - 1) * control),
            min=0,
        )
        x, _ = self.length_regulate(x, duration_rounded)
        embeddings, _ = self.length_regulate(embeddings, duration_rounded)
        return x, duration_rounded, embeddings

expand(batch, predicted)

Expands the input tensor based on the predicted values.

Parameters:

Name Type Description Default
batch Tensor

The input tensor.

 required
predicted Tensor

The tensor containing predicted expansion factors.

 required

Returns:

Type Description
Tensor

torch.Tensor: The expanded tensor.
Source code in models/tts/delightful_tts/acoustic_model/length_adaptor.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def expand(self, batch: torch.Tensor, predicted: torch.Tensor) -> torch.Tensor:
    r"""Expands the input tensor based on the predicted values.

    Args:
        batch (torch.Tensor): The input tensor.
        predicted (torch.Tensor): The tensor containing predicted expansion factors.

    Returns:
        torch.Tensor: The expanded tensor.
    """
    out = torch.jit.annotate(List[torch.Tensor], [])
    for i, vec in enumerate(batch):
        expand_size = predicted[i].item()
        out.append(vec.expand(max(int(expand_size), 0), -1))
    return torch.cat(out, 0)

length_regulate(x, duration)

Regulates the length of the input tensor using the duration tensor.

Parameters:

Name Type Description Default
x Tensor

The input tensor.

 required
duration Tensor

The tensor containing duration for each time step in x.

 required

Returns:

Type Description
Tuple[Tensor, Tensor]

Tuple[torch.Tensor, torch.Tensor]: The regulated output tensor and the tensor containing the length of each sequence in the batch.
Source code in models/tts/delightful_tts/acoustic_model/length_adaptor.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def length_regulate(
    self,
    x: torch.Tensor,
    duration: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
    r"""Regulates the length of the input tensor using the duration tensor.

    Args:
        x (torch.Tensor): The input tensor.
        duration (torch.Tensor): The tensor containing duration for each time step in x.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]: The regulated output tensor and the tensor containing the length of each sequence in the batch.
    """
    output = torch.jit.annotate(List[torch.Tensor], [])
    mel_len = torch.jit.annotate(List[int], [])
    max_len = 0
    for batch, expand_target in zip(x, duration):
        expanded = self.expand(batch, expand_target)
        if expanded.shape[0] > max_len:
            max_len = expanded.shape[0]
        output.append(expanded)
        mel_len.append(expanded.shape[0])
    output = tools.pad(output, max_len)
    return output, torch.tensor(mel_len, dtype=torch.int64)

upsample(x, x_res, src_mask, embeddings, control)

Upsamples the input tensor during inference.

Parameters:

Name Type Description Default
x Tensor

The input tensor.

 required
x_res Tensor

Another input tensor for duration prediction.

 required
src_mask Tensor

The mask tensor indicating valid entries in x and x_res.

 required
embeddings Tensor

The tensor containing phoneme embeddings.

 required
control float

A control parameter for pitch regulation.

 required

Returns:

Type Description
Tuple[Tensor, Tensor, Tensor]

Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, approximated duration, and upsampled embeddings.
Source code in models/tts/delightful_tts/acoustic_model/length_adaptor.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
def upsample(
    self,
    x: torch.Tensor,
    x_res: torch.Tensor,
    src_mask: torch.Tensor,
    embeddings: torch.Tensor,
    control: float,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    r"""Upsamples the input tensor during inference.

    Args:
        x (torch.Tensor): The input tensor.
        x_res (torch.Tensor): Another input tensor for duration prediction.
        src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res.
        embeddings (torch.Tensor): The tensor containing phoneme embeddings.
        control (float): A control parameter for pitch regulation.

    Returns:
        Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, approximated duration, and upsampled embeddings.
    """
    log_duration_prediction = self.duration_predictor(
        x_res,
        src_mask,
    )
    duration_rounded = torch.clamp(
        (torch.round(torch.exp(log_duration_prediction) - 1) * control),
        min=0,
    )
    x, _ = self.length_regulate(x, duration_rounded)
    embeddings, _ = self.length_regulate(embeddings, duration_rounded)
    return x, duration_rounded, embeddings

upsample_train(x, x_res, duration_target, embeddings, src_mask)

Upsamples the input tensor during training using ground truth durations.

Parameters:

Name Type Description Default
x Tensor

The input tensor.

 required
x_res Tensor

Another input tensor for duration prediction.

 required
duration_target Tensor

The ground truth durations tensor.

 required
embeddings Tensor

The tensor containing phoneme embeddings.

 required
src_mask Tensor

The mask tensor indicating valid entries in x and x_res.

 required

Returns:

Type Description
Tuple[Tensor, Tensor, Tensor]

Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, log duration prediction, and upsampled embeddings.
Source code in models/tts/delightful_tts/acoustic_model/length_adaptor.py 
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
def upsample_train(
    self,
    x: torch.Tensor,
    x_res: torch.Tensor,
    duration_target: torch.Tensor,
    embeddings: torch.Tensor,
    src_mask: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    r"""Upsamples the input tensor during training using ground truth durations.

    Args:
        x (torch.Tensor): The input tensor.
        x_res (torch.Tensor): Another input tensor for duration prediction.
        duration_target (torch.Tensor): The ground truth durations tensor.
        embeddings (torch.Tensor): The tensor containing phoneme embeddings.
        src_mask (torch.Tensor): The mask tensor indicating valid entries in x and x_res.

    Returns:
        Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: The upsampled x, log duration prediction, and upsampled embeddings.
    """
    x_res = x_res.detach()
    log_duration_prediction = self.duration_predictor(
        x_res,
        src_mask,
    )  # type: torch.Tensor
    x, _ = self.length_regulate(x, duration_target)
    embeddings, _ = self.length_regulate(embeddings, duration_target)
    return x, log_duration_prediction, embeddings