Commit 1b15751a authored by Phil Wang's avatar Phil Wang
Browse files

fix length extrapolation for rotary embeddings in local attention package

parent a94e32f4

audiolm_pytorch/soundstream.py

+1 −1
Original line number Diff line number Diff line
@@ -342,7 +342,7 @@ class LocalTransformerBlock(nn.Module): 
        **kwargs

    ):

        super().__init__()

        self.attn = LocalMHA(dim = dim, qk_rmsnorm = True, **kwargs)

        self.attn = LocalMHA(dim = dim, qk_rmsnorm = True, use_xpos = True, **kwargs)

        self.ff = FeedForward(dim = dim)



    def forward(self, x):

audiolm_pytorch/version.py

+1 −1
Original line number Diff line number Diff line 
__version__ = '0.18.0'
 
__version__ = '0.18.1'

setup.py

+1 −1
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ setup( 
    'fairseq',

    'joblib',

    'lion-pytorch',

    'local-attention>=1.6.0',

    'local-attention>=1.7.0',

    'scikit-learn',

    'sentencepiece',

    'torch>=1.12',