Commit 9145655b authored by Phil Wang's avatar Phil Wang
Browse files

increase the odds someone trains soundstream successfully with a new attention... 

increase the odds someone trains soundstream successfully with a new attention stabilization trick out of Brain
parent 653e6e91

audiolm_pytorch/soundstream.py

+1 −1
Original line number Diff line number Diff line
@@ -330,7 +330,7 @@ class LocalTransformerBlock(nn.Module): 
        **kwargs

    ):

        super().__init__()

        self.attn = LocalMHA(dim = dim, **kwargs)

        self.attn = LocalMHA(dim = dim, qk_rmsnorm = True, **kwargs)

        self.ff = FeedForward(dim = dim)



    def forward(self, x):

setup.py

+2 −2
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ from setuptools import setup, find_packages 
setup(

  name = 'audiolm-pytorch',

  packages = find_packages(exclude=[]),

  version = '0.11.19',

  version = '0.12.0',

  license='MIT',

  description = 'AudioLM - Language Modeling Approach to Audio Generation from Google Research - Pytorch',

  author = 'Phil Wang',
@@ -24,7 +24,7 @@ setup( 
    'ema-pytorch',

    'fairseq',

    'joblib',

    'local-attention>=1.5.7',

    'local-attention>=1.6.0',

    'Mega-pytorch',

    'scikit-learn',

    'sentencepiece',