Commit 946b1481 authored by dzy7e's avatar dzy7e
Browse files

optimize dataset loading

parent d10bbc5a

zoo/ccip/dataset.py

+3 −0
Original line number Diff line number Diff line
@@ -134,6 +134,9 @@ class FastCharacterDataset(Dataset): 
            if cid not in groups:

                groups[cid] = []

            groups[cid].append(i)

        for k,v in groups.items():

            if len(v)==1:

                v.append(v[0])

        self.groups = groups



        self.group_size = group_size