fastNLP/tests/io/test_embed_loader.py

import numpy as np

from fastNLP import Vocabulary
from fastNLP.io import EmbedLoader


class TestEmbedLoader:
    def test_load_with_vocab(self):
        vocab = Vocabulary()
        glove = "tests/data_for_tests/embedding/small_static_embedding/glove.6B.50d_test.txt"
        word2vec = "tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt"
        vocab.add_word('the')
        vocab.add_word('none')
        g_m = EmbedLoader.load_with_vocab(glove, vocab)
        assert(g_m.shape == (4, 50))
        w_m = EmbedLoader.load_with_vocab(word2vec, vocab, normalize=True)
        assert(w_m.shape ==(4, 50))
        assert np.allclose(np.linalg.norm(w_m, axis=1).sum(), 4)
    
    def test_load_without_vocab(self):
        words = ['the', 'of', 'in', 'a', 'to', 'and']
        glove = "tests/data_for_tests/embedding/small_static_embedding/glove.6B.50d_test.txt"
        word2vec = "tests/data_for_tests/embedding/small_static_embedding/word2vec_test.txt"
        g_m, vocab = EmbedLoader.load_without_vocab(glove)
        assert(g_m.shape == (8, 50))
        for word in words:
            assert(word in vocab)
        w_m, vocab = EmbedLoader.load_without_vocab(word2vec, normalize=True)
        assert(w_m.shape== (8, 50))
        assert np.allclose(np.linalg.norm(w_m, axis=1).sum(), 8)
        for word in words:
            assert(word in vocab)
        # no unk
        w_m, vocab = EmbedLoader.load_without_vocab(word2vec, normalize=True, unknown=None)
        assert(w_m.shape == (7, 50))
        assert np.allclose(np.linalg.norm(w_m, axis=1).sum(), 7)
        for word in words:
            assert(word in vocab)