NamedEntityTagger/4_2.py at master · AMaini503/NamedEntityTagger · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#!/usr/bin/python3
import numpy as np
from aux import computeEmissions, isRare


def findTagAndMaxEmission(emissions, x):
    emissions_for_x = emissions[x]
    return max(emissions_for_x.items(), key = lambda x: x[1])


def writeTagsForDev(emissions):
    with open('ner_dev.dat') as f_input, open('4_2.txt', 'w') as f_output:
        for line in f_input:
            tokens = line.strip().split()

            line_written_to_output = ''

            #not a new line
            if(len(tokens) > 0):

                #x is modified to _RARE_ if it is qualifies as _RARE_. It is used to index emissions
                x = tokens[0]

                #word is used when writing to the output file, this is not modified
                word = tokens[0]

                if(isRare(emissions, word)):
                    x = '_RARE_'

                tag, max_emission = findTagAndMaxEmission(emissions, x)
                log_max_emission = np.log(max_emission)
                line_written_to_output = ' '.join([word,
                                                   tag,
                                                  '{}'.format(log_max_emission)])
                line_written_to_output = line_written_to_output + '\n'
            else:
                line_written_to_output = '\n'

            f_output.write(line_written_to_output)


emissions = computeEmissions()
writeTagsForDev(emissions)