Natural Language Processing

1file = open("cbow.bin", "rb")
2

1file = open("cbow.bin", "rb")
2file.read(100)
3

1file = open("cbow.bin", "rb")
2file.read(100)
3b'4347907 300\n</s> H\xe1\xae:0\x16\xc1:\xbfX\xa7\xbaR8\x8f\xba\xa0\xd3\xee9K\xfe\x83::m\xa49\xbc\xbb\x938\xa4p\x9d\xbat\xdaA:UU\xbe\xba\x93_\xda9\x82N\x83\xb9\xaeG\xa7\xb9\xde\xdd\x90\xbaww$\xba\xfdba:\x14.\x84:R\xb8\x81:0\x96\x0b:\x96\xfc\x06'  
4

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18array([10, 14, 16])
19

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18array([10, 14, 16])
19third = np.where(
20    (a[:-1] == a[1:]) &
21    (a[1:]==1)
22    )[0]
23third
24

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18array([10, 14, 16])
19third = np.where(
20    (a[:-1] == a[1:]) &
21    (a[1:]==1)
22    )[0]
23third
24array([15, 19, 20])
25

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18array([10, 14, 16])
19third = np.where(
20    (a[:-1] == a[1:]) &
21    (a[1:]==1)
22    )[0]
23third
24array([15, 19, 20])
25if (a[-1] == 1) | (a[-1] == 2):
26    pen = np.concatenate((
27        starts, first, second, third, np.array([a.shape[0]-1])
28    ))
29else:
30    pen = np.concatenate((
31        starts, first, second, third, 
32    ))
33np.sort(pen).reshape(-1,2)
34

1import numpy as np
2
3a = np.array(
4    [
5     0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 1, 0, 0, 1, 2, 1, 1, 0, 0, 1, 1, 1
6    ]
7)
8desired = [[3, 6], [10, 10], [13, 14], [15, 15], [16,16], [19,19], [20,20], [21,21]]
9starts = np.where(a==1)[0]
10starts
11array([ 3, 10, 13, 15, 16, 19, 20, 21])
12first = np.where(a[:-1] - 2 == a[1:])[0]
13first 
14array([6])
15second = np.where((a[:-1] - 1 == a[1:]) & 
16    ((a[1:]==1) | (a[1:]==0)))[0]
17second
18array([10, 14, 16])
19third = np.where(
20    (a[:-1] == a[1:]) &
21    (a[1:]==1)
22    )[0]
23third
24array([15, 19, 20])
25if (a[-1] == 1) | (a[-1] == 2):
26    pen = np.concatenate((
27        starts, first, second, third, np.array([a.shape[0]-1])
28    ))
29else:
30    pen = np.concatenate((
31        starts, first, second, third, 
32    ))
33np.sort(pen).reshape(-1,2)
34array([[ 3,  6],
35       [10, 10],
36       [13, 14],
37       [15, 15],
38       [16, 16],
39       [19, 19],
40       [20, 20],
41       [21, 21]])
42

1text = "This is a string <code>1234</code> another string <code>123</code> another string <code>123456789</code> another string."
2text = re.sub("<code>.*?</code>", '', text)
3print(text)
4
5Output: This is a string  another string  another string  another string.
6

1text = "This is a string <code>1234</code> another string <code>123</code> another string <code>123456789</code> another string."
2text = re.sub("<code>.*?</code>", '', text)
3print(text)
4
5Output: This is a string  another string  another string  another string.
6"This is a string <code>1234</code> another string <code>123</code> another string another string."
7

1text = """spaCy is an open-source software library for advanced natural language processing,
2written in the programming languages Python and Cython. The MIT library is published under the MIT license and its main developers are Matthew Honnibal and Ines Honnibal, the founders of the software company Explosion."""
3

1text = """spaCy is an open-source software library for advanced natural language processing,
2written in the programming languages Python and Cython. The MIT library is published under the MIT license and its main developers are Matthew Honnibal and Ines Honnibal, the founders of the software company Explosion."""
3import spacy
4nlp = spacy.load("en_core_sci_lg")
5
6doc = nlp(text)
7my_tuple = list(set(doc.ents))
8print('original tuple', doc.ents, len(doc.ents))
9print('after set function', my_tuple, len(my_tuple))
10

1text = """spaCy is an open-source software library for advanced natural language processing,
2written in the programming languages Python and Cython. The MIT library is published under the MIT license and its main developers are Matthew Honnibal and Ines Honnibal, the founders of the software company Explosion."""
3import spacy
4nlp = spacy.load("en_core_sci_lg")
5
6doc = nlp(text)
7my_tuple = list(set(doc.ents))
8print('original tuple', doc.ents, len(doc.ents))
9print('after set function', my_tuple, len(my_tuple))
10original tuple: (spaCy, open-source software library, programming languages, Python, Cython, MIT, library, published, MIT, license, developers, Matthew Honnibal, Ines, Honnibal, founders, software company Explosion) 16
11
12after set function: [Honnibal, MIT, Ines, software company Explosion, founders, programming languages, library, Matthew Honnibal, license, Cython, Python, developers, MIT, published, open-source software library, spaCy] 16
13

1text = """spaCy is an open-source software library for advanced natural language processing,
2written in the programming languages Python and Cython. The MIT library is published under the MIT license and its main developers are Matthew Honnibal and Ines Honnibal, the founders of the software company Explosion."""
3import spacy
4nlp = spacy.load("en_core_sci_lg")
5
6doc = nlp(text)
7my_tuple = list(set(doc.ents))
8print('original tuple', doc.ents, len(doc.ents))
9print('after set function', my_tuple, len(my_tuple))
10original tuple: (spaCy, open-source software library, programming languages, Python, Cython, MIT, library, published, MIT, license, developers, Matthew Honnibal, Ines, Honnibal, founders, software company Explosion) 16
11
12after set function: [Honnibal, MIT, Ines, software company Explosion, founders, programming languages, library, Matthew Honnibal, license, Cython, Python, developers, MIT, published, open-source software library, spaCy] 16
13[Ines Honnibal, MIT, software company Explosion, founders, programming languages, library, Matthew Honnibal, license, Cython, Python, developers, published, open-source software library, spaCy]
14

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=16)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=16)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24INFO - haystack.modeling.utils -  Using devices: CUDA
25INFO - haystack.modeling.utils -  Number of GPUs: 1
26---------------------------------------------------------------------------
27ValueError                                Traceback (most recent call last)
28<ipython-input-11-77eb98038283> in <module>
29     14 
30     15 # classify using gpu, batch_size makes sure we do not run out of memory
31---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
32     17 
33     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
34
35~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
36    137         batches = self.get_batches(texts, batch_size=self.batch_size)
37    138         if self.task == 'zero-shot-classification':
38--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
39    140         elif self.task == 'text-classification':
40    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
41
42~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
43    137         batches = self.get_batches(texts, batch_size=self.batch_size)
44    138         if self.task == 'zero-shot-classification':
45--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
46    140         elif self.task == 'text-classification':
47    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
48
49~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
50    151             sequences = [sequences]
51    152 
52--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
53    154         num_sequences = len(sequences)
54    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
55
56~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
57    758 
58    759     def __call__(self, *args, **kwargs):
59--> 760         inputs = self._parse_and_tokenize(*args, **kwargs)
60    761         return self._forward(inputs)
61    762 
62
63~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in _parse_and_tokenize(self, sequences, candidate_labels, hypothesis_template, padding, add_special_tokens, truncation, **kwargs)
64     92         Parse arguments and tokenize only_first so that hypothesis (label) is not truncated
65     93         """
66---> 94         sequence_pairs = self._args_parser(sequences, candidate_labels, hypothesis_template)
67     95         inputs = self.tokenizer(
68     96             sequence_pairs,
69
70~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, labels, hypothesis_template)
71     25     def __call__(self, sequences, labels, hypothesis_template):
72     26         if len(labels) == 0 or len(sequences) == 0:
73---> 27             raise ValueError("You must include at least one label and at least one sequence.")
74     28         if hypothesis_template.format(labels[0]) == hypothesis_template:
75     29             raise ValueError(
76
77ValueError: You must include at least one label and at least one sequence.
78

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=16)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24INFO - haystack.modeling.utils -  Using devices: CUDA
25INFO - haystack.modeling.utils -  Number of GPUs: 1
26---------------------------------------------------------------------------
27ValueError                                Traceback (most recent call last)
28<ipython-input-11-77eb98038283> in <module>
29     14 
30     15 # classify using gpu, batch_size makes sure we do not run out of memory
31---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
32     17 
33     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
34
35~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
36    137         batches = self.get_batches(texts, batch_size=self.batch_size)
37    138         if self.task == 'zero-shot-classification':
38--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
39    140         elif self.task == 'text-classification':
40    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
41
42~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
43    137         batches = self.get_batches(texts, batch_size=self.batch_size)
44    138         if self.task == 'zero-shot-classification':
45--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
46    140         elif self.task == 'text-classification':
47    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
48
49~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
50    151             sequences = [sequences]
51    152 
52--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
53    154         num_sequences = len(sequences)
54    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
55
56~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
57    758 
58    759     def __call__(self, *args, **kwargs):
59--> 760         inputs = self._parse_and_tokenize(*args, **kwargs)
60    761         return self._forward(inputs)
61    762 
62
63~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in _parse_and_tokenize(self, sequences, candidate_labels, hypothesis_template, padding, add_special_tokens, truncation, **kwargs)
64     92         Parse arguments and tokenize only_first so that hypothesis (label) is not truncated
65     93         """
66---> 94         sequence_pairs = self._args_parser(sequences, candidate_labels, hypothesis_template)
67     95         inputs = self.tokenizer(
68     96             sequence_pairs,
69
70~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, labels, hypothesis_template)
71     25     def __call__(self, sequences, labels, hypothesis_template):
72     26         if len(labels) == 0 or len(sequences) == 0:
73---> 27             raise ValueError("You must include at least one label and at least one sequence.")
74     28         if hypothesis_template.format(labels[0]) == hypothesis_template:
75     29             raise ValueError(
76
77ValueError: You must include at least one label and at least one sequence.
78doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
79    task="zero-shot-classification",
80    labels=["music", "natural language processing", "history"],
81    batch_size=16
82)
83
84# ----------
85
86# convert to Document using a fieldmap for custom content fields the classification should run on
87docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
88
89# ----------
90
91# classify using gpu, batch_size makes sure we do not run out of memory
92classified_docs = doc_classifier.predict(docs_to_classify)
93
94# ----------
95
96# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
97print(classified_docs[0].to_dict())
98

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=4)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=4)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24INFO - haystack.modeling.utils -  Using devices: CUDA
25INFO - haystack.modeling.utils -  Using devices: CUDA
26INFO - haystack.modeling.utils -  Number of GPUs: 1
27INFO - haystack.modeling.utils -  Number of GPUs: 1
28---------------------------------------------------------------------------
29RuntimeError                              Traceback (most recent call last)
30<ipython-input-25-27dfca549a7d> in <module>
31     14 
32     15 # classify using gpu, batch_size makes sure we do not run out of memory
33---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
34     17 
35     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
36
37~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
38    137         batches = self.get_batches(texts, batch_size=self.batch_size)
39    138         if self.task == 'zero-shot-classification':
40--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
41    140         elif self.task == 'text-classification':
42    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
43
44~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
45    137         batches = self.get_batches(texts, batch_size=self.batch_size)
46    138         if self.task == 'zero-shot-classification':
47--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
48    140         elif self.task == 'text-classification':
49    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
50
51~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
52    151             sequences = [sequences]
53    152 
54--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
55    154         num_sequences = len(sequences)
56    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
57
58~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
59    759     def __call__(self, *args, **kwargs):
60    760         inputs = self._parse_and_tokenize(*args, **kwargs)
61--> 761         return self._forward(inputs)
62    762 
63    763     def _forward(self, inputs, return_tensors=False):
64
65~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in _forward(self, inputs, return_tensors)
66    780                 with torch.no_grad():
67    781                     inputs = self.ensure_tensor_on_device(**inputs)
68--> 782                     predictions = self.model(**inputs)[0].cpu()
69    783 
70    784         if return_tensors:
71
72~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
73   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
74   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
75-> 1102             return forward_call(*input, **kwargs)
76   1103         # Do not call functions when jit is used
77   1104         full_backward_hooks, non_full_backward_hooks = [], []
78
79~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
80   1162             output_attentions=output_attentions,
81   1163             output_hidden_states=output_hidden_states,
82-> 1164             return_dict=return_dict,
83   1165         )
84   1166         sequence_output = outputs[0]
85
86~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
87   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
88   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
89-> 1102             return forward_call(*input, **kwargs)
90   1103         # Do not call functions when jit is used
91   1104         full_backward_hooks, non_full_backward_hooks = [], []
92
93~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
94    823             output_attentions=output_attentions,
95    824             output_hidden_states=output_hidden_states,
96--> 825             return_dict=return_dict,
97    826         )
98    827         sequence_output = encoder_outputs[0]
99
100~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
101   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
102   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
103-> 1102             return forward_call(*input, **kwargs)
104   1103         # Do not call functions when jit is used
105   1104         full_backward_hooks, non_full_backward_hooks = [], []
106
107~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
108    513                     encoder_attention_mask,
109    514                     past_key_value,
110--> 515                     output_attentions,
111    516                 )
112    517 
113
114~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
115   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
116   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
117-> 1102             return forward_call(*input, **kwargs)
118   1103         # Do not call functions when jit is used
119   1104         full_backward_hooks, non_full_backward_hooks = [], []
120
121~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
122    398             head_mask,
123    399             output_attentions=output_attentions,
124--> 400             past_key_value=self_attn_past_key_value,
125    401         )
126    402         attention_output = self_attention_outputs[0]
127
128~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
129   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
130   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
131-> 1102             return forward_call(*input, **kwargs)
132   1103         # Do not call functions when jit is used
133   1104         full_backward_hooks, non_full_backward_hooks = [], []
134
135~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
136    328             encoder_attention_mask,
137    329             past_key_value,
138--> 330             output_attentions,
139    331         )
140    332         attention_output = self.output(self_outputs[0], hidden_states)
141
142~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
143   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
144   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
145-> 1102             return forward_call(*input, **kwargs)
146   1103         # Do not call functions when jit is used
147   1104         full_backward_hooks, non_full_backward_hooks = [], []
148
149~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
150    241                 attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
151    242 
152--> 243         attention_scores = attention_scores / math.sqrt(self.attention_head_size)
153    244         if attention_mask is not None:
154    245             # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
155
156RuntimeError: CUDA out of memory. Tried to allocate 3.60 GiB (GPU 0; 14.76 GiB total capacity; 7.33 GiB already allocated; 1.37 GiB free; 12.29 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
157---------------------------------------------------------------------------
158RuntimeError                              Traceback (most recent call last)
159<ipython-input-25-27dfca549a7d> in <module>
160     14 
161     15 # classify using gpu, batch_size makes sure we do not run out of memory
162---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
163     17 
164     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
165
166~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
167    137         batches = self.get_batches(texts, batch_size=self.batch_size)
168    138         if self.task == 'zero-shot-classification':
169--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
170    140         elif self.task == 'text-classification':
171    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
172
173~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
174    137         batches = self.get_batches(texts, batch_size=self.batch_size)
175    138         if self.task == 'zero-shot-classification':
176--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
177    140         elif self.task == 'text-classification':
178    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
179
180~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
181    151             sequences = [sequences]
182    152 
183--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
184    154         num_sequences = len(sequences)
185    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
186
187~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
188    759     def __call__(self, *args, **kwargs):
189    760         inputs = self._parse_and_tokenize(*args, **kwargs)
190--> 761         return self._forward(inputs)
191    762 
192    763     def _forward(self, inputs, return_tensors=False):
193
194~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in _forward(self, inputs, return_tensors)
195    780                 with torch.no_grad():
196    781                     inputs = self.ensure_tensor_on_device(**inputs)
197--> 782                     predictions = self.model(**inputs)[0].cpu()
198    783 
199    784         if return_tensors:
200
201~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
202   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
203   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
204-> 1102             return forward_call(*input, **kwargs)
205   1103         # Do not call functions when jit is used
206   1104         full_backward_hooks, non_full_backward_hooks = [], []
207
208~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
209   1162             output_attentions=output_attentions,
210   1163             output_hidden_states=output_hidden_states,
211-> 1164             return_dict=return_dict,
212   1165         )
213   1166         sequence_output = outputs[0]
214
215~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
216   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
217   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
218-> 1102             return forward_call(*input, **kwargs)
219   1103         # Do not call functions when jit is used
220   1104         full_backward_hooks, non_full_backward_hooks = [], []
221
222~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
223    823             output_attentions=output_attentions,
224    824             output_hidden_states=output_hidden_states,
225--> 825             return_dict=return_dict,
226    826         )
227    827         sequence_output = encoder_outputs[0]
228
229~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
230   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
231   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
232-> 1102             return forward_call(*input, **kwargs)
233   1103         # Do not call functions when jit is used
234   1104         full_backward_hooks, non_full_backward_hooks = [], []
235
236~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
237    513                     encoder_attention_mask,
238    514                     past_key_value,
239--> 515                     output_attentions,
240    516                 )
241    517 
242
243~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
244   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
245   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
246-> 1102             return forward_call(*input, **kwargs)
247   1103         # Do not call functions when jit is used
248   1104         full_backward_hooks, non_full_backward_hooks = [], []
249
250~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
251    398             head_mask,
252    399             output_attentions=output_attentions,
253--> 400             past_key_value=self_attn_past_key_value,
254    401         )
255    402         attention_output = self_attention_outputs[0]
256
257~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
258   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
259   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
260-> 1102             return forward_call(*input, **kwargs)
261   1103         # Do not call functions when jit is used
262   1104         full_backward_hooks, non_full_backward_hooks = [], []
263
264~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
265    328             encoder_attention_mask,
266    329             past_key_value,
267--> 330             output_attentions,
268    331         )
269    332         attention_output = self.output(self_outputs[0], hidden_states)
270
271~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
272   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
273   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
274-> 1102             return forward_call(*input, **kwargs)
275   1103         # Do not call functions when jit is used
276   1104         full_backward_hooks, non_full_backward_hooks = [], []
277
278~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
279    241                 attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
280    242 
281--> 243         attention_scores = attention_scores / math.sqrt(self.attention_head_size)
282    244         if attention_mask is not None:
283    245             # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
284
285RuntimeError: CUDA out of memory. Tried to allocate 3.60 GiB (GPU 0; 14.76 GiB total capacity; 7.33 GiB already allocated; 1.37 GiB free; 12.29 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
286

1with open('filt_gri.txt', 'r') as filehandle:
2    tags = [current_place.rstrip() for current_place in filehandle.readlines()]
3
4doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
5                                                task="zero-shot-classification",
6                                                labels=tags,
7                                                batch_size=4)
8
9# convert to Document using a fieldmap for custom content fields the classification should run on
10docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
11
12# classify using gpu, batch_size makes sure we do not run out of memory
13classified_docs = doc_classifier.predict(docs_to_classify)
14
15# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
16print(classified_docs[0].to_dict())
17
18all_docs = convert_files_to_dicts(dir_path=doc_dir)
19
20preprocessor_sliding_window = PreProcessor(split_overlap=3,
21                                           split_length=10,
22                                           split_respect_sentence_boundary=False,
23                                           split_by='passage')
24INFO - haystack.modeling.utils -  Using devices: CUDA
25INFO - haystack.modeling.utils -  Using devices: CUDA
26INFO - haystack.modeling.utils -  Number of GPUs: 1
27INFO - haystack.modeling.utils -  Number of GPUs: 1
28---------------------------------------------------------------------------
29RuntimeError                              Traceback (most recent call last)
30<ipython-input-25-27dfca549a7d> in <module>
31     14 
32     15 # classify using gpu, batch_size makes sure we do not run out of memory
33---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
34     17 
35     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
36
37~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
38    137         batches = self.get_batches(texts, batch_size=self.batch_size)
39    138         if self.task == 'zero-shot-classification':
40--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
41    140         elif self.task == 'text-classification':
42    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
43
44~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
45    137         batches = self.get_batches(texts, batch_size=self.batch_size)
46    138         if self.task == 'zero-shot-classification':
47--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
48    140         elif self.task == 'text-classification':
49    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
50
51~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
52    151             sequences = [sequences]
53    152 
54--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
55    154         num_sequences = len(sequences)
56    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
57
58~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
59    759     def __call__(self, *args, **kwargs):
60    760         inputs = self._parse_and_tokenize(*args, **kwargs)
61--> 761         return self._forward(inputs)
62    762 
63    763     def _forward(self, inputs, return_tensors=False):
64
65~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in _forward(self, inputs, return_tensors)
66    780                 with torch.no_grad():
67    781                     inputs = self.ensure_tensor_on_device(**inputs)
68--> 782                     predictions = self.model(**inputs)[0].cpu()
69    783 
70    784         if return_tensors:
71
72~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
73   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
74   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
75-> 1102             return forward_call(*input, **kwargs)
76   1103         # Do not call functions when jit is used
77   1104         full_backward_hooks, non_full_backward_hooks = [], []
78
79~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
80   1162             output_attentions=output_attentions,
81   1163             output_hidden_states=output_hidden_states,
82-> 1164             return_dict=return_dict,
83   1165         )
84   1166         sequence_output = outputs[0]
85
86~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
87   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
88   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
89-> 1102             return forward_call(*input, **kwargs)
90   1103         # Do not call functions when jit is used
91   1104         full_backward_hooks, non_full_backward_hooks = [], []
92
93~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
94    823             output_attentions=output_attentions,
95    824             output_hidden_states=output_hidden_states,
96--> 825             return_dict=return_dict,
97    826         )
98    827         sequence_output = encoder_outputs[0]
99
100~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
101   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
102   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
103-> 1102             return forward_call(*input, **kwargs)
104   1103         # Do not call functions when jit is used
105   1104         full_backward_hooks, non_full_backward_hooks = [], []
106
107~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
108    513                     encoder_attention_mask,
109    514                     past_key_value,
110--> 515                     output_attentions,
111    516                 )
112    517 
113
114~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
115   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
116   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
117-> 1102             return forward_call(*input, **kwargs)
118   1103         # Do not call functions when jit is used
119   1104         full_backward_hooks, non_full_backward_hooks = [], []
120
121~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
122    398             head_mask,
123    399             output_attentions=output_attentions,
124--> 400             past_key_value=self_attn_past_key_value,
125    401         )
126    402         attention_output = self_attention_outputs[0]
127
128~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
129   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
130   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
131-> 1102             return forward_call(*input, **kwargs)
132   1103         # Do not call functions when jit is used
133   1104         full_backward_hooks, non_full_backward_hooks = [], []
134
135~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
136    328             encoder_attention_mask,
137    329             past_key_value,
138--> 330             output_attentions,
139    331         )
140    332         attention_output = self.output(self_outputs[0], hidden_states)
141
142~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
143   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
144   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
145-> 1102             return forward_call(*input, **kwargs)
146   1103         # Do not call functions when jit is used
147   1104         full_backward_hooks, non_full_backward_hooks = [], []
148
149~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
150    241                 attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
151    242 
152--> 243         attention_scores = attention_scores / math.sqrt(self.attention_head_size)
153    244         if attention_mask is not None:
154    245             # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
155
156RuntimeError: CUDA out of memory. Tried to allocate 3.60 GiB (GPU 0; 14.76 GiB total capacity; 7.33 GiB already allocated; 1.37 GiB free; 12.29 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
157---------------------------------------------------------------------------
158RuntimeError                              Traceback (most recent call last)
159<ipython-input-25-27dfca549a7d> in <module>
160     14 
161     15 # classify using gpu, batch_size makes sure we do not run out of memory
162---> 16 classified_docs = doc_classifier.predict(docs_to_classify)
163     17 
164     18 # let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
165
166~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in predict(self, documents)
167    137         batches = self.get_batches(texts, batch_size=self.batch_size)
168    138         if self.task == 'zero-shot-classification':
169--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
170    140         elif self.task == 'text-classification':
171    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
172
173~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/haystack/nodes/document_classifier/transformers.py in <listcomp>(.0)
174    137         batches = self.get_batches(texts, batch_size=self.batch_size)
175    138         if self.task == 'zero-shot-classification':
176--> 139             batched_predictions = [self.model(batch, candidate_labels=self.labels, truncation=True) for batch in batches]
177    140         elif self.task == 'text-classification':
178    141             batched_predictions = [self.model(batch, return_all_scores=self.return_all_scores, truncation=True) for batch in batches]
179
180~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/zero_shot_classification.py in __call__(self, sequences, candidate_labels, hypothesis_template, multi_label, **kwargs)
181    151             sequences = [sequences]
182    152 
183--> 153         outputs = super().__call__(sequences, candidate_labels, hypothesis_template)
184    154         num_sequences = len(sequences)
185    155         candidate_labels = self._args_parser._parse_labels(candidate_labels)
186
187~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in __call__(self, *args, **kwargs)
188    759     def __call__(self, *args, **kwargs):
189    760         inputs = self._parse_and_tokenize(*args, **kwargs)
190--> 761         return self._forward(inputs)
191    762 
192    763     def _forward(self, inputs, return_tensors=False):
193
194~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/pipelines/base.py in _forward(self, inputs, return_tensors)
195    780                 with torch.no_grad():
196    781                     inputs = self.ensure_tensor_on_device(**inputs)
197--> 782                     predictions = self.model(**inputs)[0].cpu()
198    783 
199    784         if return_tensors:
200
201~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
202   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
203   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
204-> 1102             return forward_call(*input, **kwargs)
205   1103         # Do not call functions when jit is used
206   1104         full_backward_hooks, non_full_backward_hooks = [], []
207
208~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)
209   1162             output_attentions=output_attentions,
210   1163             output_hidden_states=output_hidden_states,
211-> 1164             return_dict=return_dict,
212   1165         )
213   1166         sequence_output = outputs[0]
214
215~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
216   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
217   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
218-> 1102             return forward_call(*input, **kwargs)
219   1103         # Do not call functions when jit is used
220   1104         full_backward_hooks, non_full_backward_hooks = [], []
221
222~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
223    823             output_attentions=output_attentions,
224    824             output_hidden_states=output_hidden_states,
225--> 825             return_dict=return_dict,
226    826         )
227    827         sequence_output = encoder_outputs[0]
228
229~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
230   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
231   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
232-> 1102             return forward_call(*input, **kwargs)
233   1103         # Do not call functions when jit is used
234   1104         full_backward_hooks, non_full_backward_hooks = [], []
235
236~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)
237    513                     encoder_attention_mask,
238    514                     past_key_value,
239--> 515                     output_attentions,
240    516                 )
241    517 
242
243~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
244   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
245   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
246-> 1102             return forward_call(*input, **kwargs)
247   1103         # Do not call functions when jit is used
248   1104         full_backward_hooks, non_full_backward_hooks = [], []
249
250~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
251    398             head_mask,
252    399             output_attentions=output_attentions,
253--> 400             past_key_value=self_attn_past_key_value,
254    401         )
255    402         attention_output = self_attention_outputs[0]
256
257~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
258   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
259   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
260-> 1102             return forward_call(*input, **kwargs)
261   1103         # Do not call functions when jit is used
262   1104         full_backward_hooks, non_full_backward_hooks = [], []
263
264~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
265    328             encoder_attention_mask,
266    329             past_key_value,
267--> 330             output_attentions,
268    331         )
269    332         attention_output = self.output(self_outputs[0], hidden_states)
270
271~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
272   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
273   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
274-> 1102             return forward_call(*input, **kwargs)
275   1103         # Do not call functions when jit is used
276   1104         full_backward_hooks, non_full_backward_hooks = [], []
277
278~/anaconda3/envs/mxnet_latest_p37/lib/python3.7/site-packages/transformers/models/roberta/modeling_roberta.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)
279    241                 attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
280    242 
281--> 243         attention_scores = attention_scores / math.sqrt(self.attention_head_size)
282    244         if attention_mask is not None:
283    245             # Apply the attention mask is (precomputed for all layers in RobertaModel forward() function)
284
285RuntimeError: CUDA out of memory. Tried to allocate 3.60 GiB (GPU 0; 14.76 GiB total capacity; 7.33 GiB already allocated; 1.37 GiB free; 12.29 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
286doc_classifier = TransformersDocumentClassifier(model_name_or_path="cross-encoder/nli-distilroberta-base",
287    task="zero-shot-classification",
288    labels=["music", "natural language processing", "history"],
289    batch_size=16
290)
291
292# ----------
293
294# convert to Document using a fieldmap for custom content fields the classification should run on
295docs_to_classify = [Document.from_dict(d) for d in docs_sliding_window]
296
297# ----------
298
299# classify using gpu, batch_size makes sure we do not run out of memory
300classified_docs = doc_classifier.predict(docs_to_classify)
301
302# ----------
303
304# let's see how it looks: there should be a classification result in the meta entry containing labels and scores.
305print(classified_docs[0].to_dict())
306

1import pandas as pd
2import requests
3from bs4 import BeautifulSoup
4import string
5import nltk
6from urllib.request import urlopen
7import sys
8import webbrowser
9import newspaper 
10import time
11from newspaper import Article
12
13Company_name1 =[]
14Article_number1=[]
15Article_Title1=[]
16Article_Authors1=[]
17Article_pub_date1=[]
18Article_Text1=[]
19Article_Summary1=[]
20Article_Keywords1=[]
21Final_dataframe=[]
22
23class Newspapr_pd:
24    def __init__(self,term):
25        self.term=term
26        self.subjectivity=0
27        self.sentiment=0
28        self.url='https://www.google.com/search?q={0}&safe=active&tbs=qdr:w,sdb:1&tbm=nws&source=lnt&dpr=1'.format(self.term)
29    
30    def NewsArticlerun_pd(self):
31        response=requests.get(self.url)
32        response.raise_for_status()
33        #print(response.text)
34        soup=bs4.BeautifulSoup(response.text,'html.parser')
35        links=soup.select(".r a")
36       
37        numOpen = min(5, len(links))
38        Article_number=0
39        for i in range(numOpen):
40            response_links = webbrower.open("https://www.google.com" + links[i].get("href"))
41            
42            
43            
44        #For different language newspaper refer above table 
45            article = Article(response_links, language="en") # en for English 
46            Article_number+=1
47            
48            print('*************************************************************************************')
49            
50            Article_number1.append(Article_number)
51            Company_name1.append(self.term)
52
53        #To download the article 
54            try:
55
56                article.download() 
57                 #To parse the article 
58                article.parse() 
59                #To perform natural language processing ie..nlp 
60                article.nlp() 
61  
62        #To extract title
63                Article_Title1.append(article.title)
64
65  
66        #To extract text
67                Article_Text1.append(article.text)
68
69  
70        #To extract Author name
71                Article_Authors1.append(article.authors)
72
73                
74        #To extract article published date
75                Article_pub_date1.append(article.publish_date)
76                
77
78                
79        #To extract summary
80                Article_Summary1.append(article.summary)
81                
82
83  
84        #To extract keywords 
85                Article_Keywords1.append(article.keywords)
86
87            except:
88                print('Error in loading page')
89                continue
90  
91        for art_num,com_name,title,text,auth,pub_dt,summaries,keywds in zip(Article_number1,Company_name1,Article_Title1,Article_Text1,Article_Authors1,Article_pub_date1,Article_Summary1,Article_Keywords1):
92            Final_dataframe.append({'Article_link_num':art_num, 'Company_name':com_name,'Article_Title':title,'Article_Text':text,'Article_Author':auth,
93                                   'Article_Published_date':pub_dt,'Article_Summary':summaries,'Article_Keywords':keywds})
94        
95list_of_companies=['Amazon','Jetairways','nirav modi']
96
97for i in list_of_companies:
98    comp = str('"'+ i + '"')
99    a=Newspapr_pd(comp)
100    a.NewsArticlerun_pd()
101
102Final_new_dataframe=pd.DataFrame(Final_dataframe)
103Final_new_dataframe.tail()    
104