elasticsearch-skroutz-greekstemmer | Greek Stemmer for elasticsearch | Natural Language Processing library

df_texts <- tibble(
  text=c("the ape and the fox", "the tortoise and the hare", "the owl and the the 
  grasshopper"),
  mammals=NA,
  reptiles=NA,
  birds=NA,
  insects=NA
)

df_animals <- tibble(animals=c("ape", "fox", "tortoise", "hare", "owl", "grasshopper"),
           type=c("mammal", "mammal", "reptile", "mammal", "bird", "insect"))

df_result <- tibble(
  text=c("the ape and the fox", "the tortoise and the hare", "the owl and the the 
  grasshopper"),
  mammals=c(2,1,0),
  reptiles=c(0,1,0),
  birds=c(0,0,1),
  insects=c(0,0,1)
)

    // text 1 has two "accredited" in a different order
    let text1: String = "accredit accredited accrediting accredited accreditation accredits"
    
    // text 2 has three "accredited" in different order
    let text2: String = "accredit accredits accredited accrediting accredited accredited accreditation"
    
    // text 3 has "accreditation"
    let text3: String = "accreditation"
    
    // text 4 has "accredited"
    let text4: String = "accredited"

import SwiftUI
import NaturalLanguage

struct ContentView: View {
    
    // text 1 has two "accredited" in a different order
    let text1: String = "accredit accredited accrediting accredited accreditation accredits"
    
    // text 2 has three "accredited" in a different order
    let text2: String = "accredit accredits accredited accrediting accredited accredited accreditation"
    
    // text 3 has "accreditation"
    let text3: String = "accreditation"
    
    // text 4 has "accredited"
    let text4: String = "accredited"
    
    var body: some View {
        ScrollView {
            VStack {
                
                Text("Text 1").bold()
                tagText(text: text1, scheme: .lemma).padding(.bottom)
                
                Text("Text 2").bold()
                tagText(text: text2, scheme: .lemma).padding(.bottom)
                
                Text("Text 3").bold()
                tagText(text: text3, scheme: .lemma).padding(.bottom)
                
                Text("Text 4").bold()
                tagText(text: text4, scheme: .lemma).padding(.bottom)
                
            }
        }
    }
    
    // MARK: - tagText
    func tagText(text: String, scheme: NLTagScheme) -> some View {
        VStack {
            ForEach(partsOfSpeechTagger(for: text, scheme: scheme)) { word in
                Text(word.description)
            }
        }
    }
    
    // MARK: - partsOfSpeechTagger
    func partsOfSpeechTagger(for text: String, scheme: NLTagScheme) -> [NLPTagResult] {
        
        var listOfTaggedWords: [NLPTagResult] = []
        let tagger = NLTagger(tagSchemes: [scheme])
        tagger.string = text
        
        let range = text.startIndex..<text.endIndex
        let options: NLTagger.Options = [.omitPunctuation, .omitWhitespace]
        
        tagger.enumerateTags(in: range, unit: .word, scheme: scheme, options: options) { tag, tokenRange in
            
            if let tag = tag {
                let word: String = String(text[tokenRange])
                let result = NLPTagResult(word: word, tag: tag)
                
                //if !word.localizedCaseInsensitiveContains(tag.rawValue) {
                listOfTaggedWords.append(result)
                //}
            }
            return true
        }
        
        return listOfTaggedWords
    }
    
    // MARK: - NLPTagResult
    struct NLPTagResult: Identifiable, Equatable, Hashable, Comparable {
        var id = UUID()
        var word: String
        var tag: NLTag?
        
        var description: String {
            var newString: String = "\(word)"
            
            if let tag = tag {
                newString += " : \(tag.rawValue)"
            }
            
            return newString
        }
        
        // MARK: - Equatable & Hashable requirements
        static func == (lhs: Self, rhs: Self) -> Bool {
            lhs.id == rhs.id
        }
        
        func hash(into hasher: inout Hasher) {
            hasher.combine(id)
        }
        
        // MARK: - Comparable requirements
        static func <(lhs: NLPTagResult, rhs: NLPTagResult) -> Bool {
            lhs.id.uuidString < rhs.id.uuidString
        }
    }
    
}

// MARK: - Previews
struct ContentView_Previews: PreviewProvider {
    static var previews: some View {
        ContentView()
    }
}

def preprocess(text):
  #remove punctuation
  text = re.sub('\b[A-Za-z]+(?:-+[A-Za-z]+)+\b', '-', text)
  text = re.sub('[^a-zA-Z]', ' ', text)
  text = text.split()
  text = " ".join(text)
  return text

"Attended pre-tender meetings" 

['attended', 'pre-tender', 'meeting'] 

['attended', 'pre', 'tender', 'meeting']

bigrams = ['data science', 'computer science', 'bachelors degree']

df = pd.DataFrame(data={'job_description': [['data', 'science', 'degree', 'expert'],
                                            ['computer', 'science', 'degree', 'masters'],
                                            ['bachelors', 'degree', 'computer', 'vision'],
                                            ['data', 'processing', 'science']]})

                         job_description  data science computer science bachelors degree
0        [data, science, degree, expert]          True            False            False
1   [computer, science, degree, masters]         False             True            False
2  [bachelors, degree, computer, vision]         False            False             True
3             [data, bachelors, science]         False            False            False

! pip install farm-haystack -q
# Install the latest master of Haystack
!pip install grpcio-tools==1.34.1 -q
!pip install git+https://github.com/deepset-ai/haystack.git -q
!wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.03.tar.gz
!tar -xvf xpdf-tools-linux-4.03.tar.gz && sudo cp xpdf-tools-linux-4.03/bin64/pdftotext /usr/local/bin
!pip install git+https://github.com/deepset-ai/haystack.git -q

# Here are the imports we need
from haystack.document_stores.elasticsearch import ElasticsearchDocumentStore
from haystack.nodes import PreProcessor, TransformersDocumentClassifier, FARMReader, ElasticsearchRetriever
from haystack.schema import Document
from haystack.utils import convert_files_to_dicts, fetch_archive_from_http, print_answers

02/02/2022 10:36:29 - INFO - faiss.loader -   Loading faiss with AVX2 support.
02/02/2022 10:36:29 - INFO - faiss.loader -   Could not load library with AVX2 support due to:
ModuleNotFoundError("No module named 'faiss.swigfaiss_avx2'",)
02/02/2022 10:36:29 - INFO - faiss.loader -   Loading faiss.
02/02/2022 10:36:29 - INFO - faiss.loader -   Successfully loaded faiss.
02/02/2022 10:36:33 - INFO - farm.modeling.prediction_head -   Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-4-6ff421127e9c> in <module>
      1 # Here are the imports we need
----> 2 from haystack.document_stores.elasticsearch import ElasticsearchDocumentStore
      3 from haystack.nodes import PreProcessor, TransformersDocumentClassifier, FARMReader, ElasticsearchRetriever
      4 from haystack.schema import Document
      5 from haystack.utils import convert_files_to_dicts, fetch_archive_from_http, print_answers

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/__init__.py in <module>
      3 import pandas as pd
      4 from haystack.schema import Document, Label, MultiLabel, BaseComponent
----> 5 from haystack.finder import Finder
      6 from haystack.pipeline import Pipeline
      7 

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/finder.py in <module>
      6 from collections import defaultdict
      7 
----> 8 from haystack.reader.base import BaseReader
      9 from haystack.retriever.base import BaseRetriever
     10 from haystack import MultiLabel

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/reader/__init__.py in <module>
----> 1 from haystack.reader.farm import FARMReader
      2 from haystack.reader.transformers import TransformersReader

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/reader/farm.py in <module>
     22 
     23 from haystack import Document
---> 24 from haystack.document_store.base import BaseDocumentStore
     25 from haystack.reader.base import BaseReader
     26 

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/document_store/__init__.py in <module>
      2 from haystack.document_store.faiss import FAISSDocumentStore
      3 from haystack.document_store.memory import InMemoryDocumentStore
----> 4 from haystack.document_store.milvus import MilvusDocumentStore
      5 from haystack.document_store.sql import SQLDocumentStore

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/haystack/document_store/milvus.py in <module>
      5 import numpy as np
      6 
----> 7 from milvus import IndexType, MetricType, Milvus, Status
      8 from scipy.special import expit
      9 from tqdm import tqdm

ModuleNotFoundError: No module named 'milvus'

pip install milvus

import milvus

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-3-91c33e248077> in <module>
----> 1 import milvus

ModuleNotFoundError: No module named 'milvus'

     |articleID | keywords                                               | 
     |:-------- |:------------------------------------------------------:| 
0    |58b61d1d  | ['Second Avenue (Manhattan, NY)']                      |     
1    |58b6393b  | ['Crossword Puzzles']                                  |          
2    |58b6556e  | ['Workplace Hazards and Violations', 'Trump, Donald J']|            
3    |58b657fa  | ['Trump, Donald J', 'Speeches and Statements'].        |  

     |articleID | keywords                                               | trumpMention |
     |:-------- |:------------------------------------------------------:| ------------:|
0    |58b61d1d  | ['Second Avenue (Manhattan, NY)']                      | False        |      
1    |58b6393b  | ['Crossword Puzzles']                                  | False        |          
2    |58b6556e  | ['Workplace Hazards and Violations', 'Trump, Donald J']| True         |           
3    |58b657fa  | ['Trump, Donald J', 'Speeches and Statements'].        | True         |       

df['trumpMention'] = np.where(any(df['keywords']) == 'Trump, Donald J', True, False) 

df['trumpMention'] = df['keywords'].apply(lambda x: any(token == 'Trump, Donald J') for token in x) 

lst = ['Trump, Donald J']  
df['trumpMention'] = df['keywords'].apply(lambda x: ([ True for token in x if any(token in lst)]))   

df = pd.DataFrame({'articleID': ['58b61d1d', '58b6393b', '58b6556e', '58b657fa'],
                   'keywords': [['Second Avenue (Manhattan, NY)'],
                                ['Crossword Puzzles'],
                                ['Workplace Hazards and Violations', 'Trump, Donald J'],
                                ['Trump, Donald J', 'Speeches and Statements']],
                   'trumpMention': [False, False, True, True]})

def score(model, tokenizer, sentence,  mask_token_id=103):
  tensor_input = tokenizer.encode(sentence, return_tensors='pt')
  repeat_input = tensor_input.repeat(tensor_input.size(-1)-2, 1)
  mask = torch.ones(tensor_input.size(-1) - 1).diag(1)[:-2]
  masked_input = repeat_input.masked_fill(mask == 1, 103)
  labels = repeat_input.masked_fill( masked_input != 103, -100)
  loss,_ = model(masked_input, masked_lm_labels=labels)
  result = np.exp(loss.item())
  return result

score(model, tokenizer, '我爱你') # returns 45.63794545581973

from transformers import pipeline, BertForMaskedLM, BertForMaskedLM, AutoTokenizer, RobertaForMaskedLM, AlbertForMaskedLM, ElectraForMaskedLM
import torch

1)
tokenizer = AutoTokenizer.from_pretrained("bioformers/bioformer-cased-v1.0")
model = BertForMaskedLM.from_pretrained("bioformers/bioformer-cased-v1.0")
2)
tokenizer = AutoTokenizer.from_pretrained("sultan/BioM-ELECTRA-Large-Generator")
model = ElectraForMaskedLM.from_pretrained("sultan/BioM-ELECTRA-Large-Generator")

sentence = "PERSON is ADJECTIVE"
dictionary = {"PERSON": ["Alice", "Bob", "Carol"], "ADJECTIVE": ["cute", "intelligent"]}

Alice is cute
Alice is intelligent
Bob is cute
Bob is intelligent
Carol is cute
Carol is intelligent

dictionary = {"PERSON": ["Alice", "Bob", "Carol"], "ADJECTIVE": ["cute", "intelligent"]}

for i in dictionary["PERSON"]:
    for j in dictionary["ADJECTIVE"]:
        print(f"{i} is {j}")

sentence = "PERSON is ADJECTIVE and is from COUNTRY" 
dictionary = {"PERSON": ["Alice", "Bob", "Carol"], "ADJECTIVE": ["cute", "intelligent"], "COUNTRY": ["USA", "Japan", "China", "India"]}

Alice is cute and is from USA
Alice is intelligent and is from USA
.
.
.
.
Carol is intelligent and is from India

from transformers import AutoModel 
model = AutoModel.from_pretrained('distilbert-base-uncase')

from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification('distilbert-base-uncase')