wrapped

app_utils.py

@@ -18,6 +18,7 @@ from sumy.nlp.tokenizers import Tokenizer
 from rouge import Rouge
 from transformers import BartForConditionalGeneration, BartTokenizer
 from transformers import T5ForConditionalGeneration, T5Tokenizer
+from transformers import AutoTokenizer, AutoModelForTokenClassification
 
 # from nltk import ne_chunk
 from nltk.tag import StanfordNERTagger
@@ -112,14 +113,18 @@ def nlp_analysis(text):
     return df
 
 
+
 def find_entities(text):
-    stan = StanfordNERTagger(stanford_ner_model_path, stanford_ner_jar_path)
-    text=text.replace("\n\n","\n")
-    tokens = nltk.word_tokenize(text)
-    tagged_tokens = stan.tag(tokens)
-    entities = [(token, tag) for token, tag in tagged_tokens if tag != 'O']
+    tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    model = AutoModelForTokenClassification.from_pretrained("dbmdz/bert-large-cased-finetuned-conll03-english")
+    nlp = pipeline("ner", model=model, tokenizer=tokenizer)
+    e=nlp(text)
+    entities=[(entity["word"], entity["entity"]) for entity in e]
     entities=HTML_WRAPPER.format(entities)
     return entities
+    
+
+    
 
 
 def file_download(data):