Filtering nsfw content for nomic

src/build_nomic.py

@@ -87,6 +87,10 @@ def preprocess_markdown(text):
 def build_nomic(dataset):
     df = dataset['train'].to_pandas()
 
+    # Filter df for nsfw content for displaying in Nomic
+    df = df[~df[['content', 'title', 'flair', 'permalink']].apply(
+        lambda x: x.str.contains('nsfw', case=False, na=False)).any(axis=1)]
+
     non_embedding_columns = ['date_utc', 'title', 'flair', 'poster', 'url', 'id', 'word_count',
                              'score', 'score_percentile', 'html_content', 'subreddit']