Spaces:
Runtime error
Runtime error
Download src/preprocessing.py from alpertml/TopicModelingForSummarization: direct link, hf CLI and curl.
- Browser
- Download file 969 Bytes
-
https://huggingface.co/spaces/alpertml/TopicModelingForSummarization/resolve/main/src/preprocessing.py
- Command line
-
hf download hf://spaces/alpertml/TopicModelingForSummarization/src/preprocessing.py
-
curl -L -o preprocessing.py https://huggingface.co/spaces/alpertml/TopicModelingForSummarization/resolve/main/src/preprocessing.py
969 Bytes
| import re | |
| from nltk.tokenize import RegexpTokenizer | |
| import spacy | |
| def remove_patterns(text): | |
| """ | |
| Remove punctions, emails, hashtags in given text | |
| """ | |
| if isinstance(text, spacy.tokens.span.Span): | |
| text = text.text | |
| # Remove return char | |
| text = re.sub(r'\n', ' ', text) | |
| # Remove emails | |
| text = re.sub(r'\S*@\S*\s?', '', text) | |
| # Remove hashtags | |
| text = re.sub(r'#\w+', '', text) | |
| # Remove punctuation | |
| text = re.sub(r'[^\w\s]', '', text) | |
| return text | |
| def extract_patterns(text): | |
| """ | |
| Extract punctions, emails, hashtags in given text | |
| """ | |
| # extract emails | |
| emails = re.findall(r'\S+@\S+', text) | |
| # extract hashtags | |
| hashtags = re.findall(r'#\w+', text) | |
| # extract punctuation | |
| punctuation = re.findall(r'[^\w\s]', text) | |
| return punctuation, emails, hashtags | |
| def remove_punct_nltk(text): | |
| tokenizer = RegexpTokenizer(r'\w+') | |
| tokenizer.tokenize(text) | |
| return text |