Spaces:
Paused
Paused
Download model/praproses/data_checking.py from SkillSync-II/skillsync-cli: direct link, hf CLI and curl.
- Browser
- Download file 1.17 kB
-
https://huggingface.co/spaces/SkillSync-II/skillsync-cli/resolve/main/model/praproses/data_checking.py
- Command line
-
hf download hf://spaces/SkillSync-II/skillsync-cli/model/praproses/data_checking.py
-
curl -L -o data_checking.py https://huggingface.co/spaces/SkillSync-II/skillsync-cli/resolve/main/model/praproses/data_checking.py
1.17 kB
| import pandas as pd | |
| import numpy as np | |
| import re | |
| from utils.text_processing import clean_text | |
| def check_data_quality(df): | |
| # Check for missing values | |
| missing_values = df.isnull().sum() | |
| # Check text length | |
| df['text_length'] = df['text'].apply(len) | |
| min_length = df['text_length'].min() | |
| max_length = df['text_length'].max() | |
| avg_length = df['text_length'].mean() | |
| # Check special characters | |
| def count_special_chars(text): | |
| return len(re.findall(r'[^\w\s]', text)) | |
| df['special_chars'] = df['text'].apply(count_special_chars) | |
| avg_special_chars = df['special_chars'].mean() | |
| return { | |
| "missing_values": missing_values.to_dict(), | |
| "text_length_stats": { | |
| "min": min_length, | |
| "max": max_length, | |
| "average": avg_length | |
| }, | |
| "avg_special_chars": avg_special_chars | |
| } | |
| def clean_dataset(df): | |
| # Remove rows with missing text | |
| df = df.dropna(subset=['text']) | |
| # Clean text | |
| df['cleaned_text'] = df['text'].apply(clean_text) | |
| # Remove duplicates | |
| df = df.drop_duplicates(subset=['cleaned_text']) | |
| return df |