Download scripts/06_label_with_fixes.py from LeeSek/binary-dockerfile-model: direct link, hf CLI and curl.
- Browser
- Download file 5.63 kB
-
https://huggingface.co/LeeSek/binary-dockerfile-model/resolve/main/scripts/06_label_with_fixes.py
- Command line
-
hf download hf://LeeSek/binary-dockerfile-model/scripts/06_label_with_fixes.py
-
curl -L -o 06_label_with_fixes.py https://huggingface.co/LeeSek/binary-dockerfile-model/resolve/main/scripts/06_label_with_fixes.py
5.63 kB
| # 06_label_with_fixes.py – wersja v4 | |
| import json | |
| import tempfile | |
| import subprocess | |
| from pathlib import Path | |
| from concurrent.futures import ProcessPoolExecutor, as_completed | |
| from datetime import datetime | |
| # === Ścieżki === | |
| INPUT_PATH = Path("data/raw/dockerfiles.jsonl") | |
| OUTPUT_PATH = Path("data/labeled/labeled_dockerfiles.jsonl") | |
| FAILED_LOG = Path("data/labeled/failed_dockerfiles.jsonl") | |
| MISSING_FIXES_LOG = Path("data/labeled/missing_fixes.txt") | |
| OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) | |
| FIXES_PATH = Path("data/fixes/fixes.json") | |
| HADOLINT_BIN = "hadolint" | |
| MAX_WORKERS = 6 | |
| TIMEOUT_SECONDS = 5 | |
| # === Globalny słownik fixów === | |
| with open(FIXES_PATH, encoding="utf-8") as f: | |
| FIXES = json.load(f) | |
| MISSING_FIXES = set() | |
| def attach_fixes(rules_triggered: list[str]) -> dict: | |
| suggestions = {} | |
| for rule in rules_triggered: | |
| if rule in FIXES: | |
| suggestions[rule] = FIXES[rule] | |
| else: | |
| MISSING_FIXES.add(rule) | |
| return suggestions | |
| def lint_dockerfile(entry: dict) -> dict: | |
| try: | |
| content = entry["content"] | |
| joined = "\n".join(content) | |
| with tempfile.NamedTemporaryFile("w", suffix=".Dockerfile", delete=False) as tmp: | |
| tmp.write(joined) | |
| tmp.flush() | |
| temp_path = tmp.name | |
| result = subprocess.run( | |
| [HADOLINT_BIN, temp_path, "-f", "json"], | |
| capture_output=True, | |
| text=True, | |
| timeout=TIMEOUT_SECONDS | |
| ) | |
| Path(temp_path).unlink(missing_ok=True) | |
| if result.returncode == 0: | |
| return { | |
| "label": "good", | |
| "rules_triggered": [], | |
| "lines": {}, | |
| "fix_suggestions": {}, | |
| "repo": entry["repo"], | |
| "path": entry["path"], | |
| "content": content, | |
| "timestamp": datetime.now().isoformat() | |
| } | |
| try: | |
| findings = json.loads(result.stdout) | |
| rules = sorted(set(item["code"] for item in findings if "code" in item)) | |
| line_map = {} | |
| for item in findings: | |
| code = item.get("code") | |
| line = item.get("line") | |
| if code and line: | |
| line_map.setdefault(code, line) | |
| fix_suggestions = attach_fixes(rules) | |
| except Exception as e: | |
| rules = ["lint-parse-error"] | |
| line_map = {} | |
| fix_suggestions = {} | |
| return { | |
| "label": "bad", | |
| "rules_triggered": rules, | |
| "lines": line_map, | |
| "fix_suggestions": fix_suggestions, | |
| "repo": entry["repo"], | |
| "path": entry["path"], | |
| "content": content, | |
| "timestamp": datetime.now().isoformat() | |
| } | |
| except subprocess.TimeoutExpired: | |
| return { | |
| "label": "bad", | |
| "rules_triggered": ["lint-timeout"], | |
| "lines": {}, | |
| "fix_suggestions": {}, | |
| "repo": entry.get("repo"), | |
| "path": entry.get("path"), | |
| "content": entry.get("content"), | |
| "timestamp": datetime.now().isoformat() | |
| } | |
| except Exception as e: | |
| return { | |
| "label": "bad", | |
| "rules_triggered": [f"lint-error:{str(e)}"], | |
| "lines": {}, | |
| "fix_suggestions": {}, | |
| "repo": entry.get("repo"), | |
| "path": entry.get("path"), | |
| "content": entry.get("content"), | |
| "timestamp": datetime.now().isoformat() | |
| } | |
| def main(): | |
| with open(INPUT_PATH, encoding="utf-8") as f: | |
| records = [json.loads(line) for line in f if line.strip()] | |
| print(f"🚀 Start analizy {len(records)} Dockerfile (wątki={MAX_WORKERS})") | |
| results, failed = [], [] | |
| with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor: | |
| futures = [executor.submit(lint_dockerfile, row) for row in records] | |
| for i, future in enumerate(as_completed(futures)): | |
| try: | |
| result = future.result() | |
| if "rules_triggered" not in result: | |
| failed.append(result) | |
| else: | |
| results.append(result) | |
| except Exception as e: | |
| failed.append({ | |
| "label": "bad", | |
| "rules_triggered": [f"future-error:{str(e)}"], | |
| "lines": {}, | |
| "fix_suggestions": {}, | |
| "repo": "unknown", | |
| "path": "unknown", | |
| "content": [], | |
| "timestamp": datetime.now().isoformat() | |
| }) | |
| if (i + 1) % 250 == 0: | |
| print(f" 🔄 {i+1}/{len(records)} przetworzonych...") | |
| with open(OUTPUT_PATH, "w", encoding="utf-8") as f_out: | |
| for rec in results: | |
| json.dump(rec, f_out) | |
| f_out.write("\n") | |
| with open(FAILED_LOG, "w", encoding="utf-8") as f_fail: | |
| for rec in failed: | |
| json.dump(rec, f_fail) | |
| f_fail.write("\n") | |
| if MISSING_FIXES: | |
| print(f"\n⚠️ Brakuje fixów dla {len(MISSING_FIXES)} reguł – zapisuję do {MISSING_FIXES_LOG}") | |
| with open(MISSING_FIXES_LOG, "w", encoding="utf-8") as f_miss: | |
| for rule in sorted(MISSING_FIXES): | |
| f_miss.write(rule + "\n") | |
| else: | |
| print("✅ Wszystkie reguły mają przypisany fix!") | |
| print(f"\n✅ Zapisano {len(results)} Dockerfile z etykietami i fixami → {OUTPUT_PATH}") | |
| print(f"❌ Nieudanych: {len(failed)} → {FAILED_LOG}") | |
| if __name__ == "__main__": | |
| main() | |