from sentence_transformers import SentenceTransformer
import faiss, json
from pathlib import Path
from PyPDF2 import PdfReader
import numpy as np

BASE = Path(__file__).resolve().parent.parent
PDF = BASE / "data" / "Permenpar 6 Tahun 2025.pdf"
OUT_INDEX = BASE / "rag" / "vector.index"
OUT_CHUNKS = BASE / "rag" / "chunks.jsonl"

model = SentenceTransformer("intfloat/multilingual-e5-large-instruct")

# baca PDF
reader = PdfReader(str(PDF))
pages = [p.extract_text() for p in reader.pages]

chunks = []
for page in pages:
    if page:
        for part in page.split("\n\n"):
            if len(part) > 50:
                chunks.append(part.strip())

# embed
emb = model.encode(chunks, convert_to_numpy=True).astype("float32")
faiss.normalize_L2(emb)

# FAISS index
index = faiss.IndexFlatIP(emb.shape[1])
index.add(emb)

faiss.write_index(index, str(OUT_INDEX))

with open(OUT_CHUNKS, "w", encoding="utf-8") as f:
    for c in chunks:
        f.write(json.dumps({"text": c}) + "\n")

print("DONE — FAISS dibuat pakai E5!")
