import re from jev import ask THRESHOLD = 0.5 CITE = re.compile(r"\[(c\d+)\]") def cited_sentences(answer): sentences = re.split(r"(?<=[.!?\]])\s+(?=[A-Z0-9\"'*(])", answer.strip()) out = [] for sentence in sentences: ids = sorted(set(CITE.findall(sentence))) text = re.sub(r"\s+([.,;:!?])", r"\1", CITE.sub("", sentence)).strip() if ids and len(text) > 3: out.append({"sentence": text, "cites": ids}) return out def verify(answer, chunks): claims = [c for c in cited_sentences(answer) if all(int(i[1:]) <= len(chunks) for i in c["cites"])] if not claims: return {"checked": 0, "passed": True, "escalate": False, "claims": []} notes = {f"c{i + 1}": c["text"] for i, c in enumerate(chunks)} state = {"notes": notes, "sentences": {f"s{i}": c["sentence"] for i, c in enumerate(claims)}} questions = { f"s{i}": { "type": "noul", "instructions": f"Is every fact in `sentences.s{i}` stated in " + " or ".join(f"`notes.{cid}`" for cid in c["cites"]) + "?", "criteria": {"true": "The cited note states the facts, names, and numbers in the sentence.", "false": "The sentence adds or changes a fact, name, or number the cited note does not state."}, } for i, c in enumerate(claims) } answers = ask(state, questions, stage="verify") for i, claim in enumerate(claims): claim["support"] = answers[f"s{i}"]["noul"] weak = [c for c in claims if c["support"] < THRESHOLD] return {"checked": len(claims), "passed": not weak, "escalate": bool(weak), "claims": claims, "weak": weak}