refaudit

 30class Cache:
 31    def __init__(self, path: str | Path, ttl_days: float = 90.0) -> None:
 32        self.path = Path(path)
 33        self._lock_path = self.path.with_name(self.path.name + ".lock")
 34        self.ttl = ttl_days * 86400.0
 35        self._data: dict[str, dict[str, Any]] = {}
 36        self._dirty = False
 37        # Checking runs across threads, and get() both reads and evicts, so the
 38        # dict needs guarding rather than relying on which operations happen to
 39        # be atomic today.
 40        self._lock = threading.RLock()
 41        self._load()
 42
 43    def _read_file(self) -> dict[str, dict[str, Any]]:
 44        """Entries currently on disk, or none if the file is absent or unusable."""
 45        try:
 46            blob = json.loads(self.path.read_text(encoding="utf-8"))
 47        except (OSError, json.JSONDecodeError):
 48            return {}  # corrupt or unreadable: start clean rather than crash
 49        if blob.get("schema") != SCHEMA_VERSION:
 50            return {}
 51        entries = blob.get("entries")
 52        return entries if isinstance(entries, dict) else {}
 53
 54    def _load(self) -> None:
 55        self._data = self._read_file()
 56
 57    def get(self, key: str) -> dict[str, Any] | None:
 58        with self._lock:
 59            item = self._data.get(key)
 60            if not item:
 61                return None
 62            if self.ttl and time.time() - item.get("stored_at", 0) > self.ttl:
 63                self._data.pop(key, None)
 64                self._dirty = True
 65                return None
 66            return item.get("value")
 67
 68    def put(self, key: str, value: dict[str, Any]) -> None:
 69        with self._lock:
 70            self._data[key] = {"stored_at": time.time(), "value": value}
 71            self._dirty = True
 72
 73    def flush(self) -> None:
 74        # Snapshot under the lock: flush() is called periodically during a run,
 75        # while worker threads are still writing results. Without this,
 76        # json.dumps walks a dict another thread is mutating.
 77        with self._lock:
 78            if not self._dirty:
 79                return
 80            mine = dict(self._data)
 81            self._dirty = False
 82
 83        # Merge rather than overwrite. Two people running in the same directory
 84        # -- or two terminals, or a cluster job array -- share this file, and a
 85        # wholesale write would replace whatever the other run had finished with
 86        # only what this one happens to hold. Entries are independent, so the
 87        # union is always the better answer, and the newer timestamp wins a tie.
 88        #
 89        # This narrows the losing window to the moment between reading and
 90        # replacing rather than closing it: a lost entry costs a repeated lookup
 91        # on the next run, never a wrong result, which does not justify making
 92        # every install depend on file locking that differs per platform.
 93        # Read, merge and replace under one lock. Merging alone only narrowed
 94        # the race -- another run could still finish a whole flush between our
 95        # read and our replace, and be overwritten by it.
 96        with exclusive(self._lock_path):
 97            merged = self._read_file()
 98            for key, item in mine.items():
 99                existing = merged.get(key)
100                if existing is None or item.get("stored_at", 0) >= existing.get("stored_at", 0):
101                    merged[key] = item
102
103            payload = json.dumps(
104                {"schema": SCHEMA_VERSION, "entries": merged}, ensure_ascii=False
105            )
106            self.path.parent.mkdir(parents=True, exist_ok=True)
107            fd, tmp = tempfile.mkstemp(dir=str(self.path.parent), suffix=".tmp")
108            try:
109                with os.fdopen(fd, "w", encoding="utf-8") as fh:
110                    fh.write(payload)
111                    fh.flush()
112                    os.fsync(fh.fileno())
113                os.replace(tmp, self.path)      # atomic on POSIX and Windows
114            finally:
115                if os.path.exists(tmp):
116                    try:
117                        os.unlink(tmp)
118                    except OSError:
119                        pass
120
121    # PYI034 prefers Self, which is 3.11+; this package supports 3.10.
122    def __enter__(self) -> Cache:  # noqa: PYI034
123        return self
124
125    def __exit__(self, *exc: object) -> None:
126        self.flush()

130@dataclass(frozen=True)
131class CheckResult:
132    key: str
133    verdict: Verdict
134    entry_title: str = ""
135    found_title: str = ""
136    source: str = ""
137    similarity: float | None = None
138    note: str = ""
139    cited: bool | None = None
140
141    def as_row(self) -> dict[str, object]:
142        return {
143            "key": self.key,
144            "verdict": self.verdict.value,
145            "is_finding": self.verdict.is_finding,
146            "source": self.source,
147            "similarity": "" if self.similarity is None else round(self.similarity, 3),
148            "cited": "" if self.cited is None else self.cited,
149            "entry_title": self.entry_title,
150            "found_title": self.found_title,
151            "note": self.note,
152        }

 70class Checker:
 71    def __init__(
 72        self,
 73        resolvers: Sequence[Resolver],
 74        *,
 75        cache: Cache | None = None,
 76        thresholds: Thresholds = DEFAULT_THRESHOLDS,
 77        doi_existence: DoiExistence | None = None,
 78    ) -> None:
 79        if not resolvers:
 80            raise ValueError("at least one resolver is required")
 81        self.resolvers = list(resolvers)
 82        self.cache = cache
 83        self.thresholds = thresholds
 84        # Optional so a caller can run fully offline against a stub; when it is
 85        # absent no DEAD_DOI can be issued, which is the safe direction.
 86        self.doi_existence = doi_existence
 87
 88    # -- public ------------------------------------------------------------
 89
 90    def check(self, entry: Entry, *, cited: bool | None = None) -> CheckResult:
 91        cached = self.cache.get(self._cache_key(entry)) if self.cache else None
 92        if cached:
 93            return CheckResult(
 94                key=entry.key,
 95                verdict=Verdict(cached["verdict"]),
 96                entry_title=entry.title,
 97                found_title=cached.get("found_title", ""),
 98                source=cached.get("source", ""),
 99                similarity=cached.get("similarity"),
100                note=cached.get("note", ""),
101                cited=cited,
102            )
103
104        result = self._check_uncached(entry, cited)
105
106        # Only cache outcomes that reflect the entry, not our connectivity.
107        if self.cache and result.verdict is not Verdict.UNVERIFIED:
108            self.cache.put(
109                self._cache_key(entry),
110                {
111                    "verdict": result.verdict.value,
112                    "found_title": result.found_title,
113                    "source": result.source,
114                    "similarity": result.similarity,
115                    "note": result.note,
116                },
117            )
118        return result
119
120    def check_all(self, entries: Iterable[Entry], cited: set[str] | None = None,
121                  workers: int = 1):
122        """Yield a result per entry, in input order.
123
124        ``workers`` overlaps *entries*, never the sources within one entry: an
125        entry stops at its first hit, so firing every source at once would cost
126        third parties requests we then throw away. Politeness does not depend on
127        this number -- each service has its own token bucket, shared by all
128        threads -- so raising it recovers time lost to network latency without
129        ever exceeding the rate a service documents.
130        """
131        entries = list(entries)
132        was_cited = (lambda e: (e.key in cited) if cited is not None else None)
133        if workers <= 1 or len(entries) < 2:
134            for entry in entries:
135                yield self.check(entry, cited=was_cited(entry))
136            return
137        with ThreadPoolExecutor(max_workers=workers) as pool:
138            # map preserves input order, so a run is reproducible and the
139            # progress line still matches the file.
140            yield from pool.map(lambda e: self.check(e, cited=was_cited(e)), entries)
141
142    # -- internals ---------------------------------------------------------
143
144    @staticmethod
145    def _cache_key(entry: Entry) -> str:
146        # Include the fields we compare, so editing an entry re-checks it.
147        return "|".join([
148            entry.key,
149            clean_doi(entry.doi),
150            clean_arxiv_id(entry.arxiv_id),
151            entry.title.strip().lower()[:200],
152            str(entry.year or ""),
153        ])
154
155    def _check_uncached(self, entry: Entry, cited: bool | None) -> CheckResult:
156        applicable = [r for r in self.resolvers if r.can_handle(entry)]
157
158        has_identifier = bool(clean_doi(entry.doi) or clean_arxiv_id(entry.arxiv_id))
159        if not applicable:
160            if entry.entry_type in NON_ARCHIVAL_TYPES and not has_identifier:
161                return CheckResult(entry.key, Verdict.SKIPPED, entry.title,
162                                   note=f"@{entry.entry_type} with no identifier", cited=cited)
163            return CheckResult(entry.key, Verdict.NOT_FOUND, entry.title,
164                               note="no resolver could handle this entry", cited=cited)
165
166        any_unavailable: list[str] = []
167        authoritative_absence: list[str] = []
168        doi_disowned_by: list[str] = []
169        best_guess: tuple[float, Record, Resolver] | None = None
170
171        for resolver in applicable:
172            outcome = resolver.resolve(entry)
173
174            if isinstance(outcome, Unavailable):
175                any_unavailable.append(f"{outcome.source}: {outcome.reason[:60]}")
176                continue
177
178            if isinstance(outcome, NotFound):
179                # No single agency speaks for the whole DOI system, so one
180                # agency's "not mine" is only a candidate for DEAD_DOI. It is
181                # settled after every resolver has had its turn.
182                if resolver.name.endswith(":doi") and clean_doi(entry.doi):
183                    doi_disowned_by.append(resolver.name)
184                authoritative_absence.append(f"{outcome.source}: {outcome.detail[:50]}")
185                continue
186
187            if isinstance(outcome, Found):
188                # An identifier lookup is authoritative either way: if it
189                # resolves to a different paper, that disagreement *is* the
190                # finding, and no later source can overturn it.
191                if _is_identifier_lookup(resolver):
192                    return self._judge(entry, outcome.record, resolver,
193                                       has_identifier, cited)
194                # A title search is a guess. A convincing one ends the search;
195                # a poor one must not, or the first index to return anything at
196                # all would mask a better answer from the next -- which is how
197                # a real book stayed "not found" behind an empty Crossref hit.
198                score = similarity(entry.title, outcome.record.title)
199                if score >= self.thresholds.title_match:
200                    return self._judge(entry, outcome.record, resolver,
201                                       has_identifier, cited)
202                if best_guess is None or score > best_guess[0]:
203                    best_guess = (score, outcome.record, resolver)
204                continue
205
206        # Every title search was unconvincing; report the closest so the note
207        # says what was actually seen rather than a bare "not found".
208        if best_guess is not None and not doi_disowned_by:
209            _, record, resolver = best_guess
210            return self._judge(entry, record, resolver, has_identifier, cited)
211
212        # Nothing resolved the entry. If a DOI was disowned by every agency we
213        # asked, ask the DOI proxy -- which answers for all of them -- before
214        # calling the reference dead.
215        if doi_disowned_by:
216            doi = clean_doi(entry.doi)
217            registered = self.doi_existence.exists(doi) if self.doi_existence else None
218            if registered is False:
219                return CheckResult(entry.key, Verdict.DEAD_DOI, entry.title,
220                                   source="doi.org",
221                                   note=f"DOI {doi} is not registered with any agency",
222                                   cited=cited)
223            if registered is True:
224                # Real DOI, but no index we can read holds metadata for it, so
225                # the reference itself is unchecked rather than wrong.
226                return CheckResult(
227                    entry.key, Verdict.UNVERIFIED, entry.title, source="doi.org",
228                    note=f"DOI {doi} resolves but is indexed by none of: "
229                         f"{', '.join(doi_disowned_by)}",
230                    cited=cited)
231            return CheckResult(
232                entry.key, Verdict.UNVERIFIED, entry.title,
233                note=f"DOI {doi} not found in {', '.join(doi_disowned_by)}; "
234                     f"could not reach doi.org to confirm",
235                cited=cited)
236
237        if any_unavailable and not authoritative_absence:
238            return CheckResult(entry.key, Verdict.UNVERIFIED, entry.title,
239                               note="; ".join(any_unavailable)[:160], cited=cited)
240        if entry.entry_type in NON_ARCHIVAL_TYPES and not has_identifier:
241            return CheckResult(entry.key, Verdict.SKIPPED, entry.title,
242                               note=f"@{entry.entry_type} not indexed", cited=cited)
243        note = "; ".join(authoritative_absence + any_unavailable)[:160]
244        return CheckResult(entry.key, Verdict.NOT_FOUND, entry.title, note=note, cited=cited)
245
246    def _judge(
247        self,
248        entry: Entry,
249        record: Record,
250        resolver: Resolver,
251        has_identifier: bool,
252        cited: bool | None,
253    ) -> CheckResult:
254        score = similarity(entry.title, record.title)
255
256        def result(verdict: Verdict, note: str = "") -> CheckResult:
257            # Built explicitly rather than by unpacking a dict: the shared
258            # fields are identical for every branch, but keyword unpacking
259            # erases their types and hides genuine mistakes from the checker.
260            return CheckResult(
261                key=entry.key,
262                verdict=verdict,
263                entry_title=entry.title,
264                found_title=record.title,
265                source=resolver.name,
266                similarity=score,
267                note=note,
268                cited=cited,
269            )
270
271        if score < self.thresholds.title_match:
272            if _is_identifier_lookup(resolver):
273                # The identifier points at a different paper. This is the
274                # signature of a fabricated or mis-copied citation.
275                return result(Verdict.TITLE_MISMATCH,
276                              "identifier resolves to a different title")
277            if score < self.thresholds.title_suspect:
278                # A dataset or web resource with no identifier was never going
279                # to be in a citation index; a stray title hit is not a finding.
280                if entry.entry_type in NON_ARCHIVAL_TYPES and not has_identifier:
281                    return result(Verdict.SKIPPED,
282                                  f"@{entry.entry_type} with no identifier; not indexed")
283                return result(Verdict.NOT_FOUND, "no close title match found")
284            return result(Verdict.UNVERIFIED,
285                          "only a weak title match; no identifier to confirm")
286
287        want = first_surname(entry.get("author"))
288        if (want and record.first_author_surname
289                and not surnames_match(want, first_surname(record.first_author_surname))):
290            return result(Verdict.AUTHOR_MISMATCH,
291                          f"bib={want} vs {record.first_author_surname.lower()}")
292
293        year_counts = getattr(resolver, "year_is_authoritative", True)
294        if (year_counts and entry.year and record.year
295                and abs(entry.year - record.year) > self.thresholds.year_slack):
296            return result(Verdict.YEAR_MISMATCH, f"bib={entry.year} vs {record.year}")
297
298        return result(Verdict.OK)

18@dataclass(frozen=True)
19class Entry:
20    """A single bibliography entry as it appears in the .bib file."""
21
22    key: str
23    entry_type: str
24    fields: dict[str, str] = field(default_factory=dict)
25
26    def get(self, name: str, default: str = "") -> str:
27        return self.fields.get(name.lower(), default)
28
29    @property
30    def title(self) -> str:
31        return self.get("title")
32
33    @property
34    def doi(self) -> str:
35        return self.get("doi")
36
37    @property
38    def arxiv_id(self) -> str:
39        """The arXiv identifier, from its own field or from free text.
40
41        ``eprint`` is the correct place for it, but exports from Google Scholar
42        and similar tools leave it in the journal or note field instead. Those
43        entries are perfectly findable, so refusing to look costs a real check.
44        """
45        from .normalize import clean_arxiv_id, find_arxiv_id
46
47        explicit = self.get("eprint") or self.get("archiveprefix_id")
48        if clean_arxiv_id(explicit):
49            return explicit
50        for field_name in ("journal", "note", "howpublished", "booktitle", "url", "doi"):
51            found = find_arxiv_id(self.get(field_name))
52            if found:
53                return found
54        return explicit
55
56    @property
57    def year(self) -> int | None:
58        import re
59
60        m = re.search(r"\d{4}", self.get("year"))
61        return int(m.group()) if m else None

81@dataclass(frozen=True)
82class Found:
83    record: Record

86@dataclass(frozen=True)
87class NotFound:
88    """The source was reached and authoritatively has no such record."""
89
90    source: str
91    detail: str = ""

64@dataclass(frozen=True)
65class Record:
66    """A bibliographic record retrieved from an external source."""
67
68    source: str
69    title: str
70    year: int | None = None
71    first_author_surname: str = ""
72    doi: str = ""
73    url: str = ""

49@dataclass(frozen=True)
50class Thresholds:
51    title_match: float = 0.75      # at or above this, the titles are the same work
52    title_suspect: float = 0.45    # below this, a resolved record is clearly a different paper
53    year_slack: int = 1            # preprint/publication years legitimately differ by one

 94@dataclass(frozen=True)
 95class Unavailable:
 96    """The source could not be reached, or refused us. Says nothing about the entry."""
 97
 98    source: str
 99    reason: str = ""
100    retry_after: float | None = None

106class Verdict(enum.Enum):
107    """Ordered worst-first: iteration order is the triage order."""
108
109    TITLE_MISMATCH = "TITLE_MISMATCH"
110    DEAD_DOI = "DEAD_DOI"
111    AUTHOR_MISMATCH = "AUTHOR_MISMATCH"
112    YEAR_MISMATCH = "YEAR_MISMATCH"
113    NOT_FOUND = "NOT_FOUND"
114    UNVERIFIED = "UNVERIFIED"
115    SKIPPED = "SKIPPED"
116    OK = "OK"
117
118    @property
119    def is_finding(self) -> bool:
120        """True if this says something about the entry, rather than about us."""
121        return self in {
122            Verdict.TITLE_MISMATCH,
123            Verdict.DEAD_DOI,
124            Verdict.AUTHOR_MISMATCH,
125            Verdict.YEAR_MISMATCH,
126            Verdict.NOT_FOUND,
127        }

114def cited_keys(tex_paths: list[Path]) -> set[str]:
115    """Cite keys appearing in live (non-commented) LaTeX.
116
117    Uncited entries never reach the reference list, so checking them is optional
118    work; separating them also keeps the report focused on what a reviewer sees.
119    """
120    keys: set[str] = set()
121    for path in tex_paths:
122        try:
123            text = path.read_text(encoding="utf-8", errors="replace")
124        except OSError:
125            continue
126        for line in text.splitlines():
127            stripped = line.lstrip()
128            if stripped.startswith("%"):
129                continue
130            # drop trailing comments, honouring \%
131            line = re.sub(r"(?<!\\)%.*$", "", line)
132            for m in _CITE.finditer(line):
133                keys.update(k.strip() for k in m.group(1).split(",") if k.strip())
134    return keys

66def default_resolvers(contact_email: str, *, only: list[str] | None = None,
67                      timeout: float = 20.0) -> list[Resolver]:
68    names = only or list(AVAILABLE)
69    unknown = [n for n in names if n not in AVAILABLE]
70    if unknown:
71        raise ValueError(f"unknown resolver(s): {', '.join(unknown)}; "
72                         f"available: {', '.join(AVAILABLE)}")
73    # Preserve registry order regardless of the order given on the command line.
74    return [AVAILABLE[n](contact_email=contact_email, timeout=timeout)
75            for n in AVAILABLE if n in set(names)]

137def find_tex(root: str | Path) -> list[Path]:
138    root = Path(root)
139    if root.is_file():
140        return [root]
141    return sorted(p for p in root.rglob("*.tex") if p.is_file())

107def parse_file(path: str | Path) -> list[Entry]:
108    return parse_string(Path(path).read_text(encoding="utf-8", errors="replace"))

 85def parse_string(src: str) -> list[Entry]:
 86    entries: list[Entry] = []
 87    for m in _ENTRY_START.finditer(src):
 88        etype = m.group("type").lower()
 89        if etype in {"comment", "preamble", "string"}:
 90            continue
 91        body_start = src.index("{", m.start()) if "{" in src[m.start():m.end()] else m.end()
 92        body_end = _match_brace(src, body_start)
 93        body = src[m.end():body_end]
 94
 95        fields: dict[str, str] = {}
 96        i = 0
 97        while True:
 98            fm = _FIELD.search(body, i)
 99            if not fm:
100                break
101            value, i = _read_value(body, fm.end())
102            fields[fm.group(1).lower()] = _clean(value)
103        entries.append(Entry(key=m.group("key").strip(), entry_type=etype, fields=fields))
104    return entries