-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Expand file tree
/
Copy pathdetr.py
More file actions
2865 lines (2578 loc) · 160 KB
/
Copy pathdetr.py
File metadata and controls
2865 lines (2578 loc) · 160 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# ------------------------------------------------------------------------
# RF-DETR
# Copyright (c) 2025 Roboflow. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 [see LICENSE for details]
# ------------------------------------------------------------------------
from __future__ import annotations
import contextlib
import importlib
import io
import json
import operator
import os
import tempfile
import warnings
from collections import defaultdict
from collections.abc import Callable
from copy import copy, deepcopy
from functools import wraps
from pathlib import Path
from typing import TYPE_CHECKING, Any, Concatenate, Literal, ParamSpec, TypeVar, cast
from urllib.parse import urlparse
import numpy as np
import requests
import torch
import torchvision.transforms.functional as F # noqa: N812
import yaml
from deprecate import deprecated
from PIL import Image
from rfdetr._namespace import _namespace_from_configs
from rfdetr.assets.coco_classes import COCO_CLASS_NAMES, COCO_CLASSES
from rfdetr.assets.model_weights import download_pretrain_weights, get_model_cache_dir
from rfdetr.config import ModelConfig, TrainConfig
from rfdetr.datasets._keypoint_schema import (
active_keypoint_counts,
infer_coco_keypoint_schema,
infer_yolo_keypoint_schema,
)
from rfdetr.datasets.coco import annotated_category_ids, filter_parent_categories, is_valid_coco_dataset
from rfdetr.datasets.yolo import REQUIRED_YOLO_YAML_FILES, is_valid_yolo_dataset
from rfdetr.inference import ModelContext, _build_model_context
from rfdetr.models.backbone.dinov2 import DinoV2
from rfdetr.utilities.distributed import is_main_process
from rfdetr.utilities.keypoints import _is_bg_first_schema, precision_cholesky_to_pixel_covariance
from rfdetr.utilities.logger import get_logger
if TYPE_CHECKING:
from supervision import Detections, KeyPoints
try:
torch.set_float32_matmul_precision("high")
except Exception:
pass
logger = get_logger()
_P = ParamSpec("_P")
_R = TypeVar("_R")
# ModelContext and _build_model_context are eagerly imported above (runtime use in get_model).
_VARIANT_EXPORTS = (
"RFDETRBase",
"RFDETRKeypointPreview",
"RFDETRLarge",
"RFDETRLargeDeprecated",
"RFDETRMedium",
"RFDETRNano",
"RFDETRSeg",
"RFDETRSeg2XLarge",
"RFDETRSegLarge",
"RFDETRSegMedium",
"RFDETRSegNano",
"RFDETRSegPreview",
"RFDETRSegSmall",
"RFDETRSegXLarge",
"RFDETRSmall",
)
__all__ = ["RFDETR", "ModelContext", *_VARIANT_EXPORTS]
_CHECKPOINT_MODEL_NAME_EXCLUDED_SYMBOLS = frozenset({"RFDETRLargeDeprecated", "RFDETRSeg"})
_CHECKPOINT_MODEL_NAME_CLASS_SYMBOLS: tuple[str, ...] = tuple(
class_symbol for class_symbol in _VARIANT_EXPORTS if class_symbol not in _CHECKPOINT_MODEL_NAME_EXCLUDED_SYMBOLS
)
_CHECKPOINT_PLUS_MODEL_NAME_CLASS_SYMBOLS: tuple[str, ...] = ("RFDETRXLarge", "RFDETR2XLarge")
_CHECKPOINT_MODEL_MAP_ENTRIES: tuple[tuple[str, str], ...] = (
("keypoint-preview", "RFDETRKeypointPreview"),
("seg-2xlarge", "RFDETRSeg2XLarge"),
("seg-xxlarge", "RFDETRSeg2XLarge"),
("seg-xlarge", "RFDETRSegXLarge"),
("seg-large", "RFDETRSegLarge"),
("seg-medium", "RFDETRSegMedium"),
("seg-small", "RFDETRSegSmall"),
("seg-nano", "RFDETRSegNano"),
("seg-preview", "RFDETRSegPreview"),
("large", "RFDETRLarge"),
("medium", "RFDETRMedium"),
("small", "RFDETRSmall"),
("nano", "RFDETRNano"),
("base", "RFDETRBase"),
)
_CHECKPOINT_PLUS_MODEL_MAP_ENTRIES: tuple[tuple[str, str], ...] = (
("2xlarge", "RFDETR2XLarge"),
("xxlarge", "RFDETR2XLarge"),
("xlarge", "RFDETRXLarge"),
)
def _validate_shape_dims(
shape: object,
block_size: int,
patch_size: int,
num_windows: int,
) -> tuple[int, int]:
"""Validate a user-supplied ``(height, width)`` shape tuple and return normalised plain-int dims.
Args:
shape: The raw value supplied by the caller (e.g. from ``export(shape=...)`` or
``predict(shape=...)``). Must be a two-element sequence of positive integers (or integer-compatible types
accepted by :func:`operator.index`).
block_size: Required divisor for both dimensions. Equals ``patch_size * num_windows``.
patch_size: Backbone patch size — used only in error messages.
num_windows: Number of attention windows — used only in error messages.
Returns:
A ``(height, width)`` tuple of plain Python :class:`int` values.
Raises:
ValueError: If ``shape`` cannot be unpacked as a two-element sequence, if either
dimension is a bool, float, or other non-integer type, if either dimension is not positive, or if either
dimension is not divisible by ``block_size``.
"""
try:
raw_height, raw_width = cast("tuple[Any, Any]", shape)
except (TypeError, ValueError):
raise ValueError(f"shape must be a sequence of two positive integers (height, width), got {shape!r}.") from None
for dim_name, dim in (("height", raw_height), ("width", raw_width)):
if isinstance(dim, bool):
raise ValueError(f"shape {dim_name} must be an integer, got {type(dim).__name__} (shape={shape!r}).")
try:
operator.index(dim)
except TypeError:
raise ValueError(
f"shape {dim_name} must be an integer, got {type(dim).__name__} (shape={shape!r}).",
) from None
if dim <= 0:
raise ValueError(f"shape must contain positive integers for height and width, got {shape!r}.")
# Normalise to plain Python ints; also accepts numpy.int64, torch scalars, etc.
height, width = operator.index(raw_height), operator.index(raw_width)
if height % block_size != 0 or width % block_size != 0:
raise ValueError(
f"shape must have both dimensions divisible by {block_size} "
f"(patch_size={patch_size} * num_windows={num_windows}), got {shape!r}.",
)
return height, width
def _resolve_patch_size(patch_size: int | None, model_config: object, caller: str) -> int:
"""Resolve and validate the ``patch_size`` argument for :meth:`RFDETR.export` and :meth:`RFDETR.predict`.
Args:
patch_size: Value supplied by the caller, or ``None`` to read from ``model_config``.
model_config: The model's configuration object. Must expose ``patch_size`` as a
positive integer attribute when ``patch_size`` is ``None`` or when a mismatch check is needed.
caller: Name of the calling method (``"export"`` or ``"predict"``) — used in
error messages to help the caller locate the problem.
Returns:
A validated, positive :class:`int` patch size.
Raises:
ValueError: If the resolved or provided ``patch_size`` is not a positive integer,
or if a caller-provided value disagrees with ``model_config.patch_size``.
"""
if patch_size is None:
patch_size = getattr(model_config, "patch_size", 14)
else:
if isinstance(patch_size, bool) or not isinstance(patch_size, int) or patch_size <= 0:
raise ValueError(f"patch_size must be a positive integer, got {patch_size!r}")
model_patch_size = getattr(model_config, "patch_size", None)
if model_patch_size is not None and patch_size != model_patch_size:
raise ValueError(
f"{caller}(patch_size={patch_size}) does not match the instantiated model's "
f"patch_size={model_patch_size}. Patch size is an architectural parameter; "
f"omit patch_size to use the model's configured value.",
)
if isinstance(patch_size, bool) or not isinstance(patch_size, int) or patch_size <= 0:
raise ValueError(f"patch_size must be a positive integer, got {patch_size!r}")
return patch_size
def _move_model_context_to_device(model_ctx: Any) -> None:
"""Move model weights to the target device recorded in *model_ctx*.
``_build_model_context`` intentionally keeps the ``nn.Module`` on CPU so that ``RFDETR.__init__`` does not
initialise CUDA (which would prevent DDP strategies from forking in notebook environments). This helper performs
the deferred ``.to(device)`` on first use.
It is safe to call on duck-typed stand-ins (e.g. ``SimpleNamespace``); the function silently returns when the
expected attributes are missing.
"""
target = getattr(model_ctx, "device", None)
inner = getattr(model_ctx, "model", None)
if target is None or inner is None or not hasattr(inner, "parameters"):
return
if isinstance(target, str):
target = torch.device(target)
if target.type == "cuda" and target.index is None:
# An index-less ``torch.device("cuda")`` never compares equal to the indexed device (e.g. ``cuda:0``) a
# real parameter reports once placed, even when they name the same physical GPU — resolve it to the index
# ``.to("cuda")`` would actually place on, so the guard below can detect "already on the right device" and
# skip re-moving every parameter on every call.
target = torch.device(target.type, torch.cuda.current_device())
first_param = next(inner.parameters(), None)
if first_param is not None and first_param.device != target:
# ``predict()`` stacks ``@torch.inference_mode()`` on top of ``@_ensure_model_on_device``, so the deferred
# move can run while inference mode is active. Tensors materialised by ``.to()`` under inference mode become
# *inference tensors*: they can never require gradients, so a later ``train()`` or auto-batch probe would
# silently produce no gradients. Disable inference mode for the move so deferred device placement is safe
# regardless of decorator order.
with torch.inference_mode(False):
model_ctx.model = inner.to(target)
def _ensure_model_on_device(method: Callable[Concatenate[Any, _P], _R]) -> Callable[Concatenate[Any, _P], _R]:
"""Decorate RF-DETR instance methods that require lazy model device placement.
The wrapped method receives the same arguments and return value as the original method. Before calling it, the
decorator moves ``self.model.model`` to ``self.model.device`` if the model context is available and the weights are
still on a different device. This keeps public inference methods clean while preserving deferred CUDA initialization
during ``RFDETR.__init__``.
"""
@wraps(method)
def wrapper(self: Any, *args: _P.args, **kwargs: _P.kwargs) -> _R:
_move_model_context_to_device(getattr(self, "model", None))
return method(self, *args, **kwargs)
typed_wrapper = cast("Callable[Concatenate[Any, _P], _R]", wrapper)
return typed_wrapper
def _prepare_run_config(
detector: RFDETR, *, for_eval: bool = False, **kwargs: Any
) -> tuple[TrainConfig, str | None, list[int] | None]:
"""Absorb the special run kwargs and build a :class:`~rfdetr.config.TrainConfig`.
Shared by :meth:`RFDETR.train` and :meth:`RFDETR.evaluate` so both accept exactly the same keyword arguments.
Handles the kwargs that are not plain ``TrainConfig`` fields: ``device`` (mapped to PyTorch Lightning
accelerator/devices), and
``resolution`` (a ``ModelConfig`` field applied to ``detector.model_config`` in place, with the positional-encoding
size and cached inference context kept in sync). Remaining kwargs are forwarded to
:meth:`RFDETR.get_train_config`; ``batch_size="auto"`` is resolved by auto-batch probing (skipped when
``for_eval=True`` — see below).
``model_config.model_name`` is set to the concrete subclass name.
Defined at module scope (rather than as a method) so that :meth:`RFDETR.train` / :meth:`RFDETR.evaluate`, which are
unit-tested by calling the unbound method with a ``MagicMock`` ``self``, still run the real preprocessing: a
module-level function is not intercepted by the mock, so no per-test monkeypatching of this helper is needed.
Args:
detector: The :class:`RFDETR` instance whose ``model_config`` and model context the run targets.
for_eval: When ``True`` (set by :meth:`RFDETR.evaluate`), ``batch_size="auto"`` is not resolved by probing —
the probe forces train mode and runs a forward+backward pass to size for the *training* memory envelope
(retained activations and gradients), which both wastes compute on a call that never backprops and
under-sizes the eval batch relative to what a no-grad forward pass can actually fit. Falls back to
:attr:`TrainConfig.batch_size`'s default instead.
**kwargs: Keyword arguments accepted by :meth:`RFDETR.train` / :meth:`RFDETR.evaluate`.
Returns:
``(train_config, accelerator, devices)`` where ``accelerator`` / ``devices`` are PTL trainer kwargs derived
from the ``device`` kwarg (``None`` when ``device`` was not provided).
Raises:
ValueError: If ``resolution`` is not a positive integer or is not divisible by
``patch_size * num_windows`` for the model variant.
"""
# Imported eagerly (not only under batch_size="auto") so a broken/absent training
# submodule surfaces its original ModuleNotFoundError here rather than being masked.
from rfdetr.training.auto_batch import resolve_auto_batch_config
# Parse `device` kwarg and map it to PTL accelerator/devices.
# Supports torch-style strings and torch.device (e.g. "cuda:1").
_device = kwargs.pop("device", None)
_accelerator, _devices = RFDETR._resolve_trainer_device_kwargs(_device)
# Apply resolution override to model_config before building the train config.
# resolution is a ModelConfig field, not a TrainConfig field, so we pop it
# here to avoid it being silently ignored by TrainConfig.
_resolution = kwargs.pop("resolution", None)
if _resolution is not None:
if isinstance(_resolution, bool):
raise ValueError("resolution must be a positive integer")
try:
_resolution = operator.index(_resolution)
except TypeError as error:
raise ValueError("resolution must be a positive integer") from error
if _resolution <= 0:
raise ValueError("resolution must be a positive integer")
block_size = detector.model_config.patch_size * detector.model_config.num_windows
if _resolution % block_size != 0:
raise ValueError(
f"resolution={_resolution} is not divisible by "
f"patch_size ({detector.model_config.patch_size}) * num_windows "
f"({detector.model_config.num_windows}) = {block_size}. "
f"Choose a resolution that is a multiple of {block_size}."
)
# Smart PE update: only recompute positional_encoding_size when the
# current config derives it formulaically (PE == resolution // patch_size).
# Configs with a pretrained-specific PE (e.g. RFDETRBase uses DINOv2's
# PE=37 at 518 px, training at 560 px) must not have PE silently changed
# — doing so causes shape mismatches when loading pretrained checkpoints.
_current_pe = detector.model_config.positional_encoding_size
_derived_pe = detector.model_config.resolution // detector.model_config.patch_size
if _current_pe == _derived_pe:
# Formula-derived: update PE proportionally to the new resolution.
new_pe = _resolution // detector.model_config.patch_size
detector.model_config.positional_encoding_size = new_pe
else:
# Pretrained-specific PE; leave it unchanged.
new_pe = _current_pe
detector.model_config.resolution = _resolution
# Keep the cached inference/export context in sync with model_config so
# predict()/export()/deployment all see the same resolution metadata.
if hasattr(detector, "model") and detector.model is not None:
if hasattr(detector.model, "resolution"):
detector.model.resolution = _resolution
model_args = getattr(detector.model, "args", None)
if model_args is not None:
if hasattr(model_args, "resolution"):
model_args.resolution = _resolution
if hasattr(model_args, "positional_encoding_size"):
model_args.positional_encoding_size = new_pe
config = detector.get_train_config(**kwargs)
if config.batch_size == "auto" and for_eval:
# The probe sizes for the *training* memory envelope (forward + retained activations +
# backward), which evaluate() never needs and cannot benefit from — falls back to the
# TrainConfig default micro-batch instead of running a wasted train-mode probe.
default_batch_size = TrainConfig.model_fields["batch_size"].default
logger.info(
"[auto-batch] batch_size='auto' is a train-only feature; evaluate() uses the default "
"micro-batch (%s) instead of probing.",
default_batch_size,
)
config.batch_size = default_batch_size
elif config.batch_size == "auto":
# Auto-batch probing runs forward/backward on the actual model, which
# must be on the target device (typically CUDA). Lazy placement keeps
# the model on CPU until first use — move it now.
_move_model_context_to_device(detector.model)
auto_batch = resolve_auto_batch_config(
model_context=detector.model,
model_config=detector.model_config,
train_config=config,
)
config.batch_size = auto_batch.safe_micro_batch
config.grad_accum_steps = auto_batch.recommended_grad_accum_steps
logger.info(
"[auto-batch] resolved train config: batch_size=%s grad_accum_steps=%s effective_batch_size=%s",
config.batch_size,
config.grad_accum_steps,
auto_batch.effective_batch_size,
)
detector.model_config.model_name = type(detector).__name__
return config, _accelerator, _devices
class RFDETR:
"""The base RF-DETR class implements the core methods for training RF-DETR models, running inference on the models,
optimising models, and uploading trained models for deployment."""
means = [0.485, 0.456, 0.406]
stds = [0.229, 0.224, 0.225]
size: str | None = None
_model_config_class: type[ModelConfig] = ModelConfig
_train_config_class: type[TrainConfig] = TrainConfig
def __init__(self, *, trust_checkpoint: bool = False, **kwargs: Any) -> None:
"""Initialize with ModelConfig fields as keyword arguments.
Passes all remaining kwargs to the variant's ModelConfig. Unknown kwargs raise
``pydantic.ValidationError``. See the variant's config class for available
parameters (e.g. ``RFDETRSmallConfig``).
Args:
trust_checkpoint: When ``True``, allow ``pretrain_weights`` to fall back to full
pickle deserialization (``weights_only=False``) if safe loading fails. Set this
only when ``pretrain_weights`` points to a checkpoint you explicitly trust (e.g.
when called internally by :meth:`from_checkpoint`); left ``False`` by default for
the ordinary construction path, which only ever downloads official Roboflow-hosted
weights.
**kwargs: ModelConfig field values (e.g. ``resolution``, ``num_classes``,
``pretrain_weights``, ``gradient_checkpointing``).
"""
self.model_config = self.get_model_config(**kwargs)
self.maybe_download_pretrain_weights()
self.model = self.get_model(self.model_config, trust_checkpoint=trust_checkpoint)
self.callbacks: dict[str, list[Callable[..., Any]]] = defaultdict(list)
self.means = list(self.means)
self.stds = list(self.stds)
# repeat means and stds for non-rgb images
if self.model_config.num_channels != 3:
from itertools import cycle
self.means = [val for _, val in zip(range(self.model_config.num_channels), cycle(self.means))]
self.stds = [val for _, val in zip(range(self.model_config.num_channels), cycle(self.stds))]
self.model.inference_model = None
self._is_optimized_for_inference = False
self._has_warned_about_not_being_optimized_for_inference = False
self._optimized_has_been_compiled = False
self._optimized_batch_size: int | None = None
self._optimized_resolution: int | None = None
self._optimized_dtype: torch.dtype | None = None
self._optimized_inplace = False
# Whether the currently optimized `inference_model` was exported with embeddings enabled.
# Unlike the eager model, the exported/traced forward pass can't take `return_embeddings`
# as a runtime argument, so this is fixed at `inference()` time.
self._optimized_return_embeddings = False
self._has_been_trained = False
def maybe_download_pretrain_weights(self) -> None:
"""Download pre-trained weights if they are not already downloaded.
Bare filenames (no directory component, e.g. ``rf-detr-base.pth``) are resolved to the model cache directory —
set the ``RF_HOME`` environment variable to override the location (default: ``~/.roboflow/models``). Resolution
happens in ``ModelConfig.expand_path`` for explicitly-provided values, and here as a fallback for field defaults
(which Pydantic does not validate by default).
Paths that already contain a directory component are used as-is; the parent directory is created if it does not
yet exist.
"""
if self.model_config.pretrain_weights is None:
return
pretrain_weights = str(self.model_config.pretrain_weights)
if not os.path.dirname(pretrain_weights):
# Field default was not processed by expand_path — resolve to cache dir.
cache_dir = get_model_cache_dir()
os.makedirs(cache_dir, exist_ok=True)
pretrain_weights = os.path.join(cache_dir, pretrain_weights)
else:
os.makedirs(os.path.dirname(pretrain_weights), exist_ok=True)
self.model_config.pretrain_weights = pretrain_weights
download_pretrain_weights(pretrain_weights)
def get_model_config(self, **kwargs: Any) -> ModelConfig:
"""Retrieve the configuration parameters used by the model."""
return self._model_config_class(**kwargs)
@classmethod
def from_checkpoint(cls, path: str | os.PathLike[str], *, trust_checkpoint: bool = False, **kwargs: Any) -> RFDETR:
"""Load an RF-DETR model from a training checkpoint, automatically inferring the model class.
The correct subclass is resolved in order of preference:
1. ``model_name`` key in the checkpoint (written by the PTL training
stack since v1.7.0).
2. ``pretrain_weights`` field in the checkpoint's ``args`` entry
(legacy fallback for older checkpoints).
3. The **filename** of *path* itself, used as a last resort when
``pretrain_weights`` is absent or an unset-like sentinel value
(empty string, ``"none"``, or ``"null"``). Starter weights
published by Roboflow store ``pretrain_weights="none"`` in their
``args``; passing the canonical filename (e.g.
``rf-detr-small.pth``) lets ``from_checkpoint`` infer the class
automatically.
Both legacy ``argparse.Namespace`` checkpoints (produced by ``engine.py``) and dict-style checkpoints (produced
by the PTL training stack) are supported.
Args:
path: Path to a checkpoint file (e.g. ``checkpoint_best_total.pth``).
trust_checkpoint: When ``True``, fall back to ``weights_only=False``
(full pickle) if safe deserialization fails. Only set this for
checkpoints from fully trusted sources; the default ``False``
keeps the safe loading path and raises if it cannot succeed.
Applies to both the initial checkpoint read here and the
constructor's own reload of the same file via
:func:`~rfdetr.models.weights.load_pretrain_weights`.
**kwargs: Additional keyword arguments forwarded to the model
constructor (e.g. ``accept_platform_model_license=True`` for XLarge / 2XLarge models).
``num_classes`` is resolved in this priority order:
1. Explicit caller kwarg — always wins.
2. Weight inference from ``class_embed.weight`` shape in the checkpoint
(``shape[0] - 1``, since the head includes a background class). This
overrides a stale ``model_config`` value written before fine-tuning
changed the class count.
3. ``saved_model_config["num_classes"]`` from the checkpoint's
``model_config`` entry — may be stale for older checkpoints.
4. Legacy ``args["num_classes"]`` dict entry.
5. Constructor default.
In cases 2–5 the field is not recorded as a user-set override, so
:meth:`train` can still adapt the detection head to the training
dataset's class count. Pass an explicit ``num_classes=N`` to pin
the head and prevent adaptation.
Returns:
An instance of the appropriate :class:`RFDETR` subclass loaded from the checkpoint.
Warning:
By default this method attempts safe deserialization
(``weights_only=True``). Pass ``trust_checkpoint=True`` only for
checkpoints from fully trusted sources, as it enables full pickle
deserialization which can execute arbitrary code.
Raises:
FileNotFoundError: If *path* does not exist.
OSError: If *path* exists but cannot be read.
KeyError: If the checkpoint does not contain an ``"args"`` key.
ValueError: If the model class cannot be inferred from ``model_name``,
``pretrain_weights``, or the checkpoint filename.
Examples:
>>> model = RFDETR.from_checkpoint("checkpoint_best_total.pth") # doctest: +SKIP
>>> model = RFDETRSmall.from_checkpoint("checkpoint_best_total.pth") # doctest: +SKIP
"""
# Local import breaks the variants → detr import cycle.
import rfdetr.variants as rfdetr_variants
_plus_available = False
_plus_symbols: dict[str, type[RFDETR]] = {}
_plus_entries: list[tuple[str, type[RFDETR]]] = []
from rfdetr.platform import _IS_RFDETR_PLUS_AVAILABLE
if _IS_RFDETR_PLUS_AVAILABLE:
try:
import rfdetr.platform.models as platform_models
for class_symbol in _CHECKPOINT_PLUS_MODEL_NAME_CLASS_SYMBOLS:
plus_obj = getattr(platform_models, class_symbol)
_plus_symbols[class_symbol] = plus_obj
_plus_entries = [
(name, _plus_symbols[class_symbol]) for name, class_symbol in _CHECKPOINT_PLUS_MODEL_MAP_ENTRIES
]
_plus_available = True
except ModuleNotFoundError as ex:
if ex.name not in {"rfdetr_plus", "rfdetr_plus.models"}:
raise
# Use the safe-load helper which tries weights_only=True first (with
# legacy argparse.Namespace safe globals), falling back to full pickle
# only when the caller explicitly passes trust_checkpoint=True.
from rfdetr.utilities.io import _safe_torch_load
ckpt: dict[str, Any] = _safe_torch_load(str(path), trust=trust_checkpoint)
args = ckpt["args"]
_variant_name_to_class: dict[str, type[RFDETR]] = {
getattr(variant_obj, "__name__", symbol): variant_obj
for symbol in dir(rfdetr_variants)
if symbol.startswith("RFDETR")
for variant_obj in [getattr(rfdetr_variants, symbol)]
}
_variant_symbols: dict[str, type[RFDETR]] = {
class_symbol: _variant_name_to_class[class_symbol] for class_symbol in _CHECKPOINT_MODEL_NAME_CLASS_SYMBOLS
}
# Build in three explicit segments: seg-* entries, then plus-model entries
# (xlarge/2xlarge), then base entries — order determines lookup priority.
_seg_map: list[tuple[str, type[RFDETR]]] = [
(name, _variant_symbols[class_symbol])
for name, class_symbol in _CHECKPOINT_MODEL_MAP_ENTRIES
if name.startswith("seg-")
]
_keypoint_map: list[tuple[str, type[RFDETR]]] = [
(name, _variant_symbols[class_symbol])
for name, class_symbol in _CHECKPOINT_MODEL_MAP_ENTRIES
if "keypoint" in name
]
_base_map: list[tuple[str, type[RFDETR]]] = [
(name, _variant_symbols[class_symbol])
for name, class_symbol in _CHECKPOINT_MODEL_MAP_ENTRIES
if not name.startswith("seg-") and "keypoint" not in name
]
_model_map: list[tuple[str, type[RFDETR]]] = _seg_map + _keypoint_map + _plus_entries + _base_map
# New checkpoints store model_name directly — use it when available.
_name_map: dict[str, type[RFDETR]] = dict(_variant_symbols)
# Plus-model classes are resolved only when rfdetr_plus is installed.
if _plus_available:
_name_map.update(_plus_symbols)
# RFDETRLargeDeprecated is excluded from _CHECKPOINT_MODEL_NAME_CLASS_SYMBOLS
# (so forward name-map lookups always reach RFDETRLarge), but it must be
# in _name_map so that checkpoints carrying model_name="RFDETRLargeDeprecated"
# are reloaded with the correct class instead of falling through to the
# substring matcher (which would wrongly pick RFDETRLarge and fail with a
# pydantic literal_error on encoder / projector_scale fields).
# Note: RFDETRLargeDeprecated is a _DeprecatedProxy (pyDeprecate) and has no
# __name__; look it up by the string key it was registered under, then inject
# into _name_map so checkpoint matching resolves directly without the substring
# fallback. Tests assert this mapping exists (see tests/inference/test_from_checkpoint.py).
_large_deprecated_cls = _variant_name_to_class.get("RFDETRLargeDeprecated")
if _large_deprecated_cls is not None:
_name_map["RFDETRLargeDeprecated"] = _large_deprecated_cls
saved_model_name = ckpt.get("model_name")
model_cls: type[RFDETR] | None = None
if isinstance(saved_model_name, str):
normalized_name = saved_model_name.strip()
if normalized_name:
model_cls = _name_map.get(normalized_name)
else:
normalized_name = ""
# Fall back to pretrain_weights (legacy) or, when unset-like, the checkpoint filename.
if isinstance(args, dict):
weights_name = str(args.get("pretrain_weights", "")).strip().lower()
else:
weights_name = str(getattr(args, "pretrain_weights", "")).strip().lower()
# The sentinel set {"", "none", "null"} covers unset-like checkpoint values:
# "" — pretrain_weights key absent entirely
# "none" — checkpoint value was None or the literal string "none";
# after str(...).strip().lower() both normalize to the same sentinel.
# This is NOT an intentional "no pretraining" flag (see
# test_pretrain_weights_none_warns, which operates at the config
# level, not the checkpoint level)
# "null" — checkpoint stored the literal string "null" (for example from a
# YAML-originated value), which is also treated as unset-like here
_filename_fallback = False
if weights_name in {"", "none", "null"}:
weights_name = os.path.basename(os.fspath(path)).lower()
_filename_fallback = True
if model_cls is None:
# Guard: plus-only checkpoints should raise an actionable install error
# when rfdetr_plus is missing, regardless of whether class inference
# relies on model_name (new format) or pretrain_weights (legacy format).
plus_by_model_name = normalized_name in _CHECKPOINT_PLUS_MODEL_NAME_CLASS_SYMBOLS
plus_by_weights_name = (
"xlarge" in weights_name and "seg-" not in weights_name and "keypoint-preview" not in weights_name
)
if not _plus_available and (plus_by_model_name or plus_by_weights_name):
from rfdetr.platform import _INSTALL_MSG
raise ImportError(
f"Checkpoint model_name={saved_model_name!r}, pretrain_weights={weights_name!r} requires the "
f"rfdetr_plus package. " + _INSTALL_MSG.format(name="platform model downloads")
)
for name, klass in _model_map:
if name in weights_name:
model_cls = klass
break
if _filename_fallback and model_cls is not None:
logger.info(
"pretrain_weights unset in checkpoint %r; inferred model class %s from filename %r",
path,
getattr(model_cls, "__name__", repr(model_cls)),
weights_name,
)
if model_cls is None:
raise ValueError(
f"Could not infer model class from checkpoint at {path!r} "
f"(model_name={saved_model_name!r}, pretrain_weights={weights_name!r}). "
f"Please instantiate the model class directly."
)
if isinstance(args, dict):
num_classes: int | None = args.get("num_classes")
else:
num_classes = getattr(args, "num_classes", None)
constructor_kwargs: dict[str, Any] = {}
checkpoint_config_keys: set[str] = set() # keys injected from checkpoint, not from caller
# Resolve model config field set once — used for both saved_model_config parsing and
# weight-based schema inference guards (BaseConfig has extra="forbid"; unknown fields raise).
_model_config_class = getattr(model_cls, "_model_config_class", None)
_mc_fields: dict[str, Any] = {}
_mc_model_fields = getattr(_model_config_class, "model_fields", None)
if isinstance(_mc_model_fields, dict):
_mc_fields = _mc_model_fields
else:
_mc_legacy = getattr(_model_config_class, "__fields__", None)
if isinstance(_mc_legacy, dict):
_mc_fields = _mc_legacy
saved_model_config = ckpt.get("model_config")
if isinstance(saved_model_config, dict):
for key, value in saved_model_config.items():
if key == "pretrain_weights":
continue
if not _mc_fields or key in _mc_fields:
constructor_kwargs[key] = value
checkpoint_config_keys.add(key)
if num_classes is not None and "num_classes" not in kwargs:
constructor_kwargs["num_classes"] = num_classes
checkpoint_config_keys.add("num_classes")
# Infer schema-critical fields from checkpoint weights — these are authoritative when
# ``model_config`` is absent or stale (saved before ``model_config`` persistence was added,
# or saved with default values before fine-tuning changed the trained schema).
# User-supplied ``kwargs`` take precedence and are applied in the ``update`` call below.
_ckpt_weights: dict[str, Any] = ckpt.get("model") or {}
if not _ckpt_weights and "state_dict" in ckpt:
_pfx = "model."
_ckpt_weights = {}
for k, v in ckpt["state_dict"].items():
if k.startswith(_pfx):
key = k[len(_pfx) :]
# Strip optional torch.compile() wrapper prefix
if key.startswith("_orig_mod."):
key = key[len("_orig_mod.") :]
_ckpt_weights[key] = v
if _ckpt_weights:
# num_keypoints_per_class — inferred from _kp_active_mask (shape [num_classes, max_kp]).
# Reflects what the model actually learned; saved model_config may carry the COCO default
# [0, 17] even after fine-tuning on a different keypoint schema.
if "num_keypoints_per_class" not in kwargs and (not _mc_fields or "num_keypoints_per_class" in _mc_fields):
_kp_mask = _ckpt_weights.get("_kp_active_mask")
if isinstance(_kp_mask, torch.Tensor) and _kp_mask.ndim == 2:
_inferred_kp = [int(n) for n in _kp_mask.sum(dim=1).tolist()]
_current_kp = constructor_kwargs.get("num_keypoints_per_class")
if _inferred_kp != _current_kp:
logger.debug(
"from_checkpoint: overriding num_keypoints_per_class %s → %s "
"(inferred from _kp_active_mask; saved model_config may be stale).",
_current_kp,
_inferred_kp,
)
constructor_kwargs["num_keypoints_per_class"] = _inferred_kp
checkpoint_config_keys.add("num_keypoints_per_class")
# num_classes — inferred from class_embed.weight shape.
# The head shape is ground truth for what num_classes the checkpoint uses.
if "num_classes" not in kwargs:
_ce_weight = _ckpt_weights.get("class_embed.weight")
if isinstance(_ce_weight, torch.Tensor) and _ce_weight.ndim == 2:
_inferred_nc = _ce_weight.shape[0] - 1 # shape[0] = num_classes + 1 (background)
_current_nc = constructor_kwargs.get("num_classes")
if _inferred_nc != _current_nc:
logger.debug(
"from_checkpoint: overriding num_classes %s → %s "
"(inferred from class_embed.weight; saved model_config may be stale).",
_current_nc,
_inferred_nc,
)
constructor_kwargs["num_classes"] = _inferred_nc
checkpoint_config_keys.add("num_classes")
constructor_kwargs.update(kwargs)
# pretrain_weights is placed after **kwargs so it always wins even if
# a caller accidentally passes pretrain_weights inside kwargs.
constructor_kwargs["pretrain_weights"] = str(path)
# Model construction reloads this same file via load_pretrain_weights(); without this,
# trust_checkpoint=True would bypass the safe-load only for the metadata read above and
# then fail identically when the constructor re-reads pretrain_weights.
constructor_kwargs["trust_checkpoint"] = trust_checkpoint
# Fields injected from the checkpoint but not supplied by the caller must not be
# treated as explicit user overrides in Pydantic's model_fields_set. Downstream
# alignment guards (e.g. _align_num_classes_from_dataset,
# _align_keypoint_schema_from_dataset, load_pretrain_weights) all read
# model_fields_set to decide whether to adapt model internals to the training
# dataset — leaving checkpoint-derived fields marked as user-set breaks them.
checkpoint_derived_keys = checkpoint_config_keys - set(kwargs)
model = model_cls(**constructor_kwargs)
# The instance now carries checkpoint (trained) weights; flag it so a later
# train() call warns that it will restart from pretrain_weights, not continue.
model._has_been_trained = True
if checkpoint_derived_keys:
loaded_config = getattr(model, "model_config", None)
# model_fields_set is the public API and returns the live backing set
# in Pydantic v2; fall back to the private attribute only if that changes.
fields_set = getattr(loaded_config, "model_fields_set", None)
if fields_set is None:
fields_set = getattr(loaded_config, "__pydantic_fields_set__", None)
if fields_set is not None:
fields_set.difference_update(checkpoint_derived_keys)
# Verify num_classes specifically — if Pydantic ever returns a snapshot instead
# of the live backing set, this assertion will catch the silent regression before
# it causes a training-time head-adaptation failure.
if "num_classes" in checkpoint_derived_keys:
assert "num_classes" not in getattr(loaded_config, "model_fields_set", set()), (
"num_classes still in model_fields_set after checkpoint load; "
"Pydantic may return a snapshot rather than the live backing set — "
"switch to model_construct(_fields_set=...) for Pydantic v3 compatibility."
)
return model
@staticmethod
def _resolve_trainer_device_kwargs(device: Any) -> tuple[str | None, list[int] | None]:
"""Map a torch-style device specifier to PTL ``accelerator``/``devices`` kwargs.
Args:
device: A device specifier accepted by ``torch.device``.
Returns:
``(accelerator, devices)`` where ``devices`` is ``None`` unless an explicit device index is provided (for
example ``cuda:1``). ``device.type == "xla"`` maps to ``accelerator="tpu"`` -- PTL's accelerator
registry has no ``"xla"`` string; ``"tpu"`` is its canonical name for the XLA backend.
Raises:
ValueError: If ``device`` is not a valid torch device specifier.
"""
if device is None:
return None, None
try:
resolved_device = torch.device(device)
except (TypeError, ValueError, RuntimeError) as exc:
raise ValueError(
f"Invalid device specifier for train(): {device!r}. "
"Expected values like 'cpu', 'cuda', 'cuda:0', or torch.device(...).",
) from exc
if resolved_device.type == "cpu":
return "cpu", None
if resolved_device.type == "cuda":
return "gpu", [resolved_device.index] if resolved_device.index is not None else None
if resolved_device.type == "mps":
return "mps", [resolved_device.index] if resolved_device.index is not None else None
if resolved_device.type == "xla":
# PTL's accelerator registry has no "xla" string -- "tpu" is its canonical name for
# the XLA backend (torch.device("xla") is valid and .type is always "xla", even on
# TPU; torch.device("tpu") itself raises RuntimeError). Bridge explicitly instead of
# falling through to the auto-detection warning below.
return "tpu", [resolved_device.index] if resolved_device.index is not None else None
warnings.warn(
f"Device type {resolved_device.type!r} is not explicitly mapped to a PyTorch Lightning "
"accelerator; falling back to PTL auto-detection. Training may use an unexpected device.",
UserWarning,
stacklevel=2,
)
return None, None
def train(self, **kwargs: Any) -> None:
"""Train an RF-DETR model via the PyTorch Lightning stack.
All keyword arguments are forwarded to :meth:`get_train_config` to build a :class:`~rfdetr.config.TrainConfig`.
Several kwargs are absorbed and handled specially so that existing call-sites do not break:
* ``resolution`` — updates the model's input resolution by mutating
:attr:`model_config.resolution` in place before the train config is built. This change persists on
:attr:`model_config` after :meth:`train` returns. The value must be a positive integer divisible by
``patch_size * num_windows`` for the model variant; a :class:`ValueError` is raised otherwise.
:attr:`model_config.positional_encoding_size` is also updated when the config derives it formulaically (``PE
== resolution // patch_size``); configs with a pretrained-specific PE value (e.g. ``RFDETRBase`` uses DINOv2's
PE=37 at 560 px) are left unchanged to preserve checkpoint compatibility.
* ``device`` — normalized via :class:`torch.device` and mapped to PyTorch
Lightning trainer arguments. ``"cpu"`` becomes ``accelerator="cpu"``; ``"cuda"`` and ``"cuda:N"`` become
``accelerator="gpu"`` and optionally ``devices=[N]``; ``"mps"`` becomes ``accelerator="mps"``; ``"xla"``
becomes ``accelerator="tpu"`` (PTL's canonical name for the XLA backend). Other valid torch device types
fall back to PTL auto-detection and emit a :class:`UserWarning`.
* ``notes`` — optional user-defined metadata (string, dict, list, or
any JSON-serialisable value) stored under the ``"notes"`` key in every ``.pth`` checkpoint produced during
training. The value is also available inside ``args["notes"]`` for full provenance. Pass the same value to
:meth:`export` to embed it in the ONNX file as well.
After training completes the underlying ``nn.Module`` is synced back onto ``self.model.model`` so that
:meth:`predict` and :meth:`export` continue to work without reloading the checkpoint.
Raises:
ImportError: If training dependencies are not installed. Install with
``pip install "rfdetr[train,loggers]"``.
ValueError: If ``resolution`` is not a positive integer or is not
divisible by ``patch_size * num_windows`` for the model variant.
"""
# The training stack lives in the `rfdetr[train]` extras group — a missing
# `pytorch_lightning` (or any other training-extras package) causes the import to fail,
# and the remediation is `pip install "rfdetr[train,loggers]"`.
try:
from rfdetr.training import RFDETRDataModule, RFDETRModelModule, build_trainer
except ModuleNotFoundError as exc:
# Preserve internal import errors so packaging/regression issues in
# rfdetr.* are not misreported as missing optional extras.
if exc.name and exc.name.startswith("rfdetr."):
raise
raise ImportError(
"RF-DETR training dependencies are missing. "
'Install them with `pip install "rfdetr[train,loggers]"` and try again.',
) from exc
if getattr(self, "_has_been_trained", False):
warnings.warn(
"Calling train() on a model that has already been trained or loaded from a checkpoint. "
"The new training run will start from the original pretrained weights (pretrain_weights), "
"NOT from the in-memory trained state. To continue training, pass resume=<checkpoint_path>.",
UserWarning,
stacklevel=2,
)
# Absorb the special train/evaluate kwargs (device, resolution, deprecated knobs),
# build the TrainConfig, and resolve any auto batch size. Shared with evaluate()
# so both accept exactly the same keyword arguments.
config, _accelerator, _devices = _prepare_run_config(self, **kwargs)
# Auto-detect num_classes from the training dataset and align model_config.
# This must run before RFDETRModelModule is constructed so that weight loading
# inside the module uses the correct (dataset-derived) class count.
dataset_dir = getattr(config, "dataset_dir", None)
if dataset_dir:
self._align_keypoint_schema_from_dataset(config)
self._align_num_classes_from_dataset(dataset_dir)
module = RFDETRModelModule(self.model_config, config)
datamodule = RFDETRDataModule(self.model_config, config)
# Guard with LOCAL_RANK env var rather than is_main_process() because torch.distributed
# is not yet initialized here (it is set up inside trainer.fit()). In Lightning DDP
# subprocesses, LOCAL_RANK is set by the launcher before the subprocess calls train(),
# so this correctly identifies rank 0 even before dist.init_process_group() runs.
if config.save_dataset_grids and os.environ.get("LOCAL_RANK", "0") == "0":
try:
from rfdetr.datasets.save_grids import DatasetGridSaver
datamodule.setup("fit")
grids_output_dir = Path(config.output_dir) / "dataset_grids"
DatasetGridSaver(datamodule.train_dataloader(), grids_output_dir, dataset_type="train").save_grid()
DatasetGridSaver(datamodule.val_dataloader(), grids_output_dir, dataset_type="val").save_grid()
except Exception:
logger.warning(
"Failed to save dataset grids; training will continue without them.",
exc_info=True,
)
if config.resume:
# BestModelCallback's four lightweight checkpoint files (unlike the trainer's own
# `last.ckpt` / `checkpoint_<epoch>.ckpt`, which retain full PTL state) intentionally
# omit optimizer/LR-scheduler state to stay small — see
# BestModelCallback._build_checkpoint_payload. They retain model weights, epoch
# metadata, and per-callback state only when the resumed callback configuration
# matches the saved keys. Checkpoints written before callback-state persistence have
# no such state. Best-score tracking additionally requires exactly the original
# output_dir because of PTL's ModelCheckpoint.load_state_dict() dirpath gate.
# Flag the optimizer/scheduler gap explicitly instead of letting it pass silently.
_light_checkpoint_names = frozenset(
{"checkpoint_best_regular.pth", "checkpoint_best_ema.pth", "checkpoint_best_total.pth", "last_ema.pth"}
)
if Path(config.resume).name in _light_checkpoint_names:
from rfdetr.utilities.io import _safe_torch_load
# checkpoint.get("callbacks") is None-checked by PTL's own
# _call_callbacks_load_state_dict(), which no-ops (skipping every callback's
# restoration) when the key is absent or empty. Checkpoints written before
# BestModelCallback._build_checkpoint_payload started persisting per-callback state
# — or from a run where every registered callback happened to have empty state —
# are exactly this case, so peek at the file rather than let the warning below
# overclaim a restoration that silently does not happen.
_resume_ckpt = _safe_torch_load(config.resume, trust=True)
_has_callback_state = bool(_resume_ckpt.get("callbacks"))
del _resume_ckpt
_resume_dir = Path(config.resume).resolve().parent
_configured_output_dir = Path(config.output_dir).resolve()
_best_score_restores = _resume_dir == _configured_output_dir
if _has_callback_state:
logger.warning(
"resume=%r points at one of BestModelCallback's lightweight checkpoints, "
"which intentionally omit optimizer/LR-scheduler state to stay small. "
"Model weights and epoch count will resume. Callback state can restore only "
"for matching configured callbacks; the optimizer and LR scheduler restart cold. "
"To resume with optimizer/scheduler state too, pass the trainer's full "
"checkpoint instead (e.g. %s/last.ckpt or %s/checkpoint_<epoch>.ckpt).",
config.resume,
config.output_dir,
config.output_dir,
)
else:
logger.warning(
"resume=%r points at one of BestModelCallback's lightweight checkpoints, "
"which intentionally omit optimizer/LR-scheduler state to stay small. "
"Model weights and epoch count will resume, but this particular file has no "
"saved callback state (it predates callback-state persistence, or every "
"registered callback had nothing to save), so best-score tracking, EMA, and "
"early-stopping state all restart cold this run too — not just the "
"optimizer and LR scheduler. To resume with full state, pass the trainer's "
"full checkpoint instead (e.g. %s/last.ckpt or %s/checkpoint_<epoch>.ckpt).",
config.resume,
config.output_dir,
config.output_dir,
)
# BestModelCallback always writes these four files directly under its own
# `dirpath` (== output_dir at save time; see BestModelCallback.__init__ and
# _build_checkpoint_payload). PTL's ModelCheckpoint.load_state_dict() only
# restores best_model_score/best_k_models/kth_value/last_model_path when the
# resumed dirpath matches the checkpoint's saved dirpath exactly (model_checkpoint.py,
# installed pytorch-lightning) — with a different output_dir this run only recovers
# best_model_path, so the first metric logged after resume looks like an automatic
# improvement over an empty best_model_score. Only worth flagging when there was
# callback state to lose in the first place.
if _has_callback_state and not _best_score_restores:
logger.warning(
"resume=%r was written under %s but output_dir=%r points elsewhere. "
"PyTorch Lightning only restores best_model_score/best_k_models when "
"output_dir matches the checkpoint's original directory exactly — with "
"this output_dir, best-score tracking (BestModelCallback's high-water "
"mark) restarts fresh in the new directory instead of resuming. Set "
"output_dir=%r to keep it.",
config.resume,