Add deprecation warnings to docstrings (#4083)

Co-authored-by: Quentin Gallouédec <[email protected]>

Author: albertvillanova

trl/trainer/online_dpo_config.py

@@ -62,6 +62,14 @@ class may differ from those in [`~transformers.TrainingArguments`].
 
         dataset_num_proc (`int`, *optional*):
             Number of processes to use for processing the dataset.
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Since OnlineDPO does not involve
+            dataset preparation, you can safely remove it.
+
+            </Deprecated>
+
         disable_dropout (`bool`, *optional*, defaults to `True`):
             Whether to disable dropout in the model and reference model.
 

trl/trainer/online_dpo_trainer.py

@@ -168,12 +168,13 @@ class OnlineDPOTrainer(Trainer):
         preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`):
             The function to use to preprocess the logits before computing the metrics.
 
-    .. deprecated:: 0.22.0
-        The following parameters are deprecated and will be removed in a future version:
+        reward_model:
 
-        * `reward_model`: Use `reward_funcs` instead. For example, change `reward_model=model` to `reward_funcs=model`.
-        * `reward_processing_class`: Use `reward_processing_classes` instead. For example, change
-          `reward_processing_class=tokenizer` to `reward_processing_classes=tokenizer`.
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `reward_funcs` instead.
+
+            </Deprecated>
     """
 
     _tag_names = ["trl", "online-dpo"]

trl/trainer/rloo_config.py

@@ -190,6 +190,142 @@ class RLOOConfig(TrainingArguments):
         wandb_log_unique_prompts (`bool`, *optional*, defaults to `False`):
             Whether to log unique prompts in wandb. If `True`, only unique prompts are logged. If `False`, all prompts
             are logged.
+
+        > Deprecated parameters
+
+        rloo_k:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `num_generations` instead.
+
+            </Deprecated>
+
+        cliprange:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `epsilon` instead.
+
+            </Deprecated>
+
+        kl_coef:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `beta` instead.
+
+            </Deprecated>
+
+        exp_name:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `run_name` instead.
+
+            </Deprecated>
+
+        normalize_reward:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `normalize_advantages` instead.
+
+            </Deprecated>
+
+        num_ppo_epochs:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `num_iterations` instead.
+
+            </Deprecated>
+
+        num_mini_batches:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `steps_per_generation` instead.
+
+            </Deprecated>
+
+        total_episodes:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `max_steps` instead.
+
+            </Deprecated>
+
+        response_length:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `max_completion_length` instead.
+
+            </Deprecated>
+
+        token_level_kl:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. KL is now computed only at the sequence
+            level.
+
+            </Deprecated>
+
+        dataset_num_proc:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. This parameter was unused, you can
+            safely remove it from your scripts.
+
+            </Deprecated>
+
+        local_rollout_forward_batch_size:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Now it is automatically set to
+            `per_device_train_batch_size` (or `per_device_eval_batch_size` during evaluation).
+
+            </Deprecated>
+
+        num_sample_generations:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `logging_steps` to control
+            generation logging frequency.
+
+            </Deprecated>
+
+        stop_token:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0.
+
+            </Deprecated>
+
+        stop_token_id:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `processing_class.eos_token_id`
+            instead.
+
+            </Deprecated>
+
+        missing_eos_penalty:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Replicate with a custom reward function
+            checking if `eos_token_id` is in `completion_ids`.
+
+            </Deprecated>
     """
 
     _VALID_DICT_FIELDS = TrainingArguments._VALID_DICT_FIELDS + ["model_init_kwargs"]

trl/trainer/rloo_trainer.py

@@ -197,6 +197,47 @@ def reward_func(completions, **kwargs):
             model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.
         peft_config ([`~peft.PeftConfig`], *optional*):
             PEFT configuration used to wrap the model. If `None`, the model is not wrapped.
+
+        config:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `args` instead.
+
+            </Deprecated>
+
+        reward_model:
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `reward_funcs` instead.
+
+            </Deprecated>
+
+        policy:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. Use `model` instead.
+
+            </Deprecated>
+
+        ref_policy:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. To use the initial model as the
+            reference model, simply omit this parameter. The parameter is ignored.
+
+            </Deprecated>
+
+        data_collator:
+
+            <Deprecated version="0.22.0">
+
+            This parameter is deprecated and will be removed in version 0.25.0. The RLOOTrainer does not use a data
+            collator, so this parameter is ignored.
+
+            </Deprecated>
     """
 
     _tag_names = ["trl", "rloo"]