mirror of
https://github.com/microsoft/SkillOpt.git
synced 2026-07-07 16:49:08 +08:00
48 lines
2.6 KiB
YAML
48 lines
2.6 KiB
YAML
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Feature: soft / mixed validation-gate metric (community-contributed, PR #25)
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
#
|
|
# This is NOT a default SkillOpt setting and was NOT used to produce the
|
|
# numbers reported in the paper. It is provided as a reference for users
|
|
# who encounter a specific scenario where the default `hard` gate is too
|
|
# coarse to drive training.
|
|
#
|
|
# When to consider this:
|
|
# - You are running on a custom environment.
|
|
# - Your held-out *selection* split has very few items (e.g. ≤ ~10).
|
|
# - Your reward function is continuous / partial-credit (e.g. F1, BLEU,
|
|
# soft match) rather than purely binary 0/1.
|
|
#
|
|
# Symptom this addresses:
|
|
# With a small selection split + continuous rewards, candidate skills
|
|
# often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but
|
|
# never flip the discrete hard outcome. The default `hard` gate then
|
|
# rejects every candidate and training stalls. Switching the gate to
|
|
# `soft` or `mixed` lets these partial improvements be accepted.
|
|
#
|
|
# When NOT to use this:
|
|
# - When reproducing the paper. The paper-reported numbers were obtained
|
|
# under the default `hard` gate.
|
|
# - When your selection split is large (dozens+ items) and / or your
|
|
# reward is already binary — `hard` is the more conservative choice
|
|
# and matches the design described in the paper.
|
|
#
|
|
# To use: inherit your env config from this file, e.g.
|
|
# _base_: ../features/soft_gate.yaml
|
|
# or copy the `evaluation:` block below into your config.
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
_base_: ../_base_/default.yaml
|
|
|
|
evaluation:
|
|
# Three options:
|
|
# 'hard' — default; exact-match accuracy. Use this to reproduce the paper.
|
|
# 'soft' — per-item soft / partial-credit score (recommended for the
|
|
# small-split + continuous-reward scenario described above).
|
|
# 'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by
|
|
# `gate_mixed_weight` below.
|
|
gate_metric: soft
|
|
|
|
# Only used when gate_metric == 'mixed'. Ignored otherwise.
|
|
gate_mixed_weight: 0.5
|