@COMMENT This file was generated by bib2html.pl <http://www.cs.cmu.edu/~pfr/misc_software/index.html#bib2html> version 0.90
@COMMENT written by Patrick Riley <http://www.cs.cmu.edu/~pfr>
@COMMENT This file came from Peter Stone's publication pages at
@COMMENT http://www.cs.utexas.edu/~pstone/papers
@Article{brad_knox_TMLR2023,
  author   = {W. Bradley Knox and Stephane Hatgis-Kessell and Serena Booth and Scott Niekum and Peter Stone and Alessandro Allievi},
  title    = {Models of human preference for learning reward functions},
  journal = {Transactions on Machine Learning Research (TMLR)},
  year     = {2023},
  abstract = {The utility of reinforcement learning is limited by the alignment of reward
functions with the interests of human stakeholders. One promising method for
alignment is to learn the reward function from human-generated preferences
between pairs of trajectory segments, a type of reinforcement learning from human
feedback (RLHF). These human preferences are typically assumed to be informed
solely by partial return, the sum of rewards along each segment. We find this
assumption to be flawed and propose modeling human preferences instead as
informed by each segment's regret, a measure of a segment's deviation from
optimal decision-making. Given infinitely many preferences generated according to
regret, we prove that we can identify a reward function equivalent to the reward
function that generated those preferences, and we prove that the previous partial
return model lacks this identifiability property in multiple contexts. We
empirically show that our proposed regret preference model outperforms the
partial return preference model with finite training data in otherwise the same
setting. Additionally, we find that our proposed regret preference model better
predicts real human preferences and also learns reward functions from these
preferences that lead to policies that are better human-aligned. Overall, this
work establishes that the choice of preference model is impactful, and our
proposed regret preference model provides an improvement upon a core assumption
of recent research. We have open sourced our experimental code, the human
preferences dataset we gathered, and our training and preference elicitation
interfaces for gathering a such a dataset.
  },
}