publications

@inproceedings{lerch_beyond_2026,

title = {Beyond Static Benchmarks: A Post-Leaderboard Evaluation Paradigm for Generative Music and Creative AI},

author = {Alexander Lerch},

url = {https://hal.science/hal-05673858},

year  = {2026},

date = {2026-01-01},

urldate = {2026-08-03},

booktitle = {Proceedings of the ACM AI Leadership Summit},

address = {Atlanta},

abstract = {Generative artificial intelligence has rapidly advanced the creation of music and other creative artifacts, yet the problem of how to evaluate these outputs remains unresolved. Current research mostly relies on static benchmarks and uninterpretable, single-score leaderboards that fail to capture the multi-faceted nature of music, ignore domain context, suffer from reproducibility issues, and are fundamentally incompatible with creative AI. This paper proposes a paradigm shift from rigid benchmarking to a modular, evolving, multidimensional, and community-governed evaluation ecosystem. The envisioned ecosystem cleanly decouples datasets, feature representations, and distance metrics to ensure continuous extensibility as generative models evolve. Crucially, Reference Benchmark Profiles are introduced to balance domain-specific flexibility with the cross-study standardization the field currently lacks, backed by rigorous metadata logging for traceability. By establishing evaluation as a transparent, evolving process rather than a static metric, this framework outlines a roadmap toward meaningful assessment of generative AI.},

keywords = {Concept, Evaluation, Evaluation Science, Framework, Generative AI, Metrics, music},

pubstate = {published},

tppubtype = {inproceedings}

}