publications

@inproceedings{liu_enhancing_2024,

title = {Enhancing Video Music Recommendation with Transformer-Driven Audio-Visual Embeddings},

author = {Shimiao Liu and Alexander Lerch},

url = {https://ieeexplore.ieee.org/abstract/document/10704086},

doi = {10.1109/IS262782.2024.10704086},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the IEEE International Symposium on the Internet of Sounds (IS2)},

address = {Erlangen},

abstract = {A fitting soundtrack can help a video better convey its content and provide a better immersive experience. This paper introduces a novel approach utilizing self-supervised learning and contrastive learning to automatically recommend audio for video content, thereby eliminating the need for manual labeling. We use a dual-branch cross-modal embedding model that maps both audio and video features into a common low-dimensional space. The fit of various audio-video pairs can then be modeled as inverse distance measure. In addition, a comparative analysis of various temporal encoding methods is presented, emphasizing the effectiveness of transformers in managing the temporal information of audio-video matching tasks. Through multiple experiments, we demonstrate that our model TIVM, which integrates transformer encoders and using InfoNCE loss, significantly improves the performance of audio-video matching and surpasses traditional methods.},

keywords = {Contrastive learning, Encoding, Fitting, Immersive experience, Internet, Labeling, Manuals, multi-modal, music, music recommendation, Recommender systems, trans-former, Transformers},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{wang_catch_2022,

title = {To Catch A Chorus, Verse, Intro, or Anything Else: Analyzing a Song with Structural Functions},

author = {Ju-Chiang Wang and Yun-Ning Hung and Jordan B. L. Smith},

url = {https://ieeexplore.ieee.org/abstract/document/9747252/authors#authors},

doi = {10.1109/ICASSP43922.2022.9747252},

year  = {2022},

date = {2022-05-01},

urldate = {2024-02-08},

booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},

pages = {416\textendash420},

abstract = {Conventional music structure analysis algorithms aim to divide a song into segments and to group them with abstract labels (e.g., ‘A’, ‘B’, and ‘C’). However, explicitly identifying the function of each segment (e.g., ‘verse’ or ‘chorus’) is rarely attempted, but has many applications. We introduce a multi-task deep learning framework to model these structural semantic labels directly from audio by estimating "verseness," "chorusness," and so forth, as a function of time. We propose a 7-class taxonomy (i.e., intro, verse, chorus, bridge, outro, instrumental, and silence) and provide rules to consolidate annotations from four disparate datasets. We also propose to use a spectral-temporal Transformer-based model, called SpecTNT, which can be trained with an additional connectionist temporal localization (CTL) loss. In cross-dataset evaluations using four public datasets, we demonstrate the effectiveness of the SpecTNT model and CTL loss, and obtain strong results overall: the proposed system outperforms state-of-the-art chorus-detection and boundary-detection methods at detecting choruses and boundaries, respectively.},

note = {ISSN: 2379-190X},

keywords = {Location awareness, music, Music structure, segmentation, semantic labeling, Semantics, Signal processing, Signal processing algorithms, SpecTNT, Taxonomy, Transformer, Transformers},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{hung_transcription_2021,

title = {Transcription Is All You Need: Learning To Separate Musical Mixtures With Score As Supervision},

author = {Yun-Ning Hung and Gordon Wichern and Jonathan Le Roux},

url = {https://ieeexplore.ieee.org/abstract/document/9413358/authors#authors},

doi = {10.1109/ICASSP39728.2021.9413358},

year  = {2021},

date = {2021-06-01},

urldate = {2024-02-08},

booktitle = {ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},

pages = {46\textendash50},

abstract = {Most music source separation systems require large collections of isolated sources for training, which can be difficult to obtain. In this work, we use musical scores, which are comparatively easy to obtain, as a weak label for training a source separation system. In contrast with previous score-informed separation approaches, our system does not require isolated sources, and score is used only as a training target, not required for inference. Our model consists of a separator that outputs a time-frequency mask for each instrument, and a transcriptor that acts as a critic, providing both temporal and frequency supervision to guide the learning of the separator. A harmonic mask constraint is introduced as another way of leveraging score information during training, and we propose two novel adversarial losses for additional fine-tuning of both the transcriptor and the separator. Results demonstrate that using score information outper-forms temporal weak-labels, and adversarial structures lead to further improvements in both separation and transcription performance.},

note = {ISSN: 2379-190X},

keywords = {audio source separation, Conferences, Instruments, music, music transcription, Particle separators, Source separation, Time-frequency analysis, Training, weakly-labeled data, weakly-supervised separation},

pubstate = {published},

tppubtype = {inproceedings}

}