@inproceedings{hu2017integrating, author = {Hu, Ting-Yao and Raman, Chirag and Maza, Salvador Medina and Gui, Liangke and Baltrusaitis, Tadas and Frederking, Robert and Morency, Louis-Philippe and Black, Alan and Eskenazi, Maxine}, title = {Integrating {Verbal} and {Nonvebval} {Input} into a {Dynamic} {Response} {Spoken} {Dialogue} {System}}, booktitle = {Proceedings of the AAAI conference on artificial intelligence}, volume = {31}, number = {1}, date = {2017}, doi = {10.1609/aaai.v31i1.10538}, abstract = {In this work, we present a dynamic response spoken dialogue system (DRSDS). It is capable of understanding the verbal and nonverbal language of users and making instant, situation-aware response. Incorporating with two external systems, MultiSense and email summarization, we built an email reading agent on mobile device to show the functionality of DRSDS.} } @inproceedings{mehta2018auto, author = {Mehta, Swapneel and Raman, Chirag and Ayer, Nitin and Sahasrabudhe, Sameer}, publisher = {IEEE}, title = {Auto-Grading for {3D} {Modeling} {Assignments} in {MOOCs}}, booktitle = {2018 IEEE 18th international conference on advanced learning technologies (ICALT)}, pages = {51-53}, date = {2018}, doi = {10.1109/ICALT.2018.00012}, abstract = {Bottlenecks such as the latency in correcting assignments and providing a grade for Massive Open Online Courses (MOOCs) could impact the levels of interest among learners. In this proposal for an auto-grading system, we present a method to simplify grading for an online course that focuses on 3D Modeling, thus addressing a critical component of the MOOC ecosystem that affects. Our approach involves a live auto-grader that is capable of attaching descriptive labels to assignments which will be deployed for evaluating submissions. This paper presents a brief overview of this auto-grading system and the reasoning behind its inception. Preliminary internal tests show that our system presents results comparable to human graders.} } @article{du2018multimodal, author = {Du, Yulun and Raman, Chirag and Black, Alan W and Morency, Louis-Philippe and Eskenazi, Maxine}, title = {Multimodal Polynomial Fusion for Detecting Driver Distraction}, journal = {arXiv preprint arXiv:1810.10565}, date = {2018}, url = {https://arxiv.org/abs/1810.10565}, note = {Author list follows the arXiv version; related conference publication: Interspeech 2018.}, abstract = {Distracted driving is deadly, claiming 3,477 lives in the U.S. in 2015 alone. Although there has been a considerable amount of research on modeling the distracted behavior of drivers under various conditions, accurate automatic detection using multiple modalities and especially the contribution of using the speech modality to improve accuracy has received little attention. This paper introduces a new multimodal dataset for distracted driving behavior and discusses automatic distraction detection using features from three modalities: facial expression, speech and car signals. Detailed multimodal feature analysis shows that adding more modalities monotonically increases the predictive accuracy of the model. Finally, a simple and effective multimodal fusion technique using a polynomial fusion layer shows superior distraction detection results compared to the baseline SVM and neural network models.} } @inproceedings{hung2019multimodal, author = {Hung, Hayley and Raman, Chirag and Gedik, Ekin and Tan, Stephanie and Quiros, Jose Vargas}, title = {Multimodal {Data} {Collection} for {Social} {Interaction} {Analysis} {In-the-Wild.}}, booktitle = {ACM multimedia}, pages = {2714-2715}, date = {2019}, doi = {10.1145/3343031.3351320}, abstract = {The benefits of exploiting multi-modality in the analysis of human-human social behaviour has been demonstrated widely in the community. An important aspect of this problem is the collection of data-sets that provide a rich and realistic representation of how people actually socialize with each other in real life. These subtle coordination patterns are influenced by individual beliefs, goals, and, desires related to what an individual stands to lose or gain in the activities they perform in their every day life. These conditions cannot be easily replicated in a lab setting and require a radical re-thinking of both how and what to collect. This tutorial provides a guide on how to create such multi-modal multi-sensor data sets when holistically considering the entire experimental design and data collection process.} } @inproceedings{raman2019towards, author = {Raman, Chirag and Hung, Hayley}, publisher = {IEEE}, title = {Towards Automatic Estimation of Conversation Floors Within {F-formations}}, booktitle = {2019 8th international conference on affective computing and intelligent interaction workshops and demos (ACIIW)}, pages = {175-181}, date = {2019}, doi = {10.1109/ACIIW.2019.8925065}, abstract = {The detection of free-standing conversing groups has received significant attention in recent years. In the absence of a formal definition, most studies operationalize the notion of a conversation group either through a spatial or a temporal lens. Spatially, the most commonly used representation is the F-formation, defined by social scientists as the configuration in which people arrange themselves to sustain an interaction. However, the use of this representation is often accompanied with the simplifying assumption that a single conversation occurs within an F-formation. Temporally, various categories have been used to organize conversational units; these include, among others, turn, topic, and floor. Some of these concepts are hard to define objectively by themselves. The present work constitutes an initial exploration into unifying these perspectives by primarily posing the question: can we use the observation of simultaneous speaker turns to infer whether multiple conversation floors exist within an F-formation? We motivate a metric for the existence of distinct conversation floors based on simultaneous speaker turns, and provide an analysis using this metric to characterize conversations across F-formations of varying cardinality. We contribute two key findings: firstly, at the average speaking turn duration of about two seconds for humans, there is evidence for the existence of multiple floors within an F-formation; and secondly, an increase in the cardinality of an F-formation correlates with a decrease in duration of simultaneous speaking turns.} } @inproceedings{raman2020modular, author = {Raman, Chirag and Tan, Stephanie and Hung, Hayley}, title = {A Modular Approach for Synchronized Wireless Multimodal Multisensor Data Acquisition in Highly Dynamic Social Settings}, booktitle = {Proceedings of the 28th ACM international conference on multimedia}, pages = {3586-3594}, date = {2020}, doi = {10.1145/3394171.3413697}, abstract = {Existing data acquisition literature for human behavior research provides wired solutions, mainly for controlled laboratory setups. In uncontrolled free-standing conversation settings, where participants are free to walk around, these solutions are unsuitable. While wireless solutions are employed in the broadcasting industry, they can be prohibitively expensive. In this work, we propose a modular and cost-effective wireless approach for synchronized multisensor data acquisition of social human behavior. Our core idea involves a cost-accuracy trade-off by using Network Time Protocol (NTP) as a source reference for all sensors. While commonly used as a reference in ubiquitous computing, NTP is widely considered to be insufficiently accurate as a reference for video applications, where Precision Time Protocol (PTP) or Global Positioning System (GPS) based references are preferred. We argue and show, however, that the latency introduced by using NTP as a source reference is adequate for human behavior research, and the subsequent cost and modularity benefits are a desirable trade-off for applications in this domain. We also describe one instantiation of the approach deployed in a real-world experiment to demonstrate the practicality of our setup in-the-wild.} } @inproceedings{raj2020defining, author = {Raj Prabhu, Navin and Raman, Chirag and Hung, Hayley}, title = {Defining and Quantifying Conversation Quality in Spontaneous Interactions}, booktitle = {Companion publication of the 2020 international conference on multimodal interaction}, pages = {196-205}, date = {2020}, doi = {10.1145/3395035.3425966}, abstract = {Social interactions in general are multifaceted and there exists a wide set of factors and events that influence them. In this paper, we quantify social interactions with a holistic viewpoint on individual experiences, particularly focusing on non-task-directed spontaneous interactions. To achieve this, we design a novel perceived measure, the perceived Conversation Quality, which intends to quantify spontaneous interactions by accounting for several socio-dimensional aspects of individual experiences. To further quantitatively study spontaneous interactions, we devise a questionnaire which measures the perceived Conversation Quality, at both the individual- and at the group- level. Using the questionnaire, we collected perceived annotations for conversation quality in a publicly available dataset using naive annotators. The results of the analysis performed on the distribution and the inter-annotator agreeability shows that naive annotators tend to agree less in cases of low conversation quality samples, especially while annotating for group-level conversation quality.} } @inproceedings{tsfasman2021towards, author = {Tsfasman, Maria and Saravanan, Avinash and Viner, Dekel and Goslinga, Daan and De Wolf, Sarah and Raman, Chirag and Jonker, Catholijn M and Oertel, Catharine}, title = {Towards a Real-Time Measure of the Perception of Anthropomorphism in Human-Robot Interaction}, booktitle = {Proceedings of the 2nd ACM multimedia workshop on multimodal conversational AI}, pages = {13-18}, date = {2021}, doi = {10.1145/3475959.3485394}, abstract = {How human-like do conversational robots need to look to enable long-term human-robot conversation? One essential aspect of long-term interaction is a human’s ability to adapt to the varying degrees of a conversational partner’s engagement and emotions. Prosodically, this can be achieved through (dis)entrainment. While speech-synthesis has been a limiting factor for many years, restrictions in this regard are increasingly mitigated. These advancements now emphasise the importance of studying the effect of robot embodiment on human entrainment. In this study, we conducted a between-subjects online human-robot interaction experiment in an educational use-case scenario where a tutor was either embodied through a human or a robot face. 43 English-speaking participants took part in the study for whom we analysed the degree of acoustic-prosodic entrainment to the human or robot face, respectively. We found that the degree of subjective and objective perception of anthropomorphism positively correlates with acoustic-prosodic entrainment.} } @inproceedings{raman2022social, author = {Raman, Chirag and Hung, Hayley and Loog, Marco}, publisher = {Springer Nature Switzerland Cham}, title = {Social Processes: {Self-supervised} Meta-Learning over Conversational Groups for Forecasting Nonverbal Social Cues}, booktitle = {Computer vision – ECCV 2022 workshops}, series = {Lecture notes in computer science}, volume = {13803}, pages = {639-659}, date = {2023}, url = {https://arxiv.org/abs/2107.13576}, doi = {10.1007/978-3-031-25066-8_37}, abstract = {Free-standing social conversations constitute a yet underexplored setting for human behavior forecasting. While the task of predicting pedestrian trajectories has received much recent attention, an intrinsic difference between these settings is how groups form and disband. Evidence from social psychology suggests that group members in a conversation explicitly self-organize to sustain the interaction by adapting to one another’s behaviors. Crucially, the same individual is unlikely to adapt similarly across different groups; contextual factors such as perceived relationships, attraction, rapport, etc., influence the entire spectrum of participants’ behaviors. A question arises: how can we jointly forecast the mutually dependent futures of conversation partners by modeling the dynamics unique to every group? In this paper, we propose the Social Process (SP) models, taking a novel meta-learning and stochastic perspective of group dynamics. Training group-specific forecasting models hinders generalization to unseen groups and is challenging given limited conversation data. In contrast, our SP models treat interaction sequences from a single group as a meta-dataset: we condition forecasts for a sequence from a given group on other observed-future sequence pairs from the same group. In this way, an SP model learns to adapt its forecasts to the unique dynamics of the interacting partners, generalizing to unseen groups in a data-efficient manner. Additionally, we first rethink the task formulation itself, motivating task requirements from social science literature that prior formulations have overlooked. For our formulation of Social Cue Forecasting, we evaluate the empirical performance of our SP models against both non-meta-learning and meta-learning approaches with similar assumptions. The SP models yield improved performance on synthetic and real-world behavior datasets.} } @inproceedings{quiros2022covfee, author = {Vargas Quiros, Jose and Tan, Stephanie and Raman, Chirag and Cabrera-Quiros, Laura and Hung, Hayley}, publisher = {PMLR}, title = {Covfee: An Extensible Web Framework for Continuous-Time Annotation of Human Behavior}, booktitle = {Understanding social behavior in dyadic and small group interactions}, series = {Proceedings of machine learning research}, volume = {173}, pages = {265-293}, date = {2022}, url = {https://proceedings.mlr.press/v173/vargas-quiros22a.html}, abstract = {Continuous-time annotation, where subjects annotate data while watching the continuous media (video, audio, or time series in general) has traditionally been applied to the annotation of continuous-value variables like arousal and valence in Affective Computing. On the other hand, machine perception tasks are most often annotated using frame-wise techniques. For actions, annotators find the start and end frame of the action of interest using a graphical interface. However, given the duration of the videos that are generally annotated in social interaction datasets, this can be a slow and frustrating process. It usually involves pausing the video at the onset or offset of the action and scrolling back and forth to identify the precise moment. A continuous annotation system, where annotators are asked to press a key when they perceive the target action to be occurring, can improve the time to do such annotations, especially in situations where single subjects are annotated for long periods of time. Keypoint annotations, where the task is to follow a particular point of interest in a video (e.g., a body joint) can also be done continuously. In this paper we present the Covfee web framework, a software package designed to support online continuous annotation tasks, with crowd-sourcing capabilities. We present results from case studies of continuous annotation of body poses (keypoints) and speaking (action) on an in-the-wild social interaction dataset. In the case of keypoints, we present a new technique allowing an easy way to follow a keypoint in a video using the mouse cursor. We found the technique to significantly reduce annotation times with no adverse effect on inter-annotator agreement. For action annotation, we used continuous annotation techniques to obtain binary speaking status labels and annotator ratings of confidence on those labels. Covfee is free software, available as a Python package documented at josedvq.github.io/covfee.} } @article{raman2022perceived, author = {Raman, Chirag and Prabhu, Navin Raj and Hung, Hayley}, title = {Perceived {Conversation} {Quality} in {Spontaneous} {Interactions}}, journal = {IEEE Transactions on Affective Computing}, volume = {14}, number = {4}, pages = {2901-2912}, date = {2023}, url = {https://arxiv.org/abs/2207.05791}, doi = {10.1109/TAFFC.2023.3233950}, abstract = {The quality of daily spontaneous conversations is of importance towards both our well-being as well as the development of interactive social agents. Prior research directly studying the quality of social conversations has operationalized it in narrow terms, associating greater quality to less small talk. Other works taking a broader perspective of interaction experience have indirectly studied quality through one of the several overlapping constructs such as rapport or engagement, in isolation. In this work we bridge this gap by proposing a holistic conceptualization of conversation quality, building upon the collaborative attributes of cooperative conversation floors. Taking a multilevel perspective of conversation, we develop and validate two instruments for perceived conversation quality (PCQ) at the individual and group levels. Specifically, we motivate capturing external raters’ gestalt impressions of participant experiences from thin slices of behavior, and collect annotations of PCQ on the publicly available MatchNMingle dataset of in-the-wild mingling conversations. Finally, we present an analysis of behavioral features that are predictive of PCQ. We find that for the conversations in MatchNMingle, raters tend to associate smaller group sizes, equitable speaking turns with fewer interruptions, and time taken for synchronous bodily coordination with higher PCQ.} } @article{raman2022conflab, author = {Raman, Chirag and Vargas Quiros, Jose and Tan, Stephanie and Islam, Ashraful and Gedik, Ekin and Hung, Hayley}, title = {ConfLab: {A} Data Collection Concept, Dataset, and Benchmark for Machine Analysis of Free-Standing Social Interactions in the Wild}, journal = {Advances in Neural Information Processing Systems}, volume = {35}, pages = {23701-23715}, date = {2022}, url = {https://openreview.net/forum?id=CNJQKM5cV2o}, abstract = {Recording the dynamics of unscripted human interactions in the wild is challenging due to the delicate trade-offs between several factors: participant privacy, ecological validity, data fidelity, and logistical overheads. To address these, following a ’datasets for the community by the community’ ethos, we propose the Conference Living Lab (ConfLab): a new concept for multimodal multisensor data collection of in-the-wild free-standing social conversations. For the first instantiation of ConfLab described here, we organized a real-life professional networking event at a major international conference. Involving 48 conference attendees, the dataset captures a diverse mix of status, acquaintance, and networking motivations. Our capture setup improves upon the data fidelity of prior in-the-wild datasets while retaining privacy sensitivity: 8 videos (1920x1080, 60 fps) from a non-invasive overhead view, and custom wearable sensors with onboard recording of body motion (full 9-axis IMU), privacy-preserving low-frequency audio (1250 Hz), and Bluetooth-based proximity. Additionally, we developed custom solutions for distributed hardware synchronization at acquisition, and time-efficient continuous annotation of body keypoints and actions at high sampling rates. Our benchmarks showcase some of the open research tasks related to in-the-wild privacy-preserving social data analysis: keypoints detection from overhead camera views, skeleton-based no-audio speaker detection, and F-formation detection.} } @inproceedings{raman2023mesh, author = {Raman, Chirag and Hewitt, Charlie and Wood, Erroll and Baltrušaitis, Tadas}, publisher = {IEEE}, title = {Mesh-Tension Driven Expression-Based Wrinkles for Synthetic Faces}, booktitle = {2023 IEEE/CVF winter conference on applications of computer vision (WACV)}, pages = {3504-3514}, date = {2023}, url = {https://openaccess.thecvf.com/content/WACV2023/html/Raman_Mesh-Tension_Driven_Expression-Based_Wrinkles_for_Synthetic_Faces_WACV_2023_paper.html}, abstract = {Recent advances in synthesizing realistic faces have shown that synthetic training data can replace real data for various face-related computer vision tasks. A question arises: how important is realism? Is the pursuit of photorealism excessive? In this work, we show otherwise. We boost the realism of our synthetic faces by introducing dynamic skin wrinkles in response to facial expressions, and observe significant performance improvements in downstream computer vision tasks. Previous approaches for producing such wrinkles either required prohibitive artist effort to scale across identities and expressions, or were not capable of reconstructing high-frequency skin details with sufficient fidelity. Our key contribution is an approach that produces realistic wrinkles across a large and diverse population of digital humans. Concretely, we formalize the concept of mesh-tension and use it to aggregate possible wrinkles from high-quality expression scans into albedo and displacement texture maps. At synthesis, we use these maps to produce wrinkles even for expressions not represented in the source scans. Additionally, to provide a more nuanced indicator of model performance under deformations resulting from compressed expressions, we introduce the 300W-winks evaluation subset and the Pexels dataset of closed eyes and winks.} } @article{wood20223d, author = {Wood, Erroll and Baltrusaitis, Tadas and Hewitt, Charlie and Johnson, Matthew and Shen, Jingjing and Milosavljevic, Nikola and Wilde, Daniel and Garbin, Stephan and Raman, Chirag and Shotton, Jamie and Sharp, Toby and Stojiljkovic, Ivan and Cashman, Tom and Valentin, Julien}, title = {3D Face Reconstruction with Dense Landmarks}, journal = {arXiv preprint arXiv:2204.02776}, date = {2022}, url = {https://arxiv.org/abs/2204.02776}, note = {Author list follows the arXiv version; related conference publication: ECCV 2022.}, abstract = {Landmarks often play a key role in face analysis, but many aspects of identity or expression cannot be represented by sparse landmarks alone. Thus, in order to reconstruct faces more accurately, landmarks are often combined with additional signals like depth images or techniques like differentiable rendering. Can we keep things simple by just using more landmarks? In answer, we present the first method that accurately predicts 10x as many landmarks as usual, covering the whole head, including the eyes and teeth. This is accomplished using synthetic training data, which guarantees perfect landmark annotations. By fitting a morphable model to these dense landmarks, we achieve state-of-the-art results for monocular 3D face reconstruction in the wild. We show that dense landmarks are an ideal signal for integrating face shape information across frames by demonstrating accurate and expressive facial performance capture in both monocular and multi-view scenarios. This approach is also highly efficient: we can predict dense landmarks and fit our 3D face model at over 150FPS on a single CPU thread. Please see our website: https://microsoft.github.io/DenseLandmarks/.} } @phdthesis{raman2023towards, author = {Raman, Chirag Anantha}, publisher = {Delft University of Technology}, title = {Towards {Artificial} {Social} {Intelligence} in the {Wild:} {Sensing,} {Synthesizing,} {Modeling,} and {Perceiving} {Nonverbal} {Social} {Human} {Behavior}}, date = {2023}, doi = {10.4233/uuid:05fe4340-31bb-4c24-a827-69189aa2622b}, abstract = {Over the last three decades, the social roots of human intelligence have come to influence the development of artificial intelligence (AI). Researchers in AI have moved beyond agents operating in isolation towards developing socially situated agents that can operate in the real world. Meanwhile, researchers in the social sciences have been leveraging AI techniques to analyze and theorize about social phenomena. Both these research endeavors came to be independently termed Artificial Social Intelligence (ASI), leading to the emergence of a field spanning several subdisciplines of the social and computational sciences.This Thesis takes a holistic view of ASI and makes contributions toward both its historical goals. Moreover, the work presented here focuses on taking ASI research into natural real-world settings in the wild. The research is organized under three themes: acquiring, modeling, and perceiving social human behavior. The Thesis begins by addressing the challenge of data acquisition. We propose a replicable data collection concept for curating datasets of real-world social human behavior, incorporating technical innovations and ethical considerations required for the noninvasive sensing of multimodal behavioral streams. To overcome the limited availability of real-world data, we also explore the potential of synthetic training data for downstream tasks. Next, we tackle the challenge of modeling real-world social behavioral cues. Evidence from social psychology suggests that individuals uniquely adapt their behaviors to different conversation partners to sustain interactions. How can we jointly forecast these mutually dependent future cues of conversation partners? We propose a stochastic meta-learning method that adapts its forecasts to the unique dynamics of a conversation group given example behavior sequences. Thereby, it generalizes to unseen groups in a data-efficient manner by avoiding the need for group-specific models. Further, to facilitate the integration of data-driven and hypothesis-driven research, we propose a post hoc explanation framework for identifying timesteps that are salient to a forecasting model’s predictions. Finally, we contribute to a nuanced perception of social interactions by establishing evidence of multiple conversation floors within a single conversing group, in contrast to the prevailing implicit assumption in the automatic detection of conversation groups. We also develop an instrument for measuring the perceived quality of conversations at the individual and group levels.Through these research themes, we provide novel contributions to the field of ASI, taking important steps toward the development of socially intelligent machines that can operate effectively in complex real-world settings.} } @article{raman2023did, author = {Raman, Chirag and Nonnemaker, Alec and Villegas-Morcillo, Amelia and Hung, Hayley and Loog, Marco}, title = {Why Did This Model Forecast This Future? {Information-theoretic} Saliency for Counterfactual Explanations of Probabilistic Regression Models}, journal = {Advances in Neural Information Processing Systems}, volume = {36}, pages = {33222-33240}, date = {2023}, url = {https://openreview.net/forum?id=IrEYkhuxup}, abstract = {We propose a post hoc saliency-based explanation framework for counterfactual reasoning in probabilistic multivariate time-series forecasting (regression) settings. Building upon Miller’s framework of explanations derived from research in multiple social science disciplines, we establish a conceptual link between counterfactual reasoning and saliency-based explanation techniques. To address the lack of a principled notion of saliency, we leverage a unifying definition of information-theoretic saliency grounded in preattentive human visual cognition and extend it to forecasting settings. Specifically, we obtain a closed-form expression for commonly used density functions to identify which observed timesteps appear salient to an underlying model in making its probabilistic forecasts. We empirically validate our framework in a principled manner using synthetic data to establish ground-truth saliency that is unavailable for real-world data. Finally, using real-world data and forecasting models, we demonstrate how our framework can assist domain experts in forming new data-driven hypotheses about the causal relationships between features in the wild.} } @inproceedings{dollack2023ensemble, author = {Dollack, Felix and Kiyokawa, Kiyoshi and Liu, Huakun and Perusquia-Hernandez, Monica and Raman, Chirag and Uchiyama, Hideaki and Wei, Xin}, publisher = {IEEE}, title = {Ensemble Learning to Assess Dynamics of Affective Experience Ratings and Physiological Change}, booktitle = {2023 11th international conference on affective computing and intelligent interaction workshops and demos (ACIIW)}, pages = {1-8}, date = {2023}, url = {https://arxiv.org/abs/2312.16036}, abstract = {The congruence between affective experiences and physiological changes has been a debated topic for centuries. Recent technological advances in measurement and data analysis provide hope to solve this epic challenge. Open science and open data practices, together with data analysis challenges open to the academic community, are also promising tools for solving this problem. In this entry to the Emotion Physiology and Experience Collaboration (EPiC) challenge, we propose a data analysis solution that combines theoretical assumptions with data-driven methodologies. We used feature engineering and ensemble selection. Each predictor was trained on subsets of the training data that would maximize the information available for training. Late fusion was used with an averaging step. We chose to average considering a “wisdom of crowds” strategy. This strategy yielded an overall RMSE of 1.19 in the test set. Future work should carefully explore if our assumptions are correct and the potential of weighted fusion.} } @article{quiros2024rewind, author = {Quiros, Jose Vargas and Raman, Chirag and Tan, Stephanie and Gedik, Ekin and Cabrera-Quiros, Laura and Hung, Hayley}, title = {REWIND {Dataset:} {Privacy-preserving} Speaking Status Segmentation from Multimodal Body Movement Signals in the Wild}, journal = {arXiv preprint arXiv:2403.01229}, date = {2024}, url = {https://arxiv.org/abs/2403.01229}, abstract = {Recognizing speaking in humans is a central task towards understanding social interactions. Ideally, speaking would be detected from individual voice recordings, as done previously for meeting scenarios. However, individual voice recordings are hard to obtain in the wild, especially in crowded mingling scenarios due to cost, logistics, and privacy concerns. As an alternative, machine learning models trained on video and wearable sensor data make it possible to recognize speech by detecting its related gestures in an unobtrusive, privacy-preserving way. These models themselves should ideally be trained using labels obtained from the speech signal. However, existing mingling datasets do not contain high quality audio recordings. Instead, speaking status annotations have often been inferred by human annotators from video, without validation of this approach against audio-based ground truth. In this paper we revisit no-audio speaking status estimation by presenting the first publicly available multimodal dataset with high-quality individual speech recordings of 33 subjects in a professional networking event. We present three baselines for no-audio speaking status segmentation: a) from video, b) from body acceleration (chest-worn accelerometer), c) from body pose tracks. In all cases we predict a 20Hz binary speaking status signal extracted from the audio, a time resolution not available in previous datasets. In addition to providing the signals and ground truth necessary to evaluate a wide range of speaking status detection methods, the availability of audio in REWIND makes it suitable for cross-modality studies not feasible with previous mingling datasets. Finally, our flexible data consent setup creates new challenges for multimodal systems under missing modalities.} } @misc{baltrusaitis2024face, author = {BALTRUSAITIS, Tadas and HEWITT, Charles Thomas and WOOD, Erroll William and RAMAN, Chirag Anantha}, title = {Face Image Generation with Wrinkles}, date = {2024}, url = {https://patents.google.com/patent/US12062140B2/en}, note = {US Patent 12,062,140}, abstract = {Computing an image depicting a face having an expression with wrinkles is described. A 3D polygon mesh model of a face has a non-neutral expression. A tension map is computed from the 3D polygon mesh model. A neutral texture, a compressed wrinkle texture and an expanded wrinkle texture are computed or obtained from a library. The neutral texture comprises a map of the first face with a neutral expression. The compressed wrinkle texture is a map of the first face formed by aggregating maps of the first face with different expressions using the tension map, and the expanded wrinkle texture comprises a map of the first face formed by aggregating maps of the first face with different expressions using the tension map. A graphics engine may be used to apply the wrinkle textures to the 3D model according to the tension map; and render the image from the 3D model.} } @article{achar2024comparative, author = {Achar, Prerana and Patel, Mayank and Mulik, Anushka and Katre, Neha and Dias, Stevina and Raman, Chirag}, title = {A Comparative Study of Garment Draping Techniques}, journal = {arXiv preprint arXiv:2405.11056}, date = {2024}, url = {https://arxiv.org/abs/2405.11056}, abstract = {We present a comparison review that evaluates popular techniques for garment draping for 3D fashion design, virtual try-ons, and animations. A comparative study is performed between various methods for garment draping of clothing over the human body. These include numerous models, such as physics and machine learning based techniques, collision handling, and more. Performance evaluations and trade-offs are discussed to ensure informed decision-making when choosing the most appropriate approach. These methods aim to accurately represent deformations and fine wrinkles of digital garments, considering the factors of data requirements, and efficiency, to produce realistic results. The research can be insightful to researchers, designers, and developers in visualizing dynamic multi-layered 3D clothing.} } @inproceedings{liu2024private, author = {Liu, Ailin and Vunderink, Pepijn and Quiros, Jose Vargas and Raman, Chirag and Hung, Hayley}, title = {How Private Is Low-Frequency Speech Audio in the Wild? An Analysis of Verbal Intelligibility by Humans and Machines}, booktitle = {Interspeech 2024}, pages = {2725-2729}, date = {2024}, url = {https://arxiv.org/abs/2407.13266}, doi = {10.21437/Interspeech.2024-1258}, abstract = {Low-frequency audio has been proposed as a promising privacy-preserving modality to study social dynamics in real-world settings. To this end, researchers have developed wearable devices that can record audio at frequencies as low as 1250 Hz to mitigate the automatic extraction of the verbal content of speech that may contain private details. This paper investigates the validity of this hypothesis, examining the degree to which low-frequency speech ensures verbal privacy. It includes simulating a potential privacy attack in various noise environments. Further, it explores the trade-off between the performance of voice activity detection, which is fundamental for understanding social behavior, and privacy-preservation. The evaluation incorporates subjective human intelligibility and automatic speech recognition performance, comprehensively analyzing the delicate balance between effective social behavior analysis and preserving verbal privacy.} } @misc{baltrusaitis2026face, author = {BALTRUSAITIS, Tadas and HEWITT, Charles Thomas and WOOD, Erroll William and RAMAN, Chirag Anantha}, title = {Face Image Generation with Wrinkles}, date = {2026}, url = {https://patents.google.com/patent/US12555324B2/en}, note = {US Patent 12,555,324}, abstract = {Computing an image depicting a face having an expression with wrinkles is described. A 3D polygon mesh model of a face has a non-neutral expression. A tension map is computed from the 3D polygon mesh model. A neutral texture, a compressed wrinkle texture and an expanded wrinkle texture are computed or obtained from a library. The neutral texture comprises a map of the first face with a neutral expression. The compressed wrinkle texture is a map of the first face formed by aggregating maps of the first face with different expressions using the tension map, and the expanded wrinkle texture comprises a map of the first face formed by aggregating maps of the first face with different expressions using the tension map. A graphics engine may be used to apply the wrinkle textures to the 3D model according to the tension map; and render the image from the 3D model.} } @article{raman2025social, author = {Jučas, Augustinas and Raman, Chirag}, title = {Social {Processes:} {Probabilistic} {Meta-learning} for {Adaptive} {Multiparty} {Interaction} {Forecasting}}, journal = {arXiv preprint arXiv:2501.01915}, date = {2025}, url = {https://arxiv.org/abs/2501.01915}, abstract = {Adaptively forecasting human behavior in social settings is an important step toward achieving Artificial General Intelligence. Most existing research in social forecasting has focused either on unfocused interactions, such as pedestrian trajectory prediction, or on monadic and dyadic behavior forecasting. In contrast, social psychology emphasizes the importance of group interactions for understanding complex social dynamics. This creates a gap that we address in this paper: forecasting social interactions at the group (conversation) level. Additionally, it is important for a forecasting model to be able to adapt to groups unseen at train time, as even the same individual behaves differently across different groups. This highlights the need for a forecasting model to explicitly account for each group’s unique dynamics. To achieve this, we adopt a meta-learning approach to human behavior forecasting, treating every group as a separate meta-learning task. As a result, our method conditions its predictions on the specific behaviors within the group, leading to generalization to unseen groups. Specifically, we introduce Social Process (SP) models, which predict a distribution over future multimodal cues jointly for all group members based on their preceding low-level multimodal cues, while incorporating other past sequences of the same group’s interactions. In this work we also analyze the generalization capabilities of SP models in both their outputs and latent spaces through the use of realistic synthetic datasets.} } @article{carciumaru2025systematic, author = {Carciumaru, Teona Z and Tang, Cadey M and Farsi, Mohsen and Bramer, Wichor M and Dankelman, Jenny and Raman, Chirag and Dirven, Clemens MF and Gholinejad, Maryam and Vasilic, Dalibor}, publisher = {Nature Publishing Group UK London}, title = {Systematic Review of Machine Learning Applications Using Nonoptical Motion Tracking in Surgery}, journal = {NPJ digital medicine}, volume = {8}, number = {1}, pages = {28}, date = {2025}, doi = {10.1038/s41746-024-01412-1}, abstract = {This systematic review explores machine learning (ML) applications in surgical motion analysis using non-optical motion tracking systems (NOMTS), alone or with optical methods. It investigates objectives, experimental designs, model effectiveness, and future research directions. From 3632 records, 84 studies were included, with Artificial Neural Networks (38\%) and Support Vector Machines (11\%) being the most common ML models. Skill assessment was the primary objective (38\%). NOMTS used included internal device kinematics (56\%), electromagnetic (17\%), inertial (15\%), mechanical (11\%), and electromyography (1\%) sensors. Surgical settings were robotic (60\%), laparoscopic (18\%), open (16\%), and others (6\%). Procedures focused on bench-top tasks (67\%), clinical models (17\%), clinical simulations (9\%), and non-clinical simulations (7\%). Over 90\% accuracy was achieved in 36\% of studies. Literature shows NOMTS and ML can enhance surgical precision, assessment, and training. Future research should advance ML in surgical environments, ensure model interpretability and reproducibility, and use larger datasets for accurate evaluation.} } @inproceedings{dudzik2024indeterminacy, author = {Dudzik, Bernd and Hrkalovic, Tiffany Matej and Hao, Chenxu and Raman, Chirag and Tsfasman, Masha}, publisher = {IEEE}, title = {Indeterminacy in {Affective} {Computing:} {Considering} {Meaning} and {Context} in {Data} {Collection} {Practices}}, booktitle = {2024 12th international conference on affective computing and intelligent interaction workshops and demos (ACIIW)}, pages = {181-185}, date = {2024}, url = {https://arxiv.org/abs/2502.09294}, abstract = {Automatic Affect Prediction (AAP) uses computational analysis of input data such as text, speech, images, and physiological signals to predict various affective phenomena (e.g., emotions or moods). These models are typically constructed using supervised machine-learning algorithms, which rely heavily on labeled training datasets. In this position paper, we posit that all AAP training data are derived from human Affective Interpretation Processes, resulting in a form of Affective Meaning. Research on human affect indicates a form of complexity that is fundamental to such meaning: it can possess what we refer to here broadly as Qualities of Indeterminacy (QIs) - encompassing Subjectivity (meaning depends on who is interpreting), Uncertainty (lack of confidence regarding meanings’ correctness), Ambiguity (meaning contains mutually exclusive concepts) and Vagueness (meaning is situated at different levels in a nested hierarchy). Failing to appropriately consider QIs leads to results incapable of meaningful and reliable predictions. Based on this premise, we argue that a crucial step in adequately addressing indeterminacy in AAP is the development of data collection practices for modeling corpora that involve the systematic consideration of 1) a relevant set of QIs and 2) context for the associated interpretation processes. To this end, we are 1) outlining a conceptual model of AIPs and the QIs associated with the meaning these produce and a conceptual structure of relevant context, supporting understanding of its role. Finally, we use our framework for 2) discussing examples of context-sensitivity-related challenges for addressing QIs in data collection setups. We believe our efforts can stimulate a structured discussion of both the role of aspects of indeterminacy and context in research on AAP, informing the development of better practices for data collection and analysis.} } @inproceedings{quamara2025massxr, author = {Quamara, Megha and Celiktutan, Oya and Viganò, Luca and Bera, Aniket and Cesar, Pablo and Durupinar, Funda and Normoyle, Aline and Raman, Chirag and Yumak, Zerrin}, publisher = {Institute of Electrical; Electronics Engineers Inc.}, title = {MASSXR 2025: {The} 3rd {Workshop} on {Multi-modal} {Affective} and {Social} {Behavior} {Analysis} and {Synthesis} in {Extended} {Reality} {(Affiliated} with {IEEE} {VR} 2025)}, booktitle = {2025 IEEE conference on virtual reality and 3D user interfaces abstracts and workshops, VRW 2025}, pages = {282-283}, date = {2025}, url = {https://ir.cwi.nl/pub/35772}, abstract = {The objective of MASSXR 2025, the 3rd Workshop on Multi-modal Affective and Social Behavior Analysis and Synthesis in Extended Reality, was to bring together researchers and practitioners from fields including cybersecurity, human-computer interaction, computer graphics/animation, multi-modal machine learning, Artificial Intelligence (AI), data privacy, and socio-technical studies to discuss the state of security in Extended Reality (XR), as well as future directions and opportunities. Through this, it aimed to achieve adaptive, context-aware security measures that are both technically robust and aligned with user trust and understanding. The workshop provided an opportunity to foster collaborative research efforts and advance the state of secure social interactions within XR, setting a foundation for future innovations in the field.} } @article{coles2025big, author = {Coles, Nicholas A. and Perz, Bartosz and Behnke, Maciej and Eichstaedt, Johannes C. and Kim, Soo Hyung and Vu, Tu N. and Raman, Chirag and Tejada, Julian and Huynh, Van-Thong and Zhang, Guangyi and Cui, Tanming and Podder, Sharanyak and Chavda, Rushi and Pandey, Shubham and Upadhyay, Arpit and Padilla-Buritica, Jorge I. and Barrera Causil, Carlos J. and Ji, Linying and Dollack, Felix and Kiyokawa, Kiyoshi and Liu, Huakun and Perusquia-Hernandez, Monica and Uchiyama, Hideaki and Wei, Xin and Cao, Houwei and Yang, Ziqing and Iancarelli, Alessia and McVeigh, Kieran and Wang, Yiyu and Berwian, Isabel M. and Chiu, Jamie C. and Mirea, Dan-Mircea and Nook, Erik C. and Vartiainen, Henna I. and Whiting, Claire and Cho, Young Won and Chow, Sy-Miin and Fisher, Zachary F. and Li, Yanling and Xiong, Xiaoyue and Shen, Yuqi and Tagliazucchi, Enzo and Bugnon, Leandro A. and Ospina, Raydonal and Bruno, Nicolas M. and D’Amelio, Tomas A. and Zamberlan, Federico and Mercado Diaz, Luis R. and Pinzon-Arenas, Javier O. and Posada-Quintero, Hugo F. and Bilalpur, Maneesh and Hinduja, Saurabh and Marmolejo-Ramos, Fernando and Canavan, Shaun and Jivnani, Liza and Saganowski, Stanisław}, title = {Big Team Science Reveals Promises and Limitations of Machine Learning Efforts to Model Physiological Markers of Affective Experience}, journal = {Royal Society Open Science}, volume = {12}, number = {6}, pages = {241778}, date = {2025}, doi = {10.1098/rsos.241778}, abstract = {Researchers are increasingly using machine learning to study physiological markers of emotion. We evaluated the promises and limitations of this approach via a big team science competition. Twelve teams competed to predict self-reported affective experiences using a multi-modal set of peripheral nervous system measures. Models were trained and tested in multiple ways: with data divided by participants, targeted emotion, inductions, and time. In 100\% of tests, teams outperformed baseline models that made random predictions. In 46\% of tests, teams also outperformed baseline models that relied on the simple average of ratings from training datasets. More notably, results uncovered a methodological challenge: multiplicative constraints on generalizability. Inferences about the accuracy and theoretical implications of machine learning efforts depended not only on their architecture, but also how they were trained, tested, and evaluated. For example, some teams performed better when tested on observations from the same (vs. different) subjects seen during training. Such results could be interpreted as evidence against claims of universality. However, such conclusions would be premature because other teams exhibited the opposite pattern. Taken together, results illustrate how big team science can be leveraged to understand the promises and limitations of machine learning methods in affective science and beyond.} } @article{oertel2025new, author = {Oertel, Catharine and Cabrera-Quirós, Laura and Hung, Hayley and Soleymani, Mohammad and Andrist, Sean and Kontogiorgos, Dimosthenis and Raman, Chirag}, publisher = {ACM New York, NY, USA}, title = {New {Initiatives} at {ICMI:} {Mentorship} in {Focus}}, journal = {Voices of SIGCHI}, volume = {1}, number = {1}, pages = {12-13}, date = {2025}, doi = {10.1145/3757867.3757873}, abstract = {In its 2024 iteration, the general chairs of SIGCHI’s International Conference on Multimodal Interaction (ICMI) decided to launch a New Initiatives programme at the conference. The goal was to give new ideas a place to form within the ICMI community, so that not only seasoned members had a say, but also, newcomers could voice their needs and find an academic home they would like to return to and contribute to for many years. Ultimately, the question motivating this programme was: How can we make ICMI an even more welcoming place for newcomers, early career researchers, and people from diverse backgrounds and perspectives?} } @inproceedings{shirekar2025multimodal, author = {Shirekar, Ojas and Pouw, Wim and Hao, Chenxu and Phadnis, Vrushank and Beeler, Thabo and Raman, Chirag}, title = {Multimodal {Quantitative} {Measures} for {Multiparty} {Behavior} {Evaluation}}, booktitle = {Proceedings of the 27th international conference on multimodal interaction}, pages = {249-264}, date = {2025}, doi = {10.1145/3716553.3750752}, abstract = {Digital humans are emerging as autonomous agents in multiparty interactions, yet existing evaluation metrics largely ignore contextual coordination dynamics. We introduce a unified, intervention-driven framework for objective assessment of multiparty social behaviour in skeletal motion data, spanning three complementary dimensions: (1) synchrony via Cross-Recurrence Quantification Analysis, (2) temporal alignment via Multiscale Empirical Mode Decomposition-based Beat Consistency, and (3) structural similarity via Soft Dynamic Time Warping. We validate metric sensitivity through three theory-driven perturbations - gesture kinematic dampening, uniform speech-gesture delays, and prosodic pitch-variance reduction - applied to ≈ 145 30-second thin slices of group interactions from the DnD dataset. Mixed-effects analyses reveal predictable, joint-independent shifts: dampening increases CRQA determinism and reduces beat consistency, delays weaken cross-participant coupling, and pitch flattening elevates F0 Soft-DTW costs. A complementary perception study (N = 27) compares judgments of full-video and skeleton-only renderings to quantify representation effects. Our three measures deliver orthogonal insights into spatial structure, timing alignment, and behavioural variability. Thereby forming a robust toolkit for evaluating and refining socially intelligent agents. Code available on GitHub.} } @inproceedings{licua2025mindforge, author = {Lică, Mircea Tudor and Shirekar, Ojas and Colle, Baptiste and Raman, Chirag}, title = {MindForge: {Empowering} {Embodied} {Agents} with {Theory} of {Mind} for {Lifelong} {Cultural} {Learning}}, booktitle = {The thirty-ninth annual conference on neural information processing systems}, date = {2025}, url = {https://papers.nips.cc/paper_files/paper/2025/hash/f82cb74b5d40004cb96d1426fc12fccf-Abstract-Conference.html}, abstract = {Embodied agents powered by large language models (LLMs), such as Voyager, promise open-ended competence in worlds such as Minecraft. However, when powered by open-weight LLMs they still falter on elementary tasks after domain-specific fine-tuning. We propose MindForge, a generative-agent framework for cultural lifelong learning through explicit perspective taking. We introduce three key innovations: (1) a structured theory of mind representation linking percepts, beliefs, desires, and actions; (2) natural inter-agent communication; and (3) a multi-component memory system. Following the cultural learning framework, we test MindForge in both instructive and collaborative settings within Minecraft. In an instructive setting with GPT-4, MindForge agents powered by open-weight LLMs significantly outperform their Voyager counterparts in basic tasks yielding 3× more tech-tree milestones and collecting 2.3× more unique items than the Voyager baseline. Furthermore, in fully collaborative settings, we find that the performance of two underachieving agents improves with more communication rounds, echoing the Condorcet Jury Theorem. MindForge agents demonstrate sophisticated behaviors, including expert-novice knowledge transfer, collaborative problem solving, and adaptation to out-of-distribution tasks through accumulated cultural experiences.} } @inproceedings{kirchner2026controlled, author = {Kirchner, Joris and Gudi, Amogh and Bittner, Marian and Raman, Chirag}, publisher = {IEEE}, title = {Controlled {Face} {Manipulation} and {Synthesis} for {Data} {Augmentation}}, booktitle = {2026 IEEE 20th international conference on automatic face and gesture recognition (FG)}, pages = {1-10}, date = {2026}, doi = {10.1109/FG67764.2026.11556947}, abstract = {Deep learning vision models excel with abundant supervision, but many applications face label scarcity and class imbalance. Controllable image editing can augment scarce labeled data, yet edits often introduce artifacts and entangle nontarget attributes. We study this in facial expression analysis, targeting Action Unit (AU) manipulation where annotation is costly and AU co-activation drives entanglement. We present a facial manipulation method that operates in the semantic latent space of a pre-trained face generator (Diffusion Autoencoder). Using lightweight linear models, we reduce entanglement of semantic features via (i) dependency-aware conditioning that accounts for AU co-activation, and (ii) orthogonal projection that removes nuisance attribute directions (e.g., glasses), together with an expression neutralization step to enable absolute AU edit. We use these edits to balance AU occurrence by editing labeled faces and to diversify identities/demographics via controlled synthesis. Augmenting AU detector training with the generated data improves accuracy and yields more disentangled predictions with fewer co-activation shortcuts, outperforming alternative data-efficient training strategies and suggesting improvements similar to what would require substantially more labeled data in our learning-curve analysis. Compared to prior methods, our edits are stronger, produce fewer artifacts, and preserve identity better.} } @article{shi2026synchronized, author = {Shi, Xiangwei and Dorta, Gara and Jong, Ruud de and Shirekar, Ojas and Raman, Chirag}, title = {A {Synchronized} {Audio-Visual} {Multi-View} {Capture} {System}}, journal = {arXiv preprint arXiv:2603.23089}, date = {2026}, url = {https://arxiv.org/abs/2603.23089}, abstract = {Multi-view capture systems have been an important tool in research for recording human motion under controlling conditions. Most existing systems are specified around video streams and provide little or no support for audio acquisition and rigorous audio-video alignment, despite both being essential for studying conversational interaction where timing at the level of turn-taking, overlap, and prosody matters. In this technical report, we describe an audio-visual multi-view capture system that addresses this gap by treating synchronized audio and synchronized video as first-class signals. The system combines a multi-camera pipeline with multi-channel microphone recording under a unified timing architecture and provides a practical workflow for calibration, acquisition, and quality control that supports repeatable recordings at scale. We quantify synchronization performance in deployment and show that the resulting recordings are temporally consistent enough to support fine-grained analysis and data-driven modeling of conversation behavior.} } @article{li2026cosi, author = {Li, Zonghuan and Li, Litian and Mercier, Arthur and Dorta, Gara and Dioszegi, Balint and Morales-Vargas, Jose and Hao, Chenxu and Kondyurin, Ivan and Begemann, Vanessa and Lehmann-Willenbrock, Nale and Dudzik, Bernd and Chakrabarty, Saunaq and Vacanas, Sotiris and Cabrera-Quirós, Laura and Wal, Anne L. J. ter and Popov, Vitaliy and Castro-Godínez, Jorge and Raman, Chirag and Tan, Stephanie and Hung, Hayley}, title = {COSI-Lab: {Conference} {Living} {Lab} for {Modeling} {Multi-Perspective} {Multimodal} {Social} {Intention}}, journal = {arXiv preprint arXiv:2607.28649}, date = {2026}, url = {https://arxiv.org/abs/2607.28649}, abstract = {COSI-Lab presents a multimodal, multi-sensor dataset of an interdisciplinary scientific workshop containing 32 academics at an international conference. It captures ecologically valid social interactions in a weakly scripted setting consisting of two 30-minute mingling sessions with real professional and social consequences for the participants involved. We argue that future intelligent systems could be better equipped to handle subjective perceptions by modeling their multiplicity not as label noise but as a explainable perspective-driven reasoning process. We focus on the Apparent Intent Inference (AII) problem as determined by ex-situ observers and conceptualize intentions to be independent of manifest future outcomes. We contribute 1. a novel annotation process for AII that accounts for a perceiver’s own interpretative tendencies, 2. quantitative and qualitative analyses of intent narratives with respect to diversity, grounding, and plausibility; 3. benchmark tasks for AII and surrounding relevant contextual factors such as social involvement; 4. speech quality audio for all participants as well as privacy preserving multi-modal data, enabling lexical and nonverbal behavior analysis; and 5. coupling of self-reported goals of each participant (30 minute to 3 hour) with annotated AII (seconds).} } @article{shirekar2026generative, author = {Shirekar, Ojas and Surange, Yash and Jučas, Augustinas and Raman, Chirag}, title = {Generative {Interactions:} {Weaving} {Multiparty} {Human} {Motion} with {Bilevel} {Latent} {Dynamics}}, journal = {arXiv preprint arXiv:2609.37708}, date = {2026}, url = {https://arxiv.org/abs/2609.37708}, abstract = {Human social behaviour is not a collection of independent motions, but a jointly organised process in which group dynamics and individual variation continuously shape one another. Yet existing social motion models often prioritise plausible trajectories while leaving interaction state implicit, limiting their ability to transfer across groups, tasks, and partial-observation regimes. To address this gap, we introduce Bilevel Representations for Agent Interaction Dynamics (BRAID), a hierarchical sequential latent-variable model for generative multi-person interaction. BRAID explicitly formulates social motion generation as a meta-transfer learning problem: shared interaction priors are learned across datasets and adapted through arbitrary context sets of observed people and joints. The model represents each scene through a group-level latent state that captures shared interaction dynamics and person-level latent states that capture individual behaviour conditioned on the evolving group context. This modelling choice enables coherent generation under full, sparse, or partial observations while exposing compact social-state vectors that can serve as an interface for downstream embodied-agent systems. We evaluate BRAID under a unified SMPL-based representation on social forecasting, tracking and in-filling, and response generation, using metrics that assess not only reconstruction accuracy but also realism, diversity, temporal alignment, and interpersonal coordination. We further analyse the hierarchical latent space, showing that it captures separable group- and individual-level structure.} } @article{fron2026overforge, author = {Fron, Oana Madalina and Shirekar, Ojas and Raman, Chirag}, title = {OverForge: {Reasoning} {Through} {Strategies} and {Tactics} {Helps} {Cooperative} {Lifelong} {Adaptation}}, journal = {arXiv preprint arXiv:2609.39727}, date = {2026}, url = {https://arxiv.org/abs/2609.39727}, abstract = {Cooperative language-model agents must coordinate over long horizons and adapt to changing environments and to partners with unfamiliar conventions, yet existing agents map observations to actions without separating persistent coordination strategies from their tactical execution. We introduce OverForge, a training-free hierarchical architecture that separates strategic reasoning over roles and divisions of labour from tactical reasoning over actions within each agent’s private, partner-conditioned world model. A metacognitive Prefrontal Cortex Module couples the two levels by forming strategy–action branches, imagining their consequences with a forward model, and committing when confident. In OvercookedV2, OverForge delivers 7 soups in a connected kitchen versus 3 for each flat LLM baseline, retains agreed roles, and adopts roles proposed by unfamiliar partners. Ablations and a fixed-strategy probe show that persistent strategies guide tactical adaptation while each reasoning level contributes to coordination. Memory restarts show that cross-episode partner knowledge supports task performance and partner prediction, linking the hierarchy to continual adaptation.} } @article{zeisler2026prism, author = {Zeisler, Mirella and Shirekar, Ojas and Licǎ, Mircea and Raman, Chirag}, title = {PRISM: {A} {Category-Theoretic} {Framework} for {Measuring} and {Refining} {Multimodal} {Analogies}}, journal = {arXiv preprint arXiv:2610.01383}, date = {2026}, url = {https://arxiv.org/abs/2610.01383}, abstract = {Analogical reasoning involves identifying and preserving relational structures across domains. However, existing approaches to AI-driven multimodal analogy generation lack an interpretable measure of whether this structure is understood and maintained in the generated output. We address this gap with Pullback Refinement via Interpretable Structural Mapping (PRISM), a modality-agnostic framework for measuring and improving relational alignment in multimodal analogies, evaluated on visual metaphor generation. PRISM represents analogies as explicit relational mappings grounded in category theory and uses VLMs to instantiate these structures across modalities. Its first component, the pullback score, quantifies relational alignment from the resulting graph representation. On the AnaloBench benchmark, selecting the correct analogy purely by pullback score achieves 82.5\% accuracy, demonstrating that the score captures meaningful relational information. PRISM’s second component is an iterative refinement loop that uses the pullback score as an in-context feedback signal to iteratively revise the generated image towards greater relational depth. VLM-as-a-judge and human evaluations show that PRISM consistently improves metaphor consistency and analogy appropriateness over zero-shot generation, with human participants preferring the refined output in 57.65\% of pairwise comparisons. However, a qualitative analysis reveals that refinement can favour visually crowded compositions rather than genuinely deeper relational correspondences.} }