


@Proceedings{NeurIPS Competition Track2025,
  title =     {Proceedings of the NeurIPS 2023-2025 Competition Track},
  booktitle = {Proceedings of the NeurIPS 2023-2025 Competition Track},
  editor =    {Tao Qin and Kun Zhang and Jes Frellsen},
  publisher = {PMLR},
  series =    {Proceedings of Machine Learning Research},
  volume =    276
}



@InProceedings{pmlr-v276-bao26a,
  title = 	 {NeurIPS 2025 Foundation Models Meet Embodied Agents Challenge: Systems, Analysis, and Lessons Learned},
  author =       {Bao, Tianwei and Wang, Qineng and Wang, Kangrui and Zhou, Bryan Yu and Lee, Tony and Li, Li Erran and Liu, Weiyu and Zhang, Ruohan and Choi, Yejin and Liang, Percy and Mao, Jiayuan and Fei-Fei, Li and Wu, Jiajun and Li, Manling},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {1--16},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/bao26a/bao26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/bao26a.html},
  abstract = 	 {While Large Language Models (LLMs) have demonstrated impressive high-level reasoning capabilities, their deployment in embodied decision-making tasks remains hindered by a lack of standardized, diagnostic evaluations. We present a retrospective on the NeurIPS 2025 Embodied Agent Interface Challenge, which systematically benchmarked LLMs across four critical reasoning modules: goal interpretation, subgoal decomposition, action sequencing, and transition modeling. Hosted on the BEHAVIOR and VirtualHome simulators, the competition departed from traditional, binary task-success metrics. Instead, it leveraged Linear Temporal Logic (LTL) to evaluate alternative viable trajectories and introduced fine-grained metrics to pinpoint specific failures, such as affordance violations, hallucinations, and missing preconditions. The competition attracted significant engagement globally, with 58 registered teams developing diverse solutions. An analysis of the winning and most innovative submissions reveals a clear trend: while hybrid neuro-symbolic pipelines and test-time compute significantly enhance trajectory feasibility, spatial relationship modeling remains a pervasive bottleneck for current LLMs. In this report, we detail the competition design, summarize the top-performing methodologies, and present key lessons learned to guide the future development of robust, interpretable embodied AI systems.}
}



@InProceedings{pmlr-v276-chan26a,
  title = 	 {{MMU-RAGent}: Massive Multi-Modal User-Centric Retrieval Augmented Generation Benchmark},
  author =       {Chan, Luo Qi and Wang, Tevin and Wang, Shuting and Zhang, Zhihan and Gomez, Alfredo and Chandrahasan, Prahaladh and Tang, Andy and Yan, Lan and Qiu, Zimeng and Ziyadi, Morteza and Wu, Tongshuang and Diab, Mona and Asai, Akari and Xiong, Chenyan and Temiraliev, Izat and Cheng, Serene and Yang, Diji and Zhang, Yi and Ran, Kun and Alaofi, Marwah and Hettiachchi, Danula and Ma, Chenglong and Dinh Anh, Khoi Nguyen and Nguyen, Khoi Vo and Pathiyan Cherumanal, Sachin and Rashidi, Lida and Scholer, Falk and Spina, Damiano and Sun, Shuoqi and Zendel, Oleg and Yamada, Ikuya and Ikeda, Wataru and Yoshida, Ko and Ye, Mengyu and Sugimoto, Hinata and Suzuki, Masatoshi and Ozaki, Hisanori and Suzuki, Jun and Nuengsigkapian, Cattalyya and Magas, Dmitrii and Fever, Quentin and Aslam, Naziha},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {17--52},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/chan26a/chan26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/chan26a.html},
  abstract = 	 {We present the results of the MMU-RAGent competition. The competition tasked participants with building systems capable of answering real-world queries drawn from MS MARCO Web Search and Chatbot Arena conversations, as well as queries collected live through the RAG-Arena evaluation platform. To support large-scale retrieval, participants were given API access to the English subsets of ClueWeb22-B and ClueWeb22-A (87M and 800M documents) and AWS-hosted infrastructure for system deployment and evaluation. Submissions were evaluated using a combination of human Likert-scale ratings, live preference judgments collected via RAG-Arena, LLM-as-a-Judge evaluation, and automatic metrics. The competition allowed both open-source and closed-source approaches, including systems leveraging proprietary search APIs or models, with separate leaderboard categories to ensure transparent comparison. Participating systems explored a wide range of RAG architectures, including hierarchical retrieval pipelines, structured document parsing, multi-stage reranking, and iterative generation with citation verification. The competition provides a new benchmark for studying retrieval-augmented generation under realistic deployment conditions and highlights emerging system design patterns for scalable and user-aligned RAG.}
}



@InProceedings{pmlr-v276-elachqar26a,
  title = 	 {{DCVLR}: Data Curation for Vision Language Reasoning Competition},
  author =       {Elachqar, Oussama and Feuer, Benjamin and Hulkund, Neha and Tripathi, Rohun and Agrawal, Rohun and Nguyen, Thao and Shabtay, Nimrod and Udandarao, Vishaal and Wang, Xiaohan and Webb, Stefan and Zhang, Yuhui and Beery, Sara and Gkioxari, Georgia and Liang, Paul and Schmidt, Ludwig and Xie, Saining and Yeung-Levy, Serena and Buriek, Michael and Bushuyeu, Pavel and Damianos, Dimitrios and Kasoura, Eleni and Katsouros, Vassilis and Kumar, Vikas and Molybog, Igor and Paraskevopoulos, Georgios and Shin, Yosub and Sobolev, Boris and Watson, Samuel and Xu, Haoyang and Koukoumidis, Emmanouil},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {53--68},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/elachqar26a/elachqar26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/elachqar26a.html},
  abstract = 	 {At NeurIPS 2025, we presented the first data-centric, open-data and open-models competition for data curation in vision-language reasoning. The competition focused on improving visual reasoning capabilities of vision-language models (VLMs) through data curation under a fixed supervised-finetuning (SFT) configuration and budget. Submissions were evaluated by fine-tuning a fixed VLM on the curated datasets and measuring performance on reasoning benchmarks of multiple-choice visual reasoning questions spanning diverse sources. The competition provided all necessary resources, including fine-tuning scripts, evaluation code, and baselines. Additionally, as a first of competitions of this kind, organizers also ran the entire SFT and evaluation pipeline on behalf of the contestants, thereby removing a key barrier to entry present in previous data-centric competitions. The result was the largest data-centric competition to date, with 130 signups and 20 submissions. DCVLR centered data curation as the key variable for scientific progress in VLM adaptation, highlighting its importance for real-world applications such as education, biomedicine, and scientific reasoning.}
}



@InProceedings{pmlr-v276-elvers26a,
  title = 	 {Benchmarking Non-Invasive Speech {BCIs}: Lessons Learned from the 2025 {PNPL} Competition},
  author =       {Elvers, Gereon and Landau, Gilad and Mantegna, Francesco and {\"O}zdogan, Miran and Kim, Tasha and Kwon, Teyun and Cho, SungJun and Ballyk, Benjamin and Kurth, Luisa and Jayalath, Dulhan and Somaiya, Pratik and Gohil, Chetan and Shillingford, Brendan and Farquhar, Greg and Jiang, Minqi and Gulcehre, Caglar and de Zuazo, Xabier and Abdelhedi, Hamza and Mantilla Ramos, Yorguin and Jerbi, Karim and Woolrich, Mark and Voets, Natalie and Parker Jones, Oiwi},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {69--89},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/elvers26a/elvers26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/elvers26a.html},
  abstract = 	 {We present a retrospective on the 2025 PNPL Competition, an open machine-learning benchmark for decoding speech from non-invasive brain recordings, which ran from June to September 2025 and culminated in a workshop at NeurIPS that December. The competition used the LibriBrain dataset — over 50 hours of within-subject MEG with standard data splits — and challenged participants to tackle two tasks: Speech Detection (binary frame-level classification) and Phoneme Classification (39-class ARPAbet prediction). Across 155 registered teams and 6,041 submissions, top F1-macro scores reached 95.6% for Speech Detection and 73.6% for Phoneme Classification, exceeding prior baselines of 68% and 44%, respectively, by wide margins. High-performing submissions converged on a common architectural template — temporal CNN or TCN backbones with task-specific recurrent or attention-based heads — whilst task-specific strategies such as temporal smoothing and ensemble voting proved critical for competitive performance. Hidden track evaluations suggest that main-track Speech Detection models generalised poorly to balanced class distributions. This could imply a reliance on class priors rather than robust neural decoding, highlighting a useful lesson: evaluation distributions should reflect intended deployment conditions. We further reflect on design choices, organisational lessons, and plans for future competitions in this series.}
}



@InProceedings{pmlr-v276-gruca26a,
  title = 	 {Weather4cast at {NeurIPS} 2023 – Data Fusion for Quantitative Hi-Res Rain Movie Prediction under Spatio-temporal Shifts},
  author =       {Gruca, Aleksandra and Calbet, Xavier and R{\'i}podas, Pilar and Lliso, Lloren{\c c} and Serva, Federico and Le Saux, Bertrand and Han, Lu and Chen, Xu-Yang and Ye, Han-Jia and Zhan, De-Chuan and Reddy, P. Jyoteeshkumar and Baki, Harish and Chinta, Sandeep and Izquierdo Ayala, Pablo and Punjabi, Akshay and Kumar, Ajitabh and Pablos Sarabia, Rafael and Assent, Ira and Nyborg, Joachim and Birk, Morten and Kreil, David P.},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {90--105},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/gruca26a/gruca26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/gruca26a.html},
  abstract = 	 {Now in its third year, Weather4cast 2023 again advanced modern algorithms in AI and machine learning through a highly topical interdisciplinary challenge at the NeurIPS Competition Track. Predicting hi-res rain radar movies from multi-band satellite sensors required data fusion of complementary signal sources and video frame prediction with super-resolution techniques. This year, quantitative prediction of rain rates was assessed using a balanced multi-class metric. Accurate rain predictions are ever more critical for everyone, with climate change increasing the frequency of extreme precipitation events. Weather4cast models thus have a particular impact for the many regions on Earth where costly weather radar data are not available. We here present highlights and insights emerging from the competition. To reward models extracting relevant mechanistic patterns reflecting the underlying complex weather systems, our evaluation tested their responses to spatio-temporal shifts. Algorithms needed to forecast 4 h or 8 h of ground-based hi-res precipitation radar from lo-res satellite spectral images in a unique cross-sensor prediction challenge, and were evaluated within and across a variety of regions on Earth with diverse climate and different distributions of heavy precipitation events. Robustness to distributional shifts over time was achieved by testing predictions on data one year after the training period. Notably, the capability of models to correctly capture spatio-temporal patterns qualitatively varied considerably and was not always in line with leaderboard scores. This suggests that more sophisticated metrics will be required to accurately reflect the effectiveness of AI models in corresponding meteorological applications. Weather4cast will continue to explore the expansive data set and advance benchmark challenges in the field en route to validated foundation models for weather.}
}



@InProceedings{pmlr-v276-gruca26b,
  title = 	 {Weather4cast at {NeurIPS} 2025 – Multi-task Challenges for Weather and Pollution Pattern Prediction on the Road to Hi-Res Foundation Models},
  author =       {Gruca, Aleksandra and Calbet, Xavier and R{\'i}podas, Pilar and Lliso, Lloren{\c c} and Serva, Federico and Harris, Levi and Chen, Tianlong and Dourado Filho, Luciano Araujo and da Silva Neto, Almir Moreira and Miyaguchi, Anthony and David, Rodrigo Pereira and Calumby, Rodrigo Tripodi and Picek, Lukas and Gopalan, Kaushik and Deshpande, Atharva and Shah, Jeet and Simu, Hrishikesh and Kushwaha, Ansh and Bhuskute, Anushree and Kreil, David P.},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {106--122},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/gruca26b/gruca26b.pdf},
  url = 	 {https://proceedings.mlr.press/v276/gruca26b.html},
  abstract = 	 {The Weather4cast competition series aims to advance AI and machine learning algorithms through a highly topical interdisciplinary challenge (www.weather4cast.net). The core task involves predicting high-resolution rain radar movies from multi-band satellite sensors, which requires the application of data fusion across complementary signal sources, multi-channel video frame prediction, and super-resolution techniques. To evaluate the models’ ability to extract relevant mechanistic patterns, the competition incorporates spatio-temporal shifts, evaluating predictions both within and across diverse Earth regions with varying climates and precipitation distributions. Now, in its fifth year, Weather4cast moves to improve forecasts world-wide on an expansive data set with over a magnitude more hi-res rain radar data, allowing a move towards Foundation Models through multi-modality, multi-scale, multi-task challenges. Accurate rain predictions are becoming ever more critical for everyone, with climate change increasing the frequency of extreme precipitation events. Notably, the new models and insights will have a particular impact for the many regions on Earth where costly weather radar data are not available. As a complementary application-specific forecasting endpoint, in 2025, for the first time, we added a pollution forecasting challenge.}
}



@InProceedings{pmlr-v276-moffitt26a,
  title = 	 {The 2025 Google Code Golf Championship},
  author =       {Moffitt, Michael D. and Thakkar, Divy and Burnell, Ryan and Firat, Orhan and Reade, Walter and Dane, Sohier and Howard, Addison and Deutschmann, Lucas and Gustafson, Luke and Kues, Adam and Senthilkumar, Mukundan and Posada, Nicolas and Ho, Lyndon and Wu, Larry and Patel, Lav and Richardson, Hunter and Kuan, Wei Heng},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {123--173},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/moffitt26a/moffitt26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/moffitt26a.html},
  abstract = 	 {For over half a decade, the Abstraction and Reasoning Corpus has played a critical role in evaluating the fluid intelligence and skill acquisition efficiency of machine learning systems. While the dataset has proven especially useful in advancing program synthesis, few investments have been made in distilling the transformational logic of these tasks into their most irreducible computational forms. In conjunction with the NeurIPS 2025 Competition Track, we organized the 2025 Google Code Golf Championship, a three-month online challenge designed to elicit minimal Python solutions for all training puzzles in the seminal ARC-AGI-1 benchmark suite. Sponsored by Google DeepMind and funded with $100,000 in prizes, the contest attracted over 1,000 teams and yielded tens of thousands of verified, open-source programs that collectively establish tight upper bounds on the Kolmogorov complexity of each task. In this paper, we offer a comprehensive retrospective of the event, including an overview of the contest format, a survey of approaches taken by top teams, and a discussion of potential implications for the future design of intelligent agents.}
}



@InProceedings{pmlr-v276-zhang26a,
  title = 	 {Lessons Learned from Auto-Bidding in the Large-Scale Auctions Competition at {NeurIPS} 2024},
  author =       {Zhang, Zhilin and Dou, Shuai and Huo, Yusen and Duan, Zhijian and Li, Ningyuan and Yu, Chuan and Deng, Xiaotie and Xu, Jian and Zheng, Bo},
  booktitle = 	 {Proceedings of the NeurIPS 2023-2025 Competition Track},
  pages = 	 {174--185},
  year = 	 {2026},
  editor = 	 {Qin, Tao and Zhang, Kun and Frellsen, Jes},
  volume = 	 {276},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {02--07 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v276/main/assets/zhang26a/zhang26a.pdf},
  url = 	 {https://proceedings.mlr.press/v276/zhang26a.html},
  abstract = 	 {Decision-making in large-scale games is an essential research area in artificial intelligence, with significant real-world impact. An agent faces the critical task of making high-frequency strategic decisions in an uncertain and competitive environment. This environment is characterized by substantial randomness and rapidly changing strategies from numerous competitors. However, the shortage of large-scale, realistic game systems and datasets has hindered research progress in this area. To provide opportunities for in-depth research on this valuable problem, we have presented the Auto-Bidding in Large-Scale Auctions Competition, derived from online advertising, at NeurIPS 2024. This competition focuses on bidding decision-making in uncertain and competitive games. It specifically includes the AIGB (AI-Generated Bidding) track, which emphasizes the innovation of generative algorithms in the field of auto-bidding. This competition has attracted over 1,500 teams from around the world. After six months of intense competition, participants have proposed numerous innovative and forward-looking solutions, significantly promoting research and technological development in this field. In this competition, we have open-sourced AuctionNet, a benchmark for bidding decision-making in large-scale ad auctions derived from a real-world online advertising platform. We have also pre-generated a substantial dataset comprising 10 million ad opportunities, 48 diverse auto-bidding agents, and over 500 million auction records. Performance evaluations of baseline algorithms, such as linear programming, reinforcement learning, and generative models, are also presented as part of AuctionNet. AuctionNet has powered the competition at NeurIPS 2024. We believe that the proposed competition and AuctionNet provide opportunities for more researchers to gain insights and conduct research in this field, driving technical innovation for both academic research and real-world practical applications.}
}



