67 lines
9.2 KiB
JSON
67 lines
9.2 KiB
JSON
[
|
|
{
|
|
"id": "2310.16818",
|
|
"ok": true,
|
|
"version": "1",
|
|
"title": "DreamCraft3D: Hierarchical 3D Generation with Bootstrapped Diffusion Prior",
|
|
"authors": "Jingxiang Sun , Bo Zhang , Ruizhi Shao , Lizhen Wang , Wen Liu , Zhenda Xie , Yebin Liu",
|
|
"abstract": "We present DreamCraft3D, a hierarchical 3D content generation method that produces high-fidelity and coherent 3D objects. We tackle the problem by leveraging a 2D reference image to guide the stages of geometry sculpting and texture boosting. A central focus of this work is to address the consistency issue that existing works encounter. To sculpt geometries that render coherently, we perform score distillation sampling via a view-dependent diffusion model. This 3D prior, alongside several training strategies, prioritizes the geometry consistency but compromises the texture fidelity. We further propose Bootstrapped Score Distillation to specifically boost the texture. We train a personalized diffusion model, Dreambooth, on the augmented renderings of the scene, imbuing it with 3D knowledge of the scene being optimized. The score distillation from this 3D-aware diffusion prior provides view-consistent guidance for the scene. Notably, through an alternating optimization of the diffusion prior and 3D scene representation, we achieve mutually reinforcing improvements: the optimized 3D scene aids in training the scene-specific diffusion model, which offers increasingly view-consistent guidance for 3D optimization. The optimization is thus bootstrapped and leads to substantial texture boosting. With tailored 3D priors throughout the hierarchical generation, DreamCraft3D generates coherent 3D objects with photorealistic renderings, advancing the state-of-the-art in 3D content generation. Code available at this https URL .",
|
|
"subjects": "Computer Vision and Pattern Recognition (cs.CV) ; Computational Geometry (cs.CG)",
|
|
"dateline": "[Submitted on 25 Oct 2023 ( v1 ), last revised 26 Oct 2023 (this version, v2)]",
|
|
"comments": "Project Page: this https URL",
|
|
"journal_ref": "",
|
|
"doi": ""
|
|
},
|
|
{
|
|
"id": "2408.15664",
|
|
"ok": true,
|
|
"version": "1",
|
|
"title": "Auxiliary-Loss-Free Load Balancing Strategy for Mixture-of-Experts",
|
|
"authors": "Lean Wang , Huazuo Gao , Chenggang Zhao , Xu Sun , Damai Dai",
|
|
"abstract": "For Mixture-of-Experts (MoE) models, an unbalanced expert load will lead to routing collapse or increased computational overhead. Existing methods commonly employ an auxiliary loss to encourage load balance, but a large auxiliary loss will introduce non-negligible interference gradients into training and thus impair the model performance. In order to control load balance while not producing undesired gradients during training, we propose Loss-Free Balancing, featured by an auxiliary-loss-free load balancing strategy. To be specific, before the top-K routing decision, Loss-Free Balancing will first apply an expert-wise bias to the routing scores of each expert. By dynamically updating the bias of each expert according to its recent load, Loss-Free Balancing can consistently maintain a balanced distribution of expert load. In addition, since Loss-Free Balancing does not produce any interference gradients, it also elevates the upper bound of model performance gained from MoE training. We validate the performance of Loss-Free Balancing on MoE models with up to 3B parameters trained on up to 200B tokens. Experimental results show that Loss-Free Balancing achieves both better performance and better load balance compared with traditional auxiliary-loss-controlled load balancing strategies.",
|
|
"subjects": "Machine Learning (cs.LG) ; Computation and Language (cs.CL)",
|
|
"dateline": "[Submitted on 28 Aug 2024]",
|
|
"comments": "",
|
|
"journal_ref": "",
|
|
"doi": ""
|
|
},
|
|
{
|
|
"id": "2411.07975",
|
|
"ok": true,
|
|
"version": "1",
|
|
"title": "JanusFlow: Harmonizing Autoregression and Rectified Flow for Unified Multimodal Understanding and Generation",
|
|
"authors": "Yiyang Ma , Xingchao Liu , Xiaokang Chen , Wen Liu , Chengyue Wu , Zhiyu Wu , Zizheng Pan , Zhenda Xie , Haowei Zhang , Xingkai yu , Liang Zhao , Yisong Wang , Jiaying Liu , Chong Ruan",
|
|
"abstract": "We present JanusFlow, a powerful framework that unifies image understanding and generation in a single model. JanusFlow introduces a minimalist architecture that integrates autoregressive language models with rectified flow, a state-of-the-art method in generative modeling. Our key finding demonstrates that rectified flow can be straightforwardly trained within the large language model framework, eliminating the need for complex architectural modifications. To further improve the performance of our unified model, we adopt two key strategies: (i) decoupling the understanding and generation encoders, and (ii) aligning their representations during unified training. Extensive experiments show that JanusFlow achieves comparable or superior performance to specialized models in their respective domains, while significantly outperforming existing unified approaches across standard benchmarks. This work represents a step toward more efficient and versatile vision-language models.",
|
|
"subjects": "Computer Vision and Pattern Recognition (cs.CV) ; Artificial Intelligence (cs.AI); Computation and Language (cs.CL)",
|
|
"dateline": "[Submitted on 12 Nov 2024 ( v1 ), last revised 24 Mar 2025 (this version, v2)]",
|
|
"comments": "Accepted by CVPR 2025",
|
|
"journal_ref": "",
|
|
"doi": ""
|
|
},
|
|
{
|
|
"id": "2502.07316",
|
|
"ok": true,
|
|
"version": "1",
|
|
"title": "CodeI/O: Condensing Reasoning Patterns via Code Input-Output Prediction",
|
|
"authors": "Junlong Li , Daya Guo , Dejian Yang , Runxin Xu , Yu Wu , Junxian He",
|
|
"abstract": "Reasoning is a fundamental capability of Large Language Models. While prior research predominantly focuses on enhancing narrow skills like math or code generation, improving performance on many other reasoning tasks remains challenging due to sparse and fragmented training data. To address this issue, we propose CodeI/O, a novel approach that systematically condenses diverse reasoning patterns inherently embedded in contextually-grounded codes, through transforming the original code into a code input-output prediction format. By training models to predict inputs/outputs given code and test cases entirely in natural language as Chain-of-Thought (CoT) rationales, we expose them to universal reasoning primitives -- like logic flow planning, state-space searching, decision tree traversal, and modular decomposition -- while decoupling structured reasoning from code-specific syntax and preserving procedural rigor. Experimental results demonstrate CodeI/O leads to consistent improvements across symbolic, scientific, logic, math & numerical, and commonsense reasoning tasks. By matching the existing ground-truth outputs or re-executing the code with predicted inputs, we can verify each prediction and further enhance the CoTs through multi-turn revision, resulting in CodeI/O++ and achieving higher performance. Our data and models are available at this https URL .",
|
|
"subjects": "Computation and Language (cs.CL) ; Artificial Intelligence (cs.AI)",
|
|
"dateline": "[Submitted on 11 Feb 2025 ( v1 ), last revised 21 May 2025 (this version, v4)]",
|
|
"comments": "ICML 2025",
|
|
"journal_ref": "",
|
|
"doi": ""
|
|
},
|
|
{
|
|
"id": "2502.11089",
|
|
"ok": true,
|
|
"version": "1",
|
|
"title": "Native Sparse Attention: Hardware-Aligned and Natively Trainable Sparse Attention",
|
|
"authors": "Jingyang Yuan , Huazuo Gao , Damai Dai , Junyu Luo , Liang Zhao , Zhengyan Zhang , Zhenda Xie , Y. X. Wei , Lean Wang , Zhiping Xiao , Yuqing Wang , Chong Ruan , Ming Zhang , Wenfeng Liang , Wangding Zeng",
|
|
"abstract": "Long-context modeling is crucial for next-generation language models, yet the high computational cost of standard attention mechanisms poses significant computational challenges. Sparse attention offers a promising direction for improving efficiency while maintaining model capabilities. We present NSA, a Natively trainable Sparse Attention mechanism that integrates algorithmic innovations with hardware-aligned optimizations to achieve efficient long-context modeling. NSA employs a dynamic hierarchical sparse strategy, combining coarse-grained token compression with fine-grained token selection to preserve both global context awareness and local precision. Our approach advances sparse attention design with two key innovations: (1) We achieve substantial speedups through arithmetic intensity-balanced algorithm design, with implementation optimizations for modern hardware. (2) We enable end-to-end training, reducing pretraining computation without sacrificing model performance. As shown in Figure 1, experiments show the model pretrained with NSA maintains or exceeds Full Attention models across general benchmarks, long-context tasks, and instruction-based reasoning. Meanwhile, NSA achieves substantial speedups over Full Attention on 64k-length sequences across decoding, forward propagation, and backward propagation, validating its efficiency throughout the model lifecycle.",
|
|
"subjects": "Computation and Language (cs.CL) ; Artificial Intelligence (cs.AI); Machine Learning (cs.LG)",
|
|
"dateline": "[Submitted on 16 Feb 2025 ( v1 ), last revised 27 Feb 2025 (this version, v2)]",
|
|
"comments": "",
|
|
"journal_ref": "",
|
|
"doi": ""
|
|
}
|
|
] |