[
 {
  "id": "2401.02954",
  "slug": "deepseek-llm-scaling-open-source-language-models-with-longtermism-240102954",
  "title": "DeepSeek LLM: Scaling Open-Source Language Models with Longtermism",
  "authors": "DeepSeek-AI : Xiao Bi , Deli Chen , Guanting Chen , Shanhuang Chen , Damai Dai , Chengqi Deng , Honghui Ding , Kai Dong , Qiushi Du , Zhe Fu , Huazuo Gao , Kaige Gao , Wenjun Gao , Ruiqi Ge , Kang Guan , Daya Guo , Jianzhong Guo , Guangbo Hao , Zhewen Hao , Ying He , Wenjie Hu , Panpan Huang , Erhang Li , Guowei Li , Jiashi Li , Yao Li , Y.K. Li , Wenfeng Liang , Fangyun Lin , A.X. Liu , Bo Liu , Wen Liu , Xiaodong Liu , Xin Liu , Yiyuan Liu , Haoyu Lu , Shanghao Lu , Fuli Luo , Shirong Ma , Xiaotao Nie , Tian Pei , Yishi Piao , Junjie Qiu , Hui Qu , Tongzheng Ren , Zehui Ren , Chong Ruan , Zhangli Sha , Zhihong Shao , Junxiao Song , Xuecheng Su , Jingxiang Sun , Yaofeng Sun , Minghui Tang , Bingxuan Wang , Peiyi Wang , Shiyu Wang , Yaohui Wang , Yongji Wang , Tong Wu , Y. Wu , Xin Xie , Zhenda Xie , Ziwei Xie , Yiliang Xiong , Hanwei Xu , R.X. Xu , Yanhong Xu , Dejian Yang , Yuxiang You , Shuiping Yu , Xingkai Yu , B. Zhang , Haowei Zhang , Lecong Zhang , Liyue Zhang , Mingchuan Zhang , Minghua Zhang , Wentao Zhang , Yichao Zhang , Chenggang Zhao , Yao Zhao , Shangyan Zhou , Shunfeng Zhou , Qihao Zhu , Yuheng Zou",
  "abstract": "The rapid development of open-source large language models (LLMs) has been truly remarkable. However, the scaling law described in previous literature presents varying conclusions, which casts a dark cloud over scaling LLMs. We delve into the study of scaling laws and present our distinctive findings that facilitate scaling of large scale models in two commonly used open-source configurations, 7B and 67B. Guided by the scaling laws, we introduce DeepSeek LLM, a project dedicated to advancing open-source language models with a long-term perspective. To support the pre-training phase, we have developed a dataset that currently consists of 2 trillion tokens and is continuously expanding. We further conduct supervised fine-tuning (SFT) and Direct Preference Optimization (DPO) on DeepSeek LLM Base models, resulting in the creation of DeepSeek Chat models. Our evaluation results demonstrate that DeepSeek LLM 67B surpasses LLaMA-2 70B on various benchmarks, particularly in the domains of code, mathematics, and reasoning. Furthermore, open-ended evaluations reveal that DeepSeek LLM 67B Chat exhibits superior performance compared to GPT-3.5.",
  "date": "5 Jan 2024",
  "dateline": "[Submitted on 5 Jan 2024]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2401.02954",
  "abs_url": "https://arxiv.org/abs/2401.02954"
 },
 {
  "id": "2401.06066",
  "slug": "deepseekmoe-towards-ultimate-expert-specialization-in-mixture-of-exper-240106066",
  "title": "DeepSeekMoE: Towards Ultimate Expert Specialization in Mixture-of-Experts Language Models",
  "authors": "Damai Dai , Chengqi Deng , Chenggang Zhao , R.X. Xu , Huazuo Gao , Deli Chen , Jiashi Li , Wangding Zeng , Xingkai Yu , Y. Wu , Zhenda Xie , Y.K. Li , Panpan Huang , Fuli Luo , Chong Ruan , Zhifang Sui , Wenfeng Liang",
  "abstract": "In the era of large language models, Mixture-of-Experts (MoE) is a promising architecture for managing computational costs when scaling up model parameters. However, conventional MoE architectures like GShard, which activate the top-$K$ out of $N$ experts, face challenges in ensuring expert specialization, i.e. each expert acquires non-overlapping and focused knowledge. In response, we propose the DeepSeekMoE architecture towards ultimate expert specialization. It involves two principal strategies: (1) finely segmenting the experts into $mN$ ones and activating $mK$ from them, allowing for a more flexible combination of activated experts; (2) isolating $K_s$ experts as shared ones, aiming at capturing common knowledge and mitigating redundancy in routed experts. Starting from a modest scale with 2B parameters, we demonstrate that DeepSeekMoE 2B achieves comparable performance with GShard 2.9B, which has 1.5 times the expert parameters and computation. In addition, DeepSeekMoE 2B nearly approaches the performance of its dense counterpart with the same number of total parameters, which set the upper bound of MoE models. Subsequently, we scale up DeepSeekMoE to 16B parameters and show that it achieves comparable performance with LLaMA2 7B, with only about 40% of computations. Further, our preliminary efforts to scale up DeepSeekMoE to 145B parameters consistently validate its substantial advantages over the GShard architecture, and show its performance comparable with DeepSeek 67B, using only 28.5% (maybe even 18.2%) of computations.",
  "date": "11 Jan 2024",
  "dateline": "[Submitted on 11 Jan 2024]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2401.06066",
  "abs_url": "https://arxiv.org/abs/2401.06066"
 },
 {
  "id": "2401.14196",
  "slug": "deepseek-coder-when-the-large-language-model-meets-programming-the-ris-240114196",
  "title": "DeepSeek-Coder: When the Large Language Model Meets Programming -- The Rise of Code Intelligence",
  "authors": "Daya Guo , Qihao Zhu , Dejian Yang , Zhenda Xie , Kai Dong , Wentao Zhang , Guanting Chen , Xiao Bi , Y. Wu , Y.K. Li , Fuli Luo , Yingfei Xiong , Wenfeng Liang",
  "abstract": "The rapid development of large language models has revolutionized code intelligence in software development. However, the predominance of closed-source models has restricted extensive research and development. To address this, we introduce the DeepSeek-Coder series, a range of open-source code models with sizes from 1.3B to 33B, trained from scratch on 2 trillion tokens. These models are pre-trained on a high-quality project-level code corpus and employ a fill-in-the-blank task with a 16K window to enhance code generation and infilling. Our extensive evaluations demonstrate that DeepSeek-Coder not only achieves state-of-the-art performance among open-source code models across multiple benchmarks but also surpasses existing closed-source models like Codex and GPT-3.5. Furthermore, DeepSeek-Coder models are under a permissive license that allows for both research and unrestricted commercial use.",
  "date": "25 Jan 2024",
  "dateline": "[Submitted on 25 Jan 2024 ( v1 ), last revised 26 Jan 2024 (this version, v2)]",
  "comments": "",
  "category": "代码模型",
  "pdf_url": "https://arxiv.org/pdf/2401.14196",
  "abs_url": "https://arxiv.org/abs/2401.14196"
 },
 {
  "id": "2402.03300",
  "slug": "deepseekmath-pushing-the-limits-of-mathematical-reasoning-in-open-lang-240203300",
  "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
  "authors": "Zhihong Shao , Peiyi Wang , Qihao Zhu , Runxin Xu , Junxiao Song , Xiao Bi , Haowei Zhang , Mingchuan Zhang , Y.K. Li , Y. Wu , Daya Guo",
  "abstract": "Mathematical reasoning poses a significant challenge for language models due to its complex and structured nature. In this paper, we introduce DeepSeekMath 7B, which continues pre-training DeepSeek-Coder-Base-v1.5 7B with 120B math-related tokens sourced from Common Crawl, together with natural language and code data. DeepSeekMath 7B has achieved an impressive score of 51.7% on the competition-level MATH benchmark without relying on external toolkits and voting techniques, approaching the performance level of Gemini-Ultra and GPT-4. Self-consistency over 64 samples from DeepSeekMath 7B achieves 60.9% on MATH. The mathematical reasoning capability of DeepSeekMath is attributed to two key factors: First, we harness the significant potential of publicly available web data through a meticulously engineered data selection pipeline. Second, we introduce Group Relative Policy Optimization (GRPO), a variant of Proximal Policy Optimization (PPO), that enhances mathematical reasoning abilities while concurrently optimizing the memory usage of PPO.",
  "date": "5 Feb 2024",
  "dateline": "[Submitted on 5 Feb 2024 ( v1 ), last revised 27 Apr 2024 (this version, v3)]",
  "comments": "",
  "category": "数学模型",
  "pdf_url": "https://arxiv.org/pdf/2402.03300",
  "abs_url": "https://arxiv.org/abs/2402.03300"
 },
 {
  "id": "2403.05525",
  "slug": "deepseek-vl-towards-real-world-vision-language-understanding-240305525",
  "title": "DeepSeek-VL: Towards Real-World Vision-Language Understanding",
  "authors": "Haoyu Lu , Wen Liu , Bo Zhang , Bingxuan Wang , Kai Dong , Bo Liu , Jingxiang Sun , Tongzheng Ren , Zhuoshu Li , Hao Yang , Yaofeng Sun , Chengqi Deng , Hanwei Xu , Zhenda Xie , Chong Ruan",
  "abstract": "We present DeepSeek-VL, an open-source Vision-Language (VL) Model designed for real-world vision and language understanding applications. Our approach is structured around three key dimensions: We strive to ensure our data is diverse, scalable, and extensively covers real-world scenarios including web screenshots, PDFs, OCR, charts, and knowledge-based content, aiming for a comprehensive representation of practical contexts. Further, we create a use case taxonomy from real user scenarios and construct an instruction tuning dataset accordingly. The fine-tuning with this dataset substantially improves the model's user experience in practical applications. Considering efficiency and the demands of most real-world scenarios, DeepSeek-VL incorporates a hybrid vision encoder that efficiently processes high-resolution images (1024 x 1024), while maintaining a relatively low computational overhead. This design choice ensures the model's ability to capture critical semantic and detailed information across various visual tasks. We posit that a proficient Vision-Language Model should, foremost, possess strong language abilities. To ensure the preservation of LLM capabilities during pretraining, we investigate an effective VL pretraining strategy by integrating LLM training from the beginning and carefully managing the competitive dynamics observed between vision and language modalities. The DeepSeek-VL family (both 1.3B and 7B models) showcases superior user experiences as a vision-language chatbot in real-world applications, achieving state-of-the-art or competitive performance across a wide range of visual-language benchmarks at the same model size while maintaining robust performance on language-centric benchmarks. We have made both 1.3B and 7B models publicly accessible to foster innovations based on this foundation model.",
  "date": "8 Mar 2024",
  "dateline": "[Submitted on 8 Mar 2024 ( v1 ), last revised 11 Mar 2024 (this version, v2)]",
  "comments": "this https URL",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2403.05525",
  "abs_url": "https://arxiv.org/abs/2403.05525"
 },
 {
  "id": "2405.04434",
  "slug": "deepseek-v2-a-strong-economical-and-efficient-mixture-of-experts-langu-240504434",
  "title": "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model",
  "authors": "DeepSeek-AI , Aixin Liu , Bei Feng , Bin Wang , Bingxuan Wang , Bo Liu , Chenggang Zhao , Chengqi Dengr , Chong Ruan , Damai Dai , Daya Guo , Dejian Yang , Deli Chen , Dongjie Ji , Erhang Li , Fangyun Lin , Fuli Luo , Guangbo Hao , Guanting Chen , Guowei Li , H. Zhang , Hanwei Xu , Hao Yang , Haowei Zhang , Honghui Ding , Huajian Xin , Huazuo Gao , Hui Li , Hui Qu , J.L. Cai , Jian Liang , Jianzhong Guo , Jiaqi Ni , Jiashi Li , Jin Chen , Jingyang Yuan , Junjie Qiu , Junxiao Song , Kai Dong , Kaige Gao , Kang Guan , Lean Wang , Lecong Zhang , Lei Xu , Leyi Xia , Liang Zhao , Liyue Zhang , Meng Li , Miaojun Wang , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Mingming Li , Ning Tian , Panpan Huang , Peiyi Wang , Peng Zhang , Qihao Zhu , Qinyu Chen , Qiushi Du , R.J. Chen , R.L. Jin , Ruiqi Ge , Ruizhe Pan , Runxin Xu , Ruyi Chen , S.S. Li , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shaoqing Wu , Shengfeng Ye , Shirong Ma , Shiyu Wang , Shuang Zhou , Shuiping Yu , Shunfeng Zhou , Size Zheng , T. Wang , Tian Pei , Tian Yuan , Tianyu Sun , W.L. Xiao , Wangding Zeng , Wei An , Wen Liu , Wenfeng Liang , Wenjun Gao , Wentao Zhang , X.Q. Li , Xiangyue Jin , Xianzu Wang , Xiao Bi , Xiaodong Liu , Xiaohan Wang , Xiaojin Shen , Xiaokang Chen , Xiaosha Chen , Xiaotao Nie , Xiaowen Sun , Xiaoxiang Wang , Xin Liu , Xin Xie , Xingkai Yu , Xinnan Song , Xinyi Zhou , Xinyu Yang , Xuan Lu , Xuecheng Su , Y. Wu , Y.K. Li , Y.X. Wei , Y.X. Zhu , Yanhong Xu , Yanping Huang , Yao Li , Yao Zhao , Yaofeng Sun , Yaohui Li , Yaohui Wang , Yi Zheng , Yichao Zhang , Yiliang Xiong , Yilong Zhao , Ying He , Ying Tang , Yishi Piao , Yixin Dong , Yixuan Tan , Yiyuan Liu , Yongji Wang , Yongqiang Guo , Yuchen Zhu , Yuduan Wang , Yuheng Zou , Yukun Zha , Yunxian Ma , Yuting Yan , Yuxiang You , Yuxuan Liu , Z.Z. Ren , Zehui Ren , Zhangli Sha , Zhe Fu , Zhen Huang , Zhen Zhang , Zhenda Xie , Zhewen Hao , Zhihong Shao , Zhiniu Wen , Zhipeng Xu , Zhongyu Zhang , Zhuoshu Li , Zihan Wang , Zihui Gu , Zilin Li , Ziwei Xie",
  "abstract": "We present DeepSeek-V2, a strong Mixture-of-Experts (MoE) language model characterized by economical training and efficient inference. It comprises 236B total parameters, of which 21B are activated for each token, and supports a context length of 128K tokens. DeepSeek-V2 adopts innovative architectures including Multi-head Latent Attention (MLA) and DeepSeekMoE. MLA guarantees efficient inference through significantly compressing the Key-Value (KV) cache into a latent vector, while DeepSeekMoE enables training strong models at an economical cost through sparse computation. Compared with DeepSeek 67B, DeepSeek-V2 achieves significantly stronger performance, and meanwhile saves 42.5% of training costs, reduces the KV cache by 93.3%, and boosts the maximum generation throughput to 5.76 times. We pretrain DeepSeek-V2 on a high-quality and multi-source corpus consisting of 8.1T tokens, and further perform Supervised Fine-Tuning (SFT) and Reinforcement Learning (RL) to fully unlock its potential. Evaluation results show that, even with only 21B activated parameters, DeepSeek-V2 and its chat versions still achieve top-tier performance among open-source models.",
  "date": "7 May 2024",
  "dateline": "[Submitted on 7 May 2024 ( v1 ), last revised 19 Jun 2024 (this version, v5)]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2405.04434",
  "abs_url": "https://arxiv.org/abs/2405.04434"
 },
 {
  "id": "2405.14333",
  "slug": "deepseek-prover-advancing-theorem-proving-in-llms-through-large-scale--240514333",
  "title": "DeepSeek-Prover: Advancing Theorem Proving in LLMs through Large-Scale Synthetic Data",
  "authors": "Huajian Xin , Daya Guo , Zhihong Shao , Zhizhou Ren , Qihao Zhu , Bo Liu , Chong Ruan , Wenda Li , Xiaodan Liang",
  "abstract": "Proof assistants like Lean have revolutionized mathematical proof verification, ensuring high accuracy and reliability. Although large language models (LLMs) show promise in mathematical reasoning, their advancement in formal theorem proving is hindered by a lack of training data. To address this issue, we introduce an approach to generate extensive Lean 4 proof data derived from high-school and undergraduate-level mathematical competition problems. This approach involves translating natural language problems into formal statements, filtering out low-quality statements, and generating proofs to create synthetic data. After fine-tuning the DeepSeekMath 7B model on this synthetic dataset, which comprises 8 million formal statements with proofs, our model achieved whole-proof generation accuracies of 46.3% with 64 samples and 52% cumulatively on the Lean 4 miniF2F test, surpassing the baseline GPT-4 at 23.0% with 64 samples and a tree search reinforcement learning method at 41.0%. Additionally, our model successfully proved 5 out of 148 problems in the Lean 4 Formalized International Mathematical Olympiad (FIMO) benchmark, while GPT-4 failed to prove any. These results demonstrate the potential of leveraging large-scale synthetic data to enhance theorem-proving capabilities in LLMs. Both the synthetic dataset and the model will be made available to facilitate further research in this promising field.",
  "date": "23 May 2024",
  "dateline": "[Submitted on 23 May 2024]",
  "comments": "",
  "category": "形式化证明",
  "pdf_url": "https://arxiv.org/pdf/2405.14333",
  "abs_url": "https://arxiv.org/abs/2405.14333"
 },
 {
  "id": "2406.11931",
  "slug": "deepseek-coder-v2-breaking-the-barrier-of-closed-source-models-in-code-240611931",
  "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence",
  "authors": "DeepSeek-AI , Qihao Zhu , Daya Guo , Zhihong Shao , Dejian Yang , Peiyi Wang , Runxin Xu , Y. Wu , Yukun Li , Huazuo Gao , Shirong Ma , Wangding Zeng , Xiao Bi , Zihui Gu , Hanwei Xu , Damai Dai , Kai Dong , Liyue Zhang , Yishi Piao , Zhibin Gou , Zhenda Xie , Zhewen Hao , Bingxuan Wang , Junxiao Song , Deli Chen , Xin Xie , Kang Guan , Yuxiang You , Aixin Liu , Qiushi Du , Wenjun Gao , Xuan Lu , Qinyu Chen , Yaohui Wang , Chengqi Deng , Jiashi Li , Chenggang Zhao , Chong Ruan , Fuli Luo , Wenfeng Liang",
  "abstract": "We present DeepSeek-Coder-V2, an open-source Mixture-of-Experts (MoE) code language model that achieves performance comparable to GPT4-Turbo in code-specific tasks. Specifically, DeepSeek-Coder-V2 is further pre-trained from an intermediate checkpoint of DeepSeek-V2 with additional 6 trillion tokens. Through this continued pre-training, DeepSeek-Coder-V2 substantially enhances the coding and mathematical reasoning capabilities of DeepSeek-V2, while maintaining comparable performance in general language tasks. Compared to DeepSeek-Coder-33B, DeepSeek-Coder-V2 demonstrates significant advancements in various aspects of code-related tasks, as well as reasoning and general capabilities. Additionally, DeepSeek-Coder-V2 expands its support for programming languages from 86 to 338, while extending the context length from 16K to 128K. In standard benchmark evaluations, DeepSeek-Coder-V2 achieves superior performance compared to closed-source models such as GPT4-Turbo, Claude 3 Opus, and Gemini 1.5 Pro in coding and math benchmarks.",
  "date": "17 Jun 2024",
  "dateline": "[Submitted on 17 Jun 2024]",
  "comments": "",
  "category": "代码模型",
  "pdf_url": "https://arxiv.org/pdf/2406.11931",
  "abs_url": "https://arxiv.org/abs/2406.11931"
 },
 {
  "id": "2407.01906",
  "slug": "let-the-expert-stick-to-his-last-expert-specialized-fine-tuning-for-sp-240701906",
  "title": "Let the Expert Stick to His Last: Expert-Specialized Fine-Tuning for Sparse Architectural Large Language Models",
  "authors": "Zihan Wang , Deli Chen , Damai Dai , Runxin Xu , Zhuoshu Li , Y. Wu",
  "abstract": "Parameter-efficient fine-tuning (PEFT) is crucial for customizing Large Language Models (LLMs) with constrained resources. Although there have been various PEFT methods for dense-architecture LLMs, PEFT for sparse-architecture LLMs is still underexplored. In this work, we study the PEFT method for LLMs with the Mixture-of-Experts (MoE) architecture and the contents of this work are mainly threefold: (1) We investigate the dispersion degree of the activated experts in customized tasks, and found that the routing distribution for a specific task tends to be highly concentrated, while the distribution of activated experts varies significantly across different tasks. (2) We propose Expert-Specialized Fine-Tuning, or ESFT, which tunes the experts most relevant to downstream tasks while freezing the other experts and modules; experimental results demonstrate that our method not only improves the tuning efficiency, but also matches or even surpasses the performance of full-parameter fine-tuning. (3) We further analyze the impact of the MoE architecture on expert-specialized fine-tuning. We find that MoE models with finer-grained experts are more advantageous in selecting the combination of experts that are most relevant to downstream tasks, thereby enhancing both the training efficiency and effectiveness. Our code is available at this https URL .",
  "date": "2 Jul 2024",
  "dateline": "[Submitted on 2 Jul 2024 ( v1 ), last revised 5 Jul 2024 (this version, v2)]",
  "comments": "",
  "category": "训练方法",
  "pdf_url": "https://arxiv.org/pdf/2407.01906",
  "abs_url": "https://arxiv.org/abs/2407.01906"
 },
 {
  "id": "2408.08152",
  "slug": "deepseek-prover-v1-5-harnessing-proof-assistant-feedback-for-reinforce-240808152",
  "title": "DeepSeek-Prover-V1.5: Harnessing Proof Assistant Feedback for Reinforcement Learning and Monte-Carlo Tree Search",
  "authors": "Huajian Xin , Z.Z. Ren , Junxiao Song , Zhihong Shao , Wanjia Zhao , Haocheng Wang , Bo Liu , Liyue Zhang , Xuan Lu , Qiushi Du , Wenjun Gao , Qihao Zhu , Dejian Yang , Zhibin Gou , Z.F. Wu , Fuli Luo , Chong Ruan",
  "abstract": "We introduce DeepSeek-Prover-V1.5, an open-source language model designed for theorem proving in Lean 4, which enhances DeepSeek-Prover-V1 by optimizing both training and inference processes. Pre-trained on DeepSeekMath-Base with specialization in formal mathematical languages, the model undergoes supervised fine-tuning using an enhanced formal theorem proving dataset derived from DeepSeek-Prover-V1. Further refinement is achieved through reinforcement learning from proof assistant feedback (RLPAF). Beyond the single-pass whole-proof generation approach of DeepSeek-Prover-V1, we propose RMaxTS, a variant of Monte-Carlo tree search that employs an intrinsic-reward-driven exploration strategy to generate diverse proof paths. DeepSeek-Prover-V1.5 demonstrates significant improvements over DeepSeek-Prover-V1, achieving new state-of-the-art results on the test set of the high school level miniF2F benchmark ($63.5\\%$) and the undergraduate level ProofNet benchmark ($25.3\\%$).",
  "date": "15 Aug 2024",
  "dateline": "[Submitted on 15 Aug 2024]",
  "comments": "",
  "category": "形式化证明",
  "pdf_url": "https://arxiv.org/pdf/2408.08152",
  "abs_url": "https://arxiv.org/abs/2408.08152"
 },
 {
  "id": "2408.14158",
  "slug": "fire-flyer-ai-hpc-a-cost-effective-software-hardware-co-design-for-dee-240814158",
  "title": "Fire-Flyer AI-HPC: A Cost-Effective Software-Hardware Co-Design for Deep Learning",
  "authors": "Wei An , Xiao Bi , Guanting Chen , Shanhuang Chen , Chengqi Deng , Honghui Ding , Kai Dong , Qiushi Du , Wenjun Gao , Kang Guan , Jianzhong Guo , Yongqiang Guo , Zhe Fu , Ying He , Panpan Huang , Jiashi Li , Wenfeng Liang , Xiaodong Liu , Xin Liu , Yiyuan Liu , Yuxuan Liu , Shanghao Lu , Xuan Lu , Xiaotao Nie , Tian Pei , Junjie Qiu , Hui Qu , Zehui Ren , Zhangli Sha , Xuecheng Su , Xiaowen Sun , Yixuan Tan , Minghui Tang , Shiyu Wang , Yaohui Wang , Yongji Wang , Ziwei Xie , Yiliang Xiong , Yanhong Xu , Shengfeng Ye , Shuiping Yu , Yukun Zha , Liyue Zhang , Haowei Zhang , Mingchuan Zhang , Wentao Zhang , Yichao Zhang , Chenggang Zhao , Yao Zhao , Shangyan Zhou , Shunfeng Zhou , Yuheng Zou",
  "abstract": "The rapid progress in Deep Learning (DL) and Large Language Models (LLMs) has exponentially increased demands of computational power and bandwidth. This, combined with the high costs of faster computing chips and interconnects, has significantly inflated High Performance Computing (HPC) construction costs. To address these challenges, we introduce the Fire-Flyer AI-HPC architecture, a synergistic hardware-software co-design framework and its best practices. For DL training, we deployed the Fire-Flyer 2 with 10,000 PCIe A100 GPUs, achieved performance approximating the DGX-A100 while reducing costs by half and energy consumption by 40%. We specifically engineered HFReduce to accelerate allreduce communication and implemented numerous measures to keep our Computation-Storage Integrated Network congestion-free. Through our software stack, including HaiScale, 3FS, and HAI-Platform, we achieved substantial scalability by overlapping computation and communication. Our system-oriented experience from DL training provides valuable insights to drive future advancements in AI-HPC.",
  "date": "26 Aug 2024",
  "dateline": "[Submitted on 26 Aug 2024 ( v1 ), last revised 31 Aug 2024 (this version, v2)]",
  "comments": "This is the preprint version of the paper accepted for presentation at the 2024 International Conference for High Performance Computing, Networking, Storage, and Analysis (SC'24). \\c{opyright} 2024 IEEE. Personal use of this material is permitted. For other uses, permission from IEEE must be obtained. Please refer to IEEE Xplore for the final published version",
  "category": "基础设施",
  "pdf_url": "https://arxiv.org/pdf/2408.14158",
  "abs_url": "https://arxiv.org/abs/2408.14158"
 },
 {
  "id": "2410.13848",
  "slug": "janus-decoupling-visual-encoding-for-unified-multimodal-understanding--241013848",
  "title": "Janus: Decoupling Visual Encoding for Unified Multimodal Understanding and Generation",
  "authors": "Chengyue Wu , Xiaokang Chen , Zhiyu Wu , Yiyang Ma , Xingchao Liu , Zizheng Pan , Wen Liu , Zhenda Xie , Xingkai Yu , Chong Ruan , Ping Luo",
  "abstract": "In this paper, we introduce Janus, an autoregressive framework that unifies multimodal understanding and generation. Prior research often relies on a single visual encoder for both tasks, such as Chameleon. However, due to the differing levels of information granularity required by multimodal understanding and generation, this approach can lead to suboptimal performance, particularly in multimodal understanding. To address this issue, we decouple visual encoding into separate pathways, while still leveraging a single, unified transformer architecture for processing. The decoupling not only alleviates the conflict between the visual encoder's roles in understanding and generation, but also enhances the framework's flexibility. For instance, both the multimodal understanding and generation components can independently select their most suitable encoding methods. Experiments show that Janus surpasses previous unified model and matches or exceeds the performance of task-specific models. The simplicity, high flexibility, and effectiveness of Janus make it a strong candidate for next-generation unified multimodal models.",
  "date": "17 Oct 2024",
  "dateline": "[Submitted on 17 Oct 2024]",
  "comments": "Technical Report",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2410.13848",
  "abs_url": "https://arxiv.org/abs/2410.13848"
 },
 {
  "id": "2412.10302",
  "slug": "deepseek-vl2-mixture-of-experts-vision-language-models-for-advanced-mu-241210302",
  "title": "DeepSeek-VL2: Mixture-of-Experts Vision-Language Models for Advanced Multimodal Understanding",
  "authors": "Zhiyu Wu , Xiaokang Chen , Zizheng Pan , Xingchao Liu , Wen Liu , Damai Dai , Huazuo Gao , Yiyang Ma , Chengyue Wu , Bingxuan Wang , Zhenda Xie , Yu Wu , Kai Hu , Jiawei Wang , Yaofeng Sun , Yukun Li , Yishi Piao , Kang Guan , Aixin Liu , Xin Xie , Yuxiang You , Kai Dong , Xingkai Yu , Haowei Zhang , Liang Zhao , Yisong Wang , Chong Ruan",
  "abstract": "We present DeepSeek-VL2, an advanced series of large Mixture-of-Experts (MoE) Vision-Language Models that significantly improves upon its predecessor, DeepSeek-VL, through two key major upgrades. For the vision component, we incorporate a dynamic tiling vision encoding strategy designed for processing high-resolution images with different aspect ratios. For the language component, we leverage DeepSeekMoE models with the Multi-head Latent Attention mechanism, which compresses Key-Value cache into latent vectors, to enable efficient inference and high throughput. Trained on an improved vision-language dataset, DeepSeek-VL2 demonstrates superior capabilities across various tasks, including but not limited to visual question answering, optical character recognition, document/table/chart understanding, and visual grounding. Our model series is composed of three variants: DeepSeek-VL2-Tiny, DeepSeek-VL2-Small and DeepSeek-VL2, with 1.0B, 2.8B and 4.5B activated parameters respectively. DeepSeek-VL2 achieves competitive or state-of-the-art performance with similar or fewer activated parameters compared to existing open-source dense and MoE-based models. Codes and pre-trained models are publicly accessible at this https URL .",
  "date": "13 Dec 2024",
  "dateline": "[Submitted on 13 Dec 2024]",
  "comments": "",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2412.10302",
  "abs_url": "https://arxiv.org/abs/2412.10302"
 },
 {
  "id": "2412.19437",
  "slug": "deepseek-v3-technical-report-241219437",
  "title": "DeepSeek-V3 Technical Report",
  "authors": "DeepSeek-AI , Aixin Liu , Bei Feng , Bing Xue , Bingxuan Wang , Bochao Wu , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chenyu Zhang , Chong Ruan , Damai Dai , Daya Guo , Dejian Yang , Deli Chen , Dongjie Ji , Erhang Li , Fangyun Lin , Fucong Dai , Fuli Luo , Guangbo Hao , Guanting Chen , Guowei Li , H. Zhang , Han Bao , Hanwei Xu , Haocheng Wang , Haowei Zhang , Honghui Ding , Huajian Xin , Huazuo Gao , Hui Li , Hui Qu , J.L. Cai , Jian Liang , Jianzhong Guo , Jiaqi Ni , Jiashi Li , Jiawei Wang , Jin Chen , Jingchang Chen , Jingyang Yuan , Junjie Qiu , Junlong Li , Junxiao Song , Kai Dong , Kai Hu , Kaige Gao , Kang Guan , Kexin Huang , Kuai Yu , Lean Wang , Lecong Zhang , Lei Xu , Leyi Xia , Liang Zhao , Litong Wang , Liyue Zhang , Meng Li , Miaojun Wang , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Mingming Li , Ning Tian , Panpan Huang , Peiyi Wang , Peng Zhang , Qiancheng Wang , Qihao Zhu , Qinyu Chen , Qiushi Du , R.J. Chen , R.L. Jin , Ruiqi Ge , Ruisong Zhang , Ruizhe Pan , Runji Wang , Runxin Xu , Ruoyu Zhang , Ruyi Chen , S.S. Li , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shaoqing Wu , Shengfeng Ye , Shengfeng Ye , Shirong Ma , Shiyu Wang , Shuang Zhou , Shuiping Yu , Shunfeng Zhou , Shuting Pan , T. Wang , Tao Yun , Tian Pei , Tianyu Sun , W.L. Xiao , Wangding Zeng , Wanjia Zhao , Wei An , Wen Liu , Wenfeng Liang , Wenjun Gao , Wenqin Yu , Wentao Zhang , X.Q. Li , Xiangyue Jin , Xianzu Wang , Xiao Bi , Xiaodong Liu , Xiaohan Wang , Xiaojin Shen , Xiaokang Chen , Xiaokang Zhang , Xiaosha Chen , Xiaotao Nie , Xiaowen Sun , Xiaoxiang Wang , Xin Cheng , Xin Liu , Xin Xie , Xingchao Liu , Xingkai Yu , Xinnan Song , Xinxia Shan , Xinyi Zhou , Xinyu Yang , Xinyuan Li , Xuecheng Su , Xuheng Lin , Y.K. Li , Y.Q. Wang , Y.X. Wei , Y.X. Zhu , Yang Zhang , Yanhong Xu , Yanhong Xu , Yanping Huang , Yao Li , Yao Zhao , Yaofeng Sun , Yaohui Li , Yaohui Wang , Yi Yu , Yi Zheng , Yichao Zhang , Yifan Shi , Yiliang Xiong , Ying He , Ying Tang , Yishi Piao , Yisong Wang , Yixuan Tan , Yiyang Ma , Yiyuan Liu , Yongqiang Guo , Yu Wu , Yuan Ou , Yuchen Zhu , Yuduan Wang , Yue Gong , Yuheng Zou , Yujia He , Yukun Zha , Yunfan Xiong , Yunxian Ma , Yuting Yan , Yuxiang Luo , Yuxiang You , Yuxuan Liu , Yuyang Zhou , Z.F. Wu , Z.Z. Ren , Zehui Ren , Zhangli Sha , Zhe Fu , Zhean Xu , Zhen Huang , Zhen Zhang , Zhenda Xie , Zhengyan Zhang , Zhewen Hao , Zhibin Gou , Zhicheng Ma , Zhigang Yan , Zhihong Shao , Zhipeng Xu , Zhiyu Wu , Zhongyu Zhang , Zhuoshu Li , Zihui Gu , Zijia Zhu , Zijun Liu , Zilin Li , Ziwei Xie , Ziyang Song , Ziyi Gao , Zizheng Pan",
  "abstract": "We present DeepSeek-V3, a strong Mixture-of-Experts (MoE) language model with 671B total parameters with 37B activated for each token. To achieve efficient inference and cost-effective training, DeepSeek-V3 adopts Multi-head Latent Attention (MLA) and DeepSeekMoE architectures, which were thoroughly validated in DeepSeek-V2. Furthermore, DeepSeek-V3 pioneers an auxiliary-loss-free strategy for load balancing and sets a multi-token prediction training objective for stronger performance. We pre-train DeepSeek-V3 on 14.8 trillion diverse and high-quality tokens, followed by Supervised Fine-Tuning and Reinforcement Learning stages to fully harness its capabilities. Comprehensive evaluations reveal that DeepSeek-V3 outperforms other open-source models and achieves performance comparable to leading closed-source models. Despite its excellent performance, DeepSeek-V3 requires only 2.788M H800 GPU hours for its full training. In addition, its training process is remarkably stable. Throughout the entire training process, we did not experience any irrecoverable loss spikes or perform any rollbacks. The model checkpoints are available at this https URL .",
  "date": "27 Dec 2024",
  "dateline": "[Submitted on 27 Dec 2024 ( v1 ), last revised 18 Feb 2025 (this version, v2)]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2412.19437",
  "abs_url": "https://arxiv.org/abs/2412.19437"
 },
 {
  "id": "2501.12948",
  "slug": "deepseek-r1-incentivizing-reasoning-capability-in-llms-via-reinforceme-250112948",
  "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
  "authors": "DeepSeek-AI , Daya Guo , Dejian Yang , Haowei Zhang , Junxiao Song , Peiyi Wang , Qihao Zhu , Runxin Xu , Ruoyu Zhang , Shirong Ma , Xiao Bi , Xiaokang Zhang , Xingkai Yu , Yu Wu , Z.F. Wu , Zhibin Gou , Zhihong Shao , Zhuoshu Li , Ziyi Gao , Aixin Liu , Bing Xue , Bingxuan Wang , Bochao Wu , Bei Feng , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chenyu Zhang , Chong Ruan , Damai Dai , Deli Chen , Dongjie Ji , Erhang Li , Fangyun Lin , Fucong Dai , Fuli Luo , Guangbo Hao , Guanting Chen , Guowei Li , H. Zhang , Han Bao , Hanwei Xu , Haocheng Wang , Honghui Ding , Huajian Xin , Huazuo Gao , Hui Qu , Hui Li , Jianzhong Guo , Jiashi Li , Jiawei Wang , Jingchang Chen , Jingyang Yuan , Junjie Qiu , Junlong Li , J.L. Cai , Jiaqi Ni , Jian Liang , Jin Chen , Kai Dong , Kai Hu , Kaige Gao , Kang Guan , Kexin Huang , Kuai Yu , Lean Wang , Lecong Zhang , Liang Zhao , Litong Wang , Liyue Zhang , Lei Xu , Leyi Xia , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Meng Li , Miaojun Wang , Mingming Li , Ning Tian , Panpan Huang , Peng Zhang , Qiancheng Wang , Qinyu Chen , Qiushi Du , Ruiqi Ge , Ruisong Zhang , Ruizhe Pan , Runji Wang , R.J. Chen , R.L. Jin , Ruyi Chen , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shengfeng Ye , Shiyu Wang , Shuiping Yu , Shunfeng Zhou , Shuting Pan , S.S. Li , Shuang Zhou , Shaoqing Wu , Shengfeng Ye , Tao Yun , Tian Pei , Tianyu Sun , T. Wang , Wangding Zeng , Wanjia Zhao , Wen Liu , Wenfeng Liang , Wenjun Gao , Wenqin Yu , Wentao Zhang , W.L. Xiao , Wei An , Xiaodong Liu , Xiaohan Wang , Xiaokang Chen , Xiaotao Nie , Xin Cheng , Xin Liu , Xin Xie , Xingchao Liu , Xinyu Yang , Xinyuan Li , Xuecheng Su , Xuheng Lin , X.Q. Li , Xiangyue Jin , Xiaojin Shen , Xiaosha Chen , Xiaowen Sun , Xiaoxiang Wang , Xinnan Song , Xinyi Zhou , Xianzu Wang , Xinxia Shan , Y.K. Li , Y.Q. Wang , Y.X. Wei , Yang Zhang , Yanhong Xu , Yao Li , Yao Zhao , Yaofeng Sun , Yaohui Wang , Yi Yu , Yichao Zhang , Yifan Shi , Yiliang Xiong , Ying He , Yishi Piao , Yisong Wang , Yixuan Tan , Yiyang Ma , Yiyuan Liu , Yongqiang Guo , Yuan Ou , Yuduan Wang , Yue Gong , Yuheng Zou , Yujia He , Yunfan Xiong , Yuxiang Luo , Yuxiang You , Yuxuan Liu , Yuyang Zhou , Y.X. Zhu , Yanhong Xu , Yanping Huang , Yaohui Li , Yi Zheng , Yuchen Zhu , Yunxian Ma , Ying Tang , Yukun Zha , Yuting Yan , Z.Z. Ren , Zehui Ren , Zhangli Sha , Zhe Fu , Zhean Xu , Zhenda Xie , Zhengyan Zhang , Zhewen Hao , Zhicheng Ma , Zhigang Yan , Zhiyu Wu , Zihui Gu , Zijia Zhu , Zijun Liu , Zilin Li , Ziwei Xie , Ziyang Song , Zizheng Pan , Zhen Huang , Zhipeng Xu , Zhongyu Zhang , Zhen Zhang",
  "abstract": "General reasoning represents a long-standing and formidable challenge in artificial intelligence. Recent breakthroughs, exemplified by large language models (LLMs) and chain-of-thought prompting, have achieved considerable success on foundational reasoning tasks. However, this success is heavily contingent upon extensive human-annotated demonstrations, and models' capabilities are still insufficient for more complex problems. Here we show that the reasoning abilities of LLMs can be incentivized through pure reinforcement learning (RL), obviating the need for human-labeled reasoning trajectories. The proposed RL framework facilitates the emergent development of advanced reasoning patterns, such as self-reflection, verification, and dynamic strategy adaptation. Consequently, the trained model achieves superior performance on verifiable tasks such as mathematics, coding competitions, and STEM fields, surpassing its counterparts trained via conventional supervised learning on human demonstrations. Moreover, the emergent reasoning patterns exhibited by these large-scale models can be systematically harnessed to guide and enhance the reasoning capabilities of smaller models.",
  "date": "22 Jan 2025",
  "dateline": "[Submitted on 22 Jan 2025 ( v1 ), last revised 4 Jan 2026 (this version, v2)]",
  "comments": "",
  "category": "推理模型",
  "pdf_url": "https://arxiv.org/pdf/2501.12948",
  "abs_url": "https://arxiv.org/abs/2501.12948"
 },
 {
  "id": "2501.17811",
  "slug": "janus-pro-unified-multimodal-understanding-and-generation-with-data-an-250117811",
  "title": "Janus-Pro: Unified Multimodal Understanding and Generation with Data and Model Scaling",
  "authors": "Xiaokang Chen , Zhiyu Wu , Xingchao Liu , Zizheng Pan , Wen Liu , Zhenda Xie , Xingkai Yu , Chong Ruan",
  "abstract": "In this work, we introduce Janus-Pro, an advanced version of the previous work Janus. Specifically, Janus-Pro incorporates (1) an optimized training strategy, (2) expanded training data, and (3) scaling to larger model size. With these improvements, Janus-Pro achieves significant advancements in both multimodal understanding and text-to-image instruction-following capabilities, while also enhancing the stability of text-to-image generation. We hope this work will inspire further exploration in the field. Code and models are publicly available.",
  "date": "29 Jan 2025",
  "dateline": "[Submitted on 29 Jan 2025]",
  "comments": "Research paper. arXiv admin note: text overlap with arXiv:2410.13848",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2501.17811",
  "abs_url": "https://arxiv.org/abs/2501.17811"
 },
 {
  "id": "2504.02495",
  "slug": "inference-time-scaling-for-generalist-reward-modeling-250402495",
  "title": "Inference-Time Scaling for Generalist Reward Modeling",
  "authors": "Zijun Liu , Peiyi Wang , Runxin Xu , Shirong Ma , Chong Ruan , Peng Li , Yang Liu , Yu Wu",
  "abstract": "Reinforcement learning (RL) has been widely adopted in post-training for large language models (LLMs) at scale. Recently, the incentivization of reasoning capabilities in LLMs from RL indicates that $\\textit{proper learning methods could enable effective inference-time scalability}$. A key challenge of RL is to obtain accurate reward signals for LLMs in various domains beyond verifiable questions or artificial rules. In this work, we investigate how to improve reward modeling (RM) with more inference compute for general queries, i.e. the $\\textbf{inference-time scalability of generalist RM}$, and further, how to improve the effectiveness of performance-compute scaling with proper learning methods. For the RM approach, we adopt pointwise generative reward modeling (GRM) to enable flexibility for different input types and potential for inference-time scaling. For the learning method, we propose Self-Principled Critique Tuning (SPCT) to foster scalable reward generation behaviors in GRMs through online RL, to generate principles adaptively and critiques accurately, resulting in $\\textbf{DeepSeek-GRM}$ models. Furthermore, for effective inference-time scaling, we use parallel sampling to expand compute usage, and introduce a meta RM to guide voting process for better scaling performance. Empirically, we show that SPCT significantly improves the quality and scalability of GRMs, outperforming existing methods and models in various RM benchmarks without severe biases, and could achieve better performance compared to training-time scaling. DeepSeek-GRM still meets challenges in some tasks, which we believe can be addressed by future efforts in generalist reward systems. The models are released at Hugging Face and ModelScope.",
  "date": "3 Apr 2025",
  "dateline": "[Submitted on 3 Apr 2025 ( v1 ), last revised 25 Sep 2025 (this version, v3)]",
  "comments": "Preprint, under review. 44 pages. Models are available at this https URL and this https URL",
  "category": "训练方法",
  "pdf_url": "https://arxiv.org/pdf/2504.02495",
  "abs_url": "https://arxiv.org/abs/2504.02495"
 },
 {
  "id": "2504.21801",
  "slug": "deepseek-prover-v2-advancing-formal-mathematical-reasoning-via-reinfor-250421801",
  "title": "DeepSeek-Prover-V2: Advancing Formal Mathematical Reasoning via Reinforcement Learning for Subgoal Decomposition",
  "authors": "Z.Z. Ren , Zhihong Shao , Junxiao Song , Huajian Xin , Haocheng Wang , Wanjia Zhao , Liyue Zhang , Zhe Fu , Qihao Zhu , Dejian Yang , Z.F. Wu , Zhibin Gou , Shirong Ma , Hongxuan Tang , Yuxuan Liu , Wenjun Gao , Daya Guo , Chong Ruan",
  "abstract": "We introduce DeepSeek-Prover-V2, an open-source large language model designed for formal theorem proving in Lean 4, with initialization data collected through a recursive theorem proving pipeline powered by DeepSeek-V3. The cold-start training procedure begins by prompting DeepSeek-V3 to decompose complex problems into a series of subgoals. The proofs of resolved subgoals are synthesized into a chain-of-thought process, combined with DeepSeek-V3's step-by-step reasoning, to create an initial cold start for reinforcement learning. This process enables us to integrate both informal and formal mathematical reasoning into a unified model. The resulting model, DeepSeek-Prover-V2-671B, achieves state-of-the-art performance in neural theorem proving, reaching 88.9% pass ratio on the MiniF2F-test and solving 49 out of 658 problems from PutnamBench. In addition to standard benchmarks, we introduce ProverBench, a collection of 325 formalized problems, to enrich our evaluation, including 15 selected problems from the recent AIME competitions (years 24-25). Further evaluation on these 15 AIME problems shows that the model successfully solves 6 of them. In comparison, DeepSeek-V3 solves 8 of these problems using majority voting, highlighting that the gap between formal and informal mathematical reasoning in large language models is substantially narrowing.",
  "date": "30 Apr 2025",
  "dateline": "[Submitted on 30 Apr 2025 ( v1 ), last revised 18 Jul 2025 (this version, v2)]",
  "comments": "",
  "category": "形式化证明",
  "pdf_url": "https://arxiv.org/pdf/2504.21801",
  "abs_url": "https://arxiv.org/abs/2504.21801"
 },
 {
  "id": "2505.09343",
  "slug": "insights-into-deepseek-v3-scaling-challenges-and-reflections-on-hardwa-250509343",
  "title": "Insights into DeepSeek-V3: Scaling Challenges and Reflections on Hardware for AI Architectures",
  "authors": "Chenggang Zhao , Chengqi Deng , Chong Ruan , Damai Dai , Huazuo Gao , Jiashi Li , Liyue Zhang , Panpan Huang , Shangyan Zhou , Shirong Ma , Wenfeng Liang , Ying He , Yuqing Wang , Yuxuan Liu , Y.X. Wei",
  "abstract": "The rapid scaling of large language models (LLMs) has unveiled critical limitations in current hardware architectures, including constraints in memory capacity, computational efficiency, and interconnection bandwidth. DeepSeek-V3, trained on 2,048 NVIDIA H800 GPUs, demonstrates how hardware-aware model co-design can effectively address these challenges, enabling cost-efficient training and inference at scale. This paper presents an in-depth analysis of the DeepSeek-V3/R1 model architecture and its AI infrastructure, highlighting key innovations such as Multi-head Latent Attention (MLA) for enhanced memory efficiency, Mixture of Experts (MoE) architectures for optimized computation-communication trade-offs, FP8 mixed-precision training to unlock the full potential of hardware capabilities, and a Multi-Plane Network Topology to minimize cluster-level network overhead. Building on the hardware bottlenecks encountered during DeepSeek-V3's development, we engage in a broader discussion with academic and industry peers on potential future hardware directions, including precise low-precision computation units, scale-up and scale-out convergence, and innovations in low-latency communication fabrics. These insights underscore the critical role of hardware and model co-design in meeting the escalating demands of AI workloads, offering a practical blueprint for innovation in next-generation AI systems.",
  "date": "14 May 2025",
  "dateline": "[Submitted on 14 May 2025 ( v1 ), last revised 23 Dec 2025 (this version, v2)]",
  "comments": "This is the author's version of the work. It is posted here for your personal use. Not for redistribution. The definitive version appeared as part of the Industry Track in Proceedings of the 52nd Annual International Symposium on Computer Architecture (ISCA '25)",
  "category": "基础设施",
  "pdf_url": "https://arxiv.org/pdf/2505.09343",
  "abs_url": "https://arxiv.org/abs/2505.09343"
 },
 {
  "id": "2510.18234",
  "slug": "deepseek-ocr-contexts-optical-compression-251018234",
  "title": "DeepSeek-OCR: Contexts Optical Compression",
  "authors": "Haoran Wei , Yaofeng Sun , Yukun Li",
  "abstract": "We present DeepSeek-OCR as an initial investigation into the feasibility of compressing long contexts via optical 2D mapping. DeepSeek-OCR consists of two components: DeepEncoder and DeepSeek3B-MoE-A570M as the decoder. Specifically, DeepEncoder serves as the core engine, designed to maintain low activations under high-resolution input while achieving high compression ratios to ensure an optimal and manageable number of vision tokens. Experiments show that when the number of text tokens is within 10 times that of vision tokens (i.e., a compression ratio < 10x), the model can achieve decoding (OCR) precision of 97%. Even at a compression ratio of 20x, the OCR accuracy still remains at about 60%. This shows considerable promise for research areas such as historical long-context compression and memory forgetting mechanisms in LLMs. Beyond this, DeepSeek-OCR also demonstrates high practical value. On OmniDocBench, it surpasses GOT-OCR2.0 (256 tokens/page) using only 100 vision tokens, and outperforms MinerU2.0 (6000+ tokens per page on average) while utilizing fewer than 800 vision tokens. In production, DeepSeek-OCR can generate training data for LLMs/VLMs at a scale of 200k+ pages per day (a single A100-40G). Codes and model weights are publicly accessible at this http URL .",
  "date": "21 Oct 2025",
  "dateline": "[Submitted on 21 Oct 2025]",
  "comments": "",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2510.18234",
  "abs_url": "https://arxiv.org/abs/2510.18234"
 },
 {
  "id": "2511.22570",
  "slug": "deepseekmath-v2-towards-self-verifiable-mathematical-reasoning-251122570",
  "title": "DeepSeekMath-V2: Towards Self-Verifiable Mathematical Reasoning",
  "authors": "Zhihong Shao , Yuxiang Luo , Chengda Lu , Z.Z. Ren , Jiewen Hu , Tian Ye , Zhibin Gou , Shirong Ma , Xiaokang Zhang",
  "abstract": "Large language models have made significant progress in mathematical reasoning, which serves as an important testbed for AI and could impact scientific research if further advanced. By scaling reasoning with reinforcement learning that rewards correct final answers, LLMs have improved from poor performance to saturating quantitative reasoning competitions like AIME and HMMT in one year. However, this approach faces fundamental limitations. Pursuing higher final answer accuracy doesn't address a key issue: correct answers don't guarantee correct reasoning. Moreover, many mathematical tasks like theorem proving require rigorous step-by-step derivation rather than numerical answers, making final answer rewards inapplicable. To push the limits of deep reasoning, we believe it is necessary to verify the comprehensiveness and rigor of mathematical reasoning. Self-verification is particularly important for scaling test-time compute, especially for open problems without known solutions. Towards self-verifiable mathematical reasoning, we investigate how to train an accurate and faithful LLM-based verifier for theorem proving. We then train a proof generator using the verifier as the reward model, and incentivize the generator to identify and resolve as many issues as possible in their own proofs before finalizing them. To maintain the generation-verification gap as the generator becomes stronger, we propose to scale verification compute to automatically label new hard-to-verify proofs, creating training data to further improve the verifier. Our resulting model, DeepSeekMath-V2, demonstrates strong theorem-proving capabilities, achieving gold-level scores on IMO 2025 and CMO 2024 and a near-perfect 118/120 on Putnam 2024 with scaled test-time compute.",
  "date": "27 Nov 2025",
  "dateline": "[Submitted on 27 Nov 2025]",
  "comments": "",
  "category": "数学模型",
  "pdf_url": "https://arxiv.org/pdf/2511.22570",
  "abs_url": "https://arxiv.org/abs/2511.22570"
 },
 {
  "id": "2512.02556",
  "slug": "deepseek-v3-2-pushing-the-frontier-of-open-large-language-models-251202556",
  "title": "DeepSeek-V3.2: Pushing the Frontier of Open Large Language Models",
  "authors": "DeepSeek-AI , Aixin Liu , Aoxue Mei , Bangcai Lin , Bing Xue , Bingxuan Wang , Bingzheng Xu , Bochao Wu , Bowei Zhang , Chaofan Lin , Chen Dong , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chenhao Xu , Chong Ruan , Damai Dai , Daya Guo , Dejian Yang , Deli Chen , Erhang Li , Fangqi Zhou , Fangyun Lin , Fucong Dai , Guangbo Hao , Guanting Chen , Guowei Li , H. Zhang , Hanwei Xu , Hao Li , Haofen Liang , Haoran Wei , Haowei Zhang , Haowen Luo , Haozhe Ji , Honghui Ding , Hongxuan Tang , Huanqi Cao , Huazuo Gao , Hui Qu , Hui Zeng , Jialiang Huang , Jiashi Li , Jiaxin Xu , Jiewen Hu , Jingchang Chen , Jingting Xiang , Jingyang Yuan , Jingyuan Cheng , Jinhua Zhu , Jun Ran , Junguang Jiang , Junjie Qiu , Junlong Li , Junxiao Song , Kai Dong , Kaige Gao , Kang Guan , Kexin Huang , Kexing Zhou , Kezhao Huang , Kuai Yu , Lean Wang , Lecong Zhang , Lei Wang , Liang Zhao , Liangsheng Yin , Lihua Guo , Lingxiao Luo , Linwang Ma , Litong Wang , Liyue Zhang , M.S. Di , M.Y Xu , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Mingxu Zhou , Panpan Huang , Peixin Cong , Peiyi Wang , Qiancheng Wang , Qihao Zhu , Qingyang Li , Qinyu Chen , Qiushi Du , Ruiling Xu , Ruiqi Ge , Ruisong Zhang , Ruizhe Pan , Runji Wang , Runqiu Yin , Runxin Xu , Ruomeng Shen , Ruoyu Zhang , S.H. Liu , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shaofei Cai , Shaoyuan Chen , Shengding Hu , Shengyu Liu , Shiqiang Hu , Shirong Ma , Shiyu Wang , Shuiping Yu , Shunfeng Zhou , Shuting Pan , Songyang Zhou , Tao Ni , Tao Yun , Tian Pei , Tian Ye , Tianyuan Yue , Wangding Zeng , Wen Liu , Wenfeng Liang , Wenjie Pang , Wenjing Luo , Wenjun Gao , Wentao Zhang , Xi Gao , Xiangwen Wang , Xiao Bi , Xiaodong Liu , Xiaohan Wang , Xiaokang Chen , Xiaokang Zhang , Xiaotao Nie , Xin Cheng , Xin Liu , Xin Xie , Xingchao Liu , Xingkai Yu , Xingyou Li , Xinyu Yang , Xinyuan Li , Xu Chen , Xuecheng Su , Xuehai Pan , Xuheng Lin , Xuwei Fu , Y.Q. Wang , Yang Zhang , Yanhong Xu , Yanru Ma , Yao Li , Yao Li , Yao Zhao , Yaofeng Sun , Yaohui Wang , Yi Qian , Yi Yu , Yichao Zhang , Yifan Ding , Yifan Shi , Yiliang Xiong , Ying He , Ying Zhou , Yinmin Zhong , Yishi Piao , Yisong Wang , Yixiao Chen , Yixuan Tan , Yixuan Wei , Yiyang Ma , Yiyuan Liu , Yonglun Yang , Yongqiang Guo , Yongtong Wu , Yu Wu , Yuan Cheng , Yuan Ou , Yuanfan Xu , Yuduan Wang , Yue Gong , Yuhan Wu , Yuheng Zou , Yukun Li , Yunfan Xiong , Yuxiang Luo , Yuxiang You , Yuxuan Liu , Yuyang Zhou , Z.F. Wu , Z.Z. Ren , Zehua Zhao , Zehui Ren , Zhangli Sha , Zhe Fu , Zhean Xu , Zhenda Xie , Zhengyan Zhang , Zhewen Hao , Zhibin Gou , Zhicheng Ma , Zhigang Yan , Zhihong Shao , Zhixian Huang , Zhiyu Wu , Zhuoshu Li , Zhuping Zhang , Zian Xu , Zihao Wang , Zihui Gu , Zijia Zhu , Zilin Li , Zipeng Zhang , Ziwei Xie , Ziyi Gao , Zizheng Pan , Zongqing Yao , Bei Feng , Hui Li , J.L. Cai , Jiaqi Ni , Lei Xu , Meng Li , Ning Tian , R.J. Chen , R.L. Jin , S.S. Li , Shuang Zhou , Tianyu Sun , X.Q. Li , Xiangyue Jin , Xiaojin Shen , Xiaosha Chen , Xinnan Song , Xinyi Zhou , Y.X. Zhu , Yanping Huang , Yaohui Li , Yi Zheng , Yuchen Zhu , Yunxian Ma , Zhen Huang , Zhipeng Xu , Zhongyu Zhang , Dongjie Ji , Jian Liang , Jianzhong Guo , Jin Chen , Leyi Xia , Miaojun Wang , Mingming Li , Peng Zhang , Ruyi Chen , Shangmian Sun , Shaoqing Wu , Shengfeng Ye , T.Wang , W.L. Xiao , Wei An , Xianzu Wang , Xiaowen Sun , Xiaoxiang Wang , Ying Tang , Yukun Zha , Zekai Zhang , Zhe Ju , Zhen Zhang , Zihua Qu",
  "abstract": "We introduce DeepSeek-V3.2, a model that harmonizes high computational efficiency with superior reasoning and agent performance. The key technical breakthroughs of DeepSeek-V3.2 are as follows: (1) DeepSeek Sparse Attention (DSA): We introduce DSA, an efficient attention mechanism that substantially reduces computational complexity while preserving model performance in long-context scenarios. (2) Scalable Reinforcement Learning Framework: By implementing a robust reinforcement learning protocol and scaling post-training compute, DeepSeek-V3.2 performs comparably to GPT-5. Notably, our high-compute variant, DeepSeek-V3.2-Speciale, surpasses GPT-5 and exhibits reasoning proficiency on par with Gemini-3.0-Pro, achieving gold-medal performance in both the 2025 International Mathematical Olympiad (IMO) and the International Olympiad in Informatics (IOI). (3) Large-Scale Agentic Task Synthesis Pipeline: To integrate reasoning into tool-use scenarios, we developed a novel synthesis pipeline that systematically generates training data at scale. This methodology facilitates scalable agentic post-training, yielding substantial improvements in generalization and instruction-following robustness within complex, interactive environments.",
  "date": "2 Dec 2025",
  "dateline": "[Submitted on 2 Dec 2025]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2512.02556",
  "abs_url": "https://arxiv.org/abs/2512.02556"
 },
 {
  "id": "2512.24880",
  "slug": "mhc-manifold-constrained-hyper-connections-251224880",
  "title": "mHC: Manifold-Constrained Hyper-Connections",
  "authors": "Zhenda Xie , Yixuan Wei , Huanqi Cao , Chenggang Zhao , Chengqi Deng , Jiashi Li , Damai Dai , Huazuo Gao , Jiang Chang , Kuai Yu , Liang Zhao , Shangyan Zhou , Zhean Xu , Zhengyan Zhang , Wangding Zeng , Shengding Hu , Yuqing Wang , Jingyang Yuan , Lean Wang , Wenfeng Liang",
  "abstract": "Recently, studies exemplified by Hyper-Connections (HC) have extended the ubiquitous residual connection paradigm established over the past decade by expanding the residual stream width and diversifying connectivity patterns. While yielding substantial performance gains, this diversification fundamentally compromises the identity mapping property intrinsic to the residual connection, which causes severe training instability and restricted scalability, and additionally incurs notable memory access overhead. To address these challenges, we propose Manifold-Constrained Hyper-Connections (mHC), a general framework that projects the residual connection space of HC onto a specific manifold to restore the identity mapping property, while incorporating rigorous infrastructure optimization to ensure efficiency. Empirical experiments demonstrate that mHC is effective for training at scale, offering tangible performance improvements and superior scalability. We anticipate that mHC, as a flexible and practical extension of HC, will contribute to a deeper understanding of topological architecture design and suggest promising directions for the evolution of foundational models.",
  "date": "31 Dec 2025",
  "dateline": "[Submitted on 31 Dec 2025 ( v1 ), last revised 5 Jan 2026 (this version, v2)]",
  "comments": "",
  "category": "架构",
  "pdf_url": "https://arxiv.org/pdf/2512.24880",
  "abs_url": "https://arxiv.org/abs/2512.24880"
 },
 {
  "id": "2601.07372",
  "slug": "conditional-memory-via-scalable-lookup-a-new-axis-of-sparsity-for-larg-260107372",
  "title": "Conditional Memory via Scalable Lookup: A New Axis of Sparsity for Large Language Models",
  "authors": "Xin Cheng , Rui Tian , Wangding Zeng , Damai Dai , Qinyu Chen , Bingxuan Wang , Zhenda Xie , Kezhao Huang , Xingkai Yu , Chengqi Deng , Shangyan Zhou , Chenggang Zhao , Zhewen Hao , Yukun Li , Han Zhang , Zhengyan Zhang , Yixu Wei , M.Y Xu , Huishuai Zhang , Dongyan Zhao , Wenfeng Liang",
  "abstract": "While Mixture-of-Experts (MoE) scales capacity via conditional computation, Transformers lack a native primitive for knowledge lookup, forcing them to inefficiently simulate retrieval through computation. To address this, we introduce conditional memory as a complementary sparsity axis, instantiated via Engram, a module that modernizes classic $N$-gram embedding for O(1) lookup. By formulating the Sparsity Allocation problem, we uncover a U-shaped scaling law that optimizes the trade-off between neural computation (MoE) and static memory (Engram). Guided by this law, we scale Engram to 27B parameters, achieving superior performance over a strictly iso-parameter and iso-FLOPs MoE baseline. Most notably, while the memory module is expected to aid knowledge retrieval (e.g., MMLU +3.4; CMMLU +4.0), we observe even larger gains in general reasoning (e.g., BBH +5.0; ARC-Challenge +3.7) and code/math domains~(HumanEval +3.0; MATH +2.4). Mechanistic analyses reveal that Engram relieves the backbone's early layers from static reconstruction, effectively deepening the network for complex reasoning. Furthermore, by delegating local dependencies to lookups, it frees up attention capacity for global context, substantially boosting long-context retrieval (e.g., Multi-Query NIAH: 84.2 to 97.0). Finally, Engram establishes infrastructure-aware efficiency: its deterministic addressing enables runtime prefetching from host memory, incurring negligible overhead. We envision conditional memory as an indispensable modeling primitive for next-generation sparse models.",
  "date": "12 Jan 2026",
  "dateline": "[Submitted on 12 Jan 2026 ( v1 ), last revised 12 Jul 2026 (this version, v2)]",
  "comments": "",
  "category": "架构",
  "pdf_url": "https://arxiv.org/pdf/2601.07372",
  "abs_url": "https://arxiv.org/abs/2601.07372"
 },
 {
  "id": "2601.20552",
  "slug": "deepseek-ocr-2-visual-causal-flow-260120552",
  "title": "DeepSeek-OCR 2: Visual Causal Flow",
  "authors": "Haoran Wei , Yaofeng Sun , Yukun Li",
  "abstract": "We present DeepSeek-OCR 2 to investigate the feasibility of a novel encoder-DeepEncoder V2-capable of dynamically reordering visual tokens upon image semantics. Conventional vision-language models (VLMs) invariably process visual tokens in a rigid raster-scan order (top-left to bottom-right) with fixed positional encoding when fed into LLMs. However, this contradicts human visual perception, which follows flexible yet semantically coherent scanning patterns driven by inherent logical structures. Particularly for images with complex layouts, human vision exhibits causally-informed sequential processing. Inspired by this cognitive mechanism, DeepEncoder V2 is designed to endow the encoder with causal reasoning capabilities, enabling it to intelligently reorder visual tokens prior to LLM-based content interpretation. This work explores a novel paradigm: whether 2D image understanding can be effectively achieved through two-cascaded 1D causal reasoning structures, thereby offering a new architectural approach with the potential to achieve genuine 2D reasoning. Codes and model weights are publicly accessible at this http URL .",
  "date": "28 Jan 2026",
  "dateline": "[Submitted on 28 Jan 2026]",
  "comments": "",
  "category": "多模态",
  "pdf_url": "https://arxiv.org/pdf/2601.20552",
  "abs_url": "https://arxiv.org/abs/2601.20552"
 },
 {
  "id": "2602.21548",
  "slug": "dualpath-breaking-the-storage-bandwidth-bottleneck-in-agentic-llm-infe-260221548",
  "title": "DualPath: Breaking the Storage Bandwidth Bottleneck in Agentic LLM Inference",
  "authors": "Yongtong Wu , Shaoyuan Chen , Yinmin Zhong , Rilin Huang , Yixuan Tan , Wentao Zhang , Liyue Zhang , Shangyan Zhou , Yuxuan Liu , Shunfeng Zhou , Mingxing Zhang , Xin Jin , Panpan Huang",
  "abstract": "The performance of multi-turn, agentic LLM inference is increasingly dominated by KV-Cache storage I/O rather than computation. In prevalent disaggregated architectures, loading the massive KV-Cache from external storage creates a fundamental imbalance: storage NICs on prefill engines become bandwidth-saturated, while those on decoding engines remain idle. This asymmetry severely constrains overall system throughput. We present DualPath, an inference system that breaks this bottleneck by introducing dual-path KV-Cache loading. Beyond the traditional storage-to-prefill path, DualPath enables a novel storage-to-decode path, in which the KV-Cache is loaded into decoding engines and then efficiently transferred to prefill engines via RDMA over the compute network. DualPath combines this optimized data path -- which inherently avoids network congestion and avoids interference with latency-critical model execution communications -- with a global scheduler that dynamically balances load across prefill and decode engines. Our evaluation on three models with production agentic workloads demonstrates that DualPath improves offline inference throughput by up to 1.87$\\times$ on our in-house inference system. It can also improve online serving throughput by an average factor of 1.96$\\times$ without violating SLO.",
  "date": "25 Feb 2026",
  "dateline": "[Submitted on 25 Feb 2026 ( v1 ), last revised 26 Feb 2026 (this version, v2)]",
  "comments": "",
  "category": "基础设施",
  "pdf_url": "https://arxiv.org/pdf/2602.21548",
  "abs_url": "https://arxiv.org/abs/2602.21548"
 },
 {
  "id": "2606.19348",
  "slug": "deepseek-v4-towards-highly-efficient-million-token-context-intelligenc-260619348",
  "title": "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence",
  "authors": "DeepSeek-AI , Anyi Xu , Bangcai Lin , Bing Xue , Bingxuan Wang , Bingzheng Xu , Bochao Wu , Bowei Zhang , Chaofan Lin , Chen Dong , Chenchen Ling , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chengyu Hou , Chenhao Xu , Chenze Shao , Chong Ruan , Conner Sun , Damai Dai , Daya Guo , Dejian Yang , Deli Chen , Donghao Li , Dongjie Ji , Erhang Li , Fang Wei , Fangyun Lin , Fangzhou Yuan , Feiyu Xia , Fucong Dai , Guangbo Hao , Guanting Chen , Guoai Cao , Guolai Meng , Guowei Li , Han Yu , Han Zhang , Hanwei Xu , Hao Li , Haofen Liang , Haoling Zhang , Haoming Luo , Haoran Wei , Haotian Yuan , Haowei Zhang , Haowen Luo , Haoyu Chen , Haozhe Ji , Hengqing Zhang , Honghui Ding , Hongxuan Tang , Huanqi Cao , Huazuo Gao , Hui Qu , Hui Zeng , J Yang , JQ Zhu , Jia Luo , Jia Song , Jia Yu , Jialiang Huang , Jialu Cai , Jian Liang , Jiangting Zhou , Jiasheng Ye , Jiashi Li , Jiaxin Xu , Jiewen Hu , Jieyu Yang , Jin Chen , Jin Yan , Jingchang Chen , Jingli Zhou , Jingting Xiang , Jingyang Yuan , Jingyuan Cheng , Jingzi Zhou , Jinhua Zhu , Jiping Yu , Joseph Sun , Jun Ran , Junguang Jiang , Junjie Qiu , Junlong Li , Junmin Zheng , Junxiao Song , Kai Dong , Kaige Gao , Kang Guan , Kexing Zhou , Kezhao Huang , Kuai Yu , Lean Wang , Lecong Zhang , Lei Wang , Leyi Xia , Li Zhang , Liang Zhao , Lihua Guo , Lingxiao Luo , Linwang Ma , Linyan Zhu , Litong Wang , Liyu Cai , Liyue Zhang , Longhao Chen , MS Di , MY Xu , Max Mei , Miaojun Wang , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Mingming Li , Mingxu Zhou , Minmin Han , Ning Wang , Panpan Huang , Panpan Wang , Peixin Cong , Peiyi Wang , Peng Zhang , Qiancheng Wang , Qihao Zhu , Qingyang Li , Qinyu Chen , Qiushi Du , Qiwei Jiang , Rui Tian , Ruifan Xu , Ruijie Lu , Ruiling Xu , Ruiqi Ge , Ruisong Zhang , Ruizhe Pan , Runji Wang , Runqian Chen , Runqiu Yin , Runxin Xu , Ruomeng Shen , Ruoyu Zhang , Ruyi Chen , SH Liu , Shanghao Lu , Shangmian Sun , Shangyan Zhou , Shanhuang Chen , Shaofei Cai , Shaoheng Nie , Shaoqing Wu , Shaoyuan Chen , Shengding Hu , Shengyu Liu , Shiqiang Hu , Shirong Ma , Shiyu Wang , Shuiping Yu , Shunfeng Zhou , Shuting Pan , Shuying Yu , Songyang Zhou , Tao Ni , Tao Yun , Tian Jin , Tian Pei , Tian Ye , Tianle Lin , Tianran Ji , Tianyi Cui , Tianyuan Yue , Tingting Yu , Tun Wang , W Zhang , WL Xiao , Wangding Zeng , Wei An , Weilin Zhao , Wen Liu , Wenfeng Liang , Wenjie Pang , Wenjing Luo , Wenjing Yao , Wenjun Gao , Wenkai Yang , Wenlve Huang , Wenqing Hou , Wentao Zhang , Wenting Ma , Xi Gao , Xiang He , Xiangwen Wang , Xianzu Wang , Xiao Bi , Xiaodong Liu , Xiaohan Wang , Xiaokang Chen , Xiaokang Zhang , Xiaotao Nie , Xiaowen Sun , Xiaoxiang Wang , Xin Cheng , Xin Liu , Xin Xie , Xingchao Liu , Xingchen Liu , Xingkai Yu , Xingyou Li , Xinyu Yang , Xinyu Zhang , Xu Chen , Xuanyu Wang , Xuecheng Su , Xueyin Chen , Xuheng Lin , Xuwei Fu , YC Yan , YQ Wang , YW Ma , Yanfeng Luo , Yang Zhang , Yanhong Xu , Yanru Ma , Yanwen Huang , Yao Li , Yao Li , Yao Xu , Yao Zhao , Yaofeng Sun , Yaohui Wang , Yi Qian , Yi Shao , Yi Yu , Yichao Zhang , Yifan Ding , Yifan Shi , Yijia Wu , Yiliang Xiong , Yiling Ma , Ying He , Ying Tang , Ying Zhou , Yingjia Luo , Yinmin Zhong , Yishi Piao , Yisong Wang , Yixiang Zhang , Yixiao Chen , Yixuan Tan , Yixuan Wei , Yiyang Ma , Yiyuan Liu , Yonglun Yang , Yongqiang Guo , Yongtong Wu , Yu Wu , YuKun Li , Yuan Cheng , Yuan Ou , Yuanfan Xu , Yuanhao Li , Yuduan Wang , Yuehan Yang , Yuer Xu , Yuhan Wu , Yuhao Meng , Yuheng Zou , Yukun Zha , Yunfan Xiong , Yupeng Chen , Yuping Lin , Yuqian Cao , Yuqian Wang , Yushun Zhang , Yuting Yan , Yutong Lin , Yuxian Gu , Yuxiang Luo , Yuxiang You , Yuxuan Liu , Yuxuan Zhou , Yuyang Zhou , Yuzhen Huang , ZF Wu , Zehao Wang , Zehua Zhao , Zehui Ren , Zekai Zhang , Zhangli Sha , Zhe Fu , Zhe Ju , Zhean Xu , Zhenda Xie , Zhengyan Zhang , Zheren Gao , Zhewen Hao , Zhibin Gou , Zhicheng Ma , Zhigang Yan , Zhihong Shao , Zhixian Huang , Zhixuan Chen , Zhiyu Wu , Zhizhou Ren , Zhongyu Wu , Zhuoshu Li , Zhuping Zhang , Zian Xu , Zihao Wang , Zihua Qu , Zihui Gu , Zijia Zhu , Zilin Li , Zipeng Zhang , Ziwei Xie , Ziyi Gao , Ziyi Wan , Zizheng Pan , Zongqing Yao",
  "abstract": "We present a preview version of DeepSeek-V4 series, including two strong Mixture-of-Experts (MoE) language models -- DeepSeek-V4-Pro with 1.6T parameters (49B activated) and DeepSeek-V4-Flash with 284B parameters (13B activated) -- both supporting a context length of one million tokens. DeepSeek-V4 series incorporate several key upgrades in architecture and optimization: (1) a hybrid attention architecture that combines Compressed Sparse Attention (CSA) and Heavily Compressed Attention (HCA) to improve long-context efficiency; (2) Manifold-Constrained Hyper-Connections (mHC) that enhance conventional residual connections; (3) and the Muon optimizer for faster convergence and greater training stability. We pre-train both models on more than 32T diverse and high-quality tokens, followed by a comprehensive post-training pipeline that unlocks and further enhances their capabilities. DeepSeek-V4-Pro-Max, the maximum reasoning effort mode of DeepSeek-V4-Pro, redefines the state-of-the-art for open models, outperforming its predecessors in core tasks. Meanwhile, DeepSeek-V4 series are highly efficient in long-context scenarios. In the one-million-token context setting, DeepSeek-V4-Pro requires only 27% of single-token inference FLOPs and 10% of KV cache compared with DeepSeek-V3.2. This enables us to routinely support one-million-token contexts, thereby making long-horizon tasks and further test-time scaling more feasible. The model checkpoints are available at this https URL .",
  "date": "26 Apr 2026",
  "dateline": "[Submitted on 26 Apr 2026]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2606.19348",
  "abs_url": "https://arxiv.org/abs/2606.19348"
 },
 {
  "id": "2609.19969",
  "slug": "deepseek-v4-1-flash-pushing-the-limits-of-kv-cache-compression-260919969",
  "title": "DeepSeek-V4.1-Flash: Pushing the Limits of KV Cache Compression",
  "authors": "DeepSeek-AI : Anyi Xu , B. Li , Bangcai Lin , Bing Xue , BingCheng Xian , Bingzheng Xu , Bochao Wu , Bowei Zhang , Boyi Deng , C.C. Yu , Chao Jin , Chaofan Lin , Chen Dong , Chenbing Wang , Chenfan Feng , Chengda Lu , Chenggang Zhao , Chengqi Deng , Chengyuan Zhang , Chenhao Xu , Chenqi Zhao , Chenze Shao , Chuhao Wang , Chuqi Zhang , Damai Dai , Dejian Yang , Deli Chen , Di Huang , Di Wu , Donghao Li , Erhang Li , Eric Fu , F. Zhou , Fangwei Zhou , Fangyun Lin , Fangzhou Yuan , Feiyu Xia , Fucong Dai , Guangbo Hao , Guanglin Li , Guanting Chen , Guoai Cao , Guofan Fan , Guolai Meng , Guowei Li , Haichuan Zhang , Haiyang Ma , Haiyang Shen , Han Li , Han Yu , Han Zhang , Hangyuan Deng , Hanwei Xu , Hanxiang Xu , Hanxun Zhong , Hao Guo , Hao Jiang , Hao Li , Hao Qin , Haodong Wen , Haofen Liang , Haofeng Huang , Haohua Liu , Haoling Zhang , Haoming Luo , Haoran Yang , Haotian Xu , Haotian Yuan , Haoting Huang , Haowen Luo , Haoyang Cai , Haoyu Chen , Haozhe Ji , Hengran Zhang , Hengrui Wang , Hengxu Wu , Honghui Ding , Hongxuan Tang , Huadong Wang , Huanqi Cao , Huazuo Gao , Hui Qu , Hui Zeng , J. Yang , J.H. Jin , J.H. Zhang , J.X. Zou , Jia Yu , Jiahui Zhou , Jiajun Chen , Jialiang Huang , Jialin Zhao , Jiamin Tang , Jian Zhou , Jianan Tong , Jianwen Li , Jiaqi Zhu , Jiarui Wang , Jiasheng Ye , Jiashi Li , Jiaxin Xu , Jiaying Ding , Jibai Lu , Jiewen Hu , Jin Yan , Jincheng Zhai , Jingchang Chen , Jingcheng Hu , Jingli Zhou , Jingsheng Xu , Jingting Xiang , Jingyan Yun , Jingyang Yuan , Jingyuan Cheng , Jinhua Zhu , Jinpeng Wang , Jinyi Chen , Jinyi Hu , Jiping Yu , Jueliang Guo , Junbo Pei , Junbo Sun , Junguang Jiang , Junjie Qiu , Junkang Zhou , Junqi Liu , Junren Li , Junxian Li , Junxiao Song , Junyi Guo , Kai Dong , Kaifeng Chen , Kaige Gao , Kang Guan , Kangdong Yuan , Ke Hong , Ke Xu , Kefan Zhao , Kexin Ji , Kexin Zhang , Kexing Zhou , Kuai Yu , Lan Zhang , Lean Wang , Lecong Zhang , Lei Wang , Letian Gao , Liang Zhao , Liansheng Xu , Lihua Guo , Lingxiao Luo , Lingyue Fu , Litao Deng , Litong Wang , Liyue Zhang , Longhao Chen , Lu Chen , Luotian Huang , Luyao Ma , Luyao Wang , M.S. Di , Max Mei , Menghao Ye , Miao Cui , Mingchuan Zhang , Minghua Zhang , Minghui Tang , Mingjing Zhang , Mingqi Wei , Mingshu Chen , Mingxing Liu , Mingxu Zhou , Mingyu Xu , Mingyu Yang , Mingze Wang , Muyang Chen , Ni Shentu , Ning Wang , Niufang Ning , Panpan Huang , Peixin Cong , Peiyi Wang , Peiyuan Xin , Pengfei Ren , Pengfei Yan , Pengle Zhang , Qi Kang , Qi Tang , Qiancheng Wang , Qiang Li , Qihao Zhu , Qingyang Li , Qinyu Chen , Qiushi Du , Qizhou Guo , Rongxian Xu , Rui Ding , Rui Hu , Rui Tian , Rui Yu , Ruidong Zhu , Ruifan Xu , Ruihan Yang , Ruihang Xia , Ruijie Lu , Ruilin Geng , Ruipeng Hong , Ruiqi Ge , Ruisong Zhang , Ruize Sun , Ruizhe Pan , Runji Wang , Runqian Chen , Runxin Xu , Ruohong Tian , Ruomeng Shen , Ruoyu Zhang , Ryan X. , S.H. Liu , Shanghao Lu , Shangyan Zhou , Shanhuang Chen , Shaofei Cai , Shaoheng Nie , Shaoyuan Chen , Shengding Hu , Shengkai Lin , Shengwen Ran , Shengyu Liu , Shengyuan Jia , Shi Bai , Shi Feng , Shicheng Xu , Shichun Liu , Shiqiang Hu , Shirong Ma , Shiyu Wang , Shiyuan Feng , Shufan Gong , Shuhan Lin , Shuiping Yu , Shunfeng Zhou , Shuo Yang , Shuomeng Wang , Shuting Guo , Shuting Pan , Shuying Yu , Sinuo Cao , Siyi Lin , Sizhe Chen , Songyang Chen , Songyang Zhou , Tao Ni , Tao Yun , Tian Jin , Tian Pei , Tian Ye , Tianle Lin , Tianran Ji , Tianyi Cui , Tianyuan Yue , Tingting Yu , Tongrui Xiong , Wangding Zeng , Wei Liu , Wei Zhang , Weibin Xu , Weihao Zeng , Weilin Zhao , Wen Liu , Wenfeng Liang , Wenjie Pang , Wenjing Luo , Wenjing Yao , Wenjun Gao , Wenkai Shao , Wenkai Yang , Wenli Zhang , Wenlu Wang , Wenlve Huang , Wenqian Yan , Wentao Zhang , Xi Gao , Xiang He , Xiang Li , Xiangli Li , Xiangwen Wang , Xiangying Zhang , Xiankui Wei , Xiao Bi , Xiaodong Liu , Xiaohan Wang , Xiaojian Qu , Xiaokang Chen , Xiaokang Zhang , Xiaotao Nie , Xiaoyao Zou , Xiaoyuan Li , Xicheng Guo , Xieting Chu , Xin Cheng , Xin Liu , Xin Xie , Xinbo Xu , Xingchao Liu , Xingchen Liu , Xingkai Yu , Xingyou Li , Xintong Yao , Xinyang Chen , Xinyong Jiang , Xinyu Yang , Xinyu Yang , Xu Chen , Xuanyu Wang , Xubei Zhong , Xuecheng Su , Xuejie Liu , Xuheng Lin , Xujie Fan , Xuncheng Zhao , Xuwei Fu , Y.C. Yan , Y.H. Jiang , Y.T. Wu , Y.W. M. , Y.Z. Wang , Yafei Gao , Yang Yang , Yang Zhang , Yanru Ma , Yanwen Huang , Yao Li , Yao Li , Yao Meng , Yao Zhao , Yaofeng Sun , Yaohui Wang , Yaoyang Ye , Yehang Yin , Yexinrui Wu , Yi Qian , Yi Tao , Yi Yu , Yichao Zhang , Yichen Jiang , Yicheng Wang , Yifan Ding , Yifan Shi , Yifeng Peng , Yifeng Zhai , Yijia Wu , Yiliang Xiong , Yilun Wang , Ying He , Ying Zhou , Yingjia Luo , Yinmin Zhong , Yiping Wang , Yisong Wang , Yixiang Zhang , Yixiao Chen , Yixuan Tan , Yixuan Wei , Yiyang Ma , Yiyao Yang , Yiyuan Liu , Yizai Cai , Yizhen Wei , Yizhi Wang , Yonglun Yang , Yongqi Zhuo , Yongqiang Guo , Yongtong Wu , Yu Wu , Yu Zhang , Yuan Bian , Yuan Cheng , Yuan Ou , Yuan Sun , Yuanfan Xu , Yuanhang Sun , Yuanhao Li , Yuchen Liu , Yuchen Yao , Yudong Han , Yuduan Wang , Yuhan Wu , Yuhao Meng , Yuheng Zou , YuKun Li , Yunchuan Wang , Yunfan Xiao , Yunfan Xiong , Yupeng Chen , Yuqian Cao , Yuqian Wang , Yuqing Chen , Yushun Zhang , Yutong Lin , Yuwei Xiao , Yuxian Gu , Yuxiang Chen , Yuxiang Huang , Yuxiang Luo , Yuxiang You , Yuxin Chen , Yuxin Xiang , Yuxuan Liu , Yuxuan Zhou , Yuyang Zhou , Yuzhe Guo , Yuzhen Huang , Yuzhuo Bai , Z.Y. Z. , Zanlin Ni , Zehao Wang , Zehua Zhao , Zehui Ren , Zejun Zhao , Zhangli Sha , Zhanying Wang , Zhaochen Zhang , Zhaoshuai Du , Zhe Fu , Zhean Xu , Zhenda Xie , Zheng Liu , Zhengyan Zhang , Zhenhua Dong , Zhewen Hao , Zhibang Wang , Zhibin Gou , Zhicheng Ma , Zhihao Li , Zhihong Shao , Zhihuan Huang , Zhijie Li , Zhirui Lu , Zhixian Huang , Zhixuan Chen , Zhixuan Chen , Zhixuan Pan , Zhiyu Wu , Zhizhou Ren , Zhu He , Zhuoshu Li , Zhuping Zhang , Zian Xu , Zihao Wang , Zihui Gu , Zijia Zhu , Zili Zhang , Zilin Li , Zilong Hou , Zilong Lyu , Ziqiao Wang , Ziwei Xie , Ziya Zhang , Ziyi Gao , Zizheng Pan , Zonglin Li , Zongqing Yao , Zui Chen , Zuofan Wu , Chenchen Ling , Chengyu Hou , Chong Chen , D. Li , Di Qi , Dongjie Ji , Fang Wei , Fanyi Xia , Fei Xie , Feiyi Tan , Hailong Guo , Haiyan Zhai , Hui Zhou , Huihui Tan , Huijie Li , Jia Luo , Jia Song , Jialu Cai , Jian Liang , Jiangting Zhou , Jiaqi Gao , Jiayi Shao , Jie Chen , Jieyu Yang , Jin Chen , Jingde Zhang , Jingzi Zhou , Jinqian Wang , Jinyang Liu , JinZhao Sun , Junhua Ling , Junmin Zheng , Kaicheng Yang , Ke Xu , Le Su , Leyi Xia , Liangfeng Ding , Lin Zhuo , Linwang Ma , Linyan Zhu , Liyu Cai , Luqi Yao , M.K. Zhang , Meng Li , Miao Lin , Miaojun Wang , Min Zhang , Mingming Li , Mingming Wang , Mingze Yin , Minmin Han , Nan Cao , Ning Wang , Ningxin Ma , Panpan Wang , Peihan Lin , Peng Sun , Peng Zhang , Qian Ying , Qiang Xiang , Qiao Wang , Qingmiao Mao , Qiwei Jiang , Rongli Jin , Ruyi Chen , Sha Tao , Shangmian Sun , Shaoqing Wu , Shichao Zou , Si Lei , Tianyang Zhang , Tianyu Sun , Tingting Yin , W.L. Xiao , Wei An , Wei Li , Wei Wang , Weiwei Lin , Wenqing Hou , X. Lin , Xiangfei Meng , Xianzhu Huang , Xiao Peng , Xiaoqian Li , Xiaoting Zhang , Xiaowen Sun , Xiaoxiang Wang , Xiaoyu Ye , Xinrou Zhang , Xinyu Zhang , Xue Cao , Xueyin Chen , Yanan Zhou , Yanhong Xu , Yao Xia , Yao Xu , Yi Shao , Yihong Zhang , Yiling Ma , Ying Tang , Yining Lou , Yiru Chen , Yishi Piao , Yixuan Chen , Yong Xiong , Yuchen Xuan , Yuehan Yang , Yuer Xu , Yukun Zha , Yunxian Ma , Yuping Lin , Yuting Yan , Yutong Xie , Yuwen Sheng , Yuxuan Zhu , Zekai Zhang , Zhe Ju , Zhenzhen Lin , Zheren Gao , Zheyang Sun , Zhigang Yan , Zhongyu Wu , Zi Wang , Zihua Qu , Ziling Yan , Ziyi Wan",
  "abstract": "The widespread adoption of long-horizon agents has made model workloads increasingly input-heavy. Although prior work has substantially reduced the cost of long-context computation, prefill remains computationally expensive, and large KV caches continue to strain HBM and SSD capacity and data-transfer bandwidth. Together, these compute, storage, and bandwidth demands constitute the primary bottleneck to further lowering deployment costs. To address this challenge, we introduce DeepSeek-V4.1-Flash, a multimodal Mixture-of-Experts (MoE) model with 552B backbone parameters and support for contexts of up to one million tokens. With its Causal Encoder-Decoder (CED) architecture, the model activates 16B parameters per token during decode but only 8B parameters during prefill, substantially improving cost efficiency for agentic workloads. To push the limits of KV cache compression, DeepSeek-V4.1-Flash combines cross-layer KV cache reuse in Compressed Sparse Attention 2 (CSA2) with FP4 KV caching. These designs reduce its global KV cache footprint (always in HBM) to 890 bytes per token, roughly 1/4 of the corresponding footprint of DeepSeek-V4-Flash. Further, through a dedicated deployment optimization known as SWA Bounded Replay, DeepSeek-V4.1-Flash reduces its persistent KV cache footprint (always on SSD or in host memory) to roughly 1/8 of that of DeepSeek-V4-Flash. Despite its much smaller KV cache footprint, the model delivers substantially better performance than the baseline. In addition, we streamline the DeepSeek-V4 architecture and introduce several efficient architectural extensions. We pretrain DeepSeek-V4.1-Flash on a multimodal corpus comprising 45T tokens and conduct comprehensive post-training, yielding strong performance across diverse text-based and multimodal agentic scenarios. Model checkpoints are available at this https URL .",
  "date": "17 Sep 2026",
  "dateline": "[Submitted on 17 Sep 2026]",
  "comments": "",
  "category": "基础模型",
  "pdf_url": "https://arxiv.org/pdf/2609.19969",
  "abs_url": "https://arxiv.org/abs/2609.19969"
 }
]