[GH-ISSUE #450] fail to export python pack and test #159

New issue

Closed

opened 2026-10-29 21:10:16 +00:00 by bsakaguchi · 1 comment

bsakaguchi commented

2026-10-29 21:10:16 +00:00

Owner

Originally created by @ActivePeter on GitHub (Jun 5, 2025).
Original GitHub issue: https://github.com/kvcache-ai/Mooncake/issues/450

Reproduce using the following script:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Mooncake Pack CI Script
自动化创建基础容器，通过运行时脚本完成包下载、安装和验证
"""

import os
import subprocess
import sys
import shutil
import argparse
from pathlib import Path

# 设置Python默认编码为UTF-8
os.environ['PYTHONIOENCODING'] = 'utf-8'
if hasattr(sys, 'set_int_max_str_digits'):
    sys.set_int_max_str_digits(0)

class MooncakePackCI:
    def __init__(self):
        # 解析命令行参数
        parser = argparse.ArgumentParser(description="Mooncake Pack CI Script")
        parser.add_argument("--python-version", default="3.10", 
                           help="Python版本 (默认: 3.10)")
        
        args = parser.parse_args()
        
        # 设置基本属性
        self.base_dir = Path.cwd()
        self.image_name = "mooncake-base"
        self.container_name = "mooncake-test-container"
        self.dockerfile_path = self.base_dir / "Dockerfile"
        self.test_script_path = self.base_dir / "test_mooncake.sh"
        
        # 执行完整流程
        success = self.run_full_pipeline(args.python_version)
        sys.exit(0 if success else 1)
    
    def run_full_pipeline(self, python_version="3.10"):
        """运行完整的CI流程"""
        print("=" * 60)
        print("Mooncake Pack CI 开始运行")
        print("=" * 60)
        
        # 步骤1: 创建Dockerfile和测试脚本
        self.create_dockerfile()
        self.create_test_script(python_version)
        print()
        
        # 步骤2: 构建基础镜像
        if not self.build_base_image():
            print("❌ 构建基础镜像失败，终止流程")
            return False
        print()
        
        # 步骤3: 运行测试容器
        if not self.run_test_container():
            print("❌ 运行测试容器失败，终止流程")
            return False
        
        print("\n" + "=" * 60)
        print("✓ CI流程执行完成！")
        print(f"容器名称: {self.container_name}")
        print("实时日志: docker logs -f mooncake-test-container")
        print("进入容器: docker exec -it mooncake-test-container bash")
        print("停止容器: docker stop mooncake-test-container")
        print("=" * 60)
        
        return True
    
    def create_dockerfile(self):
        """创建基础Dockerfile - 只安装系统依赖"""
        dockerfile_content = """FROM ubuntu:22.04

# 设置环境变量避免交互式安装
ENV DEBIAN_FRONTEND=noninteractive

# 更新包管理器并安装必要的依赖
RUN apt-get update && apt-get install -y \\
    libibverbs-dev \\
    python3 \\
    python3-pip \\
    python3-venv \\
    wget \\
    curl \\
    && rm -rf /var/lib/apt/lists/*

# 创建工作目录
WORKDIR /app

# 默认保持容器运行
CMD ["sleep", "infinity"]
"""
        
        print("正在创建基础Dockerfile...")
        with open(self.dockerfile_path, 'w', encoding='utf-8') as f:
            f.write(dockerfile_content)
        print(f"✓ Dockerfile已创建: {self.dockerfile_path}")
    
    def create_test_script(self, python_version="3.10"):
        """创建测试脚本"""
        script_content = f"""#!/bin/bash
set -e

echo "=================================================="
echo "Mooncake Transfer Engine 测试脚本"
echo "=================================================="

# 创建虚拟环境
echo "创建Python虚拟环境..."
python3 -m venv /app/venv
source /app/venv/bin/activate

# 升级pip
echo "升级pip..."
pip install --upgrade pip

# 下载mooncake-transfer-engine包
echo "下载mooncake-transfer-engine包..."
mkdir -p /app/packages
pip download mooncake-transfer-engine -d /app/packages \\
    --python-version {python_version} \\
    --platform manylinux_2_35_x86_64 \\
    --only-binary=:all:

echo "下载的包文件:"
ls -la /app/packages/

# 离线安装mooncake-transfer-engine
echo "离线安装mooncake-transfer-engine..."
pip install --find-links /app/packages --no-index mooncake-transfer-engine

# 验证安装
echo "验证安装..."
pip list | grep mooncake || echo "警告: 未找到mooncake相关包"

# 启动mooncake_http_metadata_server (后台运行)
echo "启动mooncake_http_metadata_server..."
nohup mooncake_http_metadata_server > /app/server.log 2>&1 &
SERVER_PID=$!

echo "服务PID: $SERVER_PID"

# 等待服务启动
echo "等待服务启动..."
sleep 10

# 检查服务是否在运行
if kill -0 $SERVER_PID 2>/dev/null; then
    echo "✓ mooncake_http_metadata_server 启动成功"
    echo "服务日志:"
    cat /app/server.log
    
    # 尝试简单的健康检查（如果有端口监听）
    if command -v netstat >/dev/null; then
        echo "端口监听情况:"
        netstat -tlnp 2>/dev/null | grep :8080 || echo "未检测到8080端口监听"
    fi
    
    echo "✓ 测试完成 - 服务运行正常"
    
    # 保持服务运行
    echo "服务将继续运行，可通过 docker logs $HOSTNAME 查看详细日志"
    wait $SERVER_PID
else
    echo "❌ mooncake_http_metadata_server 启动失败"
    echo "错误日志:"
    cat /app/server.log
    exit 1
fi
"""
        
        print("正在创建测试脚本...")
        # 确保以UTF-8编码保存.sh文件
        with open(self.test_script_path, 'w', encoding='utf-8', newline='\n') as f:
            f.write(script_content)
        
        # 添加执行权限
        os.chmod(self.test_script_path, 0o755)
        print(f"✓ 测试脚本已创建 (UTF-8编码): {self.test_script_path}")
    
    def build_base_image(self):
        """构建基础Docker镜像"""
        print(f"正在构建基础Docker镜像: {self.image_name}")
        
        cmd = f"docker build -t {self.image_name} ."
        print(f"执行命令: {cmd}")
        
        result = os.system(cmd)
        if result == 0:
            print("✓ 基础Docker镜像构建成功")
            return True
        else:
            print("❌ Docker镜像构建失败")
            return False
    
    def run_test_container(self):
        """运行测试容器"""
        print(f"正在运行测试容器: {self.container_name}")
        
        # 强制停止并删除可能存在的同名容器
        print(f"清理旧容器: {self.container_name}")
        os.system(f"docker stop {self.container_name} 2>/dev/null")
        os.system(f"docker rm -f {self.container_name} 2>/dev/null")
        
        # 获取工作目录的绝对路径
        work_dir = str(self.base_dir.absolute())
        print(f"挂载工作目录: {work_dir}")
        
        # 运行新容器，挂载工作目录
        cmd = f'docker run --name {self.container_name} -p 8080:8080 -v "{work_dir}:/workspace" {self.image_name} bash /workspace/test_mooncake.sh'
        
        print(f"执行命令: {cmd}")
        result = os.system(cmd)
        
        if result == 0:
            print("✓ 容器执行完成")
            return True
        else:
            print("❌ 容器执行失败")
            return False

if __name__ == "__main__":
    MooncakePackCI()

Originally created by @ActivePeter on GitHub (Jun 5, 2025). Original GitHub issue: https://github.com/kvcache-ai/Mooncake/issues/450 Reproduce using the following script: ``` #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Mooncake Pack CI Script 自动化创建基础容器，通过运行时脚本完成包下载、安装和验证 """ import os import subprocess import sys import shutil import argparse from pathlib import Path # 设置Python默认编码为UTF-8 os.environ['PYTHONIOENCODING'] = 'utf-8' if hasattr(sys, 'set_int_max_str_digits'): sys.set_int_max_str_digits(0) class MooncakePackCI: def __init__(self): # 解析命令行参数 parser = argparse.ArgumentParser(description="Mooncake Pack CI Script") parser.add_argument("--python-version", default="3.10", help="Python版本 (默认: 3.10)") args = parser.parse_args() # 设置基本属性 self.base_dir = Path.cwd() self.image_name = "mooncake-base" self.container_name = "mooncake-test-container" self.dockerfile_path = self.base_dir / "Dockerfile" self.test_script_path = self.base_dir / "test_mooncake.sh" # 执行完整流程 success = self.run_full_pipeline(args.python_version) sys.exit(0 if success else 1) def run_full_pipeline(self, python_version="3.10"): """运行完整的CI流程""" print("=" * 60) print("Mooncake Pack CI 开始运行") print("=" * 60) # 步骤1: 创建Dockerfile和测试脚本 self.create_dockerfile() self.create_test_script(python_version) print() # 步骤2: 构建基础镜像 if not self.build_base_image(): print("❌ 构建基础镜像失败，终止流程") return False print() # 步骤3: 运行测试容器 if not self.run_test_container(): print("❌ 运行测试容器失败，终止流程") return False print("\n" + "=" * 60) print("✓ CI流程执行完成！") print(f"容器名称: {self.container_name}") print("实时日志: docker logs -f mooncake-test-container") print("进入容器: docker exec -it mooncake-test-container bash") print("停止容器: docker stop mooncake-test-container") print("=" * 60) return True def create_dockerfile(self): """创建基础Dockerfile - 只安装系统依赖""" dockerfile_content = """FROM ubuntu:22.04 # 设置环境变量避免交互式安装 ENV DEBIAN_FRONTEND=noninteractive # 更新包管理器并安装必要的依赖 RUN apt-get update && apt-get install -y \\ libibverbs-dev \\ python3 \\ python3-pip \\ python3-venv \\ wget \\ curl \\ && rm -rf /var/lib/apt/lists/* # 创建工作目录 WORKDIR /app # 默认保持容器运行 CMD ["sleep", "infinity"] """ print("正在创建基础Dockerfile...") with open(self.dockerfile_path, 'w', encoding='utf-8') as f: f.write(dockerfile_content) print(f"✓ Dockerfile已创建: {self.dockerfile_path}") def create_test_script(self, python_version="3.10"): """创建测试脚本""" script_content = f"""#!/bin/bash set -e echo "==================================================" echo "Mooncake Transfer Engine 测试脚本" echo "==================================================" # 创建虚拟环境 echo "创建Python虚拟环境..." python3 -m venv /app/venv source /app/venv/bin/activate # 升级pip echo "升级pip..." pip install --upgrade pip # 下载mooncake-transfer-engine包 echo "下载mooncake-transfer-engine包..." mkdir -p /app/packages pip download mooncake-transfer-engine -d /app/packages \\ --python-version {python_version} \\ --platform manylinux_2_35_x86_64 \\ --only-binary=:all: echo "下载的包文件:" ls -la /app/packages/ # 离线安装mooncake-transfer-engine echo "离线安装mooncake-transfer-engine..." pip install --find-links /app/packages --no-index mooncake-transfer-engine # 验证安装 echo "验证安装..." pip list | grep mooncake || echo "警告: 未找到mooncake相关包" # 启动mooncake_http_metadata_server (后台运行) echo "启动mooncake_http_metadata_server..." nohup mooncake_http_metadata_server > /app/server.log 2>&1 & SERVER_PID=$! echo "服务PID: $SERVER_PID" # 等待服务启动 echo "等待服务启动..." sleep 10 # 检查服务是否在运行 if kill -0 $SERVER_PID 2>/dev/null; then echo "✓ mooncake_http_metadata_server 启动成功" echo "服务日志:" cat /app/server.log # 尝试简单的健康检查（如果有端口监听） if command -v netstat >/dev/null; then echo "端口监听情况:" netstat -tlnp 2>/dev/null | grep :8080 || echo "未检测到8080端口监听" fi echo "✓ 测试完成 - 服务运行正常" # 保持服务运行 echo "服务将继续运行，可通过 docker logs $HOSTNAME 查看详细日志" wait $SERVER_PID else echo "❌ mooncake_http_metadata_server 启动失败" echo "错误日志:" cat /app/server.log exit 1 fi """ print("正在创建测试脚本...") # 确保以UTF-8编码保存.sh文件 with open(self.test_script_path, 'w', encoding='utf-8', newline='\n') as f: f.write(script_content) # 添加执行权限 os.chmod(self.test_script_path, 0o755) print(f"✓ 测试脚本已创建 (UTF-8编码): {self.test_script_path}") def build_base_image(self): """构建基础Docker镜像""" print(f"正在构建基础Docker镜像: {self.image_name}") cmd = f"docker build -t {self.image_name} ." print(f"执行命令: {cmd}") result = os.system(cmd) if result == 0: print("✓ 基础Docker镜像构建成功") return True else: print("❌ Docker镜像构建失败") return False def run_test_container(self): """运行测试容器""" print(f"正在运行测试容器: {self.container_name}") # 强制停止并删除可能存在的同名容器 print(f"清理旧容器: {self.container_name}") os.system(f"docker stop {self.container_name} 2>/dev/null") os.system(f"docker rm -f {self.container_name} 2>/dev/null") # 获取工作目录的绝对路径 work_dir = str(self.base_dir.absolute()) print(f"挂载工作目录: {work_dir}") # 运行新容器，挂载工作目录 cmd = f'docker run --name {self.container_name} -p 8080:8080 -v "{work_dir}:/workspace" {self.image_name} bash /workspace/test_mooncake.sh' print(f"执行命令: {cmd}") result = os.system(cmd) if result == 0: print("✓ 容器执行完成") return True else: print("❌ 容器执行失败") return False if __name__ == "__main__": MooncakePackCI() ```

bsakaguchi closed this issue

2026-10-29 21:10:16 +00:00

bsakaguchi commented

2026-10-29 21:10:17 +00:00

Author

Owner

@ActivePeter commented on GitHub (Jun 5, 2025):

Issue resolved: by using the --no-deps flag during pip export, only the main package is exported. The environment is pre-configured with aiohttp installed.

@ActivePeter commented on GitHub (Jun 5, 2025): Issue resolved: by using the --no-deps flag during pip export, only the main package is exported. The environment is pre-configured with aiohttp installed.