deepseek-ai/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS.git synced 2026-11-05 00:47:47 +00:00
Code Issues 166 Projects Packages Wiki Activity

[GH-ISSUE #177] 尝试docker化构建，参考ceph,我已经提前在宿主机上build好了bin #106

New issue
Open
opened 2026-10-26 09:45:13 +00:00 by bsakaguchi · 0 comments
bsakaguchi commented 2026-10-26 09:45:13 +00:00
Owner
Copy link

Originally created by @pwxwmm on GitHub (Mar 14, 2025).
Original GitHub issue: https://github.com/deepseek-ai/3FS/issues/177


# Author: mmwei3
# Email: mmwei3@iflytek.com
# date: 20250313
# -------------------------------------------
# Stage 1: Build 3FS from source
# -------------------------------------------
FROM ubuntu:22.04 AS builder

# 时区非交互安装
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
    git cmake make clang-14 clang++-14 libfuse3-dev libssl-dev pkg-config \
    curl ca-certificates wget unzip \
    # FoundationDB client library headers (可选, 也可自行下载)
    && apt-get clean && rm -rf /var/lib/apt/lists/*

# 安装 Rust 工具链 (如果 3FS 需要 Rust >=1.68)
RUN export RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static
RUN export RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup
RUN curl --proto '=https' --tlsv1.2  | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"

WORKDIR /build

# 克隆 3FS 源码
#RUN 3fs /build

# -------------------------------------------
# Stage 2: Create minimal runtime image
# -------------------------------------------
FROM ubuntu:22.04

ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
    libfuse3-3 libssl3 ca-certificates \
    cmake libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev libdwarf-dev libunwind-dev \
    libaio-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgmock-dev clang-format-14 clang-14 clang-tidy-14 lld-14 \
    libgoogle-perftools-dev google-perftools libssl-dev gcc-12 g++-12 libboost-all-dev \
    cmake make clang-14 clang++-14 libfuse3-dev libssl-dev pkg-config \
    curl ca-certificates wget unzip \
    # FoundationDB 客户端库 (fdb.cluster 需自行挂载)
    && apt-get clean && rm -rf /var/lib/apt/lists/*

# 创建目录结构
RUN mkdir -p /opt/3fs/bin /opt/3fs/etc /var/log/3fs

# 从 builder 复制 3FS 可执行文件
#COPY --from=builder /build/build/bin/* /opt/3fs/bin/
COPY bin/* /opt/3fs/bin/


# entrypoint 脚本，用来根据参数启动不同角色
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh

WORKDIR /opt/3fs
ENTRYPOINT ["/entrypoint.sh"]
CMD ["help"]

这个是对应的入口


root@3fs-meta:/home/3fs# cat entrypoint.sh
#!/usr/bin/env bash
# Author: mmwei3
# Email: mmwei3@iflytek.com
# Date: 2025-03-13

set -e

ROLE="$1"
CFG_DIR="/opt/3fs/etc"
BIN_DIR="/opt/3fs/bin"
LOG_DIR="/var/log/3fs"

# 确保日志目录存在
mkdir -p "$LOG_DIR"

# 如果未提供角色参数，则显示帮助信息
if [[ -z "$ROLE" || "$ROLE" == "help" ]]; then
    echo "Usage: docker run <options> mmwei3.images.build/deepseek-3fs:20250313 <role>"
    echo "Available roles:"
    echo "  mgmtd     - Start management daemon"
    echo "  meta      - Start metadata service"
    echo "  storage   - Start storage service"
    echo "  client    - Start FUSE client"
    echo "  admin-cli - Start interactive admin CLI"
    echo "Example:"
    echo "  docker run --rm deepseek-3fs:latest mgmtd"
    exit 1
fi

# 角色处理
case "$ROLE" in
  mgmtd)
    exec "$BIN_DIR/mgmtd_main" -cfg "$CFG_DIR/mgmtd_main.toml"
    ;;
  meta)
    exec "$BIN_DIR/meta_main" -cfg "$CFG_DIR/meta_main.toml"
    ;;
  storage)
    exec "$BIN_DIR/storage_main" -cfg "$CFG_DIR/storage_main.toml"
    ;;
  client)
    exec "$BIN_DIR/hf3fs_fuse_main" -cfg "$CFG_DIR/hf3fs_fuse_main.toml"
    ;;
  admin-cli)
    exec "$BIN_DIR/admin_cli" -cfg "$CFG_DIR/admin_cli.toml"
    ;;
  *)
    echo "Error: Unknown role '$ROLE'"
    echo "Run 'docker run ... deepseek-3fs:latest help' for usage instructions."
    exit 1
    ;;
esac
root@3fs-meta:/home/3fs#



root@3fs-meta:/home/3fs# docker ps -a
CONTAINER ID   IMAGE                                         COMMAND                  CREATED       STATUS                     PORTS     NAMES
28e344dc4dc6   mmwei3.images.build/deepseek-3fs:20250314-6   "/entrypoint.sh mgmtd"   9 hours ago   Exited (139) 9 hours ago             3fs-mgmtd
root@3fs-meta:/home/3fs# docker images
REPOSITORY                         TAG          IMAGE ID       CREATED        SIZE
mmwei3.images.build/deepseek-3fs   20250314-5   80be8c8bda15   9 hours ago    4.12GB
mmwei3.images.build/deepseek-3fs   20250314-6   83f11c1f9078   9 hours ago    4.12GB
mmwei3.images.build/deepseek-3fs   20250314-4   d69c641ccbbf   10 hours ago   3.82GB
mmwei3.images.build/deepseek-3fs   20250314     20286b63a23b   10 hours ago   2.43GB
mmwei3.images.build/deepseek-3fs   20250314-1   1b75920c9faa   10 hours ago   2.43GB
mmwei3.images.build/deepseek-3fs   20250314-2   2840c489afe4   10 hours ago   2.43GB
mmwei3.images.build/deepseek-3fs   20250314-3   4f5b7df1291d   10 hours ago   2.43GB
mmwei3.images.build/deepseek-3fs   20250313     e690f7ed446c   22 hours ago   2.43GB
root@3fs-meta:/home/3fs#


 docker run -d -it --net=host --name 3fs-mgmtd --privileged   -v /mycluster/config:/opt/3fs/etc   -v /mycluster/logs:/var/log/3fs  --device=/dev/infiniband:/dev/infiniband  -v /usr/lib:/usr/lib   --cap-add=NET_RAW --cap-add=IPC_LOCK --cap-add=CAP_NET_ADMIN  mmwei3.images.build/deepseek-3fs:20250314-6 mgmtd
这里需要注意下，有几个.so文件需要映射到容器中，否则会有问题
比如：cp /root/3fs/build/third_party/jemalloc/lib/libjemalloc.so.2 /usr/bin/

环境ubuntu 22.04 
root@3fs-meta:/home/3fs# uname  -r
5.15.0-133-generic

安装说明：
foundation DB 可以使用dpkg安装 也可以直接docker安装
docker run -d --net=host --name fdb-server foundationdb/foundationdb:7.1.42


root@3fs-meta:/home/3fs# systemctl  status foundationdb.service
● foundationdb.service - LSB: start and stop foundationdb
     Loaded: loaded (/etc/init.d/foundationdb; generated)
     Active: active (running) since Fri 2025-03-14 22:40:54 CST; 1s ago
       Docs: man:systemd-sysv-generator(8)
    Process: 213581 ExecStart=/etc/init.d/foundationdb start (code=exited, status=0/SUCCESS)
      Tasks: 9 (limit: 4561)
     Memory: 86.2M
        CPU: 140ms
     CGroup: /system.slice/foundationdb.service
             ├─213585 /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb.conf --lockfile /var/run/fdbmonitor.pid --daemonize
             ├─213587 /usr/lib/foundationdb/backup_agent/backup_agent --cluster-file /etc/foundationdb/fdb.cluster --logdir /var/log/foundationdb
             └─213588 /usr/sbin/fdbserver --cluster-file /etc/foundationdb/fdb.cluster --datadir /var/lib/foundationdb/data/4500 --listen-address public --lo>

Mar 14 22:40:54 3fs-meta fdbmonitor[213584]: LogGroup="default" Process="fdbmonitor": Started FoundationDB Process Monitor 7.3 (v7.3.35)
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Watching conf file /etc/foundationdb/foundationdb.conf
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Watching conf dir /etc/foundationdb/ (2)
Mar 14 22:40:54 3fs-meta systemd[1]: Started LSB: start and stop foundationdb.
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Loading configuration /etc/foundationdb/foundationdb.conf
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Starting backup_agent.1
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Starting fdbserver.4500
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="backup_agent.1": Launching /usr/lib/foundationdb/backup_agent/backup_agent (213587) >
Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbserver.4500": Launching /usr/sbin/fdbserver (213588) for fdbserver.4500
Mar 14 22:40:55 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbserver.4500": FDBD joined cluster.

root@3fs-meta:/opt# ls
3fs  clickhouse  containerd  MLNX_OFED_LINUX-3.4-2.0.0.0-rhel7.2-x86_64  MLNX_OFED_LINUX-3.4-2.0.0.0-rhel7.2-x86_64.tgz
root@3fs-meta:/opt# cd /mnt/
root@3fs-meta:/mnt# ls
foundationdb-clients_7.3.35-1_amd64.deb  foundationdb-server_7.3.35-1_amd64.deb  fuse-3.16.1  fuse-3.16.1.tar.gz  rust.sh
root@3fs-meta:/mnt#
Originally created by @pwxwmm on GitHub (Mar 14, 2025). Original GitHub issue: https://github.com/deepseek-ai/3FS/issues/177 ```python # Author: mmwei3 # Email: mmwei3@iflytek.com # date: 20250313 # ------------------------------------------- # Stage 1: Build 3FS from source # ------------------------------------------- FROM ubuntu:22.04 AS builder # 时区非交互安装 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ git cmake make clang-14 clang++-14 libfuse3-dev libssl-dev pkg-config \ curl ca-certificates wget unzip \ # FoundationDB client library headers (可选, 也可自行下载) && apt-get clean && rm -rf /var/lib/apt/lists/* # 安装 Rust 工具链 (如果 3FS 需要 Rust >=1.68) RUN export RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static RUN export RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup RUN curl --proto '=https' --tlsv1.2 | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" WORKDIR /build # 克隆 3FS 源码 #RUN 3fs /build # ------------------------------------------- # Stage 2: Create minimal runtime image # ------------------------------------------- FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ libfuse3-3 libssl3 ca-certificates \ cmake libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev libdwarf-dev libunwind-dev \ libaio-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgmock-dev clang-format-14 clang-14 clang-tidy-14 lld-14 \ libgoogle-perftools-dev google-perftools libssl-dev gcc-12 g++-12 libboost-all-dev \ cmake make clang-14 clang++-14 libfuse3-dev libssl-dev pkg-config \ curl ca-certificates wget unzip \ # FoundationDB 客户端库 (fdb.cluster 需自行挂载) && apt-get clean && rm -rf /var/lib/apt/lists/* # 创建目录结构 RUN mkdir -p /opt/3fs/bin /opt/3fs/etc /var/log/3fs # 从 builder 复制 3FS 可执行文件 #COPY --from=builder /build/build/bin/* /opt/3fs/bin/ COPY bin/* /opt/3fs/bin/ # entrypoint 脚本，用来根据参数启动不同角色 COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh WORKDIR /opt/3fs ENTRYPOINT ["/entrypoint.sh"] CMD ["help"] ``` 这个是对应的入口 ```shell root@3fs-meta:/home/3fs# cat entrypoint.sh #!/usr/bin/env bash # Author: mmwei3 # Email: mmwei3@iflytek.com # Date: 2025-03-13 set -e ROLE="$1" CFG_DIR="/opt/3fs/etc" BIN_DIR="/opt/3fs/bin" LOG_DIR="/var/log/3fs" # 确保日志目录存在 mkdir -p "$LOG_DIR" # 如果未提供角色参数，则显示帮助信息 if [[ -z "$ROLE" || "$ROLE" == "help" ]]; then echo "Usage: docker run <options> mmwei3.images.build/deepseek-3fs:20250313 <role>" echo "Available roles:" echo " mgmtd - Start management daemon" echo " meta - Start metadata service" echo " storage - Start storage service" echo " client - Start FUSE client" echo " admin-cli - Start interactive admin CLI" echo "Example:" echo " docker run --rm deepseek-3fs:latest mgmtd" exit 1 fi # 角色处理 case "$ROLE" in mgmtd) exec "$BIN_DIR/mgmtd_main" -cfg "$CFG_DIR/mgmtd_main.toml" ;; meta) exec "$BIN_DIR/meta_main" -cfg "$CFG_DIR/meta_main.toml" ;; storage) exec "$BIN_DIR/storage_main" -cfg "$CFG_DIR/storage_main.toml" ;; client) exec "$BIN_DIR/hf3fs_fuse_main" -cfg "$CFG_DIR/hf3fs_fuse_main.toml" ;; admin-cli) exec "$BIN_DIR/admin_cli" -cfg "$CFG_DIR/admin_cli.toml" ;; *) echo "Error: Unknown role '$ROLE'" echo "Run 'docker run ... deepseek-3fs:latest help' for usage instructions." exit 1 ;; esac root@3fs-meta:/home/3fs# ``` ```python root@3fs-meta:/home/3fs# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 28e344dc4dc6 mmwei3.images.build/deepseek-3fs:20250314-6 "/entrypoint.sh mgmtd" 9 hours ago Exited (139) 9 hours ago 3fs-mgmtd root@3fs-meta:/home/3fs# docker images REPOSITORY TAG IMAGE ID CREATED SIZE mmwei3.images.build/deepseek-3fs 20250314-5 80be8c8bda15 9 hours ago 4.12GB mmwei3.images.build/deepseek-3fs 20250314-6 83f11c1f9078 9 hours ago 4.12GB mmwei3.images.build/deepseek-3fs 20250314-4 d69c641ccbbf 10 hours ago 3.82GB mmwei3.images.build/deepseek-3fs 20250314 20286b63a23b 10 hours ago 2.43GB mmwei3.images.build/deepseek-3fs 20250314-1 1b75920c9faa 10 hours ago 2.43GB mmwei3.images.build/deepseek-3fs 20250314-2 2840c489afe4 10 hours ago 2.43GB mmwei3.images.build/deepseek-3fs 20250314-3 4f5b7df1291d 10 hours ago 2.43GB mmwei3.images.build/deepseek-3fs 20250313 e690f7ed446c 22 hours ago 2.43GB root@3fs-meta:/home/3fs# docker run -d -it --net=host --name 3fs-mgmtd --privileged -v /mycluster/config:/opt/3fs/etc -v /mycluster/logs:/var/log/3fs --device=/dev/infiniband:/dev/infiniband -v /usr/lib:/usr/lib --cap-add=NET_RAW --cap-add=IPC_LOCK --cap-add=CAP_NET_ADMIN mmwei3.images.build/deepseek-3fs:20250314-6 mgmtd 这里需要注意下，有几个.so文件需要映射到容器中，否则会有问题 比如：cp /root/3fs/build/third_party/jemalloc/lib/libjemalloc.so.2 /usr/bin/ 环境ubuntu 22.04 root@3fs-meta:/home/3fs# uname -r 5.15.0-133-generic 安装说明： foundation DB 可以使用dpkg安装 也可以直接docker安装 docker run -d --net=host --name fdb-server foundationdb/foundationdb:7.1.42 root@3fs-meta:/home/3fs# systemctl status foundationdb.service ● foundationdb.service - LSB: start and stop foundationdb Loaded: loaded (/etc/init.d/foundationdb; generated) Active: active (running) since Fri 2025-03-14 22:40:54 CST; 1s ago Docs: man:systemd-sysv-generator(8) Process: 213581 ExecStart=/etc/init.d/foundationdb start (code=exited, status=0/SUCCESS) Tasks: 9 (limit: 4561) Memory: 86.2M CPU: 140ms CGroup: /system.slice/foundationdb.service ├─213585 /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb.conf --lockfile /var/run/fdbmonitor.pid --daemonize ├─213587 /usr/lib/foundationdb/backup_agent/backup_agent --cluster-file /etc/foundationdb/fdb.cluster --logdir /var/log/foundationdb └─213588 /usr/sbin/fdbserver --cluster-file /etc/foundationdb/fdb.cluster --datadir /var/lib/foundationdb/data/4500 --listen-address public --lo> Mar 14 22:40:54 3fs-meta fdbmonitor[213584]: LogGroup="default" Process="fdbmonitor": Started FoundationDB Process Monitor 7.3 (v7.3.35) Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Watching conf file /etc/foundationdb/foundationdb.conf Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Watching conf dir /etc/foundationdb/ (2) Mar 14 22:40:54 3fs-meta systemd[1]: Started LSB: start and stop foundationdb. Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Loading configuration /etc/foundationdb/foundationdb.conf Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Starting backup_agent.1 Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbmonitor": Starting fdbserver.4500 Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="backup_agent.1": Launching /usr/lib/foundationdb/backup_agent/backup_agent (213587) > Mar 14 22:40:54 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbserver.4500": Launching /usr/sbin/fdbserver (213588) for fdbserver.4500 Mar 14 22:40:55 3fs-meta fdbmonitor[213585]: LogGroup="default" Process="fdbserver.4500": FDBD joined cluster. root@3fs-meta:/opt# ls 3fs clickhouse containerd MLNX_OFED_LINUX-3.4-2.0.0.0-rhel7.2-x86_64 MLNX_OFED_LINUX-3.4-2.0.0.0-rhel7.2-x86_64.tgz root@3fs-meta:/opt# cd /mnt/ root@3fs-meta:/mnt# ls foundationdb-clients_7.3.35-1_amd64.deb foundationdb-server_7.3.35-1_amd64.deb fuse-3.16.1 fuse-3.16.1.tar.gz rust.sh root@3fs-meta:/mnt# ```
Sign in to join this conversation.
No Branch/Tag specified
Branches Tags
main
SF-Zhou-patch-1
No results found.
Labels
Clear labels   
pull-request
Mirrored from GitHub Pull Request

No labels
pull-request
Milestone
Clear milestone
No items
No milestone
Projects
Clear projects
No items
No project
Assignees
Clear assignees
No assignees
1 participant
Notifications
Due date
The due date is invalid or out of range. Please use the format "yyyy-mm-dd".

No due date set.

Dependencies

No dependencies set.

Reference
deepseek-ai/3FS#106
No description provided.