deepseek-ai/3FS
1
0
Fork 0
mirror of https://github.com/deepseek-ai/3FS.git synced 2026-11-05 00:47:47 +00:00
Code Issues 166 Projects Packages Wiki Activity

[GH-ISSUE #213] Meta node failed to connect mgmtd even if they are both in one node #127

New issue
Open
opened 2026-10-26 09:45:38 +00:00 by bsakaguchi · 4 comments
bsakaguchi commented 2026-10-26 09:45:38 +00:00
Owner
Copy link

Originally created by @iamazy on GitHub (Mar 26, 2025).
Original GitHub issue: https://github.com/deepseek-ai/3FS/issues/213

I deployed both meta and mgmtd on the same node (10.155.10.171), but meta consistently reports an inability to connect to the RDMA://10.155.10.171:8000 service. It can initially report heartbeats during startup but later fails. I have confirmed that the mgmtd service is running normally, and other storage nodes can also report heartbeats via RDMA://10.155.10.171:8000. Does anyone know why this might be happening?

Image

my meta_main.toml is

[[common.log.categories]]
categories = [ '.' ]
handlers = [ 'normal', 'err', 'fatal' ]
inherit = true
level = 'INFO'
propagate = 'NONE'

[[common.log.categories]]
categories = [ 'eventlog' ]
handlers = [ 'event' ]
inherit = false
level = 'INFO'
propagate = 'ERR'

[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/hf3fs_meta_main.log'
max_file_size = '100MB'
max_files = 10
name = 'normal'
rotate = true
rotate_on_open = false
start_level = 'NONE'
stream_type = 'STDERR'
writer_type = 'FILE'

[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_meta_main-err.log'
max_file_size = '100MB'
max_files = 10
name = 'err'
rotate = true
rotate_on_open = false
start_level = 'ERR'
stream_type = 'STDERR'
writer_type = 'FILE'

[[common.log.handlers]]
async = false
file_path = '/var/log/3fs/hf3fs_meta_main-fatal.log'
max_file_size = '100MB'
max_files = 10
name = 'fatal'
rotate = true
rotate_on_open = false
start_level = 'FATAL'
stream_type = 'STDERR'
writer_type = 'STREAM'

[[common.log.handlers]]
async = true
file_path = '/var/log/3fs/hf3fs_meta_main-event.log'
max_file_size = '100MB'
max_files = 10
name = 'event'
rotate = true
rotate_on_open = false
start_level = 'INFO'
stream_type = 'STDERR'
writer_type = 'EVENT'

[common.memory]
prof_active = false
prof_prefix = ''

[common.monitor]
collect_period = '1s'
num_collectors = 1

[[common.monitor.reporters]]
type = 'monitor_collector'

[common.monitor.reporters.monitor_collector]
remote_ip = '10.151.18.23:10000'

[server]
use_memkv = false

[server.background_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false

[server.background_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'

[server.background_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4

[server.background_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14

[server.background_client.io_worker.transport_pool]
max_connections = 1

[server.background_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'

[server.background_client.rdma_control]
max_concurrent_transmission = 64

[server.background_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'

[server.base.independent_thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'

[server.base.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'

[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'RDMA'
services = [ 'MetaSerde' ]
use_independent_thread_pool = false

[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'

[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4

[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14

[server.base.groups.io_worker.transport_pool]
max_connections = 1

[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 8001
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false

[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'

[[server.base.groups]]
check_connections_interval = '1min'
connection_expiration_time = '1day'
network_type = 'TCP'
services = [ 'Core' ]
use_independent_thread_pool = true

[server.base.groups.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'

[server.base.groups.io_worker.connect_concurrency_limiter]
max_concurrency = 4

[server.base.groups.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14

[server.base.groups.io_worker.transport_pool]
max_connections = 1

[server.base.groups.listener]
domain_socket_index = 1
filter_list = []
listen_port = 9001
listen_queue_depth = 4096
rdma_accept_timeout = '15s'
rdma_listen_ethernet = true
reuse_port = false

[server.base.groups.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'

[server.fdb]
casual_read_risky = false
clusterFile = '/opt/3fs/etc/fdb.cluster'
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'

[server.kv_engine]
use_memkv = false

[server.kv_engine.fdb]
casual_read_risky = false
clusterFile = ''
default_backoff = 0
enableMultipleClient = false
externalClientDir = ''
externalClientPath = ''
multipleClientThreadNum = 4
readonly = false
trace_file = ''
trace_format = 'json'

[server.meta]
acl_cache_time = '15s'
allow_directly_move_to_trash = false
allow_owner_change_immutable = false
allow_stat_deleted_inodes = true
authenticate = false
batch_stat_by_path_concurrent = 4
batch_stat_concurrent = 8
check_file_hole = false
dynamic_stripe = false
dynamic_stripe_growth = 2
dynamic_stripe_initial = 16
enable_new_chunk_engine = false
grv_cache = false
idempotent_record_clean = '1min'
idempotent_record_expire = '30min'
idempotent_remove = true
idempotent_rename = false
iflags_chain_allocation = false
iflags_chunk_engine = true
ignore_length_hint = false
inodeId_abort_on_duplicate = false
inodeId_check_unique = true
list_default_limit = 128
max_batch_operations = 4096
max_directory_depth = 64
max_remove_chunks_per_request = 32
max_symlink_count = 10
max_symlink_depth = 4
operation_timeout = '5s'
otrunc_replace_file = true
otrunc_replace_file_threshold = '1GB'
readonly = false
recursive_remove_check_owner = true
recursive_remove_perm_check = 1024
statfs_cache_time = '1min'
statfs_space_imbalance_threshold = 5
statfs_update_interval = '5s'
sync_on_prune_session = false
time_granularity = '1s'

[server.meta.background_hole_checker]
coroutines_num = 16
enable_work_stealing = false
queue_size = 4096

[server.meta.distributor]
timeout = '30s'
update_interval = '1s'

[server.meta.event_trace_log]
dump_interval = '30s'
enabled = true
max_num_writers = 1
max_row_group_length = 100000
trace_file_dir = '.'

[server.meta.forward]
addr_type = 'RDMA'
debug = true
timeout = '10s'

[server.meta.gc]
check_session = true
distributed_gc = true
enable = true
gc_delay_free_space_threshold = 5
gc_directory_concurrent = 4
gc_directory_delay = '0ns'
gc_directory_entry_batch = 32
gc_directory_entry_concurrent = 4
gc_file_concurrent = 32
gc_file_delay = '5min'
large_file_chunks = 128
recursive_perm_check = true
remove_chunks_batch_size = 32
retry_delay = '10min'
scan_batch = 4096
scan_interval = '200ms'
small_file_chunks = 32
txn_low_priority = false

[server.meta.gc.retry_remove_chunks]
init_wait_time = '10s'
max_retry_time = '30s'
max_wait_time = '10s'
retry_permanent_error = false

[server.meta.gc.workers]
coroutines_num = 8
enable_work_stealing = false
queue_size = 1024

[server.meta.retry_remove_chunks]
init_wait_time = '10s'
max_retry_time = '30s'
max_wait_time = '10s'
retry_permanent_error = false

[server.meta.retry_transaction]
max_backoff = '1s'
max_retry_count = 10

[server.meta.session_manager]
enable = true
scan_batch = 1024
scan_interval = '5min'
session_timeout = '5min'
sync_on_prune_session = false

[server.meta.session_manager.close_workers]
coroutines_num = 32
enable_work_stealing = false
queue_size = 1024

[server.meta.session_manager.scan_workers]
coroutines_num = 8
enable_work_stealing = false
queue_size = 128

[server.meta.user_cache]
buckets = 127
exist_ttl = '5min'
inexist_ttl = '10s'

[server.mgmtd_client]
accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
auto_extend_client_session_interval = '10s'
auto_heartbeat_interval = '10s'
auto_refresh_interval = '10s'
enable_auto_extend_client_session = false
enable_auto_heartbeat = true
enable_auto_refresh = true
mgmtd_server_addresses = ["RDMA://10.155.10.171:8000", "RDMA://10.153.16.171:8000"]
work_queue_size = 100

[server.storage_client]
check_overlapping_read_buffers = true
check_overlapping_write_buffers = false
chunk_checksum_type = 'CRC32C'
create_net_client_for_updates = false
implementation_type = 'RPC'
max_inline_read_bytes = '0'
max_inline_write_bytes = '0'
max_read_io_bytes = '0'

[server.storage_client.net_client]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false

[server.storage_client.net_client.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'

[server.storage_client.net_client.io_worker.connect_concurrency_limiter]
max_concurrency = 4

[server.storage_client.net_client.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14

[server.storage_client.net_client.io_worker.transport_pool]
max_connections = 1

[server.storage_client.net_client.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'

[server.storage_client.net_client.rdma_control]
max_concurrent_transmission = 64

[server.storage_client.net_client.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'

[server.storage_client.net_client_for_updates]
default_compression_level = 0
default_compression_threshold = '128KB'
default_log_long_running_threshold = '0ns'
default_report_metrics = false
default_send_retry_times = 1
default_timeout = '1s'
enable_rdma_control = false
force_use_tcp = false

[server.storage_client.net_client_for_updates.io_worker]
num_event_loop = 1
rdma_connect_timeout = '5s'
read_write_rdma_in_event_thread = false
read_write_tcp_in_event_thread = false
tcp_connect_timeout = '1s'
wait_to_retry_send = '100ms'

[server.storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter]
max_concurrency = 4

[server.storage_client.net_client_for_updates.io_worker.ibsocket]
buf_ack_batch = 8
buf_signal_batch = 8
buf_size = 16384
drain_timeout = '5s'
drop_connections = 0
event_ack_batch = 128
max_rd_atomic = 16
max_rdma_wr = 128
max_rdma_wr_per_post = 32
max_sge = 16
min_rnr_timer = 1
record_bytes_per_peer = false
record_latency_per_peer = false
retry_cnt = 7
rnr_retry = 0
send_buf_cnt = 32
sl = 0
start_psn = 0
timeout = 14

[server.storage_client.net_client_for_updates.io_worker.transport_pool]
max_connections = 1

[server.storage_client.net_client_for_updates.processor]
enable_coroutines_pool = true
max_coroutines_num = 256
max_processing_requests_num = 4096
response_compression_level = 1
response_compression_threshold = '128KB'

[server.storage_client.net_client_for_updates.rdma_control]
max_concurrent_transmission = 64

[server.storage_client.net_client_for_updates.thread_pool]
bg_thread_pool_stratetry = 'SHARED_QUEUE'
collect_stats = false
enable_work_stealing = false
io_thread_pool_stratetry = 'SHARED_QUEUE'
num_bg_threads = 2
num_connect_threads = 2
num_io_threads = 2
num_proc_threads = 2
proc_thread_pool_stratetry = 'SHARED_QUEUE'

[server.storage_client.retry]
init_wait_time = '2s'
max_failures_before_failover = 1
max_retry_time = '5s'
max_wait_time = '5s'

[server.storage_client.traffic_control.query]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

[server.storage_client.traffic_control.read]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

[server.storage_client.traffic_control.remove]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

[server.storage_client.traffic_control.truncate]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true

[server.storage_client.traffic_control.write]
max_batch_bytes = '4MB'
max_batch_size = 128
max_concurrent_requests = 32
max_concurrent_requests_per_server = 8
process_batches_in_parallel = true
random_shuffle_requests = true
Originally created by @iamazy on GitHub (Mar 26, 2025). Original GitHub issue: https://github.com/deepseek-ai/3FS/issues/213 I deployed both meta and mgmtd on the same node (10.155.10.171), but meta consistently reports an inability to connect to the RDMA://10.155.10.171:8000 service. It can initially report heartbeats during startup but later fails. I have confirmed that the mgmtd service is running normally, and other storage nodes can also report heartbeats via RDMA://10.155.10.171:8000. Does anyone know why this might be happening? <img width="1024" alt="Image" src="https://github.com/user-attachments/assets/d70abd6a-f1bb-46e2-836a-09d781be4db3" /> my meta_main.toml is ```toml [[common.log.categories]] categories = [ '.' ] handlers = [ 'normal', 'err', 'fatal' ] inherit = true level = 'INFO' propagate = 'NONE' [[common.log.categories]] categories = [ 'eventlog' ] handlers = [ 'event' ] inherit = false level = 'INFO' propagate = 'ERR' [[common.log.handlers]] async = true file_path = '/var/log/3fs/hf3fs_meta_main.log' max_file_size = '100MB' max_files = 10 name = 'normal' rotate = true rotate_on_open = false start_level = 'NONE' stream_type = 'STDERR' writer_type = 'FILE' [[common.log.handlers]] async = false file_path = '/var/log/3fs/hf3fs_meta_main-err.log' max_file_size = '100MB' max_files = 10 name = 'err' rotate = true rotate_on_open = false start_level = 'ERR' stream_type = 'STDERR' writer_type = 'FILE' [[common.log.handlers]] async = false file_path = '/var/log/3fs/hf3fs_meta_main-fatal.log' max_file_size = '100MB' max_files = 10 name = 'fatal' rotate = true rotate_on_open = false start_level = 'FATAL' stream_type = 'STDERR' writer_type = 'STREAM' [[common.log.handlers]] async = true file_path = '/var/log/3fs/hf3fs_meta_main-event.log' max_file_size = '100MB' max_files = 10 name = 'event' rotate = true rotate_on_open = false start_level = 'INFO' stream_type = 'STDERR' writer_type = 'EVENT' [common.memory] prof_active = false prof_prefix = '' [common.monitor] collect_period = '1s' num_collectors = 1 [[common.monitor.reporters]] type = 'monitor_collector' [common.monitor.reporters.monitor_collector] remote_ip = '10.151.18.23:10000' [server] use_memkv = false [server.background_client] default_compression_level = 0 default_compression_threshold = '128KB' default_log_long_running_threshold = '0ns' default_report_metrics = false default_send_retry_times = 1 default_timeout = '1s' enable_rdma_control = false force_use_tcp = false [server.background_client.io_worker] num_event_loop = 1 rdma_connect_timeout = '5s' read_write_rdma_in_event_thread = false read_write_tcp_in_event_thread = false tcp_connect_timeout = '1s' wait_to_retry_send = '100ms' [server.background_client.io_worker.connect_concurrency_limiter] max_concurrency = 4 [server.background_client.io_worker.ibsocket] buf_ack_batch = 8 buf_signal_batch = 8 buf_size = 16384 drain_timeout = '5s' drop_connections = 0 event_ack_batch = 128 max_rd_atomic = 16 max_rdma_wr = 128 max_rdma_wr_per_post = 32 max_sge = 16 min_rnr_timer = 1 record_bytes_per_peer = false record_latency_per_peer = false retry_cnt = 7 rnr_retry = 0 send_buf_cnt = 32 sl = 0 start_psn = 0 timeout = 14 [server.background_client.io_worker.transport_pool] max_connections = 1 [server.background_client.processor] enable_coroutines_pool = true max_coroutines_num = 256 max_processing_requests_num = 4096 response_compression_level = 1 response_compression_threshold = '128KB' [server.background_client.rdma_control] max_concurrent_transmission = 64 [server.background_client.thread_pool] bg_thread_pool_stratetry = 'SHARED_QUEUE' collect_stats = false enable_work_stealing = false io_thread_pool_stratetry = 'SHARED_QUEUE' num_bg_threads = 2 num_connect_threads = 2 num_io_threads = 2 num_proc_threads = 2 proc_thread_pool_stratetry = 'SHARED_QUEUE' [server.base.independent_thread_pool] bg_thread_pool_stratetry = 'SHARED_QUEUE' collect_stats = false enable_work_stealing = false io_thread_pool_stratetry = 'SHARED_QUEUE' num_bg_threads = 2 num_connect_threads = 2 num_io_threads = 2 num_proc_threads = 2 proc_thread_pool_stratetry = 'SHARED_QUEUE' [server.base.thread_pool] bg_thread_pool_stratetry = 'SHARED_QUEUE' collect_stats = false enable_work_stealing = false io_thread_pool_stratetry = 'SHARED_QUEUE' num_bg_threads = 2 num_connect_threads = 2 num_io_threads = 2 num_proc_threads = 2 proc_thread_pool_stratetry = 'SHARED_QUEUE' [[server.base.groups]] check_connections_interval = '1min' connection_expiration_time = '1day' network_type = 'RDMA' services = [ 'MetaSerde' ] use_independent_thread_pool = false [server.base.groups.io_worker] num_event_loop = 1 rdma_connect_timeout = '5s' read_write_rdma_in_event_thread = false read_write_tcp_in_event_thread = false tcp_connect_timeout = '1s' wait_to_retry_send = '100ms' [server.base.groups.io_worker.connect_concurrency_limiter] max_concurrency = 4 [server.base.groups.io_worker.ibsocket] buf_ack_batch = 8 buf_signal_batch = 8 buf_size = 16384 drain_timeout = '5s' drop_connections = 0 event_ack_batch = 128 max_rd_atomic = 16 max_rdma_wr = 128 max_rdma_wr_per_post = 32 max_sge = 16 min_rnr_timer = 1 record_bytes_per_peer = false record_latency_per_peer = false retry_cnt = 7 rnr_retry = 0 send_buf_cnt = 32 sl = 0 start_psn = 0 timeout = 14 [server.base.groups.io_worker.transport_pool] max_connections = 1 [server.base.groups.listener] domain_socket_index = 1 filter_list = [] listen_port = 8001 listen_queue_depth = 4096 rdma_accept_timeout = '15s' rdma_listen_ethernet = true reuse_port = false [server.base.groups.processor] enable_coroutines_pool = true max_coroutines_num = 256 max_processing_requests_num = 4096 response_compression_level = 1 response_compression_threshold = '128KB' [[server.base.groups]] check_connections_interval = '1min' connection_expiration_time = '1day' network_type = 'TCP' services = [ 'Core' ] use_independent_thread_pool = true [server.base.groups.io_worker] num_event_loop = 1 rdma_connect_timeout = '5s' read_write_rdma_in_event_thread = false read_write_tcp_in_event_thread = false tcp_connect_timeout = '1s' wait_to_retry_send = '100ms' [server.base.groups.io_worker.connect_concurrency_limiter] max_concurrency = 4 [server.base.groups.io_worker.ibsocket] buf_ack_batch = 8 buf_signal_batch = 8 buf_size = 16384 drain_timeout = '5s' drop_connections = 0 event_ack_batch = 128 max_rd_atomic = 16 max_rdma_wr = 128 max_rdma_wr_per_post = 32 max_sge = 16 min_rnr_timer = 1 record_bytes_per_peer = false record_latency_per_peer = false retry_cnt = 7 rnr_retry = 0 send_buf_cnt = 32 sl = 0 start_psn = 0 timeout = 14 [server.base.groups.io_worker.transport_pool] max_connections = 1 [server.base.groups.listener] domain_socket_index = 1 filter_list = [] listen_port = 9001 listen_queue_depth = 4096 rdma_accept_timeout = '15s' rdma_listen_ethernet = true reuse_port = false [server.base.groups.processor] enable_coroutines_pool = true max_coroutines_num = 256 max_processing_requests_num = 4096 response_compression_level = 1 response_compression_threshold = '128KB' [server.fdb] casual_read_risky = false clusterFile = '/opt/3fs/etc/fdb.cluster' default_backoff = 0 enableMultipleClient = false externalClientDir = '' externalClientPath = '' multipleClientThreadNum = 4 readonly = false trace_file = '' trace_format = 'json' [server.kv_engine] use_memkv = false [server.kv_engine.fdb] casual_read_risky = false clusterFile = '' default_backoff = 0 enableMultipleClient = false externalClientDir = '' externalClientPath = '' multipleClientThreadNum = 4 readonly = false trace_file = '' trace_format = 'json' [server.meta] acl_cache_time = '15s' allow_directly_move_to_trash = false allow_owner_change_immutable = false allow_stat_deleted_inodes = true authenticate = false batch_stat_by_path_concurrent = 4 batch_stat_concurrent = 8 check_file_hole = false dynamic_stripe = false dynamic_stripe_growth = 2 dynamic_stripe_initial = 16 enable_new_chunk_engine = false grv_cache = false idempotent_record_clean = '1min' idempotent_record_expire = '30min' idempotent_remove = true idempotent_rename = false iflags_chain_allocation = false iflags_chunk_engine = true ignore_length_hint = false inodeId_abort_on_duplicate = false inodeId_check_unique = true list_default_limit = 128 max_batch_operations = 4096 max_directory_depth = 64 max_remove_chunks_per_request = 32 max_symlink_count = 10 max_symlink_depth = 4 operation_timeout = '5s' otrunc_replace_file = true otrunc_replace_file_threshold = '1GB' readonly = false recursive_remove_check_owner = true recursive_remove_perm_check = 1024 statfs_cache_time = '1min' statfs_space_imbalance_threshold = 5 statfs_update_interval = '5s' sync_on_prune_session = false time_granularity = '1s' [server.meta.background_hole_checker] coroutines_num = 16 enable_work_stealing = false queue_size = 4096 [server.meta.distributor] timeout = '30s' update_interval = '1s' [server.meta.event_trace_log] dump_interval = '30s' enabled = true max_num_writers = 1 max_row_group_length = 100000 trace_file_dir = '.' [server.meta.forward] addr_type = 'RDMA' debug = true timeout = '10s' [server.meta.gc] check_session = true distributed_gc = true enable = true gc_delay_free_space_threshold = 5 gc_directory_concurrent = 4 gc_directory_delay = '0ns' gc_directory_entry_batch = 32 gc_directory_entry_concurrent = 4 gc_file_concurrent = 32 gc_file_delay = '5min' large_file_chunks = 128 recursive_perm_check = true remove_chunks_batch_size = 32 retry_delay = '10min' scan_batch = 4096 scan_interval = '200ms' small_file_chunks = 32 txn_low_priority = false [server.meta.gc.retry_remove_chunks] init_wait_time = '10s' max_retry_time = '30s' max_wait_time = '10s' retry_permanent_error = false [server.meta.gc.workers] coroutines_num = 8 enable_work_stealing = false queue_size = 1024 [server.meta.retry_remove_chunks] init_wait_time = '10s' max_retry_time = '30s' max_wait_time = '10s' retry_permanent_error = false [server.meta.retry_transaction] max_backoff = '1s' max_retry_count = 10 [server.meta.session_manager] enable = true scan_batch = 1024 scan_interval = '5min' session_timeout = '5min' sync_on_prune_session = false [server.meta.session_manager.close_workers] coroutines_num = 32 enable_work_stealing = false queue_size = 1024 [server.meta.session_manager.scan_workers] coroutines_num = 8 enable_work_stealing = false queue_size = 128 [server.meta.user_cache] buckets = 127 exist_ttl = '5min' inexist_ttl = '10s' [server.mgmtd_client] accept_incomplete_routing_info_during_mgmtd_bootstrapping = true auto_extend_client_session_interval = '10s' auto_heartbeat_interval = '10s' auto_refresh_interval = '10s' enable_auto_extend_client_session = false enable_auto_heartbeat = true enable_auto_refresh = true mgmtd_server_addresses = ["RDMA://10.155.10.171:8000", "RDMA://10.153.16.171:8000"] work_queue_size = 100 [server.storage_client] check_overlapping_read_buffers = true check_overlapping_write_buffers = false chunk_checksum_type = 'CRC32C' create_net_client_for_updates = false implementation_type = 'RPC' max_inline_read_bytes = '0' max_inline_write_bytes = '0' max_read_io_bytes = '0' [server.storage_client.net_client] default_compression_level = 0 default_compression_threshold = '128KB' default_log_long_running_threshold = '0ns' default_report_metrics = false default_send_retry_times = 1 default_timeout = '1s' enable_rdma_control = false force_use_tcp = false [server.storage_client.net_client.io_worker] num_event_loop = 1 rdma_connect_timeout = '5s' read_write_rdma_in_event_thread = false read_write_tcp_in_event_thread = false tcp_connect_timeout = '1s' wait_to_retry_send = '100ms' [server.storage_client.net_client.io_worker.connect_concurrency_limiter] max_concurrency = 4 [server.storage_client.net_client.io_worker.ibsocket] buf_ack_batch = 8 buf_signal_batch = 8 buf_size = 16384 drain_timeout = '5s' drop_connections = 0 event_ack_batch = 128 max_rd_atomic = 16 max_rdma_wr = 128 max_rdma_wr_per_post = 32 max_sge = 16 min_rnr_timer = 1 record_bytes_per_peer = false record_latency_per_peer = false retry_cnt = 7 rnr_retry = 0 send_buf_cnt = 32 sl = 0 start_psn = 0 timeout = 14 [server.storage_client.net_client.io_worker.transport_pool] max_connections = 1 [server.storage_client.net_client.processor] enable_coroutines_pool = true max_coroutines_num = 256 max_processing_requests_num = 4096 response_compression_level = 1 response_compression_threshold = '128KB' [server.storage_client.net_client.rdma_control] max_concurrent_transmission = 64 [server.storage_client.net_client.thread_pool] bg_thread_pool_stratetry = 'SHARED_QUEUE' collect_stats = false enable_work_stealing = false io_thread_pool_stratetry = 'SHARED_QUEUE' num_bg_threads = 2 num_connect_threads = 2 num_io_threads = 2 num_proc_threads = 2 proc_thread_pool_stratetry = 'SHARED_QUEUE' [server.storage_client.net_client_for_updates] default_compression_level = 0 default_compression_threshold = '128KB' default_log_long_running_threshold = '0ns' default_report_metrics = false default_send_retry_times = 1 default_timeout = '1s' enable_rdma_control = false force_use_tcp = false [server.storage_client.net_client_for_updates.io_worker] num_event_loop = 1 rdma_connect_timeout = '5s' read_write_rdma_in_event_thread = false read_write_tcp_in_event_thread = false tcp_connect_timeout = '1s' wait_to_retry_send = '100ms' [server.storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter] max_concurrency = 4 [server.storage_client.net_client_for_updates.io_worker.ibsocket] buf_ack_batch = 8 buf_signal_batch = 8 buf_size = 16384 drain_timeout = '5s' drop_connections = 0 event_ack_batch = 128 max_rd_atomic = 16 max_rdma_wr = 128 max_rdma_wr_per_post = 32 max_sge = 16 min_rnr_timer = 1 record_bytes_per_peer = false record_latency_per_peer = false retry_cnt = 7 rnr_retry = 0 send_buf_cnt = 32 sl = 0 start_psn = 0 timeout = 14 [server.storage_client.net_client_for_updates.io_worker.transport_pool] max_connections = 1 [server.storage_client.net_client_for_updates.processor] enable_coroutines_pool = true max_coroutines_num = 256 max_processing_requests_num = 4096 response_compression_level = 1 response_compression_threshold = '128KB' [server.storage_client.net_client_for_updates.rdma_control] max_concurrent_transmission = 64 [server.storage_client.net_client_for_updates.thread_pool] bg_thread_pool_stratetry = 'SHARED_QUEUE' collect_stats = false enable_work_stealing = false io_thread_pool_stratetry = 'SHARED_QUEUE' num_bg_threads = 2 num_connect_threads = 2 num_io_threads = 2 num_proc_threads = 2 proc_thread_pool_stratetry = 'SHARED_QUEUE' [server.storage_client.retry] init_wait_time = '2s' max_failures_before_failover = 1 max_retry_time = '5s' max_wait_time = '5s' [server.storage_client.traffic_control.query] max_batch_bytes = '4MB' max_batch_size = 128 max_concurrent_requests = 32 max_concurrent_requests_per_server = 8 process_batches_in_parallel = true random_shuffle_requests = true [server.storage_client.traffic_control.read] max_batch_bytes = '4MB' max_batch_size = 128 max_concurrent_requests = 32 max_concurrent_requests_per_server = 8 process_batches_in_parallel = true random_shuffle_requests = true [server.storage_client.traffic_control.remove] max_batch_bytes = '4MB' max_batch_size = 128 max_concurrent_requests = 32 max_concurrent_requests_per_server = 8 process_batches_in_parallel = true random_shuffle_requests = true [server.storage_client.traffic_control.truncate] max_batch_bytes = '4MB' max_batch_size = 128 max_concurrent_requests = 32 max_concurrent_requests_per_server = 8 process_batches_in_parallel = true random_shuffle_requests = true [server.storage_client.traffic_control.write] max_batch_bytes = '4MB' max_batch_size = 128 max_concurrent_requests = 32 max_concurrent_requests_per_server = 8 process_batches_in_parallel = true random_shuffle_requests = true ```
bsakaguchi commented 2026-10-26 09:45:39 +00:00
Author
Owner
Copy link

@bugwz commented on GitHub (Mar 26, 2025):

check logs in /var/log/3fs/

<!-- gh-comment-id:2754029472 --> @bugwz commented on GitHub (Mar 26, 2025): check logs in /var/log/3fs/
bsakaguchi commented 2026-10-26 09:45:39 +00:00
Author
Owner
Copy link

@iamazy commented on GitHub (Mar 26, 2025):

check logs in /var/log/3fs/

I have checked logs but made no progress

<!-- gh-comment-id:2754126642 --> @iamazy commented on GitHub (Mar 26, 2025): > check logs in /var/log/3fs/ I have checked logs but made no progress
bsakaguchi commented 2026-10-26 09:45:40 +00:00
Author
Owner
Copy link

@iamazy commented on GitHub (Mar 27, 2025):

when starting meta_main, in mgmtd_main-err.log will reports error

[2025-03-27T18:14:07.834425220+08:00 SvrProc0:3164660 IBConnect.cc:681 ERROR] IBSocket RDMA://2024-fhcsy-psc-p4f1-pod6-pm-os01-zs-bc-nfs-sk1-23/mlx5_bond_0:1/24010 failed to modify QP to RTR, errno 110, peer {"hostname":"2024-fhcsy-psc-p4f1-pod6-pm-os01-zs-bc-nfs-sk1-23","dev":0,"dev_name":"mlx5_bond_0","port":1,"mtu":3,"qp_num":24010,"linklayer":{"type":"IBConnectRoCEInfo","value":{"gid":"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000��\\n�\\u0010�"}}}, config {"sl":0,"traffic_class":0,"gid_index":0,"pkey_index":0,"start_psn":0,"min_rnr_timer":1,"timeout":14,"retry_cnt":7,"rnr_retry":0,"max_sge":16,"max_rdma_wr":128,"max_rdma_wr_per_post":32,"max_rd_atomic":16,"buf_size":16384,"send_buf_cnt":32,"buf_ack_batch":8,"buf_signal_batch":8,"event_ack_batch":128,"record_latency_per_peer":false}
[2025-03-27T18:14:07.834551640+08:00 SvrProc0:3164660 IBConnect.cc:168 ERROR] IBConnectService::connect failed to accept from 10.153.16.171, error RPC::ConnectFailed(2014) IBSocket failed to modify QP to RTR.

How to address error failed to modify QP to RTR. ?

<!-- gh-comment-id:2757530197 --> @iamazy commented on GitHub (Mar 27, 2025): when starting meta_main, in `mgmtd_main-err.log` will reports error ```console [2025-03-27T18:14:07.834425220+08:00 SvrProc0:3164660 IBConnect.cc:681 ERROR] IBSocket RDMA://2024-fhcsy-psc-p4f1-pod6-pm-os01-zs-bc-nfs-sk1-23/mlx5_bond_0:1/24010 failed to modify QP to RTR, errno 110, peer {"hostname":"2024-fhcsy-psc-p4f1-pod6-pm-os01-zs-bc-nfs-sk1-23","dev":0,"dev_name":"mlx5_bond_0","port":1,"mtu":3,"qp_num":24010,"linklayer":{"type":"IBConnectRoCEInfo","value":{"gid":"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000��\\n�\\u0010�"}}}, config {"sl":0,"traffic_class":0,"gid_index":0,"pkey_index":0,"start_psn":0,"min_rnr_timer":1,"timeout":14,"retry_cnt":7,"rnr_retry":0,"max_sge":16,"max_rdma_wr":128,"max_rdma_wr_per_post":32,"max_rd_atomic":16,"buf_size":16384,"send_buf_cnt":32,"buf_ack_batch":8,"buf_signal_batch":8,"event_ack_batch":128,"record_latency_per_peer":false} [2025-03-27T18:14:07.834551640+08:00 SvrProc0:3164660 IBConnect.cc:168 ERROR] IBConnectService::connect failed to accept from 10.153.16.171, error RPC::ConnectFailed(2014) IBSocket failed to modify QP to RTR. ``` How to address error `failed to modify QP to RTR.` ?
bsakaguchi commented 2026-10-26 09:45:40 +00:00
Author
Owner
Copy link

@xlcbingo1999 commented on GitHub (Aug 8, 2025):

请问是否有解决方案呢？我也在容器部署的时候遇到问题

<!-- gh-comment-id:3167594278 --> @xlcbingo1999 commented on GitHub (Aug 8, 2025): 请问是否有解决方案呢？我也在容器部署的时候遇到问题
Sign in to join this conversation.
No Branch/Tag specified
Branches Tags
main
SF-Zhou-patch-1
No results found.
Labels
Clear labels   
pull-request
Mirrored from GitHub Pull Request

No labels
pull-request
Milestone
Clear milestone
No items
No milestone
Projects
Clear projects
No items
No project
Assignees
Clear assignees
No assignees
1 participant
Notifications
Due date
The due date is invalid or out of range. Please use the format "yyyy-mm-dd".

No due date set.

Dependencies

No dependencies set.

Reference
deepseek-ai/3FS#127
No description provided.