# This file starts a full-node (i.e. both CPUs) production server with full CHORD parameters.
# Files are written through local SSD cache to a real NFS server (/mnt/cs00/data).
#
# To start server on cf05:
# pirate_frb run_server configs/frb_server/cf05_production.yml configs/dedispersion/chord_sb2_et.yml
#
# To send data from cf00 to cf05:
# pirate_frb run_fake_xengine 10.222.3.5:6000 10.222.3.5:6001
######################################## General config ########################################
# Under the hood, the frb search may consist of multiple servers (class FrbServer in the code).
# Each server processes a disjoint set of beams, and receives data from a disjoint set of TCP
# connections. In a multi-CPU machine, each server "lives" on a specific CPU. The mapping between
# servers in the config file, and physical CPUs, is determined by 'server_cpus'. (A many-to-one
# mapping is allowed, i.e. multiple servers per CPU.)
num_servers: 2
server_cpus: [ 0, 1 ] # length num_servers
# For now, the config file must declare upper bounds on the amount of memory needed
# for dedispersion and ring buffering. This is a hack, and I'll implement better
# memory management in the future!
rb_host_memory_per_server: '256 GB'
dd_host_memory_per_server: '420 GB'
gpu_memory_per_server: '40 GB'
use_hugepages: true
# Logical length of ring buffer (in time "chunks", not time+beam "frames").
# In principle, this is an independent parameter from 'memory_per_server'.
# However, if memory is a limiting factor, then the logical ring buffer will
# be partially populated by "reaped" chunks, which are only useful in corner
# cases (duplicate write -> copy or hardlink).
ringbuf_nchunks: 512
######################################## Network config ########################################
# 'data_ip_addrs' is a list (length num_servers). Each entry is either an 'ip:tcp_port'
# string, or a list of such strings. The latter case arises if a server receives data from
# multiple IP addresses. We expect this to happen in full CHORD. (Under the hood, each IP
# address corresponds to one 'class Receiver', and an FrbServer can have multiple Receivers.)
#
# The 'ip' part of each 'ip:tcp_port' entry may be written in any of three ways, so that ONE
# config file can be shared across a cluster of machines with different IP addresses:
#
# - a literal IPv4 address e.g. '10.0.0.2:5000'
# - a glob matched against this machine's IPv4 addresses e.g. '10.0.0.*:5000'
# - a network device / NIC name e.g. 'enp13s0f0np0:5000'
data_ip_addrs:
- [ '10.0.0.*:5000', '10.0.1.*:5000' ]
- [ '10.0.2.*:5000', '10.0.3.*:5000' ]
# 'rpc_ip_addrs' is a list of 'ip:tcp_port' strings (one per server). The 'ip' accepts the
# same literal / glob / device-name forms as data_ip_addrs above (see resolve_ip_spec()).
# Currently, RPC threads are not pinned to a specific CPU. This is fine for now, since all
# RPCs are "lightweight". We may revisit in the future, when we define "heavyweight" RPCs
# for e.g. injections. See discussion in include/pirate/FrbServer.hpp.
rpc_ip_addrs:
- '10.222.3.*:6000'
- '10.222.3.*:6001'
# FrbGrouper addresses (one per server). The FrbServer is the gRPC *client*;
# the grouper (downstream consumer) is the server. Must be loopback (CUDA IPC
# requires same node/GPU). Run with 'run_server -G' to disable.
grouper_ip_addrs:
- '127.0.0.1:7000'
- '127.0.0.1:7001'
# The FRB server (and fake X-engine) will throw exceptions if the MTUs are below these thresholds.
# Intended as a defensive check against forgetting to enable jumbo MTUs.
# Note: only enforced in python API, not low-level C++ classes.
min_data_mtu: 9000
min_rpc_mtu: 1500
##################################### File-writing config ######################################
# 'check_mountpoints' is intended to catch a misconfiguration where a file system isn't mounted.
# If any of the listed directories aren't mountpoints, then an exception is raised.
check_mountpoints: [ '/scratch2', '/mnt/cs00/data' ]
# 'ssd_dirs' and 'ssd_devices' are lists of length num_servers. If an ssd_dir isn't on the
# correct numa domain, then an exception is raised. (If this proves to be too restrictive
# in the future, then I might introduce a config flag to relax it.)
#
# Note: on the chord frb nodes,
# /scratch2 is the mountpoint of an SSD on CPU0
# /scratch is a subdirectory of /, which is an SSD on CPU1 (/scratch is not a mountpoint)
ssd_dirs: [ '/scratch2', '/scratch' ]
ssd_threads_per_server: 4
# Currently, all servers write to the same nfs_dir (i.e. 'nfs_dir' is a string,
# not a list of strings). The 'nfs_dir' member supports string interpolations
# {user} and {date}. The nfs_dir will be created if it doesn't already exist.
nfs_dir: '/mnt/cs00/data/{user}/{date}'
nfs_threads_per_server: 2