├── readme.md ├── .gitignore ├── alveo-unused └── init-alveo.tbs ├── alveo-katcp-svr ├── Makefile ├── requirements.txt ├── fpgparser.py ├── AlveoSensors.py ├── alveo-katcp-svr.py └── AlveoIO.py ├── alveo-linux-env ├── modprobe-configs │ └── xdma.conf ├── xdma-udev-rules │ ├── alveo_namer.sh │ └── 60-xdma.rules.example ├── xdma-dkms │ └── dkms.conf └── systemd-services │ ├── tcpbs.service │ ├── install-xdma.sh │ ├── alveo-katcp-svr.service │ └── xdma-install-check.service ├── CLONE.txt ├── alveo-utils ├── alveo-map ├── alveo-stop-tcpbs ├── alveo-jtag-dna.tcl ├── alveo-serial ├── alveo-launch-katcp ├── alveo-pci ├── alveo-jtag-list └── alveo-launch-tcpbs ├── alveo-program ├── hwserver-log-rotate.sh ├── alveo-program.sh └── alveo-program.tcl ├── .gitmodules ├── alveo-cmds └── alveo-reset-sh ├── alveo-tcpbs-fs ├── alveo-memwrite ├── alveo-memread ├── alveo-program └── alveo-reset ├── configure ├── alveo-sw-docs ├── readme.md └── troubleshooting.md └── Makefile /readme.md: -------------------------------------------------------------------------------- 1 | alveo-sw-docs/readme.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | alveo-linux-env/xdma-udev-rules/60-xdma.rules 2 | -------------------------------------------------------------------------------- /alveo-unused/init-alveo.tbs: -------------------------------------------------------------------------------- 1 | ?process alveo-reset reset\_alveo 2 | -------------------------------------------------------------------------------- /alveo-katcp-svr/Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | pex -r requirements.txt -v -o alveo_env.pex 3 | -------------------------------------------------------------------------------- /alveo-katcp-svr/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/ska-sa/aiokatcp@master#egg=aiokatcp 2 | numpy 3 | 4 | -------------------------------------------------------------------------------- /alveo-linux-env/modprobe-configs/xdma.conf: -------------------------------------------------------------------------------- 1 | #RvW, SARAO, 2022 2 | #load the xdma kernel module with given options 3 | options xdma poll_mode=1 4 | -------------------------------------------------------------------------------- /alveo-linux-env/xdma-udev-rules/alveo_namer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | #removes the prefix xdma*_ 6 | echo ${1#*xdma[0-9]*_} 7 | -------------------------------------------------------------------------------- /CLONE.txt: -------------------------------------------------------------------------------- 1 | This repo contains git submodules and thus an extra 2 | few step are required to "pull" these repos in after 3 | cloning: 4 | 5 | $ git submodule init 6 | $ git submodule update 7 | -------------------------------------------------------------------------------- /alveo-utils/alveo-map: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | ALVEOS=$(./alveo-pci | grep PCI | cut -d'@' -f 2) 6 | 7 | for DEVICE in $ALVEOS; do 8 | SER=$(./alveo-serial /dev/$(echo ${DEVICE} | cut -d'/' -f2)) 9 | echo "${DEVICE}/${SER}" 10 | done 11 | 12 | -------------------------------------------------------------------------------- /alveo-linux-env/xdma-dkms/dkms.conf: -------------------------------------------------------------------------------- 1 | #see dkms.conf in /usr/srx/xrt- for example 2 | PACKAGE_NAME="xdma" 3 | PACKAGE_VERSION="2020.2.2" 4 | MAKE[0]="cd xdma; make; cd .." 5 | CLEAN[0]="cd xdma; make clean; cd .." 6 | BUILT_MODULE_NAME[0]="xdma" 7 | BUILT_MODULE_LOCATION[0]="xdma/" 8 | DEST_MODULE_LOCATION[0]="/kernel/extras" 9 | AUTOINSTALL="yes" 10 | -------------------------------------------------------------------------------- /alveo-program/hwserver-log-rotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | set -e 6 | 7 | #$1 jtag serial 8 | #$2 port 9 | 10 | test -n "$1" 11 | test -n "$2" 12 | 13 | list=$(ls -Art1 hwserver-$1-$2* 2>/dev/null) 14 | 15 | count=$(echo ${list} | tr ' ' '\n' | wc -l) 16 | 17 | if test ${count} -ge 5; then 18 | rm $(echo ${list} | tr ' ' '\n' | head -1) 19 | fi 20 | -------------------------------------------------------------------------------- /alveo-linux-env/systemd-services/tcpbs.service: -------------------------------------------------------------------------------- 1 | #RvW, 2022, SARAO 2 | 3 | [Unit] 4 | Description=Launch tcpborphserver instances for all detected Alveo cards (with SARAO bsp running). 5 | 6 | [Service] 7 | #forking type NB here so the tcpbs process doesn't go away 8 | Type=forking 9 | ExecStart=/opt/alveo/alveo-utils/alveo-launch-tcpbs 10 | ExecStop=/opt/alveo/alveo-utils/alveo-stop-tcpbs 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "katcp"] 2 | path = katcp 3 | url = https://github.com/ska-sa/katcp.git 4 | [submodule "dma_ip_drivers"] 5 | path = dma_ip_drivers 6 | url = https://github.com/Xilinx/dma_ip_drivers 7 | [submodule "pcimem"] 8 | path = pcimem 9 | url = https://github.com/billfarrow/pcimem.git 10 | [submodule "alveo_sensmon"] 11 | path = alveo_sensmon 12 | url = https://github.com/mchirindo/alveo_sensmon.git 13 | -------------------------------------------------------------------------------- /alveo-linux-env/systemd-services/install-xdma.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #RvW, SARAO,2022 4 | 5 | set -e 6 | 7 | test ! -e /lib/modules/$(uname -r)/extra/xdma.ko || { echo "$(date) xdma kernel module already exists" && exit 1; } 8 | 9 | echo "$(date) installing xdma kernel module" 10 | 11 | cd $(dirname "$0")/xdma 12 | 13 | make install 14 | 15 | depmod -a 16 | 17 | insmod /lib/modules/$(uname -r)/extra/xdma.ko 18 | 19 | make clean 20 | 21 | cd - 22 | -------------------------------------------------------------------------------- /alveo-linux-env/systemd-services/alveo-katcp-svr.service: -------------------------------------------------------------------------------- 1 | #RvW, 2023, SARAO 2 | 3 | [Unit] 4 | Description=Launch alveo-katcp-svr instances for all detected Alveo cards (with SARAO bsp running). 5 | After=multi-user.target 6 | 7 | [Service] 8 | #forking type NB here so the tcpbs process doesn't go away 9 | Type=forking 10 | ExecStart=/opt/alveo/alveo-utils/alveo-launch-katcp 11 | ExecStop=/opt/alveo/alveo-utils/alveo-stop-katcp 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /alveo-utils/alveo-stop-tcpbs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | PIDS=$(ps fax | grep [t]cpborphserver3 | xargs | cut -d' ' -f1) 6 | 7 | for P in ${PIDS}; do 8 | ARGS=$(echo $(ps -o args -p ${P}) ) 9 | 10 | #TODO the line below has a potential bug - if the process 11 | #was started **without** a space before the arg, this may lead to issue with cut's space delimiter 12 | #check this i.e. there will be no space e.g. -p7150 13 | 14 | PORT=$(echo ${ARGS#*"-p"} | tr -s ' '| cut -d ' ' -f1) 15 | kcpcmd -s :${PORT} halt 16 | done 17 | -------------------------------------------------------------------------------- /alveo-utils/alveo-jtag-dna.tcl: -------------------------------------------------------------------------------- 1 | set jtag_serial_number [lindex $argv 0] 2 | set port_number [lindex $argv 1] 3 | 4 | append url localhost: $port_number /xilinx_tcf/Xilinx/ $jtag_serial_number A 5 | puts $url 6 | 7 | 8 | open_hw_manager 9 | exec hw_server -d -s tcp:localhost:$port_number -p0 -I2 10 | 11 | connect_hw_server -url localhost:$port_number 12 | 13 | open_hw_target $url 14 | current_hw_device [lindex [get_hw_devices] 0] 15 | refresh_hw_device -update_hw_probes false [current_hw_device] 16 | 17 | append output "DNA read " [get_property REGISTER.EFUSE.FUSE_DNA [lindex [get_hw_device] 0]] 18 | puts $output 19 | -------------------------------------------------------------------------------- /alveo-linux-env/systemd-services/xdma-install-check.service: -------------------------------------------------------------------------------- 1 | #RvW, SARAO, 2022 2 | 3 | #This service runs once at startup to ensure that the xdma 4 | #kernel module is installed. This is particularly of use 5 | #after the kernel updates since the xdma driver is an 6 | #out-of-tree module 7 | 8 | [Unit] 9 | Description=Check that the xdma kernel module is installed 10 | ConditionPathExists=/usr/src/xdma-2020.2.2 11 | 12 | [Service] 13 | Type=oneshot 14 | ExecStart=/usr/src/xdma-2020.2.2/install-xdma.sh 15 | TimeoutSec=0 16 | #if next line commented out, service output is written to syslog 17 | #StandardOutput=tty 18 | #RemainAfterExit=yes 19 | #SysVStartPriority=99 20 | 21 | [Install] 22 | WantedBy=multi-user.target 23 | -------------------------------------------------------------------------------- /alveo-program/alveo-program.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | # 3 | # Created on: 12 April 2022 4 | # Author: Mathews Chirindo 5 | # Modified: RvW 6 | 7 | # This script will start vivado in tcl mode and run a tcl script to program the alveo card. 8 | # Arguments: 9 | # $1 - the PCI address of the alveo card (use 'lspci -D -d 10ee: -vv' to get this) 10 | # $2 - the .bit file to upload to the alveo 11 | 12 | PCITAG=${1//[:.]/-} #replace all ':' and '.' with '-' 13 | logfile=vivado-${PCITAG}.log 14 | 15 | kcpmsg -s alveo-program "about to launch vivado - see $(pwd)/${logfile} for issues" 16 | 17 | /opt/Xilinx/Vivado/2021.1/bin/vivado -log ${logfile} -nojournal -mode batch -source alveo-program.tcl -tclargs $1 $2 18 | 19 | RET=$? 20 | 21 | kcpmsg -s alveo-program "vivado exited with code ${RET}" 22 | exit ${RET} 23 | -------------------------------------------------------------------------------- /alveo-cmds/alveo-reset-sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | #utility to be run from bash cmd line, as opposed 6 | #to alveo-reset which is to be run from tcpbs as 7 | #a registered process 8 | 9 | function print_usage_exit() { 10 | echo "usage: ${0} [--help] [--pci ADDR]" 11 | exit 12 | } 13 | 14 | pci_addr="" 15 | 16 | #parse cmd line args 17 | while test ! -z $1; do 18 | case $1 in 19 | "--pci") 20 | shift 21 | test -z $1 && print_usage_exit 22 | pci_addr=$1 23 | ;; 24 | *) 25 | print_usage_exit 26 | ;; 27 | 28 | esac 29 | shift 30 | done 31 | 32 | 33 | if test -z $pci_addr; then 34 | print_usage_exit 35 | fi 36 | 37 | test -e /sys/bus/pci/devices/${pci_addr} || echo "no pcie at ${pci_addr}" 38 | 39 | #test -e /sys/bus/pci/devices/0000:0f:00.0/remove && echo 1 > /sys/bus/pci/devices/0000:0f:00.0/remove && sleep 1 && echo 1 > /sys/bus/pci/rescan 40 | test -e /sys/bus/pci/devices/${pci_addr}/remove && echo 1 > /sys/bus/pci/devices/${pci_addr}/remove && sleep 1 && echo 1 > /sys/bus/pci/rescan 41 | -------------------------------------------------------------------------------- /alveo-tcpbs-fs/alveo-memwrite: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | #the inform must have the same name as the process for tcpbs to "catch" it 6 | 7 | set -x 8 | set -e 9 | 10 | HEX1=$(echo $1 | cut -f2 -d'x') 11 | HEX2=$(echo $2 | cut -f2 -d'x') 12 | 13 | #check if arg 1 is a hex value 14 | #let 16#${HEX} 2>/dev/null || exit -3 15 | test ${HEX} -eq 0 2>/dev/null || let 16#${HEX1} 2>/dev/null 16 | 17 | ##check that arg 2 is an integer 18 | #test -n "$2" 19 | #test $2 -eq $2 2>/dev/null 20 | 21 | #check if arg 2 is a hex value 22 | test ${HEX} -eq 0 2>/dev/null || let 16#${HEX2} 2>/dev/null 23 | 24 | PARENT_PID=$(ps -fp $PPID | awk "/$PPID/"' { print $2 } ') 25 | #PARENT_PID=10823 26 | 27 | ARGS=$(ps -o args -p ${PARENT_PID}) 28 | DEVICE=$(echo ${ARGS#*"-d"} | xargs | cut -d ' ' -f1) 29 | 30 | 31 | OUTPUT=$(pcimem ${DEVICE} $1 w $2 2>/dev/null) 32 | 33 | DATA=$(echo ${OUTPUT} | awk 'END{print}' | tr -d ' ' | cut -f2 -d';') 34 | 35 | DATA=${DATA/readback/} 36 | echo $DATA 37 | 38 | test -n "${DATA}" 39 | test "${DATA^^}" = "${2^^}" 40 | 41 | 42 | echo "#alveo-memwrite $1 $2" 43 | exit 0 44 | 45 | -------------------------------------------------------------------------------- /alveo-tcpbs-fs/alveo-memread: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | #the inform must have the same name as the process for tcpbs to "catch" it 5 | 6 | #set -v 7 | set -e 8 | set -x 9 | 10 | HEX=$(echo $1 | cut -f2 -d'x') 11 | 12 | #check if arg 1 is a hex value 13 | #let 16#${HEX} 2>/dev/null || exit -3 14 | test ${HEX} -eq 0 2>/dev/null || let 16#${HEX} 2>/dev/null 15 | 16 | #first have to identify which tcpborphserver called this funtion 17 | #because we need to get the associated pci address 18 | 19 | PARENT_PID=$(ps -fp $PPID | awk "/$PPID/"' { print $2 } ') 20 | #PARENT_PID=10823 21 | #kcpmsg "${PARENT_PID}" 22 | 23 | ARGS=$(ps -o args -p ${PARENT_PID}) 24 | #kcpmsg "${ARGS}" 25 | 26 | #TODO the line below has a potential bug - if the process 27 | #was started without a space before the arg, this may lead to issue with cut's space delimiter 28 | #check this i.e. there will be no space e.g. -d/dev/... 29 | DEVICE=$(echo ${ARGS#*"-d"} | xargs | cut -d ' ' -f1) 30 | 31 | #kcpmsg "${DEVICE}" 32 | 33 | 34 | 35 | OUTPUT=$(pcimem ${DEVICE} $1 w 2>/dev/null) 36 | 37 | DATA=$(echo ${OUTPUT} | awk 'END{print}' | tr -d ' ' | cut -f2 -d':') 38 | 39 | test -n "${DATA}" || exit -1 40 | 41 | echo "#alveo-memread $1 ${DATA}" 42 | exit 0 43 | 44 | -------------------------------------------------------------------------------- /alveo-tcpbs-fs/alveo-program: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | set -e 6 | 7 | #first have to identify which tcpborphserver called this funtion 8 | #because we need to get the associated pci address 9 | 10 | #take my PPID and search for it's PPID = grandparent pid 11 | GRANDPARENT_PID=$(ps -fp $PPID | awk "/$PPID/"' { print $3 } ') 12 | ARGS=$(ps -o args -p ${GRANDPARENT_PID}) 13 | #TODO the line below has a potential bug - if the process 14 | #was started without a space before the arg, this may lead to issue with cut's space delimiter 15 | #check this i.e. there will be no space e.g. -d/dev/... 16 | DEVICE=$(echo ${ARGS#*"-d"} | xargs | cut -d ' ' -f1) 17 | 18 | kcpmsg ${DEVICE} 19 | 20 | #remove substring "user" to get path 21 | DEVPATH=${DEVICE%"user"} 22 | 23 | PCIPATH=$(readlink -f ${DEVPATH}/pci) 24 | 25 | PCIADDR=$(echo ${PCIPATH} | rev | cut -d '/' -f1 | rev) 26 | 27 | PCITAG=${PCIADDR//[:.]/-} 28 | #assuming file visible with ?listbof 29 | BITFILE="$1" 30 | if test -z ${BITFILE}; then 31 | BITFILE="tcpborphserver-${PCITAG}.bit"; 32 | BITPATH="/lib/firmware" 33 | else 34 | BITPATH="/opt/alveo/alveo-tcpbs-fs" 35 | fi 36 | 37 | kcpmsg "about to program alveo at ${PCIADDR} with ${BITFILE} in ${BITPATH}" 38 | 39 | cd /opt/alveo/alveo-program/ 40 | 41 | ./alveo-program.sh ${PCIADDR} ${BITPATH}/${BITFILE} 42 | 43 | cd - 44 | -------------------------------------------------------------------------------- /alveo-utils/alveo-serial: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | #PCIMEM_PATH="../pcimem/" 6 | 7 | #check for an argument 8 | test -z $1 && echo "need a device" 1>&2 && exit 9 | 10 | #check that the arg exists and is an xdma char device 11 | test ! -c $1 && echo "need a valid char device" 1>&2 && exit 12 | 13 | #TODO crude implementation for now, should probably do more 14 | #sanity and error checking 15 | 16 | #check out the CMS app note... 17 | 18 | #TODO doing blind writes here not the best, first have to make sure that 19 | #the memory offset is indeed correct 20 | 21 | #ensure CMS subsystem out of reset first 22 | ${PCIMEM_PATH}pcimem ${1} 0x20000 w 1 > /dev/null 2>&1 23 | sleep 0.5 24 | 25 | ${PCIMEM_PATH}pcimem ${1} 0x29000 w 0x04000000 > /dev/null 2>&1 26 | sleep 0.5 27 | ${PCIMEM_PATH}pcimem ${1} 0x28018 w 0x20 > /dev/null 2>&1 28 | sleep 0.5 29 | 30 | DATA=$(I=10; ${PCIMEM_PATH}pcimem ${1} 0x29004 w*${I} | tail -n ${I} | cut -d' ' -f 2 | cut -d'x' -f2 | sed 's/.\{2\}/& /g' | awk -F' ' '{print $4" "$3" " $2" "$1}' | tr '\n' ' ') 31 | 32 | #get everything after serial number identifier "21 0d" 33 | SUBSTRING=${DATA#*"21 0D"} 34 | 35 | #read the next 12 octets and convert to ascii from hex 36 | index=0; 37 | for oct in ${SUBSTRING}; do 38 | echo $oct | xxd -r -p; 39 | index=$((index+1)); 40 | if [[ $index -eq 12 ]]; then 41 | break; 42 | fi; 43 | done; 44 | 45 | echo 46 | -------------------------------------------------------------------------------- /alveo-tcpbs-fs/alveo-reset: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | #NOTE: cannot call this utility 'reset', that already exists as a bash utility 3 | 4 | #RvW, SARAO, 2022 5 | 6 | set -e 7 | 8 | #first have to identify which tcpborphserver called this funtion 9 | #because we need to get the associated pci address 10 | 11 | #take my PPID and search for it's PPID = grandparent pid 12 | GRANDPARENT_PID=$(ps -fp $PPID | awk "/$PPID/"' { print $3 } ') 13 | ARGS=$(ps -o args -p ${GRANDPARENT_PID}) 14 | #TODO the line below has a potential bug - if the process 15 | #was started without a space before the arg, this may lead to issue with cut's space delimiter 16 | #check this i.e. there will be no space e.g. -d/dev/... 17 | DEVICE=$(echo ${ARGS#*"-d"} | xargs | cut -d ' ' -f1) 18 | 19 | kcpmsg ${DEVICE} 20 | 21 | #remove substring "user" to get path 22 | PCIPATH=${DEVICE%"user"} 23 | 24 | kcpmsg "about to reset alveo at ${PCIPATH}" 25 | 26 | PORT=$(echo ${ARGS#*"-p"} | xargs | cut -d ' ' -f1) 27 | 28 | ##need to unmap the fpga - simply issue empty ?progdev 29 | ##kcpcmd -q -s :${PORT} progdev 30 | ## TODO REMOVE ABOVE - DON'T unmap - this reset is usually issued 31 | # after the fpga has been programmed, unampping would not make sense here 32 | 33 | test -e ${PCIPATH}/pci/remove && echo 1 > ${PCIPATH}/pci/remove && sleep 1 && echo 1 > /sys/bus/pci/rescan 34 | 35 | #wait after reset 36 | sleep 0.5 37 | 38 | SERIAL=$(/opt/alveo/alveo-utils/alveo-serial ${DEVICE}) 39 | kcpmsg "alveo serial ${SERIAL}" 40 | 41 | #update the serial number since it won't be available on first tcpbs launch (cms not up yet) 42 | kcpcmd -q -s :${PORT} version add alveo-serial ${SERIAL} 43 | 44 | 45 | #exit 0 46 | #do not exit with 0, this allows failed test -e to push 1 back to caller 47 | -------------------------------------------------------------------------------- /alveo-utils/alveo-launch-katcp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | set -e 6 | 7 | echo "WARNING: starting with root permissions!!" 8 | 9 | BASE_PORT=7150 10 | 11 | ALVEOS=$(ls /dev/xdma/) 12 | 13 | PORT=${BASE_PORT} 14 | 15 | for CARD in ${ALVEOS}; do 16 | SERIAL=$(/opt/alveo/alveo-utils/alveo-serial /dev/xdma/${CARD}/user 2>/dev/null) 17 | echo $CARD $SERIAL 18 | 19 | PCIPATH=$(readlink -f /dev/xdma/${CARD}/pci) 20 | PCIADDR=$(echo ${PCIPATH} | rev | cut -d '/' -f1 | rev) 21 | PCITAG=${PCIADDR//[:.]/-} #replace all ':' and '.' with '-' 22 | 23 | BOF_DIR="/tmp/alveo-katcp-svr-fs-${PCITAG}/" 24 | if ! test -d ${BOF_DIR}; then 25 | mkdir ${BOF_DIR} 26 | fi 27 | 28 | #check if this tcpbs already running, since with each xdma.ko load, this script will be run 29 | STAT=0 30 | ps fax | grep [a]lveo-katcp-svr | grep ${PORT} &> /dev/null || STAT=1 #the || is so the process can continue (set -e) 31 | 32 | if [ ${STAT} -ne 1 ] 33 | then 34 | echo "alveo-katcp-svr: instance already running for ${CARD} with serial ${SERIAL} at port ${PORT}" 35 | else 36 | cd /opt/alveo/alveo-katcp-svr/ 37 | env PATH=${PATH}:/opt/alveo/alveo-katcp-svr/ \ 38 | /opt/alveo/alveo-katcp-svr/alveo_env.pex /opt/alveo/alveo-katcp-svr/alveo-katcp-svr.py --alveo ${CARD} \ 39 | --workdir ${BOF_DIR} \ 40 | --port ${PORT} \ 41 | --card ${CARD}-${SERIAL} \ 42 | -d 43 | cd - 44 | 45 | 46 | #TODO should we wait a bit here? 47 | sleep 0.5 48 | fi 49 | PORT=$((PORT+2)) 50 | done 51 | -------------------------------------------------------------------------------- /alveo-linux-env/xdma-udev-rules/60-xdma.rules.example: -------------------------------------------------------------------------------- 1 | #RvW, 2022, SARAO 2 | 3 | # arg %k is the kernel given name for the device file 4 | #works KERNEL=="xdma[0-9]*", KERNELS=="0000:0f:00.0", PROGRAM="/etc/udev/rules.d/alveo_namer.sh %k", RUN+="/bin/ln -f -s /sys/bus/pci/devices/0000:0f:00.0/ /dev/xdma/u50_0/pci", SYMLINK+="xdma/u50_0/%c", MODE="0666" 5 | #calling exe not working 6 | #KERNEL=="xdma[0-9]*", KERNELS=="0000:0f:00.0", PROGRAM="/etc/udev/rules.d/alveo_namer.sh %k", RUN+="/etc/udev/rules.d/alveo_dev_sym.sh u50_0 0000\:0f\:00.0", SYMLINK+="xdma/u50_0/%c", MODE="0666" 7 | 8 | KERNEL=="xdma[0-9]*", KERNELS=="0000:0f:00.0", PROGRAM="/etc/udev/rules.d/alveo_namer.sh %k", SYMLINK+="xdma/u50_0/%c", MODE="0666" 9 | #only need to do the following actions one, so only perform on xdma*_user invocation 10 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0f:00.0", ACTION=="add", RUN+="/bin/ln -f -s /sys/bus/pci/devices/0000:0f:00.0/ /dev/xdma/u50_0/pci" 11 | #can only have one program per 'RUN' directive, either need "lazy evaluation" ie '&&' or another separate 'RUN' 12 | #KERNEL=="xdma[0-9]_user", KERNELS=="0000:0f:00.0", ACTION=="add", RUN+="/usr/bin/sudo systemctl start tcpbs.service" 13 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0f:00.0", ACTION=="remove", RUN+="/bin/rm /dev/xdma/u50_0/pci" 14 | 15 | 16 | KERNEL=="xdma[0-9]*", KERNELS=="0000:0d:00.0", PROGRAM="/bin/bash /etc/udev/rules.d/alveo_namer.sh %k", SYMLINK+="xdma/u50_1/%c", MODE="0666" 17 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0d:00.0", ACTION=="add", RUN+="/bin/ln -f -s /sys/bus/pci/devices/0000:0d:00.0/ /dev/xdma/u50_1/pci" 18 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0d:00.0", ACTION=="remove", RUN+="/bin/rm /dev/xdma/u50_1/pci" 19 | 20 | 21 | KERNEL=="xdma[0-9]*", KERNELS=="0000:0e:00.0", PROGRAM="/bin/bash /etc/udev/rules.d/alveo_namer.sh %k", SYMLINK+="xdma/u50_2/%c", MODE="0666" 22 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0e:00.0", ACTION=="add", RUN+="/bin/ln -f -s /sys/bus/pci/devices/0000:0e:00.0/ /dev/xdma/u50_2/pci" 23 | KERNEL=="xdma[0-9]_user", KERNELS=="0000:0e:00.0", ACTION=="remove", RUN+="/bin/rm /dev/xdma/u50_2/pci" 24 | 25 | #TODO remove broken links after alveos are removed, xdma driver unloaded 26 | #TODO find a way to dynamically pass in the pci addr (?) 27 | -------------------------------------------------------------------------------- /alveo-utils/alveo-pci: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | 6 | function print_usage_exit() { 7 | echo "usage: ${0} [--help] [--dev ]" 8 | exit 9 | } 10 | 11 | 12 | 13 | #parse cmd line args 14 | while test ! -z $1; do 15 | case $1 in 16 | # "--force") 17 | # force=1 18 | # ;; 19 | "--dev") 20 | shift 21 | test -z $1 && print_usage_exit 22 | test $1 = "xdma" || test $1 = "xrt" || test $1 = "all" || print_usage_exit 23 | #xdma => custom builds using the xdma pcie device 24 | #xrt => alveos running the Xilinx XRT shell 25 | #all => find both xdma and xrt shell 26 | dev_opt=$1 27 | ;; 28 | *) 29 | print_usage_exit 30 | ;; 31 | 32 | esac 33 | shift 34 | done 35 | 36 | 37 | #if unset, set to "xdma" -> ie when no arg set on cmd line 38 | dev_opt=${dev_opt:-"xdma"} 39 | 40 | if test $dev_opt = "xdma"; then 41 | SRC_DEV="xdma.*user" 42 | elif test $dev_opt = "xrt"; then 43 | SRC_DEV="xclmgmt" 44 | elif test $dev_opt = "all"; then 45 | SRC_DEV="xdma.*user|xclmgmt" 46 | fi 47 | 48 | 49 | 50 | DEVS=$(ls -l /sys/dev/char/ | grep -E ${SRC_DEV} | rev | cut -f1,3 -d'/' | rev) 51 | 52 | 53 | #for i in ${DEVS}; do 54 | # PCI_SLOT_ADDR=$(echo $i | cut -f1 -d'/') 55 | # MEMSIZE=$(( $(cat /sys/bus/pci/devices/${PCI_SLOT_ADDR}/resource | head -n1 | cut -f1,2 -d' ' | awk '{ print $2 " " $1}' | tr ' ' '-') + 1)) 56 | # MEMSIZE_HUMANREAD=$(numfmt --to=iec-i --suffix=B ${MEMSIZE}) 57 | # printf "PCI Device @ %s\n" ${i} 58 | # printf "Region0 mem-size %s\n" ${MEMSIZE_HUMANREAD} 59 | #done 60 | 61 | 62 | for i in ${DEVS}; do 63 | INDEX=0 64 | printf "PCI Device @ %s\n" ${i} 65 | PCI_SLOT_ADDR=$(echo $i | cut -f1 -d'/') 66 | printf "ID %s\n" $(cat /sys/bus/pci/devices/${PCI_SLOT_ADDR}/device) 67 | #read in resource file line-by-line: 68 | while read -r line; do 69 | #echo "$line"; 70 | MEMSIZE=$(( $(echo "${line}" | cut -f1,2 -d' ' | awk '{ print $2 " " $1}' | tr ' ' '-') )); 71 | if test ${MEMSIZE} -ne 0; then 72 | MEMSIZE_HUMANREAD=$(numfmt --to=iec-i --suffix=B ${MEMSIZE}); 73 | printf "Region%d mem-size %s\n" ${INDEX} ${MEMSIZE_HUMANREAD}; 74 | fi 75 | INDEX=$((INDEX + 1)); 76 | done < /sys/bus/pci/devices/${PCI_SLOT_ADDR}/resource 77 | done 78 | -------------------------------------------------------------------------------- /alveo-utils/alveo-jtag-list: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #JTAG=$(lsusb -d 0403:6011 -v 2>/dev/null | grep iSerial | tr -s ' ' | rev | cut -f1 -d' ' | rev) 4 | 5 | #find an open port 6 | #read lower upper < /proc/sys/net/ipv4/ip_local_port_range 7 | # 8 | #while :; do 9 | # for (( port = lower ; port <= upper ; port++ )); do 10 | # lsof -i:${port} || break 2 11 | # done 12 | #done 13 | # 14 | 15 | #JTAG="500202A3099A" 16 | port=3170 17 | 18 | echo "Reading DNA via JTAG...please wait, this may take a few minutes" 19 | 20 | JTAG=$(lsusb -d 0403: -v | grep -i serial | rev | cut -d' ' -f1 | rev) 21 | 22 | #echo ${port} 23 | 24 | declare -a dna_jtag 25 | declare -a serial_jtag 26 | declare -a port_arr 27 | index=0 28 | 29 | for j in ${JTAG}; do 30 | #echo ${port} 31 | #echo ${j} 32 | serial_jtag[$index]=${j} 33 | dna=$(/opt/Xilinx/Vivado/2021.1/bin/vivado -mode batch -source alveo-jtag-dna.tcl -tclargs $j $port | grep "^DNA read") 34 | echo JTAG ${j} PORT ${port} ${dna} 35 | dna_jtag[$index]=$(echo ${dna: -8}) 36 | port_arr[$index]=${port} 37 | port=$((port+10)) 38 | index=$((index+1)) 39 | done 40 | 41 | declare -a dna_pci 42 | declare -a pci 43 | index=0 44 | 45 | for i in $(ls /dev/xdma/); do 46 | #u=$(ls -l /dev/xdma/$i | grep -o xdma.*user) 47 | pci[$index]=$(ls -l /dev/xdma/$i | grep pci | rev | cut -f2 -d'/' | rev) 48 | dna=$(pcimem /dev/xdma/$i/user 0x60000 w | tail -n1 | cut -d':' -f2 | tr -d ' ') 49 | dna_pci[$index]=$(echo ${dna: -8}) 50 | index=$((index+1)) 51 | done 52 | 53 | echo " dna via pci: ${dna_pci[*]}" 54 | echo " pci: ${pci[*]}" 55 | echo "dna via jtag: ${dna_jtag[*]}" 56 | echo " jtag serial: ${serial_jtag[*]}" 57 | echo " port: ${port_arr[*]}" 58 | 59 | #index=0 60 | #for k in ${dna_pci[*]}; do 61 | # l=0 62 | # for l in ${dna_jtag[*]}; do 63 | # if test $k=$l; then 64 | # echo $k 65 | # fi 66 | # done 67 | #done 68 | 69 | index_pci=0 70 | index_jtag=0 71 | 72 | echo -e "config:\n-------" 73 | 74 | for k in ${dna_pci[*]}; do 75 | index_jtag=0 76 | for l in ${dna_jtag[*]}; do 77 | if [ $l = $k ]; then 78 | #echo $k 79 | #echo ${pci[$index_pci]} 80 | #echo ${dna_jtag[index_jtag]} 81 | #echo ${dna_pci[index_pci]} 82 | #echo ${port_arr[index_jtag]} 83 | #echo ${serial_jtag[index_jtag]} 84 | echo ">${pci[$index_pci]} ${serial_jtag[index_jtag]} ${port_arr[index_jtag]}" 85 | fi 86 | index_jtag=$((index_jtag+1)) 87 | done 88 | index_pci=$((index_pci+1)) 89 | done 90 | -------------------------------------------------------------------------------- /alveo-program/alveo-program.tcl: -------------------------------------------------------------------------------- 1 | # Created on: 12 April 2022 2 | # Author: Mathews Chirindo 3 | # 4 | # Modified: RvW, SARAO, 2022 5 | 6 | 7 | # create two lookup structures for mapping the jtag serial number and port number to the pci addr 8 | set lookup_jtag [dict create pci jtag] 9 | set lookup_port [dict create pci port] 10 | 11 | #open the config mapping file 12 | set fp [open "alveo-pci-cfg" r] 13 | 14 | #split the file data into lines 15 | set data [split [read $fp] "\n"] 16 | 17 | close $fp 18 | 19 | #for each line 20 | foreach line $data { 21 | if { [string length $line] != 0 } { 22 | #split into columns 23 | set cols [split $line " "] 24 | #puts "[lindex $cols 0]" 25 | #puts "[lindex $cols 1]" 26 | #puts "[lindex $cols 2]" 27 | dict append lookup_jtag [lindex $cols 0] [lindex $cols 1] 28 | dict append lookup_port [lindex $cols 0] [lindex $cols 2] 29 | } 30 | } 31 | 32 | puts $lookup_jtag 33 | puts $lookup_port 34 | 35 | if { [dict exists $lookup_jtag [lindex $argv 0]] != 1 || [dict exists $lookup_port [lindex $argv 0]] != 1 } { 36 | puts "Invalid PCI ID [lindex $argv 0]" 37 | exit 2 38 | } 39 | 40 | if { [file exists [lindex $argv 1]] != 1 } { 41 | puts "Cannot access [lindex $argv 1]" 42 | exit 3 43 | } 44 | 45 | set jtag_serial_number [dict get $lookup_jtag [lindex $argv 0]] 46 | set port_number [dict get $lookup_port [lindex $argv 0]] 47 | append url localhost: $port_number /xilinx_tcf/Xilinx/ $jtag_serial_number A 48 | puts $url 49 | 50 | open_hw_manager 51 | #prevent chipscope server from launching 52 | set_param labtools.enable_cs_server false 53 | 54 | set systime [clock seconds] 55 | set timestring [clock format $systime -format "%d-%b-%H%M%S"] 56 | append logfile hwserver- $jtag_serial_number - $port_number - ${timestring} .log 57 | exec hwserver-log-rotate.sh ${jtag_serial_number} ${port_number} 58 | 59 | #hw_server requires an 'A' to be appended to serial number obtained from 'lsusb -v' 60 | append jtag_extended_serial ${jtag_serial_number} A 61 | exec hw_server -d -L $logfile -s tcp:localhost:$port_number -p0 -I1 -e "set jtag-port-filter ${jtag_extended_serial}" 62 | connect_hw_server -url localhost:$port_number 63 | open_hw_target $url 64 | current_hw_device [lindex [get_hw_devices] 0] 65 | refresh_hw_device -update_hw_probes false [current_hw_device] 66 | # set_property PARAM.FREQUENCY 10000000 [current_hw_target] 67 | puts "will attempt to program alveo with [lindex $argv 1]" 68 | set_property PROGRAM.FILE [lindex $argv 1] [current_hw_device] 69 | program_hw_devices [current_hw_device] 70 | -------------------------------------------------------------------------------- /alveo-utils/alveo-launch-tcpbs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/sudo bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | set -e 6 | 7 | echo "WARNING: starting with root permissions!!" 8 | 9 | #test dependencies (set -e will catch failures) 10 | which tcpborphserver3 || (>&2 echo "missing dependencies - tcpborphserver"; exit 1) 11 | which kcpcmd || (>&2 echo "missing dependencies - kcpcmd"; exit 1) 12 | which kcpfpg || (>&2 echo "missing dependencies - kcpfpg"; exit 1) 13 | test -d /opt/alveo/alveo-tcpbs-fs/ || (>&2 echo "missing dependencies - /opt/alveo/alveo-tcpbs-fs/"; exit 1) 14 | 15 | BASE_PORT=7150 16 | #BOF_DIR="/opt/alveo/alveo-tcpbs-fs/" 17 | #BOF_DIR="../alveo-tcpbs-fs/" 18 | #CFGPATH="/lib/firmware/tcpborphserver.bit" 19 | 20 | #export PATH=$PATH:$(pwd) 21 | export PATH=$PATH:$(pwd):${BOF_DIR} 22 | 23 | #ALVEOS=$(./alveo-map) 24 | ALVEOS=$(ls /dev/xdma/) 25 | 26 | PORT=${BASE_PORT} 27 | 28 | for CARD in ${ALVEOS}; do 29 | # PCIADDR=$(echo ${CARD} | cut -d'/' -f1) 30 | SERIAL=$(/opt/alveo/alveo-utils/alveo-serial /dev/xdma/${CARD}/user 2>/dev/null) 31 | echo $CARD $SERIAL 32 | 33 | PCIPATH=$(readlink -f /dev/xdma/${CARD}/pci) 34 | PCIADDR=$(echo ${PCIPATH} | rev | cut -d '/' -f1 | rev) 35 | PCITAG=${PCIADDR//[:.]/-} #replace all ':' and '.' with '-' 36 | CFGPATH="/lib/firmware/tcpborphserver-${PCITAG}.bit" 37 | 38 | BOF_DIR="/tmp/alveo-tcpbs-fs-${PCITAG}/" 39 | if ! test -d ${BOF_DIR}; then 40 | mkdir ${BOF_DIR} 41 | cp /opt/alveo/alveo-tcpbs-fs/alveo-* ${BOF_DIR} 42 | fi 43 | 44 | #check if this tcpbs already running, since with each xdma.ko load, this script will be run 45 | STAT=0 46 | ps fax | grep [t]cpborphserver3 | grep ${PORT} &> /dev/null || STAT=1 #the || is so the process can continue (set -e) 47 | 48 | if [ ${STAT} -ne 1 ] 49 | then 50 | echo "tcpborphserver instance already running for ${CARD} with serial ${SERIAL} at port ${PORT}" 51 | else 52 | env PATH=$PATH:$(pwd):${BOF_DIR} tcpborphserver3 -p ${PORT} -l /dev/null -b ${BOF_DIR} -d /dev/xdma/${CARD}/user -u $((PORT+1)) -c ${CFGPATH} 53 | 54 | #TODO should we wait a bit here? 55 | sleep 0.5 56 | # kcpcmd -s localhost:${PORT} version add alveo ${CARD} ${SERIAL} 57 | kcpcmd -s localhost:${PORT} version add alveo-card ${CARD} 58 | kcpcmd -s localhost:${PORT} version add alveo-serial ${SERIAL} 59 | kcpcmd -s localhost:${PORT} process alveo-reset reset\_this\_alveo 60 | kcpcmd -s localhost:${PORT} process alveo-program program\_this\_alveo 61 | kcpcmd -s localhost:${PORT} process alveo-memread read\_from\_a\_hex\_valued\_absolute\_address 62 | kcpcmd -s localhost:${PORT} process alveo-memwrite write\_to\_a\_hex\_valued\_absolute\_address 63 | fi 64 | PORT=$((PORT+2)) 65 | done 66 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #RvW, SARAO, 2022 4 | 5 | SRC_DEV="xdma.*user|xclmgmt" 6 | 7 | test ! -e configure && echo "run from ./configure location" && exit 8 | 9 | #create a temp file 10 | tmpfile=$(mktemp -q .60-xdma.rules.tmp.XXXX) 11 | #create a file descriptor to write data to after closing temp file 12 | exec 3>"$tmpfile" 13 | #create another file descriptor for later reading 14 | exec 4<"$tmpfile" 15 | #delete temp file 16 | rm "$tmpfile" 17 | 18 | 19 | #find any installed xilinx alveo cards in pcie slots - either running the shell xrt or the xdma 20 | DEVS=$(ls -l /sys/dev/char/ 2>/dev/null | grep -E ${SRC_DEV} 2>/dev/null | rev | cut -f3 -d'/' | rev) 21 | INDEX=0 22 | 23 | test -z "${DEVS}" && echo "No alveos found. Ensure Xilinx XRT is installed" && exec 3>&- && exec 4>&- && exit 24 | 25 | for PCIE_ADDR in ${DEVS}; do 26 | echo -n ${PCIE_ADDR} 27 | #if type unset, set default to general 'alveo' type 28 | ALVEO_TYPE=${ALVEO_TYPE:-"alveo"} 29 | #now try to get the actual type 30 | DEV_ID=$(cat /sys/bus/pci/devices/${PCIE_ADDR}/device 2>/dev/null) 31 | if test ${DEV_ID} = "0x5020"; then 32 | ALVEO_TYPE="u50"; 33 | elif test ${DEV_ID} = "0x9031"; then #current SARAO bsp builds for u50 34 | ALVEO_TYPE="u50"; 35 | elif test ${DEV_ID} = "0x9021"; then #current SARAO bsp builds for u50 36 | ALVEO_TYPE="u50"; 37 | elif test ${DEV_ID} = "0x500c"; then 38 | ALVEO_TYPE="u280"; 39 | else 40 | ALVEO_TYPE="unknown"; 41 | fi 42 | 43 | echo " ${ALVEO_TYPE}" 44 | 45 | echo "#alveo_${INDEX}" >&3 46 | echo "KERNEL==\"xdma\[0-9\]*\", KERNELS==\"${PCIE_ADDR}\", PROGRAM=\"/etc/udev/rules.d/alveo_namer.sh %k\", SYMLINK+=\"xdma/alveo_${INDEX}_${ALVEO_TYPE}/%c\", MODE=\"0666\"" >&3 47 | echo "KERNEL==\"xdma\[0-9\]*_user\", KERNELS==\"${PCIE_ADDR}\", ACTION==\"add\", RUN+=\"/bin/ln -f -s /sys/bus/pci/devices/${PCIE_ADDR}/ /dev/xdma/alveo_${INDEX}_${ALVEO_TYPE}/pci\"" >&3 48 | echo "KERNEL==\"xdma\[0-9\]*_user\", KERNELS==\"${PCIE_ADDR}\", ACTION==\"remove\", RUN+=\"/bin/rm /dev/xdma/alveo_${INDEX}_${ALVEO_TYPE}/pci\"" >&3 49 | echo >&3 50 | INDEX=$((INDEX+1)) 51 | done 52 | 53 | echo >&3 54 | echo "#all alveos" >&3 55 | echo "KERNEL==\"xdma\[0-9\]*_user\", ACTION==\"add\", RUN+=\"/usr/bin/sudo systemctl start alveo-katcp-svr.service\"" >&3 56 | 57 | 58 | UDEV_OUTPUT_PATH=./alveo-linux-env/xdma-udev-rules 59 | test -e ${UDEV_OUTPUT_PATH}/60-xdma.rules && cp --backup=t ${UDEV_OUTPUT_PATH}/60-xdma.rules ${UDEV_OUTPUT_PATH}/60-xdma.rules.bak 60 | 61 | #create Makefile.inc if it doesn't exist 62 | #touch "${UDEV_OUTPUT_PATH}/60-xdma.rules" 63 | 64 | #clean file 65 | > ${UDEV_OUTPUT_PATH}/60-xdma.rules 66 | 67 | #copy and append temp file into Makefile.inc 68 | while read -u 4 line;do echo "$line" >> ${UDEV_OUTPUT_PATH}/60-xdma.rules;done 69 | 70 | #close the file descriptor pointing to the temp file 71 | exec 3>&- 72 | exec 4>&- 73 | 74 | echo "configuring pci to jtag-serial map...this may take a few minutes..." 75 | cd alveo-utils 76 | ./alveo-jtag-list | grep "^>" | cut -d'>' -f 2 > ../alveo-program/alveo-pci-cfg 77 | cd - >/dev/null 78 | -------------------------------------------------------------------------------- /alveo-sw-docs/readme.md: -------------------------------------------------------------------------------- 1 | # alveo-sw 2 | 3 | This repo contains the supporting software infrstructure to control and monitor Alveos with the SARAO (CASPER) toolflow. 4 | It runs in a linux environment and has been tested on Ubuntu 18.04. It also relies on the Alveo cards being "FLASHED" with the SARAO BSP. For further details on this, please see [Alveo Board Support Packages](https://github.com/ska-sa/alveo_bsp). 5 | It furthermore expects a Xilinx *jtag* programmer to be used to configure (program) the Alveo with the "runtime" fpga images (ie *.fpg* or *.bit* files). 6 | 7 | ## Getting Started 8 | 9 | From the shell, do the following steps: 10 | 11 | Clone this repo 12 | ``` 13 | git clone https://github.com/ska-sa/alveo-sw.git 14 | cd alveo-sw/ 15 | ``` 16 | 17 | Initialise the *git* submodules 18 | ``` 19 | git submodule init 20 | git submodule update 21 | ``` 22 | 23 | Now it is necessary to configure the infrastructure for the current host. This can be done by executing the following script on the command line 24 | ``` 25 | ./configure 26 | ``` 27 | 28 | The software infrastructure relies on some python dependencies. Among these is the creation of a *python executable (pex)* environment which contains the dependencies required to run the python alveo katcp server. In order to generate this *pex* dependency, the following needs to be done 29 | 30 | 1. create a python3.8 virtual environment 31 | 32 | ``` 33 | python3.8 -m venv alveo_venv 34 | ``` 35 | 2. launch the virtual environmnet 36 | ``` 37 | source alveo_venv/bin/activate 38 | ``` 39 | 3. install the *pex* package 40 | ``` 41 | pip3.8 install pex 42 | ``` 43 | 4. change directory to the *alveo-katcp-svr* directory and run *make* 44 | ``` 45 | cd alveo-katcp-svr 46 | make 47 | cd .. 48 | ``` 49 | 5. deactivate the vrtual environment 50 | ``` 51 | deactivate 52 | ``` 53 | 54 | Now that the python dependencies have been built, the software infrastructure can be installed from within the parent *alveo-sw* directory 55 | ``` 56 | sudo make install 57 | ``` 58 | Reboot the host machine 59 | ``` 60 | sudo reboot 61 | ``` 62 | At this point, the Alveo supporting software infratructure shoud have be deployed and running. Some basic sanity checks to ensure that the required software is running correctly can be found in the [alveo-sw troubleshooting](./troubleshooting.md) guide. 63 | 64 | The [casperfpga branch with alveo support](https://github.com/ska-sa/casperfpga/tree/devel#alveo-support) can now be used to connect to the Alveo cards. 65 | 66 | --- 67 | ## System startup 68 | The following block diagram shows the various linux processes involved in launching the *alveo-katcp-svr*. 69 | ``` 70 | ┌───────────────┐ 71 | │ │ 72 | │ linux boot │ 73 | │ │ 74 | └──────┬────────┘ 75 | │ 76 | │ 77 | ┌──────▼────────┐ 78 | │ modprobe │ /etc/modprobe.d/xdma.conf 79 | ├───────────────┤ 80 | │ load xdma │ 81 | │ kernel module │ 82 | └──────┬────────┘ 83 | │ 84 | │ 85 | │ 86 | ┌──────▼────────┐ 87 | │ udev │ /etc/udev/rules.d/60-xdma.rules 88 | ├───────────────┤ 89 | │ configure /dev│ 90 | │ entries │ 91 | └──────┬────────┘ 92 | │ 93 | │ 94 | │ 95 | ┌──────▼────────┐ 96 | │ systemd │ /etc/systemd/system/alveo-katcp-svr.service 97 | ├───────────────┤ 98 | │launch alveo │ 99 | │services │ 100 | └──────┬────────┘ 101 | │ 102 | │ 103 | │ 104 | ┌──────▼────────┐ 105 | │ │ 106 | │alveo-katcp-svr│ 107 | │ │ 108 | └───────────────┘ 109 | ``` -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #RvW, SARAO, 2022 2 | 3 | ALVEOPATH="/opt/alveo/" 4 | #TBSFS="${ALVEOPATH}/alveo-tcpbs-fs/" 5 | ALVEOUTILS="${ALVEOPATH}/alveo-utils/" 6 | UDEVPATH="/etc/udev/rules.d/" 7 | SYSTEMDPATH="/etc/systemd/system/" 8 | PREFIX=usr/local 9 | EXISTS=$(shell test -e ./alveo-linux-env/xdma-udev-rules/60-xdma.rules || echo 'NO') 10 | 11 | install: .check .alveo_katcp_svr_install .kcpfpg_install .kcpcmd_install .pcimem_install .kcpmsg_install xdma_mod_install 12 | # @test -d ${TBSFS} || mkdir -p ${TBSFS} 13 | # cp -i ./alveo-tcpbs-fs/* ${TBSFS} 14 | cp -i ./alveo-linux-env/xdma-udev-rules/60-xdma.rules ./alveo-linux-env/xdma-udev-rules/alveo_namer.sh ${UDEVPATH} 15 | #install alveo-utils 16 | @test -d ${ALVEOUTILS} || mkdir -p ${ALVEOUTILS} 17 | cp -i ./alveo-utils/* ${ALVEOUTILS} 18 | #install tcpbs service 19 | # cp -i ./alveo-linux-env/systemd-services/tcpbs.service ${SYSTEMDPATH} 20 | cp -i ./alveo-linux-env/systemd-services/alveo-katcp-svr.service ${SYSTEMDPATH} 21 | #install kernel module src and supporting build/service scripts 22 | test -d /usr/src/xdma-2020.2.2 || mkdir /usr/src/xdma-2020.2.2 23 | cp -r dma_ip_drivers/XDMA/linux-kernel/* /usr/src/xdma-2020.2.2 24 | cp alveo-linux-env/systemd-services/install-xdma.sh /usr/src/xdma-2020.2.2/. 25 | cp -i ./alveo-linux-env/systemd-services/xdma-install-check.service ${SYSTEMDPATH} 26 | systemctl enable xdma-install-check.service 27 | #install modprobe config 28 | cp -i ./alveo-linux-env/modprobe-configs/xdma.conf /etc/modprobe.d/xdma.conf 29 | #install alveo programming scripts 30 | install -D ./alveo-program/alveo-program.sh ${ALVEOPATH}/alveo-program/alveo-program.sh 31 | install -D ./alveo-program/alveo-program.tcl ${ALVEOPATH}/alveo-program/alveo-program.tcl 32 | install -D ./alveo-program/alveo-pci-cfg ${ALVEOPATH}/alveo-program/alveo-pci-cfg 33 | install -D ./alveo-program/hwserver-log-rotate.sh ${ALVEOPATH}/alveo-program/hwserver-log-rotate.sh 34 | 35 | .check: .FORCE 36 | ifeq ("$(EXISTS)","NO") 37 | $(error "First run ./configure to set up environment") 38 | endif 39 | 40 | #.tcpborphserver3_install: .katcp_install 41 | # prefix=${PREFIX} ${MAKE} -C katcp/tcpborphserver3/ install 42 | 43 | .alveo_katcp_svr_install: 44 | test -d ${ALVEOPATH}/alveo-katcp-svr || mkdir -p ${ALVEOPATH}/alveo-katcp-svr 45 | cp ./alveo-katcp-svr/alveo-katcp-svr.py ${ALVEOPATH}/alveo-katcp-svr/. 46 | cp ./alveo-katcp-svr/AlveoIO.py ${ALVEOPATH}/alveo-katcp-svr/. 47 | cp ./alveo-katcp-svr/fpgparser.py ${ALVEOPATH}/alveo-katcp-svr/. 48 | cp ./alveo-katcp-svr/alveo_env.pex ${ALVEOPATH}/alveo-katcp-svr/. 49 | 50 | 51 | .kcpfpg_install: .katcp_install 52 | prefix=${PREFIX} ${MAKE} -C katcp/fpg install 53 | ln -sf /usr/local/bin/kcpfpg /bin/kcpfpg 54 | 55 | .kcpcmd_install: .katcp_install 56 | prefix=${PREFIX} ${MAKE} -C katcp/cmd install 57 | 58 | .katcp_install: 59 | ${MAKE} -C katcp/katcp/ 60 | 61 | .pcimem_install: 62 | ${MAKE} -C pcimem/ 63 | install pcimem/pcimem /usr/local/bin/ 64 | 65 | .kcpmsg_install: .katcp_install 66 | ${MAKE} -C katcp/msg install 67 | 68 | xdma_mod_install: 69 | ${MAKE} -C dma_ip_drivers/XDMA/linux-kernel/xdma install 70 | ${MAKE} -C dma_ip_drivers/XDMA/linux-kernel/xdma clean 71 | depmod -a 72 | modprobe xdma || insmod /lib/modules/$(shell uname -r)/extra/xdma.ko 73 | #test -d /usr/src/xdma-2020.2.2 || mkdir /usr/src/xdma-2020.2.2 74 | #cp -r dma_ip_drivers/XDMA/linux-kernel/* /usr/src/xdma-2020.2.2 75 | #cp alveo-linux-env/xdma-dkms/dkms.conf /usr/src/xdma-2020.2.2/. 76 | #dkms add -m xdma -v 2020.2.2 77 | #dkms build -m xdma -v 2020.2.2 78 | #dkms install -m xdma -v 2020.2.2 79 | 80 | uninstall: .kcpfpg_uninstall .kcpcmd_uninstall .kcpmsg_uninstall 81 | $(RM) ${TBSFS}/* 82 | $(RM) ${ALVEOUTILS}/* 83 | 84 | #.tcpborphserver3_uninstall: 85 | # prefix=${PREFIX} ${MAKE} -C katcp/tcpborphserver3/ uninstall 86 | 87 | .kcpfpg_uninstall: 88 | prefix=${PREFIX} ${MAKE} -C katcp/fpg uninstall 89 | 90 | .kcpcmd_uninstall: 91 | prefix=${PREFIX} ${MAKE} -C katcp/cmd uninstall 92 | 93 | .kcpmsg_uninstall: 94 | prefix=${PREFIX} ${MAKE} -C katcp/msg uninstall 95 | 96 | 97 | .PHONY: .check .alveo_katcp_svr_install .pcimem_install .katcp_install .kcpfpg_install .kcpmsg_install .kcpcmd_install .kcpfpg_uninstall .kcpcmd_uninstall .kcpmsg_uninstall 98 | 99 | .FORCE: 100 | -------------------------------------------------------------------------------- /alveo-katcp-svr/fpgparser.py: -------------------------------------------------------------------------------- 1 | #NOTE this was copy-pasted from casperfpga 2 | #to eliminate the need to install it 3 | 4 | import logging 5 | 6 | LOGGER = logging.getLogger(__name__) 7 | 8 | def parse_fpg(filename, isbuf=False): 9 | """ 10 | Read the meta information from the FPG file. 11 | 12 | :param filename: the name of the fpg file to parse 13 | :param isbuf: If True, the filename is not actually a name, it is a 14 | BytesIO buffer. 15 | :return: device info dictionary, memory map info (coreinfo.tab) dictionary 16 | """ 17 | 18 | 19 | LOGGER.debug('Parsing file %s for system information' % filename) 20 | if filename is not None: 21 | if not isbuf: 22 | fptr = open(filename, 'rb') 23 | else: 24 | fptr = filename 25 | firstline = fptr.readline().decode('latin-1').strip().rstrip('\n') 26 | if firstline != '#!/bin/kcpfpg': 27 | fptr.close() 28 | raise RuntimeError('%s does not look like an fpg file we can ' 29 | 'parse.' % filename) 30 | else: 31 | raise IOError('No such file %s' % filename) 32 | 33 | memorydict = {} 34 | metalist = [] 35 | while True: 36 | line = fptr.readline().decode('latin-1').strip().rstrip('\n') 37 | if line.lstrip().rstrip() == '?quit': 38 | break 39 | elif line.startswith('?meta'): 40 | # some versions of mlib_devel may mistakenly have put spaces 41 | # as delimiters where tabs should have been used. Rectify that 42 | # here. 43 | if line.startswith('?meta '): 44 | LOGGER.warn('An old version of mlib_devel generated %s. Please ' 45 | 'update. Meta fields are seperated by spaces, ' 46 | 'should be tabs.' % filename) 47 | line = line.replace(' ', '\t') 48 | # and carry on as usual. 49 | line = line.replace('\_', ' ').replace('?meta', '') 50 | line = line.replace('\n', '').lstrip().rstrip() 51 | #line_split = line.split('\t') 52 | # Rather split on any space 53 | line_split = line.split() 54 | name = line_split[0] 55 | tag = line_split[1] 56 | param = line_split[2] 57 | if len(line_split[3:]) == 1: 58 | value = line_split[3:][0] 59 | else: 60 | value = ' '.join(line_split[3:]) 61 | # name, tag, param, value = line.split('\t') 62 | name = name.replace('/', '_') 63 | metalist.append((name, tag, param, value)) 64 | elif line.startswith('?register'): 65 | if line.startswith('?register '): 66 | register = line.replace('\_', ' ').replace('?register ', '') 67 | register = register.replace('\n', '').lstrip().rstrip() 68 | name, address, size_bytes = register.split(' ') 69 | elif line.startswith('?register\t'): 70 | register = line.replace('\_', ' ').replace('?register\t', '') 71 | register = register.replace('\n', '').lstrip().rstrip() 72 | name, address, size_bytes = register.split('\t') 73 | else: 74 | raise ValueError('Cannot find ?register entries in ' 75 | 'correct format.') 76 | address = int(address, 16) 77 | size_bytes = int(size_bytes, 16) 78 | if name in memorydict.keys(): 79 | raise RuntimeError('%s: mem device %s already in ' 80 | 'dictionary' % (filename, name)) 81 | memorydict[name] = {'address': address, 'bytes': size_bytes} 82 | fptr.close() 83 | return create_meta_dictionary(metalist), memorydict 84 | 85 | 86 | 87 | 88 | def create_meta_dictionary(metalist): 89 | """ 90 | Build a meta information dictionary from a provided raw meta info list. 91 | 92 | :param metalist: a list of all meta information about the system 93 | :return: a dictionary of device info, keyed by unique device name 94 | """ 95 | meta_items = {} 96 | try: 97 | for name, tag, param, value in metalist: 98 | if name not in meta_items: 99 | meta_items[name] = {} 100 | try: 101 | if meta_items[name]['tag'] != tag: 102 | raise ValueError( 103 | 'Different tags - %s, %s - for the same item %s' % ( 104 | meta_items[name]['tag'], tag, name)) 105 | except KeyError: 106 | meta_items[name]['tag'] = tag 107 | meta_items[name][param] = value 108 | except ValueError as e: 109 | for ctr, contents in enumerate(metalist): 110 | print(ctr, end='') 111 | print(contents) 112 | raise e 113 | return meta_items 114 | -------------------------------------------------------------------------------- /alveo-sw-docs/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting the alveo-sw infrastructure 2 | 3 | Listed below are some basic sanity checks to verify that various parts of the Alveo software infrastructure is running correctly. 4 | 5 | --- 6 | Currently, the Alveo cards are configured using *jtag* programmers. In order for the Alveo software infrastructure to configure the correct card, a mapping between a specific Alveo's *pcie* bus and the *jtag* programmer is configured during installation. To check this mapping, the following command can be executed 7 | 8 | ``` 9 | $ cat /opt/alveo/alveo-program/alveo-pci-cfg 10 | 0000:c2:00.0 500202A3099A 3150 11 | 0000:c1:00.0 500202A30D7A 3160 12 | ``` 13 | The columns shown are the *pcie* address, the *jtag* serial number and the port number used by the hardware server during programming. 14 | 15 | --- 16 | To ensure that 1) the Alveo has been correctly flashed 2) that it is correctly detected and enumerated on the pcie bus and 3) that the XDMA driver has loaded correctly, the following command can be issued from the linux cmd line 17 | ``` 18 | lspci -d 10ee: -vvv -D 19 | ``` 20 | This should list ALL Xilinx cards on the pcie bus. It would include Alveos FLASHED with the SARAO BSP, as well as Alveos running the default (factory) shell image. The output should look as follows (or similar): 21 | ``` 22 | 81:00.0 Processing accelerators: Xilinx Corporation Device 500c 23 | Subsystem: Xilinx Corporation Device 000e 24 | Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx- 25 | Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- 30 | Kernel modules: xclmgmt 31 | 32 | 81:00.1 Processing accelerators: Xilinx Corporation Device 500d 33 | Subsystem: Xilinx Corporation Device 000e 34 | Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx- 35 | Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- 43 | Kernel driver in use: xocl 44 | Kernel modules: xocl 45 | 46 | c1:00.0 Serial controller: Xilinx Corporation Device 9031 (prog-if 01 [16450]) 47 | Subsystem: Xilinx Corporation Device 0007 48 | Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx- 49 | Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- 56 | Kernel driver in use: xdma 57 | Kernel modules: xdma 58 | 59 | c2:00.0 Serial controller: Xilinx Corporation Device 9031 (prog-if 01 [16450]) 60 | Subsystem: Xilinx Corporation Device 0007 61 | Control: I/O- Mem+ BusMaster+ SpecCycle- MemWINV- VGASnoop- ParErr- Stepping- SERR- FastB2B- DisINTx- 62 | Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- 69 | Kernel driver in use: xdma 70 | Kernel modules: xdma 71 | ``` 72 | 73 | From the above output, we can see that there are three Alveo cards on the pcie bus. The first (at address 81:00.0) is running the Xilinx (factory) shell, while the other two (addresses c1:00 and c2:00) are running the SARAO BSP. We can see that the latter two cards have loaded the *xdma* kernel driver module, as expected. The other important part to note is the BAR size of Region 0, given in the square parenthese. This should be 256MBytes in size. If the memory size does not equal 256M while the *xdma* module is indeed loaded, it may mean that the running image has been built with a different BAR size, which is set in the Xilinx XDMA IP module in *Vivado* during development. 74 | 75 | --- 76 | 77 | In order to ensure that the ***alveo-katcp-svr.py*** process has been started for each of the Alveo cards running the SARAO BSP, the linux process listing can be inspected as follows 78 | ``` 79 | $ ps fax | grep "[a]lveo-katcp-svr.py" 80 | ``` 81 | The output should look as follows for each of the Alveo cards: 82 | ``` 83 | 1714 ? S 0:01 /usr/bin/python3.8 /root/.pex/unzipped_pexes/9406cbbd41ebb12539c12a574d2b2d45c83b5599 /opt/alveo/alveo-katcp-svr/alveo-katcp-svr.py --alveo alveo_1_u50 --workdir /tmp/alveo-katcp-svr-fs-0000-c1-00-0/ --port 7150 --card alveo_1_u50-501211207V5X -d 84 | ``` 85 | 86 | If the above process is not running, or there is not an instance for each of the Alveo cards (ie two Alveo cards require two alveo-katcp-svr instances in the process listing), the following command can be run in an attempt to start the processes. 87 | 88 | ``` 89 | sudo systemctl start alveo-katcp-svr.service 90 | ``` 91 | 92 | --- 93 | 94 | It is also useful to query a known register value to ensure that the pcie-to-axi bus is functioning as expected. For this the `pcimem` utility [reference: https://github.com/billfarrow/pcimem] can be used to read the REG_MAP_ID_REG register in the Alveo's *Card Management Solution (CMS)* Subsystem. This query is run at a lower level than the alveo-sw infrastructure, thereby bypassing it, and this is therefore a useful sanity check to verify the state of the Alveo card and *pcie* bus. 95 | 96 | To do this, ensure that the Microblaze internal to the CMS is running. This is the case when the nRESET register is high (as readback below) 97 | ``` 98 | $ sudo pcimem /dev/xdma0_user 0x20000 w 99 | /dev/xdma0_user opened. 100 | Target offset is 0x20000, page size is 4096 101 | mmap(0, 4096, 0x3, 0x1, 3, 0x20000) 102 | PCI Memory mapped to address 0x7f547be04000. 103 | 0x20000: 0x00000001 104 | ``` 105 | 106 | Now we can check that we can communicate over the axi bus via pcie by confirming the register value equals 0x74736574 107 | ``` 108 | $ sudo pcimem /dev/xdma0_user 0x28000 w 109 | /dev/xdma0_user opened. 110 | Target offset is 0x28000, page size is 4096 111 | mmap(0, 4096, 0x3, 0x1, 3, 0x28000) 112 | PCI Memory mapped to address 0x7f2a89309000. 113 | 0x28000: 0x74736574 114 | ``` 115 | See CMS Product Guide for details: https://docs.xilinx.com/r/en-US/pg348-cms-subsystem/Register-Space 116 | 117 | --- 118 | 119 | There are also various custom tools that have been scripted to display information about the Alveo cards. These can be found in `/opt/alveo/alveo-utils/` after installation. Example outputs are given below: 120 | 121 | ``` 122 | $ cd /opt/alveo/alveo-utils/ 123 | 124 | /opt/alveo/alveo-utils$ ./alveo-pci --help 125 | usage: ./alveo-pci [--help] [--dev ] 126 | 127 | /opt/alveo/alveo-utils$ ./alveo-pci --dev all 128 | PCI Device @ 0000:81:00.0/xclmgmt33024 129 | ID 0x500c 130 | Region0 mem-size 32MiB 131 | Region2 mem-size 128KiB 132 | PCI Device @ 0000:c1:00.0/xdma0_user 133 | ID 0x9031 134 | Region0 mem-size 256MiB 135 | Region1 mem-size 64KiB 136 | ``` 137 | The above output shows the details for two Alveo pcie cards. It shows the pcie address, the relevant device file, the device ID which has been set during development and lastly the BAR regions which are mapped into memory. 138 | 139 | ``` 140 | /opt/alveo/alveo-utils$ ./alveo-map 141 | 0000:c1:00.0/xdma0_user/501211207V5X 142 | ``` 143 | The output above gives the mapping of the pcie to card serial number for each of the Alveo cards. Note this is not the jtag serial number. It also shows the device file used to interface with the axi-lite control bus. 144 | 145 | ``` 146 | /opt/alveo/alveo-utils$ ./alveo-serial /dev/xdma0_user 147 | 501211207V5X 148 | ``` 149 | The above shows the Alveo card serial number read via the relevant xdma*_user device file entry. 150 | -------------------------------------------------------------------------------- /alveo-katcp-svr/AlveoSensors.py: -------------------------------------------------------------------------------- 1 | alveo_sensor_map = { 2 | 0: {'name': '12v.pex.max.mV', 'address': 0x28020, 'description': '12V_PEX_MAX_REG', 'units': 'millivolts'}, 3 | 1: {'name': '12v.pex.avg.mV', 'address': 0x28024, 'description': '12V_PEX_AVG_REG', 'units': 'millivolts'}, 4 | 2: {'name': '12v.pex.ins.mV', 'address': 0x28028, 'description': '12V_PEX_INS_REG', 'units': 'millivolts'}, 5 | 3: {'name': '3v3.pex.max.mV', 'address': 0x2802C, 'description': '3V3_PEX_MAX_REG', 'units': 'millivolts'}, 6 | 4: {'name': '3v3.pex.avg.mV', 'address': 0x28030, 'description': '3V3_PEX_AVG_REG', 'units': 'millivolts'}, 7 | 5: {'name': '3v3.pex.ins.mV', 'address': 0x28034, 'description': '3V3_PEX_INS_REG', 'units': 'millivolts'}, 8 | 6: {'name': '3v3.aux.max.mV', 'address': 0x28038, 'description': '3V3_AUX_MAX_REG', 'units': 'millivolts'}, 9 | 7: {'name': '3v3.aux.avg.mV', 'address': 0x2803C, 'description': '3V3_AUX_AVG_REG', 'units': 'millivolts'}, 10 | 8: {'name': '3v3.aux.ins.mV', 'address': 0x28040, 'description': '3V3_AUX_INS_REG', 'units': 'millivolts'}, 11 | 9: {'name': '12v.aux.max.mV', 'address': 0x28044, 'description': '12V_AUX_MAX_REG', 'units': 'millivolts'}, 12 | 10: {'name': '12v.aux.avg.mV', 'address': 0x28048, 'description': '12V_AUX_AVG_REG', 'units': 'millivolts'}, 13 | 11: {'name': '12v.aux.ins.mV', 'address': 0x2804C, 'description': '12V_AUX_INS_REG', 'units': 'millivolts'}, 14 | 12: {'name': 'ddr4.vpp.btm.max.mV', 'address': 0x28050, 'description': 'DDR4_VPP_BTM_MAX_REG', 'units': 'millivolts'}, 15 | 13: {'name': 'ddr4.vpp.btm.avg.mV', 'address': 0x28054, 'description': 'DDR4_VPP_BTM_AVG_REG', 'units': 'millivolts'}, 16 | 14: {'name': 'ddr4.vpp.btm.ins.mV', 'address': 0x28058, 'description': 'DDR4_VPP_BTM_INS_REG', 'units': 'millivolts'}, 17 | 15: {'name': 'sys.5v5.max.mV', 'address': 0x2805C, 'description': 'SYS_5V5_MAX_REG', 'units': 'millivolts'}, 18 | 16: {'name': 'sys.5v5.avg.mV', 'address': 0x28060, 'description': 'SYS_5V5_AVG_REG', 'units': 'millivolts'}, 19 | 17: {'name': 'sys.5v5.ins.mV', 'address': 0x28064, 'description': 'SYS_5V5_INS_REG', 'units': 'millivolts'}, 20 | 18: {'name': 'vcc1v2.top.max.mV', 'address': 0x28068, 'description': 'VCC1V2_TOP_MAX_REG', 'units': 'millivolts'}, 21 | 19: {'name': 'vcc1v2.top.avg.mV', 'address': 0x2806C, 'description': 'VCC1V2_TOP_AVG_REG', 'units': 'millivolts'}, 22 | 20: {'name': 'vcc1v2.top.ins.mV', 'address': 0x28070, 'description': 'VCC1V2_TOP_INS_REG', 'units': 'millivolts'}, 23 | 21: {'name': 'vcc1v8.max.mV', 'address': 0x28074, 'description': 'VCC1V8_MAX_REG', 'units': 'millivolts'}, 24 | 22: {'name': 'vcc1v8.avg.mV', 'address': 0x28078, 'description': 'VCC1V8_AVG_REG', 'units': 'millivolts'}, 25 | 23: {'name': 'vcc1v8.ins.mV', 'address': 0x2807C, 'description': 'VCC1V8_INS_REG', 'units': 'millivolts'}, 26 | 24: {'name': 'vcc0v85.max.mV', 'address': 0x28080, 'description': 'VCC0V85_MAX_REG', 'units': 'millivolts'}, 27 | 25: {'name': 'vcc0v85.avg.mV', 'address': 0x28084, 'description': 'VCC0V85_AVG_REG', 'units': 'millivolts'}, 28 | 26: {'name': 'vcc0v85.ins.mV', 'address': 0x28088, 'description': 'VCC0V85_INS_REG', 'units': 'millivolts'}, 29 | 27: {'name': 'ddr4.vpp.top.max.mV', 'address': 0x2808C, 'description': 'DDR4_VPP_TOP_MAX_REG', 'units': 'millivolts'}, 30 | 28: {'name': 'ddr4.vpp.top.avg.mV', 'address': 0x28090, 'description': 'DDR4_VPP_TOP_AVG_REG', 'units': 'millivolts'}, 31 | 29: {'name': 'ddr4.vpp.top.ins.mV', 'address': 0x28094, 'description': 'DDR4_VPP_TOP_INS_REG', 'units': 'millivolts'}, 32 | 30: {'name': 'mgt0v9avcc.max.mV', 'address': 0x28098, 'description': 'MGT0V9AVCC_MAX_REG', 'units': 'millivolts'}, 33 | 31: {'name': 'mgt0v9avcc.avg.mV', 'address': 0x2809C, 'description': 'MGT0V9AVCC_AVG_REG', 'units': 'millivolts'}, 34 | 32: {'name': 'mgt0v9avcc.ins.mV', 'address': 0x280A0, 'description': 'MGT0V9AVCC_INS_REG', 'units': 'millivolts'}, 35 | 33: {'name': '12v.sw.max.mV', 'address': 0x280A4, 'description': '12V_SW_MAX_REG', 'units': 'millivolts'}, 36 | 34: {'name': '12v.sw.avg.mV', 'address': 0x280A8, 'description': '12V_SW_AVG_REG', 'units': 'millivolts'}, 37 | 35: {'name': '12v.sw.ins.mV', 'address': 0x280AC, 'description': '12V_SW_INS_REG', 'units': 'millivolts'}, 38 | 36: {'name': 'mgtavtt.max.mV', 'address': 0x280B0, 'description': 'MGTAVTT_MAX_REG', 'units': 'millivolts'}, 39 | 37: {'name': 'mgtavtt.avg.mV', 'address': 0x280B4, 'description': 'MGTAVTT_AVG_REG', 'units': 'millivolts'}, 40 | 38: {'name': 'mgtavtt.ins.mV', 'address': 0x280B8, 'description': 'MGTAVTT_INS_REG', 'units': 'millivolts'}, 41 | 39: {'name': 'vcc1v2.btm.max.mV', 'address': 0x280BC, 'description': 'VCC1V2_BTM_MAX_REG', 'units': 'millivolts'}, 42 | 40: {'name': 'vcc1v2.btm.avg.mV', 'address': 0x280C0, 'description': 'VCC1V2_BTM_AVG_REG', 'units': 'millivolts'}, 43 | 41: {'name': 'vcc1v2.btm.ins.mV', 'address': 0x280C4, 'description': 'VCC1V2_BTM_INS_REG', 'units': 'millivolts'}, 44 | 42: {'name': '12vpex.i.in.max.mA', 'address': 0x280C8, 'description': '12VPEX_I_IN_MAX_REG', 'units': 'milliamps'}, 45 | 43: {'name': '12vpex.i.in.avg.mA', 'address': 0x280CC, 'description': '12VPEX_I_IN_AVG_REG', 'units': 'milliamps'}, 46 | 44: {'name': '12vpex.i.in.ins.mA', 'address': 0x280D0, 'description': '12VPEX_I_IN_INS_REG', 'units': 'milliamps'}, 47 | 45: {'name': '12v.aux.i.in.max.mA', 'address': 0x280D4, 'description': '12V_AUX_I_IN_MAX_REG', 'units': 'milliamps'}, 48 | 46: {'name': '12v.aux.i.in.avg.mA', 'address': 0x280D8, 'description': '12V_AUX_I_IN_AVG_REG', 'units': 'milliamps'}, 49 | 47: {'name': '12v.aux.i.in.ins.mA', 'address': 0x280DC, 'description': '12V_AUX_I_IN_INS_REG', 'units': 'milliamps'}, 50 | 48: {'name': 'vccint.max.mV', 'address': 0x280E0, 'description': 'VCCINT_MAX_REG', 'units': 'millivolts'}, 51 | 49: {'name': 'vccint.avg.mV', 'address': 0x280E4, 'description': 'VCCINT_AVG_REG', 'units': 'millivolts'}, 52 | 50: {'name': 'vccint.ins.mV', 'address': 0x280E8, 'description': 'VCCINT_INS_REG', 'units': 'millivolts'}, 53 | 51: {'name': 'vccint.i.max.mA', 'address': 0x280EC, 'description': 'VCCINT_I_MAX_REG', 'units': 'milliamps'}, 54 | 52: {'name': 'vccint.i.avg.mA', 'address': 0x280F0, 'description': 'VCCINT_I_AVG_REG', 'units': 'milliamps'}, 55 | 53: {'name': 'vccint.i.ins.mA', 'address': 0x280F4, 'description': 'VCCINT_I_INS_REG', 'units': 'milliamps'}, 56 | 54: {'name': 'fpga.temp.max.degC', 'address': 0x280F8, 'description': 'FPGA_TEMP_MAX_REG', 'units': 'celsius'}, 57 | 55: {'name': 'fpga.temp.avg.degC', 'address': 0x280FC, 'description': 'FPGA_TEMP_AVG_REG', 'units': 'celsius'}, 58 | 56: {'name': 'fpga.temp.ins.degC', 'address': 0x28100, 'description': 'FPGA_TEMP_INS_REG', 'units': 'celsius'}, 59 | 57: {'name': 'fan.temp.max.degC', 'address': 0x28104, 'description': 'FAN_TEMP_MAX_REG', 'units': 'celsius'}, 60 | 58: {'name': 'fan.temp.avg.degC', 'address': 0x28108, 'description': 'FAN_TEMP_AVG_REG', 'units': 'celsius'}, 61 | 59: {'name': 'fan.temp.ins.degC', 'address': 0x2810C, 'description': 'FAN_TEMP_INS_REG', 'units': 'celsius'}, 62 | 60: {'name': 'dimm.temp0.max.degC', 'address': 0x28110, 'description': 'DIMM_TEMP0_MAX_REG', 'units': 'celsius'}, 63 | 61: {'name': 'dimm.temp0.avg.degC', 'address': 0x28114, 'description': 'DIMM_TEMP0_AVG_REG', 'units': 'celsius'}, 64 | 62: {'name': 'dimm.temp0.ins.degC', 'address': 0x28118, 'description': 'DIMM_TEMP0_INS_REG', 'units': 'celsius'}, 65 | 63: {'name': 'dimm.temp1.max.degC', 'address': 0x2811C, 'description': 'DIMM_TEMP1_MAX_REG', 'units': 'celsius'}, 66 | 64: {'name': 'dimm.temp1.avg.degC', 'address': 0x28120, 'description': 'DIMM_TEMP1_AVG_REG', 'units': 'celsius'}, 67 | 65: {'name': 'dimm.temp1.ins.degC', 'address': 0x28124, 'description': 'DIMM_TEMP1_INS_REG', 'units': 'celsius'}, 68 | 66: {'name': 'dimm.temp2.max.degC', 'address': 0x28128, 'description': 'DIMM_TEMP2_MAX_REG', 'units': 'celsius'}, 69 | 67: {'name': 'dimm.temp2.avg.degC', 'address': 0x2812C, 'description': 'DIMM_TEMP2_AVG_REG', 'units': 'celsius'}, 70 | 68: {'name': 'dimm.temp2.ins.degC', 'address': 0x28130, 'description': 'DIMM_TEMP2_INS_REG', 'units': 'celsius'}, 71 | 69: {'name': 'dimm.temp3.max.degC', 'address': 0x28134, 'description': 'DIMM_TEMP3_MAX_REG', 'units': 'celsius'}, 72 | 70: {'name': 'dimm.temp3.avg.degC', 'address': 0x28138, 'description': 'DIMM_TEMP3_AVG_REG', 'units': 'celsius'}, 73 | 71: {'name': 'dimm.temp3.ins.degC', 'address': 0x2813C, 'description': 'DIMM_TEMP3_INS_REG', 'units': 'celsius'}, 74 | 72: {'name': 'se98.temp0.max.degC', 'address': 0x28140, 'description': 'SE98_TEMP0_MAX_REG', 'units': 'celsius'}, 75 | 73: {'name': 'se98.temp0.avg.degC', 'address': 0x28144, 'description': 'SE98_TEMP0_AVG_REG', 'units': 'celsius'}, 76 | 74: {'name': 'se98.temp0.ins.degC', 'address': 0x28148, 'description': 'SE98_TEMP0_INS_REG', 'units': 'celsius'}, 77 | 75: {'name': 'se98.temp1.max.degC', 'address': 0x2814C, 'description': 'SE98_TEMP1_MAX_REG', 'units': 'celsius'}, 78 | 76: {'name': 'se98.temp1.avg.degC', 'address': 0x28150, 'description': 'SE98_TEMP1_AVG_REG', 'units': 'celsius'}, 79 | 77: {'name': 'se98.temp1.ins.degC', 'address': 0x28154, 'description': 'SE98_TEMP1_INS_REG', 'units': 'celsius'}, 80 | 78: {'name': 'se98.temp2.max.degC', 'address': 0x28158, 'description': 'SE98_TEMP2_MAX_REG', 'units': 'celsius'}, 81 | 79: {'name': 'se98.temp2.avg.degC', 'address': 0x2815C, 'description': 'SE98_TEMP2_AVG_REG', 'units': 'celsius'}, 82 | 80: {'name': 'se98.temp2.ins.degC', 'address': 0x28160, 'description': 'SE98_TEMP2_INS_REG', 'units': 'celsius'}, 83 | 81: {'name': 'fan.speed.max.rpm', 'address': 0x28164, 'description': 'FAN_SPEED_MAX_REG', 'units': 'rpm'}, 84 | 82: {'name': 'fan.speed.avg.rpm', 'address': 0x28168, 'description': 'FAN_SPEED_AVG_REG', 'units': 'rpm'}, 85 | 83: {'name': 'fan.speed.ins.rpm', 'address': 0x2816C, 'description': 'FAN_SPEED_INS_REG', 'units': 'rpm'}, 86 | 84: {'name': 'cage.temp0.max.degC', 'address': 0x28170, 'description': 'CAGE_TEMP0_MAX_REG', 'units': 'celsius'}, 87 | 85: {'name': 'cage.temp0.avg.degC', 'address': 0x28174, 'description': 'CAGE_TEMP0_AVG_REG', 'units': 'celsius'}, 88 | 86: {'name': 'cage.temp0.ins.degC', 'address': 0x28178, 'description': 'CAGE_TEMP0_INS_REG', 'units': 'celsius'}, 89 | 87: {'name': 'cage.temp1.max.degC', 'address': 0x2817C, 'description': 'CAGE_TEMP1_MAX_REG', 'units': 'celsius'}, 90 | 88: {'name': 'cage.temp1.avg.degC', 'address': 0x28180, 'description': 'CAGE_TEMP1_AVG_REG', 'units': 'celsius'}, 91 | 89: {'name': 'cage.temp1.ins.degC', 'address': 0x28184, 'description': 'CAGE_TEMP1_INS_REG', 'units': 'celsius'}, 92 | 90: {'name': 'cage.temp2.max.degC', 'address': 0x28188, 'description': 'CAGE_TEMP2_MAX_REG', 'units': 'celsius'}, 93 | 91: {'name': 'cage.temp2.avg.degC', 'address': 0x2818C, 'description': 'CAGE_TEMP2_AVG_REG', 'units': 'celsius'}, 94 | 92: {'name': 'cage.temp2.ins.degC', 'address': 0x28190, 'description': 'CAGE_TEMP2_INS_REG', 'units': 'celsius'}, 95 | 93: {'name': 'cage.temp3.max.degC', 'address': 0x28194, 'description': 'CAGE_TEMP3_MAX_REG', 'units': 'celsius'}, 96 | 94: {'name': 'cage.temp3.avg.degC', 'address': 0x28198, 'description': 'CAGE_TEMP3_AVG_REG', 'units': 'celsius'}, 97 | 95: {'name': 'cage.temp3.ins.degC', 'address': 0x2819C, 'description': 'CAGE_TEMP3_INS_REG', 'units': 'celsius'}, 98 | 96: {'name': 'hbm.temp1.max.degC', 'address': 0x28260, 'description': 'HBM_TEMP1_MAX_REG', 'units': 'celsius'}, 99 | 97: {'name': 'hbm.temp1.avg.degC', 'address': 0x28264, 'description': 'HBM_TEMP1_AVG_REG', 'units': 'celsius'}, 100 | 98: {'name': 'hbm.temp1.ins.degC', 'address': 0x28268, 'description': 'HBM_TEMP1_INS_REG', 'units': 'celsius'}, 101 | 99: {'name': 'vcc3v3.max.mV', 'address': 0x2826C, 'description': 'VCC3V3_MAX_REG', 'units': 'millivolts'}, 102 | 100: {'name': 'vcc3v3.avg.mV', 'address': 0x28270, 'description': 'VCC3V3_AVG_REG', 'units': 'millivolts'}, 103 | 101: {'name': 'vcc3v3.ins.mV', 'address': 0x28274, 'description': 'VCC3V3_INS_REG', 'units': 'millivolts'}, 104 | 102: {'name': '3v3pex.i.in.max.mA', 'address': 0x28278, 'description': '3V3PEX_I_IN_MAX_REG', 'units': 'milliamps'}, 105 | 103: {'name': '3v3pex.i.in.avg.mA', 'address': 0x2827C, 'description': '3V3PEX_I_IN_AVG_REG', 'units': 'milliamps'}, 106 | 104: {'name': '3v3pex.i.in.ins.mA', 'address': 0x28280, 'description': '3V3PEX_I_IN_INS_REG', 'units': 'milliamps'}, 107 | 105: {'name': 'vcc0v85.i.max.mA', 'address': 0x28284, 'description': 'VCC0V85_I_MAX_REG', 'units': 'milliamps'}, 108 | 106: {'name': 'vcc0v85.i.avg.mA', 'address': 0x28288, 'description': 'VCC0V85_I_AVG_REG', 'units': 'milliamps'}, 109 | 107: {'name': 'vcc0v85.i.ins.mA', 'address': 0x2828C, 'description': 'VCC0V85_I_INS_REG', 'units': 'milliamps'}, 110 | 108: {'name': 'hbm.1v2.max.mV', 'address': 0x28290, 'description': 'HBM_1V2_MAX_REG', 'units': 'millivolts'}, 111 | 109: {'name': 'hbm.1v2.avg.mV', 'address': 0x28294, 'description': 'HBM_1V2_AVG_REG', 'units': 'millivolts'}, 112 | 110: {'name': 'hbm.1v2.ins.mV', 'address': 0x28298, 'description': 'HBM_1V2_INS_REG', 'units': 'millivolts'}, 113 | 111: {'name': 'vpp2v5.max.mV', 'address': 0x2829C, 'description': 'VPP2V5_MAX_REG', 'units': 'millivolts'}, 114 | 112: {'name': 'vpp2v5.avg.mV', 'address': 0x282A0, 'description': 'VPP2V5_AVG_REG', 'units': 'millivolts'}, 115 | 113: {'name': 'vpp2v5.ins.mV', 'address': 0x282A4, 'description': 'VPP2V5_INS_REG', 'units': 'millivolts'}, 116 | 114: {'name': 'vccint.bram.max.mV', 'address': 0x282A8, 'description': 'VCCINT_BRAM_MAX_REG', 'units': 'millivolts'}, 117 | 115: {'name': 'vccint.bram.avg.mV', 'address': 0x282AC, 'description': 'VCCINT_BRAM_AVG_REG', 'units': 'millivolts'}, 118 | 116: {'name': 'vccint.bram.ins.mV', 'address': 0x282B0, 'description': 'VCCINT_BRAM_INS_REG', 'units': 'millivolts'}, 119 | 117: {'name': 'hbm.temp2.max.degC', 'address': 0x282B4, 'description': 'HBM_TEMP2_MAX_REG', 'units': 'celsius'}, 120 | 118: {'name': 'hbm.temp2.avg.degC', 'address': 0x282B8, 'description': 'HBM_TEMP2_AVG_REG', 'units': 'celsius'}, 121 | 119: {'name': 'hbm.temp2.ins.degC', 'address': 0x282BC, 'description': 'HBM_TEMP2_INS_REG', 'units': 'celsius'}, 122 | 120: {'name': '12v.aux1.max.mV', 'address': 0x282C0, 'description': '12V_AUX1_MAX_REG', 'units': 'millivolts'}, 123 | 121: {'name': '12v.aux1.avg.mV', 'address': 0x282C4, 'description': '12V_AUX1_AVG_REG', 'units': 'millivolts'}, 124 | 122: {'name': '12v.aux1.ins.mV', 'address': 0x282C8, 'description': '12V_AUX1_INS_REG', 'units': 'millivolts'}, 125 | 123: {'name': 'vccint.temp.max.degC', 'address': 0x282CC, 'description': 'VCCINT_TEMP_MAX_REG', 'units': 'millivolts'}, 126 | 124: {'name': 'vccint.temp.avg.degC', 'address': 0x282D0, 'description': 'VCCINT_TEMP_AVG_REG', 'units': 'millivolts'}, 127 | 125: {'name': 'vccint.temp.ins.degC', 'address': 0x282D4, 'description': 'VCCINT_TEMP_INS_REG', 'units': 'millivolts'}, 128 | 126: {'name': 'pex.12v.power.max.mW', 'address': 0x282D8, 'description': 'PEX_12V_POWER_MAX_REG', 'units': 'milliwatts'}, 129 | 127: {'name': 'pex.12v.power.avg.mW', 'address': 0x282DC, 'description': 'PEX_12V_POWER_AVG_REG', 'units': 'milliwatts'}, 130 | 128: {'name': 'pex.12v.power.ins.mW', 'address': 0x282E0, 'description': 'PEX_12V_POWER_INS_REG', 'units': 'milliwatts'}, 131 | 129: {'name': 'pex.3v3.power.max.mW', 'address': 0x282E4, 'description': 'PEX_3V3_POWER_MAX_REG', 'units': 'milliwatts'}, 132 | 130: {'name': 'pex.3v3.power.avg.mW', 'address': 0x282E8, 'description': 'PEX_3V3_POWER_AVG_REG', 'units': 'milliwatts'}, 133 | 131: {'name': 'pex.3v3.power.ins.mW', 'address': 0x282EC, 'description': 'PEX_3V3_POWER_INS_REG', 'units': 'milliwatts'}, 134 | 132: {'name': 'aux.3v3.i.max.mA', 'address': 0x282F0, 'description': 'AUX_3V3_I_MAX_REG', 'units': 'milliamps'}, 135 | 133: {'name': 'aux.3v3.i.avg.mA', 'address': 0x282F4, 'description': 'AUX_3V3_I_AVG_REG', 'units': 'milliamps'}, 136 | 134: {'name': 'aux.3v3.i.ins.mA', 'address': 0x282F8, 'description': 'AUX_3V3_I_INS_REG', 'units': 'milliamps'}, 137 | 135: {'name': 'vcc1v2.i.max.mA', 'address': 0x28314, 'description': 'VCC1V2_I_MAX_REG', 'units': 'milliamps'}, 138 | 136: {'name': 'vcc1v2.i.avg.mA', 'address': 0x28318, 'description': 'VCC1V2_I_AVG_REG', 'units': 'milliamps'}, 139 | 137: {'name': 'vcc1v2.i.ins.mA', 'address': 0x2831C, 'description': 'VCC1V2_I_INS_REG', 'units': 'milliamps'}, 140 | 138: {'name': 'v12.in.i.max.mA', 'address': 0x28320, 'description': 'V12_IN_I_MAX_REG', 'units': 'milliamps'}, 141 | 139: {'name': 'v12.in.i.avg.mA', 'address': 0x28324, 'description': 'V12_IN_I_AVG_REG', 'units': 'milliamps'}, 142 | 140: {'name': 'v12.in.i.ins.mA', 'address': 0x28328, 'description': 'V12_IN_I_INS_REG', 'units': 'milliamps'}, 143 | 141: {'name': 'v12.in.aux0.i.max.mA', 'address': 0x2832C, 'description': 'V12_IN_AUX0_I_MAX_REG', 'units': 'milliamps'}, 144 | 142: {'name': 'v12.in.aux0.i.avg.mA', 'address': 0x28330, 'description': 'V12_IN_AUX0_I_AVG_REG', 'units': 'milliamps'}, 145 | 143: {'name': 'v12.in.aux0.i.ins.mA', 'address': 0x28334, 'description': 'V12_IN_AUX0_I_INS_REG', 'units': 'milliamps'}, 146 | 144: {'name': 'v12.in.aux1.i.max.mA', 'address': 0x28338, 'description': 'V12_IN_AUX1_I_MAX_REG', 'units': 'milliamps'}, 147 | 145: {'name': 'v12.in.aux1.i.avg.mA', 'address': 0x2833C, 'description': 'V12_IN_AUX1_I_AVG_REG', 'units': 'milliamps'}, 148 | 146: {'name': 'v12.in.aux1.i.ins.mA', 'address': 0x28340, 'description': 'V12_IN_AUX1_I_INS_REG', 'units': 'milliamps'}, 149 | 147: {'name': 'vccaux.max.mV', 'address': 0x28344, 'description': 'VCCAUX_MAX_REG', 'units': 'millivolts'}, 150 | 148: {'name': 'vccaux.avg.mV', 'address': 0x28348, 'description': 'VCCAUX_AVG_REG', 'units': 'millivolts'}, 151 | 149: {'name': 'vccaux.ins.mV', 'address': 0x2834C, 'description': 'VCCAUX_INS_REG', 'units': 'millivolts'}, 152 | 150: {'name': 'vccaux.pmc.max.mV', 'address': 0x28350, 'description': 'VCCAUX_PMC_MAX_REG', 'units': 'millivolts'}, 153 | 151: {'name': 'vccaux.pmc.avg.mV', 'address': 0x28354, 'description': 'VCCAUX_PMC_AVG_REG', 'units': 'millivolts'}, 154 | 152: {'name': 'vccaux.pmc.ins.mV', 'address': 0x28358, 'description': 'VCCAUX_PMC_INS_REG', 'units': 'millivolts'}, 155 | 153: {'name': 'vccram.max.mV', 'address': 0x2835C, 'description': 'VCCRAM_MAX_REG', 'units': 'millivolts'}, 156 | 154: {'name': 'vccram.avg.mV', 'address': 0x28360, 'description': 'VCCRAM_AVG_REG', 'units': 'millivolts'}, 157 | 155: {'name': 'vccram.ins.mV', 'address': 0x28364, 'description': 'VCCRAM_INS_REG', 'units': 'millivolts'}, 158 | 156: {'name': 'power.good.ins.not_status','address': 0x28370, 'description': 'POWER_GOOD_INS_REG', 'units': 'not_status'} 159 | } 160 | -------------------------------------------------------------------------------- /alveo-katcp-svr/alveo-katcp-svr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import aiokatcp 4 | import asyncio 5 | import socket 6 | import os 7 | import argparse 8 | import logging 9 | import numpy as np 10 | import sys, subprocess 11 | 12 | from AlveoIO import AlveoUtils 13 | 14 | #NOTES: 15 | # all filesystem operation only allowed on the given working directory 16 | 17 | PCIE_BAR_SIZE = 1 << 28 18 | 19 | class MyServer(aiokatcp.DeviceServer): 20 | 21 | try: 22 | commit = subprocess.check_output('./version.sh').decode() 23 | 24 | except: 25 | commit = 'unknown' 26 | 27 | VERSION = 'serial-number-unknown' 28 | BUILD_STATE = f'alveo-katcp-server-{commit}' 29 | #wd = os.path.abspath('.') 30 | #alveo = AlveoPcieUtils('alveo_0_u50', wd) 31 | 32 | def __init__(self, ip, port, alveo_ref, wd, card, *args, **kwargs): 33 | super().__init__(ip, port, *args, **kwargs) 34 | self.wd = wd #os.path.abspath('.') 35 | #self.alveo = AlveoPcieUtils('alveo_0_u50', self.wd) 36 | self.alveo = AlveoUtils(alveo_ref, self.wd, PCIE_BAR_SIZE) 37 | sensor = aiokatcp.Sensor( 38 | str, 39 | "alveo-sn", 40 | "alveo serial number", 41 | default=card, 42 | initial_status=aiokatcp.Sensor.Status.NOMINAL, 43 | ) 44 | self.sensors.add(sensor) 45 | self.VERSION = card 46 | 47 | self._populate_sensors() 48 | self.add_service_task(asyncio.create_task(self._sensor_update())) 49 | 50 | # sensor = aiokatcp.Sensor( 51 | # float, 52 | # "uptime", 53 | # "uptime of server since boot", 54 | # default=0, 55 | # initial_status=aiokatcp.Sensor.Status.NOMINAL, 56 | # ) 57 | # self.sensors.add(sensor) 58 | # self.add_service_task(asyncio.create_task(self._alter_sensors())) 59 | # #TODO check if there's not a callback that gets run when sensor-value 60 | # # is called to hook into 61 | # 62 | # async def _service_task(self) -> None: 63 | # """Example service task that broadcasts to clients.""" 64 | # while True: 65 | # #TODO this feels like a lot of filesystem IO too often (?) 66 | # await asyncio.sleep(1) 67 | # with open('/proc/uptime', 'r') as f: 68 | # uptime_seconds = float(f.readline().split()[0]) 69 | # #self.mass_inform("uptime", uptime_seconds) 70 | # self.sensors["uptime"].value = uptime_seconds 71 | 72 | 73 | def _populate_sensors(self): 74 | sensors = self.alveo.get_alveo_sensor_map 75 | for idx in range(0,len(sensors)): 76 | katcp_sensor = aiokatcp.Sensor( 77 | int, 78 | sensors[idx]['name'], 79 | sensors[idx]['description'], 80 | units=sensors[idx]['units'], 81 | default=0, 82 | initial_status=aiokatcp.Sensor.Status.NOMINAL, 83 | ) 84 | self.sensors.add(katcp_sensor) 85 | 86 | 87 | async def _sensor_update(self) -> None: 88 | """Update the sensor values every second""" 89 | #TODO: this is written fro demonstration purposes first, there may be 90 | # a more performance efficient way to do this than single sensor updates 91 | sensors = self.alveo.get_alveo_sensor_map 92 | while True: 93 | await asyncio.sleep(1) 94 | for idx in range(0,len(sensors)): 95 | value = self.alveo.wordread(sensors[idx]['address'], type_obj='int') 96 | self.sensors[sensors[idx]['name']].value = value[0] 97 | 98 | 99 | async def request_greet(self, ctx, name: str) -> None: 100 | """Take a person's name and greet them""" 101 | ctx.inform(name) 102 | #return 'hello', name 103 | raise aiokatcp.FailReply(f"{name} error") 104 | #return 105 | 106 | #TODO turn the params like dev path and pci addr into sensors rather 107 | async def request_alveo_dev(self, ctx): 108 | """get alveo dev reference""" 109 | return self.alveo.get_dev_path 110 | 111 | async def request_alveo_pci(self, ctx): 112 | """get alveo pci addr""" 113 | return self.alveo.get_pci_addr 114 | 115 | async def request_alveo_reset(self, ctx): 116 | """reset alveo""" 117 | return self.alveo.reset() 118 | 119 | 120 | 121 | 122 | async def request_alveo_memread(self, ctx, addr: str, words='1'): 123 | """read addr from memory (?alveo-memread addr [words])""" 124 | #TODO perhaps only allow hex values - throw error here if not 125 | addr_int = int(addr, 0) 126 | 127 | #NOTE it's possible to read multiple words but for now, keeping it 128 | #to one word reads 129 | #words_int = int(words, 0) 130 | words_int = 1 131 | #return self.alveo.mem_read(addr_int, words_int) 132 | ctx.inform(addr, *self.alveo.wordread(addr_int, words_int, 'hex')) 133 | return 134 | 135 | 136 | 137 | async def request_alveo_memwrite(self, ctx, addr: str, data: str) -> None: 138 | """read addr from memory""" 139 | 140 | #TODO perhaps only allow hex values - throw error here if not 141 | print(data) 142 | 143 | addr_int = int(addr, 0) #set arg to string and then 144 | #convert in order to allow for 0x.. prefix 145 | data_int = int(data, 0) 146 | 147 | try: 148 | data = int(data, 0).to_bytes(4, 'little') 149 | except OverflowError as err: 150 | raise aiokatcp.FailReply(f'{err} - 32-bit data words required') 151 | except ValueError: 152 | raise aiokatcp.FailReply("only numeric data allowed") 153 | 154 | #TODO - multiple word writes 155 | 156 | #return self.alveo.mem_write(addr_int, data_int) 157 | self.alveo.wordwrite(addr_int, data) 158 | 159 | #TODO redback!!!! 160 | # #read back and verify 161 | (readback_data,) = self.alveo.wordread(addr_int, 1, 'hex') 162 | # #print(readback_data) 163 | # #print(type(readback_data)) 164 | if readback_data == hex(data_int): 165 | ctx.inform(addr, readback_data) 166 | else: 167 | raise aiokatcp.FailReply(f'write-error @ {addr}') 168 | 169 | 170 | async def request_alveo_program(self, ctx, bitfile='default'): 171 | """program bitfile""" 172 | if bitfile == 'default': 173 | bf = self.alveo.assoc_binary 174 | else: 175 | bf = bitfile.decode() #bytes to string 176 | 177 | bf = os.path.abspath(bf) 178 | 179 | if not os.path.exists(bf): 180 | raise aiokatcp.connection.FailReply("bit file note found") 181 | else: 182 | self.alveo.program(bf) 183 | #print(self.wd) 184 | return 185 | 186 | async def request_listbof(self, ctx): 187 | """list files""" 188 | ls = os.listdir(self.wd) 189 | #print(ls) 190 | for d in ls: 191 | ctx.inform(d) 192 | return 193 | 194 | 195 | async def request_listdev(self, ctx): 196 | """ lists available registers""" 197 | #self.alveo._parse_fpg('casper_pfb_au50_1k_8in_1pol_2022-11-11_1146.fpg') 198 | devs = self.alveo.get_named_reg_map 199 | if not devs: 200 | ctx.inform('fpg not mapped') 201 | raise aiokatcp.FailReply('unampped') 202 | else: 203 | keys = [name for name in devs.keys()] 204 | #print(keys) 205 | for d in keys: 206 | ctx.inform(d) 207 | #ctx.inform(*tuple(d for d in devs.keys())) 208 | return 209 | 210 | 211 | async def request_wordread(self, ctx, 212 | named_reg: str, 213 | offsets=str(0), 214 | size=str(1)): 215 | """read a number of words from the given byte offset (wordread register_name offset_in_words size_in_words)""" 216 | 217 | #offsets should be in the format word_offset:bit_offset 218 | #OR simply just word_offset - this is to maintain backw 219 | #compatibility with tcpbs 220 | #print(type(offsets)) 221 | #print(type(size)) 222 | 223 | if type(offsets) == bytes: 224 | offsets = offsets.decode() 225 | 226 | if type(size) == bytes: 227 | size = size.decode() 228 | 229 | if not offsets.count(':') in [0 , 1]: 230 | raise aiokatcp.FailReply('offset syntax error') 231 | 232 | try: 233 | word_offset_int = int(offsets.split(':')[0], 0) 234 | except ValueError: 235 | raise aiokatcp.FailReply('offset syntax error') 236 | 237 | try: #user may not have supplied ':bit_offset' 238 | bit_offset_int = int(offsets.split(':')[1], 0) 239 | except IndexError: 240 | #TODO bit_offset seems unused in tcpbs impl anyway? 241 | pass 242 | 243 | try: 244 | size_int = int(size, 0) 245 | except ValueError: 246 | raise aiokatcp.FailReply('length syntax error') 247 | 248 | if size_int < 1: 249 | raise aiokatcp.FailReply('read length less than one') 250 | 251 | #do some addressing calculations 252 | 253 | 254 | devs = self.alveo.get_named_reg_map 255 | if not devs: 256 | ctx.inform('fpg not mapped') 257 | raise aiokatcp.FailReply('unmapped') 258 | 259 | if named_reg in devs: 260 | try: 261 | data = self.alveo.named_wordread(named_reg, word_offset_int, size_int, 'hex') 262 | except BufferError: 263 | raise aiokatcp.FailReply('attempting to read past register boundary') 264 | except ValueError as err: 265 | raise aiokatcp.FailReply(err) 266 | #except: 267 | #TODO: return better error info to user - raise custom exceptions from alveo obj 268 | #and catch them here 269 | # raise aiokatcp.FailReply('read error') 270 | else: 271 | raise aiokatcp.FailReply(f'{named_reg} not found') 272 | 273 | return data 274 | 275 | 276 | async def request_wordwrite(self, ctx, named_reg: str, index: str, *data: str) -> None: 277 | """wordwrite""" 278 | """write a number of wordsto the given byte offset (wordwrite register_name offset_in_words data)""" 279 | 280 | try: 281 | data = tuple([int(d, 0).to_bytes(4, 'little') for d in data]) 282 | except OverflowError as err: 283 | raise aiokatcp.FailReply(f'{err} - 32-bit data words required') 284 | except ValueError: 285 | raise aiokatcp.FailReply("only numeric data allowed") 286 | 287 | index = int(index, 0) 288 | 289 | # data_int = int(data, 0) #set arg to string and then 290 | # #convert in order to allow for 0x.. prefix 291 | # 292 | # index_int = int(index, 0) #TODO for backwrd compatibility with tcpbs 293 | # 294 | # #TODO: do we want to guard this if fpg parsed ie we have device list but 295 | # #fpga not programmed - since these are two decoupled commands. Should 296 | # #really think about combining this functionality (progdev and program) - 297 | # #this is a legacy artefact 298 | # 299 | devs = self.alveo.get_named_reg_map 300 | if not devs: 301 | ctx.inform('fpg not mapped') 302 | raise aiokatcp.FailReply('unmapped') 303 | 304 | if named_reg in devs: 305 | try: 306 | self.alveo.named_wordwrite(named_reg, index, *data) 307 | except BufferError: 308 | raise aiokatcp.FailReply('attempting to write past register boundary') 309 | except ValueError as err: 310 | raise aiokatcp.FailReply(err) 311 | except: 312 | raise aiokatcp.FailReply('write error') 313 | else: 314 | raise aiokatcp.FailReply(f'{named_reg} not found') 315 | 316 | # 317 | # #read back and verify 318 | # (readback_data,) = self.alveo.named_read(named_reg, 1, index_int) 319 | # #print(readback_data) 320 | # #print(type(readback_data)) 321 | # if readback_data == hex(data_int): 322 | # ctx.inform(named_reg, readback_data) 323 | # else: 324 | # raise aiokatcp.FailReply(f'write-error @ {named_reg}') 325 | # 326 | 327 | async def request_progdev(self, ctx, filename: str) -> None: 328 | """progdev""" 329 | self.alveo._parse_fpg(filename) 330 | 331 | async def request_status(self, ctx): 332 | """fpga status""" 333 | #TODO actually check status and state 334 | #dummy function for now to satify casperfpga program cmd flow 335 | return 336 | 337 | async def request_delbof(self, ctx, filename): 338 | """fpga status""" 339 | #TODO actually do the work 340 | #dummy function for now to satify casperfpga program cmd flow 341 | logging.getLogger().info("testing the logging") 342 | return 343 | 344 | 345 | ###################################################################### 346 | 347 | async def request_write(self, ctx, named_reg: str, offset: str, data: bytes) -> None: 348 | """write""" 349 | #print(type(data)) 350 | #print('data', data) 351 | #print('offset', offset) 352 | 353 | data_bytes = tuple([el.to_bytes(1,'little') for el in data]) 354 | 355 | #TODO have to sort out the write sizes and offsets in all read and write functions 356 | 357 | #data_int = int.from_bytes(data, byteorder='little') 358 | offset_int = int(offset, 0) #TODO for backwrd compatibility with tcpbs 359 | #print('wr offset ', offset_int) 360 | 361 | devs = self.alveo.get_named_reg_map 362 | if not devs: 363 | ctx.inform('fpg not mapped') 364 | raise aiokatcp.FailReply('unmapped') 365 | 366 | try: 367 | self.alveo.named_bytewrite(named_reg, offset_int, *data_bytes) 368 | except Exception as err: 369 | raise aiokatcp.FailReply(err) 370 | 371 | 372 | async def request_read(self, ctx, 373 | named_reg: str, 374 | offset=str(0), #offset in bytes 375 | read_len_bytes=str(4)): #length in bytes 376 | """read""" 377 | len_int = int(read_len_bytes, 0) 378 | offset_int = int(offset, 0) 379 | 380 | devs = self.alveo.get_named_reg_map 381 | if not devs: 382 | ctx.inform('fpg not mapped') 383 | raise aiokatcp.FailReply('unmapped') 384 | 385 | try: 386 | data = self.alveo.named_byteread(named_reg, offset_int, len_int) 387 | #above function returns a tuple of one-byte elements 388 | #need to concatenate into one long byte-string 389 | bytestring = b''.join(data) 390 | 391 | except Exception as err: 392 | raise aiokatcp.FailReply(err) 393 | else: 394 | return bytestring 395 | 396 | # 397 | # #NOTES: historiclly, this function allowed bit- and byte-grained 398 | # #access to register reads. In bid to get it working on the alveo, 399 | # #will only do word accesses for now - ie. round up to the nearest 400 | # #word boundary in byte length 401 | # read_len_bytes = int(4 * np.ceil(len_int/4)) 402 | # print(read_len_bytes) 403 | # read_len_words = (read_len_bytes) >> 2 404 | # print(read_len_words) 405 | # 406 | # FIXME actually receiving a byte offset 407 | # word_offset_int = int(offset, 0) >> 2 408 | # 409 | # data_string = self.alveo.named_wordread(named_reg, read_len_words, word_offset_int) 410 | # data_string = self.alveo.named_read(named_reg, read_len_words, 0) 411 | # data_string = (b'\xde\xad\xbe\xef',) 412 | # 413 | ## print(data_string) 414 | ## for word in data_string: 415 | ## print(word) 416 | ## for b in range(0,4): 417 | ## byte = (int(word,0) >> (b*8)) & 0xff 418 | ## print(f'0x{byte:x}') 419 | # 420 | # #f'{int(b[i],0) : x}') 421 | # data_string_fmtd = f'{int(data_string[0],0):x}' 422 | # print(data_string_fmtd) 423 | 424 | #return b'\x31\x32\x33\x00' 425 | #return bytes.fromhex(data_string_fmtd) 426 | # byte_str = b'' 427 | # for word in data_string: 428 | # for b in range(0,4): 429 | # byte = (int(word,0) >> (b*8)) & 0xff 430 | # byte_str = b''.join([byte_str,byte.to_bytes(1,'big')]) 431 | 432 | #print(byte_str) 433 | #print(type(byte_str)) 434 | # return byte_str 435 | 436 | ####################################################################### 437 | async def request_saveremote(self, ctx, port: str, filename: str): 438 | """Upload a file to the remote filesystem""" 439 | SVR = "0.0.0.0" 440 | #PORT = 10000 441 | PORT = int(port) 442 | 443 | CHUNK = 4096 444 | 445 | socket.setdefaulttimeout(10) 446 | #socket.settimeout(10) 447 | sock = socket.socket() 448 | sock.bind((SVR, PORT)) 449 | sock.listen(1) 450 | 451 | print(f"[*] Listening as {SVR}:{PORT}") 452 | 453 | try: 454 | client_socket, address = sock.accept() 455 | except socket.timeout: 456 | raise aiokatcp.FailReply(f'timeout') 457 | 458 | 459 | print(f"[+] {address} is connected.") 460 | 461 | count=0 462 | 463 | with open(filename, "wb") as f: 464 | while True: 465 | bytes_read = client_socket.recv(CHUNK) 466 | if not bytes_read: 467 | # nothing is received 468 | # file transmitting is done 469 | break 470 | # write to the file the bytes we just received 471 | if count % 100 == 0: 472 | print(count) 473 | count=count+1 474 | f.write(bytes_read) 475 | 476 | f.close() 477 | # close the client socket 478 | client_socket.close() 479 | # close the server socket 480 | sock.close() 481 | return 482 | 483 | ####################################################################### 484 | async def main(foo): 485 | 486 | #alveo = args.alveo 487 | #workdir=args.workdir 488 | 489 | server = MyServer('0.0.0.0', foo.port, foo.alveo, foo.workdir, foo.card) 490 | 491 | logging.basicConfig(level=logging.INFO) 492 | handler = MyServer.LogHandler(server) 493 | logging.getLogger().addHandler(handler) 494 | 495 | await server.start() 496 | await server.join() 497 | 498 | def daemonise(): 499 | try: 500 | pid = os.fork() 501 | if pid > 0: 502 | # exit first parent 503 | sys.exit(0) 504 | except OSError as e: 505 | sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) 506 | sys.exit(1) 507 | 508 | # decouple from parent environment 509 | os.chdir("/") 510 | os.setsid() 511 | os.umask(0) 512 | 513 | # do second fork 514 | try: 515 | pid = os.fork() 516 | if pid > 0: 517 | # exit from second parent 518 | sys.exit(0) 519 | except OSError as e: 520 | sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) 521 | sys.exit(1) 522 | 523 | # # redirect standard file descriptors 524 | # sys.stdout.flush() 525 | # sys.stderr.flush() 526 | # si = file(self.stdin, 'r') 527 | # so = file(self.stdout, 'a+') 528 | # se = file(self.stderr, 'a+', 0) 529 | # os.dup2(si.fileno(), sys.stdin.fileno()) 530 | # os.dup2(so.fileno(), sys.stdout.fileno()) 531 | # os.dup2(se.fileno(), sys.stderr.fileno()) 532 | 533 | 534 | if __name__ == '__main__': 535 | parser = argparse.ArgumentParser() 536 | parser.add_argument('-a', '--alveo', required=True, metavar='', type=str) 537 | parser.add_argument('-w', '--workdir', required=True, metavar='', type=str) 538 | parser.add_argument('-p', '--port', required=True, metavar='', type=str) 539 | parser.add_argument('-c', '--card', required=True, metavar='', type=str) 540 | parser.add_argument('-d', '--daemon', action='store_true') 541 | 542 | args = parser.parse_args() 543 | 544 | if args.daemon == True: 545 | daemonise() 546 | #at this point we should be in the daemon process??? 547 | 548 | asyncio.get_event_loop().run_until_complete(main(args)) 549 | asyncio.get_event_loop().close() 550 | -------------------------------------------------------------------------------- /alveo-katcp-svr/AlveoIO.py: -------------------------------------------------------------------------------- 1 | import os, mmap, sys, subprocess, zlib 2 | import numpy as np 3 | from time import sleep 4 | #from casperfpga import utils 5 | import fpgparser 6 | import AlveoSensors as sensors 7 | 8 | 9 | class _AlveoMemMap: 10 | """ 11 | Class representing Alveo memory map and associated I/O 12 | """ 13 | 14 | def __init__(self, devfile, size): 15 | """ 16 | Instantiate the object representing the memory map 17 | :param devfile: typically a device file but regular files could be used for testing 18 | :param size: size to map 19 | """ 20 | self.__pcie_bar_size = size 21 | fd = os.open(devfile, os.O_RDWR | os.O_SYNC) 22 | self.__mm = mmap.mmap(fd, 23 | self.__pcie_bar_size, 24 | mmap.MAP_SHARED, 25 | mmap.PROT_READ | mmap.PROT_WRITE, 26 | offset=0) 27 | #we don't need to keep the fd around 28 | os.close(fd) 29 | 30 | 31 | def __del__(self): 32 | self.__mm.close() 33 | self.__mm = None 34 | 35 | 36 | def wordread_aligned(self, offset, size=1) -> tuple: 37 | """ 38 | Read 32-bit aligned words from the mapped memory region 39 | :param offset: offset in bytes 40 | :param size: number of words to read, default=1 41 | :return: tuple containing bytes of read data 42 | """ 43 | if (offset % 4) != 0: 44 | raise MemoryError("offset not aligned to 32-bit word boundary") 45 | 46 | size_in_bytes = size << 2 47 | 48 | if (offset + size_in_bytes) > self.__pcie_bar_size: 49 | raise MemoryError("attempting buffer over-read") 50 | 51 | #expose mem-region-of-interest as 1-D np array 52 | buff = np.frombuffer(self.__mm, np.uint32, size, offset) 53 | 54 | #tmp = [hex(x) for x in buff] 55 | tmp = [x.tobytes() for x in buff] 56 | return (*tmp,) #return tuple here since this is what katcp lib expects 57 | 58 | 59 | def wordwrite_aligned(self, offset, *data_words): 60 | """ 61 | Write 32-bit data words to word aligned offsets 62 | :param offset: offset in bytes 63 | :param data: data to write in byte-like format 64 | e.g. wordwrite_aligned(0, b'1234', b'\\x01\\x02\\x03\\x04') 65 | """ 66 | 67 | #slightly weird way to check if all elements in the data arg (tuple) 68 | #are byte-strings - relying on min([...]) to return False if any item 69 | #in the list is False 70 | if min([isinstance(el,bytes) for el in data_words]) == False: 71 | raise ValueError("expecting byte string data") 72 | 73 | if offset < 0: 74 | raise ValueError("offset must be positive") 75 | 76 | if (offset % 4) != 0: 77 | raise MemoryError("unaligned access") 78 | 79 | #check that each word has 4 bytes 80 | if max(len(d) for d in data_words) != 4: 81 | raise ValueError("each byte-element must be of size 4") 82 | 83 | if min(len(d) for d in data_words) != 4: 84 | raise ValueError("each byte-element must be of size 4") 85 | 86 | #get the number of words requested to be written 87 | size_in_words = len(data_words) 88 | 89 | size_in_bytes = size_in_words << 2 90 | 91 | if (offset + size_in_bytes) > self.__pcie_bar_size: 92 | raise MemoryError("attempting buffer over-write") 93 | 94 | #expose mem-mapped-region-of-interest as 1-D array 95 | buff_array = np.frombuffer(self.__mm, np.uint32, size_in_words, offset) 96 | 97 | #create a np array with the data 98 | data_array = np.frombuffer(b''.join(data_words), np.uint32, size_in_words, 0) 99 | 100 | #assign data to memory region 101 | buff_array[0:size_in_words] = data_array 102 | 103 | 104 | def byteread(self, offset, size=1): 105 | """ 106 | Read byte(s) in mapped memory from a given offset 107 | :param offset: offset in bytes to start reading from 108 | :param size: number of bytes to read 109 | :return: tuple of bytes read containing byte-strings 110 | """ 111 | 112 | if offset < 0: 113 | raise ValueError("offset must be positive") 114 | 115 | if (offset + size) > self.__pcie_bar_size: 116 | raise MemoryError("attempting buffer over-read") 117 | 118 | #expose mem-region-of-interest as 1-D np array of uint8 elements 119 | buff = np.frombuffer(self.__mm, np.uint8, size, offset) 120 | 121 | tmp = [x.tobytes() for x in buff] 122 | return (*tmp,) #return tuple here since this is what katcp lib expects 123 | 124 | 125 | def bytewrite(self, offset, *data_bytes): 126 | """ 127 | Write byte(s) to mapped memory at a given byte offset 128 | :param offset: offset in bytes 129 | :param data_bytes: bytes to write in b'' format 130 | """ 131 | 132 | if min([isinstance(el,bytes) for el in data_bytes]) == False: 133 | raise ValueError("expecting byte string data") 134 | 135 | if offset < 0: 136 | raise ValueError("offset must be positive") 137 | 138 | #check that each byte is one byte long 139 | if max(len(d) for d in data_bytes) != 1: 140 | raise ValueError("each byte-element must be of size 1") 141 | 142 | if min(len(d) for d in data_bytes) != 1: 143 | raise ValueError("each byte-element must be of size 1") 144 | 145 | size_in_bytes = len(data_bytes) 146 | 147 | if (offset + size_in_bytes) > self.__pcie_bar_size: 148 | raise MemoryError("attempting buffer over-write") 149 | 150 | #expose mem-mapped-region-of-interest as 1-D array 151 | buff_array = np.frombuffer(self.__mm, np.uint8, size_in_bytes, offset) 152 | 153 | #create a np array with the data 154 | data_array = np.frombuffer(b''.join(data_bytes), np.uint8, size_in_bytes, 0) 155 | 156 | #assign data to memory region 157 | buff_array[0:size_in_bytes] = data_array 158 | 159 | 160 | 161 | class AlveoUtils: 162 | """ 163 | Class to wrap Alveo untilities nd I/O 164 | """ 165 | 166 | def __init__(self, alveo_ref, work_dir, pcie_bar_size): 167 | """ 168 | Instantiate the AlveoUtils object 169 | :param alveo_ref: specify the taget alveo in format alveo_N_uXXX 170 | where N is the index and XXX is the type e.g. alveo_0_u50 171 | run bash cmd 'ls -l /dev/xdma' to get references 172 | :param work_dir: working directory for file I/O 173 | :param pcie_bar_size: size of the pcie bar set in the xdma axi4lite interface 174 | """ 175 | 176 | #firstly verify referenced alveo 177 | if not self.__check_valid_alveo(alveo_ref): 178 | raise RuntimeError("not a valid alveo reference") 179 | 180 | #private variables 181 | self.__base_path = f'/dev/xdma/' 182 | self.__devpath = f'{self.__base_path}{alveo_ref}' 183 | self.__alveo_ref = alveo_ref 184 | self.__pciaddr = self.__get_pci_addr() 185 | self.__pcie_bar_size = pcie_bar_size 186 | self.__mm = _AlveoMemMap(f'{self.__devpath}/user', pcie_bar_size) 187 | self.__named_reg_map = {} 188 | self.__DSP_BASE_OFFSET = 0x8000000 189 | self.__work_dir = work_dir 190 | self.__assoc_binary = f'' 191 | 192 | 193 | #private methods 194 | def __check_valid_alveo(self, alveo_ref): 195 | """check if the given alveo reference is valid""" 196 | exists = os.path.exists(f'/dev/xdma/{alveo_ref}') 197 | return exists 198 | 199 | 200 | def __get_pci_addr(self): 201 | """return alveo pci addr in format ::.""" 202 | return os.readlink(f'{self.__devpath}/pci') 203 | 204 | 205 | def __tear_down(self): 206 | """ 207 | this method destroys the mmap obj among other things 208 | usually done before reprogramming the fpga 209 | """ 210 | del self.__mm 211 | 212 | 213 | #less private methods 214 | def _remove_from_bus(self): 215 | if os.geteuid() != 0: 216 | raise EnvironmentError('Root permissions required.') 217 | 218 | with open(f'{self.__devpath}/pci/remove', 'w') as f: 219 | f.write(str(1)) 220 | 221 | 222 | def _rescan_bus(self): 223 | if os.geteuid() != 0: 224 | raise EnvironmentError('Root permissions required.') 225 | 226 | with open(f'/sys/bus/pci/rescan', 'w') as f: 227 | f.write(str(1)) 228 | 229 | 230 | def _reset_pci(self): 231 | if os.geteuid() != 0: 232 | raise EnvironmentError('Root permissions required.') 233 | self._remove_from_bus() 234 | sleep(2) 235 | self._rescan_bus() 236 | sleep(2) 237 | #TODO at this point, should we re-read all the private variables in case something changed? 238 | #although it shouldn't since it is "statically" set up in the linux alveo infrastructure 239 | 240 | 241 | #public getters 242 | @property 243 | def get_dev_path(self): 244 | """getter for alveo device path reference""" 245 | return (self.__devpath) 246 | 247 | 248 | @property 249 | def get_pci_addr(self): 250 | """getter for alveo pci address in format ::.""" 251 | tmp = self.__pciaddr.strip('/') #remove trailing & leading '/' 252 | return (tmp.split('/')[-1]) 253 | 254 | 255 | @property 256 | def assoc_binary(self): 257 | """path of bitfile/binfile associated with this alveo instance""" 258 | return self.__assoc_binary 259 | 260 | 261 | @assoc_binary.setter 262 | def set_assoc_binary_ext(self, extension): 263 | """setter for the name of the associated binary - .bit or .bin""" 264 | 265 | if extension not in ['bit', 'bin']: 266 | raise RuntimeError('not a valid extension') 267 | 268 | tmp = self.get_pci_addr 269 | for char in [':','.']: 270 | if char in tmp: 271 | tmp = tmp.replace(char, '-') 272 | self.__assoc_binary = f'{self.__work_dir}/alveo-{tmp}.{extension}' 273 | 274 | 275 | @property 276 | def get_named_reg_map(self): 277 | """getter to return the named registers from the fpg in the current context state""" 278 | return self.__named_reg_map 279 | 280 | 281 | @property 282 | def get_dsp_reg_offset(self): 283 | """getter to return the offset to dsp reg space on AXI bus (fixed at design time)""" 284 | return self.__DSP_BASE_OFFSET 285 | 286 | @property 287 | def get_alveo_sensor_map(self): 288 | """getter to return the map of the sensors internal to the alveo""" 289 | return sensors.alveo_sensor_map 290 | 291 | 292 | #public methods 293 | def reset(self) -> None: 294 | self.__tear_down() 295 | self._reset_pci() 296 | 297 | #remap memmap 298 | self.__mm = _AlveoMemMap(f'{self.__devpath}/user', self.__pcie_bar_size) 299 | self.__mm.bytewrite(0x20000,b'\x01') #start CMS internal ublaze 300 | 301 | 302 | def wordread(self, addr, size=1, type_obj='bytes') -> tuple: 303 | """ 304 | Read words from mapped memory 305 | :param addr: base address (32-bit aligned) to read from 306 | :param size: number of 32-bit words to read 307 | :param type_obj: return type of data - 'bytes', 'int' or 'hex' 308 | :return: tuple containing words read in byte-string format 309 | """ 310 | #TODO some sanity checks on args 311 | if type_obj not in ['int', 'bytes', 'hex']: 312 | raise ValueError("type %s not available" % type_obj) 313 | 314 | data_bytes = self.__mm.wordread_aligned(addr, size) 315 | 316 | if type_obj == 'bytes': 317 | return data_bytes 318 | elif type_obj == 'int': 319 | #convert each element to int then return tuple 320 | return tuple([int.from_bytes(el, byteorder='little') for el in data_bytes]) 321 | elif type_obj == 'hex': 322 | return tuple([hex(int.from_bytes(el, byteorder='little')) for el in data_bytes]) 323 | 324 | 325 | def wordwrite(self, addr, *data): 326 | self.__mm.wordwrite_aligned(addr, *data) 327 | #TODO read back, compare and return status, or leave this to caller to do? 328 | 329 | 330 | def named_wordread(self, reg_name, offset=0, size=1, type_obj='bytes'): 331 | """ 332 | Read a named register 333 | :param offset: offset in word lengths from register base 334 | :param size: number of 32-bit words to read 335 | """ 336 | 337 | if offset < 0: 338 | raise ValueError("offset must be positive") 339 | 340 | if type_obj not in ['int', 'bytes', 'hex']: 341 | raise ValueError("type %s not available" % type_obj) 342 | 343 | #sanity check to see if symbolic name of register exists 344 | if not reg_name in self.__named_reg_map: 345 | raise RuntimeError("not a valid named register") 346 | 347 | #lookup params from reg name map dictionary 348 | register_offset = self.__named_reg_map[reg_name]['address'] 349 | byte_length = self.__named_reg_map[reg_name]['bytes'] 350 | 351 | if ((offset + size) << 2) > byte_length: 352 | raise BufferError("attempting to read block past the mapped length") 353 | 354 | byte_offset = offset << 2 355 | 356 | return self.wordread(self.__DSP_BASE_OFFSET + register_offset + byte_offset, 357 | size=size, 358 | type_obj=type_obj) 359 | 360 | 361 | def named_wordwrite(self, reg_name, offset, *data): 362 | """ 363 | :param offset: offset in word lengths from register base 364 | """ 365 | if offset < 0: 366 | raise ValueError("offset must be positive") 367 | 368 | #sanity check to see if symbolic name of register exists 369 | if not reg_name in self.__named_reg_map: 370 | raise RuntimeError("not a valid named register") 371 | 372 | #TODO check length of data +++++++++++++++++++++++++++++++++ 373 | 374 | #lookup from reg name map dictionary 375 | register_offset = self.__named_reg_map[reg_name]['address'] 376 | byte_length = self.__named_reg_map[reg_name]['bytes'] 377 | 378 | if (offset + len(data)) << 2 > byte_length: 379 | raise BufferError("attempting to write past the mapped length") 380 | 381 | byte_offset = offset << 2 382 | 383 | self.wordwrite(self.__DSP_BASE_OFFSET + register_offset + byte_offset, *data) 384 | 385 | 386 | def named_byteread(self, reg_name, offset, size): 387 | """ 388 | :param offset: offset in bytes from register base 389 | :param size: number of bytes to read 390 | """ 391 | 392 | if offset < 0: 393 | raise ValueError("offset must be positive") 394 | 395 | #sanity check to see if symbolic name of register exists 396 | if not reg_name in self.__named_reg_map: 397 | raise RuntimeError("not a valid named register") 398 | 399 | #lookup params from reg name map dictionary 400 | register_offset = self.__named_reg_map[reg_name]['address'] 401 | byte_length = self.__named_reg_map[reg_name]['bytes'] 402 | 403 | if (offset + size) > byte_length: 404 | raise BufferError("attempting to read past the register boundary") 405 | 406 | return self.__mm.byteread(self.__DSP_BASE_OFFSET + register_offset + offset, size) 407 | 408 | 409 | def named_bytewrite(self, reg_name, offset, *data): 410 | """ 411 | :param offset: in bytes 412 | """ 413 | 414 | if offset < 0: 415 | raise ValueError("offset must be positive") 416 | 417 | #sanity check to see if symbolic name of register exists 418 | if not reg_name in self.__named_reg_map: 419 | raise RuntimeError("not a valid named register") 420 | 421 | #lookup params from reg name map dictionary 422 | register_offset = self.__named_reg_map[reg_name]['address'] 423 | byte_length = self.__named_reg_map[reg_name]['bytes'] 424 | 425 | if (offset + len(data)) > byte_length: 426 | raise BufferError("attempting to write past the register boundary") 427 | 428 | return self.__mm.bytewrite(self.__DSP_BASE_OFFSET + register_offset + offset, *data) 429 | 430 | 431 | def _parse_fpg(self, fpg_filename, inplace=0): 432 | """ 433 | fpg_filename: fpg file to parse 434 | inplace : create the bitstream file in same dir as fpg 435 | """ 436 | #TODO - sanity check - although casperfpga.utils does check 437 | #but do anyway in case downstream function changes 438 | #TODO check line 1 for #! kcpfpg to check valid fpg 439 | meta = fpgparser.parse_fpg(fpg_filename) 440 | self.__named_reg_map = meta[1] 441 | 442 | __fpgbasename = os.path.splitext(fpg_filename)[0] 443 | print(__fpgbasename) 444 | 445 | if inplace: 446 | __bitstream = f'{__fpgbasename}.bitstream' #this will be renamed just now 447 | else: 448 | __bitstream = f'{self.__work_dir}/bitstream.bitstream' 449 | 450 | #decompress gzip'd fpg on the fly - skip over gzip header 451 | dec = zlib.decompressobj(32 + zlib.MAX_WBITS) 452 | 453 | with open(fpg_filename, 'rb') as f, mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm, open(__bitstream, 'wb') as wr: 454 | count = 0 455 | bitstream = 0 456 | look = 0 457 | firstline = 1 458 | 459 | for line in iter(mm.readline, b''): 460 | if firstline == 1: 461 | firstline = 0 462 | if not line.strip() == b'#!/bin/kcpfpg': 463 | raise RuntimeError("not a valid fpg") 464 | elif bitstream == 0: 465 | if line.strip() == b'?quit': 466 | bitstream = 1 467 | #count = count + 1 468 | #print(count, line) 469 | continue 470 | else: 471 | data = dec.decompress(line) 472 | if data == b'': 473 | continue 474 | if look == 0: 475 | look = 1 #look only in first non-empty chunk 476 | # 'aa995566' usually found within first 200 bytes of bitstream (bin/bit) 477 | # and should be within the first non zero chunk 478 | token_loc = data.find(b'\xaa\x99\x55\x66') 479 | if token_loc == -1: 480 | raise RuntimeError("is this a valid bitstream?") #TODO 481 | else: 482 | #distinguish between a bit or bin file. 483 | #In a bit file, there's some header 484 | #meta data in the first 129 bytes while in the .bin 485 | #file there's not, so the apporximate 486 | #location of aa996655 token could be used to distinguish 487 | #between bit / bin files. Upon inspection, 100 bytes seems 488 | #like resonable threshold 489 | 490 | if token_loc > 100: 491 | ext = f'bit' 492 | else: 493 | ext = f'bin' 494 | self.set_assoc_binary_ext = ext 495 | 496 | wr.write( data ) 497 | #now rename accordingly 498 | if inplace: 499 | os.rename(__bitstream, f'{__fpgbasename}{ext}') 500 | else: 501 | os.rename(__bitstream, self.assoc_binary) 502 | 503 | 504 | def program(self, progfile): 505 | """ 506 | program with either .bit or .fpg file 507 | """ 508 | #TODO do sanity check to see if it's a valid prog file 509 | #for now just check file extension, can do a bit deeper inspection 510 | 511 | #TODO : check the programming call chain and how it should affect the locking 512 | #mechanisms like statuses and device mapping 513 | 514 | #self.__named_reg_map = {} 515 | #self.__progstat = 0 516 | 517 | ext = os.path.splitext(progfile)[1] 518 | if ext == '.fpg': 519 | self._parse_fpg(progfile, 0) 520 | self.__program_bitfile(self.assoc_binary) 521 | #self.__progstat = 1 #TODO this must be guarded by fpg condition 522 | elif ext == '.bit': 523 | self.__program_bitfile(progfile) 524 | 525 | else: 526 | raise RuntimeError("not a valid programming file extension") 527 | 528 | 529 | #TODO check return code after programming !! 530 | 531 | 532 | def __program_bitfile(self, bitfile): 533 | """ 534 | program a bit file onto the alveo 535 | """ 536 | #TODO sanity checks 537 | 538 | if os.geteuid() != 0: 539 | raise EnvironmentError('Root permissions required.') 540 | bf = os.path.abspath(bitfile) 541 | #subprocess.call(["ls", "-lha"]) 542 | PROGPATH=f'/opt/alveo/alveo-program/' 543 | z = subprocess.run([f'{PROGPATH}alveo-program.sh',self.get_pci_addr, bf ], cwd=f'{PROGPATH}') 544 | #TODO check return code 545 | --------------------------------------------------------------------------------