├── LICENSE ├── README.md ├── afu └── thymesisflow │ ├── aurora_global_poweron.v │ ├── hls_modules │ ├── memory_egress │ │ ├── build_hls_module.tcl │ │ ├── globals.hpp │ │ ├── ocx_flit.cpp │ │ ├── ocx_flit.hpp │ │ ├── ocx_memory_egress_top.cpp │ │ ├── ocx_resp_eng.cpp │ │ ├── ocx_resp_eng.hpp │ │ ├── ocx_rty_eng.cpp │ │ ├── ocx_rty_eng.hpp │ │ └── types.hpp │ └── memory_egress_lookup │ │ ├── build_hls_module.tcl │ │ ├── globals.hpp │ │ ├── ocx_cmd_fifo_load.cpp │ │ ├── ocx_cmd_fifo_load.hpp │ │ ├── ocx_flit.cpp │ │ ├── ocx_flit.hpp │ │ ├── ocx_memory_egress_lookup_top.cpp │ │ └── types.hpp │ ├── ip │ ├── aurora_qsfp0.tcl │ ├── aurora_qsfp1.tcl │ ├── capptag_fifo.tcl │ ├── clock_domain_cross_fifo.tcl │ ├── cmd_lookup_bram.tcl │ ├── data_lookup_bram.tcl │ ├── dataflit_fifo.tcl │ ├── llc_framer_bram_32B.tcl │ ├── memory_egress.tcl │ ├── memory_egress_lookup.tcl │ ├── rty_data_fifo.tcl │ └── tl_resp_fifo.tcl │ ├── lpc_cmdfifo.v │ ├── lpc_errary.v │ ├── lpc_tlx_afu_credit_mgr.v │ ├── mem_cmd_fifo.v │ ├── mem_data_fifo.v │ ├── mem_respfifo.v │ ├── net_cmdfifo.v │ ├── oc_function.v │ ├── rmem_afu.v │ ├── rmem_mmio_regs.v │ ├── rmem_respfifo.v │ ├── thymesisflow_32B_64B_routing_compute_ingress.v │ ├── thymesisflow_32B_64B_routing_memory_ingress.v │ ├── thymesisflow_32B_bckpressure_egress.v │ ├── thymesisflow_32B_bckpressure_ingress.v │ ├── thymesisflow_64B_32B_routing_egress.v │ ├── thymesisflow_64B_compute_egress_adapter.v │ ├── thymesisflow_credit_mgr.v │ ├── thymesisflow_llc │ ├── thymesisflow_32B_llc_top.v │ ├── thymesisflow_llc_egress_driver.v │ ├── thymesisflow_llc_framer_crc_replay.v │ ├── thymesisflow_llc_ingress_driver.v │ └── thymesisflow_llc_parser_crc_replay.v │ ├── thymesisflow_memory_ingress.v │ ├── thymesisflow_rr_arbiter.v │ └── thymesisflow_top.v ├── board_support_packages └── ad9v3 │ ├── ip │ ├── axi_hwicap.tcl │ ├── axi_quad_spi.tcl │ ├── create_DLx_PHY_bypass_20.0g.tcl │ ├── create_DLx_PHY_bypass_25.78125g.tcl │ ├── create_DLx_PHY_elastic_20.0g.tcl │ ├── create_vio_DLx_phy_vio_0.tcl │ └── create_vio_reset_n.tcl │ ├── verilog │ ├── cfg_reg_to_axi4lite.v │ ├── cfg_tieoffs.v │ ├── flash_sub_system.v │ ├── oc_bsp.v │ ├── oc_fpga_top.v │ ├── vpd_stub.v │ └── xilinx │ │ ├── encrypted_buffer_bypass │ │ ├── DLx_phy_example_bit_sync.v │ │ ├── DLx_phy_example_gtwiz_buffbypass_rx.v │ │ ├── DLx_phy_example_gtwiz_buffbypass_tx.v │ │ ├── DLx_phy_example_gtwiz_buffbypass_tx_mstrln3.v │ │ ├── DLx_phy_example_gtwiz_reset.v │ │ ├── DLx_phy_example_gtwiz_userclk_rx.v │ │ ├── DLx_phy_example_gtwiz_userclk_tx.v │ │ ├── DLx_phy_example_gtwiz_userclk_tx_mstrln3.v │ │ ├── DLx_phy_example_init.v │ │ ├── DLx_phy_example_reset_inv_sync.v │ │ ├── DLx_phy_example_reset_sync.v │ │ ├── DLx_phy_example_wrapper.v │ │ ├── DLx_phy_example_wrapper_functions.v │ │ ├── DLx_phy_example_wrapper_functions_mstrln3.v │ │ ├── DLx_phy_example_wrapper_mstrln3.v │ │ ├── dlx_phy_wrap.v │ │ ├── drp_read_modify_write.v │ │ ├── elastic_buffer_dlx_phy_wrap.v │ │ ├── tx_mod_da_fsm.v │ │ └── tx_mod_da_fsm_mstrln3.v │ │ └── encrypted_elastic_buffer │ │ ├── DLx_phy_example_bit_sync.v │ │ ├── DLx_phy_example_gtwiz_buffbypass_rx.v │ │ ├── DLx_phy_example_gtwiz_buffbypass_tx.v │ │ ├── DLx_phy_example_gtwiz_userclk_tx.v │ │ ├── DLx_phy_example_init.v │ │ ├── DLx_phy_example_reset_inv_sync.v │ │ ├── DLx_phy_example_reset_sync.v │ │ ├── DLx_phy_example_wrapper.v │ │ ├── DLx_phy_example_wrapper_functions.v │ │ └── dlx_phy_wrap.v │ └── xdc │ ├── extra.xdc │ ├── gty_properties.xdc │ ├── main_pinout.xdc │ ├── main_placement_bypass.xdc │ ├── main_placement_bypass_thymesisflow_cpu.xdc │ ├── main_placement_bypass_thymesisflow_mem.xdc │ ├── main_placement_elastic.xdc │ ├── main_timing.xdc │ ├── main_timing_thymesisflow_cpu.xdc │ ├── main_timing_thymesisflow_mem.xdc │ ├── qsfp_pinout.xdc │ ├── qspi_pinout.xdc │ └── qspi_timing.xdc ├── config_subsystem ├── cfg_cmdfifo.v ├── cfg_descriptor.v ├── cfg_fence.v ├── cfg_func0.v ├── cfg_func0_init.v ├── cfg_func1.v ├── cfg_func1_init.v ├── cfg_reg_to_axi4lite.v ├── cfg_respfifo.v ├── cfg_seq.v └── oc_cfg.v ├── create_project.tcl ├── dlx ├── ocx_bram_infer.v ├── ocx_dlx_crc.v ├── ocx_dlx_rx_lane.v ├── ocx_dlx_rx_lane_66.v ├── ocx_dlx_rx_main.v ├── ocx_dlx_rxdf.v ├── ocx_dlx_top.v ├── ocx_dlx_tx_ctl.v ├── ocx_dlx_tx_flt.v ├── ocx_dlx_tx_gbx.v ├── ocx_dlx_tx_que.v ├── ocx_dlx_txdf.v └── ocx_dlx_xlx_if.v ├── libtfshmem ├── Makefile ├── README.md ├── include │ ├── custom_mm.h │ ├── tf_shmem.h │ ├── tf_shmem_api.h │ └── utils.h ├── kernel_module │ ├── Kbuild │ ├── Makefile │ ├── tfshmem.conf │ ├── tfshmem.h │ ├── tfshmem_mod.c │ └── tfshmem_ops.c ├── src │ ├── custom_mm.c │ └── tf_shmem.c └── test │ ├── test_compute.c │ └── test_memory.c ├── libthymesisflow ├── Dockerfile ├── Doxyfile ├── Makefile ├── README.md ├── cmd │ ├── thymesisf-agent.c │ └── thymesisf-cli.c ├── include │ ├── agent.h │ ├── client.h │ ├── connection.h │ ├── errcode.h │ ├── iport.h │ ├── logger.h │ ├── proto.h │ ├── protoconst.h │ ├── stats.h │ └── thymesisflow.h ├── scripts │ └── read_counters.py ├── src │ ├── agent.c │ ├── client.c │ ├── connection.c │ ├── iport.c │ ├── proto.c │ ├── stats.c │ └── thymesisflow.c └── unittest │ └── unittest.c ├── reference_design_doc ├── TLX_3.0_Reference_Design.pdf ├── ThymesisFlow Design and Deployment v1.1.pdf └── thymesis_arch.jpg ├── scripts ├── setup_environment.csh └── setup_environment.ksh ├── tcl ├── write_bitstream.tcl ├── write_mcs.tcl └── write_proj_config.tcl └── tlx ├── bram_syn_test.v ├── dram_syn_test.v ├── ocx_leaf_inferd_regfile.v ├── ocx_tlx_513x32_fifo.v ├── ocx_tlx_514x16_fifo.v ├── ocx_tlx_bdi_mac.v ├── ocx_tlx_cfg_mac.v ├── ocx_tlx_cmd_fifo_mac.v ├── ocx_tlx_ctl_fsm.v ├── ocx_tlx_data_arb.v ├── ocx_tlx_data_fifo_mac.v ├── ocx_tlx_dcp_fifo_ctl.v ├── ocx_tlx_fifo_cntlr.v ├── ocx_tlx_flit_parser.v ├── ocx_tlx_framer.v ├── ocx_tlx_framer_cmd_fifo.v ├── ocx_tlx_framer_rsp_fifo.v ├── ocx_tlx_parse_mac.v ├── ocx_tlx_parser_err_mac.v ├── ocx_tlx_rcv_mac.v ├── ocx_tlx_rcv_top.v ├── ocx_tlx_resp_fifo_mac.v ├── ocx_tlx_top.v ├── ocx_tlx_vc0_fifo_ctl.v └── ocx_tlx_vc1_fifo_ctl.v /README.md: -------------------------------------------------------------------------------- 1 | # Welcome to the Home of ThymesisFlow 2 | ThymesisFlow is a HW-SW co-designed prototype enabling 3 | hardware disaggregation of compute resources on POWER9 processor systems. 4 | The current design supports disaggregation of memory by "stealing" it from a 5 | neighbour node, and is based on OpenCAPI. 6 | 7 | ![ThymesisFlow Architecture](./reference_design_doc/thymesis_arch.jpg) 8 | 9 | The ThymesisFlow approach consists of a compute node (left side of the figure) 10 | that is stealing the memory, and a memory node (righ side of the figure) from 11 | which the memory is stolen. The compute endpoint of the ThymesisFlow design is 12 | based on the OpenCAPI M1 mode (or LPC), while the memory endpoint uses the 13 | OpenCAPI C1 mode. On the compute endpoint the disaggregated memory is mapped at 14 | a specific range of addresses in the physical address space and can be 15 | dynamically hotplugged to a running Linux system. No software modification is 16 | needed to access disaggregated memory, neither in the Linux kernel nor from the 17 | user applications. 18 | 19 | This repository contains the complete design (Verilog/HLS) of memory and compute 20 | side, including the OpenCAPI 3.0 reference design. 21 | 22 | Please refer to the [ThymesisFlow documentation](./reference_design_doc/ThymesisFlow%20Design%20and%20Deployment%20v1.1.pdf) for a complete description of the 23 | design and its bringup. 24 | 25 | ## Supported Cards 26 | 27 | The ThymesisFlow design currently supports the following cards: 28 | 29 | - [Alphadata 9V3](https://www.alpha-data.com/dcp/products.php?product=adm-pcie-9v3) 30 | 31 | ## How-to bring-up the design 32 | The ThymesisFlow build and bring-up procedure is described in detail in the [design reference manual](./reference_design_doc/ThymesisFlow%20Design%20and%20Deployment%20v1.1.pdf). 33 | Please, refer also to the [OpenCAPI3.0 34 | Wiki](https://github.com/OpenCAPI/OpenCAPI3.0_Client_RefDesign/wiki) for further details on the Vivado projects creation. 35 | 36 | ### Create Compute side Vivado project 37 | ```console 38 | vivado -source create_project.tcl -tclargs --speed 20.0 --afu thymesisflow --buffer bypass --tftype compute 39 | ``` 40 | 41 | ### Create Memory side Vivado project 42 | ```console 43 | vivado -source create_project.tcl -tclargs --speed 20.0 --afu thymesisflow --buffer bypass --tftype memory 44 | ``` 45 | 46 | ## Follow us: 47 | 48 | 49 | ### Conferences: 50 | - [OpenPOWER Summit North America 2020 - September 15th 2020](https://openpowerna2020.sched.com/event/eDqy/thymesisflow-a-hardwaresoftware-open-framework-for-software-defined-disaggregation-based-on-opencapi-christian-pinto-ibm-research-europe?iframe=yes&w=100%&sidebar=yes&bg=no) 51 | - [FPL 2020 Tutorial](https://www.youtube.com/watch?v=eWhraT06K2w&feature=emb_logo&ab_channel=FPL2020) 52 | - [OpenPOWER Summit North America 2019](https://www.youtube.com/watch?v=XcjRL3Lh8Ig) 53 | - [OpenPOWER Summit Europe 2018](https://www.youtube.com/watch?v=vSKUeGeEkoA) 54 | 55 | ### Press releases 56 | - [Teleporting Memory Across Two Servers - IBM Research blog](https://www.ibm.com/blogs/research/2020/08/teleporting-memory-across-two-servers/) 57 | 58 | 59 | ## What's Next 60 | 61 | Our immediate next steps are related to the release of the software support for 62 | ThymesisFlow: 63 | 64 | - Memory stealing process: this is the user application that allocates a buffer in 65 | the memory node for access from a remote node. 66 | - ThymesisStats: Software Library for accessing performance countersi embedded in the design. 67 | - ThymesisFlow orchestration software: we gave a stub at our vision of a 68 | disaggregated system, and came up with a proposal control plane for 69 | dynamic orchestration of disaggregated memory segments. 70 | - Stay tuned! 71 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/build_hls_module.tcl: -------------------------------------------------------------------------------- 1 | # Create a project 2 | open_project -reset ocx_memory_egress 3 | 4 | # The source files 5 | add_files ocx_memory_egress_top.cpp 6 | add_files ocx_flit.cpp 7 | add_files ocx_resp_eng.cpp 8 | add_files ocx_rty_eng.cpp 9 | 10 | # Specify the top-level function for synthesis 11 | set_top ocx_memory_egress_top 12 | 13 | ########################### 14 | # Solution settings 15 | 16 | # Create solution1 17 | open_solution -reset solution1 18 | 19 | set_part {xcvu3p-ffvc1517-2-i} 20 | create_clock -period 2.5 21 | config_rtl -reset_level low 22 | 23 | ## Synthesize design 24 | csynth_design 25 | ## Export for vivado IP catalog integration 26 | export_design -rtl verilog -format ip_catalog -description "ThymesisFlow Memory Egress module" -vendor "ibm.com" -library "ThymesisFlow" -display_name "ocx_memory_egress" 27 | 28 | exit 29 | 30 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/globals.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | #ifndef GLOBALS_HH 25 | #define GLOBALS_HH 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | using namespace hls; 38 | 39 | #define TL_RD_RESPOK 0x4 40 | #define TL_RD_RESPFAIL 0x5 41 | #define TL_WR_RESPOK 0x8 42 | #define TL_WR_RESPFAIL 0x9 43 | #define TL_XLATE_DONE 0x18 44 | 45 | 46 | #define TLRESP_DATA 0x2 47 | #define TLRESP_CODE_RTY 0x2 48 | #define TLRESP_CODE_PEND 0x4 49 | 50 | #define TLRESP_RTY 0x1 51 | #define TLRESP_DATA 0x2 52 | 53 | #define ADDR_MASK 0xc000 54 | #define LOOKUPSIZE 512 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/ocx_flit.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #ifndef OCX_FLIT_HH 26 | #define OCX_FLIT_HH 27 | 28 | /* This class provides all flit parsing functions 29 | and preserves the datapath maximum size to 512 30 | (standard opencapi flit size) and in some cases 31 | 513 (flit size + Bad data indicator bit). 32 | The only types used here are: 33 | ocxflitbdi_t: 513 bits (ocxflit + bdi) 34 | ocxflit_t: 512 bits ocxflit 35 | 36 | according to context. 37 | 38 | */ 39 | 40 | #include "globals.hpp" 41 | #include "types.hpp" 42 | 43 | class OCxFlit { 44 | 45 | private: 46 | ocxflitbdi_t flit; 47 | 48 | public: 49 | OCxFlit(); 50 | void fromOCxFlit(ocxflit_t fl); 51 | ocxflit_t toOCxFlit(void); 52 | void fromOCxBdiFlit(ocxflitbdi_t fl); 53 | ocxflitbdi_t toOCxBdiFlit(void); 54 | void setBdi(valid_t bdi); 55 | 56 | //TL Response CMD flit parsing 57 | void fromTLRespData(tlrflit_t fl); 58 | tlrflit_t toTLRespData(void); 59 | sel_t getTLRespState(void); 60 | dl_t getTLRespDl(void); 61 | dp_t getTLRespDp(void); 62 | tag_t getRespCappTag(void); 63 | void setRespOrigCappTag(tag_t tag); 64 | tag_t getRespOrigCappTag(void); 65 | code_t getRespCode(void); 66 | opcode_t getRespOpcode(void); 67 | bool TLRespIsXlateDone(void); 68 | bool TLRespCodeIsFailure(void); 69 | 70 | 71 | //TLx Command CMD flit parsing 72 | void fromTLxCmd(tlxcflit_t fl); 73 | tlxcflit_t toTLxCmdFlit(void); 74 | void fromTLxCStore(tlxscflit_t fl); 75 | tlxscflit_t toTLxCStore(void); 76 | bool TLxCmdIsWrite(void); 77 | bool TLxCmdIsRead(void); 78 | bool TLxCmdIsAssignActag(void); 79 | dl_t getDl(void); 80 | void setOpcode(opcode_t opc); 81 | void setDl(dl_t dl); 82 | void setEA(addr_t addr); 83 | void setBe(addr_t be); 84 | void setTLxCmdRespAnno(code_t cd); 85 | void setActag(actag_t ac); 86 | void setPasid(pasid_t pasid); 87 | code_t getTLxCmdRespAnno(void); 88 | tag_t getCappTagLocal(void); 89 | void setCappTagLocal(tag_t tag); 90 | void setCappTagOrig(tag_t tag); 91 | tag_t getCappTagOrig(void); 92 | 93 | //Network part 94 | nid_t getNetId(void); 95 | void setNetId(nid_t netid); 96 | }; 97 | 98 | 99 | 100 | 101 | #endif 102 | 103 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/ocx_resp_eng.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #ifndef OCX_RESP_ENG_HH 26 | #define OCX_RESP_ENG_HH 27 | 28 | #include "globals.hpp" 29 | #include "types.hpp" 30 | 31 | #include "ocx_flit.hpp" 32 | 33 | class TLRespEng{ 34 | 35 | public: 36 | TLRespEng(); 37 | //Cmd Response from TL 38 | void RespEngSM(valid_t &enable,\ 39 | stream &tlresp_fifo_in, \ 40 | //Data Response from TL 41 | stream &tlresp_data_in, \ 42 | ocxflit_t data0lookup[LOOKUPSIZE],\ 43 | ocxflit_t data1lookup[LOOKUPSIZE],\ 44 | ocxflit_t data2lookup[LOOKUPSIZE],\ 45 | ocxflit_t data3lookup[LOOKUPSIZE],\ 46 | /*OUTPUTS*/ 47 | //capptag return.. 48 | stream &lookup_out,\ 49 | //rty command out 50 | stream &rty_do_cmd_out, \ 51 | //network command out - should be many 52 | stream &network_cmd_out0,\ 53 | count_t &valex_memegress_cmdf_cnt,\ 54 | count_t &valex_memegress_data_cnt,\ 55 | count_t &valex_memegress_cmdf_torty,\ 56 | count_t &valex_memegress_data_torty); 57 | 58 | }; 59 | 60 | 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/ocx_rty_eng.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #ifndef OCX_RTY_ENG_HH 26 | #define OCX_RTY_ENG_HH 27 | 28 | #include "globals.hpp" 29 | #include "types.hpp" 30 | 31 | #include "ocx_flit.hpp" 32 | 33 | class RtyEng { 34 | 35 | public: 36 | 37 | RtyEng(); 38 | //Get Input from Retry engine and accordingly forwards to command engine 39 | void RtyEngSM(cutimer_t timeout,\ 40 | stream &input_fifo, \ 41 | stream &rty_jail,\ 42 | stream &output_fifo, 43 | count_t &valex_memingress_cmdf_rty_in,\ 44 | count_t &valex_memingress_cmdf_jail_in,\ 45 | count_t &valex_memingress_cmdf_rty_out,\ 46 | count_t &valex_memingress_cmdf_jail_out,\ 47 | count_t &valex_memingress_data_rty_in,\ 48 | count_t &valex_memingress_data_rty_out,\ 49 | count_t &valex_memingress_data_jail_in,\ 50 | count_t &valex_memingress_data_jail_out); 51 | 52 | 53 | 54 | }; 55 | 56 | 57 | 58 | 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress/types.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | #ifndef TYPES_HH 25 | #define TYPES_HH 26 | 27 | #include 28 | 29 | 30 | typedef ap_uint<253> resp_combo_t; //tlrflit (252:215) + tlxscflit_t (214:0) 31 | typedef ap_uint<32> count_t; 32 | typedef ap_uint<513> ocxflitbdi_t; 33 | typedef ap_uint<512> ocxflit_t; 34 | typedef ap_uint<256> ocxflitH_t; 35 | typedef ap_uint<1> valid_t; 36 | typedef ap_uint<199> tlxcflit_t; 37 | typedef ap_uint<38> tlrflit_t; 38 | typedef ap_uint<215> tlxscflit_t; 39 | typedef ap_uint<3> dlsize_t; 40 | typedef ap_uint<36> cutimer_t; 41 | typedef ap_uint<20> pasid_t; 42 | typedef ap_uint<4> stream_t; 43 | typedef ap_uint<12> actag_t; 44 | typedef ap_uint<16> bdf_t; 45 | typedef ap_uint<3> pl_t; 46 | typedef ap_uint<64> addr_t; 47 | typedef ap_uint<4> code_t; 48 | typedef ap_uint<2> dp_t; 49 | typedef ap_uint<1> pad_t; 50 | typedef ap_uint<6> nid_t; 51 | typedef ap_uint<8> opcode_t; 52 | typedef ap_uint<2> dl_t; 53 | typedef ap_uint<16> tag_t; 54 | typedef ap_uint<1> signal_t; 55 | typedef ap_uint<3> sel_t; 56 | typedef ap_uint<10> combo_t; 57 | typedef ap_uint<10> fifocnt_t; 58 | typedef ap_uint<8> token_t; 59 | typedef ap_uint<4> xlatecnt_t; 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/build_hls_module.tcl: -------------------------------------------------------------------------------- 1 | # Create a project 2 | open_project -reset ocx_memory_egress_lookup 3 | 4 | # The source file and test bench 5 | add_files ocx_memory_egress_lookup_top.cpp 6 | add_files ocx_flit.cpp 7 | add_files ocx_cmd_fifo_load.cpp 8 | # Specify the top-level function for synthesis 9 | set_top ocx_memory_egress_lookup_top 10 | 11 | ########################### 12 | # Solution settings 13 | 14 | # Create solution1 15 | open_solution -reset solution1 16 | 17 | set_part {xcvu3p-ffvc1517-2-i} 18 | create_clock -period 2.5 19 | config_rtl -reset_level low 20 | 21 | 22 | ## Synthesize design 23 | csynth_design 24 | ## Export for vivado IP catalog integration 25 | export_design -rtl verilog -format ip_catalog -description "ThymesisFlow-P Memory Egress Lookup module" -vendor "ibm.com" -library "ThymesisFlow" -display_name "ocx_memory_egress_lookup" 26 | 27 | exit 28 | 29 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/globals.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | #ifndef GLOBALS_HH 25 | #define GLOBALS_HH 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | using namespace hls; 38 | 39 | #define TL_RD_RESPOK 0x4 40 | #define TL_RD_RESPFAIL 0x5 41 | #define TL_WR_RESPOK 0x8 42 | #define TL_WR_RESPFAIL 0x9 43 | #define TL_XLATE_DONE 0x18 44 | 45 | 46 | #define TLRESP_DATA 0x2 47 | #define TLRESP_CODE_RTY 0x2 48 | #define TLRESP_CODE_PEND 0x8 49 | 50 | #define TLRESP_RTY 0x1 51 | #define TLRESP_DATA 0x2 52 | 53 | #define LOOKUPSIZE 512 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/ocx_cmd_fifo_load.cpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #include "ocx_cmd_fifo_load.hpp" 26 | #include "globals.hpp" 27 | 28 | CmdFifoLoad::CmdFifoLoad() 29 | { 30 | return; 31 | } 32 | 33 | void CmdFifoLoad::CmdFifoLoadSM(stream &input_fifo,\ 34 | tlxscflit_t cmdlookup[LOOKUPSIZE], \ 35 | count_t &valex_memegress_cmddata_ld_cnt,\ 36 | stream &output_fifo) 37 | { 38 | #pragma HLS pipeline II=1 enable_flush 39 | 40 | OCxFlit tlresp; 41 | OCxFlit cmdf; 42 | static count_t valex_memegress_cmddata_ld_cntin = 0x0; 43 | static resp_combo_t to_send; 44 | 45 | if (!input_fifo.empty()) { 46 | tlresp.fromTLRespData(input_fifo.read()); 47 | cmdf.fromTLxCStore(cmdlookup[tlresp.getRespCappTag()]); 48 | valex_memegress_cmddata_ld_cnt = valex_memegress_cmddata_ld_cntin; 49 | valex_memegress_cmddata_ld_cntin = valex_memegress_cmddata_ld_cntin + 1; 50 | to_send.range(252,215) = tlresp.toTLRespData().range(37,0); 51 | to_send.range(214,0) = cmdf.toTLxCStore().range(214,0); 52 | output_fifo.write(to_send); 53 | } 54 | 55 | return; 56 | } 57 | 58 | 59 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/ocx_cmd_fifo_load.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #ifndef OCX_CMD_FIFO_LOAD_HH 26 | #define OCX_CMD_FIFO_LOAD_HH 27 | 28 | #include "globals.hpp" 29 | #include "types.hpp" 30 | #include "ocx_flit.hpp" 31 | 32 | class CmdFifoLoad { 33 | 34 | 35 | public: 36 | 37 | CmdFifoLoad(); 38 | 39 | void CmdFifoLoadSM(stream &input_fifo,\ 40 | tlxscflit_t cmdlookup[LOOKUPSIZE],\ 41 | count_t &valex_memegress_cmddata_ld_cnt,\ 42 | stream &output_fifo); 43 | 44 | 45 | }; 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/ocx_flit.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | #ifndef OCX_FLIT_HH 26 | #define OCX_FLIT_HH 27 | 28 | /* This class provides all flit parsing functions 29 | and preserves the datapath maximum size to 512 30 | (standard opencapi flit size) and in some cases 31 | 513 (flit size + Bad data indicator bit). 32 | The only types used here are: 33 | ocxflitbdi_t: 513 bits (ocxflit + bdi) 34 | ocxflit_t: 512 bits ocxflit 35 | 36 | according to context. 37 | 38 | */ 39 | 40 | #include "globals.hpp" 41 | #include "types.hpp" 42 | 43 | class OCxFlit { 44 | 45 | private: 46 | ocxflitbdi_t flit; 47 | 48 | public: 49 | OCxFlit(); 50 | void fromOCxFlit(ocxflit_t fl); 51 | ocxflit_t toOCxFlit(void); 52 | void fromOCxBdiFlit(ocxflitbdi_t fl); 53 | ocxflitbdi_t toOCxBdiFlit(void); 54 | void setBdi(valid_t bdi); 55 | 56 | //TL Response CMD flit parsing 57 | void fromTLRespData(tlrflit_t fl); 58 | tlrflit_t toTLRespData(void); 59 | sel_t getTLRespState(void); 60 | dl_t getTLRespDl(void); 61 | dp_t getTLRespDp(void); 62 | tag_t getRespCappTag(void); 63 | void setRespOrigCappTag(tag_t tag); 64 | tag_t getRespOrigCappTag(void); 65 | code_t getRespCode(void); 66 | opcode_t getRespOpcode(void); 67 | bool TLRespIsXlateDone(void); 68 | bool TLRespCodeIsFailure(void); 69 | 70 | 71 | //TLx Command CMD flit parsing 72 | void fromTLxCmd(tlxcflit_t fl); 73 | tlxcflit_t toTLxCmdFlit(void); 74 | void fromTLxCStore(tlxscflit_t fl); 75 | tlxscflit_t toTLxCStore(void); 76 | bool TLxCmdIsWrite(void); 77 | bool TLxCmdIsRead(void); 78 | bool TLxCmdIsAssignActag(void); 79 | dl_t getDl(void); 80 | void setOpcode(opcode_t opc); 81 | void setDl(dl_t dl); 82 | void setEA(addr_t addr); 83 | void setBe(addr_t be); 84 | void setTLxCmdRespAnno(code_t cd); 85 | void setActag(actag_t ac); 86 | void setPasid(pasid_t pasid); 87 | code_t getTLxCmdRespAnno(void); 88 | tag_t getCappTagLocal(void); 89 | void setCappTagLocal(tag_t tag); 90 | void setCappTagOrig(tag_t tag); 91 | tag_t getCappTagOrig(void); 92 | 93 | //Network part 94 | nid_t getNetId(void); 95 | void setNetId(nid_t netid); 96 | }; 97 | 98 | 99 | 100 | 101 | #endif 102 | 103 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/ocx_memory_egress_lookup_top.cpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | 25 | 26 | #include "globals.hpp" 27 | 28 | #include "types.hpp" 29 | #include "ocx_flit.hpp" 30 | #include "ocx_cmd_fifo_load.hpp" 31 | 32 | 33 | 34 | void ocx_memory_egress_lookup_top(stream &tl_resp, \ 35 | /*External BRAM Input Interfaces */ 36 | tlxscflit_t cmdlookup_in[LOOKUPSIZE],\ 37 | /*outputs*/ 38 | count_t &valex_memegress_cmddata_ld_cnt,\ 39 | stream &tl_fifo_resp_out) 40 | { 41 | 42 | 43 | #pragma HLS INTERFACE bram register port=cmdlookup_in 44 | #pragma HLS RESOURCE variable=cmdlookup_in core=RAM_1P_BRAM latency=1 45 | 46 | 47 | #pragma HLS INTERFACE ap_none register port=valex_memegress_cmddata_ld_cnt 48 | #pragma HLS INTERFACE axis register port=tl_fifo_resp_out 49 | #pragma HLS INTERFACE axis register port=tl_resp 50 | #pragma HLS INTERFACE ap_ctrl_none port=return 51 | 52 | #pragma HLS dataflow interval=1 53 | 54 | /* 55 | This Fifo Holds Responses coming from the local TL 56 | We need this Buffer as the Resp State machine will 57 | might not be ready to pull, spending cycles on 58 | rertry CAM updates or data push. We put a generous 8 59 | depth here, but i think we will not need more than 4. 60 | */ 61 | 62 | /*Various State machines Declarations*/ 63 | CmdFifoLoad CmdFifold; 64 | 65 | 66 | /* 67 | Fifo load for TL cmd 68 | Input: 69 | tl_resp comming from TL 70 | Output: 71 | tl_fifo_resp a fifo that feeds Resp egine. 72 | */ 73 | 74 | CmdFifold.CmdFifoLoadSM(tl_resp,\ 75 | cmdlookup_in,\ 76 | valex_memegress_cmddata_ld_cnt,\ 77 | tl_fifo_resp_out); 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /afu/thymesisflow/hls_modules/memory_egress_lookup/types.hpp: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | #ifndef TYPES_HH 25 | #define TYPES_HH 26 | 27 | #include 28 | 29 | 30 | 31 | typedef ap_uint<253> resp_combo_t; //tlrflit (252:215) + tlxscflit_t (214:0) 32 | typedef ap_uint<513> ocxflitbdi_t; 33 | typedef ap_uint<512> ocxflit_t; 34 | typedef ap_uint<256> ocxflitH_t; 35 | typedef ap_uint<1> valid_t; 36 | typedef ap_uint<199> tlxcflit_t; 37 | typedef ap_uint<38> tlrflit_t; 38 | typedef ap_uint<215> tlxscflit_t; 39 | typedef ap_uint<3> dlsize_t; 40 | typedef ap_uint<36> cutimer_t; 41 | typedef ap_uint<20> pasid_t; 42 | typedef ap_uint<4> stream_t; 43 | typedef ap_uint<12> actag_t; 44 | typedef ap_uint<16> bdf_t; 45 | typedef ap_uint<3> pl_t; 46 | typedef ap_uint<64> addr_t; 47 | typedef ap_uint<4> code_t; 48 | typedef ap_uint<2> dp_t; 49 | typedef ap_uint<1> pad_t; 50 | typedef ap_uint<6> nid_t; 51 | typedef ap_uint<8> opcode_t; 52 | typedef ap_uint<2> dl_t; 53 | typedef ap_uint<16> tag_t; 54 | typedef ap_uint<1> signal_t; 55 | typedef ap_uint<3> sel_t; 56 | typedef ap_uint<10> combo_t; 57 | typedef ap_uint<10> fifocnt_t; 58 | typedef ap_uint<8> token_t; 59 | typedef ap_uint<4> xlatecnt_t; 60 | typedef ap_uint<32> count_t; 61 | 62 | #endif 63 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/aurora_qsfp0.tcl: -------------------------------------------------------------------------------- 1 | ## Aurora 64B66B QSFP0 configuration for 9V3 QSFP0 cage - 4x25.78125 and 309.375 reference clock 2 | 3 | create_ip -name aurora_64b66b -vendor xilinx.com -library ip -version 11.2 -module_name aurora_qsfp0 4 | 5 | 6 | set_property -dict [list CONFIG.CHANNEL_ENABLE {X0Y16 X0Y17 X0Y18 X0Y19} CONFIG.C_AURORA_LANES {4} CONFIG.C_LINE_RATE {25.78125} CONFIG.C_REFCLK_FREQUENCY {309.375} \ 7 | CONFIG.C_INIT_CLK {100} CONFIG.C_UCOLUMN_USED {left} CONFIG.C_START_QUAD {Quad_X0Y4} CONFIG.C_START_LANE {X0Y16} CONFIG.C_REFCLK_SOURCE {MGTREFCLK0_of_Quad_X0Y4} \ 8 | CONFIG.C_GT_LOC_4 {4} CONFIG.C_GT_LOC_3 {3} CONFIG.C_GT_LOC_2 {2} CONFIG.crc_mode {true} CONFIG.SupportLevel {1} CONFIG.C_USE_BYTESWAP {true} CONFIG.drp_mode {Native}] [get_ips aurora_qsfp0] 9 | 10 | 11 | generate_target {all} [get_ips aurora_qsfp0] 12 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/aurora_qsfp1.tcl: -------------------------------------------------------------------------------- 1 | ## Aurora 64B66B QSFP1 configuration for 9V3 cage QSFP1 - 4x25.78125 and 309.375 reference clock 2 | 3 | create_ip -name aurora_64b66b -vendor xilinx.com -library ip -version 11.2 -module_name aurora_qsfp1 4 | 5 | set_property -dict [list CONFIG.CHANNEL_ENABLE {X0Y12 X0Y13 X0Y14 X0Y15} CONFIG.C_AURORA_LANES {4} CONFIG.C_LINE_RATE {25.78125} CONFIG.C_REFCLK_FREQUENCY {309.375} \ 6 | CONFIG.C_INIT_CLK {100} CONFIG.C_UCOLUMN_USED {left} CONFIG.C_START_QUAD {Quad_X0Y3} CONFIG.C_START_LANE {X0Y12} CONFIG.C_REFCLK_SOURCE {MGTREFCLK0_of_Quad_X0Y3} \ 7 | CONFIG.C_GT_LOC_4 {4} CONFIG.C_GT_LOC_3 {3} CONFIG.C_GT_LOC_2 {2} CONFIG.crc_mode {true} CONFIG.SupportLevel {1} CONFIG.C_USE_BYTESWAP {true} CONFIG.drp_mode {Native}] [get_ips aurora_qsfp1] 8 | 9 | 10 | generate_target {all} [get_ips aurora_qsfp1] 11 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/capptag_fifo.tcl: -------------------------------------------------------------------------------- 1 | ## capptag_fifo that stores capptags and enables their recycling. 2 | 3 | create_ip -name axis_data_fifo -vendor xilinx.com -library ip -version 1.1 -module_name capptag_fifo 4 | 5 | set_property -dict [list CONFIG.TDATA_NUM_BYTES {2} CONFIG.FIFO_DEPTH {512}] [get_ips capptag_fifo] 6 | 7 | generate_target {all} [get_ips capptag_fifo] 8 | 9 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/clock_domain_cross_fifo.tcl: -------------------------------------------------------------------------------- 1 | ## clock domain cross fifo that bridges dataflow between OpenCAPI DLx and Aurora transceiver clock domains 2 | 3 | create_ip -name axis_data_fifo -vendor xilinx.com -library ip -version 1.1 -module_name clock_domain_cross_fifo 4 | 5 | set_property -dict [list CONFIG.TDATA_NUM_BYTES {32} CONFIG.FIFO_DEPTH {256} CONFIG.IS_ACLK_ASYNC {1} CONFIG.SYNCHRONIZATION_STAGES {8}] [get_ips clock_domain_cross_fifo] 6 | 7 | generate_target {all} [get_ips clock_domain_cross_fifo] 8 | 9 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/cmd_lookup_bram.tcl: -------------------------------------------------------------------------------- 1 | ## cmd lookup bram of depth 512 for storing command flits that have to be reissued 2 | 3 | create_ip -name blk_mem_gen -vendor xilinx.com -library ip -version 8.4 -module_name cmd_lookup_bram 4 | 5 | set_property -dict [list CONFIG.Memory_Type {True_Dual_Port_RAM} CONFIG.Enable_32bit_Address {true} CONFIG.Use_Byte_Write_Enable {true} CONFIG.Byte_Size {8} CONFIG.Write_Width_A {256} CONFIG.Write_Depth_A {512} CONFIG.Read_Width_A {256} CONFIG.Write_Width_B {256} CONFIG.Read_Width_B {256} CONFIG.Enable_B {Use_ENB_Pin} CONFIG.Register_PortA_Output_of_Memory_Primitives {false} CONFIG.Register_PortB_Output_of_Memory_Primitives {false} CONFIG.Use_RSTA_Pin {true} CONFIG.Use_RSTB_Pin {true} CONFIG.Port_B_Clock {100} CONFIG.Port_B_Write_Rate {50} CONFIG.Port_B_Enable_Rate {100} CONFIG.use_bram_block {BRAM_Controller} CONFIG.EN_SAFETY_CKT {true}] [get_ips cmd_lookup_bram] 6 | 7 | generate_target {all} [get_ips cmd_lookup_bram] 8 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/data_lookup_bram.tcl: -------------------------------------------------------------------------------- 1 | ## data lookup bram of depth 512 for storing command flits that have to be reissued 2 | 3 | create_ip -name blk_mem_gen -vendor xilinx.com -library ip -version 8.4 -module_name data_lookup_bram 4 | 5 | set_property -dict [list CONFIG.Memory_Type {True_Dual_Port_RAM} CONFIG.Enable_32bit_Address {true} CONFIG.Use_Byte_Write_Enable {true} CONFIG.Byte_Size {8} CONFIG.Write_Width_A {512} CONFIG.Write_Depth_A {512} CONFIG.Read_Width_A {512} CONFIG.Write_Width_B {512} CONFIG.Read_Width_B {512} CONFIG.Enable_B {Use_ENB_Pin} CONFIG.Register_PortA_Output_of_Memory_Primitives {false} CONFIG.Register_PortB_Output_of_Memory_Primitives {false} CONFIG.Use_RSTA_Pin {true} CONFIG.Use_RSTB_Pin {true} CONFIG.Port_B_Clock {100} CONFIG.Port_B_Write_Rate {50} CONFIG.Port_B_Enable_Rate {100} CONFIG.use_bram_block {BRAM_Controller} CONFIG.EN_SAFETY_CKT {true}] [get_ips data_lookup_bram] 6 | 7 | generate_target {all} [get_ips data_lookup_bram] 8 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/dataflit_fifo.tcl: -------------------------------------------------------------------------------- 1 | ## tlx_afu_data_fifo fifo that stores cmddata flits that are emmited by M1 (or compute side) stack towards the network 2 | 3 | create_ip -name axis_data_fifo -vendor xilinx.com -library ip -version 1.1 -module_name dataflit_fifo 4 | 5 | set_property -dict [list CONFIG.TDATA_NUM_BYTES {65} CONFIG.FIFO_DEPTH {128}] [get_ips dataflit_fifo] 6 | 7 | generate_target {all} [get_ips dataflit_fifo] 8 | 9 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/llc_framer_bram_32B.tcl: -------------------------------------------------------------------------------- 1 | ## LLC Framer BRAM (native interface) that acts as the retransmission fifo buffer to relax timing closure 2 | 3 | create_ip -name blk_mem_gen -vendor xilinx.com -library ip -version 8.4 -module_name thymesisflow_llc_framer_bram_32B 4 | 5 | set_property -dict [list CONFIG.Memory_Type {Simple_Dual_Port_RAM} CONFIG.Enable_32bit_Address {false} CONFIG.Use_Byte_Write_Enable {false} CONFIG.Byte_Size {9} \ 6 | CONFIG.Assume_Synchronous_Clk {true} CONFIG.Write_Width_A {256} CONFIG.Write_Depth_A {256} CONFIG.Read_Width_A {256} CONFIG.Operating_Mode_A {READ_FIRST} \ 7 | CONFIG.Enable_A {Always_Enabled} CONFIG.Write_Width_B {256} CONFIG.Read_Width_B {256} CONFIG.Operating_Mode_B {READ_FIRST} CONFIG.Enable_B {Always_Enabled} \ 8 | CONFIG.Register_PortA_Output_of_Memory_Primitives {false} CONFIG.Register_PortB_Output_of_Memory_Primitives {false} CONFIG.Use_RSTA_Pin {false} \ 9 | CONFIG.Port_B_Clock {100} CONFIG.Port_B_Enable_Rate {100} CONFIG.use_bram_block {Stand_Alone} CONFIG.EN_SAFETY_CKT {false}] [get_ips thymesisflow_llc_framer_bram_32B] 10 | 11 | generate_target {all} [get_ips thymesisflow_llc_framer_bram_32B] 12 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/memory_egress.tcl: -------------------------------------------------------------------------------- 1 | ## ocx_memory_egress ip creation 2 | 3 | create_ip -name ocx_memory_egress_top -vendor ibm.com -library ThymesisFlow -version 1.0 -module_name ocx_memory_egress 4 | 5 | 6 | generate_target {all} [get_ips ocx_memory_egress] 7 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/memory_egress_lookup.tcl: -------------------------------------------------------------------------------- 1 | ## ocx_memory_egress_lookup HLS module ip creation 2 | 3 | create_ip -name ocx_memory_egress_lookup_top -vendor ibm.com -library ThymesisFlow -version 1.0 -module_name ocx_memory_egress_lookup 4 | 5 | 6 | generate_target {all} [get_ips ocx_memory_egress_lookup] 7 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/rty_data_fifo.tcl: -------------------------------------------------------------------------------- 1 | ## fifo that stores command and dataflits that need to be retried. 2 | 3 | create_ip -name axis_data_fifo -vendor xilinx.com -library ip -version 1.1 -module_name rty_data_fifo 4 | 5 | set_property -dict [list CONFIG.TDATA_NUM_BYTES {64} CONFIG.FIFO_DEPTH {128}] [get_ips rty_data_fifo] 6 | 7 | generate_target {all} [get_ips rty_data_fifo] 8 | 9 | -------------------------------------------------------------------------------- /afu/thymesisflow/ip/tl_resp_fifo.tcl: -------------------------------------------------------------------------------- 1 | ## tl_resp_fifo stores response flits 2 | 3 | create_ip -name axis_data_fifo -vendor xilinx.com -library ip -version 1.1 -module_name tl_resp_fifo 4 | 5 | set_property -dict [list CONFIG.TDATA_NUM_BYTES {32} CONFIG.FIFO_DEPTH {32}] [get_ips tl_resp_fifo] 6 | 7 | generate_target {all} [get_ips tl_resp_fifo] 8 | 9 | -------------------------------------------------------------------------------- /afu/thymesisflow/thymesisflow_credit_mgr.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | `timescale 1ns / 1ps 25 | 26 | 27 | // module implements credit manager logic that acts as housekeeping for backpressure credits. 28 | 29 | // ============================================================================================================================== 30 | // @@@ Module Declaration 31 | // ============================================================================================================================== 32 | module thymesisflow_credit_mgr 33 | #(parameter MSB = 2) // Bit number of most significant bit. i.e. a 3 bit counter, [2:0] has MSB=2 34 | ( 35 | input clock 36 | , input resetn //active high 37 | , input reset_counter 38 | 39 | , input [MSB:0] initial_credits // Number of starting credits from TLX 40 | , input [MSB:0] returned_credits // Increment 1 credit each cycle the TLX sets this to 1 41 | , input get_all_credits 42 | 43 | , input consume_credit 44 | 45 | , output [MSB:0] credits_available // Number of credits currently available 46 | , output credit_overflow // Error - received excess credits from TLX 47 | , output credit_underflow // Error - want to consume more credits than are currently available 48 | 49 | ) ; 50 | 51 | 52 | // ============================================================================================================================== 53 | // @@@ Main counter 54 | // ============================================================================================================================== 55 | reg [MSB+1:0] credit_reg; // Make one bit larger so can check for overflow 56 | wire [MSB:0] zerovec; 57 | wire [MSB+1:0] onevec; 58 | 59 | assign zerovec = {(MSB+1){1'b0}}; // Define a string of 0s of the correct length to make comparisons easier to understand 60 | assign onevec = {zerovec, 1'b1}; 61 | 62 | always @(posedge(clock)) 63 | begin 64 | if ((resetn == 1'b0) || (reset_counter == 1'b1)) 65 | credit_reg <= {1'b0,initial_credits}; 66 | else //get_all_credits gets all available credits (consume_credits has no effect in same cycle) else normal accounting is happening. 67 | credit_reg <= (get_all_credits == 1'b1) ? {1'b0,returned_credits} : credit_reg + {1'b0,returned_credits} - {8'b0,consume_credit}; 68 | end 69 | 70 | assign credits_available = credit_reg[MSB:0]; // Pass credits available to external logic 71 | assign credit_overflow = credit_reg[MSB+1]; // Overflow if accumulate more credits than should be allowed 72 | assign credit_underflow = ({7'b0,consume_credit} > credit_reg[MSB:0]) ? 1'b1 : 1'b0; // Underflow if ask for more credits than are available 73 | 74 | endmodule 75 | -------------------------------------------------------------------------------- /afu/thymesisflow/thymesisflow_llc/thymesisflow_llc_ingress_driver.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | `timescale 1ns / 10ps 25 | // ============================================================================================================================== 26 | // @@@ Module Declaration 27 | // ============================================================================================================================== 28 | 29 | module thymesisflow_llc_ingress_driver ( 30 | 31 | input clock // Clock - samples & launches data on rising edge 32 | , input reset_n // Active Low 33 | 34 | // 32B incoming data Interface 35 | , input [255:0] driver_in_tdata 36 | , input driver_in_tvalid 37 | , output driver_in_tready 38 | 39 | // 32B AXI-STREAM 40 | , output [255:0] driver_out_tdata 41 | , output driver_out_tvalid 42 | , input driver_out_tready 43 | ); 44 | 45 | 46 | reg [255:0] outdata_q; 47 | reg outdata_vld; 48 | 49 | assign driver_out_tdata = outdata_q; 50 | assign driver_out_tvalid = outdata_vld; 51 | 52 | 53 | 54 | //Encoding of the various flit cases based on OpenCAPI TL/TLx protocol 55 | reg [1:0] ocapi_sm; 56 | 57 | parameter CMDFLIT = 2'b01; 58 | parameter DATAFLIT = 2'b10; 59 | 60 | wire is_cmd; 61 | assign is_cmd = ocapi_sm[0]; 62 | 63 | wire is_data; 64 | assign is_data = ocapi_sm[1]; 65 | 66 | 67 | wire [1:0] cmd_dl; 68 | assign cmd_dl = driver_in_tdata[167:166]; //according to the mapping 69 | 70 | wire [1:0] resp_dl; 71 | assign resp_dl = driver_in_tdata[227:226]; 72 | 73 | //Check command types 74 | wire is_writecmd; 75 | assign is_writecmd = driver_in_tdata[253]; 76 | 77 | wire is_readcmd; 78 | assign is_readcmd = driver_in_tdata[252]; 79 | 80 | wire is_nopcmd; 81 | assign is_nopcmd = ((driver_in_tdata[254] == 1'b1) && (driver_in_tdata[249] == 1'b0)); 82 | 83 | wire is_respRok; 84 | assign is_respRok = ((driver_in_tdata[250] == 1'b1) && (driver_in_tdata[248] == 1'b0)); 85 | 86 | wire is_respWok; 87 | assign is_respWok = ((driver_in_tdata[251] == 1'b1) && (driver_in_tdata[248] == 1'b0)); 88 | 89 | //we eat up nop cmds here so the other side does not have to be ready to process those. -- this is interfaced to aurora parser so it needs to be always ready 90 | assign driver_in_tready = (driver_out_tready == 1'b1) || ((is_nopcmd == 1'b1) && (is_cmd == 1'b1) && (driver_in_tvalid == 1'b1)); 91 | 92 | reg [2:0] dflit_size; 93 | 94 | 95 | always @(posedge(clock)) 96 | begin 97 | if (reset_n == 1'b0) 98 | begin 99 | dflit_size <= 3'b0; 100 | ocapi_sm <= CMDFLIT; 101 | end 102 | else 103 | case (ocapi_sm) 104 | CMDFLIT: 105 | begin 106 | if ((driver_out_tready == 1'b1) && (driver_in_tvalid == 1'b1) && 107 | ((is_writecmd == 1'b1) || (is_respRok == 1'b1))) //Commands with dataflits 108 | begin 109 | ocapi_sm <= DATAFLIT; 110 | dflit_size <= 3'b100; 111 | end 112 | end 113 | DATAFLIT: 114 | begin 115 | if ((driver_out_tready == 1'b1) && (driver_in_tvalid == 1'b1) && (dflit_size != 3'b001)) //Commands with dataflits 116 | dflit_size <= dflit_size - 3'b001; 117 | else if ((driver_out_tready == 1'b1) && (driver_in_tvalid == 1'b1)) 118 | ocapi_sm <= CMDFLIT; 119 | end 120 | endcase 121 | end 122 | 123 | 124 | //Data and command 125 | always @(posedge(clock)) 126 | begin //this is the first network Rx moduls so make sure only allowed commands get through. 127 | if (reset_n == 1'b0) 128 | begin 129 | outdata_q <= 256'b0; 130 | outdata_vld <= 1'b0; 131 | end 132 | else if ((is_cmd == 1'b1) && (driver_in_tvalid == 1'b1) && (driver_out_tready == 1'b1) && (is_nopcmd == 1'b0)) //Cmd flit out 133 | begin 134 | outdata_q <= driver_in_tdata; 135 | outdata_vld <= 1'b1; 136 | end 137 | else if ((is_data == 1'b1) && (driver_in_tvalid == 1'b1) && (driver_out_tready == 1'b1)) //Data flit out 138 | begin 139 | outdata_q <= driver_in_tdata; 140 | outdata_vld <= 1'b1; 141 | end 142 | else if (driver_out_tready == 1'b1) 143 | begin 144 | outdata_q <= 256'b0; 145 | outdata_vld <= 1'b0; 146 | end 147 | end 148 | 149 | 150 | endmodule 151 | -------------------------------------------------------------------------------- /afu/thymesisflow/thymesisflow_rr_arbiter.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | // Module designer: Dimitris Syrivelis 22 | // Backup: Christian Pinto, Michele Gazzetti 23 | 24 | `timescale 1ns / 10ps 25 | // ****************************************************************************************************************************** 26 | // File Name : thymesisflow_rr_arbiter 27 | // Project : ThymesisFlow disagreggated memory 28 | // Round robin arbiter 29 | // ****************************************************************************************************************************** 30 | // ============================================================================================================================== 31 | // @@@ Module Declaration 32 | // ============================================================================================================================== 33 | 34 | module thymesisflow_rr_arbiter 35 | #(parameter SIZE = 2) ( 36 | 37 | input clock // Clock - samples & launches data on rising edge 38 | , input reset_n // Active Low 39 | 40 | // fifo_in AXI-STREAM 41 | , input [SIZE-1:0] request_vector 42 | , input request_nxt 43 | , output [SIZE-1:0] selected 44 | 45 | ); 46 | 47 | reg [SIZE-1:0] masked_req_vector;//snapshot of the request input. 48 | 49 | 50 | //masked req vector 51 | always @(posedge(clock)) 52 | begin 53 | if (reset_n == 1'b0) //Reset active 54 | begin 55 | masked_req_vector <= 2'b0; 56 | end 57 | else if ((request_nxt == 1'b1) && (masked_req_vector == 2'b0)) //active 58 | begin 59 | masked_req_vector <= (request_vector ^ selected) & request_vector; //empty mask means no history or rollover - so we just remove previous selection. 60 | end 61 | else if ((request_nxt == 1'b1)) 62 | begin 63 | masked_req_vector <= (masked_req_vector ^ selected) & request_vector; // 64 | end 65 | end 66 | 67 | //Combinational to return selection immediately 68 | assign selected = (masked_req_vector == 2'b0) ? request_vector & (~request_vector + 1) : masked_req_vector & (~masked_req_vector + 1); 69 | 70 | endmodule 71 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/axi_hwicap.tcl: -------------------------------------------------------------------------------- 1 | ## LPC axi_hwicap 2 | create_ip -name axi_hwicap -vendor xilinx.com -library ip -version 3.0 -module_name axi_hwicap_0 3 | set_property -dict [list CONFIG.C_BRAM_SRL_FIFO_TYPE {0} CONFIG.C_ENABLE_ASYNC {1}] [get_ips axi_hwicap_0] 4 | generate_target {all} [get_ips axi_hwicap_0] 5 | 6 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/axi_quad_spi.tcl: -------------------------------------------------------------------------------- 1 | ## LPC axi_quad_spi 2 | create_ip -name axi_quad_spi -vendor xilinx.com -library ip -version 3.2 -module_name axi_quad_spi_0 3 | set_property -dict [list CONFIG.C_SPI_MEMORY {2} CONFIG.C_SPI_MODE {2} CONFIG.C_DUAL_QUAD_MODE {1} CONFIG.C_NUM_SS_BITS {2} CONFIG.C_FIFO_DEPTH {256} CONFIG.Async_Clk {1} CONFIG.C_USE_STARTUP {1} CONFIG.C_SCK_RATIO {2}] [get_ips axi_quad_spi_0] 4 | generate_target {all} [get_ips axi_quad_spi_0] 5 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/create_DLx_PHY_bypass_20.0g.tcl: -------------------------------------------------------------------------------- 1 | # Generates Transceive IP with buffer-bypass at 20.0 Gbps Using lane 4 as the master lane 2 | create_ip -name gtwizard_ultrascale -vendor xilinx.com -library ip -version 1.7 -module_name DLx_phy 3 | set_property -dict [list CONFIG.CHANNEL_ENABLE {X0Y7 X0Y6 X0Y5 X0Y4 X0Y3 X0Y2 X0Y1 X0Y0} \ 4 | CONFIG.TX_MASTER_CHANNEL {X0Y4} \ 5 | CONFIG.RX_MASTER_CHANNEL {X0Y4} \ 6 | CONFIG.TX_LINE_RATE {20} \ 7 | CONFIG.TX_PLL_TYPE {QPLL1} \ 8 | CONFIG.TX_REFCLK_FREQUENCY {156.25} \ 9 | CONFIG.TX_DATA_ENCODING {64B66B} \ 10 | CONFIG.TX_USER_DATA_WIDTH {64} \ 11 | CONFIG.TX_INT_DATA_WIDTH {64} \ 12 | CONFIG.TX_BUFFER_MODE {0} \ 13 | CONFIG.TX_OUTCLK_SOURCE {TXPROGDIVCLK} \ 14 | CONFIG.RX_LINE_RATE {20} \ 15 | CONFIG.RX_PLL_TYPE {QPLL1} \ 16 | CONFIG.RX_REFCLK_FREQUENCY {156.25} \ 17 | CONFIG.RX_DATA_DECODING {64B66B} \ 18 | CONFIG.RX_USER_DATA_WIDTH {64} \ 19 | CONFIG.RX_INT_DATA_WIDTH {64} \ 20 | CONFIG.RX_BUFFER_MODE {0} \ 21 | CONFIG.RX_JTOL_FC {10} \ 22 | CONFIG.RX_CB_MAX_LEVEL {4} \ 23 | CONFIG.RX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 24 | CONFIG.TX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 25 | CONFIG.LOCATE_RESET_CONTROLLER {EXAMPLE_DESIGN} \ 26 | CONFIG.LOCATE_TX_BUFFER_BYPASS_CONTROLLER {EXAMPLE_DESIGN} \ 27 | CONFIG.LOCATE_RX_BUFFER_BYPASS_CONTROLLER {EXAMPLE_DESIGN} \ 28 | CONFIG.LOCATE_IN_SYSTEM_IBERT_CORE {EXAMPLE_DESIGN} \ 29 | CONFIG.TXPROGDIV_FREQ_SOURCE {QPLL1} \ 30 | CONFIG.TXPROGDIV_FREQ_VAL {312.5} \ 31 | CONFIG.Component_Name {DLx_phy}] [get_ips DLx_phy] 32 | generate_target {all} [get_ips DLx_phy] 33 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/create_DLx_PHY_bypass_25.78125g.tcl: -------------------------------------------------------------------------------- 1 | # Generates Transceive IP with buffer-bypass at 25.78125 Gbps Using lane 4 as the master lane 2 | create_ip -name gtwizard_ultrascale -vendor xilinx.com -library ip -version 1.7 -module_name DLx_phy 3 | set_property -dict [list CONFIG.CHANNEL_ENABLE {X0Y7 X0Y6 X0Y5 X0Y4 X0Y3 X0Y2 X0Y1 X0Y0} \ 4 | CONFIG.TX_MASTER_CHANNEL {X0Y4} \ 5 | CONFIG.RX_MASTER_CHANNEL {X0Y4} \ 6 | CONFIG.TX_LINE_RATE {25.78125} \ 7 | CONFIG.TX_PLL_TYPE {QPLL1} \ 8 | CONFIG.TX_REFCLK_FREQUENCY {156.25} \ 9 | CONFIG.TX_DATA_ENCODING {64B66B} \ 10 | CONFIG.TX_USER_DATA_WIDTH {64} \ 11 | CONFIG.TX_INT_DATA_WIDTH {64} \ 12 | CONFIG.TX_BUFFER_MODE {0} \ 13 | CONFIG.TX_QPLL_FRACN_NUMERATOR {8388608} \ 14 | CONFIG.TX_OUTCLK_SOURCE {TXPROGDIVCLK} \ 15 | CONFIG.RX_LINE_RATE {25.78125} \ 16 | CONFIG.RX_PLL_TYPE {QPLL1} \ 17 | CONFIG.RX_REFCLK_FREQUENCY {156.25} \ 18 | CONFIG.RX_DATA_DECODING {64B66B} \ 19 | CONFIG.RX_USER_DATA_WIDTH {64} \ 20 | CONFIG.RX_INT_DATA_WIDTH {64} \ 21 | CONFIG.RX_BUFFER_MODE {0} \ 22 | CONFIG.RX_QPLL_FRACN_NUMERATOR {8388608} \ 23 | CONFIG.RX_JTOL_FC {10} \ 24 | CONFIG.RX_CB_MAX_LEVEL {4} \ 25 | CONFIG.RX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 26 | CONFIG.TX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 27 | CONFIG.LOCATE_RESET_CONTROLLER {EXAMPLE_DESIGN} \ 28 | CONFIG.LOCATE_TX_BUFFER_BYPASS_CONTROLLER {EXAMPLE_DESIGN} \ 29 | CONFIG.LOCATE_RX_BUFFER_BYPASS_CONTROLLER {EXAMPLE_DESIGN} \ 30 | CONFIG.LOCATE_IN_SYSTEM_IBERT_CORE {EXAMPLE_DESIGN} \ 31 | CONFIG.TXPROGDIV_FREQ_SOURCE {QPLL1} \ 32 | CONFIG.TXPROGDIV_FREQ_VAL {402.8320312} \ 33 | CONFIG.FREERUN_FREQUENCY {150} \ 34 | CONFIG.Component_Name {DLx_phy}] [get_ips DLx_phy] 35 | generate_target {all} [get_ips DLx_phy] 36 | 37 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/create_DLx_PHY_elastic_20.0g.tcl: -------------------------------------------------------------------------------- 1 | # Generates Transceive IP 2 | create_ip -name gtwizard_ultrascale -vendor xilinx.com -library ip -version 1.7 -module_name DLx_phy 3 | set_property -dict [list CONFIG.CHANNEL_ENABLE {X0Y7 X0Y6 X0Y5 X0Y4 X0Y3 X0Y2 X0Y1 X0Y0} \ 4 | CONFIG.TX_MASTER_CHANNEL {X0Y4} \ 5 | CONFIG.RX_MASTER_CHANNEL {X0Y4} \ 6 | CONFIG.TX_LINE_RATE {20.0} \ 7 | CONFIG.TX_PLL_TYPE {QPLL1} \ 8 | CONFIG.TX_REFCLK_FREQUENCY {156.25} \ 9 | CONFIG.TX_DATA_ENCODING {64B66B} \ 10 | CONFIG.TX_OUTCLK_SOURCE {TXPROGDIVCLK} \ 11 | CONFIG.RX_LINE_RATE {20.0} \ 12 | CONFIG.RX_PLL_TYPE {QPLL1} \ 13 | CONFIG.RX_REFCLK_FREQUENCY {156.25} \ 14 | CONFIG.RX_DATA_DECODING {64B66B} \ 15 | CONFIG.RX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 16 | CONFIG.TX_REFCLK_SOURCE {X0Y7 clk1 X0Y6 clk1 X0Y5 clk1 X0Y4 clk1 X0Y3 clk1 X0Y2 clk1 X0Y1 clk1 X0Y0 clk1} \ 17 | CONFIG.LOCATE_IN_SYSTEM_IBERT_CORE {EXAMPLE_DESIGN} \ 18 | CONFIG.TX_USER_DATA_WIDTH {64} \ 19 | CONFIG.TX_INT_DATA_WIDTH {64} \ 20 | CONFIG.RX_USER_DATA_WIDTH {64} \ 21 | CONFIG.RX_INT_DATA_WIDTH {64} \ 22 | CONFIG.RX_JTOL_FC {10} \ 23 | CONFIG.RX_CB_MAX_LEVEL {4} \ 24 | CONFIG.TXPROGDIV_FREQ_SOURCE {QPLL1} \ 25 | CONFIG.TXPROGDIV_FREQ_VAL {312.5} \ 26 | CONFIG.ENABLE_OPTIONAL_PORTS {rxpolarity_in}] [get_ips DLx_phy] 27 | generate_target {all} [get_ips DLx_phy] 28 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/create_vio_DLx_phy_vio_0.tcl: -------------------------------------------------------------------------------- 1 | # Create DLx_phy VIO 2 | create_ip -name vio -vendor xilinx.com -library ip -version 3.0 -module_name DLx_phy_vio_0 3 | set_property -dict [list CONFIG.C_PROBE_OUT1_INIT_VAL {0x1} CONFIG.C_PROBE_IN6_WIDTH {8} CONFIG.C_PROBE_IN5_WIDTH {8} CONFIG.C_PROBE_IN4_WIDTH {8} CONFIG.C_PROBE_IN3_WIDTH {4} CONFIG.C_NUM_PROBE_OUT {7} CONFIG.C_NUM_PROBE_IN {13}] [get_ips DLx_phy_vio_0] 4 | generate_target all [get_ips DLx_phy_vio_0] 5 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/ip/create_vio_reset_n.tcl: -------------------------------------------------------------------------------- 1 | # IP is created in Vivado project this script is sourced from 2 | 3 | 4 | create_ip -name vio -vendor xilinx.com -library ip -version 3.0 -module_name vio_reset_n 5 | set_property -dict [list CONFIG.C_PROBE_OUT0_INIT_VAL {0x1} CONFIG.C_NUM_PROBE_IN {2}] [get_ips vio_reset_n] 6 | generate_target all [get_ips vio_reset_n] 7 | 8 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/verilog/xilinx/encrypted_buffer_bypass/DLx_phy_example_bit_sync.v: -------------------------------------------------------------------------------- 1 | `pragma protect begin_protected 2 | `pragma protect version = 2 3 | `pragma protect encrypt_agent = "XILINX" 4 | `pragma protect encrypt_agent_info = "Xilinx Encryption Tool 2015" 5 | `pragma protect begin_commonblock 6 | `pragma protect end_commonblock 7 | `pragma protect begin_toolblock 8 | `pragma protect rights_digest_method="sha256" 9 | `pragma protect key_keyowner = "Xilinx", key_keyname= "xilinxt_2017_05", key_method = "rsa", key_block 10 | Q+o3AsIy1pK++Bj28yuvgUQHbYOMOwk0dHZG3YFeEp1Dgkjhde3pJoRXLgU3spWc5/Z7rKozaEcM 11 | /t3r+RxQU0fzciMThN747HPDXD/nXsjJc11u/J1/Ng10aPeSr+QUHGb1XDKZQs4Nfxg6qf2JBUWk 12 | 8SwNX9MAiHFrbTIBYPCKBrkgxA5N+BXKx2SUHmq9l3MH7/UzUcIGYMp75e6jLD/MitWLRIQy2p71 13 | F8ERpBBJVWgGc7NxXoqQolCffoBCB1gKkf+G236ORi2tj/hTnEZhyh+Kzl4NJ3f4+N/v4414t0YI 14 | C0AZRD5rTqpO7Dxit6F5r4FAtpQFP9us+Z6l1A== 15 | 16 | `pragma protect end_toolblock="QlwR5jmthaPt81/EM167DG/YXfoFXLXbsHcoOngZzps=" 17 | `pragma protect data_method = "AES128-CBC" 18 | `pragma protect encoding = (enctype = "BASE64", line_length = 76, bytes = 3824) 19 | `pragma protect data_block 20 | g+F0MmXBYzMpHsaoJuiyMsKgrQXPgROCPBMbgwZKlpdK8SIjtNZEADAAJw/t7XL1MTBwnJN+RmCy 21 | 8iSoZmpLwIeeRthd5a7M4eu2w8CVbiQ8gpiQXBdJwp174SXt34VcvGfYNzRPBz/AVhiBWYfOq+rp 22 | BJHoWSLYXtKuFI9++OLBRRRFuK1yfqviUfH2EzIxd5l98Y0ljXVWM1mmY/ZEyIA0O8PWoF5u734P 23 | zZzejEAhkkQSj31pKlbK3SPrmL3h5BiLOW+QWgjE7aN1G3MEd+PF5Ay5zCL0Ber0hq2DVcMkpbKB 24 | NGQN9MeG+DunzBNvF8Ixh20JOP3aEpn3/JrFI93adeOyRhN60EFSWZRxK4h5ou7+YYYd2Khbsiq1 25 | dAutIzIg+Kv+tVO5ZdkV/xphXSAYNxSRz2q/PBL25TxaRhJQn8A+K3zPUtez/yyamyKfVbptfqCR 26 | +e+E6b4ar7MFOq8g+lVslSa/fM1mADQlZp8FERmE4nkTFlVpFe1FE9DyR3nOzjmj2eERP4v2K4K3 27 | 4+grgYXFcE2RWhv4tn7TCsYvJJ2zwmRxMS2ez42BQaJx7BBjBgzjiyjVCLxrjOnuRP7tdq+JwbjU 28 | wKYnROxR+Zu8ZAZL+Y06LGRzwzg1VuUOWBI9FDrX5ocvZv/V6Ts1DaDcDQ0T0zOC6s9yMnexJoHl 29 | Fa72zKupw5XQZ6TiZ+6lgO3P3Raf2Bbz9MXeeJhKLtrHhhOWdgdbSVTxmoPqTqIsJcXcCHJU/asw 30 | AwyuSxcBIyGjQ/uUlKnOwtFQ2GdPudQgI4827J3se+4dzzfqmX5BzHFHz6LpxJDULtZ1/rTvJUz+ 31 | 67wEWCFpCjK+7Vwqkr3XQR2BXqO0VIHgGMG5POGI9RJrSXKXAE2GoNwTPLTBXUY2HrY0AIBszmZY 32 | HvDHl8XUI5jIJf+48pafv/P5xme8B2VP8EhAi6gmkx/W8brQqS/xW24ofb5AnbTUoSPxu7tEUjsZ 33 | vllOr8Bfn/gaypuLBtnWJiFtE++XqdiF5sj+1uB1G49nQdu1jMfOcneG2kDXarW3PnHlwrMVZhyw 34 | ZAYEpRTXKD+/ibjeT2ABcEN52CIHbV7rMiSemdTc3y6GzjRJLSBAmzTMMmUPcVjcAZu8nIJVfRQ7 35 | Um0dwR0kLmrSdZuVNoSlPFfU9ZBP9oYyCV0+5MaD3GebSodpxfj0R6kRn5WgohvqFsJ2DuBxiEWw 36 | oZHUPqHmpmmeFG6+O2+GYt9fm4CaVK117Q9k7r8ccLIZOZA2PBpqiqAZIYiO5fN6OAb6Wv4WBWyM 37 | qMINXfA208WxoPFC95LIUtHKht+dsNcr300xa5Yt4dqymU+efxVeJxHm2ltj07AU7lCRcX9nxL/K 38 | pERcj1IHUTJdZvKxh07PNPmNo+4BVYDsf91A4tbkfkmrLPv1dAtgYyhXcisK8kUH7/xjj8HxLuGR 39 | IRwamLPWXkJCMEL+HNG7gCn5pXXmika13OtZAcX/HxP2lkbL0qWtw2c7bj6m2a9UqI9yBKjNSZFH 40 | MOczb5v0pCSQNuxDLM3AxaCb3UT4KCorXPAiMgUQhWHqP1XiBqDPARoNOeVPZHfAbjZCWO4fw759 41 | CMPBJh+fzZej6anZQURHGEM/9SJe0aCbm/A6aHeHL/17Gct2q+r8kW75twAvMIlF/BnQJl6DcBgz 42 | B82FNuwmbw7f/UYPrlbHMnpnaaY4TARNDGWexQSQqOV1hYX6HyRdtLHvBuwSxT5M4gMYZE+IYWze 43 | wmJaeW7jJn/taggGx4B/Ng+pjkwceJMLVQSKd31tsxhRQ3Uit/kwEOsTTUD0gh/cWrptIhPohFF2 44 | 670fSSopaDRJebQZKWrMMrfjzPzj7PECFhP9F3jumMr7C0A5nPkLIbpI4QTEKxTGm92QnQWBRtbY 45 | D36+SPIX1NE5DzxlhFV5K79+a5ZZCbNzHcpKmCXLMBu392iOGU18VnYHbUifzS8wrU+JQOPFEaxa 46 | jIf/dyLplkZoxt/5iIJN8wuN2INmCJ3y6Le/aUmaFvvls1HypupTqZL3LtF6cVHQQpeEtmAj4Eop 47 | o4eobDMYCJtiv8mqVcEVrT45vUMNkBXLqnVNVju+NokV+yllPzVv3y6/9z2WVm2n3Ab2nuvTdtbO 48 | P0sww4NtqP1vK7lbVr0QQuM0zr91a9uYypq4BXBNG7tyS2FMMBAxmt0Qe3WQu7zzbSa8KWYOe7yC 49 | KVlWfmXmuIgCrd/cKdhVuPcIMxXa0sHTl8lstPv4CXqQE/oKsZ9Wj5dkAprRt/FT5uvYHxJc4a2z 50 | BXyykH4BoDanyk6hiq9CRVY1disXod1mxLmVVTahMk8WUCZA8+YifmktSg+yYHYsPGVPm/g4mB6A 51 | eiSCPys/6jKac7QrpeyxXye/CJcH7hj0y906V6r3zul9dkTyhv4uX0adj3k9MUszOwArV1C0UDy0 52 | Ywif/Z+Fbf51sW226CKtSNAKKBnAitg9j5dKdU2KQfDbb+vAGueXj1G6Z23QfOnTfZfxqBOqT8OT 53 | LXNSfsd6aUF9VUvx+DdUDo3gMjzLuqt1acol8o+9lheBwNm8Am8CaZg1LdBgPIephZ+rF3yGNfwd 54 | /+CeLHOIIQOVxUQGRUR3mk4RudWUyuZ79hqNqVa0r1nGh0R499uHjQBtda1bot1splLeWuCWGknN 55 | 47U8FSMNwPpZxKJ+5y6WyC8Q3cj1yr1SCGmaZ0U+o6q6OSbXivUbz7m32GMppT8papiVTO6FYZSO 56 | 5uvRJJOfp2N0GzbzTbj8XMDHEMn3BSW4HZMkaK0UHhmRauBxC23CN7e20VS7e8Y1EE0rqlMKGSN/ 57 | vcrupRHdfdAwjHrSs3Ks/gnT1xfR7nP3TES9+MhACGUDrtDZIgc3IpdBXBCy69pnQHCa0bfYzEKz 58 | p02RZPIOOO/f5canFNFoWWu4AM5ZSFGM49sPGMSlMMyu+hpgoZ8chymTDjo07AHFb+s3SHWY6l4R 59 | oVhNQl3bpGpEO0tcd7vELw6STd9ndXOUciPrtPC/LSK5WcMFjHyY6tyuGOqIFJlpjbZ4w9e6eGJW 60 | ejVIoS1tdQ2r6f1Pa5hkGlg9eJbN+9/aONf54Ohv6ifEhLDwDTDTMT2PVE++l9P9j5npMdjI+IBm 61 | /Y/TseOLhv4IZPKjF7DQ2ln5CkyQKvfwr4JCsAg6wtYngFF8nErg3ovvM0wOTdPdh8YGooHVpHZR 62 | pcmjTOeOHip/zG0hPWc2PkLGHbTbn8nirVv7iOqC56GrBfZUgah2LQT5xJXMYaxNv9aSJ/qA2ZjM 63 | xv5MTYEWys4YqNXMwoQ3KYrm3qWc0+ItEfY020Z/+cGGwgoPR9WCOcbJOU0KwW40OXyfIe5Bk6n6 64 | eO3xxCzxp5NzW5plsDhjbjgWNAUiFFbzb9sc7ZdwzrJftiWB2sPEMaRVONrh0kYHmKoZcNqXPd0+ 65 | B1uWii92TYMhfTlxGHxBiGM9iIE77pr00bsrZ5HXqtbF+esOk3sANFLZJM6WZFhRaOsHJspQao68 66 | MpQ9CSb2XuSdn6PUujBRgmNjHCRFaDTfPd+lQoFYw+4IY8MFs8GM/4kjeSeibrFkygriwftUTu8R 67 | HShlxX5SWwtPZCW9ZW/wkAp7i56ZeJWOMJnVQLhoGiGZqXMT0L1KFxQCuFdRiV8ejtQsy3F5eCa+ 68 | H+6F+fEFj6tIHyOXUrQdJx7CX5FaTzfrTqaTW+n2JFTqC7PTayXUzcnarwdJUNFpnSI62dzoM04R 69 | 2yWSqzYAqDSCl8HT12QNQkbTHs3TBo9zjSslZ3jrGIu6RjrC8T8xjcwNVTtTROobm8kPWGwepQUC 70 | z9CoZxNgqw5dlHVXdmdSaCYon5jfMczCpUdW6fo+rGw35YNFcfyLjWDcH7bHcj73EjJ2XJsNtTGn 71 | GVqTWGL+Dfuri37NmWfdLaGK5ErCGIPlMomeM6kQy+/fLUxTs3/NkzO+w+9aRvgajT/NmEr+q3xj 72 | 71tsBWlhnzaVqDeEthrIJx3qFDMnrq1FrM0BiGGEHE2XbQ03YwhfA3kd862IPQw6IljbKps1u2m9 73 | o0mGwCOA5nTGVmtSCpCk1AZ+mJ+oQ/GKlDO8/JEyairHNU+OBmeV6KtxvU1b1kvdqFCkgWpAeqZ6 74 | 7myJraiKN+yzeI3+t4QdmEunH3QMQIaUR/KQnpgWoOgyNyaAKwscgZKXJ4AJTy3O6JxpQsbY1uKo 75 | eUtHWpLjnsBmXuPnKRZ0Ls1mcxLe+MbojFki9dHaGEGpoeH7BtnJIQqnC0sazijiykpPdEKotYKS 76 | U6Nz3jDSaGIR3WZGTi7LXcuLxayxjJOIHTqXJrF31fIbsa8kVbJXAmDvU+VeSwi/EimFoZ11gzvy 77 | 0Nfw4c9DoPtRXlaL9Kea0WS3eLuL8+BQzSDxx3aWfSBn9BVkIzDkNYDiVJadmQQmyauvFByBCr6j 78 | nEF3cnW1E3xkwm7FAxLeGUxHgyGxr2JqV4pm2+jNkac0bNJjNcGz9ruPU3u1LQEQAD2NXn1AEbGZ 79 | XFtmybZxCvOiOHvp4W/oHbgWmWniyG09uTTVV1BmLiBHLAlGrrtklwR0+U6cWIG/nOQk2R2ZLDOO 80 | pB/sUcNw+BsfSp5Xws/oQgSNRiMgGMDMEMBkM5JIdclzrDJPei1D2oZQooT+Yh5cS1/B+qDN7laL 81 | Q6qc83j3LsUNw4eYa3FNjKr91D+PEihvesG+wCFskusDmx2+XOkYkxsbspkelqJbNa8LQQMTP0LD 82 | ZMTHbMt6IVGt0T52uyRE7L1IdD/tlFTufcFPMmXdzz6fk5IEN5Y3XCsKGAWSWSS9CCf7f/X8NFpx 83 | uCDZcfmF9HodqA3HrIW5Mm0cBnXiBdi4TR8XUYQh6EFByyAqo4H1sIvm9g7/uOHxZXMFky+t+H1W 84 | y2ybqH+GpgVfo0NsAI+3//BG/0vhGm/pf33mpJakXQTpYw9vGu6CSSzoWMTfKiLx/rN5ZfrvcqxE 85 | N0hbkep0hHMU6G+gYl4p2ssCujH2gStXf1t3PLbhzF3sSYDcavM/Xv7GDTuK6tkk+zP1zwiWhpjs 86 | nvYa9W4L4E21Jen1KvxVzz67Umc38za1qWAx9BGFfodjmD06bNkUvWoo9bc8LEl2IqL77fWVPip7 87 | dkvxhTE= 88 | `pragma protect end_protected 89 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/verilog/xilinx/encrypted_elastic_buffer/DLx_phy_example_bit_sync.v: -------------------------------------------------------------------------------- 1 | `pragma protect begin_protected 2 | `pragma protect version = 2 3 | `pragma protect encrypt_agent = "XILINX" 4 | `pragma protect encrypt_agent_info = "Xilinx Encryption Tool 2015" 5 | `pragma protect begin_commonblock 6 | `pragma protect end_commonblock 7 | `pragma protect begin_toolblock 8 | `pragma protect rights_digest_method="sha256" 9 | `pragma protect key_keyowner = "Xilinx", key_keyname= "xilinxt_2017_05", key_method = "rsa", key_block 10 | qY/FxZbbT7BVU+DWDmzlnJAe76p/9xthWWBvi65stzOSI2z2zH9VIy1Xaa/sQIq3+3GorqRTvBPk 11 | 2jheRHJ42MqTMUfj0Ty+2m70vv3iLjd5pVf2lUTlbH6aVWdPc8VGbaP95P053GIV1ymkraCmY35g 12 | F5I7MbSgYebE7tm4JBGq3+KbAOEYx9zSmBh177FBNEBb0vSoNlLjZnc2yYfa44+V53xTPiSpK14o 13 | C9bRs+s5T5xMy+uyrx1cQs1TCzlDvYIQtbppf+AI/OMaDfIkBdIHaw0MgkjF/kxON+FoWtyr733V 14 | ACiK/ZxoT7pKRCuTPCPVaHrVNYY7tDpoFZZz7w== 15 | 16 | `pragma protect end_toolblock="YuoBYKBwbAJX7w+6cp2n8rUxqhyFk5OK1rlzfhrls2Q=" 17 | `pragma protect data_method = "AES128-CBC" 18 | `pragma protect encoding = (enctype = "BASE64", line_length = 76, bytes = 3824) 19 | `pragma protect data_block 20 | xPtSX9juFkKlgfmjK+hpnSHUpW6z6eKa63FyMM9T2ViDN/9W+T6RHMLhb6q/qBpYzmXyt0PbrhjR 21 | 7Bs4Ids8JUkDmuHUFY+YEcXmLNxJLEmFTNkX1JMfnguNK9sDg1Nx228b6RMhCRzj6HIZx00F0qXQ 22 | hUrKmdN0wG8bf7yA+eqGiNWBME2wu4Z4el7wgMPeC13wiDyHht3GLIWoUpkfp4HJi7QXpe+BSJJY 23 | WaJ+LjR311qtUZ8hCLLsGQfzy22IJmVfP0eyRIomaoEOyfkEZb2nM05IL66+ItYz8a+EcTCOwbWA 24 | NZsOaMSryur4yGP9U/xrNQvdQTQFlcEUIQWsh9WifzhaOv1hcBoHOJs8SZc7Rn1dZ0/aJuie3ZOR 25 | fvj7VyGcCkfcnXzWdyVt1XD8VpyLFFfSvUkiW/bxZOulAZlR5y8IDMQpaXVbfALsXyF7p4/cZV8v 26 | UTGFWlra5vANgDSGMTSwM3DUulejOrsjmphUmwKBrtBLzXsaktTaUYiqRxby0ONxi8q1xXv5wpzZ 27 | Pq6dOcu+Q72a/e+TJAi9NiyfnGB2Dng9uG4/MaUUy0SDqxlkTr3YHQw1C09crhRM6bQkfT+m067o 28 | kxrTv0mXmCzKluQKc+9YJDVr6nVlt0r/y/MsEHz4z/6zfN0h3g3cebQ5LlAE9quiEI55Y9aKrsUp 29 | wD0Nm6H1GiJMPN3n1tLTz0bn7O6iwQttBw/6cVMd5DeGUTb302elRWlyNbkvx/gl49zCKRFEBJ+b 30 | tBZRo5sTzcAV3DCwygkBQioEazFmN8Zo6Doxd4o1t6fBe/jORTjwOaDERJ+bsow+U3RYlRIghBEq 31 | ETbeS7DBWZb9sRONzsoycVHCMpowZqSWdIGTNG3vYnpgO4EK1FlQKdOXZH4Wq9O45jlX/whmeIGb 32 | nKC2L4T9vm68O5e8/NahHPN9X1Zn74MzMcGZY35VXoNWLom5I14oMTVlhDFq/5ifRPLB+Z59ai87 33 | okRoGc3CUuHpoF5bRZU5ZPOXrlu1Q48iz0suGRc8V8JCtxbam15L1pDqVuU0XRVRQ9KmjzJ1mDrT 34 | uv3+Fj3S2Gzbtt/cQgecck/7jEDidur89gwHOn9oYrlGGNanbSjgS1QDmLtpdGfn/OFM5W4mtGI0 35 | Vb9fyy0ZY7bHris+lRUIf0pazycfiky4EjYXghjZdxMIS8Lj5GJmed9fvupMpuay/9Di1lnZvwTI 36 | saky2rFFZdFMQnipHVlDhABK3NABI/1P/JHclDIeAYFK5mBpXg/WLFP3z+eiihJvk4XM7+PLqUVD 37 | ONmWfD5T2Y0y/V34fy0HBOKbJINrtCZWpISngqKwDKqeODa//g214UuUGqCzTnlsKCByh1WyiW6f 38 | kv0XOyKEEki38AMbsS/qL3LD3Sq0jkM48eFevQDfh/aqT3S7cCI1fKpZTyIxQ0ybS4n7I7K/rFg7 39 | r6tj2ykElJKt62Ps5ZaAPIugjB1Kz+o/IiqYET53qdtXpDPu1opbQ7NP0g6zc1saboVD8M4KXubx 40 | W6yD7KDASnmZgfqIKU4493VwtPv1QSnRdaN8Br5rB3map4eq0dse43qGnB+9EZmzSxsG07JPjAib 41 | Rm6Df3xrYucMr5Cxd9944E4EQFemWETSWbP5+ulMIipb74kADj9mmLyW4uMSUIQmAAVLzKeOJTfF 42 | IBhW+Z77LKJnvaLQGgdOG3OzNXVnvPmciH2EfH5OTyAMGAoMn3DOEylTJvsTzrOaNbHvZlEHOfEV 43 | Kyn7Jy+Yuzq+57q1j/LJ3cVNWfuB9CEtYTbahyAOpRf1oOH6FVLJsqpRrC5Or18iPpcpgYOENM/w 44 | Cx6H42TMLIehm4bKjRqAY3KG7WphLxcFYZMyg/nL+KT67TXcs6VVS0sEEVaFLI4wDu4nQNOi3CZE 45 | HkZ7M6KXOrlV8fqKPl60FOroB+az1MHx8djW7FRbkTGHsggFuQ+b20OaSzd0GrJpSCFXfe72YKQ/ 46 | lqSR2Out0BdNhToGuouc9eNlZq9bZYqvEantvQ/nd0kNm0e4xlygVyoUTMstRmxSHlhg1NzbVzs/ 47 | JeHfsYzL7QiZ4UHvJAF16hRWwXOH4e94n1+y914m4zX2dEOEQXyLT5W1qMMzzfTbYGGIhva81/j1 48 | 4Vvow/UXjPo7/EmJipx1n5rfkN7GCErVOs8ZQNZMbgytfJyw39Sx3t72YUzB+09c4Hqd2CX4bZGd 49 | cb7jnEeO+xzJwh03rfME/gnzU96yxXjw5QaQioNEMQbWpPmiTGeboM+xGgrUaUsQm9CYPNf5DxoW 50 | TgZxB7PAN2pz5BSVTUUFpmVXWm2rlkMoKtSYXA7GNWrxczqJUY4E6VphGJUzwNv5fStPsLq1IA9H 51 | MbdHInbQ7VGBgsi27C2/HkK9xLzItDcFEwl56cfuqXNVVYf2RfCayozXDKb/cYFFxyg+4sXo8dOK 52 | LhA5zGA2E6t3IIdIt93Mdby8g6bwWOncENZ45SMnUJpmshnxCOi0jdHlmdDciv7IRdjXf+CiPeZp 53 | 53/I3Tr8JOREvgcn5ySss9QYsXXCH9iRpT3dMA6Pf/nCmvYplbr620/fvzDN1+UW7610UIQeBedr 54 | yenwOOOKYc4fAnJMCtzXCE92TTafIOKJa5eCWTC/Bn+jNeVEzXRj3pf7oRiHQ9nZx0zium466eUp 55 | QXZB+j2m9L5s6V3pBd3RyCXLiwO1wpaieX+CHvCzzTY0uOuYV73ZPZES5VWFNfXCitLgttkO4J+x 56 | sOnrSX+EWZGPalCeeZl6E0NM6VRCeGjTYpXUt2p9f8sMLH8ZIbpRzpj1FGjph3NfG9bHADJ6SL9F 57 | hYQClNjeY9O1gG0PaQjPYMHi6Be/kOW6mRGnaar/3ACeIMdWLIo8NpT2LLt8WpZanDWBrLUMpy+n 58 | 7f3BiEEsrrc0tskUNSs7QsIMpyf/cuE2ceOtQEVGKXcUqiS1DkGd8dPS6wmAFrAHN6ICjDRHTzFk 59 | kA9PEH3meqQ6frUBmY5I/Vm2rKw5avK23NMojQnY0+S+YryJypYUMYzG8ypjDehbReeOc/W7D9Uq 60 | gAUHCYNR5CNUl//Wvpv8+j9AXKLPy4WHWiGSauxcZYhqvcf/xfAll/HnsTNP0aigJvtcMI0LxX3F 61 | jJ1/Q6clK//clWZzO3NW0QxqgVIZfY/8IU/MVfXjQgdrmLrygfTnCVdkKHcF7sfyQ1/4yjIKlC0Q 62 | rocnS5j3GGcUB/OMtwG+/M3ulZNWIf/8q2bcoUyU2REt1nD/93Hav7aHmnOkO5s8rdbdfcAd8tLj 63 | rTeKOMG3aBf0WkKdpCleL3grNZJKUFMEdI2CppzrNUyLrWFwjp61W79IK6MCjDjK/W3mQ77rouIy 64 | 8MyRJJwYeLIdafoKQ9yDWfmauyz8ArvNsqY7ggSDrLz6Towg20lmMVd6ruMaZkWu77S5cbU6Ywpw 65 | kHkz0GmAbIvdn93iZkNLGizS1JgsGJ+0t/K8w1ORF/0WNnLudxbFd4f1aBP5u1YD/ADHL/mCDyMZ 66 | NEteAFi3zFR1R+HvtMek5BxGLeAHcIWaddxIYGxhK5tHCoJVMcpFQ8Qgo7smVP3Ze5bABZPlXUnU 67 | WlSBeuoaDpwFitsbUNjVAhQ4ExQ1MeIXmQsn1WmWPEXJAt78PtASDuSzHKAgVpkQIWsGOIwjPi+/ 68 | ODtIxIiD2Imwo6qM8OTocCc/WRgY2LKQXh34heZukMvTT+esdOpwyFft9UDpdLmvHFS3mcicPWAa 69 | e1tqUzu5fpBrctktiMXEvpGt8DCAopSSyrA8bB+TLc2PgQB6bNKMJ4iKYvPghAdvY8EuDMFKFEux 70 | euEQAxfYzwf/tTWIfyKJTC0YW53f5ZJRFg2lnFWYWXPBcWP8wuiHqrgl9CImfvoIQHyCBru3OLw7 71 | 64RMXKLoDlIkR5r6mD+THULlYDE6dEu8822/qV5LwIfkJ2oGK8zNOCSsJKn56bkwl2oCl3nMvndG 72 | PQIh2C/au3DjYGJdB12ayW4GHzqPlPCEJaGrggjOjwXP7GoBwFQFA66VJ+fzCVXHHkBc6x7c1hsd 73 | MQJHLG4RIFFccLe6JQjQ8TaRreP6g2j9XiPdty0nU8BlLiRYRFP0f2NNY2+M3j9/ur2ULovJ2/ae 74 | rhJLX45z4xH6lubBCZNvrNjJTXDusY1TG34vyWQL2P1J5THI/xCnf/WsJs81k80fBAuHpy1RXNDZ 75 | o8MrOn+NaarpibIJRvnbESanW6jwEqdpcr9B8LS/KsX5AVB7tuYfntCuKV5FR/a4wssrc2KM3Nae 76 | DKmVxethxxGZrQeCJ9jUaUSORhlplzBXGvDI6l0qh8saBvT6D7UOUS12AoQ9zp2foocgUsJR0FvD 77 | gkQJEcO+LG1TmLqmLF4M+ecO4RKV+MQKDmicf7kKLD1g7/z4lJwzXCGgHWJOg5aM/etovgm4F8cd 78 | 7YBS9YvBbNpjy2mECKrJ/s81qgasJ8wmRC6jW/iSzlfDE9Bb4+aKjOtnfu0rOHI0B2hQDoli4L1u 79 | w72p3pdeByxum2DH7SYjtArc9n9FPSGv752z4r27Y6MhFMP8sbyzQzrYsa/HWCl9D2EC/upcNdre 80 | sAeLer4AgdetGSF31UjFV9LSjtQOTfiO6kOdk4xsGt72+V+G4DVRbagR6X2NbRXbCvZXOHW2Iwog 81 | UIQ46qf3lpyzXCIhpmrpg6kmV8fgTBoyLYBgfxP47z/pepI+HFSP441BRVxO1GzfkkpvP+VbCCxv 82 | GJF7vwtsUJ+ahCUKB2dGsk5quZvJnwszQ1D2CTYyY1hcv8nRnGOLbd1fnElCZNVvKMtb1BMvOeL0 83 | CtafwFduJQ3V6WZYFiCFVX+CiOBYUDTUlROCkJQ7NaFh+K0CwD8HkI+s9mVRfUwxp0zMHiFVeNou 84 | MQy1gQUh05g+TaeBkviCs/FJbYdGVCw5Yu5SofbzW0HhgaN67abK6i7uvM3bFcIbQbGHl1qBTayq 85 | 5rlZ6O1GZMFU2ImHAL2q1Tz06DclRLRb8GoCybnpXLKkh3n40LTk4IkqMOl7Om+cBsp7ztXhfVFF 86 | juNWeJO7blciiGXc4QWYs7ToBH/nfZJtfg6tE1X1q+I9GhQDxKCT1GvrGtIQKSqeCqQe5m3dTksO 87 | RKN9JTU= 88 | `pragma protect end_protected 89 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/extra.xdc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenCAPI/ThymesisFlow/f69f7ae0bdc245e4e969e488e118359dde9c8b40/board_support_packages/ad9v3/xdc/extra.xdc -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_pinout.xdc: -------------------------------------------------------------------------------- 1 | ############################################################### 2 | ## Placement Constraints: xcvu3p-ffvc1517 3 | ############################################################### 4 | 5 | set_property PACKAGE_PIN A30 [get_ports ocde] 6 | set_property IOSTANDARD LVCMOS18 [get_ports ocde] 7 | 8 | ######################## 9 | ## Bottom Left Corner ## 10 | ######################## 11 | 12 | # DLX0 Clocks 13 | set_property PACKAGE_PIN AG34 [get_ports mgtrefclk1_x0y0_n] 14 | set_property PACKAGE_PIN AG33 [get_ports mgtrefclk1_x0y0_p] 15 | set_property PACKAGE_PIN AC34 [get_ports mgtrefclk1_x0y1_n] 16 | set_property PACKAGE_PIN AC33 [get_ports mgtrefclk1_x0y1_p] 17 | 18 | # Free run clock constraint 19 | # page 13 adm-pcie-9v3 user manual.pdf 20 | # Signal: FABRIC_CLK 21 | set_property PACKAGE_PIN AP27 [get_ports freerun_clk_n] 22 | set_property PACKAGE_PIN AP26 [get_ports freerun_clk_p] 23 | set_property IOSTANDARD LVDS [get_ports freerun_clk_p] 24 | set_property IOSTANDARD LVDS [get_ports freerun_clk_n] 25 | 26 | # DLX0 TRANSMIT/RECEIVE Channels 27 | set_property PACKAGE_PIN AV36 [get_ports ch0_gtyrxp_in] 28 | set_property PACKAGE_PIN AV37 [get_ports ch0_gtyrxn_in] 29 | set_property PACKAGE_PIN AW33 [get_ports ch0_gtytxp_out] 30 | set_property PACKAGE_PIN AW34 [get_ports ch0_gtytxn_out] 31 | set_property PACKAGE_PIN AU38 [get_ports ch1_gtyrxp_in] 32 | set_property PACKAGE_PIN AU39 [get_ports ch1_gtyrxn_in] 33 | set_property PACKAGE_PIN AU33 [get_ports ch1_gtytxp_out] 34 | set_property PACKAGE_PIN AU34 [get_ports ch1_gtytxn_out] 35 | set_property PACKAGE_PIN AR38 [get_ports ch2_gtyrxp_in] 36 | set_property PACKAGE_PIN AR39 [get_ports ch2_gtyrxn_in] 37 | set_property PACKAGE_PIN AT35 [get_ports ch2_gtytxp_out] 38 | set_property PACKAGE_PIN AT36 [get_ports ch2_gtytxn_out] 39 | set_property PACKAGE_PIN AN38 [get_ports ch3_gtyrxp_in] 40 | set_property PACKAGE_PIN AN39 [get_ports ch3_gtyrxn_in] 41 | set_property PACKAGE_PIN AP35 [get_ports ch3_gtytxp_out] 42 | set_property PACKAGE_PIN AP36 [get_ports ch3_gtytxn_out] 43 | set_property PACKAGE_PIN AL38 [get_ports ch4_gtyrxp_in] 44 | set_property PACKAGE_PIN AL39 [get_ports ch4_gtyrxn_in] 45 | set_property PACKAGE_PIN AM35 [get_ports ch4_gtytxp_out] 46 | set_property PACKAGE_PIN AM36 [get_ports ch4_gtytxn_out] 47 | set_property PACKAGE_PIN AJ38 [get_ports ch5_gtyrxp_in] 48 | set_property PACKAGE_PIN AJ39 [get_ports ch5_gtyrxn_in] 49 | set_property PACKAGE_PIN AK35 [get_ports ch5_gtytxp_out] 50 | set_property PACKAGE_PIN AK36 [get_ports ch5_gtytxn_out] 51 | set_property PACKAGE_PIN AG38 [get_ports ch6_gtyrxp_in] 52 | set_property PACKAGE_PIN AG39 [get_ports ch6_gtyrxn_in] 53 | set_property PACKAGE_PIN AH35 [get_ports ch6_gtytxp_out] 54 | set_property PACKAGE_PIN AH36 [get_ports ch6_gtytxn_out] 55 | set_property PACKAGE_PIN AE38 [get_ports ch7_gtyrxp_in] 56 | set_property PACKAGE_PIN AE39 [get_ports ch7_gtyrxn_in] 57 | set_property PACKAGE_PIN AF35 [get_ports ch7_gtytxp_out] 58 | set_property PACKAGE_PIN AF36 [get_ports ch7_gtytxn_out] 59 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_placement_bypass.xdc: -------------------------------------------------------------------------------- 1 | ############################################################### 2 | ## FPGA: xcvu3p-ffvc1517-2-i 3 | ############################################################### 4 | 5 | ## Settings to generate MSC file 6 | # Configuration from SPI Flash as per XAPP1233 7 | set_property BITSTREAM.GENERAL.COMPRESS {TRUE} [ current_design ] 8 | set_property BITSTREAM.CONFIG.EXTMASTERCCLK_EN {DIV-1} [current_design] 9 | set_property BITSTREAM.CONFIG.SPI_32BIT_ADDR YES [current_design] 10 | set_property BITSTREAM.CONFIG.SPI_BUSWIDTH 8 [current_design] 11 | set_property BITSTREAM.CONFIG.SPI_FALL_EDGE YES [current_design] 12 | 13 | # Set CFGBVS to GND to match schematics 14 | set_property CFGBVS GND [current_design] 15 | 16 | # Set CONFIG_VOLTAGE to 1.8V to match schematics 17 | set_property CONFIG_VOLTAGE 1.8 [current_design] 18 | 19 | # Set safety trigger to power down FPGA at 125degC 20 | set_property BITSTREAM.CONFIG.OVERTEMPSHUTDOWN Enable [current_design] 21 | 22 | 23 | # Pblock for placing VIO IPs and Debug Hub IP closer to DLx logic. This Pblock includes everything in the design except for a couple of I/O and clock buffers and BSCAN primitives which are constrained outside of this Pblock 24 | create_pblock pblock_1 25 | resize_pblock pblock_1 -add CLOCKREGION_X0Y0:CLOCKREGION_X1Y1 26 | add_cells_to_pblock pblock_1 [get_cells [list dbg_hub bsp/dlx_phy bsp/DLx_phy_vio_0_inst bsp/vio_reset_n_inst_tlx bsp/tlx]] 27 | 28 | remove_cells_from_pblock pblock_1 [get_cells bsp/dlx_phy/IBUFDS_freerun] 29 | # remove_cells_from_pblock pblock_1 [get_cells dbg_hub/inst/BSCANID.u_xsdbm_id/SWITCH_N_EXT_BSCAN.u_bufg_icon_tck] 30 | remove_cells_from_pblock pblock_1 [get_cells bsp/dlx_phy/BUFGCE_DIV_inst] 31 | # remove_cells_from_pblock pblock_1 [get_cells dbg_hub/inst/BSCANID.u_xsdbm_id/SWITCH_N_EXT_BSCAN.bscan_inst/SERIES7_BSCAN.bscan_inst] 32 | 33 | set_property USER_CLOCK_ROOT X0Y1 [get_nets {bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/rxusrclk2_in[0]}] 34 | 35 | set_property EXTRACT_ENABLE NO [get_cells {bsp/dlx_phy/ocx_dlx_top_inst/rx/main/replay_deskew_cntr_q_reg[*] bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/ts1_cntr_q_reg[*] bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/lfsr_q_reg[*] bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/deskew_buffer?_q_reg[*] bsp/dlx_phy/ocx_dlx_top_inst/rx/main/crc_bits_q_reg[*]}] 36 | set_power_opt -exclude_cells [get_cells {bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/bram/ram_sdp_reg_?}] 37 | 38 | set_property MAX_FANOUT 60 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/tx/ctl/ctl_x4_not_x8_tx_mode}] 39 | # set_property MAX_FANOUT 60 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/rx/main/deskew_all_valid_l0}] 40 | set_property MAX_FANOUT 80 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/rx/reset}] 41 | set_property MAX_FANOUT 50 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/crc_zero_d1_q}] 42 | set_property MAX_FANOUT 50 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/rx/training_q}] 43 | set_property MAX_FANOUT 26 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/crc_zero_d2_q}] 44 | 45 | # 1024: Added more constraints to disable inference of enable/reset for some regs 46 | set_property EXTRACT_ENABLE NO [get_cells {bsp/tlx/OCX_TLX_FRAMER/cmd_cntl_flit_reg_reg[*]}] 47 | set_property EXTRACT_ENABLE NO [get_cells {bsp/tlx/OCX_TLX_FRAMER/vc0_fifo/valid_entry_counter_reg[*]}] 48 | set_property EXTRACT_ENABLE NO [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_Parser/flit_parser/ctl_flit_dout_reg[*]}] 49 | 50 | set_property EXTRACT_RESET NO [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/CMD_FIFO_MAC/CMD_INFO_CTL/ctl_cnt_dout_reg[*]}] 51 | set_property EXTRACT_RESET NO [get_cells {bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/pre_crc_data_q_reg[*]}] 52 | set_property EXTRACT_RESET NO [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/RESP_FIFO_MAC/RESP_INFO_CTL/data_wr_cnt_dout_reg[*]}] 53 | set_property EXTRACT_RESET NO [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/CMD_FIFO_MAC/CMD_INFO_CTL/data_wr_cnt_dout_reg[*]}] 54 | 55 | set_property C_CLK_INPUT_FREQ_HZ 300000000 [get_debug_cores dbg_hub] 56 | set_property C_ENABLE_CLK_DIVIDER false [get_debug_cores dbg_hub] 57 | set_property C_USER_SCAN_CHAIN 1 [get_debug_cores dbg_hub] 58 | connect_debug_port dbg_hub/clk [get_nets clock_tlx] 59 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_placement_bypass_thymesisflow_cpu.xdc: -------------------------------------------------------------------------------- 1 | 2 | create_pblock aurora_qsfp_pblock 3 | add_cells_to_pblock [get_pblocks aurora_qsfp_pblock] [get_cells -quiet [list oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP0/TFLLC_32B_FRAMER oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP1/TFLLC_32B_FRAMER]] 4 | resize_pblock [get_pblocks aurora_qsfp_pblock] -add {CLOCKREGION_X0Y3:CLOCKREGION_X0Y4} 5 | 6 | 7 | create_pblock driver_pblock 8 | add_cells_to_pblock [get_pblocks driver_pblock] [get_cells -quiet [list oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP0/TF_32B_DRIVER_EGR oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP1/TF_32B_DRIVER_EGR ]] 9 | resize_pblock [get_pblocks driver_pblock] -add {CLOCKREGION_X1Y4:CLOCKREGION_X1Y3} -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_placement_bypass_thymesisflow_mem.xdc: -------------------------------------------------------------------------------- 1 | 2 | 3 | create_pblock aurora_qsfp_pblock 4 | add_cells_to_pblock [get_pblocks aurora_qsfp_pblock] [get_cells -quiet [list oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP0/TFLLC_32B_FRAMER oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP1/TFLLC_32B_FRAMER ]] 5 | resize_pblock [get_pblocks aurora_qsfp_pblock] -add {CLOCKREGION_X0Y3:CLOCKREGION_X0Y4} 6 | 7 | 8 | create_pblock driver_pblock 9 | add_cells_to_pblock [get_pblocks driver_pblock] [get_cells -quiet [list oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP1/TF_32B_DRIVER_EGR oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP0/TF_32B_DRIVER_EGR ]] 10 | resize_pblock [get_pblocks driver_pblock] -add {CLOCKREGION_X1Y4:CLOCKREGION_X1Y3} 11 | 12 | #create_pblock tfllc_framer 13 | #add_cells_to_pblock [get_pblocks tfllc_framer] [get_cells -quiet [list oc_func/AFU00/TF_TOP/TFLLC_32B_QSFP1/TF_LLC_32B_FRAMER oc_func/AFU00/TF_TOP/TF_LLC_FRAMER_BRAM1]] 14 | #resize_pblock [get_pblocks tfllc_framer] -add {SLICE_X0Y120:SLICE_X16Y179} 15 | #resize_pblock [get_pblocks tfllc_framer] -add {DSP48E2_X0Y48:DSP48E2_X1Y71} 16 | #resize_pblock [get_pblocks tfllc_framer] -add {RAMB18_X0Y48:RAMB18_X1Y71} 17 | #resize_pblock [get_pblocks tfllc_framer] -add {RAMB36_X0Y24:RAMB36_X1Y35} 18 | 19 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_placement_elastic.xdc: -------------------------------------------------------------------------------- 1 | ############################################################### 2 | ## FPGA: xcvu3p-ffvc1517-2-i 3 | ############################################################### 4 | 5 | ## Settings to generate MSC file 6 | # Configuration from SPI Flash as per XAPP1233 7 | set_property BITSTREAM.GENERAL.COMPRESS TRUE [current_design] 8 | set_property BITSTREAM.CONFIG.EXTMASTERCCLK_EN DIV-1 [current_design] 9 | set_property BITSTREAM.CONFIG.SPI_32BIT_ADDR YES [current_design] 10 | set_property BITSTREAM.CONFIG.SPI_BUSWIDTH 8 [current_design] 11 | set_property BITSTREAM.CONFIG.SPI_FALL_EDGE YES [current_design] 12 | 13 | # Set CFGBVS to GND to match schematics 14 | set_property CFGBVS GND [current_design] 15 | 16 | # Set CONFIG_VOLTAGE to 1.8V to match schematics 17 | set_property CONFIG_VOLTAGE 1.8 [current_design] 18 | 19 | # Set safety trigger to power down FPGA at 125degC 20 | set_property BITSTREAM.CONFIG.OVERTEMPSHUTDOWN Enable [current_design] 21 | 22 | 23 | 24 | 25 | # Added by milans to improve timing results 26 | 27 | # Pblock for placing VIO IPs and Debug Hub IP closer to DLx logic. This Pblock includes everything in the design except for a couple of I/O and clock buffers and BSCAN primitives which are constrained outside of this Pblock 28 | create_pblock dlx_pblock 29 | add_cells_to_pblock [get_pblocks dlx_pblock] [get_cells -quiet [list DLx_phy_vio_0_inst GND VCC bsp/dlx_phy/GND bsp/dlx_phy/IBUFDS_GTE4_MGTREFCLK0_X0Y0_INST bsp/dlx_phy/IBUFDS_GTE4_MGTREFCLK0_X0Y1_INST bsp/dlx_phy/VCC bsp/dlx_phy/bit_synchronizer_vio_gtwiz_buffbypass_rx_done_0_inst bsp/dlx_phy/bit_synchronizer_vio_gtwiz_buffbypass_rx_error_0_inst bsp/dlx_phy/bit_synchronizer_vio_gtwiz_buffbypass_tx_done_0_inst bsp/dlx_phy/bit_synchronizer_vio_gtwiz_buffbypass_tx_error_0_inst bsp/dlx_phy/bit_synchronizer_vio_gtwiz_reset_rx_done_0_inst bsp/dlx_phy/bit_synchronizer_vio_gtwiz_reset_tx_done_0_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_0_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_1_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_2_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_3_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_4_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_5_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_6_inst bsp/dlx_phy/bit_synchronizer_vio_rxpmaresetdone_7_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_0_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_1_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_2_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_3_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_4_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_5_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_6_inst bsp/dlx_phy/bit_synchronizer_vio_txpmaresetdone_7_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_0_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_1_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_2_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_3_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_4_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_5_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_6_inst bsp/dlx_phy/bit_synchronizer_vio_txprgdivresetdone_7_inst bsp/dlx_phy/example_init_inst bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst bsp/dlx_phy/example_wrapper_inst/GND bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/GND bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/VCC bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/gen_gtwiz_userclk_tx_main.gtwiz_userclk_tx_active_meta_reg bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/gen_gtwiz_userclk_tx_main.gtwiz_userclk_tx_active_sync_reg bsp/dlx_phy/ocx_dlx_top_inst]] 30 | resize_pblock [get_pblocks dlx_pblock] -add {CLOCKREGION_X0Y0:CLOCKREGION_X0Y1} 31 | 32 | set_property USER_CLOCK_ROOT {X0Y1} [get_nets {bsp/dlx_phy/example_wrapper_inst/gtwiz_userclk_tx_inst/rxusrclk2_in[0]}] 33 | 34 | set_property EXTRACT_ENABLE false [get_cells {{bsp/dlx_phy/ocx_dlx_top_inst/rx/main/replay_deskew_cntr_q_reg[*]} {bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/ts1_cntr_q_reg[*]} {bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/lfsr_q_reg[*]} {bsp/dlx_phy/ocx_dlx_top_inst/rx/lane?/deskew_buffer?_q_reg[*]} {bsp/dlx_phy/ocx_dlx_top_inst/rx/main/crc_bits_q_reg[*]}}] 35 | set_power_opt -exclude_cells [get_cells bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/bram/ram_sdp_reg_?] 36 | 37 | set_property MAX_FANOUT 60 [get_nets bsp/dlx_phy/ocx_dlx_top_inst/tx/ctl/ctl_x4_not_x8_tx_mode] 38 | # set_property MAX_FANOUT 60 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/rx/main/deskew_all_valid_l0}] 39 | set_property MAX_FANOUT 80 [get_nets bsp/dlx_phy/ocx_dlx_top_inst/rx/reset] 40 | set_property MAX_FANOUT 50 [get_nets bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/crc_zero_d1_q] 41 | set_property MAX_FANOUT 50 [get_nets bsp/dlx_phy/ocx_dlx_top_inst/rx/training_q] 42 | # set_property MAX_FANOUT 35 [get_nets {bsp/dlx_phy/ocx_dlx_top_inst/tx/ctl/FSM_onehot_tsm_q_reg_n_0_[7]}] 43 | set_property MAX_FANOUT 26 [get_nets bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/crc_zero_d2_q] 44 | 45 | # 1024: Added more constraints to disable inference of enable/reset for some regs 46 | set_property EXTRACT_ENABLE false [get_cells {bsp/tlx/OCX_TLX_FRAMER/cmd_cntl_flit_reg_reg[*]}] 47 | set_property EXTRACT_ENABLE false [get_cells {bsp/tlx/OCX_TLX_FRAMER/vc0_fifo/valid_entry_counter_reg[*]}] 48 | set_property EXTRACT_ENABLE false [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_Parser/flit_parser/ctl_flit_dout_reg[*]}] 49 | set_property EXTRACT_RESET false [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/CMD_FIFO_MAC/CMD_INFO_CTL/ctl_cnt_dout_reg[*]}] 50 | set_property EXTRACT_RESET false [get_cells {bsp/dlx_phy/ocx_dlx_top_inst/tx/flt/pre_crc_data_q_reg[*]}] 51 | set_property EXTRACT_RESET false [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/RESP_FIFO_MAC/RESP_INFO_CTL/data_wr_cnt_dout_reg[*]}] 52 | set_property EXTRACT_RESET false [get_cells {bsp/tlx/OCX_TLX_PARSER/TLX_RCV_FIFO/CMD_FIFO_MAC/CMD_INFO_CTL/data_wr_cnt_dout_reg[*]}] 53 | 54 | 55 | 56 | set_property C_CLK_INPUT_FREQ_HZ 300000000 [get_debug_cores dbg_hub] 57 | set_property C_ENABLE_CLK_DIVIDER false [get_debug_cores dbg_hub] 58 | set_property C_USER_SCAN_CHAIN 1 [get_debug_cores dbg_hub] 59 | connect_debug_port dbg_hub/clk [get_nets clock_tlx] 60 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_timing.xdc: -------------------------------------------------------------------------------- 1 | ############################################################### 2 | ## FPGA: xcvu3p-ffvc1517-2-i 3 | ############################################################### 4 | 5 | ############################################################### 6 | ## Synthesis Constraints 7 | ############################################################### 8 | create_clock -period 6.400 -name mgtrefclk1_x0y0_p [get_ports mgtrefclk1_x0y0_p] 9 | create_clock -period 6.400 -name mgtrefclk1_x0y1_p [get_ports mgtrefclk1_x0y1_p] 10 | 11 | create_clock -period 3.333 -name freerun_clk_p [get_ports freerun_clk_p] 12 | 13 | # Timing analysis tools are unable to calculate fabric clock correctly because of the fractional-N in the 14 | # transceiver. Therefore, overconstrain the design to get the correct clock calculation. 15 | # create_clock -period 6.361 -name mgtrefclk1_x0y0_p [get_ports mgtrefclk1_x0y0_p] 16 | # create_clock -period 6.361 -name mgtrefclk1_x0y1_p [get_ports mgtrefclk1_x0y1_p] 17 | 18 | 19 | # False path constraints 20 | # ---------------------------------------------------------------------------------------------------------------------- 21 | set_false_path -to [get_cells -hierarchical -filter {NAME =~ *bit_synchronizer*inst/i_in_meta_reg}] 22 | set_false_path -to [get_cells -hierarchical -filter {NAME =~ *bit_synchronizer*inst/i_in_meta_reg}] 23 | set_false_path -to [get_cells -hierarchical -filter {NAME =~ *reset_synchronizer*inst/rst_in_*_reg}] 24 | set_false_path -to [get_cells -hierarchical -filter {NAME =~ *reset_synchronizer*inst/rst_in_*_reg}] 25 | set_false_path -to [get_cells -hierarchical -filter {NAME =~ *gtwiz_userclk_tx_inst/*gtwiz_userclk_tx_active_*_reg}] 26 | set_false_path -from [get_clocks mgtrefclk1_x0y0_p] -to [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] 27 | set_false_path -from [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst/O]] 28 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_timing_thymesisflow_cpu.xdc: -------------------------------------------------------------------------------- 1 | # FPGA-external clock definitions to help timing analysis 2 | # probably not needed because we configure the reference clock frequency in the aurora core 3 | # but adding anyway for completeness. 4 | 5 | ##QSFP reference clocks 309.25Mhz 6 | create_clock -period 3.232 -name QSFP0_REF_CLK_P [get_ports QSFP0_REF_CLK_P] 7 | 8 | 9 | #set false path between aurora reset module (clock domain clock_100) and actual reset pins on aurora module. 10 | set_false_path -from [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] 11 | set_false_path -from [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] -to [get_clocks -of_objects [get_pins {oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/aurora_qsfp0_gt_i/inst/gen_gtwizard_gtye4_top.aurora_qsfp0_gt_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[4].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[2].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] 12 | 13 | # false path constraints for example design paths on aurora 14 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.rx_active_stg2_reg/CLR] 15 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.gtwiz_userclk_rx_active_out_reg/CLR] 16 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP0_CORE/inst/aurora_qsfp0_core_i/aurora_qsfp0_wrapper_i/aurora_qsfp0_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.rx_active_aurora_qsfp0_cdc_to_reg/CLR] 17 | 18 | 19 | set_false_path -to [get_pins -hier *aurora_qsfp0_cdc_to*/D] 20 | 21 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/main_timing_thymesisflow_mem.xdc: -------------------------------------------------------------------------------- 1 | # FPGA-external clock definitions to help timing analysis 2 | # probably not needed because we configure the reference clock frequency in the aurora core 3 | # but adding anyway for completeness. 4 | 5 | ##QSFP reference clocks 309.25Mhz 6 | create_clock -period 3.232 -name QSFP1_REF_CLK_P [get_ports QSFP1_REF_CLK_P] 7 | 8 | 9 | #set false path between aurora reset module (clock domain clock_100) and actual reset pins on aurora module. 10 | set_false_path -from [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] 11 | 12 | set_false_path -from [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] 13 | 14 | 15 | # false path constraints for example design paths on aurora 16 | 17 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.rx_active_stg2_reg/CLR] 18 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.gtwiz_userclk_rx_active_out_reg/CLR] 19 | set_false_path -from [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/gtwiz_userclk_rx_reset_in_r_reg/C] -to [get_pins oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/ultrascale_rx_userclk/gen_gtwiz_userclk_rx_main.rx_active_aurora_qsfp1_cdc_to_reg/CLR] 20 | 21 | 22 | set_false_path -from [get_clocks -of_objects [get_pins {bsp/dlx_phy/example_wrapper_inst/DLx_phy_inst/inst/gen_gtwizard_gtye4_top.DLx_phy_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[1].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[0].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] -to [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] 23 | set_false_path -from [get_clocks -of_objects [get_pins bsp/dlx_phy/BUFGCE_DIV_inst2/O]] -to [get_clocks -of_objects [get_pins {oc_func/AFU00/TF_TOP/AURORA_QSFP1_CORE/inst/aurora_qsfp1_core_i/aurora_qsfp1_wrapper_i/aurora_qsfp1_multi_gt_i/aurora_qsfp1_gt_i/inst/gen_gtwizard_gtye4_top.aurora_qsfp1_gt_gtwizard_gtye4_inst/gen_gtwizard_gtye4.gen_channel_container[3].gen_enabled_channel.gtye4_channel_wrapper_inst/channel_inst/gtye4_channel_gen.gen_gtye4_channel_inst[2].GTYE4_CHANNEL_PRIM_INST/TXOUTCLK}]] 24 | 25 | set_false_path -to [get_pins -hier *aurora_qsfp1_cdc_to*/D] 26 | 27 | 28 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/qsfp_pinout.xdc: -------------------------------------------------------------------------------- 1 | #Aurora DataLink serDes 2 | 3 | set_property PACKAGE_PIN N34 [get_ports QSFP0_REF_CLK_N] 4 | set_property PACKAGE_PIN N33 [get_ports QSFP0_REF_CLK_P] 5 | 6 | set_property PACKAGE_PIN U34 [get_ports QSFP1_REF_CLK_N] 7 | set_property PACKAGE_PIN U33 [get_ports QSFP1_REF_CLK_P] 8 | 9 | 10 | 11 | #domain X0Y4 12 | set_property PACKAGE_PIN G38 [get_ports {QSFP0_RX_P[0]}] 13 | set_property PACKAGE_PIN G39 [get_ports {QSFP0_RX_N[0]}] 14 | set_property PACKAGE_PIN F35 [get_ports {QSFP0_TX_P[0]}] 15 | set_property PACKAGE_PIN F36 [get_ports {QSFP0_TX_N[0]}] 16 | set_property PACKAGE_PIN E38 [get_ports {QSFP0_RX_P[1]}] 17 | set_property PACKAGE_PIN E39 [get_ports {QSFP0_RX_N[1]}] 18 | set_property PACKAGE_PIN D35 [get_ports {QSFP0_TX_P[1]}] 19 | set_property PACKAGE_PIN D36 [get_ports {QSFP0_TX_N[1]}] 20 | set_property PACKAGE_PIN C38 [get_ports {QSFP0_RX_P[2]}] 21 | set_property PACKAGE_PIN C39 [get_ports {QSFP0_RX_N[2]}] 22 | set_property PACKAGE_PIN C33 [get_ports {QSFP0_TX_P[2]}] 23 | set_property PACKAGE_PIN C34 [get_ports {QSFP0_TX_N[2]}] 24 | set_property PACKAGE_PIN B36 [get_ports {QSFP0_RX_P[3]}] 25 | set_property PACKAGE_PIN B37 [get_ports {QSFP0_RX_N[3]}] 26 | set_property PACKAGE_PIN A33 [get_ports {QSFP0_TX_P[3]}] 27 | set_property PACKAGE_PIN A34 [get_ports {QSFP0_TX_N[3]}] 28 | 29 | #domain X0Y3 30 | set_property PACKAGE_PIN R38 [get_ports {QSFP1_RX_P[0]}] 31 | set_property PACKAGE_PIN R39 [get_ports {QSFP1_RX_N[0]}] 32 | set_property PACKAGE_PIN P35 [get_ports {QSFP1_TX_P[0]}] 33 | set_property PACKAGE_PIN P36 [get_ports {QSFP1_TX_N[0]}] 34 | set_property PACKAGE_PIN N38 [get_ports {QSFP1_RX_P[1]}] 35 | set_property PACKAGE_PIN N39 [get_ports {QSFP1_RX_N[1]}] 36 | set_property PACKAGE_PIN M35 [get_ports {QSFP1_TX_P[1]}] 37 | set_property PACKAGE_PIN M36 [get_ports {QSFP1_TX_N[1]}] 38 | set_property PACKAGE_PIN L38 [get_ports {QSFP1_RX_P[2]}] 39 | set_property PACKAGE_PIN L39 [get_ports {QSFP1_RX_N[2]}] 40 | set_property PACKAGE_PIN K35 [get_ports {QSFP1_TX_P[2]}] 41 | set_property PACKAGE_PIN K36 [get_ports {QSFP1_TX_N[2]}] 42 | set_property PACKAGE_PIN J38 [get_ports {QSFP1_RX_P[3]}] 43 | set_property PACKAGE_PIN J39 [get_ports {QSFP1_RX_N[3]}] 44 | set_property PACKAGE_PIN H35 [get_ports {QSFP1_TX_P[3]}] 45 | set_property PACKAGE_PIN H36 [get_ports {QSFP1_TX_N[3]}] 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/qspi_pinout.xdc: -------------------------------------------------------------------------------- 1 | # Following ports should be added as outputs in top level, and assigned to data outputs of AXI QSPI Core passed through IOBUF 2 | #set_property PACKAGE_PIN AB8 [get_ports FPGA_FLASH_DQ0] 3 | #set_property PACKAGE_PIN AD8 [get_ports FPGA_FLASH_DQ1] 4 | #set_property PACKAGE_PIN Y8 [get_ports FPGA_FLASH_DQ2] 5 | #set_property PACKAGE_PIN AC8 [get_ports FPGA_FLASH_DQ3] 6 | set_property PACKAGE_PIN AF30 [get_ports FPGA_FLASH_DQ4] 7 | set_property PACKAGE_PIN AG30 [get_ports FPGA_FLASH_DQ5] 8 | set_property PACKAGE_PIN AF28 [get_ports FPGA_FLASH_DQ6] 9 | set_property PACKAGE_PIN AG28 [get_ports FPGA_FLASH_DQ7] 10 | 11 | # Slave select output of AXI QSPI Core 12 | set_property PACKAGE_PIN AV30 [get_ports FPGA_FLASH_CE2_L] 13 | 14 | #set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ0] 15 | #set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ1] 16 | #set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ2] 17 | #set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ3] 18 | set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ4] 19 | set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ5] 20 | set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ6] 21 | set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_DQ7] 22 | set_property IOSTANDARD LVCMOS18 [get_ports FPGA_FLASH_CE2_L] 23 | 24 | -------------------------------------------------------------------------------- /board_support_packages/ad9v3/xdc/qspi_timing.xdc: -------------------------------------------------------------------------------- 1 | # All the delay numbers have to be provided by the user 2 | # We need to consider the max delay for worst case analysis 3 | set cclk_delay 6.7 4 | # Following are the SPI device parameters, for MT25QL512ABB8E12-0SIT 5 | # Max Tco 6 | set tco_max 6 7 | # Min Tco 8 | set tco_min 1.5 9 | # Setup time requirement 10 | set tsu 1.75 11 | # Hold time requirement 12 | set th 2.3 13 | # Following are the board/trace delay numbers 14 | # Assumption is that all Data lines are matched 15 | set tdata_trace_delay_max 0.25 16 | set tdata_trace_delay_min 0.25 17 | set tclk_trace_delay_max 0.2 18 | set tclk_trace_delay_min 0.2 19 | ### End of user provided delay numbers 20 | # This is to ensure min routing delay from SCK generation to STARTUP input 21 | # User should change this value based on the results 22 | # Having more delay on this net reduces the Fmax 23 | # Following constraint should be commented when STARTUP block is disabled 24 | 25 | # set_max_delay 1.5 -from [get_pins -hier *SCK_O_reg_reg/C] -to [get_pins -hier *USRCCLKO] -datapath_only 26 | # set_min_delay 0.1 -from [get_pins -hier *SCK_O_reg_reg/C] -to [get_pins -hier *USRCCLKO] 27 | # set_max_delay 1.5 -from [get_clocks tx_clk_201MHz] -to [get_pins -hier *USRCCLKO] -datapath_only 28 | # set_min_delay 0.1 -from [get_clocks tx_clk_201MHz] -to [get_pins -hier *USRCCLKO] 29 | set_max_delay 1.5 -from [get_clocks clock_afu] -to [get_pins -hier *USRCCLKO] -datapath_only 30 | set_min_delay 0.1 -from [get_clocks clock_afu] -to [get_pins -hier *USRCCLKO] 31 | 32 | # Following command creates a divide by 2 clock 33 | # It also takes into account the delay added by STARTUP block to route the CCLK 34 | # This constraint is not needed when STARTUP block is disabled 35 | # Following constraint should be commented when STARTUP block is disabled 36 | 37 | # create_generated_clock -name clk_sck -source [get_pins -hierarchical *axi_quad_spi_0/ext_spi_clk] [get_pins -hierarchical *USRCCLKO] -edges {3 5 7} -edge_shift [list $cclk_delay $cclk_delay $cclk_delay] 38 | create_generated_clock -name clk_sck -source [get_pins bsp/FLASH/QSPI/ext_spi_clk] [get_pins -hierarchical *USRCCLKO] -edges {3 5 7} -edge_shift [list $cclk_delay $cclk_delay $cclk_delay] 39 | 40 | # Enable following constraint when STARTUP block is disabled 41 | #create_generated_clock -name clk_virt -source [get_pins -hierarchical 42 | #*axi_quad_spi_0/ext_spi_clk] [get_ports ] -edges {3 5 7} 43 | # Data is captured into FPGA on the second rising edge of ext_spi_clk after the SCK falling edge 44 | # Data is driven by the FPGA on every alternate rising_edge of ext_spi_clk 45 | 46 | set_input_delay -clock clk_sck -max [expr $tco_max + $tdata_trace_delay_max + $tclk_trace_delay_max] [get_ports *FPGA_FLASH*] -clock_fall; 47 | set_input_delay -clock clk_sck -min [expr $tco_min + $tdata_trace_delay_min + $tclk_trace_delay_min] [get_ports *FPGA_FLASH*] -clock_fall; 48 | set_multicycle_path 2 -setup -from clk_sck -to [get_clocks -of_objects [get_pins -hierarchical */ext_spi_clk]] 49 | set_multicycle_path 1 -hold -end -from clk_sck -to [get_clocks -of_objects [get_pins -hierarchical */ext_spi_clk]] 50 | 51 | # Data is captured into SPI on the following rising edge of SCK 52 | # Data is driven by the IP on alternate rising_edge of the ext_spi_clk 53 | 54 | set_output_delay -clock clk_sck -max [expr $tsu + $tdata_trace_delay_max - $tclk_trace_delay_min] [get_ports *FPGA_FLASH*]; 55 | set_output_delay -clock clk_sck -min [expr $tdata_trace_delay_min -$th - $tclk_trace_delay_max] [get_ports *FPGA_FLASH*]; 56 | set_multicycle_path 2 -setup -start -from [get_clocks -of_objects [get_pins -hierarchical */ext_spi_clk]] -to clk_sck 57 | set_multicycle_path 1 -hold -from [get_clocks -of_objects [get_pins -hierarchical */ext_spi_clk]] -to clk_sck 58 | -------------------------------------------------------------------------------- /dlx/ocx_bram_infer.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | //------------------------------------------------------------------------ 22 | //-- 23 | //-- TITLE: ocx_bram_infer.v 24 | //-- FUNCTION: 128x512 Xilinx BRAM 25 | //-- 26 | //------------------------------------------------------------------------ 27 | 28 | 29 | module ocx_bram_infer( 30 | input clka, 31 | input clkb, 32 | input ena, 33 | input enb, 34 | input wea, 35 | input rstb, 36 | //input regceb, 37 | input [6:0] addra, 38 | input [6:0] addrb, 39 | input [512-1:0] dina, 40 | output [512-1:0] doutb, 41 | output sbiterr, 42 | output dbiterr, 43 | output wire [6:0] rdaddrecc 44 | ); 45 | 46 | (* ram_style="block" *) 47 | // wizard generated memory has 2 clocks, use clka for both ports 48 | wire clk; 49 | assign clk = clka; 50 | //wire rstb; 51 | wire regceb; 52 | //assign rstb = 1'b0; 53 | assign regceb = 1'b1; 54 | assign sbiterr = 1'b0; 55 | assign dbiterr = 1'b0; 56 | assign rdaddrecc = {7{1'b0}}; 57 | 58 | // Xilinx Simple Dual Port Single Clock RAM 59 | // This code implements a parameterizable SDP single clock memory. 60 | // If a reset or enable is not necessary, it may be tied off or removed from the code. 61 | 62 | parameter RAM_WIDTH = 512; // Specify RAM data width 63 | parameter RAM_DEPTH = 128; // Specify RAM depth (number of entries) 64 | parameter RAM_PERFORMANCE = "HIGH_PERFORMANCE"; // Select "HIGH_PERFORMANCE" or "LOW_LATENCY" 65 | parameter INIT_FILE = ""; // Specify name/location of RAM initialization file if using one (leave blank if not) 66 | 67 | reg [RAM_WIDTH-1:0] ram_sdp [RAM_DEPTH-1:0]; 68 | reg [RAM_WIDTH-1:0] ram_sdp_data = {RAM_WIDTH{1'b0}}; 69 | 70 | // The following code either initializes the memory values to a specified file or to all zeros to match hardware 71 | generate 72 | if (INIT_FILE != "") begin: use_init_file 73 | initial 74 | $readmemh(INIT_FILE, ram_sdp, 0, RAM_DEPTH-1); 75 | end else begin: init_bram_to_zero 76 | integer ram_index; 77 | initial 78 | for (ram_index = 0; ram_index < RAM_DEPTH; ram_index = ram_index + 1) 79 | ram_sdp[ram_index] = {RAM_WIDTH{1'b0}}; 80 | end 81 | endgenerate 82 | 83 | always @(posedge clk) begin 84 | if (wea) 85 | ram_sdp[addra] <= dina; 86 | if (enb) 87 | ram_sdp_data <= ram_sdp[addrb]; 88 | end 89 | 90 | // The following code generates HIGH_PERFORMANCE (use output register) or LOW_LATENCY (no output register) 91 | generate 92 | if (RAM_PERFORMANCE == "LOW______LATENCY") begin: no_output_register 93 | 94 | // The following is a 1 clock cycle read latency at the cost of a longer clock-to-out timing 95 | assign doutb = ram_sdp_data; 96 | 97 | end else begin: output_register 98 | 99 | // The following is a 2 clock cycle read latency with improve clock-to-out timing 100 | 101 | reg [RAM_WIDTH-1:0] doutb_reg = {RAM_WIDTH{1'b0}}; 102 | 103 | always @(posedge clk) 104 | if (rstb) 105 | doutb_reg <= {RAM_WIDTH{1'b0}}; 106 | else if (regceb) 107 | doutb_reg <= ram_sdp_data; 108 | 109 | assign doutb = doutb_reg; 110 | 111 | end 112 | endgenerate 113 | 114 | // The following function calculates the address width based on specified RAM depth 115 | function integer clogb2; 116 | input integer depth; 117 | for (clogb2=0; depth>0; clogb2=clogb2+1) 118 | depth = depth >> 1; 119 | endfunction 120 | 121 | 122 | endmodule 123 | -------------------------------------------------------------------------------- /libtfshmem/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-fPIC -Wall 3 | ODIR=objs 4 | SRC_DIR=src 5 | LIB_DIR=lib 6 | BIN_DIR=bin 7 | OUT_DIRS= $(ODIR) $(LIB_DIR) $(BIN_DIR) 8 | INSTALL_LIBDIR=/usr/lib64 9 | INSTALL_INCDIR=/usr/include 10 | 11 | 12 | TF_FOLDER=../libthymesisflow 13 | DEBUG=1 14 | 15 | ifeq ($(DEBUG),1) 16 | CFLAGS+=-D DEBUG 17 | endif 18 | 19 | CFLAGS+=-D MOCK -I./include -I$(TF_FOLDER)/include 20 | 21 | CORE_SRCS := $(wildcard src/*.c) 22 | CORE_OBJ:= $(addprefix $(ODIR)/,$(notdir $(CORE_SRCS:.c=.o))) 23 | TEST_SRCS := $(wildcard test/*.c) 24 | TEST_BIN:= $(addprefix $(BIN_DIR)/,$(notdir $(TEST_SRCS:.c=))) 25 | 26 | VPATH= src test 27 | 28 | all: make_dirs libtfshmem tests 29 | 30 | make_dirs: 31 | mkdir -p $(OUT_DIRS) 32 | 33 | $(ODIR)/%.o: src/%.c 34 | $(info making $<) 35 | $(CC) $(CFLAGS) -c -o $@ $< 36 | 37 | libtfshmem: $(CORE_OBJ) 38 | $(CC) -shared -o lib/$@.so $(CORE_OBJ) $(TF_FOLDER)/lib/libthymesisflow.a 39 | 40 | $(BIN_DIR)/%: %.c 41 | $(CC) $(CFLAGS) -o $@ $< -L$(LIB_DIR) -ltfshmem -locxl -lpthread 42 | 43 | tests: $(TEST_BIN) 44 | 45 | clean: 46 | rm -fr cscope.* 47 | rm -rf $(OUT_DIRS) 48 | 49 | install: 50 | sudo install include/tf_shmem_api.h $(INSTALL_INCDIR) 51 | sudo install lib/libtfshmem.so $(INSTALL_LIBDIR) 52 | 53 | uninstall: 54 | sudo rm $(INSTALL_INCDIR)/tf_shmem_api.h $(INSTALL_LIBDIR)/libtfshmem.so 55 | 56 | 57 | cscope: 58 | -find . -name "*.[ch]" > cscope.files 59 | -cscope -b -R 60 | -------------------------------------------------------------------------------- /libtfshmem/README.md: -------------------------------------------------------------------------------- 1 | ## LibTFShmem: a library enabling shared memory on Thymesisflow 2 | 3 | This library enables creating a shared memory segment across processes running on two 4 | machines interfaced via ThymesisFlow. The idea is that the two processes will be able 5 | to map a buffer in memory and access it concurrently. 6 | ALthough this is possible, there are a few caveats to take into consideration: 7 | - memory accesses across the two machines are neither coherent nor consistent by nature. 8 | It will be up-to the programmer to make sure memory is properly handled. 9 | - (unexpectedly) The data-cache is your worst enemy here. The two sides sharing memory 10 | might as well keep accessing a local copy of the shared data in their data cache as 11 | no cache-coherence protocol is in place across the two machines. It will be again up-to 12 | the programmer to make sure data is casted out from one processor cache before the 13 | other side can be aware of it. There are sample applications showing how to deal with 14 | the cache, have a look [here](libtfshmem/test). 15 | - Do not try make the mapping un-cacheable by hacking the mmap in [here](libtfshmem/kernel_module), 16 | as this would make the side borrowing memory (a.k.a. the compute) fail miserably at 17 | the first memory access. ThymesisFlow is only supporting 128B memory transactions and 18 | and disabling the data cache does not help enforcing this. 19 | 20 | Having said that, have fun experimentign with shared memory on thymesisflow. 21 | 22 | ### Overview 23 | This library is composed of two main parts: a user-space library (libtfshmem) and a kernel module (tfshmem). As you might already 24 | know at this stage, ThymesisFlow is divided in the compute (borrower) and memory node (lender). The user-space 25 | library is needed on both sides, while the kernel module is needed only on the compute node. The reason being the memory node 26 | accesses the shared memory locally, while the compute node needs something to map the shared memory into processes VA space. 27 | When using this library there's no need for you to start the thymesisflow-agent or to use the CLI, the only thing you need to do 28 | yourself is to make sure the memory node is initialized first, send the EA (Effective Address) generated over to the application runnning on the compute 29 | node and execute the initialization in there. 30 | Have a look at the examples [here](libtfshmem/test). 31 | 32 | ### How-to compile the library 33 | First make sure you have compiled `libthymesisflow`. Then just run `make` in this folder. 34 | There's a `DEBUG` flag in the Makefile that is set to 1 by default. Set it to 0 if you want 35 | a less verbose but **less informative** run. 36 | Have a look at the makefile for more details. 37 | 38 | ### How-to compile the kernel module 39 | Building the kernel module is as easy as running `make`. The Makefile will default the `KDIR` to the location where you running kernel build files are located. 40 | If you want to build for another kernel you can run: 41 | ``` 42 | make KDIR=/path/to/configured/kernel 43 | ``` 44 | 45 | the kernel odule can be manually probed with: 46 | ``` 47 | insmod tfshmem.ko 48 | ``` 49 | or automatically at machine boot by typing: 50 | ``` 51 | make install 52 | ``` 53 | 54 | in both cases a sign of success is if you can find the below device: 55 | ``` 56 | /dev/tfshmem 57 | ``` 58 | 59 | if not `dmesg` is your best friend. 60 | 61 | Please note ***installing this module will taint your kernel*** and it is strongly discouraged on production machines (i.e., machines running services people rely on). 62 | The module itself, before you start using it, is relatively harmless. However, your machine could crash when you start operating it. 63 | 64 | ***Install this module at your own risk***. 65 | 66 | ### Test applications 67 | There are test applications for you to get a taste on how to use it. The documentation is the code :D. 68 | 69 | Tips: 70 | - make sure you initialize the memory node first and the compute after. 71 | - The EA generated by the memroy node at initialization is needed for initializing the compute node. 72 | -------------------------------------------------------------------------------- /libtfshmem/include/custom_mm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | * Panagiotis Koutsovasilis - koutsovasilis.panagiotis1@ibm.com 18 | */ 19 | 20 | #ifndef CUSTOM_MM_H 21 | #define CUSTOM_MM_H 22 | 23 | #define ALIGN(x,a) __ALIGN_MASK(x,(typeof(x))(a)-1) 24 | #define __ALIGN_MASK(x,mask) (((x)+(mask))&~(mask)) 25 | 26 | #define BLOCK_T_ALIGNED (ALIGN(sizeof(block_t), 8)) // block_t aligned 8 bytes 27 | 28 | #define PAGE_SIZE 64*1024 29 | 30 | typedef struct _block_t 31 | { 32 | unsigned long len; 33 | struct _block_t *next; 34 | struct _block_t *prev; 35 | } block_t; 36 | 37 | 38 | void init_custom_mm(void *base_addr, uint64_t size); 39 | void fini_custom_mm(); 40 | void * custom_malloc(size_t size); 41 | void * custom_malloc_aligned(size_t alignment, size_t size); 42 | void custom_free(void *ptr); 43 | 44 | #endif 45 | 46 | -------------------------------------------------------------------------------- /libtfshmem/include/tf_shmem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #ifndef TF_SHMEM_H 20 | #define TF_SHMEM_H 21 | 22 | #define COMPUTE_PORT 0x2 23 | #define MEMORY_PORT 0x1 24 | #define AFU_NAME "IBM,RMEM" 25 | /* 26 | * This is the physical address whete the ThymesisFlow OpenCAPI accelerator 27 | * is mapped and represents the base physical address for the attached 28 | * disaggregated memory pool. 29 | */ 30 | #define OCAPI_PHYS_BASE 0x2000000000000 31 | 32 | struct tf_shmem_state { 33 | uint64_t ea; 34 | uint64_t pool_size; 35 | int memory_node; 36 | }; 37 | 38 | #endif 39 | 40 | -------------------------------------------------------------------------------- /libtfshmem/include/tf_shmem_api.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #ifndef TF_SHMEM_API_H 20 | #define TF_SHMEM_API_H 21 | 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | /** 29 | * Initialize the memory inception interface for the compute node side. 30 | * This requires the effective address (EA) one the memory side to be provided as input. 31 | * The need for the EA is not a requirement of Memory Inception but rather linked to how the 32 | * ThymesysFlow prototype works. 33 | * arguments: 34 | * ea: effective address at which the main pool is mapped on the memory side. 35 | * pool_size: size of the mail remote memory pool in bytes. 36 | * 37 | * return value: 38 | * This function returns 0 in case of success or -1 in case of error and -2 if the library 39 | * is already initialized. 40 | */ 41 | int tf_compute_init(void *ea, uint64_t pool_size); 42 | 43 | /** 44 | * Initialize the memory inception interface for the memory node side. 45 | * This method allocates a memory pool of the required size and makes it available for 46 | * remote access. 47 | * arguments: 48 | * pool_size: size of the mail remote memory pool in bytes. 49 | * 50 | * return value: 51 | * This function returns 0 in case of success or -1 in case of error and -2 if the library 52 | * is already initialized. 53 | */ 54 | int tf_memory_init(uint64_t pool_size); 55 | 56 | /** 57 | * Tear down the tf_shmem library. The function takes care or resetting the state and FPGAs. 58 | * 59 | * return value: 60 | * 0 in case of success -1 otherwise. 61 | */ 62 | int tf_shmem_teardown(void); 63 | 64 | /** 65 | * Carve out a buffer of the requested size from the main memory pool. This function can only be invoked 66 | * on the memory node. 67 | * 68 | * arguments: 69 | * buffer_size: the size in bytes of the buffer to be allocated 70 | * 71 | * return value: 72 | * this function return NULL in case of error or a valid pointer otherwise. 73 | */ 74 | void* tf_buffer_alloc(uint64_t buffer_size); 75 | 76 | /** 77 | * Map a buffer from the remote memory pool. This function is to be invoked on the compute node only. 78 | * arguments: 79 | * ea: the EA of buffer base on the memory node side. 80 | * buffer_size: the size in bytes of the buffer to be mapped. 81 | * 82 | * return value: 83 | * this function return NULL in case of error or a valid pointer otherwise. 84 | */ 85 | void* tf_buffer_map(void *ea, uint64_t buffer_size); 86 | 87 | /** 88 | * Returns the effective address of the main shared memory pool 89 | */ 90 | void * tf_get_pool_ea(void); 91 | 92 | /** 93 | * Returns true if the library is initialized. 94 | * Can be invoked from both memory and compute node. 95 | */ 96 | bool tf_initialized(void); 97 | 98 | /** 99 | * Frees a buffer allocated from the pool. This should only be invoked by a memory node 100 | */ 101 | void tf_buffer_free(void *buf); 102 | 103 | /** 104 | * Unmaps a buffer from a compute note memory map. SHould only be invoked by a compute node. 105 | * 106 | * return value: 107 | * 0 in case of success, -1 otherwise. 108 | */ 109 | int tf_buffer_unmap(void *buf, size_t length); 110 | 111 | #ifdef __cplusplus 112 | } 113 | #endif 114 | 115 | #endif //#ifndef TF_SHMEM_H 116 | -------------------------------------------------------------------------------- /libtfshmem/include/utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #ifndef UTILS_H 20 | #define UTILS_H 21 | 22 | #ifdef DEBUG 23 | #define DBG_PRINT(fmt,args...) \ 24 | do {printf("\n%s - "fmt, __func__, ##args); fflush(stdout);} while (0) 25 | #else 26 | #define DBG_PRINT(fmt,...) ((void) (0)) 27 | #endif 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/Kbuild: -------------------------------------------------------------------------------- 1 | obj-m := tfshmem.o 2 | tfshmem-y := tfshmem_mod.o tfshmem_ops.o 3 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/Makefile: -------------------------------------------------------------------------------- 1 | #Change this path if your kernel sources are somewhere else 2 | KDIR ?= /lib/modules/`uname -r`/build 3 | 4 | default: 5 | $(MAKE) -C $(KDIR) M=$$PWD 6 | 7 | clean: 8 | rm -f built-in.a *.o *.ko *.cmd *.mod.c *.order *.symvers.cmd *.ko.cmd *.symvers .[!.]* 9 | 10 | install: 11 | sudo cp tfshmem.ko /lib/modules/`uname -r`/kernel/ 12 | sudo depmod 13 | sudo cp tfshmem.conf /etc/modules-load.d/ 14 | 15 | uninstall: 16 | sudo rm -rf /lib/modules/`uname -r`/kernel/tfshmem.ko 17 | sudo depmod 18 | sudo rm -rf /etc/modules-load.d/tfshmem.ko 19 | 20 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/tfshmem.conf: -------------------------------------------------------------------------------- 1 | tfshmem 2 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/tfshmem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | #ifndef _TFSHMEM_H_ 19 | #define _TFSHMEM_H_ 20 | 21 | #define OCAPI_PHY_BASE 0x2000000000000 22 | 23 | int tfshmem_open(struct inode *inode, struct file *filp); 24 | int tfshmem_release(struct inode *inode, struct file *filp); 25 | int tfshmem_mmap (struct file *filp, struct vm_area_struct *vma); 26 | 27 | #endif 28 | 29 | 30 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/tfshmem_mod.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include "tfshmem.h" 26 | 27 | #define DRIVER_NAME "tf_shmem" 28 | #define TFSHMEM_DEVICE_NAME "tfshmem" 29 | #define TFSHMEM_DEV_COUNT 1 30 | 31 | static int major; 32 | static dev_t first = 0; 33 | static struct class *cl = NULL; 34 | static struct cdev mm_dev; 35 | static struct device *dev; 36 | 37 | struct file_operations mm_fops = { 38 | .open = tfshmem_open, 39 | .owner = THIS_MODULE, 40 | .release = tfshmem_release, 41 | .mmap = tfshmem_mmap, 42 | }; 43 | 44 | /* 45 | * I want to use the same module for client (compute node) and server (memory node). 46 | * Users will set this param while probing the module and I'll use it just to make sure 47 | * nothing illegal is executed. 48 | * 49 | */ 50 | int client_mode = 0; 51 | module_param(client_mode, int, 0); 52 | 53 | static char *remap_devnode(struct device *dev, umode_t *mode) { 54 | if (!mode) 55 | return NULL; 56 | *mode = 0666; 57 | return NULL; 58 | } 59 | 60 | static int __init tfshmem_driver_register(void) { 61 | major = alloc_chrdev_region(&first, 0, TFSHMEM_DEV_COUNT, DRIVER_NAME); 62 | 63 | if(major < 0) { 64 | pr_err("Cannot allocate major and minor numbers\n" ); 65 | return -1; 66 | } 67 | 68 | if ( (cl = class_create( THIS_MODULE, "tf" ) ) == NULL ) { 69 | pr_err("Class creation failed\n" ); 70 | unregister_chrdev_region(first, TFSHMEM_DEV_COUNT); 71 | return -1; 72 | } 73 | 74 | cl->devnode = remap_devnode; 75 | 76 | return 0; 77 | } 78 | 79 | static int __init tfshmem_device_register(char *device_name, struct file_operations *fops, 80 | 81 | dev_t first, struct cdev *c_dev) { 82 | if ((dev = device_create( cl, NULL, first, NULL, device_name)) == NULL ) { 83 | pr_err("%s device creation failed\n", device_name); 84 | return -1; 85 | } 86 | 87 | cdev_init(c_dev, fops); 88 | 89 | if (cdev_add(c_dev, first, 1) == -1) { 90 | pr_err("%s device addition failed\n", device_name); 91 | device_destroy(cl, first); 92 | return -1; 93 | } 94 | 95 | pr_info("\"%s\" character device successfully created\n", device_name); 96 | 97 | return 0; 98 | } 99 | 100 | static void tfshmem_driver_unregister(void) { 101 | 102 | if (mm_dev.dev != 0) { 103 | device_destroy(cl, first); 104 | } 105 | 106 | if (cl != NULL) { 107 | class_destroy(cl); 108 | } 109 | 110 | if (first != 0) { 111 | unregister_chrdev_region(first, TFSHMEM_DEV_COUNT); 112 | } 113 | } 114 | 115 | int __init tf_shmem_init(void) { 116 | 117 | int ret; 118 | pr_info("Starting TFSHMEM device driver initialization"); 119 | if ((ret = tfshmem_driver_register())) { 120 | goto except_fail; 121 | } 122 | 123 | if ((ret = tfshmem_device_register(TFSHMEM_DEVICE_NAME, &mm_fops, first, &mm_dev))) { 124 | goto except_fail; 125 | } 126 | 127 | except_fail: 128 | if (ret < 0) { 129 | tfshmem_driver_unregister(); 130 | } 131 | 132 | return ret; 133 | } 134 | 135 | void __exit module_teardown(void) { 136 | tfshmem_driver_unregister(); 137 | } 138 | 139 | 140 | module_init(tf_shmem_init); 141 | module_exit(module_teardown); 142 | MODULE_LICENSE("GPL"); 143 | -------------------------------------------------------------------------------- /libtfshmem/kernel_module/tfshmem_ops.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include "tfshmem.h" 23 | 24 | int tfshmem_open(struct inode *inode, struct file *filp) { 25 | 26 | return 0; 27 | } 28 | 29 | int tfshmem_release(struct inode *inode, struct file *filp) { 30 | 31 | return 0; 32 | } 33 | 34 | 35 | int tfshmem_mmap (struct file *filp, struct vm_area_struct *vma) { 36 | 37 | int ret; 38 | unsigned long size, addr, pfn; 39 | 40 | size = vma->vm_end - vma->vm_start; 41 | pfn = vma->vm_pgoff + (OCAPI_PHY_BASE >> PAGE_SHIFT); 42 | addr = vma->vm_start; 43 | 44 | pr_debug("size: %.16lx, pfn: %.16lx, addr, %.16lx", size, pfn, addr); 45 | 46 | /* 47 | * This is mapping the contiguous physical addres space window where the shared buffer 48 | * is located into the virtual address space of the calling process. Mapping is 1-to-1 as 49 | * as it is both virtually and physically contiguous. 50 | */ 51 | if ((ret = remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot)) < 0) { 52 | printk("remap_pfn_range failed\n"); 53 | return ret; 54 | } 55 | 56 | pr_debug("Mapping to user space successful\n"); 57 | 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /libtfshmem/src/custom_mm.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | * Panagiotis Koutsovasilis - koutsovasilis.panagiotis1@ibm.com 18 | */ 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include "custom_mm.h" 28 | #include "utils.h" 29 | 30 | static void *base_address = NULL; 31 | static void *next = NULL; 32 | static long unsigned PREALLOC_SIZE = 0; 33 | static bool initialized = false; 34 | 35 | void custom_free(void *ptr) 36 | { 37 | if (ptr == NULL) 38 | { 39 | return; 40 | } 41 | ptr=NULL; 42 | } 43 | 44 | void *custom_malloc(size_t size) 45 | { 46 | if (size == 0 || !initialized){ 47 | return NULL; 48 | } 49 | 50 | void *ret; 51 | size = ALIGN(size, PAGE_SIZE); 52 | 53 | if (next == NULL){ 54 | ret = base_address; 55 | next = base_address + size; 56 | } 57 | else { 58 | ret = next; 59 | next = (char*)next + size; 60 | } 61 | 62 | DBG_PRINT("size: %.16lX, next %p, ret %p", size, next, ret); 63 | return ret; 64 | 65 | } 66 | 67 | void * custom_malloc_aligned(size_t alignment, size_t size) { 68 | void * memptr; 69 | if (!initialized) { 70 | return NULL; 71 | } 72 | size = ALIGN(size, alignment); 73 | memptr = custom_malloc(size); 74 | 75 | if (memptr == NULL) { 76 | DBG_PRINT("custom malloc failed"); 77 | } 78 | 79 | return memptr; 80 | } 81 | 82 | void init_custom_mm(void *base_addr, uint64_t size) { 83 | base_address = base_addr; 84 | PREALLOC_SIZE = size; 85 | initialized = true; 86 | } 87 | 88 | void fini_custom_mm() { 89 | base_address = NULL; 90 | PREALLOC_SIZE = 0; 91 | initialized = false; 92 | } 93 | -------------------------------------------------------------------------------- /libtfshmem/test/test_compute.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #include 20 | #include 21 | #include "tf_shmem_api.h" 22 | 23 | /* POWER9 cacheline size 24 | * TODO: make this more robust*/ 25 | #define CACHELINE_SIZE 0x80 26 | #define CACHELINE_MASK (CACHELINE_SIZE - 1) 27 | 28 | 29 | #define FLUSH_LINE(addr) \ 30 | asm volatile ( \ 31 | "dcbf 0x0,%0\n" \ 32 | : \ 33 | : "b"(addr) \ 34 | : "memory" \ 35 | ) 36 | 37 | 38 | #define FLUSH_LINE_SYNC(addr) \ 39 | asm volatile ( \ 40 | "dcbf 0x0,%0\n" \ 41 | "sync\n" \ 42 | : \ 43 | : "b"(addr) \ 44 | : "memory" \ 45 | ) 46 | 47 | 48 | static inline void flush_memory_block (void *addr, uint64_t size, int sync) { 49 | uint64_t start_addr = (uint64_t) addr; 50 | uint64_t end_addr = (start_addr + size); 51 | unsigned int count = 0; 52 | printf("\nstarting flusshing buffer @ address: %" PRIx64 " of size %" PRIu64, start_addr, size); 53 | 54 | if ((start_addr & CACHELINE_MASK) != 0) { 55 | if (sync) 56 | FLUSH_LINE_SYNC(start_addr); 57 | else { 58 | // printf("\n#%u flushing line @ address: %" PRIx64, count, start_addr); 59 | fflush(stdout); 60 | FLUSH_LINE(start_addr); 61 | } 62 | start_addr += CACHELINE_SIZE - (start_addr & CACHELINE_MASK); 63 | } 64 | 65 | for ( ; start_addr < end_addr; start_addr += CACHELINE_SIZE, count++){ 66 | if (sync) 67 | FLUSH_LINE_SYNC(start_addr); 68 | else { 69 | // printf("\n#%u flushing line @ address: %" PRIx64, count, start_addr); 70 | fflush(stdout); 71 | FLUSH_LINE(start_addr); 72 | } 73 | } 74 | } 75 | 76 | int main() { 77 | 78 | uint64_t pool_size = 100ULL*1024*1024*1024; 79 | uint64_t pool_ea; 80 | uint64_t buf_size = 100ULL*1024*1024; 81 | uint64_t buf_ea; 82 | void *buf_ptr; 83 | int result; 84 | int i; 85 | unsigned long long int sum = 0; 86 | 87 | printf("\nClient test waiting for base ea of OCAPI pool to continue with initialization: "); 88 | result = scanf("%" PRIx64, &pool_ea); 89 | 90 | printf("\nRead: %" PRIx64 ", continuing initialization\n", pool_ea); 91 | 92 | /* 93 | * Initialize the TF shared memory library by providing the ea of the OCAPI memory pool 94 | * and the size of the pool itself. The ea is not valid here (compute node) as it is 95 | * an address generated at the memory node side. However this is needed to configure the 96 | * ThymesisFlow design. 97 | * The ea is usually provided, somehow, by the memory node. 98 | * */ 99 | if (tf_compute_init((void *)pool_ea, pool_size) == -1){ 100 | printf ("Error initializing shared memory pool"); 101 | return -1; 102 | } 103 | printf("\nOCAPI shared memory initialized correctly"); 104 | 105 | printf("\nClient test waiting for buffer base ea to continue with testing mapping it: "); 106 | result = scanf("%" PRIx64, &buf_ea); 107 | printf("\nRead: %" PRIx64 ", continuing initialization", buf_ea); 108 | 109 | /* 110 | * Here we map a first 100MB buffer allocated at the memory node side. Again the ea is not valid here 111 | * and it is used only to compute the offset from the main pool start. 112 | * */ 113 | buf_ptr = tf_buffer_map((void *)buf_ea, buf_size); 114 | if (buf_ptr == NULL){ 115 | printf ("Error mapping buffer"); 116 | return -1; 117 | } 118 | 119 | printf("\n %" PRIu64 "bytes mapped in a buffer at ea %p", buf_size, buf_ptr); 120 | for (i=0; i < buf_size/(sizeof (unsigned int)); i++) { 121 | ((unsigned int *)buf_ptr)[i] = i; 122 | sum += i; 123 | } 124 | printf("\nBuffer 1 sum is %llu", sum); 125 | 126 | flush_memory_block(buf_ptr, buf_size, 0); 127 | 128 | tf_buffer_unmap(buf_ptr, buf_size); 129 | 130 | 131 | printf("\nClient test waiting for second buffer base ea to continue with testing mapping it: "); 132 | result = scanf("%" PRIx64, &buf_ea); 133 | printf("\nRead: %" PRIx64 ", continuing initialization\n", buf_ea); 134 | 135 | /* 136 | * Here we map a second 100MB buffer allocated at the memory node side. Again the ea is not valid here 137 | * and it is used only to compute the offset from the main pool start. 138 | * */ 139 | buf_ptr = tf_buffer_map((void *)buf_ea, 2*buf_size); 140 | if (buf_ptr == NULL){ 141 | printf ("Error mapping buffer"); 142 | return -1; 143 | } 144 | 145 | sum =0; 146 | printf("\n %" PRIu64 "mapped in a buffer at ea %p\n", 2*buf_size, buf_ptr); 147 | for (i=0; i < (2*buf_size)/(sizeof (unsigned int)); i++) { 148 | ((unsigned int *)buf_ptr)[i] = (2*buf_size)/(sizeof (unsigned int)) - i; 149 | sum += (2*buf_size)/(sizeof (unsigned int)) - i; 150 | } 151 | printf("\nBuffer 2 sum is %llu", sum); 152 | 153 | flush_memory_block(buf_ptr, buf_size, 1); 154 | 155 | tf_buffer_unmap(buf_ptr, 2 * buf_size); 156 | 157 | if (tf_shmem_teardown() == -1) { 158 | printf("Failed tearing down tfshmem library"); 159 | } 160 | 161 | } 162 | -------------------------------------------------------------------------------- /libtfshmem/test/test_memory.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Author: Christian Pinto - christian.pinto@ibm.com 17 | */ 18 | 19 | #include 20 | #include 21 | #include "tf_shmem_api.h" 22 | 23 | int main() { 24 | 25 | uint64_t size = 100ULL*1024*1024*1024; 26 | uint64_t buf_size = 100ULL*1024*1024; 27 | void* ea; 28 | void* buf_ptr; 29 | void* buf2_ptr; 30 | char ch; 31 | int i; 32 | unsigned long long int sum = 0; 33 | unsigned long long int val = 0; 34 | unsigned long long int zeroes = 0; 35 | 36 | printf("\nTesting initializing OCAPI shared memory pool\n"); 37 | 38 | 39 | /* 40 | * Initialize the memory pool used to allocate buffers from user applications. 41 | * The only info needed at this stage is the size of the pool itself. 42 | * */ 43 | if (tf_memory_init(size) == -1){ 44 | printf ("Error initializing shared memory pool"); 45 | return -1; 46 | } 47 | 48 | /* 49 | * Retrieve the base ea of the pool. This value must be shared with the counterpart 50 | * on the compute node. 51 | * */ 52 | ea = tf_get_pool_ea(); 53 | printf("\nOCAPI shared memory pool initialised correctly at ea %p\n", ea); 54 | 55 | /* 56 | * Carves out a 100MB buffer from the main pool. The resulting ea is to be share 57 | * with the application on the memory node for mapping 58 | * */ 59 | buf_ptr = tf_buffer_alloc(buf_size); 60 | if (buf_ptr == NULL){ 61 | printf ("Error allocating buffer"); 62 | return -1; 63 | } 64 | 65 | printf("\n %" PRIu64 "bytes allocated in a buffer at ea %p\n", buf_size, buf_ptr); 66 | 67 | /* 68 | * Carves out a 200MB buffer from the main pool. The resulting ea is to be share 69 | * with the application on the memory node for mapping 70 | * */ 71 | buf2_ptr = tf_buffer_alloc(2*buf_size); 72 | if (buf_ptr == NULL){ 73 | printf ("Error allocating buffer"); 74 | return -1; 75 | } 76 | 77 | printf("\n %" PRIu64 "bytes allocated in a buffer at ea %p\n", 2*buf_size, buf2_ptr); 78 | 79 | printf("\nConfigure compute node with the pool and buffers effective address\n"); 80 | printf("\nPress any key to exit when mapping is completed on the compute side\n"); 81 | ch = getchar(); 82 | 83 | for (i=0; i < (buf_size)/(sizeof (unsigned int)); i++){ 84 | val = ((unsigned int *)buf_ptr)[i]; 85 | if (val == 0) 86 | zeroes += 1; 87 | else 88 | sum += val; 89 | } 90 | 91 | printf("\nBuffer 1 sum is %llu, zeroes are %llu", sum, zeroes); 92 | 93 | sum = 0; 94 | zeroes = 0; 95 | for (i=0; i < (2 * buf_size)/(sizeof (unsigned int)); i++){ 96 | val = ((unsigned int *)buf2_ptr)[i]; 97 | if (val == 0) 98 | zeroes += 1; 99 | else 100 | sum += val; 101 | } 102 | 103 | printf("\nBuffer 2 sum is %llu, zeroes are %llu", sum, zeroes); 104 | 105 | } 106 | -------------------------------------------------------------------------------- /libthymesisflow/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | 3 | MAINTAINER michele.gazzetti1@ibm.com 4 | 5 | RUN apt-get update && \ 6 | apt-get install -y libocxl-dev 7 | 8 | COPY ./bin/thymesisf-agent /usr/bin/ 9 | 10 | ENTRYPOINT ["/usr/bin/thymesisf-agent"] 11 | -------------------------------------------------------------------------------- /libthymesisflow/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CCFLAGS=-fPIC -Wall 3 | #LDFLAGS=-shared 4 | #LIBOCXL_DIR=../libocxl/lib 5 | 6 | # mock thymesisflow backend 7 | MOCK ?=0 8 | 9 | IDIR=include 10 | ODIR=obj 11 | BINDIR=bin 12 | LIB_DIR=lib 13 | TESTBIN_DIR=testbin 14 | 15 | OUT_DIRS= $(ODIR) $(BINDIR) $(LIB_DIR) $(TESTBIN_DIR) 16 | 17 | CCFLAGS+=-I $(IDIR)/ 18 | CCFLAGS+= -I/usr/include/ 19 | 20 | 21 | CORE_SRCS := $(wildcard src/*.c) 22 | CORE_OBJ:= $(addprefix $(ODIR)/,$(notdir $(CORE_SRCS:.c=.o))) 23 | 24 | TEST_SRCS := $(wildcard unittest/*.c) 25 | TEST_BIN:= $(addprefix $(TESTBIN_DIR)/,$(notdir $(TEST_SRCS:.c=))) 26 | 27 | CMD_SRCS := $(wildcard cmd/*.c) 28 | CMD_BIN:= $(addprefix $(BINDIR)/,$(notdir $(CMD_SRCS:.c=))) 29 | 30 | 31 | VPATH= src cmd unittest 32 | 33 | GIT_VERSION=`git rev-parse --abbrev-ref HEAD` 34 | 35 | ifeq ($(DEBUG),1) 36 | CCFLAGS+= -g -O0 37 | else 38 | CCFLAGS+= -O2 39 | endif 40 | 41 | ifeq ($(MOCK),1) 42 | CCFLAGS+= -DMOCK 43 | else 44 | LDFLAGS+= -L$(LIBOCXL_DIR) -locxl -lpthread -locxl 45 | endif 46 | 47 | ifeq ($(PREFIX),) 48 | PREFIX := /usr/local 49 | endif 50 | 51 | all: make_dirs libthymesisflow cmds 52 | 53 | make_dirs: 54 | $(info making output directories: $(OUT_DIRS)) 55 | mkdir -p $(OUT_DIRS) 56 | 57 | $(ODIR)/%.o: %.c 58 | $(CC) $(CCFLAGS) -c -o $@ $< $(LDFLAGS) 59 | 60 | libthymesisflow: $(CORE_OBJ) 61 | ar rcs lib/$@.a $(CORE_OBJ) 62 | 63 | 64 | $(TESTBIN_DIR)/%: $(ODIR)/%.o 65 | $(CC) $(CCFLAGS) -o $@ $^ -L$(LIB_DIR) -lthymesisflow $(LDFLAGS) 66 | 67 | tests: $(TEST_BIN) 68 | 69 | 70 | $(BINDIR)/%: $(ODIR)/%.o 71 | $(CC) $(CCFLAGS) -o $@ $^ -L$(LIB_DIR) -lthymesisflow $(LDFLAGS) 72 | 73 | cmds: $(CMD_BIN) 74 | 75 | docker: 76 | docker build --network=host -t thymesisflow:$(GIT_VERSION) . 77 | 78 | 79 | .PHONY: all clean clean_tests cmds tests docs 80 | 81 | cscope: 82 | find ./* -type f \( -name \*.c -o -name \*.h \) > ./cscope.files 83 | cscope -b -q 84 | 85 | clean: 86 | rm -rf $(OUT_DIRS) 87 | rm -rf cscope.files 88 | rm -rf cscope.in.out 89 | rm -rf cscope.out 90 | rm -rf cscope.po.out 91 | 92 | -------------------------------------------------------------------------------- /libthymesisflow/README.md: -------------------------------------------------------------------------------- 1 | ## Libthymesisflow 2 | 3 | Low levels APIs and configuration scripts to test and setup disaggregated memory 4 | via [ThymesisFlow](https://github.com/OpenCAPI/ThymesisFlow). 5 | 6 | We consider two possible scenarios. 7 | The first case is a mock of the AFUs (Accelerator Function Unit ) setup. 8 | This is mainly used for unit testing the APIs. 9 | The second scenario describes step by step how to setup remote memory using 10 | Thymesisflow AFUs on Power 9 machines. 11 | 12 | - [Dependencies](#dependencies) 13 | - [Build Mock](#build-mock) 14 | - [Unit Tests](#unit-tests) 15 | - [Remote Memory Setup](#remote-memory-setup) 16 | - [Find your AFU](#find-your-afu) 17 | 18 | 19 | ### Dependencies 20 | 21 | ``` 22 | sudo apt-get install libocxl-dev 23 | ``` 24 | 25 | 26 | ### Build Mock 27 | 28 | Build the library as following in case you want to test client and server interactions 29 | 30 | ``` 31 | make MOCK=1 32 | ``` 33 | 34 | Add `DEBUG=1` for debugging information 35 | 36 | ### Unit Tests 37 | 38 | The following steps describe how to setup and run unit tests. 39 | 40 | The first step requires to build unit tests. 41 | 42 | ``` 43 | make MOCK=1 44 | make MOCK=1 tests 45 | ``` 46 | 47 | To launch unit tests, run the following command. 48 | 49 | ``` 50 | ./testbin/unittest 51 | ``` 52 | 53 | 54 | ### Remote memory setup 55 | 56 | 57 | In this scenario we assume that you flashed both memory and compute design on your FPGAS. 58 | If not, you can find the instructions to [bring up the design](https://github.com/OpenCAPI/ThymesisFlow#how-to-bring-up-the-design). 59 | 60 | While the [mock](#build-mock) build is used for testing, in this scenario each requests will configure compute or memory node AFUs. 61 | For this scenario, recompile agent and CLI as following. 62 | 63 | ``` 64 | make 65 | ``` 66 | 67 | **Note:** use `DEBUG=1` for debugging information 68 | 69 | On both Compute and Memory side you can now run the server process. 70 | 71 | ``` 72 | sudo ./bin/thymesisf-agent -s 73 | ``` 74 | 75 | Where `SOCKET_PATH` is the path of the UNIX stream socket. 76 | This is used for local communications between agent and clients. 77 | The default value is `/tmp/thymesisflow.sock`. 78 | 79 | #### Setup Memory side 80 | 81 | On the **Memory node** you can reserve memory as following: 82 | 83 | ``` 84 | sudo ./bin/thymesisf-cli attach-memory \ 85 | --afu \ 86 | --cid \ 87 | --size 88 | ``` 89 | 90 | Where: 91 | 92 | * `AFU_NAME` is the prefix of the character device that can be found in `/dev/ocxl/`. Reference [Find your AFU](#find-your-afu) 93 | 94 | * `CIRCUIT_ID` is an alphanumeric string representing your connection 95 | * `MEMORY_SIZE` represents the amount of memory (in bytes) to be reserved. This needs to be a multiple of 256 MB 96 | 97 | An output example: 98 | 99 | ``` 100 | 2020-07-29 16:26:55, INFO ,Successfully allocated memory - EA: 0x79a3ff0d0080 101 | ``` 102 | 103 | `EA` is the effective address required to configure the compute side. 104 | 105 | #### Setup Compute side 106 | 107 | On the **Compute node** we can now attach the remote memory as following: 108 | 109 | ``` 110 | sudo ./bin/thymesisf-cli attach-compute \ 111 | --afu \ 112 | --cid \ 113 | --size \ 114 | --ea \ 115 | [--no-hotplug] 116 | ``` 117 | 118 | Where: 119 | 120 | * `AFU_NAME` is the prefix of the character device that can be found in `/dev/ocxl/`. Reference [Find your AFU](#find-your-afu) 121 | * `CIRCUIT_ID` is an alphanumeric string representing your connection 122 | * `MEMORY_SIZE` represents the amount of memory (in bytes) to be reserved. This needs to be a multiple of 256 MB 123 | * `EFFECTIVE_ADDRESS` address returned as `EA` during the [memory side setup](# Setup Memory side) 124 | * `--no-hotplug` attaches the memory without actual hotplug to the running Linux kenrel 125 | 126 | #### Stats counters 127 | 128 | To keep track of the traffic on the channel, Thymesisflow provides three counters reporting: 129 | 130 | * `rx (flits/sec)`: number of flits read from the channel. 131 | * `tx (flits/sec)`: number of flits transmitted on the channel. 132 | * `latency (cycles/sec)`: average latency on the channel. 133 | 134 | **Note:** Each counter resets at each read. Therefore, a read each seconds would report the `flits/sec`. 135 | Given that each flit is 32B, you can compute `rx` and `tx` bandwidth in `B/s` respectively by multiplying `tx*32` and `rx*32` 136 | 137 | To read the counters you can run the following command on the compute node: 138 | 139 | ``` 140 | python ./scripts/read_counters.py 141 | ``` 142 | 143 | ### Detach remote memory 144 | 145 | Once the connection is not needed anymore, we can release the preallocated resources. 146 | 147 | **Note:** it is necessary to detach the compute side first to avoid machine faults. 148 | 149 | 150 | #### Detach Compute side 151 | 152 | ``` 153 | sudo ./bin/thymesisf-cli detach-compute --cid 154 | ``` 155 | 156 | #### Detach Memory side 157 | 158 | ``` 159 | sudo ./bin/thymesisf-cli detach-compute --cid 160 | ``` 161 | 162 | #### Run agent in a Docker container 163 | 164 | ``` 165 | make docker 166 | docker run -it --privileged \ 167 | --mount type=bind,source=/dev/ocxl,target=/dev/ocxl \ 168 | --mount type=bind,source=/tmp,target=/tmp \ 169 | thymesisflow:$(git rev-parse --abbrev-ref HEAD) 170 | 171 | ``` 172 | 173 | 174 | ### Find your AFU 175 | 176 | AFUs respective character devices are available in `/dev/ocxl/` 177 | using the pattern `/dev/ocxl/..`. 178 | 179 | You can inspect the available AFUs as following: 180 | ``` 181 | $ ls /dev/ocxl/ 182 | IBM,RMEM.0006:00:00.1.0 183 | ``` 184 | 185 | In case of Thymesisflow, AFUs are exposed with the name `IBM,RMEM`. 186 | 187 | -------------------------------------------------------------------------------- /libthymesisflow/cmd/thymesisf-agent.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "proto.h" 18 | #include "agent.h" 19 | 20 | #include 21 | 22 | void help() { 23 | fprintf(stderr, "Usage thymesisf-agent [options]\n\n"); 24 | fprintf(stderr, 25 | "thymesisf-agent is a server responsible for configuring AFUs\n"); 26 | fprintf( 27 | stderr, 28 | "on each node participating in a disaggregated memory connection\n\n"); 29 | fprintf(stderr, "Options (all required):\n\n"); 30 | fprintf(stderr, 31 | " -s SOCKET_PATH\n\t\t linux socket fullpath, used by the " 32 | "server to receive client requests\n"); 33 | fflush(stdout); 34 | } 35 | 36 | 37 | 38 | int main(int argc, char *argv[]) { 39 | 40 | char *sock_path = NULL; 41 | 42 | 43 | int opt; 44 | 45 | while ((opt = getopt(argc, argv, ":s")) != -1) { 46 | switch (opt) { 47 | case 's': 48 | sock_path = optarg; 49 | break; 50 | case '?': 51 | help(); 52 | exit(EXIT_FAILURE); 53 | default: 54 | help(); 55 | exit(EXIT_FAILURE); 56 | } 57 | } 58 | run_agent(sock_path); 59 | 60 | } 61 | -------------------------------------------------------------------------------- /libthymesisflow/include/agent.h: -------------------------------------------------------------------------------- 1 | /// @file agent.h 2 | #if !defined(THYMESISFLOW_CMD_H) 3 | #define THYMESISFLOW_CMD_H 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "logger.h" 15 | #include 16 | #include 17 | #include 18 | #include "proto.h" 19 | 20 | #ifndef MOCK 21 | #include "stats.h" 22 | #endif 23 | 24 | /*! \def BACKLOG 25 | * \brief max incoming connections 26 | */ 27 | #define BACKLOG 5 28 | 29 | 30 | void run_agent(const char* sock_path); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /libthymesisflow/include/client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file client.h 18 | #if !defined(THYMESISFLOW_CLIENT_H) 19 | #define THYMESISFLOW_CLIENT_H 20 | 21 | #include "errcode.h" 22 | #include "iport.h" 23 | #include "logger.h" 24 | #include "proto.h" 25 | 26 | // fixed afu port in current design 27 | #define AFU_PORT 2 28 | 29 | /** 30 | * Send setup request to the memory node daemon through the linux socket 31 | * 32 | * @param[in] circuit_id: circuit identifier 33 | * @param[in] afu: AFU name 34 | * @param[in] iport_list: list of ports to be used for the attachemnt (Currently 35 | * single port supported) 36 | * @param[in] size: size of memory to be allocated 37 | * @param[in] sock_path: path to the linux socket 38 | * @param[out] pmessage unmarshalled response from the server 39 | */ 40 | pmessage send_attach_memory_msg(const char *circuitid, const char *afu, 41 | const iport_list *ports, const uint64_t size, 42 | const char *sock_path); 43 | 44 | /** 45 | * Send setup request to the compute node daemon through the linux socket 46 | * 47 | * @param[in] circuit_id: circuit identifier 48 | * @param[in] afu: AFU name 49 | * @param[in] iport_list: list of ports to be used for the attachemnt (Currently 50 | * single port supported) 51 | * @param[in] size: size of memory to be allocated 52 | * @param[in] ea: effective address to be passed to the AFU during the setup 53 | * @param[in] no_hotplug: do not hotploug memory 54 | * @param[in] sock_path: path to the linux socket 55 | * @param[out] pmessage unmarshalled response from the server 56 | */ 57 | pmessage send_attach_compute_msg(const char *circuitid, const char *afu, 58 | const iport_list *ports, const uint64_t size, 59 | const uint64_t ea, int no_hotplug, const char *sock_path); 60 | 61 | /** 62 | * Send detachment request to the memory node daemon through the linux socket 63 | * 64 | * @param[in] circuit_id: circuit identifier 65 | * @param[in] sock_path: path to the linux socket 66 | * @param[out] pmessage unmarshalled response from the server 67 | */ 68 | pmessage send_detach_memory_msg(const char *circuitid, const char *sock_path); 69 | 70 | /** 71 | * Send detachment request to the compute node daemon through the linux socket 72 | * 73 | * @param[in] circuit_id: circuit identifier 74 | * @param[in] sock_path: path to the linux socket 75 | * @param[out] pmessage unmarshalled response from the server 76 | */ 77 | pmessage send_detach_compute_msg(const char *circuitid, const char *sock_path); 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /libthymesisflow/include/connection.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file connection.h 18 | #if !defined(CONNECTION_H) 19 | #define CONNECTION_H 20 | 21 | #include "errcode.h" 22 | #include "iport.h" 23 | #include "logger.h" 24 | #include "protoconst.h" 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #ifndef MOCK 31 | #include "libocxl.h" 32 | 33 | #define AURORA_CTRL 0x78 34 | 35 | #define AURORA_CHAN 0x02 36 | 37 | #define MEM_SEG0_CONFIG 0x20 38 | 39 | #define MEM_SEG0_OFFSET 0x0 40 | 41 | #define MEM_SEG1_CONFIG 0x28 42 | 43 | #define MEM_SEG1_OFFSET (1UL << 40) 44 | #include "stats.h" 45 | 46 | typedef union { 47 | struct { 48 | unsigned int test_enable : 1; 49 | unsigned int trigger_sfw_intrp : 1; 50 | unsigned int ignore_nomatch_on_read : 1; 51 | unsigned int unassigned1 : 1; 52 | unsigned int enable_pipeline : 1; 53 | } reg; 54 | uint64_t val; 55 | } global_mmio_control; 56 | 57 | #endif 58 | 59 | /** 60 | * Contains all variables related to an active remote memory allocation 61 | */ 62 | struct connection_s { 63 | char circuit_id[CIRCUIT_ID_SIZE + 1]; 64 | char afu_name[AFUNAME_SIZE]; 65 | uint64_t size; 66 | unsigned char *ea; 67 | struct connection_s *next; 68 | int no_hotplug; 69 | 70 | #ifndef MOCK 71 | ocxl_afu_h afu; 72 | pthread_t *stat_tid; 73 | ocxl_mmio_h global; 74 | ocxl_mmio_h pp_mmio; 75 | #endif 76 | }; 77 | 78 | typedef struct connection_s connection; 79 | 80 | connection *clist; 81 | 82 | /** 83 | * Add a new connection 84 | * 85 | * @param[in] connection: connection to be added 86 | * @param[out] outcome: 0 if operations finished with success 87 | */ 88 | int add_conn(connection *conn); 89 | 90 | /** 91 | * Get connection from circuit_id 92 | * 93 | * @param[in] circuit_id: circuit identifier 94 | * @param[out] connection: first connection having the specified circuit id 95 | */ 96 | connection *get_conn(const char *circuit_id); 97 | 98 | /** 99 | * Delete first circuit matching cirguit_id 100 | * 101 | * @param[in] circuit_id: circuit identifier 102 | * @param[out] outcome: 0 if operations finished with success 103 | */ 104 | int del_conn(const char *circuit_id); 105 | 106 | /** 107 | * List all active connections 108 | */ 109 | void list_connections(); 110 | 111 | /** 112 | * Free dynamically allocated connection struc 113 | * 114 | * @param[in] connection: connection to be freed 115 | * @param[out] outcome: 0 if operations finished with success 116 | */ 117 | int free_conn(connection *conn); 118 | 119 | /** 120 | * Dynamically allocate a new connection 121 | * 122 | * @param[in] circuit_id: circuit identifier 123 | * @param[in] afu_name: AFU to be used 124 | * @param[in] size: disaggregated memory size 125 | * @param[in] no_hotplug: no_hotplug flag value 126 | * @param[out] connection: connection to be freed 127 | */ 128 | connection *new_conn(const char *circuit_id, const char *afu_name, 129 | const uint64_t size, int no_hotplug); 130 | 131 | int setup_afu_compute(connection *conn, uint64_t effective_addr, 132 | iport_list *ports); 133 | 134 | int setup_afu_memory(connection *conn); 135 | 136 | #define CONN_SUCCESS 0 137 | 138 | #define CONN_DUP 1 139 | #define CONN_NULL 2 140 | 141 | #define CONN_NOT_FOUND 3 142 | 143 | #endif 144 | -------------------------------------------------------------------------------- /libthymesisflow/include/errcode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file errcode.h 18 | #if !defined(THYMESISFLOW_ERR_H) 19 | #define THYMESISFLOW_ERR_H 20 | #define THYMESISFLOW_ERR_H 21 | 22 | /*! \def ATTACH_OK 23 | * \brief Successfully attached memory 24 | */ 25 | #define ATTACH_OK 100 26 | 27 | /*! \def ERR_MLOCK 28 | * \brief Error locking memory 29 | */ 30 | #define ERR_MLOCK 101 31 | 32 | /*! \def ERR_AFU_OPEN 33 | * \brief Error opening AFU 34 | */ 35 | #define ERR_AFU_OPEN 102 36 | 37 | /*! \def ERR_AFU_ATTACH 38 | * \brief Error attaching AFU 39 | */ 40 | #define ERR_AFU_ATTACH 103 41 | 42 | /*! \def ERR_MMIO_MAP_GLOBAL 43 | * \brief Error mapping global memory area 44 | */ 45 | #define ERR_MMIO_MAP_GLOBAL 104 46 | 47 | /*! \def ERR_MMIO_MAP_LOCAL 48 | * \brief Error mapping per process MMIO Area 49 | */ 50 | #define ERR_MMIO_MAP_LOCAL 105 51 | 52 | /*! \def ERR_WRITE_CTRL_REGISTER 53 | * \brief Error writing control register 54 | */ 55 | #define ERR_WRITE_CTRL_REGISTER 106 56 | 57 | /*! \def ERR_ENABLING_NETWORK 58 | * \brief Error enabling network 59 | */ 60 | #define ERR_ENABLING_NETWORK 107 61 | 62 | /*! \def ERR_PORT_UNSUPPORTED 63 | * \brief Only single port setup supported 64 | */ 65 | #define ERR_PORT_UNSUPPORTED 108 66 | 67 | /*! \def ERR_AURORA_CHAN 68 | * \brief Error bringing up the AURORA channel 69 | */ 70 | #define ERR_AURORA_CHAN 109 71 | 72 | /*! \def ERR_ATTACH_SEG 73 | * \brief Error while attaching the memory segment 74 | */ 75 | #define ERR_ATTACH_SEG 110 76 | 77 | /*! \def ERR_PROBE 78 | * \brief Error while probing the memory 79 | */ 80 | #define ERR_PROBE 111 81 | 82 | /*! \def DETACH_OK 83 | * \brief Successfully detached memory 84 | */ 85 | #define DETACH_OK 200 86 | 87 | /*! \def ERR_REGISTER_DETACH 88 | * \brief error while updating the internal connection status 89 | */ 90 | #define ERR_REGISTER_DETACH 201 91 | 92 | /*! \def ERR_MISSING_CID 93 | * \brief Missing connection related to circuit id 94 | */ 95 | #define ERR_MISSING_CID 201 96 | 97 | #endif 98 | -------------------------------------------------------------------------------- /libthymesisflow/include/iport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file iport.h 18 | #if !defined(IPORT_H) 19 | #define IPORT_H 20 | 21 | #include "logger.h" 22 | #include 23 | #include 24 | 25 | struct port_s { 26 | int nport; 27 | struct port_s *next; 28 | }; 29 | 30 | typedef struct port_s iport_list; 31 | 32 | /** 33 | * Print interface (AFU) ports 34 | * 35 | * @param[in] iport_list: list of ports passed in the socket request 36 | */ 37 | void cycle_ports(iport_list *pl); 38 | 39 | /** 40 | * Add port to the list 41 | * 42 | * @param[in] iport_list: list of ports 43 | * @param[in] port_number: port number 44 | */ 45 | iport_list *add_port(iport_list *pl, u_int8_t port_number); 46 | 47 | /** 48 | * Convert from binary representation of the ports to list structure 49 | * 50 | * @param[in] n: binary representation of the porst. 51 | */ 52 | iport_list *unmarshal_iports(u_int8_t n); 53 | 54 | /** 55 | * Free dynamically allocated list structure 56 | * 57 | * @param[in] iport_list: list of port numbers 58 | */ 59 | void free_iport_list(iport_list *pl); 60 | 61 | /** 62 | * Convert port list to binary represetnation 63 | * 64 | * @param[in] iport_list: list of port numbers 65 | * @param[out] binary_rep: binary representation of the ports currently in use 66 | */ 67 | u_int8_t marshal_iports(const iport_list *pl); 68 | #endif 69 | -------------------------------------------------------------------------------- /libthymesisflow/include/logger.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file logger.h 18 | 19 | #ifndef LOGGER_H 20 | #define LOGGER_H 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #define ERROR_TAG "ERROR" 29 | #define WARN_TAG "WARN" 30 | #define INFO_TAG "INFO" 31 | #define DEBUG_TAG "DEBUG" 32 | 33 | /** 34 | * Get current time 35 | * 36 | * @param[out] time_string: string reporting the time as %Y-%m-%d %H:%M:%S 37 | */ 38 | static inline char *current_time() { 39 | static char time_string[64]; 40 | time_t rawtime; 41 | struct tm *timeinfo; 42 | 43 | time(&rawtime); 44 | timeinfo = localtime(&rawtime); 45 | 46 | strftime(time_string, 64, "%Y-%m-%d %H:%M:%S", timeinfo); 47 | 48 | return time_string; 49 | } 50 | 51 | static inline void print_log(const char *tag, const char *curr_time, 52 | const char *message, va_list args) { 53 | printf("%s [%s] ", curr_time, tag); 54 | vprintf(message, args); 55 | fflush(stdout); // printf("\n"); 56 | } 57 | 58 | static inline void log_info(char *message, ...) { 59 | va_list args; 60 | va_start(args, message); 61 | print_log(INFO_TAG, current_time(), message, args); 62 | va_end(args); 63 | } 64 | static inline void log_debug(char *message, ...) { 65 | va_list args; 66 | va_start(args, message); 67 | print_log(DEBUG_TAG, current_time(), message, args); 68 | va_end(args); 69 | } 70 | 71 | static inline void log_warn(char *message, ...) { 72 | va_list args; 73 | va_start(args, message); 74 | print_log(WARN_TAG, current_time(), message, args); 75 | va_end(args); 76 | } 77 | 78 | static inline void log_error(char *message, ...) { 79 | va_list args; 80 | va_start(args, message); 81 | print_log(ERROR_TAG, current_time(), message, args); 82 | va_end(args); 83 | } 84 | 85 | static inline void print_log_ext(const char *tag, const char *curr_time, 86 | const char *filename, int line, 87 | const char *message, ...) { 88 | va_list args; 89 | va_start(args, message); 90 | printf("%s [%s] %s:%d, ", curr_time, tag, filename, line); 91 | vprintf(message, args); 92 | va_end(args); 93 | fflush(stdout); 94 | } 95 | 96 | #define log_info_ext(message, args...) \ 97 | print_log_ext(INFO_TAG, current_time(), basename(__FILE__), __LINE__, \ 98 | message, ##args) 99 | 100 | #define log_debug_ext(message, args...) \ 101 | print_log_ext(DEBUG_TAG, current_time(), basename(__FILE__), __LINE__, \ 102 | message, ##args) 103 | 104 | #define log_warn_ext(message, args...) \ 105 | print_log_ext(WARN_TAG, current_time(), basename(__FILE__), __LINE__, \ 106 | message, ##args) 107 | 108 | #define log_error_ext(message, args...) \ 109 | print_log_ext(ERROR_TAG, current_time(), basename(__FILE__), __LINE__, \ 110 | message, ##args) 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /libthymesisflow/include/proto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file proto.h 18 | #if !defined(THYMESISFLOW_PROTO_H) 19 | #define THYMESISFLOW_PROTO_H 20 | 21 | #include "errcode.h" 22 | #include "iport.h" 23 | #include "logger.h" 24 | #include "protoconst.h" 25 | #include "thymesisflow.h" 26 | #include 27 | #include 28 | #include 29 | 30 | typedef struct proto_response { 31 | char circuitid[CIRCUIT_ID_SIZE + 1]; 32 | char msgtype[MSGTYPE_SIZE + 1]; 33 | char afu[AFUNAME_SIZE + 1]; 34 | int status; 35 | uint64_t size; 36 | uint64_t ea; 37 | int no_hotplug; 38 | } pmessage; 39 | 40 | /** 41 | * Open linux socket connection towards the server 42 | * 43 | * @param[in] circuit_id: circuit identifier 44 | * @param[out] sfd: socket file descriptor if success, -1 if failure 45 | */ 46 | int open_client_connection(const char *sock_path); 47 | 48 | /** 49 | * Print the content of the a message sent on the socket. Used for DEBUG 50 | * purposes 51 | * 52 | * @param[in] msg: message sent through the socket 53 | */ 54 | void print_proto_msg(char *msg); 55 | 56 | /** 57 | * Print the content of the a message sent on the socket. Used for DEBUG 58 | * purposes 59 | * 60 | * @param[in] msg: message sent through the socket 61 | * @param[in] rsp: pointer to data struct that will contain the unmarshalled 62 | * response 63 | */ 64 | void unmarshal_response(char *msg, pmessage *rsp); 65 | 66 | /** 67 | * Read message from from the socket connection and unmarshal the content 68 | * 69 | * @param[in] cfd: socket connection 70 | * @param[in] rsp: pointer to data struct that will contain the unmarshalled 71 | * response 72 | */ 73 | int read_pmessage(int cfd, pmessage *rsp); 74 | 75 | /** 76 | * Read message from the socket connection 77 | * 78 | * @param[in] cfd: socket connection 79 | * @param[in] msg: array containing the marshalled response 80 | */ 81 | char *read_message(int cfd); 82 | 83 | /** 84 | * Write message on a socket connection 85 | * 86 | * @param[in] fd: socket connection 87 | * @param[in] msg: array containing the marshalled response 88 | * @param[out] result: 0 if success, -1 if error in writing the message 89 | */ 90 | int write_message(int fd, const char *msg); 91 | 92 | /** 93 | * Marshal request to setup new thymesisflow on memory-stealing node 94 | * 95 | * @param[in] circuitid: circuit uuid 96 | * @param[in] afu: name of the Accelerated Function Unit 97 | * @param[in] ports: AFU port to be used 98 | * @param[in] size: memory allocation size (in bytes) 99 | * @param[out] msg: array containing the marshalled request 100 | */ 101 | char *marshal_attach_memory_request(const char *circuitid, const char *afu, 102 | const iport_list *ports, 103 | const uint64_t size); 104 | 105 | /** 106 | * Marshal request to setup new thymesisflow on memory-stealing node 107 | * 108 | * @param[in] circuitid: circuit uuid 109 | * @param[in] afu: name of the Accelerated Function Unit 110 | * @param[in] ports: AFU port to be used 111 | * @param[in] size: memory allocation size (in bytes) 112 | * @param[in] ea: effective address used for memory access translation 113 | * @param[in] no_hotplug: don't hoplug memory 114 | * @param[out] msg: array containing the marshalled request 115 | */ 116 | char *marshal_attach_compute_request(const char *circuitid, const char *afu, 117 | const iport_list *ports, 118 | const uint64_t memsize, const uint64_t ea, int no_hotplug); 119 | 120 | /** 121 | * Marshal request to tear down a thymesisflow on memory-stealing node 122 | * 123 | * @param[in] circuitid: circuit uuid associate to the connection to tear down 124 | * @param[out] msg: array containing the marshalled request 125 | */ 126 | char *marshal_detach_memory_request(const char *circuitid); 127 | 128 | /** 129 | * Marshal request to tear down a thymesisflow on comute node 130 | * 131 | * @param[in] circuitid: circuit uuid associate to the connection to tear down 132 | * @param[out] msg: array containing the marshalled request 133 | */ 134 | char *marshal_detach_compute_request(const char *circuitid); 135 | 136 | /** 137 | * Perform the steps required to setup a new thymesisflow on the compute node 138 | * 139 | * @param[in] msg: array containing the marshalled request 140 | * @param[out] msg: array containing the marshalled response 141 | */ 142 | char *proto_attach_compute(const char *msg); 143 | 144 | /** 145 | * Perform the steps required to setup a new thymesisflow on the memory-stealing 146 | * node 147 | * 148 | * @param[in] msg: array containing the marshalled request 149 | * @param[out] msg: array containing the marshalled response 150 | */ 151 | char *proto_attach_memory(const char *msg); 152 | 153 | /** 154 | * Perform the steps required to tear down a thymesisflow on the memory-stealing 155 | * node 156 | * 157 | * @param[in] msg: array containing the marshalled request 158 | * @param[out] msg: array containing the marshalled response 159 | */ 160 | char *proto_detach_memory(const char *msg); 161 | 162 | /** 163 | * Perform the steps required to setup a new thymesisflow on the compute node 164 | * 165 | * @param[in] msg: array containing the marshalled request 166 | * @param[out] msg: array containing the marshalled response 167 | */ 168 | char *proto_detach_compute(const char *msg); 169 | 170 | /** 171 | * Return error message if the message type is not recognized 172 | * 173 | * @param[in] msg: array containing the marshalled request 174 | * @param[out] msg: array containing the marshalled response 175 | */ 176 | char *set_unknown_mode_response(char *msg); 177 | 178 | /** 179 | * Dynamically allocate memory for a protocol message 180 | * 181 | * @param[out] ptr: return a pointer to newly allocated memory 182 | */ 183 | char *new_proto_msg(); 184 | 185 | #endif 186 | -------------------------------------------------------------------------------- /libthymesisflow/include/protoconst.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file protoconst.h 18 | #if !defined(PROTOCONST_H) 19 | #define PROTOCONST_H 20 | 21 | /* 22 | 23 | Possible messages: 24 | 25 | - | circuitid | memory | afu | port | size | 26 | - | circuitid | compute | afu | port | size | ea | no_hotplug | 27 | - | circuitid | mresponse | afu | | | ea | | error | 28 | - | circuitid | cresponse | afu | | size | | | error | 29 | 30 | */ 31 | 32 | /*! \def SOCK_PATH 33 | * \brief default linux socket path used for communication with libthymesisflow 34 | * server 35 | */ 36 | #define SOCK_PATH "/tmp/thymesisflow.sock" 37 | 38 | /*! \def CIRCUIT_ID_SIZE 39 | * \brief circuit uuid lenght 40 | */ 41 | #define CIRCUIT_ID_SIZE 36 42 | 43 | /*! \def MSGTYPE_SIZE 44 | * \brief maximum size for section identifying the type of message sent/received 45 | */ 46 | #define MSGTYPE_SIZE 16 47 | 48 | /*! \def AFUNAME_SIZE 49 | * \brief maximum size for section identifying the AFU name (or path) 50 | */ 51 | #define AFUNAME_SIZE 256 52 | 53 | /*! \def IPORT_SIZE 54 | * \brief maximum size for section reporting the AFU ports to be used 55 | * (current maximum is 8 given from the binary representation) 56 | */ 57 | #define IPORT_SIZE sizeof(uint8_t) 58 | 59 | /*! \def IPORT_SIZE 60 | * \brief maximum size for section reporting the remote memory to be attached 61 | */ 62 | #define MEM_SIZE sizeof(uint64_t) 63 | 64 | /*! \def EA_SIZE 65 | * \brief maximum size for sectiction containing the effective address 66 | */ 67 | #define EA_SIZE sizeof(uint64_t) 68 | 69 | /*! \def NO_HOTPLUG_SIZE 70 | * \brief size of the no_hotplug flag field 71 | */ 72 | #define NO_HOTPLUG_SIZE sizeof(int) 73 | 74 | /*! \def ERROR_SIZE 75 | * \brief maximum size for the error section 76 | */ 77 | #define ERROR_SIZE sizeof(uint32_t) 78 | 79 | /*! \def MSG_SIZE 80 | * \brief maximum message size 81 | */ 82 | #define MSG_SIZE \ 83 | CIRCUIT_ID_SIZE + MSGTYPE_SIZE + AFUNAME_SIZE + IPORT_SIZE + MEM_SIZE + \ 84 | EA_SIZE + ERROR_SIZE + NO_HOTPLUG_SIZE 85 | 86 | #define CIRCUIT_ID_OFFSET 0 87 | 88 | #define MSGTYPE_OFFSET CIRCUIT_ID_OFFSET + CIRCUIT_ID_SIZE 89 | 90 | #define AFUNAME_OFFSET MSGTYPE_OFFSET + MSGTYPE_SIZE 91 | 92 | #define IPORT_OFFSET AFUNAME_OFFSET + AFUNAME_SIZE 93 | 94 | #define MEM_SIZE_OFFSET IPORT_OFFSET + IPORT_SIZE 95 | 96 | #define EA_OFFSET MEM_SIZE_OFFSET + MEM_SIZE 97 | 98 | #define NO_HOTPLUG_OFFSET EA_OFFSET + EA_SIZE 99 | 100 | #define ERROR_OFFSET NO_HOTPLUG_OFFSET + NO_HOTPLUG_SIZE 101 | 102 | #define MEMORY_ATTACH "memory_attach" 103 | 104 | #define COMPUTE_ATTACH "compute_attach" 105 | 106 | #define ERROR_ATTACH "error_attach" 107 | 108 | #define MRESPONSE_ATTACH "mresponse_attach" 109 | 110 | #define CRESPONSE_ATTACH "cresponse_attach" 111 | 112 | #define MEMORY_DETACH "memory_detach" 113 | 114 | #define COMPUTE_DETACH "compute_detach" 115 | 116 | #define ERROR_DETACH "error_detach" 117 | 118 | #define MRESPONSE_DETACH "mresponse_detach" 119 | 120 | #define CRESPONSE_DETACH "cresponse_detach" 121 | 122 | /*! \def ERR_MSGTYPE 123 | * \brief message type is not one of the default message types 124 | * specified (MEMORY_ATTACH/MEMORY_DETACH/COMPUTE_ATTACH/COMPUTE_DETACH) 125 | */ 126 | #define ERR_MSGTYPE -3 127 | 128 | /*! \def ERR_MSGNULL 129 | * \brief provided a null reference to a message 130 | */ 131 | #define ERR_MSGNULL -2 132 | 133 | /*! \def ERR_NULL_PARAM 134 | * \brief provided a null reference input parameter 135 | */ 136 | #define ERR_NULL_PARAM -1 137 | 138 | /*! \def ERR_EMPTY_PARAM 139 | * \brief provided an empty input parameter 140 | */ 141 | #define ERR_EMPTY_PARAM -4 142 | 143 | /*! \def ERR_INT_PARAM 144 | * \brief provided a negative or zero input parameter 145 | */ 146 | #define ERR_INT_PARAM -5 147 | 148 | /*! \def ERR_MEMBLOCK_SIZE 149 | * \brief memory hotplugged needs to be a multiple of MEMBLOCK_SIZE 150 | * (/sys/devices/system/memory/block_size_bytes) 151 | */ 152 | #define ERR_MEMBLOCK_SIZE -5 153 | #endif 154 | -------------------------------------------------------------------------------- /libthymesisflow/include/stats.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef THYMESISSTATS_H 18 | #define THYMESISSTATS_H 19 | 20 | #include "logger.h" 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #define TX_STATS 0x60 28 | #define RX_STATS 0x68 29 | #define LATENCY_STATS 0x70 30 | #define PERIOD_SEC 1 31 | #define THYMESISCOUNT_PATH "/tmp/thymesispcount" 32 | 33 | struct thread_args { 34 | ocxl_mmio_h global; 35 | int fd; 36 | int period; 37 | }; 38 | 39 | typedef struct thread_args t_args; 40 | 41 | // int tf_init_stats_thread(pthread_t * tid, int fd, ocxl_mmio_h global, int 42 | // period); 43 | int start_stats_thread(pthread_t **tid, ocxl_mmio_h global); 44 | int stop_stats_thread(pthread_t *tid); 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /libthymesisflow/include/thymesisflow.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file thymesisflow.h 18 | #if !defined(THYMESISFLOW_H) 19 | #define THYMESISFLOW_H 20 | 21 | #include "connection.h" 22 | #include "errcode.h" 23 | #include "iport.h" 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | //#include "ty_afu.h" 30 | 31 | #ifndef MOCK 32 | #include "libocxl.h" 33 | #endif 34 | 35 | #define GLOBAL_MMIO_REG_COUNT (0x300 / 8) 36 | 37 | #define CTRL_REG 0x000 38 | 39 | #define INIT_PROBE_ADDR 0x2000000000000 40 | 41 | #define AFU_NAME "IBM,RMEM" 42 | 43 | #define MEMBLOCK_SIZE 0x10000000 44 | 45 | #define PROBE_PATH "/sys/devices/system/memory/probe" 46 | 47 | #define MEMORY_PATH "/sys/devices/system/memory" 48 | 49 | #define CACHE_ALIGNMENT 128 50 | 51 | /** 52 | * Setup AFU and allocate memory 53 | * Note: ports are passed but not needed in this current hw implementation 54 | * 55 | * @param[in] circuit_id: circuit identifier 56 | * @param[in] afu_name: AFU name 57 | * @param[in] iport_list: list of ports to be used for the attachemnt (Currently 58 | * single port supported) 59 | * @param[in] size: size of memory to be allocated 60 | * @param[in] eaptr: effective address to return to the compute node for memory 61 | * attachment #param[out] error_code 62 | */ 63 | int attach_memory(const char *circuit_id, const char *afu_name, 64 | iport_list *ports, const uint64_t size, uint64_t *eaptr); 65 | 66 | /** 67 | * Setup AFU and attach memory 68 | * 69 | * @param[in] circuit_id: circuit identifier 70 | * @param[in] afu_name: AFU name 71 | * @param[in] iport_list: list of ports to be used for the attachemnt (Currently 72 | * single port supported) 73 | * @param[in] effective_addr: address passed the AFU to attach the remove memory 74 | * @param[in] size: size of memory to be allocated 75 | * @param[out] error_code 76 | */ 77 | int attach_compute(const char *circuit_id, const char *afu_name, 78 | iport_list *ports, const uint64_t effective_addr, 79 | const uint64_t size, int no_hotplug); 80 | /** 81 | * Disconnect and free allocated memory 82 | * 83 | * @param[in] circuit_id: circuit identifier 84 | * @param[out] error_code 85 | */ 86 | int detach_memory(const char *circuit_id); 87 | 88 | /** 89 | * Register detachment of memory on compute side 90 | * 91 | * @param[in] circuit_id: circuit identifier 92 | * @param[out] error_code 93 | */ 94 | int detach_compute(const char *circuit_id); 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /libthymesisflow/scripts/read_counters.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | FIFO ="/tmp/thymesispcount" 4 | fd=open(FIFO,"r") 5 | 6 | def read_fifo(): 7 | while True: 8 | line = fd.readline() 9 | if line !="": 10 | sys.stdout.write(line) 11 | 12 | if __name__ == '__main__': 13 | read_fifo() 14 | -------------------------------------------------------------------------------- /libthymesisflow/src/agent.c: -------------------------------------------------------------------------------- 1 | #include "agent.h" 2 | 3 | void handle_request(int cfd) { 4 | char *msg = read_message(cfd); 5 | char *response_msg; 6 | 7 | if (msg == NULL) { 8 | // TODO: return error reading message 9 | 10 | response_msg = new_proto_msg(); 11 | return; 12 | } else { 13 | char *msgtype = (char *)malloc(MSGTYPE_SIZE); 14 | memcpy(msgtype, msg + CIRCUIT_ID_SIZE, MSGTYPE_SIZE); 15 | 16 | if (strncmp(msgtype, MEMORY_ATTACH, MSGTYPE_SIZE) == 0) { 17 | response_msg = proto_attach_memory(msg); 18 | 19 | } else if (strncmp(msgtype, MEMORY_DETACH, MSGTYPE_SIZE) == 0) { 20 | response_msg = proto_detach_memory(msg); 21 | 22 | } else if (strncmp(msgtype, COMPUTE_ATTACH, MSGTYPE_SIZE) == 0) { 23 | response_msg = proto_attach_compute(msg); 24 | 25 | } else if (strncmp(msgtype, COMPUTE_DETACH, MSGTYPE_SIZE) == 0) { 26 | response_msg = proto_detach_compute(msg); 27 | 28 | } else { 29 | log_warn("unknown mode %s\n", msgtype); 30 | response_msg = set_unknown_mode_response(msg); 31 | // fill error response 32 | } 33 | free(msgtype); 34 | } 35 | 36 | write_message(cfd, response_msg); 37 | // error checking 38 | 39 | if (msg != NULL) 40 | free(msg); 41 | if (response_msg != NULL) 42 | free(response_msg); 43 | } 44 | 45 | void graceful_termination(){ 46 | log_info("Graceful agent termination...\n"); 47 | exit(0); 48 | } 49 | 50 | 51 | void run_agent(const char* sock_path){ 52 | 53 | struct sockaddr_un addr; 54 | int sfd, cfd; 55 | 56 | if (sock_path == NULL) { 57 | sock_path = SOCK_PATH; 58 | } 59 | log_info("using socket: %s\n", sock_path); 60 | 61 | log_info("starting server with sock_path: %s\n", sock_path); 62 | 63 | // avoid exiting when reading on closed pipe, need better handling 64 | signal(SIGPIPE, SIG_IGN); 65 | signal(SIGTERM, graceful_termination); 66 | 67 | sfd = socket(AF_UNIX, SOCK_STREAM, 0); 68 | if (sfd == -1) { 69 | log_error("error creating socket: %s\n", strerror(errno)); 70 | exit(1); 71 | } 72 | 73 | if (remove(sock_path) == -1 && errno != ENOENT) { 74 | log_info("error removing %s.. \n", sock_path); 75 | exit(2); 76 | } 77 | 78 | memset(&addr, 0, sizeof(addr.sun_path) - 1); 79 | 80 | addr.sun_family = AF_UNIX; // use unix socket 81 | strncpy(addr.sun_path, sock_path, sizeof(addr.sun_path) - 1); 82 | 83 | if (bind(sfd, (struct sockaddr *)&addr, sizeof(struct sockaddr_un)) == -1) { 84 | log_error("error while binding the socket\n"); 85 | exit(3); 86 | } 87 | 88 | if (listen(sfd, BACKLOG) == -1) { 89 | log_error("error during listen\n"); 90 | } 91 | 92 | log_info("Starting thymesisflow server...\n"); 93 | 94 | for (;;) { 95 | log_info("Ready to accept new requests...\n"); 96 | 97 | cfd = accept(sfd, NULL, NULL); 98 | if (cfd == -1) { 99 | log_info("error during accept\n"); 100 | exit(4); 101 | } 102 | handle_request(cfd); 103 | 104 | log_info("closing connection...\n"); 105 | 106 | if (close(cfd) == -1) { 107 | log_error("error during close\n"); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /libthymesisflow/src/client.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file client.c 18 | #include "client.h" 19 | 20 | pmessage send_cmd(const char *msg, const char *sock_path) { 21 | int sfd = open_client_connection(sock_path); 22 | if (write_message(sfd, msg) != 0) { 23 | log_warn("error sending message\n"); 24 | } 25 | 26 | pmessage rsp; 27 | read_pmessage(sfd, &rsp); 28 | 29 | if (close(sfd) == -1) { 30 | log_error("error while closing socket file descriptor\n"); 31 | } 32 | return rsp; 33 | } 34 | 35 | pmessage send_detach_memory_msg(const char *circuitid, const char *sock_path) { 36 | char *msg = marshal_detach_memory_request(circuitid); 37 | 38 | pmessage rsp = send_cmd(msg, sock_path); 39 | 40 | log_debug("circuitid: %s - type: %s - status: %d\n", rsp.circuitid, 41 | rsp.msgtype, rsp.status); 42 | 43 | free(msg); 44 | return rsp; 45 | } 46 | 47 | pmessage send_detach_compute_msg(const char *circuitid, const char *sock_path) { 48 | char *msg = marshal_detach_compute_request(circuitid); 49 | 50 | pmessage rsp = send_cmd(msg, sock_path); 51 | 52 | log_debug("circuitid: %s - type: %s - status: %d\n", rsp.circuitid, 53 | rsp.msgtype, rsp.status); 54 | 55 | free(msg); 56 | return rsp; 57 | } 58 | 59 | pmessage send_attach_memory_msg(const char *circuitid, const char *afu, 60 | const iport_list *ports, const uint64_t size, 61 | const char *sock_path) { 62 | 63 | char *msg = marshal_attach_memory_request(circuitid, afu, ports, size); 64 | 65 | pmessage rsp = send_cmd(msg, sock_path); 66 | 67 | log_debug("circuitid: %s - type: %s - afu %s - size %lu - ea(hex): %lx - " 68 | "ea(int):%lu - status: %d\n", 69 | rsp.circuitid, rsp.msgtype, rsp.afu, rsp.size, rsp.ea, rsp.ea, 70 | rsp.status); 71 | 72 | free(msg); 73 | return rsp; 74 | } 75 | 76 | pmessage send_attach_compute_msg(const char *circuitid, const char *afu, 77 | const iport_list *ports, const uint64_t size, 78 | const uint64_t ea, int no_hotplug, const char *sock_path) { 79 | 80 | char *msg = marshal_attach_compute_request(circuitid, afu, ports, size, ea, no_hotplug); 81 | 82 | pmessage rsp = send_cmd(msg, sock_path); 83 | 84 | log_debug("circuitid: %s - type: %s - afu %s - size %lu - status: %d\n", 85 | rsp.circuitid, rsp.msgtype, rsp.afu, rsp.size, rsp.status); 86 | 87 | free(msg); 88 | return rsp; 89 | } 90 | -------------------------------------------------------------------------------- /libthymesisflow/src/iport.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /// @file iport.c 18 | #include "iport.h" 19 | 20 | void cycle_ports(iport_list *pl) { 21 | iport_list *plrunner = pl; 22 | while (plrunner) { 23 | log_info("selected port: %d\n", plrunner->nport); 24 | plrunner = plrunner->next; 25 | } 26 | } 27 | 28 | void free_iport_list(iport_list *pl) { 29 | 30 | while (pl) { 31 | iport_list *pl_next = pl->next; 32 | free(pl); 33 | pl = pl_next; 34 | } 35 | } 36 | 37 | iport_list *add_port(iport_list *pl, u_int8_t port_number) { 38 | if (pl == NULL) { 39 | pl = (iport_list *)malloc(sizeof(iport_list)); 40 | pl->nport = port_number; 41 | pl->next = NULL; 42 | return pl; 43 | } 44 | iport_list *new_plhead = (iport_list *)malloc(sizeof(iport_list)); 45 | new_plhead->nport = port_number; 46 | new_plhead->next = pl; 47 | return new_plhead; 48 | } 49 | 50 | u_int8_t marshal_iports(const iport_list *pl) { 51 | 52 | u_int8_t port_rep = 0; 53 | 54 | const iport_list *plrunner = pl; 55 | 56 | while (plrunner) { 57 | port_rep += (1 << (plrunner->nport - 1)); 58 | plrunner = plrunner->next; 59 | } 60 | return port_rep; 61 | } 62 | 63 | iport_list *unmarshal_iports(u_int8_t n) { 64 | 65 | u_int8_t i = 1, pos = 1; 66 | 67 | iport_list *pl = NULL; 68 | 69 | while (i) { 70 | 71 | if (i & n) { 72 | pl = add_port(pl, pos); 73 | } 74 | 75 | // Unset current bit and set the next bit in 'i' 76 | i = i << 1; 77 | 78 | // increment position 79 | ++pos; 80 | } 81 | 82 | return pl; 83 | } 84 | -------------------------------------------------------------------------------- /libthymesisflow/src/stats.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 International Business Machines 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "stats.h" 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | void *_stats_thread(void *arg) { 25 | 26 | uint64_t tx, rx, latency; 27 | t_args *targs = (t_args *)arg; 28 | ocxl_err rc; 29 | 30 | int fd = open(THYMESISCOUNT_PATH, O_WRONLY); 31 | log_info_ext("successfully opened file: %s\n", THYMESISCOUNT_PATH); 32 | 33 | if (targs == NULL) { 34 | log_warn("%s: thread arguments are null\n", __func__); 35 | pthread_exit(NULL); 36 | } 37 | int cs; 38 | 39 | while (1) { 40 | if (pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) { 41 | log_error("error setting pthread_setcancelstate\n"); 42 | exit(EXIT_FAILURE); 43 | } 44 | 45 | rc = ocxl_mmio_read64(targs->global, TX_STATS, OCXL_MMIO_LITTLE_ENDIAN, 46 | &tx); 47 | if (rc != OCXL_OK) { 48 | log_info("Cannot read tx counter, exiting stat thread\n"); 49 | close(fd); 50 | pthread_exit(NULL); 51 | } 52 | rc = ocxl_mmio_read64(targs->global, RX_STATS, OCXL_MMIO_LITTLE_ENDIAN, 53 | &rx); 54 | if (rc != OCXL_OK) { 55 | log_info("Cannot read rx counter, exiting stat thread\n"); 56 | close(fd); 57 | pthread_exit(NULL); 58 | } 59 | 60 | rc = ocxl_mmio_read64(targs->global, LATENCY_STATS, 61 | OCXL_MMIO_LITTLE_ENDIAN, &latency); 62 | if (rc != OCXL_OK) { 63 | log_info("Cannot read latency counter, exiting stat thread\n"); 64 | close(fd); 65 | pthread_exit(NULL); 66 | } 67 | 68 | // latency 64-bit register contains the average 69 | dprintf(fd, "%ld,%ld,%ld\n", tx, rx, 70 | (((1UL << 37) - 1) & latency) / (latency >> 37)); 71 | 72 | cs = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); 73 | if (cs != 0) { 74 | log_error_ext("error setting pthread_setcancelstate\n"); 75 | close(fd); 76 | exit(EXIT_FAILURE); 77 | } 78 | sleep(targs->period); 79 | } 80 | } 81 | 82 | int tf_init_stats_thread(pthread_t *tid, ocxl_mmio_h global, int period) { 83 | 84 | t_args *targs; 85 | 86 | // Checking values 87 | if (tid == NULL) { 88 | fprintf(stderr, "%s: pthread_t is NULL\n", __func__); 89 | return -1; 90 | } 91 | if (global == NULL) { 92 | fprintf(stderr, "%s: global arg is NULL\n", __func__); 93 | return -1; 94 | } 95 | if (period <= 0) { 96 | fprintf(stderr, "%s: period is <= 0 \n", __func__); 97 | return -1; 98 | } 99 | 100 | targs = malloc(sizeof(t_args)); 101 | if (targs == NULL) { 102 | fprintf(stderr, "%s: arg allocation failed\n", __func__); 103 | } 104 | 105 | targs->global = global; 106 | targs->period = period; 107 | 108 | return pthread_create(tid, NULL, _stats_thread, (void *)targs); 109 | } 110 | 111 | int start_stats_thread(pthread_t **tid, ocxl_mmio_h global) { 112 | 113 | unlink(THYMESISCOUNT_PATH); 114 | log_info_ext("opening thymesisstats file: %s\n", THYMESISCOUNT_PATH); 115 | int res = mkfifo(THYMESISCOUNT_PATH, 0666); 116 | if (res != 0) { 117 | log_error_ext("error opening fifo - err: %d\n", res); 118 | // set thread it to null so that we don't try to release it during while 119 | // we delete the connection 120 | *tid = NULL; 121 | return -1; 122 | } 123 | *tid = (pthread_t *)malloc(sizeof(pthread_t)); 124 | 125 | return tf_init_stats_thread(*tid, global, PERIOD_SEC); 126 | } 127 | 128 | int stop_stats_thread(pthread_t *tid) { 129 | 130 | if (tid == NULL) { 131 | log_warn("stat thread reference cannot be null\n"); 132 | return -1; 133 | } 134 | void *res; 135 | 136 | int s = pthread_cancel(*tid); 137 | if (s != 0) { 138 | log_error("error while cancelling stat thread\n"); 139 | exit(EXIT_FAILURE); 140 | } 141 | 142 | /* Join with thread to see what its exit status was */ 143 | s = pthread_join(*tid, &res); 144 | if (s != 0) { 145 | log_error("error while joining the stat thread"); 146 | exit(EXIT_FAILURE); 147 | } 148 | if (res == PTHREAD_CANCELED) { 149 | log_info("stat thread successfully cancelled\n"); 150 | } else 151 | log_error("stat thread wasn't cancelled, this should not happen\n"); 152 | free(tid); 153 | return 0; 154 | } 155 | -------------------------------------------------------------------------------- /reference_design_doc/TLX_3.0_Reference_Design.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenCAPI/ThymesisFlow/f69f7ae0bdc245e4e969e488e118359dde9c8b40/reference_design_doc/TLX_3.0_Reference_Design.pdf -------------------------------------------------------------------------------- /reference_design_doc/ThymesisFlow Design and Deployment v1.1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenCAPI/ThymesisFlow/f69f7ae0bdc245e4e969e488e118359dde9c8b40/reference_design_doc/ThymesisFlow Design and Deployment v1.1.pdf -------------------------------------------------------------------------------- /reference_design_doc/thymesis_arch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenCAPI/ThymesisFlow/f69f7ae0bdc245e4e969e488e118359dde9c8b40/reference_design_doc/thymesis_arch.jpg -------------------------------------------------------------------------------- /scripts/setup_environment.csh: -------------------------------------------------------------------------------- 1 | #!/bin/tcsh 2 | set s=`pwd` 3 | setenv OPENCAPI_BUILD_DIR $s:h 4 | 5 | -------------------------------------------------------------------------------- /scripts/setup_environment.ksh: -------------------------------------------------------------------------------- 1 | #!/bin/ksh 2 | s=`pwd` 3 | export OPENCAPI_BUILD_DIR=${s%/scripts} 4 | 5 | 6 | -------------------------------------------------------------------------------- /tcl/write_bitstream.tcl: -------------------------------------------------------------------------------- 1 | # Creates a bitstream for the current Vivado project run 2 | 3 | set bitstream_dir [get_property DIRECTORY [current_run]] 4 | set bitstream_name [get_property TOP [current_fileset]] 5 | set bitstream_name ${bitstream_name}.bit 6 | 7 | # Sometimes the current_fileset TOP property is empty, so give the bitstream a generic name if this happens 8 | if { $bitstream_name eq ".bit" } { 9 | set bitstream_name "fpga_top${bitstream_name}" 10 | } 11 | 12 | puts "HELP-----" 13 | puts "bitstream_dir: |$bitstream_dir|" 14 | puts "bitstream_name: |$bitstream_name|" 15 | report_property [current_fileset] -all 16 | report_property [current_project] -all 17 | report_property [current_run] -all 18 | 19 | write_bitstream -force ./${bitstream_name} 20 | write_debug_probes -force ./debug_nets.ltx -------------------------------------------------------------------------------- /tcl/write_mcs.tcl: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Creates MCS files for the following OpenCAPI cards 3 | # Usage: ./write_mcs card_name 4 | # card_name can be one of the following options: ad9v3, ad9h7, ad9h3 5 | 6 | # tcl wrapper - next line treated as comment by vivado \ 7 | if [ "$#" -ne 1 ]; then echo "Need to pass 1 command line argument (card eg: ad9v3, ad9h7, ad9h3)"; exit 3; else exec vivado -notrace -nojournal -nolog -source "$0" -mode batch -tclargs "$1"; fi 8 | 9 | set card "" 10 | 11 | if {$::argc == 1} { 12 | set card [string tolower [lindex $::argv 0]] 13 | puts "$card" 14 | } 15 | 16 | set myformat "MCS" 17 | set mysize 64 18 | set myinterface "SPIx8" 19 | 20 | if {$card eq "ad9v3"} { 21 | set mysize 64 22 | 23 | } elseif {$card eq "ad9h7"} { 24 | set mysize 128 25 | } elseif {$card eq "ad9h3"} { 26 | set mysize 64 27 | } else { 28 | puts "Card not recognized" 29 | exit; 30 | } 31 | 32 | set bitstream_name [string trim [lindex [glob *.bit] 0]] 33 | 34 | #remove .bit from name 35 | set bitstream_name [string trimright $bitstream_name ".bit"] 36 | 37 | # Write MCS file for Alphadata 9V3 card's flash memory 38 | write_cfgmem -format $myformat -size $mysize -interface $myinterface -loadbit "up 0x0000000 ./$bitstream_name.bit" ./mcs_files/$bitstream_name.mcs 39 | -------------------------------------------------------------------------------- /tcl/write_proj_config.tcl: -------------------------------------------------------------------------------- 1 | # Writes a hidden config file, so package_bitstreams.pl knows how to 2 | # package the bitstream results 3 | 4 | set origin_dir "." 5 | if { [info exists ::env(OPENCAPI_BUILD_DIR)] } { 6 | set origin_dir $::env(OPENCAPI_BUILD_DIR) 7 | } else { 8 | puts "Need to create environmental variable OPENCAPI_BUILD_DIR" 9 | exit 0 10 | } 11 | 12 | 13 | # AFU TYPE | PHY TYPE | Speed 14 | # config_contents is a global variable created in the create_project* tcl script 15 | set data $::config_contents; 16 | set hidden_dir "$origin_dir/viv_proj/.my_settings"; 17 | set filename "$hidden_dir/config" 18 | 19 | file mkdir $hidden_dir; 20 | 21 | # Open FILE handle 22 | set FILE [open $filename "w"] 23 | 24 | puts $FILE $data 25 | close $FILE 26 | 27 | -------------------------------------------------------------------------------- /tlx/bram_syn_test.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | `timescale 1ns / 1ps 22 | ////////////////////////////////////////////////////////////////////////////////// 23 | // Company: 24 | // Engineer: 25 | // 26 | // Create Date: 11/04/2016 11:59:53 AM 27 | // Design Name: 28 | // Module Name: bram_syn_test 29 | // Project Name: 30 | // Target Devices: 31 | // Tool Versions: 32 | // Description: 33 | // 34 | // Dependencies: 35 | // 36 | // Revision: 37 | // Revision 0.01 - File Created 38 | // Additional Comments: 39 | // 40 | ////////////////////////////////////////////////////////////////////////////////// 41 | 42 | 43 | module bram_syn_test(a, dpra, clk, din, we, qdpo_ce, qdpo, reset_n); 44 | 45 | parameter ADDRESSWIDTH = 6; 46 | parameter BITWIDTH = 1; 47 | parameter DEPTH = 34; 48 | 49 | input clk, we, qdpo_ce; 50 | input reset_n; 51 | input [BITWIDTH-1:0] din; 52 | input [ADDRESSWIDTH-1:0] a, dpra; 53 | 54 | output [BITWIDTH-1:0] qdpo; 55 | 56 | // (* ram_style = "distributed", ARRAY_UPDATE="RW"*) 57 | (* ram_style="block" *) 58 | reg [BITWIDTH-1:0] ram [DEPTH-1:0]; 59 | reg [BITWIDTH-1:0] qdpo_reg; 60 | 61 | always @(posedge clk) begin 62 | if (we) begin ram [a] <= din; end 63 | 64 | if (!reset_n) begin qdpo_reg <= {BITWIDTH{1'b0}}; end 65 | else if (qdpo_ce) begin qdpo_reg <= ram[dpra]; end // read 66 | end 67 | 68 | assign qdpo = qdpo_reg; 69 | 70 | endmodule 71 | -------------------------------------------------------------------------------- /tlx/dram_syn_test.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | `timescale 1ns / 1ps 22 | ////////////////////////////////////////////////////////////////////////////////// 23 | // Company: 24 | // Engineer: 25 | // 26 | // Create Date: 11/04/2016 11:59:53 AM 27 | // Design Name: 28 | // Module Name: dram_syn_test 29 | // Project Name: 30 | // Target Devices: 31 | // Tool Versions: 32 | // Description: 33 | // 34 | // Dependencies: 35 | // 36 | // Revision: 37 | // Revision 0.01 - File Created 38 | // Additional Comments: 39 | // 40 | ////////////////////////////////////////////////////////////////////////////////// 41 | 42 | 43 | module dram_syn_test(a, dpra, clk, din, we, qdpo_ce, qdpo, reset_n); 44 | 45 | parameter ADDRESSWIDTH = 6; 46 | parameter BITWIDTH = 1; 47 | parameter DEPTH = 34; 48 | 49 | input clk, we, qdpo_ce; 50 | input reset_n; 51 | input [BITWIDTH-1:0] din; 52 | input [ADDRESSWIDTH-1:0] a, dpra; 53 | 54 | output [BITWIDTH-1:0] qdpo; 55 | 56 | (* ram_style = "distributed", ARRAY_UPDATE="RW"*) 57 | reg [BITWIDTH-1:0] ram [DEPTH-1:0]; 58 | reg [BITWIDTH-1:0] qdpo_reg; 59 | 60 | always @(posedge clk) begin 61 | if (we) begin ram [a] <= din; end 62 | 63 | if (!reset_n) begin qdpo_reg <= {BITWIDTH{1'b0}}; end 64 | else if (qdpo_ce) begin qdpo_reg <= ram[dpra]; end // read 65 | end 66 | 67 | assign qdpo = qdpo_reg; 68 | 69 | endmodule 70 | -------------------------------------------------------------------------------- /tlx/ocx_leaf_inferd_regfile.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | `timescale 1ns / 10ps 22 | 23 | // ****************************************************************************************************************************** 24 | // File Name : ocx_leaf_inferd_regfile.v 25 | // Project : TLX 0.7x Reference Design (External Transaction Layer logic for attaching to the IBM P9 OpenCAPI Interface) 26 | // Module Name : ocx_leaf_inferd_regfile 27 | // 28 | // Module Description : This logic does the following: 29 | // - 30 | // 31 | // Module Sub-sections: (search for @@@) 32 | // 33 | // ****************************************************************************************************************************** 34 | // Modification History : 35 | // | Version | | Author | Date | Description of change 36 | // | --------- | | -------- | ----------- | --------------------- 37 | `define OCX_LEAF_INFERD_REGFILE_VERSION 19_Jul_2016 // | | Jul.19,2016 | Initial creation 38 | // 39 | // ****************************************************************************************************************************** 40 | 41 | 42 | // ============================================================================================================================== 43 | // @@@ Module Declaration 44 | // ============================================================================================================================== 45 | module ocx_leaf_inferd_regfile 46 | ( 47 | // Port A module ports (This is a WRITE-ONLY port.) 48 | clka , 49 | ena , 50 | addra , 51 | dina , 52 | 53 | // Port B module ports (This is a READ-ONLY port.) 54 | clkb , 55 | rstb_n , 56 | enb , 57 | addrb , 58 | doutb 59 | ) ; 60 | 61 | 62 | // ============================================================================================================================== 63 | // @@@ Parameters (These can be overwritten by module instatiation.) 64 | // ============================================================================================================================== 65 | 66 | parameter REGFILE_DEPTH = 16 ; //positive integer 67 | parameter REGFILE_WIDTH = 576 ; //positive integer 68 | parameter ADDR_WIDTH = 4 ; //positive integer 69 | 70 | 71 | // ============================================================================================================================== 72 | // @@@ Port Declarations 73 | // ============================================================================================================================== 74 | 75 | // Port A module ports (This is a WRITE-ONLY port.) 76 | input clka ; 77 | input ena ; 78 | input [ADDR_WIDTH-1:0] addra ; 79 | input [REGFILE_WIDTH-1:0] dina ; 80 | 81 | // Port B module ports (This is a READ-ONLY port.) 82 | input clkb ; 83 | input rstb_n ; 84 | input enb ; 85 | input [ADDR_WIDTH-1:0] addrb ; 86 | output [REGFILE_WIDTH-1:0] doutb ; 87 | 88 | 89 | 90 | // ============================================================================================================================== 91 | // @@@ Wires and Variables (Regs) 92 | // ============================================================================================================================== 93 | 94 | (* RAM_STYLE="DISTRIBUTED" *) 95 | reg [REGFILE_WIDTH-1:0] regfile [REGFILE_DEPTH-1:0] ; 96 | reg [REGFILE_WIDTH-1:0] output_reg ; 97 | 98 | 99 | // ============================================================================================================================== 100 | // @@@ ocx_leaf_inferd_regfile Logic 101 | // ============================================================================================================================== 102 | 103 | // Regfile memory array 104 | always @ (posedge clka) begin 105 | if(ena) begin regfile[addra] <= dina; end 106 | end 107 | 108 | // Output register 109 | always @ (posedge clkb) begin 110 | if (!rstb_n) begin output_reg <= {REGFILE_WIDTH{1'b0}}; end 111 | else if (enb) begin output_reg <= regfile[addrb]; end 112 | end 113 | 114 | assign doutb = output_reg ; 115 | 116 | 117 | endmodule // ocx_leaf_inferd_regfile 118 | -------------------------------------------------------------------------------- /tlx/ocx_tlx_cfg_mac.v: -------------------------------------------------------------------------------- 1 | // *!*************************************************************************** 2 | // *! Copyright 2019 International Business Machines 3 | // *! 4 | // *! Licensed under the Apache License, Version 2.0 (the "License"); 5 | // *! you may not use this file except in compliance with the License. 6 | // *! You may obtain a copy of the License at 7 | // *! http://www.apache.org/licenses/LICENSE-2.0 8 | // *! 9 | // *! The patent license granted to you in Section 3 of the License, as applied 10 | // *! to the "Work," hereby includes implementations of the Work in physical form. 11 | // *! 12 | // *! Unless required by applicable law or agreed to in writing, the reference design 13 | // *! distributed under the License is distributed on an "AS IS" BASIS, 14 | // *! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | // *! See the License for the specific language governing permissions and 16 | // *! limitations under the License. 17 | // *! 18 | // *! The background Specification upon which this is based is managed by and available from 19 | // *! the OpenCAPI Consortium. More information can be found at https://opencapi.org. 20 | // *!*************************************************************************** 21 | `timescale 1ns / 1ps 22 | 23 | ////////////////////////////////////////////////////////////////////////////////// 24 | // Company: 25 | // Engineer: 26 | // 27 | // Create Date: 04/11/2017 11:54:54 AM 28 | // Design Name: 29 | // Module Name: ocx_tlx_cfg_mac 30 | // Project Name: 31 | // Target Devices: 32 | // Tool Versions: 33 | // Description: 34 | // 35 | // Dependencies: 36 | // 37 | // Revision: 38 | // Revision 0.01 - File Created 39 | // Additional Comments: 40 | // 41 | ////////////////////////////////////////////////////////////////////////////////// 42 | 43 | 44 | module ocx_tlx_cfg_mac 45 | #(parameter addr_width = 6) 46 | ( 47 | input tlx_clk, 48 | input reset_n, 49 | input crc_error, 50 | input cfg_data_v, 51 | input [31:0] cfg_data_bus, 52 | input cfg_rd_ena, 53 | output [31:0] tlx_cfg_data_bus, 54 | output rcv_xmt_tl_crd_cfg_dcp1_valid, 55 | input good_crc, 56 | input crc_flush_inprog, 57 | input crc_flush_done 58 | ); 59 | wire [addr_width:0] fifo_wr_ptr_din,fifo_rd_ptr_din,fifo_wr_verif_ptr_din; 60 | reg [addr_width:0] fifo_wr_ptr_dout,fifo_rd_ptr_dout,fifo_wr_verif_ptr_dout; 61 | wire crc_error_din; 62 | reg crc_error_dout; 63 | wire crc_flush_inprog_din; 64 | reg crc_flush_inprog_dout; 65 | reg crc_flush_done_s1_dout; 66 | reg crc_flush_done_s2_dout; 67 | reg crc_flush_inprog_s1_dout; 68 | reg crc_flush_inprog_s2_dout; 69 | reg good_crc_s1_dout; 70 | reg good_crc_s2_dout; 71 | reg crc_error_s2_reg; 72 | always @(posedge tlx_clk) 73 | begin 74 | if (!reset_n) 75 | begin 76 | fifo_wr_ptr_dout <= {addr_width+1{1'b0}}; 77 | fifo_wr_verif_ptr_dout <= {addr_width+1{1'b0}}; 78 | fifo_rd_ptr_dout <= {addr_width+1{1'b0}}; 79 | crc_error_dout <= 1'b0; 80 | crc_flush_inprog_dout <= 1'b0; 81 | crc_flush_done_s1_dout <= 1'b0; 82 | crc_flush_done_s2_dout <= 1'b0; 83 | crc_flush_inprog_s1_dout <= 1'b0; 84 | crc_flush_inprog_s2_dout <= 1'b0; 85 | good_crc_s1_dout <= 1'b0; 86 | good_crc_s2_dout <= 1'b0; 87 | end 88 | else 89 | begin 90 | fifo_wr_ptr_dout <= fifo_wr_ptr_din; 91 | fifo_wr_verif_ptr_dout <= fifo_wr_verif_ptr_din; 92 | fifo_rd_ptr_dout <= fifo_rd_ptr_din; 93 | crc_error_dout <= crc_error_din; 94 | crc_error_s2_reg <= crc_error_dout; 95 | crc_flush_inprog_dout <= crc_flush_inprog_din; 96 | crc_flush_done_s1_dout <= crc_flush_done; 97 | crc_flush_done_s2_dout <= crc_flush_done_s1_dout; 98 | crc_flush_inprog_s1_dout <= crc_flush_inprog_dout; 99 | crc_flush_inprog_s2_dout <= crc_flush_inprog_s1_dout; 100 | good_crc_s1_dout <= good_crc; 101 | good_crc_s2_dout <= good_crc_s1_dout; 102 | end 103 | end 104 | assign rcv_xmt_tl_crd_cfg_dcp1_valid = cfg_rd_ena; 105 | assign crc_flush_inprog_din = crc_flush_inprog; 106 | assign crc_error_din = crc_error; 107 | assign fifo_wr_ptr_din = (crc_flush_done_s2_dout || (~crc_flush_inprog_s1_dout && crc_error_s2_reg && ~good_crc_s2_dout)) ? fifo_wr_verif_ptr_dout : 108 | cfg_data_v ? fifo_wr_ptr_dout + {{addr_width{1'b0}}, 1'b1} : fifo_wr_ptr_dout; 109 | 110 | assign fifo_wr_verif_ptr_din = (good_crc_s2_dout && cfg_data_v) ? fifo_wr_ptr_din : 111 | (good_crc_s2_dout || crc_flush_inprog_s2_dout) ? fifo_wr_ptr_dout : fifo_wr_verif_ptr_dout; 112 | assign fifo_rd_ptr_din = cfg_rd_ena ? fifo_rd_ptr_dout + {{addr_width{1'b0}}, 1'b1} : fifo_rd_ptr_dout; 113 | dram_syn_test #( 114 | .ADDRESSWIDTH(addr_width), 115 | .BITWIDTH(32), 116 | .DEPTH((2**addr_width)) 117 | ) CFG_DATA_FIFO ( 118 | .a(fifo_wr_ptr_dout[addr_width-1:0]), 119 | .dpra(fifo_rd_ptr_dout[addr_width-1:0]), 120 | .clk(tlx_clk), 121 | .din(cfg_data_bus), 122 | .we(cfg_data_v), 123 | .reset_n(reset_n), 124 | .qdpo_ce(cfg_rd_ena), 125 | .qdpo(tlx_cfg_data_bus) 126 | ); 127 | endmodule 128 | --------------------------------------------------------------------------------