├── .clang-format ├── .gitignore ├── CMakeLists.txt ├── Readme.md ├── assets ├── kshark.png ├── ksharkzoom.png ├── menuconfig │ ├── 1.1.png │ ├── 1.2.png │ ├── 1.3.png │ ├── 1.4.png │ ├── 2.2.png │ ├── 2.3.png │ ├── 3.2.png │ ├── 3.3.png │ ├── 4.1.png │ ├── 4.2.png │ ├── 4.3.png │ ├── 4.4.png │ └── gifs │ │ ├── cpu-gov.gif │ │ ├── prempt-model.gif │ │ ├── timer-freq.gif │ │ └── timer-tick.gif └── plot.png ├── easy_prempt.sh ├── examples ├── CMakeLists.txt ├── cpu_affinity.cpp ├── cpu_affinity_array.cpp ├── periodic_thread.cpp ├── rt_thread.cpp └── thread_priority.cpp ├── include └── ThreadUtils │ ├── ThreadWrapper.h │ ├── TimeStampLogger.h │ └── timer_math.h ├── instruction.md ├── raspberry-pi-rt.md ├── scripts └── plot.py └── src └── ThreadUtils └── ThreadWrapper.cpp /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: WebKit 4 | AccessModifierOffset: -4 5 | AlignAfterOpenBracket: DontAlign 6 | AlignConsecutiveMacros: false 7 | AlignConsecutiveAssignments: false 8 | AlignConsecutiveDeclarations: false 9 | AlignEscapedNewlines: Right 10 | AlignOperands: false 11 | AlignTrailingComments: false 12 | AllowAllArgumentsOnNextLine: true 13 | AllowAllConstructorInitializersOnNextLine: true 14 | AllowAllParametersOfDeclarationOnNextLine: true 15 | AllowShortBlocksOnASingleLine: Empty 16 | AllowShortCaseLabelsOnASingleLine: false 17 | AllowShortFunctionsOnASingleLine: All 18 | AllowShortLambdasOnASingleLine: All 19 | AllowShortIfStatementsOnASingleLine: Never 20 | AllowShortLoopsOnASingleLine: false 21 | AlwaysBreakAfterDefinitionReturnType: None 22 | AlwaysBreakAfterReturnType: None 23 | AlwaysBreakBeforeMultilineStrings: false 24 | AlwaysBreakTemplateDeclarations: MultiLine 25 | BinPackArguments: true 26 | BinPackParameters: true 27 | BraceWrapping: 28 | AfterCaseLabel: false 29 | AfterClass: false 30 | AfterControlStatement: false 31 | AfterEnum: false 32 | AfterFunction: true 33 | AfterNamespace: false 34 | AfterObjCDeclaration: false 35 | AfterStruct: false 36 | AfterUnion: false 37 | AfterExternBlock: false 38 | BeforeCatch: false 39 | BeforeElse: false 40 | IndentBraces: false 41 | SplitEmptyFunction: true 42 | SplitEmptyRecord: true 43 | SplitEmptyNamespace: true 44 | BreakBeforeBinaryOperators: All 45 | BreakBeforeBraces: WebKit 46 | BreakBeforeInheritanceComma: false 47 | BreakInheritanceList: BeforeColon 48 | BreakBeforeTernaryOperators: true 49 | BreakConstructorInitializersBeforeComma: false 50 | BreakConstructorInitializers: BeforeComma 51 | BreakAfterJavaFieldAnnotations: false 52 | BreakStringLiterals: true 53 | ColumnLimit: 0 54 | CommentPragmas: '^ IWYU pragma:' 55 | CompactNamespaces: false 56 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 57 | ConstructorInitializerIndentWidth: 4 58 | ContinuationIndentWidth: 4 59 | Cpp11BracedListStyle: false 60 | DeriveLineEnding: true 61 | DerivePointerAlignment: false 62 | DisableFormat: false 63 | ExperimentalAutoDetectBinPacking: false 64 | FixNamespaceComments: false 65 | ForEachMacros: 66 | - foreach 67 | - Q_FOREACH 68 | - BOOST_FOREACH 69 | IncludeBlocks: Preserve 70 | IncludeCategories: 71 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 72 | Priority: 2 73 | SortPriority: 0 74 | - Regex: '^(<|"(gtest|gmock|isl|json)/)' 75 | Priority: 3 76 | SortPriority: 0 77 | - Regex: '.*' 78 | Priority: 1 79 | SortPriority: 0 80 | IncludeIsMainRegex: '(Test)?$' 81 | IncludeIsMainSourceRegex: '' 82 | IndentCaseLabels: false 83 | IndentGotoLabels: true 84 | IndentPPDirectives: None 85 | IndentWidth: 4 86 | IndentWrappedFunctionNames: false 87 | JavaScriptQuotes: Leave 88 | JavaScriptWrapImports: true 89 | KeepEmptyLinesAtTheStartOfBlocks: true 90 | MacroBlockBegin: '' 91 | MacroBlockEnd: '' 92 | MaxEmptyLinesToKeep: 1 93 | NamespaceIndentation: Inner 94 | ObjCBinPackProtocolList: Auto 95 | ObjCBlockIndentWidth: 4 96 | ObjCSpaceAfterProperty: true 97 | ObjCSpaceBeforeProtocolList: true 98 | PenaltyBreakAssignment: 2 99 | PenaltyBreakBeforeFirstCallParameter: 19 100 | PenaltyBreakComment: 300 101 | PenaltyBreakFirstLessLess: 120 102 | PenaltyBreakString: 1000 103 | PenaltyBreakTemplateDeclaration: 10 104 | PenaltyExcessCharacter: 1000000 105 | PenaltyReturnTypeOnItsOwnLine: 60 106 | PointerAlignment: Left 107 | ReflowComments: true 108 | SortIncludes: true 109 | SortUsingDeclarations: true 110 | SpaceAfterCStyleCast: false 111 | SpaceAfterLogicalNot: false 112 | SpaceAfterTemplateKeyword: true 113 | SpaceBeforeAssignmentOperators: true 114 | SpaceBeforeCpp11BracedList: true 115 | SpaceBeforeCtorInitializerColon: true 116 | SpaceBeforeInheritanceColon: true 117 | SpaceBeforeParens: ControlStatements 118 | SpaceBeforeRangeBasedForLoopColon: true 119 | SpaceInEmptyBlock: true 120 | SpaceInEmptyParentheses: false 121 | SpacesBeforeTrailingComments: 1 122 | SpacesInAngles: false 123 | SpacesInConditionalStatement: false 124 | SpacesInContainerLiterals: true 125 | SpacesInCStyleCastParentheses: false 126 | SpacesInParentheses: false 127 | SpacesInSquareBrackets: false 128 | SpaceBeforeSquareBrackets: false 129 | Standard: Latest 130 | StatementMacros: 131 | - Q_UNUSED 132 | - QT_REQUIRE_VERSION 133 | TabWidth: 8 134 | UseCRLF: false 135 | UseTab: Never 136 | ... 137 | 138 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | CMakeLists.txt.user 2 | CMakeCache.txt 3 | CMakeFiles 4 | CMakeScripts 5 | Testing 6 | Makefile 7 | cmake_install.cmake 8 | install_manifest.txt 9 | compile_commands.json 10 | CTestTestfile.cmake 11 | _deps 12 | 13 | /bin/ 14 | /build/ 15 | /.vscode/ -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14) 2 | project(RT-THREAD) 3 | 4 | set(CMAKE_CXX_STANDARD 11) 5 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 6 | 7 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/ ) 8 | set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/bin/ ) 9 | set(CMAKE_ROOT_PATH ${PROJECT_SOURCE_DIR}) 10 | 11 | if (WIN32) 12 | SET( CMAKE_EXECUTABLE_OUTPUT_PATH_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 13 | SET( CMAKE_EXECUTABLE_OUTPUT_PATH_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 14 | SET( CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 15 | SET( CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 16 | SET( CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 17 | SET( CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 18 | SET( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_DEBUG ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 19 | SET( CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) 20 | endif() 21 | 22 | execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory 23 | ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/logs) 24 | execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink 25 | ${CMAKE_CURRENT_SOURCE_DIR}/scripts/plot.py 26 | ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/plot.py) 27 | 28 | include_directories(include) 29 | 30 | add_library(ThreadWrapper src/ThreadUtils/ThreadWrapper.cpp) 31 | target_link_libraries(ThreadWrapper pthread) 32 | 33 | add_subdirectory(examples) -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Real Time Threads with PREEMPT_RT 2 | Example implementaion of real time threads with wrapper for POSIX thread. 3 | 4 | # Prerequisite 5 | ⚠️ NOTE: Make sure you have already patched `PREEMPT_RT` kernel, follow [instruction.md](instruction.md) to patch and install. To patch Raspberry PI 4 kernel with `PREEMPT_RT` follow instructions from [raspberry-pi-rt.md](raspberry-pi-rt.md) 6 | 7 | # Building real time periodic thread 8 | Download the source code and compile 9 | ```console 10 | git clone https://github.com/siddharthdeore/preempt.git 11 | cd preempt 12 | mkdir build && cd build 13 | cmake .. 14 | make 15 | ``` 16 | 17 | Execute example program with PREEMPT_RT 18 | ```console 19 | cd ../bin 20 | sudo ./rt_thread 21 | ``` 22 | Test CPU affinity (pin thread to core) 23 | ```console 24 | ./cpu_affinity 25 | ``` 26 | To check CPU affinity use [htop](https://en.wikipedia.org/wiki/Htop) system monitor, cores which have thread pinned succussfully shall show 100% CPU consumption. 27 | Or use `taskset` to check on which CPU core a process running 28 | ``` 29 | taskset -c -p $(pidof cpu_affinity) 30 | ``` 31 | 32 | # Tracing and diagnostic 33 | install kernel tracer tools trace-cmd and KernelShark 34 | ```console 35 | sudo apt-get install -y trace-cmd 36 | sudo apt-get install -y kernelshark 37 | ``` 38 | Trace CPU `sched_switch` event 39 | ```console 40 | sudo trace-cmd record -e sched_switch ./rt_thread 41 | ``` 42 | Analize CPU trace with KernelShark GUI 43 | 44 | ```console 45 | kernelshark trace.dat 46 | ``` 47 |
48 | 49 | 50 | Fig: Kernelshark CPU trace for sched_switch event 51 | 52 | 53 | 54 | Fig: Kernelshark sched_switch event ticks (zoom). 55 |
56 | 57 | # Troubleshoot 58 | 59 | Issue: Unexpected freeze issue on COM Express Type 6 60 | 61 | Description: 62 | 63 | COM Express running Ubuntu 20.04 LTS freezes unexpectedly, this recurrent behavior has been noticed when processor usage goes close to 100% 64 | similar issue arise on when program/threads uses all processor cores, and when real time thread is affine to specific core. 65 | 66 | Solution: 67 | 68 | This behaviour obsereved with Ubuntu 18.04 LTS and 22.04 LTS and expected cause of such freezes is related to "CPU Power Management". 69 | Disabling all CPU power managements seems to have solved problem. 70 | Steps to Disable 71 | 72 | Press "F2" or "DEL" key to enter Aptio Bios Setup utility. 73 | Select Advanced Tab 74 | ```console 75 | > Advanced 76 | > Power & Performance 77 | > CPU - Power Management Control 78 | Set Boot mode [Turbo Performance] 79 | Intel(R) SpeedStep(tm) [Disable] 80 | Race To Halt (RTH) [Disable] 81 | Intel(R) Speed Shift Technology [Disable] 82 | HDC Control [Disable] 83 | ``` 84 | 85 | ⚠️ Important (Only for Xenomai): Problems on [Xenomai](https://source.denx.de/Xenomai/xenomai/-/wikis/home) ipipe kernel CPU affinity 86 |

87 | Current kernel `cmdline` flags can be checked with following command, 88 | 89 | ```console 90 | cat /proc/cmdline 91 | ``` 92 | To disable DWC features add following flags to kernel cmdline 93 | ```console 94 | dwc_otg.fiq_enable=0 dwc_otg.fiq_fsm_enable=0 dwc_otg.nak_holdoff=0 95 | ``` 96 | ## CPU Isolation (Xenomai) 97 | Remove the CPU 0 and CPU 1 from the general kernel SMP balance and scheduler algorithms. 98 | 99 | ```console 100 | isolcpus=0,1 xenomai.supported_cpus=0x3 101 | # above line isolates cpu 0 and 1 from SMP balance, and set xenomai supported cpu mask to 0011 102 | ``` 103 | To set flags edit `/etc/default/grub` and add flags to GRUB_CMDLINE_LINUX_DEFAULT, finaly update grub with `sudo update-grub` 104 | 105 | 106 | RPI : flags can be set by adding above lines to `/boot/cmdline.txt`, 107 |

108 | 109 | 110 | # TODO 111 | - ~~realtime periodic thread Program~~ 112 | - ~~cpu affinity~~ 113 | - ~~thread priority~~ 114 | 115 | - Some of the tools listed below to be tested and integrated 116 | - Linux Trace Toolkit 117 | - Dynamic Probes 118 | - kGDB (kernel debugger) 119 | - Linux Kernel Crash Dump 120 | - Linux Test Project 121 | - LMBench 122 | - Ballista 123 | - strace 124 | - System Call Tracker 125 | 126 | # Maintainers 127 | This repository is maintained by: 128 | 129 | | | [Siddharth Deore](https://github.com/siddharthdeore) | 130 | |--|--| 131 | -------------------------------------------------------------------------------- /assets/kshark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/kshark.png -------------------------------------------------------------------------------- /assets/ksharkzoom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/ksharkzoom.png -------------------------------------------------------------------------------- /assets/menuconfig/1.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/1.1.png -------------------------------------------------------------------------------- /assets/menuconfig/1.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/1.2.png -------------------------------------------------------------------------------- /assets/menuconfig/1.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/1.3.png -------------------------------------------------------------------------------- /assets/menuconfig/1.4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/1.4.png -------------------------------------------------------------------------------- /assets/menuconfig/2.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/2.2.png -------------------------------------------------------------------------------- /assets/menuconfig/2.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/2.3.png -------------------------------------------------------------------------------- /assets/menuconfig/3.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/3.2.png -------------------------------------------------------------------------------- /assets/menuconfig/3.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/3.3.png -------------------------------------------------------------------------------- /assets/menuconfig/4.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/4.1.png -------------------------------------------------------------------------------- /assets/menuconfig/4.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/4.2.png -------------------------------------------------------------------------------- /assets/menuconfig/4.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/4.3.png -------------------------------------------------------------------------------- /assets/menuconfig/4.4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/4.4.png -------------------------------------------------------------------------------- /assets/menuconfig/gifs/cpu-gov.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/gifs/cpu-gov.gif -------------------------------------------------------------------------------- /assets/menuconfig/gifs/prempt-model.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/gifs/prempt-model.gif -------------------------------------------------------------------------------- /assets/menuconfig/gifs/timer-freq.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/gifs/timer-freq.gif -------------------------------------------------------------------------------- /assets/menuconfig/gifs/timer-tick.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/menuconfig/gifs/timer-tick.gif -------------------------------------------------------------------------------- /assets/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siddharthdeore/preempt/20e3bf456844acff551da290eb45414251016bb1/assets/plot.png -------------------------------------------------------------------------------- /easy_prempt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Script prepares system to build system, downloads most suitable RT patch and kernel source, 3 | # and patches the kernel. 4 | # author : siddharth deore 5 | 6 | NC='\033[0m' # No Color 7 | 8 | # Regular Colors 9 | Black='\033[0;30m' # Black 10 | Red='\033[0;31m' # Red 11 | Green='\033[0;32m' # Green 12 | Yellow='\033[0;33m' # Yellow 13 | Blue='\033[0;34m' # Blue 14 | Purple='\033[0;35m' # Purple 15 | Cyan='\033[0;36m' # Cyan 16 | White='\033[0;37m' # White 17 | 18 | # Bold 19 | BBlack='\033[1;30m' # Black 20 | BRed='\033[1;31m' # Red 21 | BGreen='\033[1;32m' # Green 22 | BYellow='\033[1;33m' # Yellow 23 | BBlue='\033[1;34m' # Blue 24 | BPurple='\033[1;35m' # Purple 25 | BCyan='\033[1;36m' # Cyan 26 | BWhite='\033[1;37m' # White 27 | 28 | # Underline 29 | UBlack='\033[4;30m' # Black 30 | URed='\033[4;31m' # Red 31 | UGreen='\033[4;32m' # Green 32 | UYellow='\033[4;33m' # Yellow 33 | UBlue='\033[4;34m' # Blue 34 | UPurple='\033[4;35m' # Purple 35 | UCyan='\033[4;36m' # Cyan 36 | UWhite='\033[4;37m' # White 37 | 38 | # Background 39 | On_Black='\033[40m' # Black 40 | On_Red='\033[41m' # Red 41 | On_Green='\033[42m' # Green 42 | On_Yellow='\033[43m' # Yellow 43 | On_Blue='\033[44m' # Blue 44 | On_Purple='\033[45m' # Purple 45 | On_Cyan='\033[46m' # Cyan 46 | On_White='\033[47m' # White 47 | 48 | # High Intensity 49 | IBlack='\033[0;90m' # Black 50 | IRed='\033[0;91m' # Red 51 | IGreen='\033[0;92m' # Green 52 | IYellow='\033[0;93m' # Yellow 53 | IBlue='\033[0;94m' # Blue 54 | IPurple='\033[0;95m' # Purple 55 | ICyan='\033[0;96m' # Cyan 56 | IWhite='\033[0;97m' # White 57 | 58 | # Bold High Intensity 59 | BIBlack='\033[1;90m' # Black 60 | BIRed='\033[1;91m' # Red 61 | BIGreen='\033[1;92m' # Green 62 | BIYellow='\033[1;93m' # Yellow 63 | BIBlue='\033[1;94m' # Blue 64 | BIPurple='\033[1;95m' # Purple 65 | BICyan='\033[1;96m' # Cyan 66 | BIWhite='\033[1;97m' # White 67 | 68 | # High Intensity backgrounds 69 | On_IBlack='\033[0;100m' # Black 70 | On_IRed='\033[0;101m' # Red 71 | On_IGreen='\033[0;102m' # Green 72 | On_IYellow='\033[0;103m' # Yellow 73 | On_IBlue='\033[0;104m' # Blue 74 | On_IPurple='\033[0;105m' # Purple 75 | On_ICyan='\033[0;106m' # Cyan 76 | On_IWhite='\033[0;107m' # White 77 | 78 | # kernel version 79 | VER=`uname -r` 80 | MAJ=`uname -r | cut -d '.' -f1` 81 | MIN=`uname -r | cut -d '.' -f2` 82 | REV=`uname -r | cut -d '.' -f3` 83 | 84 | # Install Prerequisites 85 | # download kernel requirements 86 | # sudo apt-get install -y lftp 87 | # menuconfig GUI requirements 88 | # building and compiling requirements 89 | 90 | echo ${info} ${Red}"Installing Prerequisites"${NC} 91 | 92 | sudo apt-get install build-essential 93 | sudo apt-get install -y fakeroot libnuma-dev 94 | sudo apt install -y kernel-package libncurses5 libncurses5-dev libncurses-dev qtbase5-dev-tools flex 95 | sudo apt install -y bison openssl libssl-dev dkms libelf-dev libudev-dev libpci-dev libiberty-dev autoconf 96 | sudo apt install -y dwarves 97 | sudo apt install -y zstd 98 | 99 | sleep 0.1 100 | 101 | # kernel source code path 102 | if [ "$MAJ" -gt "2" ]; then 103 | KER_PATH="https://cdn.kernel.org/pub/linux/kernel/v"$MAJ.x/ 104 | else 105 | KER_PATH="https://cdn.kernel.org/pub/linux/kernel/v"$MAJ.$MIN/ 106 | fi 107 | # RT patch code path 108 | RTK_PATH="https://cdn.kernel.org/pub/linux/kernel/projects/rt/"$MAJ.$MIN 109 | 110 | info="${Green}[ INFO ]${NC}" 111 | echo ${info} "Searching for kernel patch suitable for " ${BRed}$VER${NC}"\n" 112 | 113 | PATCH_COMPRESSED_FILE_NAME=`lftp -e "cls -1 *.patch.gz; exit" ${RTK_PATH}` 114 | echo "\n"${info} Found patch ${Green}$PATCH_COMPRESSED_FILE_NAME${NC} 115 | PATCH_URL=${RTK_PATH}/${PATCH_COMPRESSED_FILE_NAME} 116 | echo ${info} Downloading patch ${UYellow}${PATCH_URL}${NC} 117 | ################################################################# 118 | # downlad patch 119 | wget ${PATCH_URL} 120 | # extract patch 121 | gunzip ${PATCH_COMPRESSED_FILE_NAME} 122 | PATCH_NAME=`echo ${PATCH_COMPRESSED_FILE_NAME%.gz}` 123 | ################################################################# 124 | 125 | echo ${info} "Searching for kernel source suitable for " ${BRed}$VER${NC}"\n" 126 | # get list of available kernel sources 127 | KERN_FILE_NAME=`lftp -e "set ftp:ssl-allow no; cls -1 linux-${MAJ}.${MIN}.*.tar.gz; exit" ${KER_PATH}` 128 | KERN_FILE_NAME=`echo ${KERN_FILE_NAME} | cut -d ' ' -f1` 129 | echo "\n"${info} Found kernel source ${Green} ${KERN_FILE_NAME} ${NC} 130 | SOURCE_URL=${KER_PATH}${KERN_FILE_NAME} 131 | echo ${info} Downloading kernel ${UYellow}${SOURCE_URL}${NC} 132 | ################################################################# 133 | # download kernel source 134 | wget ${SOURCE_URL} 135 | # extract source 136 | tar -xzf ${KERN_FILE_NAME} 137 | ################################################################# 138 | KERN_DIR_NAME=`echo ${KERN_FILE_NAME%.tar.gz}` 139 | # change directory 140 | cd ${KERN_DIR_NAME} 141 | ################################################################# 142 | # patch kernel 143 | patch -p1 < ../${PATCH_NAME} 144 | ################################################################# 145 | # copy current configration settings 146 | cp /boot/config-$(uname -r) .config 147 | yes '' | make oldconfig 148 | make menuconfig 149 | 150 | scripts/config --set-str CONFIG_SYSTEM_REVOCATION_KEYS "" 151 | scripts/config --set-str CONFIG_SYSTEM_TRUSTED_KEYS "" 152 | 153 | echo "\n"${info} ${BICyan}"now you can make kernel\n"${NC} 154 | # scripts/config --set-val CONFIG_HAVE_PREEMPT_LAZY y 155 | # scripts/config --set-val CONFIG_PREEMPT_LAZY y 156 | # scripts/config -d CONFIG_PREEMPT_VOLUNTARY 157 | 158 | # sudo make modules_install -j$(nproc) 159 | # sudo make install -j$(nproc) 160 | # sudo update-grub 161 | # sudo reboot 162 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.14) 2 | project(ThreadExamples) 3 | 4 | include_directories(../include) 5 | 6 | add_executable(cpu_affinity cpu_affinity.cpp ) 7 | target_link_libraries(cpu_affinity ThreadWrapper) 8 | 9 | add_executable(cpu_affinity_array cpu_affinity_array.cpp ) 10 | target_link_libraries(cpu_affinity_array ThreadWrapper) 11 | 12 | add_executable(thread_priority thread_priority.cpp ) 13 | target_link_libraries(thread_priority ThreadWrapper) 14 | 15 | add_executable(rt_thread rt_thread.cpp ) 16 | target_link_libraries(rt_thread ThreadWrapper) 17 | 18 | add_executable(periodic_thread periodic_thread.cpp ) 19 | target_link_libraries(periodic_thread ThreadWrapper) -------------------------------------------------------------------------------- /examples/cpu_affinity.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | // some cpu intensive callable function 4 | void* callback_function() 5 | { 6 | float x = 1.5f; 7 | 8 | while (1) { 9 | x *= sin(x) / atan(x) * tanh(x) * sqrt(x); 10 | usleep(1); 11 | } 12 | } 13 | int main(int argc, char const* argv[]) 14 | { 15 | ThreadWrapper th1(callback_function); 16 | 17 | // pin thread to CPU core 0 18 | th1.setAffinity(0); 19 | 20 | ThreadWrapper th2(callback_function); 21 | 22 | // pin thread to CPU core 1 23 | th2.setAffinity(1); 24 | 25 | // wait for threads to finish 26 | th1.join(); 27 | th2.join(); 28 | 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /examples/cpu_affinity_array.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | // some cpu intensive callable function 4 | void* callback_function(void* arg) 5 | { 6 | float x = 1.5f; 7 | 8 | while (1) { 9 | x *= sin(x) / atan(x) * tanh(x) * sqrt(x); 10 | } 11 | } 12 | int main(int argc, char const* argv[]) 13 | { 14 | int count = 1; 15 | if(argc>1){ 16 | count = atoi(argv[1]); 17 | } 18 | ThreadWrapper th[count]; 19 | int x; 20 | for (size_t i = 0; i < count; i++) 21 | { 22 | ThreadWrapper temp(callback_function,&x); 23 | th[i] = temp; 24 | // pin thread to CPU core i 25 | th[i].setAffinity(i); 26 | 27 | } 28 | 29 | // wait for threads to finish 30 | for (size_t i = 0; i < count; i++) 31 | { 32 | th[i].join(); 33 | } 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /examples/periodic_thread.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | void* non_real_time_function(void* arg); 7 | void* real_time_function(void* arg); 8 | 9 | #include 10 | #include 11 | 12 | static std::atomic done(false); // signal flag 13 | static std::atomic x(10); 14 | std::mutex mtx; 15 | int main(int argc, char const* argv[]) 16 | { 17 | // Ctrl + C signal handler with lambda 18 | signal(SIGINT, [](int sig_num) { done = true; }); 19 | 20 | struct timespec _rqtp; 21 | clock_gettime(CLOCK_REALTIME, &_rqtp); 22 | _rqtp.tv_nsec = 0; 23 | _rqtp.tv_sec += 1; 24 | clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &_rqtp, NULL); 25 | 26 | period_info info_rt; 27 | int rate = 1000; // 1 kHz 28 | periodic_timer_init(&info_rt, 1e9 / (rate)); 29 | 30 | ThreadWrapper rt_thread1k(real_time_function, &info_rt); 31 | rt_thread1k.setPriority(SCHED_FIFO, 99); 32 | rt_thread1k.setAffinity(1); // pin to core 0 33 | 34 | period_info info_nrt; 35 | rate = 5000; // 5 kHz 36 | periodic_timer_init(&info_nrt, 1e9 / (rate)); 37 | 38 | ThreadWrapper rt_thread5k(non_real_time_function, &info_nrt); 39 | rt_thread5k.setPriority(SCHED_FIFO, 99); 40 | rt_thread5k.setAffinity(0); // pin to core 0 41 | 42 | if (!done.is_lock_free()) { 43 | return 10; // error 44 | } 45 | 46 | rt_thread1k.join(); 47 | rt_thread5k.join(); 48 | 49 | /* code */ 50 | return 0; 51 | } 52 | 53 | /** 54 | * @brief Real Time callback function handle 55 | * 56 | * @param arg 57 | * @return void* 58 | */ 59 | void* real_time_function(void* arg) 60 | { 61 | // initalize time stamp looger file 62 | TimeStampLogger t_stamp_log("logs/rt1k.txt"); 63 | period_info* _p_info = (period_info*)arg; 64 | 65 | // static double x; 66 | 67 | period_info info_rt; 68 | 69 | // syncronize time 70 | clock_gettime(CLOCK_MONOTONIC, &(_p_info->next_deadline)); 71 | //_p_info->next_deadline.tv_nsec = 1000000000 - _p_info->next_deadline.tv_nsec; 72 | _p_info->next_deadline.tv_nsec = 0; 73 | _p_info->next_deadline.tv_sec += 1; 74 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &_p_info->next_deadline, NULL); 75 | 76 | while (!done) { 77 | t_stamp_log.stamp(); 78 | /* Code to be called periodicaly */ 79 | // std::cout << "RT Thread " << x++ << "\n"; 80 | mtx.lock(); 81 | x = x - 5; 82 | mtx.unlock(); 83 | // give other threads some cpu time 84 | wait_rest_of_period(_p_info); 85 | } 86 | 87 | std::lock_guard guard(mtx); 88 | std::cout << "x in rt is " << x << std::endl; 89 | return 0; 90 | } 91 | 92 | /** 93 | * @brief Non real time function 94 | * 95 | * @param arg 96 | * @return void* 97 | */ 98 | void* non_real_time_function(void* arg) 99 | { 100 | TimeStampLogger t_stamp_log("logs/rt5k.txt"); 101 | period_info* _p_info = (period_info*)arg; 102 | 103 | // static double x; 104 | 105 | // syncronize time 106 | clock_gettime(CLOCK_MONOTONIC, &(_p_info->next_deadline)); 107 | //_p_info->next_deadline.tv_nsec = 1000000000 - _p_info->next_deadline.tv_nsec; 108 | _p_info->next_deadline.tv_nsec = 0; 109 | _p_info->next_deadline.tv_sec += 1; 110 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &_p_info->next_deadline, NULL); 111 | 112 | while (!done) { 113 | t_stamp_log.stamp(); 114 | /* Code to be called periodicaly */ 115 | // std::cout << "NRT Thread " << x++ << "\n"; 116 | // for (size_t i = 0; i < 400; i++) { 117 | mtx.lock(); 118 | x = x + 1; 119 | mtx.unlock(); 120 | 121 | //} 122 | 123 | // give other threads some cpu time 124 | wait_rest_of_period(_p_info); 125 | } 126 | std::lock_guard guard(mtx); 127 | std::cout << "x in nrt is " << x << std::endl; 128 | return 0; 129 | } 130 | -------------------------------------------------------------------------------- /examples/rt_thread.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void* non_real_time_function(void* arg); 6 | void* real_time_function(void* arg); 7 | 8 | int main(int argc, char const* argv[]) 9 | { 10 | period_info info_rt; 11 | int rate = 1000; // 1000 Hz 12 | periodic_timer_init(&info_rt, 1e9 / (rate)); 13 | 14 | ThreadWrapper rt_thread(real_time_function, &info_rt); 15 | rt_thread.setPriority(SCHED_FIFO, 99); 16 | rt_thread.setAffinity(0); // pin to core 0 17 | 18 | period_info info_nrt; 19 | rate = 2000; // 2000 Hz 20 | periodic_timer_init(&info_nrt, 1e9 / (rate)); 21 | 22 | ThreadWrapper nrt_thread(non_real_time_function, &info_nrt); 23 | nrt_thread.setPriority(SCHED_FIFO, 20); 24 | nrt_thread.setAffinity(0); // pin to core 1 25 | 26 | // wait for threads to finish 27 | rt_thread.join(); 28 | nrt_thread.join(); 29 | /* code */ 30 | return 0; 31 | } 32 | 33 | /** 34 | * @brief Real Time callback function handle 35 | * 36 | * @param arg 37 | * @return void* 38 | */ 39 | void* real_time_function(void* arg) 40 | { 41 | period_info* _p_info = (period_info*)arg; 42 | double x = 1.73; 43 | 44 | while (1) { 45 | /* Code to be called periodicaly */ 46 | // std::cout << "RT Thread " << x++ << "\n"; 47 | x *= sin(x) / atan(x) * tanh(x) * sqrt(x); 48 | 49 | // give other threads some cpu time 50 | wait_rest_of_period(_p_info); 51 | } 52 | } 53 | 54 | /** 55 | * @brief Non real time function 56 | * 57 | * @param arg 58 | * @return void* 59 | */ 60 | void* non_real_time_function(void* arg) 61 | { 62 | period_info* _p_info = (period_info*)arg; 63 | double x = 1.37; 64 | 65 | while (1) { 66 | /* Code to be called periodicaly */ 67 | // std::cout << "NRT Thread " << x++ << "\n"; 68 | 69 | for (size_t i = 0; i < 100; i++) { 70 | x *= sin(x) / atan(x) * tanh(x) * sqrt(x); 71 | } 72 | // give other threads some cpu time 73 | wait_rest_of_period(_p_info); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /examples/thread_priority.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | void* callback_function_hp() 6 | { 7 | struct timespec time_requested, time_remaining; 8 | time_requested.tv_sec = 0; 9 | time_requested.tv_nsec = 10000; // 10 milissec 10 | 11 | float x = 1.5f; 12 | 13 | while (1) { 14 | x *= sin(x) / atan(x) * tanh(x) * sqrt(x); 15 | // need to sleep so lower priority threads get some cpu time 16 | nanosleep(&time_requested, &time_remaining); 17 | } 18 | } 19 | int main(int argc, char const* argv[]) 20 | { 21 | ThreadWrapper th1(callback_function_hp); 22 | 23 | // set scheduling policy FIFO, priority 20 24 | th1.setPriority(SCHED_FIFO, 20); 25 | 26 | ThreadWrapper th2(callback_function_hp); 27 | 28 | // set scheduling policy Round-Robin, priority 50 29 | th2.setPriority(SCHED_RR, 50); 30 | 31 | // wait for threads to finish 32 | th1.join(); 33 | th2.join(); 34 | 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /include/ThreadUtils/ThreadWrapper.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file ThreadWrapper.h 3 | * @author Siddharth Deore (siddharth.deore@iit.it) 4 | * @brief Wrapper class to implement POSIX Thread on C++11 5 | * @version 0.1 6 | * @date 2022-04-28 7 | * 8 | * @copyright Copyright (c) Istituto Italiano di Tecnologia, Genova, Italy 2022 9 | * 10 | */ 11 | #ifndef THREADWRAPPER_H 12 | #define THREADWRAPPER_H 13 | 14 | #pragma once 15 | 16 | #include 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | class ThreadWrapper : public std::thread { 25 | public: 26 | // make sure we inherit all constructors 27 | using thread::thread; 28 | /** 29 | * @brief Set thread scheduling policy and priority 30 | * possible policies : 31 | * SCHED_BATCH, SCHED_DEADLINE, SCHED_FIFO, SCHED_IDLE, SCHED_ISO, 32 | * SCHED_OTHER, SCHED_RESET_ON_FORK, SCHED_RR 33 | * 34 | * @param policy 35 | * @param priority 36 | */ 37 | void setPriority(int policy, int priority); 38 | ~ThreadWrapper() 39 | { 40 | if (joinable()) 41 | std::terminate(); 42 | } 43 | /** 44 | * @brief Pin thread to specific processsor core 45 | * 46 | * @param core processsor core number 47 | */ 48 | void setAffinity(int core); 49 | 50 | /** 51 | * @brief Get the Max Priority possible 52 | * 53 | * @return int maximum priority 54 | */ 55 | int getMaxPriority(); 56 | 57 | /** 58 | * @brief Get the Min Priority possible 59 | * 60 | * @return int minimum priority 61 | */ 62 | int getMinPriority(); 63 | 64 | /** 65 | * @brief Get number of CPU cores 66 | * 67 | * @return int CPU core count 68 | */ 69 | int nproc(); 70 | 71 | // Carefull!! This quick hack to impliment assignement is working for now. 72 | ThreadWrapper& operator=(ThreadWrapper& __t) 73 | { 74 | thread::swap(__t); 75 | return *this; 76 | } 77 | 78 | private: 79 | // policy type 80 | int _policy = SCHED_OTHER; 81 | 82 | // data structure to describe a process' schedulability 83 | sched_param _sch_params; 84 | 85 | // data structure to describe CPU mask 86 | cpu_set_t _cpuset; 87 | 88 | // standard mutex 89 | std::mutex _mtx; 90 | }; 91 | 92 | #endif -------------------------------------------------------------------------------- /include/ThreadUtils/TimeStampLogger.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #define timespec_to_second(tsp) (double)tsp.tv_sec + (double)tsp.tv_nsec / 1000000000 6 | class TimeStampLogger { 7 | private: 8 | timespec _time_stamp; 9 | double init_time_in_sec; 10 | std::vector _vec; 11 | std::string _filename; 12 | std::ofstream _logfile; 13 | 14 | public: 15 | TimeStampLogger(std::string name) 16 | { 17 | _vec.reserve(1000000); 18 | _filename = name; 19 | std::cout << "Stamp File :" << _filename << std::endl; 20 | clock_gettime(CLOCK_MONOTONIC, &_time_stamp); 21 | init_time_in_sec = timespec_to_second(_time_stamp); 22 | } 23 | void stamp() 24 | { 25 | clock_gettime(CLOCK_MONOTONIC, &_time_stamp); 26 | _vec.push_back(_time_stamp); 27 | } 28 | ~TimeStampLogger() 29 | { 30 | std::cout << "Writing Logger file :" << _filename << std::endl; 31 | _logfile.open(_filename); 32 | for (const auto& value : _vec) { 33 | _logfile << std::setprecision(std::numeric_limits::digits10) 34 | << timespec_to_second(value) << "\n"; 35 | //<< timespec_to_second(value) - init_time_in_sec << "\n"; 36 | } 37 | _logfile.close(); 38 | } 39 | // void flush(); // to do : save to file and clear ram; 40 | }; 41 | -------------------------------------------------------------------------------- /include/ThreadUtils/timer_math.h: -------------------------------------------------------------------------------- 1 | #ifndef TIMER_UTILS_H 2 | #define TIMER_UTILS_H 3 | 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | extern "C" { 13 | 14 | /** 15 | * periodic information 16 | */ 17 | struct period_info { 18 | struct timespec next_deadline; 19 | long period_ns; 20 | }; 21 | // struct timespec holds time in seconds and nanoseconds 22 | void timespec_add_us(struct timespec* t, long us); 23 | int timespec_cmp(struct timespec* a, struct timespec* b); 24 | int timespec_sub(struct timespec* diff, struct timespec* a, struct timespec* b); 25 | static void increment_period(struct period_info* pinfo); 26 | static void periodic_task_init(struct period_info* pinfo, long period_ns); 27 | static void wait_rest_of_period(struct period_info* pinfo); 28 | static void set_policy(const long policy); 29 | 30 | /** 31 | * @brief addd microseconds to timespec 32 | * 33 | * @param t 34 | * @param us 35 | */ 36 | void timespec_add_us(struct timespec* t, long us) 37 | { 38 | t->tv_nsec += us * 1000; 39 | if (t->tv_nsec > 1000000000) { 40 | t->tv_nsec = t->tv_nsec - 1000000000; // + ms*1000000; 41 | t->tv_sec += 1; 42 | } 43 | } 44 | 45 | /** 46 | * @brief Wait thread to start before clock monotonic timer reaches next millisecond 47 | * 48 | */ 49 | void sync_to_milisec() 50 | { // Try to syncronize 51 | timespec sync_t; 52 | clock_gettime(CLOCK_MONOTONIC, &(sync_t)); 53 | // wait for remaining nanoseconds 54 | // sync_t.tv_nsec = __syscall_slong_t(1000000000) - sync_t.tv_nsec; 55 | // sync time till next millisecond 56 | sync_t.tv_nsec = (sync_t.tv_nsec / 1000000) * 1000000 + 1000000; 57 | 58 | while (sync_t.tv_nsec >= 1000000000) { 59 | /* timespec nsec overflow */ 60 | sync_t.tv_sec++; 61 | sync_t.tv_nsec -= 1000000000; 62 | } 63 | 64 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &sync_t, NULL); 65 | } 66 | 67 | /** 68 | * @brief Wait thread to start before clock monotonic timer reaches next second 69 | * 70 | */ 71 | void sync_to_sec() 72 | { 73 | // Try to syncronize 74 | timespec sync_t; 75 | clock_gettime(CLOCK_MONOTONIC, &(sync_t)); 76 | // wait for remaining nanoseconds 77 | sync_t.tv_nsec = __syscall_slong_t(1000000000) - sync_t.tv_nsec; 78 | // sync time till next millisecond 79 | sync_t.tv_sec += 1; 80 | 81 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &sync_t, NULL); 82 | } 83 | 84 | int timespec_cmp(struct timespec* a, struct timespec* b) 85 | { 86 | if (a->tv_sec > b->tv_sec) { 87 | return 1; 88 | } else if (a->tv_sec < b->tv_sec) { 89 | return -1; 90 | } else //(a->tv_sec == b->tv_sec) 91 | { 92 | if (a->tv_nsec > b->tv_nsec) { 93 | return 1; 94 | } else if (a->tv_nsec == b->tv_nsec) { 95 | return 0; 96 | } else { 97 | return -1; 98 | } 99 | } 100 | } 101 | 102 | /** 103 | * @brief Substract timespec b from timespec a 104 | * 105 | * @param diff 106 | * @param a 107 | * @param b 108 | * @return int 109 | */ 110 | int timespec_sub(struct timespec* diff, struct timespec* a, 111 | struct timespec* b) 112 | { 113 | diff->tv_nsec = a->tv_nsec - b->tv_nsec; 114 | diff->tv_sec = a->tv_sec - b->tv_sec; 115 | if (a->tv_nsec < b->tv_nsec) { 116 | diff->tv_nsec += 1000000000; 117 | diff->tv_sec -= 1; 118 | } 119 | return 1; 120 | } 121 | 122 | /** 123 | * @brief increment the timespec period info 124 | * 125 | * @param pinfo 126 | */ 127 | static void increment_period(struct period_info* pinfo) 128 | { 129 | pinfo->next_deadline.tv_nsec += pinfo->period_ns; 130 | 131 | while (pinfo->next_deadline.tv_nsec >= 1000000000) { 132 | /* timespec nsec overflow */ 133 | pinfo->next_deadline.tv_sec++; 134 | pinfo->next_deadline.tv_nsec -= 1000000000; 135 | } 136 | } 137 | /** 138 | * @brief Initialize periodic information 139 | * 140 | * @param pinfo 141 | * @param period_ns 142 | */ 143 | static void periodic_timer_init(struct period_info* pinfo, 144 | const long period_ns) 145 | { 146 | /* for simplicity, hardcoding a 1ms period */ 147 | pinfo->period_ns = period_ns; 148 | 149 | clock_gettime(CLOCK_MONOTONIC, &(pinfo->next_deadline)); 150 | } 151 | 152 | /** 153 | * @brief sleep until clock reaches next deadline 154 | * 155 | * @param pinfo 156 | */ 157 | static void wait_rest_of_period(struct period_info* pinfo) 158 | { 159 | // increment deadline 160 | increment_period(pinfo); 161 | 162 | /* for simplicity, ignoring possibilities of signal wakes */ 163 | clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &pinfo->next_deadline, NULL); 164 | } 165 | } 166 | 167 | #endif -------------------------------------------------------------------------------- /instruction.md: -------------------------------------------------------------------------------- 1 | # The Real Time Linux 2 | 3 | 4 | This repository contains instruction to patch and install linux PREEMPT_RT, test the installation with standard rt-tests routines and example codes to create real time threads. Before starting installation please go through [reference](#reference) section mentioned at the end of this file. 5 | 6 | 7 | Setting up Preempt RT real time kernel involves steps mentioned below 8 | - [Download Linux Kernel Source Code](https://cdn.kernel.org/pub/linux/kernel/) 9 | - [Download the PREEMPT_RT patch](https://rt.wiki.kernel.org/index.php/Main_Page) 10 | - [Patch the kernel](https://www.kernel.org/doc/html/v4.11/process/applying-patches.html) 11 | - Build and Install the kernel 12 | - Update the grub boot and restart 13 | - Select perefered RT kernel from grub boot menu 14 | - [Validate the installation with rt-tests](https://wiki.linuxfoundation.org/realtime/documentation/howto/tools/rt-tests) 15 | 16 | - Make your own Real time application 17 | # Patch Build and Install 18 | Following installation instructions are tested on system running 64bit Ubuntu 20.04.3 LTS with 5.11.0-36-generic kernel. 19 | 20 | We need some tools to build kernel. 21 | ```console 22 | sudo apt-get install build-essential libnuma-dev 23 | sudo apt-get install libncurses5-dev flex bison openssl libssl-dev dkms 24 | sudo apt-get install libelf-dev libudev-dev libpci-dev libiberty-dev 25 | sudo apt-get install autoconf fakeroot 26 | sudo apt install dwarves 27 | sudo apt install zstd 28 | 29 | ``` 30 | 31 | Make a directory named kernel in desired location 32 | ```console 33 | mkdir -p ~/kernel 34 | cd -p ~/kernel 35 | ``` 36 | 37 | print kernel version and machine related information, here we have linux kernel version 5.11.3, I would prefere to build and patch nearest kernel version to existing one. 38 | ```console 39 | uname -a 40 | ``` 41 | Download suitable linux kernel source from [Linux kernel page](https://cdn.kernel.org/pub/linux/kernel/), preferebly same version already installed on system. 42 | ```console 43 | wget https://mirrors.edge.kernel.org/pub/linux/kernel/v5.x/linux-5.11.3.tar.gz 44 | ``` 45 | Extract source code in current directory. 46 | ```console 47 | tar -xzf linux-5.11.3.tar.gz 48 | ``` 49 | Download and extract sutaible PREEMPT_RT version source code from [Linux RT page](https://cdn.kernel.org/pub/linux/kernel/projects/rt/). 50 | 51 | ```console 52 | wget http://cdn.kernel.org/pub/linux/kernel/projects/rt/5.11/older/patch-5.11-rt7.patch.gz 53 | 54 | gunzip patch-5.11-rt7.patch.gz 55 | ``` 56 | Apply the patch to kernel source code. 57 | ```console 58 | cd linux-5.11.3 59 | patch -p1 < ../patch-5.11-rt7.patch 60 | ``` 61 | To ensure that the RT kernel supports the current distribution, we need to copy current configration settings 62 | ```console 63 | # cp /boot/config-5.11.0-36-generic .config 64 | cp /boot/config-$(uname -r) .config 65 | ``` 66 | Keep defualt settings by automaticaly setting yes to old configration. 67 | ```console 68 | yes '' | make oldconfig 69 | ``` 70 | Menuconfig allows us to choose linux features, in this case PREEMPT_RT patch related functionality. 71 | ```console 72 | make menuconfig 73 | ``` 74 | Configure following settings by navigating GUI menu 75 | 76 | 77 | ### 1. Timer tick handling (Full dynticks system (tickless)) 78 | ``` 79 | # Enable CONFIG_NO_HZ_FULL 80 | -> General setup 81 | -> Timers subsystem 82 | -> Timer tick handling (Full dynticks system (tickless)) 83 | (X) Full dynticks system (tickless) 84 | ``` 85 | 86 | 87 | 88 | ### 2. Preemption Model (Fully Preemptible Kernel (Real-Time)) 89 | ``` 90 | # Enable CONFIG_PREEMPT_RT 91 | -> General Setup 92 | -> Preemption Model (Fully Preemptible Kernel (Real-Time)) 93 | (X) Fully Preemptible Kernel (Real-Time) 94 | ``` 95 | 96 | 97 | 98 | ### 3. Timer frequency 99 | 100 | ``` 101 | # Set CONFIG_HZ_1000 (note: this is no longer in the General Setup menu, go back twice) 102 | -> Processor type and features 103 | -> Timer frequency (1000 HZ) 104 | (X) 1000 HZ 105 | ``` 106 | 107 | 108 | 109 | ### 4. Default CPUFreq governor 110 | 111 | ``` 112 | # Set CPU_FREQ_DEFAULT_GOV_PERFORMANCE [=y] 113 | -> Power management and ACPI options 114 | -> CPU Frequency scaling 115 | -> CPU Frequency scaling (CPU_FREQ [=y]) 116 | -> Default CPUFreq governor ( [=y]) 117 | (X) performance 118 | ``` 119 | 120 | 121 | 122 | 123 | In your kernel configuration file spesificaly when compiling on Ubuntu (debian)to avoid any error during compilation process related to SYSTEM_TRUSTED_KEYS and CONFIG_SYSTEM_REVOCATION_KEYS update following lines: 124 | 125 | ```console 126 | CONFIG_SYSTEM_TRUSTED_KEYS="debian/canonical-certs.pem" 127 | CONFIG_SYSTEM_REVOCATION_KEYS="debian/canonical-revoked-certs.pem" 128 | ``` 129 | Change it to this: 130 | ```console 131 | CONFIG_SYSTEM_TRUSTED_KEYS="" 132 | CONFIG_SYSTEM_REVOCATION_KEYS="" 133 | ``` 134 | Instead of manually editing, you can also update above configs by running following commands, 135 | 136 | ```console 137 | scripts/config --set-str SYSTEM_TRUSTED_KEYS "" 138 | scripts/config --set-str SYSTEM_REVOCATION_KEYS "" 139 | ``` 140 | 141 | Compile Kernel source file with -jX flag, here X number of core 142 | ```console 143 | sudo make -j20 144 | ``` 145 | Make modules and install 146 | ```console 147 | sudo make modules_install 148 | sudo make install 149 | ``` 150 | Now update grub and restart the system and select patched kernel version from grub boot menu 151 | ```console 152 | sudo update-grub 153 | reboot 154 | ``` 155 | Check the kernel version 156 | ```console 157 | uname -a 158 | ``` 159 | # Troubleshoot 160 | If you are not able to boot in to letest pached kernel and get message simillar to loading ramdisk or below, 161 | ```console 162 | Kernel panic - not syncing 163 | VFS: unable to mount root fs on unknown block(0,0) 164 | ``` 165 | check installed kernels 166 | ```console 167 | find /boot/vmli* # for list of all kernels 168 | find /boot/vmli* | grep rt # for rt kernels 169 | ``` 170 | Edit initramfs and set `MODULES=dep` 171 | ``` 172 | sudo gedit /etc/initramfs-tools/initramfs.conf 173 | ``` 174 | Update initramfs with your rt kerenel version, in this case "5.11-rt7" 175 | 176 | ``` 177 | sudo update-initramfs -c -k 5.11-rt7 178 | sudo update-grub 179 | reboot 180 | ``` 181 | 182 | # Testing 183 | rt-tests is a test suite, that contains programs to test various real time Linux features. 184 | ```console 185 | git clone git://git.kernel.org/pub/scm/utils/rt-tests/rt-tests.git 186 | cd rt-tests 187 | make all 188 | make install 189 | ``` 190 | 191 | Download the test bash script and run test, (Take few hours) script generates latency plot and requires gnuplot do make plot.png. 192 | ```console 193 | https://www.osadl.org/uploads/media/mklatencyplot.bash 194 | chmod +x mklatencyplot.bash 195 | sudo ./mklatencyplot.bash 196 | ``` 197 | N.B. We need super user permision to use PREEMPT_RT 198 | 199 | Show or change the real-time scheduling attributes of a process 200 | ```console 201 | # Set policy: 202 | # chrt [options] [...] 203 | # chrt [options] --pid 204 | # 205 | #Get policy: 206 | # chrt [options] -p 207 | # Policy options: 208 | # -b, --batch set policy to SCHED_BATCH 209 | # -d, --deadline set policy to SCHED_DEADLINE 210 | # -f, --fifo set policy to SCHED_FIFO 211 | # -i, --idle set policy to SCHED_IDLE 212 | # -o, --other set policy to SCHED_OTHER 213 | # -r, --rr set policy to SCHED_RR (default) 214 | sudo chrt --rr 99 ./app_name 215 | 216 | ``` 217 | To set process priorities 218 | ```console 219 | # https://www.howtoforge.com/linux-chrt-command/ 220 | # Get process ID 221 | pidof 222 | # check priority status 223 | sudo chrt -p 224 | # change policy to real time SCHED_RR 225 | sudo chrt -r -p 99 226 | ``` 227 | 228 | Latency Plot 229 | 230 | ![latency](assets/plot.png) 231 | 232 | 233 | # References 234 | 1. [F. Reghenzani, The real-time Linux kernel: a Survey on PREEMPT_RT](https://web.archive.org/web/20210305000638/https://re.public.polimi.it/retrieve/handle/11311/1076057/344112/paper.pdf) 235 | 2. [A realtime preemption overview](https://web.archive.org/web/20220126075157/https://lwn.net/Articles/146861/) 236 | 3. [The Linux Foundation Real Time Linux](https://wiki.linuxfoundation.org/realtime/documentation/start) 237 | 4. [Giuseppe Lipari, Lesson notes - Programming RT systems with pthreads](https://web.archive.org/web/20170517102136/http://retis.sssup.it/~lipari/courses/str09/10.rtprogramming-handout.pdf) 238 | 5. [Frank Vasquez, Chris Simmonds, "Mastering Embedded Linux Programming".](https://books.google.it/books?id=4Hc5DwAAQBAJ&lpg=PP1&hl=it&pg=PP1#v=onepage&q&f=false) 239 | 6. [P. Ficheux, Using real-time with Linux](https://web.archive.org/web/20210924153548/https://etr2021.ensma.fr/files/p-ficheux-realtime-linux.pdf) 240 | 241 | # Maintainers 242 | This repository is maintained by: 243 | 244 | | | [Siddharth Deore](https://github.com/siddharthdeore) | 245 | |--|--| 246 | -------------------------------------------------------------------------------- /raspberry-pi-rt.md: -------------------------------------------------------------------------------- 1 | # Raspberry PI 4B preempt-rt kernel cross compilation 2 | 3 | # Step 1: Essenttial tools to build kernel. 4 | Install essential utilities and build tools 5 | ```sh 6 | sudo apt-get install -y build-essential libgmp-dev libmpfr-dev libmpc-dev 7 | sudo apt-get install -y libisl-dev libncurses5-dev bc git-core bison flex 8 | sudo apt-get install -y libncurses-dev libssl-dev 9 | ``` 10 | 11 | # Step 2: Install Cross Compiler 12 | Download and install cross compiler utilities for aarch-64 13 | ```sh 14 | # sudo apt install gcc make gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu 15 | # or build from source 16 | cd ~/Downloads 17 | wget https://ftp.gnu.org/gnu/binutils/binutils-2.35.tar.bz2 18 | tar xf binutils-2.35.tar.bz2 19 | cd binutils-2.35/ 20 | ./configure --prefix=/opt/aarch64 --target=aarch64-linux-gnu --disable-nls 21 | 22 | # make and install binutils 23 | make -j$(nproc) 24 | sudo make install 25 | 26 | # Export path 27 | export PATH=$PATH:/opt/aarch64/bin/ 28 | 29 | # Build and install gcc from source 30 | cd .. 31 | # download and gcc source 32 | wget https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz 33 | 34 | # unpack archive 35 | tar xf gcc-8.4.0.tar.xz 36 | cd gcc-8.4.0/ 37 | ./contrib/download_prerequisites 38 | ./configure --prefix=/opt/aarch64 --target=aarch64-linux-gnu --with-newlib --without-headers \ 39 | --disable-nls --disable-shared --disable-threads --disable-libssp --disable-decimal-float \ 40 | --disable-libquadmath --disable-libvtv --disable-libgomp --disable-libatomic \ 41 | --enable-languages=c --disable-multilib 42 | 43 | # Make and install gcc compiler 44 | make all-gcc -j$(nproc) 45 | sudo make install-gcc 46 | 47 | # Check if compiler works 48 | /opt/aarch64/bin/aarch64-linux-gnu-gcc -v 49 | # or aarch64-linux-gnu-gcc if installed with apt-get 50 | ``` 51 | 52 | # Step 3: Patching Kernel and Configure 53 | ```sh 54 | mkdir ~/rpi-kernel 55 | cd ~/rpi-kernel 56 | 57 | # Downlad stable Raspberry PI kernel source 58 | git clone https://github.com/raspberrypi/linux.git -b rpi-5.15.y 59 | # Download rt patch 60 | wget https://mirrors.edge.kernel.org/pub/linux/kernel/projects/rt/5.15/patch-5.15.44-rt46.patch.gz 61 | 62 | mkdir kernel-build 63 | 64 | cd linux 65 | 66 | # unpack the patch. 67 | gzip -cd ../patch-5.15.44-rt46.patch.gz | patch -p1 --verbose 68 | 69 | # configuration for Raspberry PI 4B 70 | make O=../kernel-build/ ARCH=arm64 CROSS_COMPILE=/opt/aarch64/bin/aarch64-linux-gnu- bcm2711_defconfig 71 | 72 | # enter the menuconfig 73 | make O=../kernel-build/ ARCH=arm64 CROSS_COMPILE=/opt/aarch64/bin/aarch64-linux-gnu- menuconfig 74 | ``` 75 | 76 | # Step 4: Building RT Kernel 77 | 78 | ```sh 79 | make -j$(nproc) O=../kernel-build/ ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- 80 | 81 | # pack our Kernel 82 | export INSTALL_MOD_PATH=~/rpi-kernel/rt-kernel 83 | export INSTALL_DTBS_PATH=~/rpi-kernel/rt-kernel 84 | make O=../kernel-build/ ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- modules_install dtbs_install 85 | cp ../kernel-build/arch/arm64/boot/Image ../rt-kernel/boot/kernel8.img 86 | cd $INSTALL_MOD_PATH 87 | tar czf ../rt-kernel.tgz * 88 | cd .. 89 | 90 | ``` 91 | # Step 5: Installing New Kernel 92 | ```sh 93 | # secure copy kernel archive to raspberi pi 94 | 95 | scp rt-kernel.tgz pi@:/tmp 96 | 97 | # login pi through ssh 98 | ssh pi@ 99 | # copy our files 100 | cd /tmp 101 | tar xzf rt-kernel.tgz 102 | cd boot 103 | sudo cp -rd * /boot/ 104 | cd ../lib 105 | sudo cp -dr * /lib/ 106 | cd ../overlays 107 | sudo cp -dr * /boot/overlays 108 | cd ../broadcom 109 | sudo cp -dr bcm* /boot/ 110 | # add following line to `/boot/config.txt`. 111 | # kernel=kernel8.img 112 | 113 | reboot 114 | 115 | uname -a 116 | ``` 117 | # Step 6: Isolate CPUs from the kernel scheduler. 118 | Remove the given CPUs, as indicated by the cpu number values, from the general kernel SMP balance and scheduler algorithms. The CPU affinity syscalls are the sole mechanism to transfer a process onto or off of a "isolated" CPU. To do so add following line to `/boot/cmdline.txt` 119 | ``` sh 120 | isolcpus=3 # isolate the CPU nr 3 121 | # or 122 | isolcpus=1-3 # isolate the CPUs nr 1, 2 & 3 123 | ``` 124 | establish the perticular interrupt's affinity towards a specific CPU type e.g.: 125 | ```sh 126 | sudo echo 3 > /proc/irq/62/smp_affinity 127 | sudo echo 3 > /proc/irq/62/smp_affinity_list 128 | ``` 129 | Examine which interruptions are being scheduled and which CPU is responding to them. 130 | ```sh 131 | cat /proc/interrupts 132 | ``` -------------------------------------------------------------------------------- /scripts/plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | 6 | def deviation_plot(filename,delta_t = 1000): 7 | arr = np.loadtxt(filename) 8 | #arr = arr[1:-1] 9 | # start time as initial time 10 | arr = arr - arr[0] 11 | #arr = arr[1:-1] 12 | # convert to microseconds 13 | delta_arr = (np.diff(arr)) * 1e6 - delta_t # sec to usec 14 | # get deviation in consecutive readigs 15 | # delta_arr = delta_arr - delta_arr[0] # sec to usec 16 | # remove first and last 10 rows 17 | delta_arr = delta_arr[10:-10] 18 | #np.set_printoptions(precision=8) 19 | #with np.printoptions(precision=3, suppress=False): 20 | # print(arr) 21 | x = range(1,len(delta_arr)+1) 22 | 23 | fig, axes = plt.subplots(nrows=2, ncols=1, sharey=False) 24 | axes[0].plot(x,delta_arr,'k.') 25 | axes[0].set_ylim(ymax=max(delta_arr)*1.2,ymin=min(delta_arr)*1.2) 26 | axes[0].set_ylabel("$\Delta T ( \mu sec )$") 27 | # statistics 28 | sigma = np.std(delta_arr) 29 | mu = np.mean(delta_arr) 30 | # An "interface" to matplotlib.axes.Axes.hist() method 31 | n, bins, patches = axes[1].hist(x=delta_arr, bins='auto', color='#0504aa', alpha=0.7) 32 | axes[1].grid(axis='y', alpha=0.75) 33 | axes[1].set_xlabel("$deviation ( \mu sec )$") 34 | axes[1].set_ylabel('Frequency') 35 | pdfunc = ((1 / (np.sqrt(2 * np.pi) * sigma)) * np.exp(-0.5 * (1 / sigma * (bins - mu))**2)) 36 | axes[1].plot(bins, pdfunc, '--') 37 | #plt.title('Histogram') 38 | #axes[1].set_text(0, 0, r'$\mu=, b= $') 39 | maxfreq = n.max() 40 | # Set a clean upper y-axis limit. 41 | axes[1].set_ylim(ymax=np.ceil(maxfreq / 10) * 10 if maxfreq % 10 else maxfreq + 10) 42 | plt.tight_layout() 43 | plt.show() 44 | 45 | def jitter_plot(filename1,filename2): 46 | arr = np.loadtxt(filename1) 47 | arr2 = np.loadtxt(filename2) 48 | 49 | sigma = np.std(arr) 50 | mu = np.mean(arr) 51 | n, bins, patches = plt.hist(x=arr, bins='auto', color='#0504aa', alpha=0.7,log=True) 52 | n, bins, patches = plt.hist(x=arr2, bins='auto', color='#ff04aa', alpha=0.7,log=True) 53 | plt.tight_layout() 54 | plt.show() 55 | 56 | if __name__ == "__main__": 57 | try: 58 | # periodic deviation plots 59 | deviation_plot('logs/rt1k.txt',delta_t=1000) 60 | deviation_plot('logs/rt5k.txt',delta_t=200) 61 | 62 | finally: 63 | pass 64 | -------------------------------------------------------------------------------- /src/ThreadUtils/ThreadWrapper.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void ThreadWrapper::setPriority(int policy = SCHED_RR, int priority = 20) 4 | { 5 | _policy = policy; 6 | _sch_params.sched_priority = priority; 7 | 8 | if (pthread_setschedparam(this->native_handle(), policy, &_sch_params)) { 9 | std::lock_guard lock(_mtx); // unlocks when lock destroyed 10 | std::cerr << "Failed to set thread scheduling, try with sudo : " 11 | << std::strerror(errno) << std::endl; 12 | } else { 13 | std::lock_guard lock(_mtx); 14 | std::cout << "Thread priority scheduling set to " << priority << std::endl; 15 | } 16 | } 17 | 18 | 19 | void ThreadWrapper::setAffinity(int core) 20 | { 21 | int nproc = sysconf(_SC_NPROCESSORS_ONLN); 22 | 23 | if (nproc < core) { 24 | core = nproc; 25 | } 26 | 27 | CPU_ZERO(&_cpuset); 28 | CPU_SET(core, &_cpuset); 29 | 30 | int ret = pthread_setaffinity_np(this->native_handle(), sizeof(cpu_set_t), 31 | &_cpuset); 32 | if (ret != 0) { 33 | std::lock_guard lock(_mtx); 34 | std::cerr << "Failed to set Thread Affinity : " << std::strerror(errno) 35 | << std::endl; 36 | } else { 37 | std::lock_guard lock(_mtx); 38 | std::cout << "Thread Affinity set to CPU " << core << std::endl; 39 | } 40 | } 41 | 42 | int ThreadWrapper::getMaxPriority() { return sched_get_priority_max(_policy); } 43 | 44 | int ThreadWrapper::getMinPriority() { return sched_get_priority_min(_policy); } 45 | 46 | int ThreadWrapper::nproc() { return sysconf(_SC_NPROCESSORS_ONLN); } 47 | --------------------------------------------------------------------------------