├── .clangd ├── .editorconfig ├── .githooks └── pre-commit ├── .github ├── actions │ ├── install-deps-action │ │ └── action.yml │ └── install-nix │ │ └── action.yml ├── include │ ├── build-kernel.nix │ ├── ci.py │ ├── flake.lock │ ├── flake.nix │ ├── list-integration-tests.py │ ├── update-kernels.py │ └── veristat.nix └── workflows │ ├── 6_12.yml │ ├── bpf-next-test.yml │ ├── build-kernels.yml │ ├── caching-build.yml │ ├── integration-tests.yml │ ├── stable.yml │ └── update-kernels.yml ├── .gitignore ├── .gitmodules ├── BREAKING_CHANGES.md ├── Cargo.lock ├── Cargo.toml ├── DEVELOPER_GUIDE.md ├── INSTALL.md ├── LICENSE ├── OVERVIEW.md ├── README.md ├── cargo-publish.py ├── kernel-versions.json ├── kernel.config ├── lib ├── arena.bpf.c ├── bitmap.bpf.c ├── cpumask.bpf.c ├── meson.build ├── sdt_alloc.bpf.c ├── sdt_task.bpf.c └── topology.bpf.c ├── libalpm ├── openrc │ ├── 90-scx-scheds-upgrade.hook │ ├── meson.build │ └── scx-openrc-restart └── systemd │ ├── 90-scx-scheds-upgrade.hook │ ├── meson.build │ └── scx-systemd-restart ├── meson-scripts ├── bpftool_build_skel ├── bpftool_build_skel_lib ├── bpftool_dummy.c ├── build_bpftool ├── build_libbpf ├── cargo_fetch ├── cc_cflags_probe.c ├── compile_scx_lib ├── fetch_bpftool ├── fetch_libbpf ├── get_bpftool_ver ├── get_clang_ver ├── get_sys_incls ├── install_rust_user_scheds ├── run_stress_tests ├── stress_tests.ini ├── test_sched ├── veristat └── veristat_diff ├── meson.build ├── meson.options ├── pgo-lto.sh ├── rust-toolchain.toml ├── rust ├── .gitignore ├── scx_loader │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── configuration.md │ ├── meson.build │ ├── org.scx.Loader.conf │ ├── org.scx.Loader.xml │ └── src │ │ ├── config.rs │ │ ├── dbus.rs │ │ ├── lib.rs │ │ ├── logger.rs │ │ └── main.rs ├── scx_rustland_core │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── assets │ │ ├── bpf.rs │ │ └── bpf │ │ │ ├── intf.h │ │ │ └── main.bpf.c │ ├── bindings.h │ ├── bpf_h │ ├── build.rs │ └── src │ │ ├── alloc.rs │ │ ├── bindings.rs │ │ ├── bpf_intf.rs │ │ ├── bpf_skel.rs │ │ ├── lib.rs │ │ └── rustland_builder.rs ├── scx_stats │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── examples │ │ ├── client.rs │ │ ├── server.rs │ │ └── stats_defs.rs.h │ ├── scripts │ │ └── scxstats_to_openmetrics.py │ ├── scx_stats_derive │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ └── src │ │ │ └── lib.rs │ └── src │ │ ├── client.rs │ │ ├── lib.rs │ │ ├── server.rs │ │ └── stats.rs ├── scx_userspace_arena │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ ├── alloc.rs │ │ ├── bpf │ │ └── intf.h │ │ ├── bpf_intf.rs │ │ └── lib.rs └── scx_utils │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ ├── bpf_h │ ├── build.rs │ ├── examples │ └── mangolog.rs │ ├── perf_wrapper.h │ └── src │ ├── autopower.rs │ ├── bindings.rs │ ├── bpf_builder.rs │ ├── build_id.rs │ ├── builder.rs │ ├── clang_info.rs │ ├── compat.rs │ ├── cpumask.rs │ ├── energy_model.rs │ ├── enums.rs │ ├── gpu.rs │ ├── infeasible.rs │ ├── lib.rs │ ├── libbpf_logger.rs │ ├── mangoapp.rs │ ├── misc.rs │ ├── netdev.rs │ ├── perf.rs │ ├── pm.rs │ ├── ravg.rs │ ├── topology.rs │ └── user_exit_info.rs ├── rustfmt.toml ├── scheds ├── README.md ├── c │ ├── README.md │ ├── meson.build │ ├── scx_central.bpf.c │ ├── scx_central.c │ ├── scx_flatcg.bpf.c │ ├── scx_flatcg.c │ ├── scx_flatcg.h │ ├── scx_nest.bpf.c │ ├── scx_nest.c │ ├── scx_nest.h │ ├── scx_nest_stats_table.h │ ├── scx_pair.bpf.c │ ├── scx_pair.c │ ├── scx_pair.h │ ├── scx_prev.bpf.c │ ├── scx_prev.c │ ├── scx_qmap.bpf.c │ ├── scx_qmap.c │ ├── scx_sdt.bpf.c │ ├── scx_sdt.c │ ├── scx_sdt.h │ ├── scx_simple.bpf.c │ ├── scx_simple.c │ ├── scx_userland.bpf.c │ ├── scx_userland.c │ └── scx_userland.h ├── include │ ├── .gitignore │ ├── arch │ │ ├── arm │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ ├── arm64 │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ ├── mips │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ ├── powerpc │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ ├── riscv │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ ├── s390 │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ │ └── x86 │ │ │ ├── vmlinux-v6.16-g9b30400ff652.h │ │ │ └── vmlinux.h │ ├── bpf-compat │ │ └── gnu │ │ │ └── stubs.h │ ├── lib │ │ ├── arena.h │ │ ├── cpumask.h │ │ ├── percpu.h │ │ ├── sdt_task.h │ │ └── topology.h │ ├── scx │ │ ├── arena_userspace_interrop.bpf.h │ │ ├── bpf_arena_common.h │ │ ├── bpf_arena_spin_lock.h │ │ ├── bpf_atomic.h │ │ ├── common.bpf.h │ │ ├── common.h │ │ ├── compat.bpf.h │ │ ├── compat.h │ │ ├── enum_defs.autogen.h │ │ ├── enums.autogen.bpf.h │ │ ├── enums.autogen.h │ │ ├── enums.bpf.h │ │ ├── enums.h │ │ ├── namespace.bpf.h │ │ ├── namespace_impl.bpf.h │ │ ├── ravg.bpf.h │ │ ├── ravg_impl.bpf.h │ │ └── user_exit_info.h │ └── vmlinux.h ├── meson.build ├── rust │ ├── .gitignore │ ├── README.md │ ├── scx_bpfland │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ └── main.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_chaos │ │ ├── Cargo.toml │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ ├── lib │ │ │ └── main.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── lib.rs │ │ │ └── main.rs │ ├── scx_flash │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ └── main.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_lavd │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── balance.bpf.c │ │ │ ├── idle.bpf.c │ │ │ ├── intf.h │ │ │ ├── introspec.bpf.c │ │ │ ├── lavd.bpf.h │ │ │ ├── lock.bpf.c │ │ │ ├── main.bpf.c │ │ │ ├── power.bpf.c │ │ │ ├── preempt.bpf.c │ │ │ ├── sys_stat.bpf.c │ │ │ └── util.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_layered │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ ├── examples │ │ │ ├── avgruntime.json │ │ │ ├── cgrp_contains.json │ │ │ ├── cmdjoin.c │ │ │ ├── cmdjoin.json │ │ │ ├── cpus_pct.json │ │ │ ├── cpuset.json │ │ │ ├── exclude.json │ │ │ ├── preempt_rt_sched_class.json │ │ │ ├── prefix_suffix.json │ │ │ ├── protected.json │ │ │ ├── same_over_idle.json │ │ │ └── template.json │ │ ├── integration │ │ │ ├── layer_llc.bt │ │ │ ├── layer_node.bt │ │ │ ├── llc.json │ │ │ ├── numa.json │ │ │ └── run_tests.sh │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ ├── main.bpf.c │ │ │ ├── timer.bpf.c │ │ │ ├── timer.bpf.h │ │ │ ├── util.bpf.c │ │ │ └── util.bpf.h │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── config.rs │ │ │ ├── layer_core_growth.rs │ │ │ ├── lib.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_mitosis │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── .clang-format │ │ │ ├── intf.h │ │ │ └── mitosis.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ └── main.rs │ ├── scx_p2dq │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ ├── lib │ │ │ ├── main.bpf.c │ │ │ └── types.h │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── lib.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_rlfifo │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ └── main.rs │ ├── scx_rustland │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ ├── scx_rusty │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ ├── lb_domain.h │ │ │ ├── main.bpf.c │ │ │ ├── sdt_alloc.bpf.c │ │ │ ├── sdt_task.bpf.c │ │ │ ├── sdt_task.h │ │ │ └── types.h │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── domain.rs │ │ │ ├── load_balance.rs │ │ │ ├── main.rs │ │ │ ├── stats.rs │ │ │ └── tuner.rs │ ├── scx_tickless │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ │ ├── bpf │ │ │ ├── intf.h │ │ │ └── main.bpf.c │ │ │ ├── bpf_intf.rs │ │ │ ├── bpf_skel.rs │ │ │ ├── main.rs │ │ │ └── stats.rs │ └── scx_wd40 │ │ ├── Cargo.lock │ │ ├── Cargo.toml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── build.rs │ │ └── src │ │ ├── bpf │ │ ├── common.bpf.c │ │ ├── common.h │ │ ├── deadline.bpf.c │ │ ├── deadline.h │ │ ├── intf.h │ │ ├── lb_domain.bpf.c │ │ ├── lb_domain.h │ │ ├── main.bpf.c │ │ ├── placement.bpf.c │ │ └── types.h │ │ ├── bpf_intf.rs │ │ ├── bpf_skel.rs │ │ ├── domain.rs │ │ ├── load_balance.rs │ │ ├── main.rs │ │ ├── stats.rs │ │ └── tuner.rs └── sync-to-kernel.sh ├── scripts ├── bpftrace_stress_wrapper.sh ├── dsq_lat.bt ├── freq_trace.bt ├── gen_enum_defs.py ├── gen_enums.py ├── gen_vmlinux_h.sh ├── process_runqlat.bt ├── sched_ftrace.py ├── scxtop.bt ├── slicesnoop.bt └── vtime_dist.bt ├── services ├── README.md ├── openrc │ ├── meson.build │ └── scx.initrd ├── scx └── systemd │ ├── meson.build │ ├── org.scx.Loader.service │ ├── scx.service │ └── scx_loader.service ├── tools ├── scxctl │ ├── Cargo.toml │ ├── LICENSE │ ├── README.md │ └── src │ │ ├── cli.rs │ │ └── main.rs ├── scxtop │ ├── Cargo.toml │ ├── README.md │ ├── benches │ │ └── search_benchmark.rs │ ├── build.rs │ └── src │ │ ├── app.rs │ │ ├── bpf │ │ ├── intf.h │ │ └── main.bpf.c │ │ ├── bpf_intf.rs │ │ ├── bpf_skel.rs │ │ ├── bpf_stats.rs │ │ ├── cli.rs │ │ ├── config.rs │ │ ├── cpu_data.rs │ │ ├── edm.rs │ │ ├── event_data.rs │ │ ├── keymap.rs │ │ ├── lib.rs │ │ ├── llc_data.rs │ │ ├── main.rs │ │ ├── mangoapp.rs │ │ ├── node_data.rs │ │ ├── perf_event.rs │ │ ├── perfetto_trace.rs │ │ ├── protos │ │ ├── mod.rs │ │ └── perfetto_scx.proto │ │ ├── search.rs │ │ ├── stats.rs │ │ ├── theme.rs │ │ ├── tracer.rs │ │ ├── tui.rs │ │ └── util.rs └── vmlinux_docify │ ├── Cargo.toml │ ├── README.md │ └── src │ └── main.rs └── version-tool.py /.clangd: -------------------------------------------------------------------------------- 1 | CompileFlags: 2 | Add: [-DLSP, -Wno-missing-declarations, -Wno-typedef-redefinition, -Wno-ignored-attributes, -xc] -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0-only 2 | 3 | root = true 4 | 5 | [{*.{awk,c,dts,dtsi,dtso,h,mk,s,S},Kconfig,Makefile,Makefile.*}] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | indent_style = tab 10 | indent_size = 8 11 | -------------------------------------------------------------------------------- /.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Running cargo fmt..." 4 | cargo fmt --all 5 | 6 | # Stage any files modified by cargo fmt 7 | git add $(git diff --name-only) 8 | 9 | exit 0 10 | 11 | -------------------------------------------------------------------------------- /.github/actions/install-deps-action/action.yml: -------------------------------------------------------------------------------- 1 | name: install-deps 2 | 3 | runs: 4 | using: 'composite' 5 | steps: 6 | ### OTHER REPOS #### 7 | # turn off interactive, refresh pkgs 8 | - run: | 9 | echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections 10 | sudo rm /var/lib/man-db/auto-update 11 | sudo apt-get update -y 12 | sudo apt-get install -y tasksel 13 | sudo tasksel remove ubuntu-desktop 14 | shell: bash 15 | 16 | ### DOWNLOAD AND INSTALL DEPENDENCIES ### 17 | 18 | # Download dependencies packaged by Ubuntu 19 | - run: | 20 | sudo apt install -f -y bison busybox-static cmake coreutils \ 21 | cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod \ 22 | libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev \ 23 | linux-headers-generic linux-tools-common linux-tools-generic make \ 24 | ninja-build pahole pkg-config python3-dev python3-pip python3-requests \ 25 | qemu-kvm rsync stress-ng udev zstd libseccomp-dev libcap-ng-dev \ 26 | llvm-19 clang-19 python3-full curl meson bpftrace dwarves rustup \ 27 | protobuf-compiler 28 | 29 | echo /usr/lib/llvm-19/bin >> $GITHUB_PATH 30 | shell: bash 31 | 32 | # ensure some toolchain is installed 33 | - run: | 34 | rustup default nightly 35 | shell: bash 36 | 37 | # virtme-ng 38 | - run: sudo pip3 install virtme-ng --break-system-packages 39 | shell: bash 40 | 41 | # Setup KVM support 42 | - run: | 43 | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules 44 | sudo udevadm control --reload-rules 45 | sudo udevadm trigger --name-match=kvm 46 | shell: bash 47 | 48 | ### END DEPENDENCIES ### 49 | -------------------------------------------------------------------------------- /.github/actions/install-nix/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Install Nix with caching' 2 | description: 'Install Nix and configure caching on both self-hosted and GitHub Hosted runners' 3 | inputs: 4 | cachix-auth-token: 5 | required: true 6 | type: string 7 | 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Install Nix 12 | uses: cachix/install-nix-action@v25 13 | with: 14 | nix_path: nixpkgs=channel:nixos-unstable 15 | 16 | - name: Load dependencies 17 | if: ${{ runner.environment == 'self-hosted' }} 18 | run: | 19 | nix run ./.github/include#nix-develop-gha -- ./.github/include#gha-common 20 | shell: bash 21 | 22 | - uses: cachix/cachix-action@v14 23 | name: Configure Cachix (self-hosted) 24 | if: ${{ runner.environment == 'self-hosted' }} 25 | with: 26 | name: sched-ext 27 | authToken: '${{ inputs.cachix-auth-token }}' 28 | skipAddingSubstituter: true 29 | useDaemon: false 30 | 31 | - uses: cachix/cachix-action@v14 32 | name: Configure Cachix (github-hosted) 33 | if: ${{ runner.environment != 'self-hosted' }} 34 | with: 35 | name: sched-ext 36 | authToken: '${{ inputs.cachix-auth-token }}' 37 | skipAddingSubstituter: false 38 | useDaemon: false 39 | -------------------------------------------------------------------------------- /.github/include/build-kernel.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , stdenv 3 | , fetchgit 4 | , virtme-ng 5 | , linuxManualConfig 6 | , linuxPackages_latest 7 | , name 8 | , repo 9 | , branch 10 | , commitHash 11 | , narHash 12 | , version 13 | }: 14 | 15 | let 16 | src = fetchgit { 17 | url = repo; 18 | rev = commitHash; 19 | branchName = branch; 20 | 21 | hash = narHash; 22 | }; 23 | 24 | configfile = stdenv.mkDerivation { 25 | inherit src; 26 | name = name + "-configfile"; 27 | 28 | buildInputs = linuxPackages_latest.kernel.buildInputs; 29 | nativeBuildInputs = linuxPackages_latest.kernel.nativeBuildInputs; 30 | 31 | buildPhase = '' 32 | ${virtme-ng}/bin/virtme-ng -v --kconfig --config ${../../kernel.config} 33 | ''; 34 | installPhase = '' 35 | mv .config $out 36 | ''; 37 | }; 38 | 39 | headers = stdenv.mkDerivation { 40 | name = "linux-headers-${version}"; 41 | inherit src version; 42 | 43 | buildInputs = linuxPackages_latest.kernel.buildInputs; 44 | nativeBuildInputs = linuxPackages_latest.kernel.nativeBuildInputs; 45 | 46 | buildPhase = '' 47 | cp ${configfile} .config 48 | make headers 49 | ''; 50 | installPhase = '' 51 | mkdir -p $out 52 | find . -type f \ 53 | \( -path 'usr/include/*' -o -name '*.h' \) \ 54 | -exec cp --parents '{}' $out \; 55 | ''; 56 | }; 57 | in 58 | (linuxManualConfig { 59 | inherit src version configfile; 60 | }).overrideAttrs { 61 | passthru = { 62 | inherit headers; 63 | }; 64 | } 65 | -------------------------------------------------------------------------------- /.github/include/veristat.nix: -------------------------------------------------------------------------------- 1 | { lib 2 | , stdenv 3 | , fetchFromGitHub 4 | , llvmPackages 5 | , clang 6 | , libbpf 7 | , elfutils 8 | , zlib 9 | , pkg-config 10 | , version 11 | , src 12 | }: 13 | 14 | stdenv.mkDerivation { 15 | pname = "veristat"; 16 | inherit version src; 17 | 18 | buildInputs = [ 19 | elfutils 20 | zlib 21 | ]; 22 | 23 | buildPhase = '' 24 | # The Makefile expects to build libbpf from source. We already have a built 25 | # version, and this is a single C file with minimal dependencies, so compile 26 | # and link it by hand. 27 | 28 | cd src 29 | $CC $CFLAGS -I${libbpf}/include -DVERISTAT_VERSION='"${version}"' \ 30 | -c veristat.c -o veristat.o 31 | $CC veristat.o ${libbpf}/lib/libbpf.a -lelf -lz -o veristat 32 | ''; 33 | 34 | installPhase = '' 35 | mkdir -p $out/bin 36 | install -m755 veristat $out/bin/ 37 | ''; 38 | 39 | meta = with lib; { 40 | description = "Tool to provide statistics from the BPF verifier for BPF programs"; 41 | homepage = "https://github.com/libbpf/veristat"; 42 | license = licenses.bsd2; 43 | platforms = platforms.linux; 44 | }; 45 | } 46 | -------------------------------------------------------------------------------- /.github/workflows/6_12.yml: -------------------------------------------------------------------------------- 1 | name: 6_12-test 2 | 3 | on: 4 | # only runs on main, every 6 hours. specify hours explicitly so scheduled runs can be offset and use a random minute. 5 | schedule: 6 | - cron: "46 1,7,13,19 * * *" 7 | 8 | jobs: 9 | build-kernels: 10 | uses: ./.github/workflows/build-kernels.yml 11 | secrets: inherit 12 | 13 | integration-test: 14 | uses: ./.github/workflows/integration-tests.yml 15 | needs: build-kernels 16 | secrets: inherit 17 | with: 18 | repo-name: stable/6_12 19 | 20 | notify-job: 21 | runs-on: ubuntu-latest 22 | if: ${{ failure() && github.ref == 'refs/heads/main' }} 23 | needs: 24 | - integration-test 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Slack Notification 28 | uses: rtCamp/action-slack-notify@v2 29 | env: 30 | SLACK_USERNAME: ci 31 | SLACK_ICON: https://www.dictionary.com/e/wp-content/uploads/2018/03/thisisfine-1.jpg 32 | SLACK_TITLE: Workflow failed 33 | SLACK_MESSAGE: 6.12 ci job failed. 34 | SLACK_COLOR: failure 35 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/bpf-next-test.yml: -------------------------------------------------------------------------------- 1 | name: bpf-next-test 2 | 3 | on: 4 | # only runs on main, every 6 hours. specify hours explicitly so scheduled runs can be offset and use a random minute. 5 | schedule: 6 | - cron: "35 0,6,12,18 * * *" 7 | 8 | jobs: 9 | build-kernels: 10 | uses: ./.github/workflows/build-kernels.yml 11 | secrets: inherit 12 | 13 | integration-test: 14 | uses: ./.github/workflows/integration-tests.yml 15 | needs: build-kernels 16 | secrets: inherit 17 | with: 18 | repo-name: bpf/bpf-next 19 | 20 | notify-job: 21 | runs-on: ubuntu-latest 22 | if: ${{ failure() && github.ref == 'refs/heads/main' }} 23 | needs: 24 | - integration-test 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Slack Notification 28 | uses: rtCamp/action-slack-notify@v2 29 | env: 30 | SLACK_USERNAME: ci 31 | SLACK_ICON: https://www.dictionary.com/e/wp-content/uploads/2018/03/thisisfine-1.jpg 32 | SLACK_TITLE: Workflow failed 33 | SLACK_MESSAGE: bpf-next ci job failed. 34 | SLACK_COLOR: failure 35 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} 36 | 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/build-kernels.yml: -------------------------------------------------------------------------------- 1 | name: build-and-test 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-kernels: 8 | runs-on: ${{ github.repository_owner == 'sched-ext' && fromJSON('[ "self-hosted", "linux", "x64" ]') || 'ubuntu-latest' }} 9 | steps: 10 | - uses: actions/checkout@v4 11 | 12 | - name: Install Nix 13 | uses: ./.github/actions/install-nix 14 | with: 15 | cachix-auth-token: '${{ secrets.CACHIX_AUTH_TOKEN }}' 16 | 17 | - name: Load dependencies 18 | run: nix run ./.github/include#nix-develop-gha -- ./.github/include#gha-build-kernels 19 | 20 | - name: Build all kernels 21 | id: nix-build 22 | run: | 23 | drvnames=$(nix flake show ./.github/include --json | jq -r ' 24 | .packages["x86_64-linux"] | keys[] | select(startswith("kernel_"))' | 25 | awk '{ print "./.github/include#" $0 "\n./.github/include#" $0 ".headers" }') 26 | 27 | echo 'kernel-paths<> $GITHUB_OUTPUT 28 | xargs nix build --no-link --print-out-paths <<< "$drvnames" >>$GITHUB_OUTPUT 29 | echo 'EOF' >> "$GITHUB_OUTPUT" 30 | 31 | - name: Explicitly push to cachix 32 | run: | 33 | # address an edge case where the dedicated runner has a local cache hit 34 | # but that entry wasn't uploaded to cachix 35 | if [ -f "$HOME/.config/cachix/cachix.dhall" ]; then 36 | KERNEL_PATHS="${{ steps.nix-build.outputs.kernel-paths }}" 37 | printf '%s\n' "$KERNEL_PATHS" | xargs cachix push sched-ext 38 | else 39 | echo "no auth token; skipping cache push" 40 | fi 41 | -------------------------------------------------------------------------------- /.github/workflows/stable.yml: -------------------------------------------------------------------------------- 1 | name: stable-test 2 | 3 | on: 4 | # only runs on main, every 6 hours. specify hours explicitly so scheduled runs can be offset and use a random minute. 5 | schedule: 6 | - cron: "40 2,8,14,20 * * *" 7 | 8 | jobs: 9 | build-kernels: 10 | uses: ./.github/workflows/build-kernels.yml 11 | secrets: inherit 12 | 13 | integration-test: 14 | uses: ./.github/workflows/integration-tests.yml 15 | needs: build-kernels 16 | secrets: inherit 17 | with: 18 | repo-name: stable/linux-rolling-stable 19 | 20 | notify-job: 21 | runs-on: ubuntu-latest 22 | if: ${{ failure() && github.ref == 'refs/heads/main' }} 23 | needs: 24 | - integration-test 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Slack Notification 28 | uses: rtCamp/action-slack-notify@v2 29 | env: 30 | SLACK_USERNAME: ci 31 | SLACK_ICON: https://www.dictionary.com/e/wp-content/uploads/2018/03/thisisfine-1.jpg 32 | SLACK_TITLE: Workflow failed 33 | SLACK_MESSAGE: stable ci job failed. 34 | SLACK_COLOR: failure 35 | SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} 36 | 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | libbpf/ 2 | *.gitignore 3 | PKGBUILD 4 | target 5 | *.swp 6 | .cache/ 7 | .vscode/ 8 | tags 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sched-ext/scx/41160ea2ab71d57f025100dd5cb3184b869d4c17/.gitmodules -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "rust/scx_loader", 4 | "rust/scx_rustland_core", 5 | "rust/scx_stats", 6 | "rust/scx_stats/scx_stats_derive", 7 | "rust/scx_userspace_arena", 8 | "rust/scx_utils", 9 | "scheds/rust/scx_bpfland", 10 | "scheds/rust/scx_chaos", 11 | "scheds/rust/scx_flash", 12 | "scheds/rust/scx_lavd", 13 | "scheds/rust/scx_layered", 14 | "scheds/rust/scx_mitosis", 15 | "scheds/rust/scx_p2dq", 16 | "scheds/rust/scx_rlfifo", 17 | "scheds/rust/scx_rustland", 18 | "scheds/rust/scx_rusty", 19 | "scheds/rust/scx_tickless", 20 | "tools/scxctl", 21 | "tools/scxtop", 22 | "tools/vmlinux_docify", 23 | ] 24 | resolver = "2" 25 | 26 | [profile.release] 27 | lto = "thin" 28 | 29 | [profile.release-fast] 30 | inherits = "release" 31 | target-cpu = "native" 32 | lto = false 33 | incremental = true 34 | opt-level = 0 35 | debuginfo = 0 36 | -------------------------------------------------------------------------------- /kernel-versions.json: -------------------------------------------------------------------------------- 1 | { 2 | "sched_ext/for-next": { 3 | "repo": "https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git", 4 | "branch": "for-next", 5 | "commitHash": "9465f44d2df21b43f062de303e59e2b391eeab16", 6 | "lastModified": 1749550997, 7 | "narHash": "sha256-EVOxBnqN+YpFyfuOfyADymxaIUzEsghF8iHvXmopB34=", 8 | "kernelVersion": "6.16.0-rc1" 9 | }, 10 | "bpf/bpf-next": { 11 | "repo": "https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git", 12 | "branch": "master", 13 | "commitHash": "2d72dd14d77f31a7caa619fe0b889304844e612e", 14 | "lastModified": 1749614328, 15 | "narHash": "sha256-9DuXsvyf0d7/iGyTgRTUFEiFYoOQSKqXzsG6Hycm6Hw=", 16 | "kernelVersion": "6.15.0" 17 | }, 18 | "stable/6_12": { 19 | "repo": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git", 20 | "branch": "linux-6.12.y", 21 | "commitHash": "e03ced99c437f4a7992b8fa3d97d598f55453fd0", 22 | "lastModified": 1749614323, 23 | "narHash": "sha256-9TLCE0q7A25F1UIpFB6oHNWJpYiGaFTQ1sYbmT0depY=", 24 | "kernelVersion": "6.12.33" 25 | }, 26 | "stable/linux-rolling-stable": { 27 | "repo": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git", 28 | "branch": "linux-rolling-stable", 29 | "commitHash": "b0d369dbad20a8c9702607e7294d63c4a0c8e0fe", 30 | "lastModified": 1749614376, 31 | "narHash": "sha256-9kmPz/+vDXYsdwDK2IAEBPO7KkfSYAZp4uP7gXwU/8U=", 32 | "kernelVersion": "6.15.2" 33 | } 34 | } -------------------------------------------------------------------------------- /kernel.config: -------------------------------------------------------------------------------- 1 | # sched-ext mandatory options 2 | # 3 | CONFIG_BPF=y 4 | CONFIG_BPF_SYSCALL=y 5 | CONFIG_BPF_JIT=y 6 | CONFIG_DEBUG_INFO_BTF=y 7 | CONFIG_BPF_JIT_ALWAYS_ON=y 8 | CONFIG_BPF_JIT_DEFAULT_ON=y 9 | CONFIG_SCHED_CLASS_EXT=y 10 | 11 | # Required on arm64 12 | # 13 | # CONFIG_DEBUG_INFO_REDUCED is not set 14 | 15 | # Enable scheduling debugging 16 | # 17 | CONFIG_SCHED_DEBUG=y 18 | 19 | # Enable extra scheduling features (for a better code coverage while testing 20 | # the schedulers) 21 | # 22 | CONFIG_SCHED_AUTOGROUP=y 23 | CONFIG_SCHED_CORE=y 24 | CONFIG_SCHED_MC=y 25 | 26 | # Enable fully preemptible kernel for a better test coverage of the schedulers 27 | # 28 | # CONFIG_PREEMPT_NONE is not set 29 | # CONFIG_PREEMPT_VOLUNTARY is not set 30 | CONFIG_PREEMPT=y 31 | CONFIG_PREEMPT_COUNT=y 32 | CONFIG_PREEMPTION=y 33 | CONFIG_PREEMPT_DYNAMIC=y 34 | CONFIG_PREEMPT_RCU=y 35 | 36 | # Additional debugging information (useful to catch potential locking issues) 37 | # 38 | CONFIG_DEBUG_LOCKDEP=y 39 | CONFIG_DEBUG_ATOMIC_SLEEP=y 40 | CONFIG_PROVE_LOCKING=y 41 | 42 | # Bpftrace headers (for additional debug info) 43 | CONFIG_BPF=y 44 | CONFIG_BPF_SYSCALL=y 45 | CONFIG_BPF_JIT=y 46 | CONFIG_HAVE_EBPF_JIT=y 47 | CONFIG_BPF_EVENTS=y 48 | CONFIG_FTRACE_SYSCALLS=y 49 | CONFIG_FUNCTION_TRACER=y 50 | CONFIG_HAVE_DYNAMIC_FTRACE=y 51 | CONFIG_DYNAMIC_FTRACE=y 52 | CONFIG_HAVE_KPROBES=y 53 | CONFIG_KPROBES=y 54 | CONFIG_KPROBE_EVENTS=y 55 | CONFIG_ARCH_SUPPORTS_UPROBES=y 56 | CONFIG_UPROBES=y 57 | CONFIG_UPROBE_EVENTS=y 58 | CONFIG_DEBUG_FS=y 59 | # more bpftrace to make that work 60 | CONFIG_IKHEADERS=y 61 | CONFIG_IKCONFIG_PROC=y 62 | CONFIG_IKCONFIG=y 63 | 64 | -------------------------------------------------------------------------------- /lib/arena.bpf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: GPL-2.0 3 | * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 4 | */ 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | /* 14 | * "System-call" based API for arenas. 15 | */ 16 | 17 | struct task_ctx; 18 | 19 | SEC("syscall") 20 | int arena_init(struct arena_init_args *args) 21 | { 22 | int ret; 23 | 24 | ret = scx_static_init(args->static_pages); 25 | if (ret) 26 | return ret; 27 | 28 | /* How many types to store all CPU IDs? */ 29 | ret = scx_bitmap_init(div_round_up(nr_cpu_ids, 8)); 30 | if (ret) 31 | return ret; 32 | 33 | ret = scx_percpu_storage_init(); 34 | if (ret) 35 | return ret; 36 | 37 | ret = scx_task_init(args->task_ctx_size); 38 | if (ret) 39 | return ret; 40 | 41 | return 0; 42 | } 43 | 44 | SEC("syscall") 45 | int arena_alloc_mask(struct arena_alloc_mask_args *args) 46 | { 47 | scx_bitmap_t bitmap; 48 | 49 | bitmap = scx_bitmap_alloc(); 50 | if (!bitmap) 51 | return -ENOMEM; 52 | 53 | args->bitmap = (u64)&bitmap->bits; 54 | 55 | return 0; 56 | } 57 | 58 | SEC("syscall") 59 | int arena_topology_node_init(struct arena_topology_node_init_args *args) 60 | { 61 | scx_bitmap_t bitmap = (scx_bitmap_t)container_of(args->bitmap, struct scx_bitmap, bits); 62 | int ret; 63 | 64 | ret = topo_init(bitmap, args->data_size, args->id); 65 | if (ret) 66 | return ret; 67 | 68 | return 0; 69 | } 70 | 71 | SEC("syscall") 72 | int arena_topology_print(void) 73 | { 74 | scx_arena_subprog_init(); 75 | 76 | topo_print(); 77 | 78 | return 0; 79 | } 80 | 81 | -------------------------------------------------------------------------------- /lib/meson.build: -------------------------------------------------------------------------------- 1 | libs = ['sdt_alloc', 'sdt_task'] 2 | 3 | objs = [] 4 | 5 | foreach src: libs 6 | # Copy of gen_bpf_o(), because custom targets and generators don't mix 7 | bpf_o = custom_target(src, 8 | output: src + '.bpf.o', 9 | input: src + '.bpf.c', 10 | command: [bpf_clang, bpf_base_cflags, '-target', 'bpf', libbpf_c_headers, bpf_includes, 11 | '-c', '@INPUT@', '-o', '@OUTPUT@'], 12 | depends: [libbpf, bpftool_target] 13 | ) 14 | 15 | objs += [bpf_o] 16 | endforeach 17 | 18 | scx_lib = custom_target(scx_lib_name, 19 | output: scx_lib_name + '.bpf.o', 20 | command: [compile_scx_lib, bpftool_exe_path, '@OUTPUT@', scx_lib_name, objs], 21 | depends: objs, 22 | ) 23 | -------------------------------------------------------------------------------- /lib/sdt_task.bpf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-License-Identifier: GPL-2.0 3 | * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 | * Copyright (c) 2024 Tejun Heo 5 | * Copyright (c) 2024 Emil Tsalapatis 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | /* 12 | * Task BPF map entry recording the task's assigned ID and pointing to the data 13 | * area allocated in arena. 14 | */ 15 | struct scx_task_map_val { 16 | union sdt_id tid; 17 | __u64 tptr; 18 | struct sdt_data __arena *data; 19 | }; 20 | 21 | struct { 22 | __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 23 | __uint(map_flags, BPF_F_NO_PREALLOC); 24 | __type(key, int); 25 | __type(value, struct scx_task_map_val); 26 | } scx_task_map SEC(".maps"); 27 | 28 | struct scx_allocator scx_task_allocator; 29 | 30 | __hidden 31 | void __arena *scx_task_alloc(struct task_struct *p) 32 | { 33 | struct sdt_data __arena *data = NULL; 34 | struct scx_task_map_val *mval; 35 | 36 | mval = bpf_task_storage_get(&scx_task_map, p, 0, 37 | BPF_LOCAL_STORAGE_GET_F_CREATE); 38 | if (!mval) 39 | return NULL; 40 | 41 | data = scx_alloc(&scx_task_allocator); 42 | if (unlikely(!data)) 43 | return NULL; 44 | 45 | mval->tid = data->tid; 46 | mval->tptr = (__u64) p; 47 | mval->data = data; 48 | 49 | return (void __arena *)data->payload; 50 | } 51 | 52 | __hidden 53 | int scx_task_init(__u64 data_size) 54 | { 55 | return scx_alloc_init(&scx_task_allocator, data_size); 56 | } 57 | 58 | __hidden 59 | void __arena *scx_task_data(struct task_struct *p) 60 | { 61 | struct sdt_data __arena *data; 62 | struct scx_task_map_val *mval; 63 | 64 | scx_arena_subprog_init(); 65 | 66 | mval = bpf_task_storage_get(&scx_task_map, p, 0, 0); 67 | if (!mval) 68 | return NULL; 69 | 70 | data = mval->data; 71 | 72 | return (void __arena *)data->payload; 73 | } 74 | 75 | __hidden 76 | void scx_task_free(struct task_struct *p) 77 | { 78 | struct scx_task_map_val *mval; 79 | 80 | scx_arena_subprog_init(); 81 | 82 | mval = bpf_task_storage_get(&scx_task_map, p, 0, 0); 83 | if (!mval) 84 | return; 85 | 86 | scx_alloc_free_idx(&scx_task_allocator, mval->tid.idx); 87 | bpf_task_storage_delete(&scx_task_map, p); 88 | } 89 | -------------------------------------------------------------------------------- /libalpm/openrc/90-scx-scheds-upgrade.hook: -------------------------------------------------------------------------------- 1 | [Trigger] 2 | Type = Path 3 | Operation = Upgrade 4 | Target = etc/conf.d/scx 5 | Target = etc/default/scx 6 | Target = etc/init.d/scx 7 | Target = usr/bin/scx_* 8 | 9 | [Trigger] 10 | Type = Package 11 | Operation = Upgrade 12 | Target = scx-scheds* 13 | 14 | [Action] 15 | Description = Checking scx_scheduler... 16 | When = PostTransaction 17 | Exec = /usr/share/libalpm/scripts/scx-openrc-restart 18 | NeedsTargets 19 | -------------------------------------------------------------------------------- /libalpm/openrc/meson.build: -------------------------------------------------------------------------------- 1 | # Install the 'scx-openrc-restart' file to the '/usr/share/libalpm/scripts' directory 2 | install_data('scx-openrc-restart', install_dir: '/usr/share/libalpm/scripts') 3 | # Install the '90-scx-scheds-upgrade.hook' file to the '/usr/share/libalpm/hooks' directory 4 | install_data('90-scx-scheds-upgrade.hook', install_dir: '/usr/share/libalpm/hooks') 5 | -------------------------------------------------------------------------------- /libalpm/openrc/scx-openrc-restart: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: GPL-2.0-only 3 | 4 | # Check the status of the service 5 | if /etc/init.d/scx status > /dev/null; then 6 | echo "The service is active. Restarting..." 7 | /usr/bin/rc-service scx restart 8 | echo "Service has been restarted." 9 | else 10 | echo "The service is not active." 11 | fi 12 | -------------------------------------------------------------------------------- /libalpm/systemd/90-scx-scheds-upgrade.hook: -------------------------------------------------------------------------------- 1 | [Trigger] 2 | Type = Path 3 | Operation = Upgrade 4 | Target = etc/default/scx 5 | Target = usr/bin/scx_* 6 | Target = usr/lib/systemd/system/scx.service 7 | 8 | [Trigger] 9 | Type = Package 10 | Operation = Upgrade 11 | Target = scx-scheds* 12 | 13 | [Action] 14 | Description = Checking scx_scheduler... 15 | When = PostTransaction 16 | Exec = /usr/share/libalpm/scripts/scx-systemd-restart 17 | NeedsTargets 18 | -------------------------------------------------------------------------------- /libalpm/systemd/meson.build: -------------------------------------------------------------------------------- 1 | # Install the 'scx-systemd-restart' file to the '/usr/share/libalpm/scripts' directory 2 | install_data('scx-systemd-restart', install_dir: '/usr/share/libalpm/scripts') 3 | # Install the '90-scx-scheds-upgrade.hook' file to the '/usr/share/libalpm/hooks' directory 4 | install_data('90-scx-scheds-upgrade.hook', install_dir: '/usr/share/libalpm/hooks') 5 | -------------------------------------------------------------------------------- /libalpm/systemd/scx-systemd-restart: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: GPL-2.0-only 3 | 4 | # Check the status of the service 5 | if systemctl is-active --quiet scx.service; then 6 | echo "The service is active. Restarting..." 7 | systemctl daemon-reload 8 | systemctl restart scx.service 9 | echo "Service has been restarted." 10 | else 11 | echo "The service is not active." 12 | fi 13 | -------------------------------------------------------------------------------- /meson-scripts/bpftool_build_skel: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | bpftool="$1" 6 | input="$2" 7 | skel="$3" 8 | subskel="$4" 9 | 10 | stem="${input%.o}" 11 | name="${input%.bpf.o}" 12 | name="${name##*/}" 13 | 14 | "$bpftool" gen object "$stem".l1o "$input" 15 | "$bpftool" gen object "$stem".l2o "$stem".l1o 16 | "$bpftool" gen object "$stem".l3o "$stem".l2o 17 | cmp "$stem".l2o "$stem".l3o 18 | "$bpftool" gen skeleton "$stem".l3o name "$name" > "$skel" 19 | "$bpftool" gen subskeleton "$stem".l3o name "$name" > "$subskel" 20 | -------------------------------------------------------------------------------- /meson-scripts/bpftool_build_skel_lib: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | bpftool="$1" 6 | input="$2" 7 | skel="$3" 8 | subskel="$4" 9 | lib="$5" 10 | 11 | stem="${input%.o}" 12 | name="${input%.bpf.o}" 13 | name="${name##*/}" 14 | 15 | if [ -z "$lib" ] || [ `basename $lib` == $name ]; 16 | then 17 | "$bpftool" gen object "$stem".l1o "$input" 18 | else 19 | "$bpftool" gen object "$stem".l1o "$input" "$lib".bpf.o 20 | "$bpftool" gen object "$stem".l2o "$stem".l1o 21 | "$bpftool" gen object "$stem".l3o "$stem".l2o 22 | cmp "$stem".l2o "$stem".l3o 23 | "$bpftool" gen skeleton "$stem".l3o name "$name" > "$skel" 24 | "$bpftool" gen subskeleton "$stem".l3o name "$name" > "$subskel" 25 | fi 26 | -------------------------------------------------------------------------------- /meson-scripts/bpftool_dummy.c: -------------------------------------------------------------------------------- 1 | // This is just used as a dummy target file for meson 2 | int main(void) { 3 | return 0; 4 | } 5 | -------------------------------------------------------------------------------- /meson-scripts/build_bpftool: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | out=$("$1" 'map(select(.["file"] | contains ("cc_cflags_probe.c"))) | first | .["command"]' < compile_commands.json) 6 | out=${out#\"} 7 | out=${out%\"} 8 | args=($out) 9 | 10 | idx=0 11 | cc=${args[idx]} 12 | if [ "$cc" = "ccache" -o "$cc" = "sccache" ]; then 13 | idx=$((idx+1)) 14 | cc="$cc ${args[idx]}" 15 | fi 16 | 17 | declare -a cflags=() 18 | 19 | for arg in ${args[@]:(idx+1)}; do 20 | case $arg in 21 | -I*|-M*|-o|-c) ;; 22 | -*) cflags+="$arg ";; 23 | esac 24 | done 25 | 26 | extra_args=() 27 | # When using clang, `LLVM=1` needs to be passed as an argument. Otherwise some 28 | # parts of the build system will attempt to invoke GCC. 29 | if cc --version 2>&1 | grep clang; then 30 | extra_args+="LLVM=1" 31 | fi 32 | 33 | cd $3 34 | make_out=$(env CC="$cc" CFLAGS="$cflags" "$2" -j"$4" $extra_args) 35 | exit $? 36 | -------------------------------------------------------------------------------- /meson-scripts/build_libbpf: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | out=$("$1" 'map(select(.["file"] | contains ("cc_cflags_probe.c"))) | first | .["command"]' < compile_commands.json) 6 | out=${out#\"} 7 | out=${out%\"} 8 | args=($out) 9 | 10 | idx=0 11 | cc=${args[idx]} 12 | if [ "$cc" = "ccache" ]; then 13 | idx=$((idx+1)) 14 | cc="$cc ${args[idx]}" 15 | fi 16 | 17 | if [ "$cc" = "sccache" ]; then 18 | idx=$((idx+1)) 19 | cc="$cc ${args[idx]}" 20 | fi 21 | 22 | if [ "$cc" = "ccache" ]; then 23 | idx=$((idx+1)) 24 | cc="$cc ${args[idx]}" 25 | fi 26 | 27 | if [ "$cc" = "sccache" ]; then 28 | idx=$((idx+1)) 29 | cc="$cc ${args[idx]}" 30 | fi 31 | 32 | declare -a cflags=() 33 | 34 | for arg in ${args[@]:(idx+1)}; do 35 | case $arg in 36 | -I*|-M*|-o|-c) ;; 37 | -*) cflags+="$arg ";; 38 | esac 39 | done 40 | 41 | make_out=$(env CC="$cc" CFLAGS="$cflags" BUILD_STATIC_ONLY=y DESTDIR=. "$2" install -C "$3" -j"$4") 42 | exit $? 43 | -------------------------------------------------------------------------------- /meson-scripts/cargo_fetch: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | shopt -s globstar 4 | set -e 5 | 6 | cargo="$1" 7 | 8 | # 9 | # Run `cargo fetch` for each Cargo.toml found in the source directory. This 10 | # can lead to fetching unnecessary packages if the source directory contains 11 | # past cargo build artifacts. While not ideal, this is a lot simpler and 12 | # shouldn't cause problems in practice. 13 | # 14 | for manifest in ${MESON_SOURCE_ROOT}/**/Cargo.toml; do 15 | echo "Fetching for $manifest" 16 | "$cargo" fetch --manifest-path="$manifest" 17 | done 18 | -------------------------------------------------------------------------------- /meson-scripts/cc_cflags_probe.c: -------------------------------------------------------------------------------- 1 | // This exists only to get the same compiler and flags meson uses. 2 | // See build_libbpf script for more info 3 | int main(void) { 4 | return 0; 5 | } 6 | -------------------------------------------------------------------------------- /meson-scripts/compile_scx_lib: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | bpftool="$1" 6 | output="$2" 7 | lib="$3" 8 | dir="$PWD/lib" 9 | 10 | libs=`find $dir -type f -name *.bpf.o | grep -v $lib.bpf.o` 11 | 12 | ${bpftool} gen object "$output" $libs 13 | -------------------------------------------------------------------------------- /meson-scripts/fetch_bpftool: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | URL=https://github.com/libbpf/bpftool.git 4 | 5 | cd $1 6 | rm -rf bpftool 7 | git clone --depth=1 ${URL} 8 | cd bpftool 9 | git fetch --depth=1 origin $2 || { 10 | echo "commit $2 does not exists in ${URL}" 11 | exit 1 12 | } 13 | git checkout $2 14 | git submodule update --init --recursive 15 | -------------------------------------------------------------------------------- /meson-scripts/fetch_libbpf: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | URL=https://github.com/libbpf/libbpf.git 4 | 5 | unset CDPATH 6 | cd $1 7 | rm -rf libbpf 8 | git clone --depth=1 ${URL} 9 | cd libbpf 10 | git fetch --depth=1 origin $2 || { 11 | echo "commit $2 does not exists in ${URL}" 12 | exit 1 13 | } 14 | git checkout $2 15 | 16 | # This is needed because we haven't built libbpf yet 17 | # and this is where the headers will end up 18 | mkdir src/usr 19 | mkdir src/usr/include 20 | -------------------------------------------------------------------------------- /meson-scripts/get_bpftool_ver: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | "$1" --version | sed -nr 's/^.*bpftool v([\.0-9]*)(git)?( .*)?$/\1/p' 4 | -------------------------------------------------------------------------------- /meson-scripts/get_clang_ver: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | "$1" --version | sed -nr 's/^.*clang version ([\.0-9]*)(git)?(-rc.*)?(\+.*)?( .*)?$/\1/p' 4 | -------------------------------------------------------------------------------- /meson-scripts/get_sys_incls: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | clang="$1" 6 | 7 | # 8 | # Get Clang's default includes on this system, as opposed to those seen by 9 | # '-target bpf'. This fixes "missing" files on some architectures/distros, 10 | # such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. 11 | # 12 | # Use '-idirafter': Don't interfere with include mechanics except where the 13 | # build would have failed anyways. 14 | # 15 | 16 | "$clang" -v -E - < /dev/null 2>&1 \ 17 | | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }' \ 18 | 19 | "$clang" -dM -E - < /dev/null \ 20 | | grep '__riscv_xlen ' \ 21 | | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $3, $3)}' 22 | -------------------------------------------------------------------------------- /meson-scripts/stress_tests.ini: -------------------------------------------------------------------------------- 1 | [scx_lavd] 2 | sched: scx_lavd 3 | sched_args: -v 4 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 5 | timeout_sec: 45 6 | 7 | [scx_rusty] 8 | sched: scx_rusty 9 | sched_args: -v 10 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 11 | timeout_sec: 45 12 | 13 | [scx_rustland] 14 | sched: scx_rustland 15 | sched_args: -v 16 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 17 | timeout_sec: 45 18 | 19 | [scx_bpfland] 20 | sched: scx_bpfland 21 | sched_args: -v 22 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 23 | timeout_sec: 45 24 | 25 | [scx_flash] 26 | sched: scx_flash 27 | sched_args: 28 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 29 | timeout_sec: 45 30 | 31 | [scx_layered] 32 | sched: scx_layered 33 | sched_args: --run-example -v --stats 1 --enable-gpu-support 34 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 35 | timeout_sec: 45 36 | bpftrace_scripts: dsq_lat.bt,process_runqlat.bt 37 | 38 | [scx_p2dq] 39 | sched: scx_p2dq 40 | sched_args: 41 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 42 | timeout_sec: 45 43 | 44 | [scx_chaos] 45 | sched: scx_chaos 46 | sched_args: --random-delay-frequency 0.5 --random-delay-min-us 1000 --random-delay-max-us 2000 47 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 48 | timeout_sec: 45 49 | 50 | [scx_tickless] 51 | sched: scx_tickless 52 | sched_args: -v 53 | stress_cmd: stress-ng -t 31 --aggressive -M -c `nproc` -f `nproc` --affinity 1 --affinity-delay 1 --affinity-pin 54 | timeout_sec: 45 55 | -------------------------------------------------------------------------------- /meson-scripts/veristat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | GUEST_TIMEOUT=25 3 | BUILD_DIR=$1 4 | SCHED=$2 5 | KERNEL=$3 6 | VNG_RW_ARG=$4 7 | 8 | if [ "${KERNEL}" == "vmlinuz" ]; then 9 | unset KERNEL 10 | fi 11 | 12 | VNG_RW='' 13 | if [ "${VNG_RW_ARG}" == "VNG_RW=true" ]; then 14 | VNG_RW=' --rw ' 15 | fi 16 | 17 | cd $BUILD_DIR || exit 1 18 | 19 | if [ -n "${KERNEL}" ] && [ ! -x `which vng` ]; then 20 | echo "vng not found, please install virtme-ng to enable testing" 21 | exit 1 22 | fi 23 | 24 | if [ -n "${SCHED}" ]; then 25 | BPF_PATH=$(find ${BUILD_DIR} -type f -name bpf.bpf.o | grep ${SCHED}) 26 | echo "Running veristat on ${BPF_PATH}" 27 | if [ -n "${KERNEL}" ]; then 28 | timeout --preserve-status ${GUEST_TIMEOUT} \ 29 | vng --user root -m 10G --cpu 8 -v --user root -r ${KERNEL} -- \ 30 | sudo $(which veristat) ${BPF_PATH} 2>&1 | tee veristat.ci.log 31 | exit $? 32 | else 33 | sudo veristat ${BPF_PATH} 2>&1 | tee veristat.ci.log 34 | exit $? 35 | fi 36 | fi 37 | 38 | for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do 39 | if [ -n "${KERNEL}" ]; then 40 | timeout --preserve-status ${GUEST_TIMEOUT} \ 41 | vng --user root -m 10G --cpu 8 $VNG_RW -v --user root -r ${KERNEL} -- \ 42 | sudo $(which veristat) ${BPF_PATH} 2>&1 | tee "$(basename "${BPF_PATH}")-veristat.ci.log" 43 | else 44 | echo "$BPF_PATH" 45 | sudo veristat ${BPF_PATH} 2>&1 | tee "$(basename "${BPF_PATH}")-veristat.ci.log" 46 | fi 47 | done 48 | -------------------------------------------------------------------------------- /pgo-lto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eou pipefail 3 | 4 | PGO_TMPDIR=${1:-/tmp/pgo-data} 5 | SCHED=${2:-scx_rustland} 6 | PGO_DUR=${3:-30} 7 | 8 | rm -rf "$PGO_TMPDIR" 9 | mkdir -p "$PGO_TMPDIR" 10 | 11 | RUSTFLAGS="-C profile-generate=$PGO_TMPDIR -C link-arg=-lgcov" \ 12 | cargo build --release --bin "$SCHED" 13 | 14 | echo "Running sched to generate PGO" 15 | for i in {0..3}; do 16 | sudo "./target/release/$SCHED" & 17 | sleep "$PGO_DUR" 18 | sudo kill -9 $! || echo "$SCHED already dead" 19 | sleep 1 20 | done 21 | 22 | # Merge the `.profraw` files into a `.profdata` file 23 | llvm-profdata merge --failure-mode=warn \ 24 | -o "$PGO_TMPDIR/merged.profdata" \ 25 | "$PGO_TMPDIR" 26 | 27 | # Use the `.profdata` file for guiding optimizations 28 | RUSTFLAGS="-Cprofile-use=$PGO_TMPDIR/merged.profdata" \ 29 | cargo build --release --bin "$SCHED" 30 | 31 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "stable" 3 | -------------------------------------------------------------------------------- /rust/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | -------------------------------------------------------------------------------- /rust/scx_loader/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_loader" 3 | version = "1.0.12" 4 | authors = ["Vladislav Nepogodin "] 5 | edition = "2021" 6 | description = "DBUS on-demand loader of sched-ext schedulers" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 12 | colored = "3.0.0" 13 | ctrlc = { version = "3.1", features = ["termination"] } 14 | log = "0.4.17" 15 | nix = { features = ["process", "signal"], default-features = false, version = "0.29" } 16 | serde = { version = "1.0.215", features = ["derive"] } 17 | sysinfo = "0.33.1" 18 | tokio = { version = "1.42.0", features = ["macros", "sync", "rt-multi-thread", "process"] } 19 | tokio-util = "0.7.13" 20 | toml = "0.8.19" 21 | zbus = { version = "5.3.1", features = ["tokio"], default-features = false } 22 | zvariant = "5.1" 23 | 24 | [lib] 25 | path = "src/lib.rs" 26 | 27 | [[bin]] 28 | name = "scx_loader" 29 | path = "src/main.rs" 30 | 31 | [lints.clippy] 32 | not_unsafe_ptr_arg_deref = "allow" 33 | -------------------------------------------------------------------------------- /rust/scx_loader/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /rust/scx_loader/meson.build: -------------------------------------------------------------------------------- 1 | sched = custom_target('scx_loader', 2 | output: '@PLAINNAME@.__PHONY__', 3 | input: 'Cargo.toml', 4 | command: [cargo, 'build', '--manifest-path=@INPUT@', '--target-dir=@OUTDIR@', 5 | cargo_build_args], 6 | env: cargo_env, 7 | depends: sched_deps, 8 | build_by_default: false, 9 | build_always_stale: true) 10 | -------------------------------------------------------------------------------- /rust/scx_loader/org.scx.Loader.conf: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /rust/scx_loader/src/logger.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Vladislav Nepogodin 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | 8 | use std::env; 9 | 10 | use colored::Colorize; 11 | use log::Level; 12 | use log::Metadata; 13 | use log::Record; 14 | 15 | struct SimpleLogger; 16 | 17 | static LOGGER: SimpleLogger = SimpleLogger; 18 | 19 | impl log::Log for SimpleLogger { 20 | fn enabled(&self, _: &Metadata) -> bool { 21 | true 22 | } 23 | 24 | fn log(&self, record: &Record) { 25 | if self.enabled(record.metadata()) { 26 | let level_str = match record.level() { 27 | Level::Error => "[ERROR]".red(), 28 | Level::Warn => "[WARN]".yellow(), 29 | Level::Info => "[INFO]".red(), 30 | Level::Debug => "[DEBUG]".white(), 31 | Level::Trace => "[TRACE]".black(), 32 | }; 33 | println!("{level_str}: {}", record.args()); 34 | } 35 | } 36 | 37 | fn flush(&self) { 38 | // use std::io::Write; 39 | // io::stdout().flush().unwrap(); 40 | } 41 | } 42 | 43 | pub fn init_logger() -> Result<(), log::SetLoggerError> { 44 | // set log level 45 | let max_log_level = if let Ok(env_log) = env::var("RUST_LOG") { 46 | let env_log = env_log.to_lowercase(); 47 | match env_log.as_str() { 48 | "trace" => log::LevelFilter::Trace, 49 | "debug" => log::LevelFilter::Debug, 50 | "warn" => log::LevelFilter::Warn, 51 | "error" => log::LevelFilter::Error, 52 | _ => log::LevelFilter::Info, 53 | } 54 | } else { 55 | log::LevelFilter::Info 56 | }; 57 | 58 | log::set_logger(&LOGGER).map(|()| log::set_max_level(max_log_level)) 59 | } 60 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_rustland_core" 3 | version = "2.3.1" 4 | edition = "2021" 5 | authors = ["Andrea Righi "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "Framework to implement sched_ext schedulers running in user space" 9 | 10 | [dependencies] 11 | anyhow = "1.0.65" 12 | plain = "0.2.3" 13 | libbpf-rs = "=0.25.0-beta.1" 14 | libc = "0.2.137" 15 | seccomp = "0.1" 16 | scx_utils = { path = "../scx_utils", version = "1.0.15" } 17 | 18 | [build-dependencies] 19 | tar = "0.4" 20 | walkdir = "2.5" 21 | scx_utils = { path = "../scx_utils", version = "1.0.15" } 22 | 23 | [lib] 24 | name = "scx_rustland_core" 25 | path = "src/lib.rs" 26 | include = [ 27 | "assets/bpf/intf.h", 28 | "assets/bpf/main.bpf.c", 29 | "assets/bpf.rs", 30 | ] 31 | 32 | [lints.clippy] 33 | not_unsafe_ptr_arg_deref = "allow" 34 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /rust/scx_rustland_core/bindings.h: -------------------------------------------------------------------------------- 1 | #include "bpf_h/vmlinux/vmlinux.h" 2 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/bpf_h: -------------------------------------------------------------------------------- 1 | ../../scheds/include -------------------------------------------------------------------------------- /rust/scx_rustland_core/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use scx_utils::Builder; 7 | 8 | fn main() { 9 | Builder::new().build() 10 | } 11 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/src/bindings.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_upper_case_globals)] 2 | #![allow(non_camel_case_types)] 3 | #![allow(non_snake_case)] 4 | #![allow(dead_code)] 5 | 6 | include!(concat!(env!("OUT_DIR"), "/bindings.rs")); 7 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | #![allow(non_upper_case_globals)] 5 | #![allow(non_camel_case_types)] 6 | #![allow(non_snake_case)] 7 | #![allow(dead_code)] 8 | 9 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 10 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 5 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod bindings; 2 | 3 | pub const VERSION: &'static str = env!("CARGO_PKG_VERSION"); 4 | 5 | mod alloc; 6 | pub use alloc::ALLOCATOR; 7 | 8 | mod rustland_builder; 9 | pub use rustland_builder::RustLandBuilder; 10 | -------------------------------------------------------------------------------- /rust/scx_rustland_core/src/rustland_builder.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Andrea Righi 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use anyhow::Result; 7 | 8 | use scx_utils::BpfBuilder; 9 | use std::fs; 10 | use std::fs::File; 11 | use std::io::Write; 12 | use std::path::Path; 13 | 14 | pub struct RustLandBuilder { 15 | inner_builder: BpfBuilder, 16 | } 17 | 18 | impl RustLandBuilder { 19 | pub fn new() -> Result { 20 | Ok(Self { 21 | inner_builder: BpfBuilder::new()?, 22 | }) 23 | } 24 | 25 | fn create_file(&self, file_name: &str, content: &[u8]) { 26 | let path = Path::new(file_name); 27 | 28 | // Limit file writing to when file contents differ (for caching) 29 | if fs::read(path).map_or(false, |b| b == content) { 30 | return; 31 | } 32 | 33 | let mut file = File::create(path).expect("Unable to create file"); 34 | file.write_all(content).expect("Unable to write to file"); 35 | } 36 | 37 | pub fn build(&mut self) -> Result<()> { 38 | // Embed the BPF source files. 39 | let intf = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/assets/bpf/intf.h")); 40 | let skel = include_bytes!(concat!( 41 | env!("CARGO_MANIFEST_DIR"), 42 | "/assets/bpf/main.bpf.c" 43 | )); 44 | let bpf = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/assets/bpf.rs")); 45 | 46 | // Generate BPF backend code (C). 47 | self.create_file("intf.h", intf); 48 | self.create_file("main.bpf.c", skel); 49 | 50 | self.inner_builder.enable_intf("intf.h", "bpf_intf.rs"); 51 | self.inner_builder.enable_skel("main.bpf.c", "bpf"); 52 | 53 | // Generate user-space BPF connector code (Rust). 54 | self.create_file("src/bpf.rs", bpf); 55 | 56 | // Build the scheduler. 57 | self.inner_builder.build() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /rust/scx_stats/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_stats" 3 | version = "1.0.12" 4 | edition = "2021" 5 | authors = ["Tejun Heo "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "Statistics transport library for sched_ext schedulers" 9 | 10 | [dependencies] 11 | anyhow = "1.0.65" 12 | crossbeam = "0.8.4" 13 | libc = "0.2.137" 14 | log = "0.4.17" 15 | proc-macro2 = "1.0" 16 | quote = "1.0" 17 | serde = { version = "1.0.215", features = ["derive"] } 18 | serde_json = "1.0.133" 19 | syn = { version = "2.0", features = ["extra-traits", "full"] } 20 | 21 | [dev-dependencies] 22 | scx_stats_derive = { path = "scx_stats_derive" } 23 | simple_logger = "5.0" 24 | 25 | [lints.clippy] 26 | not_unsafe_ptr_arg_deref = "allow" 27 | -------------------------------------------------------------------------------- /rust/scx_stats/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /rust/scx_stats/examples/client.rs: -------------------------------------------------------------------------------- 1 | use scx_stats::prelude::*; 2 | use scx_stats_derive::Stats; 3 | use serde::{Deserialize, Serialize}; 4 | use std::collections::BTreeMap; 5 | use std::env::args; 6 | 7 | // Hacky definition sharing. See stats_def.rs.h. 8 | include!("stats_defs.rs.h"); 9 | 10 | fn main() { 11 | simple_logger::SimpleLogger::new() 12 | .with_level(log::LevelFilter::Info) 13 | .env() 14 | .init() 15 | .unwrap(); 16 | 17 | std::assert_eq!(args().len(), 2, "Usage: client UNIX_SOCKET_PATH"); 18 | let path = args().nth(1).unwrap(); 19 | 20 | let mut client = StatsClient::new().set_path(path).connect().unwrap(); 21 | 22 | println!("===== Requesting \"stats_meta\":"); 23 | let resp = client.request::>("stats_meta", vec![]); 24 | println!("{:#?}", resp); 25 | 26 | println!("\n===== Requesting \"stats\" without arguments:"); 27 | let resp = client.request::("stats", vec![]); 28 | println!("{:#?}", resp); 29 | 30 | println!("\n===== Requesting \"stats\" with \"target\"=\"non-existent\":"); 31 | let resp = 32 | client.request::("stats", vec![("target".into(), "non-existent".into())]); 33 | println!("{:#?}", resp); 34 | 35 | println!("\n===== Requesting \"stats\" with \"target\"=\"all\":"); 36 | let resp = client.request::("stats", vec![("target".into(), "top".into())]); 37 | println!("{:#?}", resp); 38 | 39 | println!("\n===== Requesting \"stats\" but receiving with serde_json::Value:"); 40 | let resp = client.request::("stats", vec![("target".into(), "top".into())]); 41 | println!("{:#?}", resp); 42 | 43 | println!("\n===== Requesting \"stats_meta\" but receiving with serde_json::Value:"); 44 | let resp = client 45 | .request::("stats_meta", vec![]) 46 | .unwrap(); 47 | println!("{}", serde_json::to_string_pretty(&resp).unwrap()); 48 | } 49 | -------------------------------------------------------------------------------- /rust/scx_stats/examples/stats_defs.rs.h: -------------------------------------------------------------------------------- 1 | // To be included from server.rs and client.rs examples. This would usually 2 | // be done through the usual pub struct definitions but it's cumbersome to 3 | // do in the examples directory, so work around with c-like includes. 4 | 5 | #[derive(Clone, Debug, Serialize, Deserialize, Stats)] 6 | #[stat(desc = "domain statistics", _om_prefix="d_", _om_label="domain_name")] 7 | struct DomainStats { 8 | pub name: String, 9 | #[stat(desc = "an event counter")] 10 | pub events: u64, 11 | #[stat(desc = "a gauge number")] 12 | pub pressure: f64, 13 | } 14 | 15 | #[derive(Clone, Debug, Serialize, Deserialize, Stats)] 16 | #[stat(desc = "cluster statistics", top)] 17 | struct ClusterStats { 18 | pub name: String, 19 | #[stat(desc = "update timestamp")] 20 | pub at: u64, 21 | #[stat(desc = "some bitmap we want to report", _om_skip)] 22 | pub bitmap: Vec, 23 | #[stat(desc = "domain statistics")] 24 | pub doms_dict: BTreeMap, 25 | } 26 | -------------------------------------------------------------------------------- /rust/scx_stats/scx_stats_derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_stats_derive" 3 | version = "1.0.12" 4 | edition = "2021" 5 | authors = ["Tejun Heo "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "Derive macro for scx_stats" 9 | 10 | [lib] 11 | proc-macro = true 12 | 13 | [dependencies] 14 | proc-macro2 = "1.0" 15 | quote = "1.0" 16 | scx_stats = { path = "..", version = "1.0.12" } 17 | serde_json = "1.0.133" 18 | syn = { version = "2.0", features = ["extra-traits", "full"] } 19 | 20 | [lints.clippy] 21 | not_unsafe_ptr_arg_deref = "allow" 22 | -------------------------------------------------------------------------------- /rust/scx_stats/scx_stats_derive/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /rust/scx_stats/scx_stats_derive/README.md: -------------------------------------------------------------------------------- 1 | # Derive macro crate for `scx_stats` 2 | 3 | See [`scx_stats`](../README.md). 4 | -------------------------------------------------------------------------------- /rust/scx_stats/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use serde_json; 2 | 3 | mod stats; 4 | pub use stats::{ 5 | Meta, StatsAttr, StatsData, StatsField, StatsFieldAttrs, StatsKind, StatsMeta, StatsMetaAux, 6 | StatsStructAttrs, 7 | }; 8 | 9 | mod server; 10 | pub use server::{ 11 | StatsCloser, StatsErrno, StatsOpener, StatsOps, StatsReader, StatsReaderSend, StatsReaderSync, 12 | StatsRequest, StatsResponse, StatsServer, StatsServerData, ToJson, 13 | }; 14 | 15 | mod client; 16 | pub use client::StatsClient; 17 | 18 | pub mod prelude { 19 | pub use crate::*; 20 | } 21 | -------------------------------------------------------------------------------- /rust/scx_userspace_arena/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_userspace_arena" 3 | version = "1.0.12" 4 | edition = "2021" 5 | authors = ["Jake Hillion "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "Utilities for interacting with BPF arenas from userspace in sched_ext schedulers" 9 | 10 | [package.metadata.scx] 11 | ci.use_clippy = true 12 | 13 | [dependencies] 14 | scx_utils = { path = "../scx_utils", version = "1.0.15" } 15 | 16 | anyhow = "1.0.65" 17 | buddy_system_allocator = { version = "0.11.0", default-features = false } 18 | libbpf-rs = "=0.25.0-beta.1" 19 | 20 | [build-dependencies] 21 | scx_utils = { path = "../scx_utils", version = "1.0.15" } 22 | -------------------------------------------------------------------------------- /rust/scx_userspace_arena/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .build() 11 | .unwrap(); 12 | } 13 | -------------------------------------------------------------------------------- /rust/scx_userspace_arena/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 4 | * Copyright (c) 2025 Jake Hillion 5 | */ 6 | #include 7 | -------------------------------------------------------------------------------- /rust/scx_userspace_arena/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /rust/scx_userspace_arena/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | pub mod alloc; 7 | 8 | mod bpf_intf; 9 | -------------------------------------------------------------------------------- /rust/scx_utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_utils" 3 | version = "1.0.15" 4 | edition = "2021" 5 | authors = ["Tejun Heo "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "Utilities for sched_ext schedulers" 9 | 10 | [dependencies] 11 | anyhow = "1.0.65" 12 | bitvec = { version = "1.0", features = ["serde"] } 13 | bindgen = ">=0.69" 14 | glob = "0.3.2" 15 | hex = "0.4.3" 16 | lazy_static = "1.5.0" 17 | libbpf-sys = "=1.4.6" 18 | libbpf-cargo = "=0.25.0-beta.1" 19 | libbpf-rs = "=0.25.0-beta.1" 20 | log = "0.4.17" 21 | nvml-wrapper = { version = "0.11.0", optional = true } 22 | nvml-wrapper-sys = { version = "0.9.0", optional = true } 23 | paste = "1.0" 24 | regex = "1.11.1" 25 | scx_stats = { path = "../scx_stats", version = "1.0.12" } 26 | serde = { version = "1.0.215", features = ["derive"] } 27 | sscanf = "0.4" 28 | tar = "0.4" 29 | walkdir = "2.5" 30 | version-compare = "0.1" 31 | libc = "0.2.137" 32 | zbus = { version = "5.3.1", optional = true } 33 | 34 | [dev-dependencies] 35 | tempfile = "3.19.1" 36 | 37 | [build-dependencies] 38 | anyhow = "1.0.65" 39 | bindgen = ">=0.69" 40 | glob = "0.3.2" 41 | lazy_static = "1.5.0" 42 | libbpf-cargo = "=0.25.0-beta.1" 43 | sscanf = "0.4" 44 | tar = "0.4" 45 | vergen = { version = "8.0.0", features = ["cargo", "git", "gitcl"] } 46 | version-compare = "0.1" 47 | walkdir = "2.5" 48 | 49 | [features] 50 | default = [] 51 | gpu-topology = ["dep:nvml-wrapper", "dep:nvml-wrapper-sys"] 52 | autopower = ["dep:zbus"] 53 | 54 | [[example]] 55 | name = "mangolog" 56 | crate-type = ["bin"] 57 | 58 | [lints.clippy] 59 | not_unsafe_ptr_arg_deref = "allow" 60 | 61 | -------------------------------------------------------------------------------- /rust/scx_utils/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /rust/scx_utils/README.md: -------------------------------------------------------------------------------- 1 | # Utility collection for sched_ext schedulers 2 | 3 | [sched_ext](https://github.com/sched-ext/scx) is a Linux kernel feature 4 | which enables implementing kernel thread schedulers in BPF and dynamically 5 | loading them. 6 | 7 | This crate is a collection of utilities for sched_ext scheduler 8 | implementations which use Rust for userspace component. This enables 9 | implementing hot paths in BPF while offloading colder and more complex 10 | operations to userspace Rust code which can be significantly more convenient 11 | and powerful. 12 | 13 | Please see [documentation](https://docs.rs/scx_utils/latest/scx_utils/) for 14 | more details. 15 | -------------------------------------------------------------------------------- /rust/scx_utils/bpf_h: -------------------------------------------------------------------------------- 1 | ../../scheds/include -------------------------------------------------------------------------------- /rust/scx_utils/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use vergen::EmitBuilder; 7 | include!("src/builder.rs"); 8 | 9 | fn main() { 10 | Builder::new().build(); 11 | EmitBuilder::builder() 12 | .git_sha(true) 13 | .git_dirty(true) 14 | .cargo_target_triple() 15 | .emit() 16 | .unwrap(); 17 | 18 | let bindings = bindgen::Builder::default() 19 | .header("perf_wrapper.h") 20 | .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) 21 | .prepend_enum_name(false) 22 | .derive_default(true) 23 | .generate() 24 | .expect("Unable to generate bindings"); 25 | 26 | let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); 27 | bindings 28 | .write_to_file(out_path.join("perf_bindings.rs")) 29 | .expect("Couldn't write bindings!"); 30 | } 31 | -------------------------------------------------------------------------------- /rust/scx_utils/perf_wrapper.h: -------------------------------------------------------------------------------- 1 | // This file is consumed by bindgen, called from our build.rs file. 2 | 3 | #include 4 | 5 | // for __NR_perf_event_open 6 | #include 7 | #include 8 | 9 | // bindgen won't capture preprocessor macro definitions, so we have to do this. 10 | enum perf_event_ioctls { 11 | ENABLE = PERF_EVENT_IOC_ENABLE, 12 | DISABLE = PERF_EVENT_IOC_DISABLE, 13 | REFRESH = PERF_EVENT_IOC_REFRESH, 14 | RESET = PERF_EVENT_IOC_RESET, 15 | PERIOD = PERF_EVENT_IOC_PERIOD, 16 | SET_OUTPUT = PERF_EVENT_IOC_SET_OUTPUT, 17 | SET_FILTER = PERF_EVENT_IOC_SET_FILTER, 18 | ID = PERF_EVENT_IOC_ID, 19 | SET_BPF = PERF_EVENT_IOC_SET_BPF, 20 | PAUSE_OUTPUT = PERF_EVENT_IOC_PAUSE_OUTPUT, 21 | QUERY_BPF = PERF_EVENT_IOC_QUERY_BPF, 22 | MODIFY_ATTRIBUTES = PERF_EVENT_IOC_MODIFY_ATTRIBUTES, 23 | }; 24 | -------------------------------------------------------------------------------- /rust/scx_utils/src/bindings.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_upper_case_globals)] 2 | #![allow(non_camel_case_types)] 3 | #![allow(non_snake_case)] 4 | #![allow(dead_code)] 5 | 6 | include!(concat!(env!("OUT_DIR"), "/bindings.rs")); 7 | -------------------------------------------------------------------------------- /rust/scx_utils/src/build_id.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use std::fmt::Write; 7 | 8 | lazy_static::lazy_static! { 9 | static ref GIT_VERSION: String = { 10 | let mut ver = String::new(); 11 | match option_env!("VERGEN_GIT_SHA") { 12 | Some(v) if v != "VERGEN_IDEMPOTENT_OUTPUT" => { 13 | ver += "g"; 14 | ver += v; 15 | if let Some("true") = option_env!("VERGEN_GIT_DIRTY") { 16 | ver += "-dirty"; 17 | } 18 | } 19 | _ => {} 20 | } 21 | ver 22 | }; 23 | static ref BUILD_TAG: String = { 24 | let mut tag = env!("VERGEN_CARGO_TARGET_TRIPLE").to_string(); 25 | if cfg!(debug_assertions) { 26 | write!(tag, "/debug").unwrap(); 27 | } 28 | tag 29 | }; 30 | } 31 | 32 | pub fn full_version(semver: &str) -> String { 33 | let mut ver = semver.to_string(); 34 | if !GIT_VERSION.is_empty() { 35 | write!(ver, "-{}", &*GIT_VERSION).unwrap(); 36 | } 37 | if !BUILD_TAG.is_empty() { 38 | write!(ver, " {}", &*BUILD_TAG).unwrap(); 39 | } 40 | ver 41 | } 42 | 43 | lazy_static::lazy_static! { 44 | pub static ref SCX_CARGO_VERSION: &'static str = env!("CARGO_PKG_VERSION"); 45 | pub static ref SCX_FULL_VERSION: String = full_version(*SCX_CARGO_VERSION); 46 | } 47 | 48 | #[cfg(test)] 49 | mod tests { 50 | #[test] 51 | fn test_cargo_ver() { 52 | //assert_eq!(super::*SCX_CARGO_VERSION, 1); 53 | println!("{}", *super::SCX_CARGO_VERSION); 54 | } 55 | 56 | #[test] 57 | fn test_full_ver() { 58 | //assert_eq!(super::*SCX_CARGO_VERSION, 1); 59 | println!("{}", *super::SCX_FULL_VERSION); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rust/scx_utils/src/builder.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use std::{ 7 | fs::File, 8 | path::{Path, PathBuf}, 9 | }; 10 | 11 | include!("clang_info.rs"); 12 | 13 | const BPF_H: &str = "bpf_h"; 14 | 15 | pub struct Builder; 16 | 17 | impl Builder { 18 | pub fn new() -> Self { 19 | Builder 20 | } 21 | 22 | fn gen_bpf_h(&self) { 23 | let out_dir = env::var("OUT_DIR").unwrap(); 24 | let file = File::create(PathBuf::from(&out_dir).join(format!("{}.tar", BPF_H))).unwrap(); 25 | let mut ar = tar::Builder::new(file); 26 | 27 | ar.follow_symlinks(false); 28 | ar.append_dir_all(".", BPF_H).unwrap(); 29 | ar.finish().unwrap(); 30 | 31 | for ent in walkdir::WalkDir::new(BPF_H) { 32 | let ent = ent.unwrap(); 33 | if !ent.file_type().is_dir() { 34 | println!("cargo:rerun-if-changed={}", ent.path().to_string_lossy()); 35 | } 36 | } 37 | } 38 | 39 | fn gen_bindings(&self) { 40 | let out_dir = env::var("OUT_DIR").unwrap(); 41 | let clang = ClangInfo::new().unwrap(); 42 | let kernel_target = clang.kernel_target().unwrap(); 43 | let vmlinux_h = Path::new(&BPF_H) 44 | .join("arch") 45 | .join(kernel_target) 46 | .join("vmlinux.h") 47 | .to_str() 48 | .unwrap() 49 | .to_string(); 50 | 51 | let bindings = bindgen::Builder::default() 52 | .header(vmlinux_h) 53 | .allowlist_type("scx_exit_kind") 54 | .allowlist_type("scx_consts") 55 | .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) 56 | .generate() 57 | .expect("Unable to generate bindings"); 58 | 59 | bindings 60 | .write_to_file(PathBuf::from(&out_dir).join("bindings.rs")) 61 | .expect("Couldn't write bindings"); 62 | } 63 | 64 | pub fn build(self) { 65 | self.gen_bpf_h(); 66 | self.gen_bindings(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /rust/scx_utils/src/libbpf_logger.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use libbpf_rs::{set_print, PrintLevel}; 7 | 8 | fn print_to_log(level: PrintLevel, msg: String) { 9 | match level { 10 | PrintLevel::Debug => log::debug!("{}", msg), 11 | PrintLevel::Info => log::info!("{}", msg), 12 | PrintLevel::Warn => log::warn!("{}", msg), 13 | } 14 | } 15 | 16 | pub fn init_libbpf_logging(level: Option) { 17 | set_print(Some((level.unwrap_or(PrintLevel::Debug), print_to_log))); 18 | } 19 | -------------------------------------------------------------------------------- /rust/scx_utils/src/perf.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_upper_case_globals)] 2 | #![allow(non_camel_case_types)] 3 | #![allow(non_snake_case)] 4 | 5 | use libc::pid_t; 6 | use std::os::raw::{c_int, c_ulong}; 7 | 8 | /// The `perf_event_open` system call. 9 | /// 10 | /// See the [`perf_event_open(2) man page`][man] for details. 11 | /// 12 | /// On error, this returns -1, and the C `errno` value (accessible via 13 | /// `std::io::Error::last_os_error`) is set to indicate the error. 14 | /// 15 | /// Note: The `attrs` argument needs to be a `*mut` because if the `size` field 16 | /// is too small or too large, the kernel writes the size it was expecing back 17 | /// into that field. It might do other things as well. 18 | /// 19 | /// # Safety 20 | /// 21 | /// The `attrs` argument must point to a properly initialized 22 | /// `perf_event_attr` struct. The measurements and other behaviors its 23 | /// contents request must be safe. 24 | /// 25 | /// [man]: https://www.mankier.com/2/perf_event_open 26 | pub unsafe fn perf_event_open( 27 | attrs: *mut bindings::perf_event_attr, 28 | pid: pid_t, 29 | cpu: c_int, 30 | group_fd: c_int, 31 | flags: c_ulong, 32 | ) -> c_int { 33 | unsafe { 34 | libc::syscall( 35 | bindings::__NR_perf_event_open as libc::c_long, 36 | attrs as *const bindings::perf_event_attr, 37 | pid, 38 | cpu, 39 | group_fd, 40 | flags, 41 | ) as c_int 42 | } 43 | } 44 | 45 | pub mod bindings { 46 | include!(concat!(env!("OUT_DIR"), "/perf_bindings.rs")); 47 | } 48 | 49 | pub mod ioctls { 50 | use crate::perf; 51 | use std::os::raw::{c_int, c_uint}; 52 | 53 | #[allow(clippy::missing_safety_doc)] 54 | pub unsafe fn enable(fd: c_int, arg: c_uint) -> c_int { 55 | unsafe { libc::ioctl(fd, perf::bindings::ENABLE as libc::Ioctl, arg) } 56 | } 57 | 58 | #[allow(clippy::missing_safety_doc)] 59 | pub unsafe fn reset(fd: c_int, arg: c_uint) -> c_int { 60 | unsafe { libc::ioctl(fd, perf::bindings::RESET as libc::Ioctl, arg) } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /rust/scx_utils/src/pm.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use anyhow::{anyhow, Result}; 7 | use std::fs::{File, OpenOptions}; 8 | use std::io::Write; 9 | use std::path::Path; 10 | 11 | /// Updates the global idle resume latency. When the returned file is closed the request is 12 | /// dropped. See the following kernel docs for more details: 13 | /// https://www.kernel.org/doc/html/latest/admin-guide/pm/cpuidle.html#power-management-quality-of-service-for-cpus 14 | pub fn update_global_idle_resume_latency(value_us: i32) -> Result { 15 | if value_us < 0 { 16 | return Err(anyhow!("Latency value must be non-negative")); 17 | } 18 | 19 | let mut file = OpenOptions::new() 20 | .write(true) 21 | .open("/dev/cpu_dma_latency")?; 22 | let bytes = value_us.to_le_bytes(); // Convert to little-endian bytes 23 | file.write_all(&bytes)?; 24 | Ok(file) // return file descriptor so it can be closed later 25 | } 26 | 27 | /// Updates per cpu idle resume latency. 28 | pub fn update_cpu_idle_resume_latency(cpu_num: usize, value_us: i32) -> Result<()> { 29 | if value_us < 0 { 30 | return Err(anyhow!("Latency value must be non-negative")); 31 | } 32 | 33 | let path = format!( 34 | "/sys/devices/system/cpu/cpu{}/power/pm_qos_resume_latency_us", 35 | cpu_num 36 | ); 37 | 38 | let mut file = File::create(Path::new(&path))?; 39 | write!(file, "{}", value_us)?; 40 | Ok(()) 41 | } 42 | 43 | /// Returns if idle resume latency is supported. 44 | pub fn cpu_idle_resume_latency_supported() -> bool { 45 | std::fs::exists("/sys/devices/system/cpu/cpu0/power/pm_qos_resume_latency_us").unwrap_or(false) 46 | } 47 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # this file is here to prevent build breakages on arch 2 | edition = "2021" 3 | 4 | -------------------------------------------------------------------------------- /scheds/README.md: -------------------------------------------------------------------------------- 1 | SCHED_EXT SCHEDULERS 2 | ==================== 3 | 4 | # Introduction 5 | 6 | This directory contains the repo's schedulers. 7 | 8 | Some of these schedulers are simply examples of different types of schedulers 9 | that can be built using sched_ext. They can be loaded and used to schedule on 10 | your system, but their primary purpose is to illustrate how various features of 11 | sched_ext can be used. 12 | 13 | Other schedulers are actually performant, production-ready schedulers. That is, 14 | for the correct workload and with the correct tuning, they may be deployed in a 15 | production environment with acceptable or possibly even improved performance. 16 | Some of the examples could be improved to become production schedulers. 17 | 18 | Please see the following README files for details on each of the various types 19 | of schedulers: 20 | 21 | - [rust](rust/README.md) describes all of the schedulers with rust 22 | user space components. All of these schedulers are production ready. 23 | - [c](c/README.md) describes all of the schedulers with C user space 24 | components. All of these schedulers are production ready. 25 | 26 | ## Note on syncing 27 | 28 | Note that there is a [sync-to-kernel.sh](sync-to-kernel.sh) script in this 29 | directory. This is used to sync any changes to the specific schedulers 30 | with the Linux kernel tree. If you've made any changes to a scheduler in please 31 | use the script to synchronize with the sched_ext Linux 32 | kernel tree: 33 | 34 | ``` 35 | $ ./sync-to-kernel.sh /path/to/kernel/tree 36 | ``` 37 | -------------------------------------------------------------------------------- /scheds/c/meson.build: -------------------------------------------------------------------------------- 1 | c_scheds = ['scx_simple', 'scx_qmap', 'scx_central', 'scx_userland', 'scx_nest', 2 | 'scx_flatcg', 'scx_pair', 'scx_prev'] 3 | 4 | c_scheds_lib = ['scx_sdt'] 5 | 6 | thread_dep = dependency('threads') 7 | 8 | foreach sched: c_scheds 9 | bpf_o = gen_bpf_o.process(sched + '.bpf.c') 10 | bpf_skel = gen_bpf_skel.process(bpf_o) 11 | executable(sched, [bpf_skel, sched + '.c'], 12 | include_directories: [user_c_includes], 13 | dependencies: [kernel_dep, libbpf_dep, thread_dep], 14 | install: true) 15 | endforeach 16 | 17 | foreach sched: c_scheds_lib 18 | bpf_o = gen_bpf_o.process(sched + '.bpf.c') 19 | bpf_skel = gen_bpf_skel_lib.process(bpf_o) 20 | executable(sched, [bpf_skel, sched + '.c'], 21 | include_directories: [user_c_includes], 22 | dependencies: [kernel_dep, libbpf_dep, thread_dep], 23 | install: true) 24 | endforeach 25 | -------------------------------------------------------------------------------- /scheds/c/scx_flatcg.h: -------------------------------------------------------------------------------- 1 | #ifndef __SCX_EXAMPLE_FLATCG_H 2 | #define __SCX_EXAMPLE_FLATCG_H 3 | 4 | enum { 5 | FCG_HWEIGHT_ONE = 1LLU << 16, 6 | }; 7 | 8 | enum fcg_stat_idx { 9 | FCG_STAT_ACT, 10 | FCG_STAT_DEACT, 11 | FCG_STAT_LOCAL, 12 | FCG_STAT_GLOBAL, 13 | 14 | FCG_STAT_HWT_UPDATES, 15 | FCG_STAT_HWT_CACHE, 16 | FCG_STAT_HWT_SKIP, 17 | FCG_STAT_HWT_RACE, 18 | 19 | FCG_STAT_ENQ_SKIP, 20 | FCG_STAT_ENQ_RACE, 21 | 22 | FCG_STAT_CNS_KEEP, 23 | FCG_STAT_CNS_EXPIRE, 24 | FCG_STAT_CNS_EMPTY, 25 | FCG_STAT_CNS_GONE, 26 | 27 | FCG_STAT_PNC_NO_CGRP, 28 | FCG_STAT_PNC_NEXT, 29 | FCG_STAT_PNC_EMPTY, 30 | FCG_STAT_PNC_GONE, 31 | FCG_STAT_PNC_RACE, 32 | FCG_STAT_PNC_FAIL, 33 | 34 | FCG_STAT_BAD_REMOVAL, 35 | 36 | FCG_NR_STATS, 37 | }; 38 | 39 | struct fcg_cgrp_ctx { 40 | u32 nr_active; 41 | u32 nr_runnable; 42 | u32 queued; 43 | u32 weight; 44 | u32 hweight; 45 | u64 child_weight_sum; 46 | u64 hweight_gen; 47 | s64 cvtime_delta; 48 | u64 tvtime_now; 49 | }; 50 | 51 | #endif /* __SCX_EXAMPLE_FLATCG_H */ 52 | -------------------------------------------------------------------------------- /scheds/c/scx_nest.h: -------------------------------------------------------------------------------- 1 | #ifndef __SCX_NEST_H 2 | #define __SCX_NEST_H 3 | 4 | enum nest_stat_group { 5 | STAT_GRP_WAKEUP, 6 | STAT_GRP_NEST, 7 | STAT_GRP_CONSUME, 8 | }; 9 | 10 | #define NEST_STAT(__stat) BPFSTAT_##__stat 11 | #define NEST_ST(__stat, __grp, __desc) NEST_STAT(__stat), 12 | enum nest_stat_idx { 13 | #include "scx_nest_stats_table.h" 14 | NEST_ST(NR, 0, 0) 15 | }; 16 | #undef NEST_ST 17 | 18 | #endif /* __SCX_NEST_H */ 19 | -------------------------------------------------------------------------------- /scheds/c/scx_nest_stats_table.h: -------------------------------------------------------------------------------- 1 | NEST_ST(WAKEUP_ATTACHED, STAT_GRP_WAKEUP, "Attached CPU was idle, and in primary nest") 2 | NEST_ST(WAKEUP_PREV_PRIMARY, STAT_GRP_WAKEUP, "Previous CPU was idle, and in primary nest") 3 | NEST_ST(WAKEUP_FULLY_IDLE_PRIMARY, STAT_GRP_WAKEUP, "Woken up to fully idle primary nest core") 4 | NEST_ST(WAKEUP_ANY_IDLE_PRIMARY, STAT_GRP_WAKEUP, "Woken up to idle logical primary nest core") 5 | NEST_ST(WAKEUP_FULLY_IDLE_RESERVE, STAT_GRP_WAKEUP, "Woken up to fully idle reserve nest core") 6 | NEST_ST(WAKEUP_ANY_IDLE_RESERVE, STAT_GRP_WAKEUP, "Woken up to idle logical reserve nest core") 7 | NEST_ST(WAKEUP_IDLE_OTHER, STAT_GRP_WAKEUP, "Woken to any idle logical core in p->cpus_ptr") 8 | 9 | NEST_ST(TASK_IMPATIENT, STAT_GRP_NEST, "A task was found to be impatient") 10 | NEST_ST(PROMOTED_TO_PRIMARY, STAT_GRP_NEST, "A core was promoted into the primary nest") 11 | NEST_ST(PROMOTED_TO_RESERVED, STAT_GRP_NEST, "A core was promoted into the reserve nest") 12 | NEST_ST(DEMOTED_TO_RESERVED, STAT_GRP_NEST, "A core was demoted into the reserve nest") 13 | NEST_ST(RESERVED_AT_CAPACITY, STAT_GRP_NEST, "Reserved nest was at capacity") 14 | NEST_ST(SCHEDULED_COMPACTION, STAT_GRP_NEST, "Scheduled a primary core to be compacted") 15 | NEST_ST(CANCELLED_COMPACTION, STAT_GRP_NEST, "Cancelled a primary core from being compacted at task wakeup time") 16 | NEST_ST(EAGERLY_COMPACTED, STAT_GRP_NEST, "A core was compacted in ops.dispatch()") 17 | NEST_ST(CALLBACK_COMPACTED, STAT_GRP_NEST, "A core was compacted in the scheduled timer callback") 18 | 19 | NEST_ST(CONSUMED, STAT_GRP_CONSUME, "A task was consumed from the global DSQ") 20 | NEST_ST(NOT_CONSUMED, STAT_GRP_CONSUME, "There was no task in the global DSQ") 21 | -------------------------------------------------------------------------------- /scheds/c/scx_pair.h: -------------------------------------------------------------------------------- 1 | #ifndef __SCX_EXAMPLE_PAIR_H 2 | #define __SCX_EXAMPLE_PAIR_H 3 | 4 | enum { 5 | MAX_QUEUED = 4096, 6 | MAX_CGRPS = 4096, 7 | }; 8 | 9 | #endif /* __SCX_EXAMPLE_PAIR_H */ 10 | -------------------------------------------------------------------------------- /scheds/c/scx_prev.bpf.c: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * A variation on scx_simple with CPU selection that prioritizes an idle 4 | * previous CPU over finding a fully idle core (as is done in scx_simple and 5 | * scx_rusty). 6 | * 7 | * Outperforms the in-kernel fair class (v6.12), scx_simple, and scx_rusty on 8 | * OLTP workloads run on systems with simple topology (i.e. non-NUMA, single 9 | * LLC). 10 | * 11 | * Copyright (c) 2025, Oracle and/or its affiliates. 12 | * Copyright (c) 2025, Daniel Jordan 13 | */ 14 | #include 15 | 16 | char _license[] SEC("license") = "GPL"; 17 | 18 | UEI_DEFINE(uei); 19 | 20 | struct { 21 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 22 | __uint(key_size, sizeof(u32)); 23 | __uint(value_size, sizeof(u64)); 24 | __uint(max_entries, 4); /* [local, select_fail, prev_cpu, idle_cpu] */ 25 | } stats SEC(".maps"); 26 | 27 | static void stat_inc(u32 idx) 28 | { 29 | u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx); 30 | if (cnt_p) 31 | (*cnt_p)++; 32 | } 33 | 34 | s32 BPF_STRUCT_OPS(prev_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) 35 | { 36 | s32 cpu; 37 | 38 | if (scx_bpf_test_and_clear_cpu_idle(prev_cpu)) { 39 | stat_inc(2); /* prev_cpu */ 40 | cpu = prev_cpu; 41 | goto insert; 42 | } 43 | 44 | cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); 45 | if (cpu >= 0) { 46 | stat_inc(3); /* idle_cpu */ 47 | goto insert; 48 | } 49 | 50 | stat_inc(1); /* select_fail */ 51 | 52 | return prev_cpu; 53 | 54 | insert: 55 | stat_inc(0); /* local */ 56 | scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0); 57 | 58 | return cpu; 59 | } 60 | 61 | void BPF_STRUCT_OPS(prev_exit, struct scx_exit_info *ei) 62 | { 63 | UEI_RECORD(uei, ei); 64 | } 65 | 66 | SCX_OPS_DEFINE(prev_ops, 67 | .select_cpu = (void *)prev_select_cpu, 68 | .exit = (void *)prev_exit, 69 | .name = "prev" 70 | ); 71 | -------------------------------------------------------------------------------- /scheds/c/scx_sdt.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | struct scx_stats { 3 | int seq; 4 | pid_t pid; 5 | __u64 enqueue; 6 | __u64 exit; 7 | __u64 init; 8 | __u64 select_busy_cpu; 9 | __u64 select_idle_cpu; 10 | }; 11 | -------------------------------------------------------------------------------- /scheds/c/scx_userland.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | /* Copyright (c) 2022 Meta, Inc */ 3 | 4 | #ifndef __SCX_USERLAND_COMMON_H 5 | #define __SCX_USERLAND_COMMON_H 6 | 7 | /* 8 | * An instance of a task that has been enqueued by the kernel for consumption 9 | * by a user space global scheduler thread. 10 | */ 11 | struct scx_userland_enqueued_task { 12 | __s32 pid; 13 | u64 sum_exec_runtime; 14 | u64 weight; 15 | }; 16 | 17 | #endif // __SCX_USERLAND_COMMON_H 18 | -------------------------------------------------------------------------------- /scheds/include/.gitignore: -------------------------------------------------------------------------------- 1 | vmlinux.h 2 | -------------------------------------------------------------------------------- /scheds/include/arch/arm/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/arm64/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/mips/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/powerpc/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/riscv/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/s390/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/arch/x86/vmlinux.h: -------------------------------------------------------------------------------- 1 | vmlinux-v6.16-g9b30400ff652.h -------------------------------------------------------------------------------- /scheds/include/bpf-compat/gnu/stubs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Dummy gnu/stubs.h. clang can end up including /usr/include/gnu/stubs.h when 3 | * compiling BPF files although its content doesn't play any role. The file in 4 | * turn includes stubs-64.h or stubs-32.h depending on whether __x86_64__ is 5 | * defined. When compiling a BPF source, __x86_64__ isn't set and thus 6 | * stubs-32.h is selected. However, the file is not there if the system doesn't 7 | * have 32bit glibc devel package installed leading to a build failure. 8 | * 9 | * The problem is worked around by making this file available in the include 10 | * search paths before the system one when building BPF. 11 | */ 12 | -------------------------------------------------------------------------------- /scheds/include/lib/arena.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct arena_init_args { 4 | u64 static_pages; 5 | u64 task_ctx_size; 6 | }; 7 | 8 | int arena_init(struct arena_init_args *args); 9 | 10 | struct arena_alloc_mask_args { 11 | u64 bitmap; 12 | }; 13 | 14 | int arena_alloc_mask(struct arena_alloc_mask_args *args); 15 | 16 | struct arena_topology_node_init_args { 17 | u64 bitmap; 18 | u64 data_size; 19 | u64 id; 20 | }; 21 | 22 | int arena_topology_node_init(struct arena_topology_node_init_args *args); 23 | 24 | int arena_topology_print(void); 25 | -------------------------------------------------------------------------------- /scheds/include/scx/arena_userspace_interrop.bpf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. 4 | * Copyright (c) 2025 Jake Hillion 5 | */ 6 | #ifndef __SCX_ARENA_USERSPACE_INTERROP_H 7 | #define __SCX_ARENA_USERSPACE_INTERROP_H 8 | 9 | #ifndef __arena 10 | #define __arena 11 | #endif 12 | 13 | #ifndef __KERNEL__ 14 | typedef unsigned char u8; 15 | typedef unsigned int u32; 16 | typedef unsigned long long u64; 17 | #endif 18 | 19 | struct scx_userspace_arena_alloc_pages_args 20 | { 21 | u32 sz; 22 | void __arena *ret; 23 | }; 24 | 25 | struct scx_userspace_arena_free_pages_args 26 | { 27 | void __arena *addr; 28 | u32 sz; 29 | }; 30 | 31 | #endif /* __SCX_ARENA_USERSPACE_INTERROP_H */ 32 | -------------------------------------------------------------------------------- /scheds/include/scx/enums.bpf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Convenience macros for getting/setting struct scx_enums instances. 4 | * 5 | * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 6 | */ 7 | #ifndef __SCX_ENUMS_BPF_H 8 | #define __SCX_ENUMS_BPF_H 9 | 10 | #include "enums.autogen.bpf.h" 11 | 12 | #endif /* __SCX_ENUMS_BPF_H */ 13 | -------------------------------------------------------------------------------- /scheds/include/scx/enums.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Define struct scx_enums that stores the load-time values of enums 4 | * used by the BPF program. 5 | * 6 | * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 7 | */ 8 | 9 | #ifndef __SCX_ENUMS_H 10 | #define __SCX_ENUMS_H 11 | 12 | static inline void __ENUM_set(u64 *val, char *type, char *name) 13 | { 14 | bool res; 15 | 16 | res = __COMPAT_read_enum(type, name, val); 17 | if (!res) 18 | *val = 0; 19 | } 20 | 21 | #define SCX_ENUM_SET(skel, type, name) do { \ 22 | __ENUM_set(&skel->rodata->__##name, #type, #name); \ 23 | } while (0) 24 | 25 | 26 | #include "enums.autogen.h" 27 | 28 | #endif /* __SCX_ENUMS_H */ 29 | -------------------------------------------------------------------------------- /scheds/include/scx/namespace.bpf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 | */ 5 | #ifndef __SCHED_EXT_NAMESPACE_BPF_H 6 | #define __SCHED_EXT_NAMESPACE_BPF_H 7 | 8 | #ifdef LSP 9 | #define __bpf__ 10 | #include "../vmlinux.h" 11 | #else 12 | #include "vmlinux.h" 13 | #endif 14 | 15 | struct pid_namespace* get_task_pid_ns(const struct task_struct* task, enum pid_type); 16 | struct pid* get_task_pid_ptr(const struct task_struct* task, enum pid_type type); 17 | pid_t get_task_ns_pid(const struct task_struct* task); 18 | 19 | pid_t get_pid_nr_ns(struct pid* pid, struct pid_namespace* ns); 20 | pid_t get_ns_pid(void); 21 | 22 | #endif /* __SCHED_EXT_NAMESPACE_BPF_H */ 23 | -------------------------------------------------------------------------------- /scheds/include/scx/namespace_impl.bpf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2024 Meta Platforms, Inc. and affiliates. 4 | */ 5 | #include "namespace.bpf.h" 6 | 7 | #include 8 | #include 9 | 10 | 11 | __hidden struct pid* get_task_pid_ptr(const struct task_struct* task, 12 | enum pid_type type) 13 | { 14 | // Returns the pid pointer of the given task. See get_task_pid_ptr for 15 | // the kernel implementation. 16 | return (type == PIDTYPE_PID) ? BPF_CORE_READ(task, thread_pid) : 17 | BPF_CORE_READ(task, signal, pids[type]); 18 | } 19 | 20 | __hidden struct pid_namespace* get_task_pid_ns(const struct task_struct* task, 21 | enum pid_type type) 22 | { 23 | struct pid_namespace* ns; 24 | struct pid* p; 25 | int level; 26 | 27 | // See kernel function task_active_pid_ns in pid.c which calls into 28 | // ns_of_pid. Returns the pid namespace of the given task. 29 | if (!task) 30 | task = (struct task_struct*)bpf_get_current_task(); 31 | 32 | if (!task) 33 | return NULL; 34 | 35 | p = get_task_pid_ptr(task, type); 36 | if (!p) 37 | return NULL; 38 | 39 | level = BPF_CORE_READ(p, level); 40 | ns = BPF_CORE_READ(p, numbers[level].ns); 41 | return ns; 42 | } 43 | 44 | __hidden pid_t get_pid_nr_ns(struct pid* pid, struct pid_namespace* ns) 45 | { 46 | int level, ns_level; 47 | pid_t nr = 0; 48 | 49 | /* This function implements the kernel equivalent pid_nr_ns in linux/pid.h */ 50 | if (!pid || !ns) 51 | return nr; 52 | 53 | level = BPF_CORE_READ(pid, level); 54 | ns_level = BPF_CORE_READ(ns, level); 55 | if (ns_level <= level) { 56 | struct upid upid; 57 | 58 | upid = BPF_CORE_READ(pid, numbers[ns_level]); 59 | if (upid.ns == ns) 60 | nr = upid.nr; 61 | } 62 | return nr; 63 | } 64 | 65 | __hidden pid_t get_task_ns_pid(const struct task_struct* task) 66 | { 67 | struct pid_namespace* ns; 68 | struct pid* p; 69 | 70 | if (!task) 71 | task = (struct task_struct*)bpf_get_current_task(); 72 | 73 | ns = get_task_pid_ns(task, PIDTYPE_TGID); 74 | p = get_task_pid_ptr(task, PIDTYPE_PID); 75 | return get_pid_nr_ns(p, ns); 76 | } 77 | 78 | __hidden pid_t get_ns_pid(void) 79 | { 80 | return get_task_ns_pid(NULL); 81 | } 82 | -------------------------------------------------------------------------------- /scheds/include/scx/ravg.bpf.h: -------------------------------------------------------------------------------- 1 | #ifndef __SCX_RAVG_BPF_H__ 2 | #define __SCX_RAVG_BPF_H__ 3 | 4 | /* 5 | * Running average helpers to be used in BPF progs. Assumes vmlinux.h has 6 | * already been included. 7 | */ 8 | enum ravg_consts { 9 | RAVG_VAL_BITS = 44, /* input values are 44bit */ 10 | RAVG_FRAC_BITS = 20, /* 1048576 is 1.0 */ 11 | }; 12 | 13 | /* 14 | * Running avg mechanism. Accumulates values between 0 and RAVG_MAX_VAL in 15 | * arbitrary time intervals. The accumulated values are halved every half_life 16 | * with each period starting when the current time % half_life is 0. Zeroing is 17 | * enough for initialization. 18 | * 19 | * See ravg_accumulate() and ravg_read() for more details. 20 | */ 21 | struct ravg_data { 22 | /* current value */ 23 | u64 val; 24 | 25 | /* 26 | * The timestamp of @val. The latest completed seq #: 27 | * 28 | * (val_at / half_life) - 1 29 | */ 30 | u64 val_at; 31 | 32 | /* running avg as of the latest completed seq */ 33 | u64 old; 34 | 35 | /* 36 | * Accumulated value of the current period. Input value is 48bits and we 37 | * normalize half-life to 16bit, so it should fit in a u64. 38 | */ 39 | u64 cur; 40 | }; 41 | 42 | #endif /* __SCX_RAVG_BPF_H__ */ 43 | -------------------------------------------------------------------------------- /scheds/include/vmlinux.h: -------------------------------------------------------------------------------- 1 | arch/x86/vmlinux.h -------------------------------------------------------------------------------- /scheds/meson.build: -------------------------------------------------------------------------------- 1 | # Common include paths for user C compilation. The following should be 2 | # passed in as executable::include_directories. 3 | user_c_includes = include_directories('include') 4 | 5 | install_subdir(join_paths(meson.current_source_dir(), 'include/scx'), install_dir: 'include', install_tag: 'devel') 6 | 7 | subdir('c') 8 | -------------------------------------------------------------------------------- /scheds/rust/.gitignore: -------------------------------------------------------------------------------- 1 | */src/bpf/.output 2 | target 3 | -------------------------------------------------------------------------------- /scheds/rust/README.md: -------------------------------------------------------------------------------- 1 | RUST SCHEDULERS 2 | =============== 3 | 4 | # Introduction 5 | 6 | This directory contains schedulers with user space rust components. 7 | 8 | The README in each scheduler directory provides some background and describes 9 | the types of workloads or scenarios they're designed to accommodate. For more 10 | details on any of these schedulers, please see the header comment in their 11 | main.rs or \*.bpf.c files. 12 | 13 | # Schedulers 14 | 15 | - [scx_bpfland](scx_bpfland/README.md) 16 | - [scx_flash](scx_flash/README.md) 17 | - [scx_lavd](scx_lavd/README.md) 18 | - [scx_layered](scx_layered/README.md) 19 | - [scx_p2dq](scx_p2dq/README.md) 20 | - [scx_rlfifo](scx_rlfifo/README.md) 21 | - [scx_rustland](scx_rustland/README.md) 22 | - [scx_rusty](scx_rusty/README.md) 23 | - [scx_tickless](scx_tickless/README.md) 24 | -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_bpfland" 3 | version = "1.0.12" 4 | authors = ["Andrea Righi "] 5 | edition = "2021" 6 | description = "A vruntime-based sched_ext scheduler that prioritizes interactive workloads. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | ctrlc = { version = "3.1", features = ["termination"] } 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | crossbeam = "0.8.4" 14 | libbpf-rs = "=0.25.0-beta.1" 15 | log = "0.4.17" 16 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 17 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 18 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15", features = ["autopower"] } 19 | serde = { version = "1.0.215", features = ["derive"] } 20 | simplelog = "0.12" 21 | 22 | [build-dependencies] 23 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 24 | 25 | [features] 26 | enable_backtrace = [] 27 | -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Andrea Righi 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .build() 12 | .unwrap(); 13 | } 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2024 Andrea Righi 4 | * 5 | * This software may be used and distributed according to the terms of the GNU 6 | * General Public License version 2. 7 | */ 8 | #ifndef __INTF_H 9 | #define __INTF_H 10 | 11 | #include 12 | 13 | #define MAX(x, y) ((x) > (y) ? (x) : (y)) 14 | #define MIN(x, y) ((x) < (y) ? (x) : (y)) 15 | #define CLAMP(val, lo, hi) MIN(MAX(val, lo), hi) 16 | #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 17 | 18 | enum consts { 19 | NSEC_PER_USEC = 1000ULL, 20 | NSEC_PER_MSEC = (1000ULL * NSEC_PER_USEC), 21 | NSEC_PER_SEC = (1000ULL * NSEC_PER_MSEC), 22 | 23 | /* Kernel definitions */ 24 | CLOCK_BOOTTIME = 7, 25 | }; 26 | 27 | #ifndef __VMLINUX_H__ 28 | typedef unsigned char u8; 29 | typedef unsigned short u16; 30 | typedef unsigned int u32; 31 | typedef unsigned long u64; 32 | 33 | typedef signed char s8; 34 | typedef signed short s16; 35 | typedef signed int s32; 36 | typedef signed long s64; 37 | 38 | typedef int pid_t; 39 | #endif /* __VMLINUX_H__ */ 40 | 41 | struct cpu_arg { 42 | s32 cpu_id; 43 | }; 44 | 45 | struct domain_arg { 46 | s32 lvl_id; 47 | s32 cpu_id; 48 | s32 sibling_cpu_id; 49 | }; 50 | 51 | #endif /* __INTF_H */ 52 | -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | #![allow(non_upper_case_globals)] 8 | #![allow(non_camel_case_types)] 9 | #![allow(non_snake_case)] 10 | #![allow(dead_code)] 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_bpfland/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | 8 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 9 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_chaos" 3 | version = "1.0.15" 4 | edition = "2021" 5 | authors = ["Jake Hillion "] 6 | description = "scx_chaos A general purpose sched_ext scheduler designed to amplify race conditions" 7 | license = "GPL-2.0-only" 8 | 9 | [package.metadata.scx] 10 | ci.use_clippy = true 11 | 12 | [dependencies] 13 | scx_userspace_arena = { path = "../../../rust/scx_userspace_arena", version = "1.0.12" } 14 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 15 | scx_p2dq = { path = "../../../scheds/rust/scx_p2dq", version = "1.0.16" } 16 | 17 | anyhow = "1.0.65" 18 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 19 | ctrlc = { version = "3.1", features = ["termination"] } 20 | libbpf-rs = "=0.25.0-beta.1" 21 | libc = "0.2.137" 22 | log = "0.4.17" 23 | nix = { version = "0.29", features = ["process"] } 24 | simplelog = "0.12" 25 | 26 | [build-dependencies] 27 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 28 | scx_p2dq = { path = "../../../scheds/rust/scx_p2dq", version = "1.0.15" } 29 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | use std::io::Write; 6 | 7 | fn main() { 8 | let include_path = std::env::var("OUT_DIR").unwrap() + "/include"; 9 | 10 | std::fs::create_dir_all(include_path.clone() + "/scx_p2dq").unwrap(); 11 | let mut file = std::fs::File::create(include_path.clone() + "/scx_p2dq/main.bpf.c").unwrap(); 12 | file.write_all(scx_p2dq::bpf_srcs::main_bpf_c()).unwrap(); 13 | let mut file = std::fs::File::create(include_path.clone() + "/scx_p2dq/intf.h").unwrap(); 14 | file.write_all(scx_p2dq::bpf_srcs::intf_h()).unwrap(); 15 | let mut file = std::fs::File::create(include_path.clone() + "/scx_p2dq/types.h").unwrap(); 16 | file.write_all(scx_p2dq::bpf_srcs::types_h()).unwrap(); 17 | 18 | // TODO: this is a massive hack. BpfBuilder should be rewritten to have an explicit change of 19 | // state between "builder" mode (where options are set) and "actualised" mode where they are 20 | // turned into the useful arguments. As it is cflags is generated too early, and refactoring it 21 | // is a pain and will require a change of interface. 22 | let mut extra_flags = vec!["-I".to_string() + &include_path]; 23 | if let Ok(e) = std::env::var("BPF_EXTRA_CFLAGS_POST_INCL") { 24 | extra_flags.push(e); 25 | } 26 | 27 | unsafe { 28 | std::env::set_var("BPF_EXTRA_CFLAGS_POST_INCL", extra_flags.join(" ")); 29 | } 30 | 31 | scx_utils::BpfBuilder::new() 32 | .unwrap() 33 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 34 | .enable_skel("src/bpf/main.bpf.c", "bpf") 35 | .add_source("src/bpf/lib/sdt_task.bpf.c") 36 | .add_source("src/bpf/lib/sdt_alloc.bpf.c") 37 | .add_source("src/bpf/lib/bitmap.bpf.c") 38 | .add_source("src/bpf/lib/topology.bpf.c") 39 | .add_source("src/bpf/lib/arena.bpf.c") 40 | .compile_link_gen() 41 | .unwrap(); 42 | } 43 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #ifndef __CHAOS_INTF_H 6 | #define __CHAOS_INTF_H 7 | 8 | #ifndef __KERNEL__ 9 | typedef unsigned long long u64; 10 | #endif 11 | 12 | enum chaos_consts { 13 | CHAOS_DSQ_BASE_SHIFT = 16, 14 | CHAOS_DSQ_BASE = 1 << CHAOS_DSQ_BASE_SHIFT, 15 | 16 | CHAOS_NUM_PPIDS_CHECK = 1 << 20, 17 | }; 18 | 19 | enum chaos_match { 20 | CHAOS_MATCH_UNKNOWN = 0, 21 | CHAOS_MATCH_COMPLETE = 1 << 0, 22 | CHAOS_MATCH_EXCLUDED = 1 << 1, 23 | CHAOS_MATCH_HAS_PARENT = 1 << 2, 24 | 25 | CHAOS_MATCH_MAX = 1 << 3, 26 | }; 27 | 28 | enum chaos_trait_kind { 29 | CHAOS_TRAIT_NONE, 30 | CHAOS_TRAIT_RANDOM_DELAYS, 31 | CHAOS_TRAIT_CPU_FREQ, 32 | CHAOS_TRAIT_DEGRADATION, 33 | CHAOS_TRAIT_MAX, 34 | }; 35 | 36 | struct chaos_task_ctx { 37 | // chaos_task_ctx is initialised zero'd 38 | enum chaos_match match; 39 | 40 | enum chaos_trait_kind next_trait; 41 | u64 enq_flags; 42 | u64 p2dq_vtime; 43 | }; 44 | 45 | #endif /* __CHAOS_INTF_H */ 46 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/src/bpf/lib: -------------------------------------------------------------------------------- 1 | ../../../../../lib -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 7 | -------------------------------------------------------------------------------- /scheds/rust/scx_chaos/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | use scx_chaos::run; 6 | use scx_chaos::Args; 7 | 8 | use clap::Parser; 9 | 10 | fn main() -> anyhow::Result<()> { 11 | let args = Args::parse(); 12 | 13 | let llv = match &args.verbose { 14 | 0 => simplelog::LevelFilter::Info, 15 | 1 => simplelog::LevelFilter::Debug, 16 | _ => simplelog::LevelFilter::Trace, 17 | }; 18 | 19 | simplelog::TermLogger::init( 20 | llv, 21 | simplelog::ConfigBuilder::new() 22 | .set_time_offset_to_local() 23 | .unwrap() 24 | .set_time_level(simplelog::LevelFilter::Error) 25 | .set_location_level(simplelog::LevelFilter::Off) 26 | .set_target_level(simplelog::LevelFilter::Off) 27 | .set_thread_level(simplelog::LevelFilter::Off) 28 | .build(), 29 | simplelog::TerminalMode::Stderr, 30 | simplelog::ColorChoice::Auto, 31 | )?; 32 | 33 | run(args) 34 | } 35 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_flash" 3 | version = "1.0.10" 4 | authors = ["Andrea Righi "] 5 | edition = "2021" 6 | description = "A scheduler designed for multimedia and real-time audio processing workloads. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | ctrlc = { version = "3.1", features = ["termination"] } 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | crossbeam = "0.8.4" 14 | libbpf-rs = "=0.25.0-beta.1" 15 | log = "0.4.17" 16 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 17 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 18 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15", features = ["autopower"] } 19 | serde = { version = "1.0.215", features = ["derive"] } 20 | simplelog = "0.12" 21 | 22 | [build-dependencies] 23 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 24 | 25 | [features] 26 | enable_backtrace = [] 27 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_flash/README.md: -------------------------------------------------------------------------------- 1 | # scx_flash 2 | 3 | This is a single user-defined scheduler used within [sched_ext](https://github.com/sched-ext/scx/tree/main), which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | A scheduler that focuses on ensuring fairness among tasks and performance 8 | predictability. 9 | 10 | It operates using an earliest deadline first (EDF) policy, where each task is 11 | assigned a "latency" weight. This weight is dynamically adjusted based on how 12 | often a task release the CPU before its full time slice is used. Tasks that 13 | release the CPU early are given a higher latency weight, prioritizing them over 14 | tasks that fully consume their time slice. 15 | 16 | ## Typical Use Case 17 | 18 | The combination of dynamic latency weights and EDF scheduling ensures 19 | responsive and consistent performance, even in overcommitted systems. 20 | 21 | This makes the scheduler particularly well-suited for latency-sensitive 22 | workloads, such as multimedia or real-time audio processing. 23 | 24 | ## Production Ready? 25 | 26 | Yes. 27 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Andrea Righi 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .build() 12 | .unwrap(); 13 | } 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2024 Andrea Righi 4 | * 5 | * This software may be used and distributed according to the terms of the GNU 6 | * General Public License version 2. 7 | */ 8 | #ifndef __INTF_H 9 | #define __INTF_H 10 | 11 | #include 12 | 13 | #define MAX(x, y) ((x) > (y) ? (x) : (y)) 14 | #define MIN(x, y) ((x) < (y) ? (x) : (y)) 15 | #define CLAMP(val, lo, hi) MIN(MAX(val, lo), hi) 16 | #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 17 | 18 | enum consts { 19 | NSEC_PER_USEC = 1000ULL, 20 | NSEC_PER_MSEC = (1000ULL * NSEC_PER_USEC), 21 | NSEC_PER_SEC = (1000ULL * NSEC_PER_MSEC), 22 | 23 | /* Kernel definitions */ 24 | CLOCK_BOOTTIME = 7, 25 | }; 26 | 27 | #ifndef __VMLINUX_H__ 28 | typedef unsigned char u8; 29 | typedef unsigned short u16; 30 | typedef unsigned int u32; 31 | typedef unsigned long u64; 32 | 33 | typedef signed char s8; 34 | typedef signed short s16; 35 | typedef signed int s32; 36 | typedef signed long s64; 37 | 38 | typedef int pid_t; 39 | #endif /* __VMLINUX_H__ */ 40 | 41 | struct cpu_arg { 42 | s32 cpu_id; 43 | }; 44 | 45 | struct domain_arg { 46 | s32 lvl_id; 47 | s32 cpu_id; 48 | s32 sibling_cpu_id; 49 | }; 50 | 51 | #endif /* __INTF_H */ 52 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | #![allow(non_upper_case_globals)] 8 | #![allow(non_camel_case_types)] 9 | #![allow(non_snake_case)] 10 | #![allow(dead_code)] 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_flash/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | 8 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 9 | -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_lavd" 3 | version = "1.0.12" 4 | authors = ["Changwoo Min ", "Igalia"] 5 | edition = "2021" 6 | description = "A Latency-criticality Aware Virtual Deadline (LAVD) scheduler based on sched_ext, which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | bitvec = { version = "1.0", features = ["serde"] } 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | clap-num = { version = "1.2.0" } 14 | crossbeam = "0.8.4" 15 | ctrlc = { version = "3.1", features = ["termination"] } 16 | fb_procfs = "0.7" 17 | hex = "0.4.3" 18 | itertools = "0.13.0" 19 | libbpf-rs = "=0.25.0-beta.1" 20 | libc = "0.2.137" 21 | log = "0.4.17" 22 | ordered-float = "3.4.0" 23 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 24 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 25 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15", features = ["autopower"] } 26 | serde = { version = "1.0.215", features = ["derive"] } 27 | simplelog = "0.12" 28 | static_assertions = "1.1.0" 29 | plain = "0.2.3" 30 | gpoint = "0.2" 31 | 32 | [build-dependencies] 33 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 34 | 35 | [features] 36 | enable_backtrace = [] 37 | -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/README.md: -------------------------------------------------------------------------------- 1 | # scx_lavd 2 | 3 | This is a single user-defined scheduler used within [sched_ext](https://github.com/sched-ext/scx/tree/main), which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | scx_lavd is a BPF scheduler that implements an LAVD (Latency-criticality Aware 8 | Virtual Deadline) scheduling algorithm. While LAVD is new and still evolving, 9 | its core ideas are 1) measuring how much a task is latency critical and 2) 10 | leveraging the task's latency-criticality information in making various 11 | scheduling decisions (e.g., task's deadline, time slice, etc.). As the name 12 | implies, LAVD is based on the foundation of deadline scheduling. This scheduler 13 | consists of the BPF part and the rust part. The BPF part makes all the 14 | scheduling decisions; the rust part provides high-level information (e.g., CPU 15 | topology) to the BPF code, loads the BPF code and conducts other chores (e.g., 16 | printing sampled scheduling decisions). 17 | 18 | ## Typical Use Case 19 | 20 | scx_lavd is initially motivated by gaming workloads. It aims to improve 21 | interactivity and reduce stuttering while playing games on Linux. Hence, this 22 | scheduler's typical use case involves highly interactive applications, such as 23 | gaming, which requires high throughput and low tail latencies. 24 | 25 | ## Production Ready? 26 | 27 | Yes, scx_lavd should be performant across various CPU architectures. It creates 28 | a separate scheduling domain per-LLC, per-core type (e.g., P or E core on 29 | Intel, big or LITTLE on ARM), and per-NUMA domain, so the default balanced 30 | profile or autopilot mode should be performant. It mainly targets single CCX 31 | / single-socket systems. 32 | 33 | -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Changwoo Min 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .build() 12 | .unwrap(); 13 | } 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Valve Corporation. 4 | // Author: Changwoo Min 5 | 6 | // This software may be used and distributed according to the terms of the 7 | // GNU General Public License version 2. 8 | #![allow(non_upper_case_globals)] 9 | #![allow(non_camel_case_types)] 10 | #![allow(non_snake_case)] 11 | #![allow(dead_code)] 12 | 13 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_lavd/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2024 Valve Corporation. 4 | // Author: Changwoo Min 5 | 6 | // This software may be used and distributed according to the terms of the 7 | // GNU General Public License version 2. 8 | 9 | // We can't directly include the generated skeleton in main.rs as it may 10 | // contain compiler attributes that can't be `include!()`ed via macro and we 11 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 12 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 13 | // https://github.com/rust-lang/rust/pull/83366). 14 | 15 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 16 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_layered" 3 | version = "1.0.13" 4 | authors = ["Tejun Heo ", "Meta"] 5 | edition = "2021" 6 | description = "A highly configurable multi-layer BPF / user space hybrid scheduler used within sched_ext, which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | bitvec = "1.0" 12 | chrono = "0.4" 13 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 14 | crossbeam = "0.8.4" 15 | ctrlc = { version = "3.1", features = ["termination"] } 16 | fastrand = "2.1.1" 17 | fb_procfs = "0.7" 18 | lazy_static = "1.5.0" 19 | libbpf-rs = "=0.25.0-beta.1" 20 | libc = "0.2.137" 21 | log = "0.4.17" 22 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 23 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 24 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 25 | serde = { version = "1.0.215", features = ["derive"] } 26 | serde_json = "1.0.133" 27 | simplelog = "0.12" 28 | once_cell = "1.20.2" 29 | walkdir = "2.5" 30 | 31 | [build-dependencies] 32 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 33 | 34 | [features] 35 | enable_backtrace = [] 36 | 37 | [package.metadata.appimage] 38 | auto_link = true 39 | 40 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_layered/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .add_source("src/bpf/timer.bpf.c") 12 | .add_source("src/bpf/util.bpf.c") 13 | .compile_link_gen() 14 | .unwrap(); 15 | } 16 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/avgruntime.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "short", 4 | "comment": "Layers running for up to 100us", 5 | "matches": [ 6 | [ 7 | { 8 | "AvgRuntime": [0, 100] 9 | } 10 | ] 11 | ], 12 | "kind": { 13 | "Confined": { 14 | "protected": true, 15 | "slice_us": 100000, 16 | "util_range": [0.01, 0.99], 17 | "cpus_range": [5, 5], 18 | "placement": "Floating" 19 | } 20 | } 21 | }, 22 | 23 | { 24 | "name": "mid", 25 | "comment": "Layers running for up to 10ms", 26 | "matches": [ 27 | [ 28 | { 29 | "AvgRuntime": [100, 10000] 30 | } 31 | ] 32 | ], 33 | "kind": { 34 | "Confined": { 35 | "protected": true, 36 | "slice_us": 100000, 37 | "util_range": [0.01, 0.99], 38 | "cpus_range": [5, 5], 39 | "placement": "Sticky" 40 | } 41 | } 42 | }, 43 | 44 | { 45 | "name": "long", 46 | "comment": "Layers running from 10ms to 1s", 47 | "matches": [ 48 | [ 49 | { 50 | "AvgRuntime": [10000, 1000000] 51 | } 52 | ] 53 | ], 54 | "kind": { 55 | "Confined": { 56 | "protected": true, 57 | "slice_us": 100000, 58 | "util_range": [0.01, 0.99], 59 | "cpus_range": [5, 5], 60 | "placement": "Sticky" 61 | } 62 | } 63 | }, 64 | 65 | { 66 | "name": "rest", 67 | "comment": "the rest", 68 | "matches":[[]], 69 | "kind": { 70 | "Grouped": { 71 | "util_range": [ 72 | 0.05, 73 | 0.60 74 | ] 75 | } 76 | } 77 | } 78 | ] 79 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/cgrp_contains.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "match one container", 4 | "matches": [ 5 | [ 6 | { "CgroupContains": "nope" } 7 | ] 8 | ], 9 | "kind": { 10 | "Grouped": { 11 | "cpus_range_frac": [0.25, 0.5], 12 | "util_range": [0.4, 0.85], 13 | "growth_algo": "RandomTopo", 14 | "disallow_preempt_after_us": 0, 15 | "protected": true 16 | } 17 | } 18 | }, 19 | { 20 | "name": "match other containers", 21 | "matches": [ 22 | [{ "CgroupContains": "docker" }] 23 | ], 24 | "kind": { 25 | "Grouped": { 26 | "cpus_range_frac": [0.5, 0.5], 27 | "util_range": [0.4, 0.85], 28 | "growth_algo": "RandomTopo", 29 | "protected": true 30 | } 31 | } 32 | }, 33 | { 34 | "name": "third", 35 | "matches": [ 36 | [{ "PcommPrefix": "stress-ng" }] 37 | ], 38 | "kind": { 39 | "Grouped": { 40 | "cpus_range": [5, 5], 41 | "util_range": [0.4, 0.85], 42 | "growth_algo": "RandomTopo", 43 | "protected": true 44 | } 45 | } 46 | }, 47 | { 48 | "name": "fourth", 49 | "matches": [ 50 | [] 51 | ], 52 | "kind": { 53 | "Open": { 54 | "growth_algo": "RandomTopo" 55 | } 56 | } 57 | } 58 | ] 59 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/cmdjoin.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define SCX_PREFIX 0x5C10 10 | 11 | /* Maximum size, accounting for the prefix. */ 12 | #define SCX_LEN 13 13 | 14 | #define LAYER_NAME "test" 15 | 16 | #define CMD_JOIN 1 17 | #define CMD_LEAVE 2 18 | 19 | /* 20 | * Command structure passed to the scheduler. The prefix must 21 | * match SCX_PREFIX to avoid accidentally parsing a process 22 | * name as a command. The opcode allows a process to request joining 23 | * or leaving the layer with prefix LAYER_NAME. 24 | * 25 | * LAYER_NAME does not need to match the name of the layer exactly: The 26 | * rule will match as long as the layer name is a _prefix_ of the task 27 | * name. 28 | */ 29 | struct scxcmd { 30 | uint16_t prefix; 31 | uint8_t opcode; 32 | char name[SCX_LEN]; 33 | }; 34 | 35 | int main(int argc, char *argv[]) 36 | { 37 | struct scxcmd cmd; 38 | 39 | cmd.prefix = SCX_PREFIX; 40 | cmd.opcode = CMD_JOIN; 41 | strncpy(cmd.name, LAYER_NAME, SCX_LEN); 42 | 43 | printf("Joining...\n"); 44 | pthread_setname_np(pthread_self(), (const char *)&cmd); 45 | 46 | sleep(3); 47 | 48 | cmd.prefix = SCX_PREFIX; 49 | cmd.opcode = CMD_LEAVE; 50 | strncpy(cmd.name, LAYER_NAME, SCX_LEN); 51 | 52 | printf("Leaving...\n"); 53 | pthread_setname_np(pthread_self(), (const char *)&cmd); 54 | 55 | sleep(3); 56 | 57 | return (0); 58 | } 59 | 60 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/cmdjoin.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "cmd", 4 | "comment": "Layers join using pthread_setname_np() to send a command to the scheduler. See examples/cmdjoin.c for more details.", 5 | "matches": [ 6 | [ 7 | { 8 | "CmdJoin": "test" 9 | } 10 | ] 11 | ], 12 | "kind": { 13 | "Confined": { 14 | "util_range": [ 15 | 0.4, 16 | 0.90 17 | ] 18 | } 19 | } 20 | }, 21 | { 22 | "name": "reset", 23 | "comment": "the rest", 24 | "matches":[[]], 25 | "kind": { 26 | "Grouped": { 27 | "util_range": [ 28 | 0.05, 29 | 0.60 30 | ] 31 | } 32 | } 33 | } 34 | ] 35 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/cpus_pct.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "first", 4 | "matches": [ 5 | [{ "PcommPrefix": "htop" }], 6 | [{ "PcommPrefix": "scxtop" }], 7 | [{ "PcommPrefix": "bash" }], 8 | [{ "PcommPrefix": "yes" }] 9 | ], 10 | "kind": { 11 | "Grouped": { 12 | "cpus_range_frac": [0.25, 0.5], 13 | "util_range": [0.4, 0.85], 14 | "growth_algo": "RandomTopo", 15 | "disallow_preempt_after_us": 0, 16 | "protected": true 17 | } 18 | } 19 | }, 20 | { 21 | "name": "second", 22 | "matches": [ 23 | [{ "PcommPrefix": "stress-ng" }] 24 | ], 25 | "kind": { 26 | "Grouped": { 27 | "cpus_range_frac": [0.5, 0.5], 28 | "util_range": [0.4, 0.85], 29 | "growth_algo": "RandomTopo", 30 | "protected": true 31 | } 32 | } 33 | }, 34 | { 35 | "name": "third", 36 | "matches": [ 37 | [{ "PcommPrefix": "stress-ng" }] 38 | ], 39 | "kind": { 40 | "Grouped": { 41 | "cpus_range": [5, 5], 42 | "util_range": [0.4, 0.85], 43 | "growth_algo": "RandomTopo", 44 | "protected": true 45 | } 46 | } 47 | }, 48 | { 49 | "name": "fourth", 50 | "matches": [ 51 | [] 52 | ], 53 | "kind": { 54 | "Open": { 55 | "growth_algo": "RandomTopo" 56 | } 57 | } 58 | } 59 | ] 60 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/cpuset.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "hhvmworkers", 4 | "comment": "hhvmworker", 5 | "matches": [ 6 | [ 7 | { 8 | "CommPrefix": "geekbench" 9 | } 10 | ] 11 | ], 12 | "kind": { 13 | "Grouped": { 14 | "min_exec_us": 100, 15 | "util_range": [ 16 | 0.8, 17 | 0.9 18 | ], 19 | "growth_algo": "CpuSetSpread" 20 | } 21 | } 22 | }, 23 | { 24 | "name": "normal", 25 | "comment": "the rest", 26 | "matches": [ 27 | [] 28 | ], 29 | "kind": { 30 | "Open": { 31 | "util_range": [ 32 | 0.8, 33 | 0.9 34 | ], 35 | "min_exec_us": 100, 36 | "allow_node_aligned": true 37 | } 38 | } 39 | } 40 | ] 41 | 42 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/exclude.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "not stress-ng", 4 | "matches": [ 5 | [{ "CommPrefixExclude": "stress-ng" }] 6 | ], 7 | "kind": { 8 | "Grouped": { 9 | "cpus_range": [5, 5], 10 | "util_range": [0.01, 0.99], 11 | "growth_algo": "RandomTopo", 12 | "protected": true 13 | } 14 | } 15 | }, 16 | { 17 | "name": "stress-ng", 18 | "matches": [ 19 | [] 20 | ], 21 | "kind": { 22 | "Open": { 23 | "growth_algo": "RandomTopo" 24 | } 25 | } 26 | } 27 | ] 28 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/preempt_rt_sched_class.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "second", 4 | "matches": [ 5 | [{ "CommPrefix": "stress" }, {"UIDEquals": 1000}] 6 | ], 7 | "kind": { 8 | "Confined": { 9 | "cpus_range": [6, 6], 10 | "util_range": [0.4, 0.85], 11 | "preempt": true, 12 | "preempt_first": true, 13 | "growth_algo": "Reverse", 14 | "slice_us": 1000, 15 | "yield_ignore": 0.50 16 | } 17 | } 18 | }, 19 | { 20 | "name": "third", 21 | "matches": [ 22 | [] 23 | ], 24 | "kind": { 25 | "Open": { 26 | "growth_algo": "Linear" 27 | } 28 | } 29 | } 30 | ] 31 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/prefix_suffix.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "match one container", 4 | "matches": [ 5 | [ 6 | { "CgroupPrefix": "system.slice/docker" }, 7 | { "CgroupSuffix": "f.scope/" } 8 | ] 9 | ], 10 | "kind": { 11 | "Grouped": { 12 | "cpus_range_frac": [0.25, 0.5], 13 | "util_range": [0.4, 0.85], 14 | "growth_algo": "RandomTopo", 15 | "disallow_preempt_after_us": 0, 16 | "protected": true 17 | } 18 | } 19 | }, 20 | { 21 | "name": "match other containers", 22 | "matches": [ 23 | [{ "CgroupPrefix": "system.slice/docker" }] 24 | ], 25 | "kind": { 26 | "Grouped": { 27 | "cpus_range_frac": [0.5, 0.5], 28 | "util_range": [0.4, 0.85], 29 | "growth_algo": "RandomTopo", 30 | "protected": true 31 | } 32 | } 33 | }, 34 | { 35 | "name": "third", 36 | "matches": [ 37 | [{ "PcommPrefix": "stress-ng" }] 38 | ], 39 | "kind": { 40 | "Grouped": { 41 | "cpus_range": [5, 5], 42 | "util_range": [0.4, 0.85], 43 | "growth_algo": "RandomTopo", 44 | "protected": true 45 | } 46 | } 47 | }, 48 | { 49 | "name": "fourth", 50 | "matches": [ 51 | [] 52 | ], 53 | "kind": { 54 | "Open": { 55 | "growth_algo": "RandomTopo" 56 | } 57 | } 58 | } 59 | ] 60 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/protected.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "first", 4 | "matches": [ 5 | [{ "PcommPrefix": "htop" }], 6 | [{ "PcommPrefix": "scxtop" }], 7 | [{ "PcommPrefix": "bash" }], 8 | [{ "PcommPrefix": "yes" }] 9 | ], 10 | "kind": { 11 | "Grouped": { 12 | "cpus_range": [5, 5], 13 | "util_range": [0.4, 0.85], 14 | "growth_algo": "RandomTopo", 15 | "disallow_preempt_after_us": 0, 16 | "protected": true 17 | } 18 | } 19 | }, 20 | { 21 | "name": "second", 22 | "matches": [ 23 | [{ "PcommPrefix": "stress-ng" }] 24 | ], 25 | "kind": { 26 | "Grouped": { 27 | "cpus_range": [5, 5], 28 | "util_range": [0.4, 0.85], 29 | "growth_algo": "RandomTopo", 30 | "protected": true 31 | } 32 | } 33 | }, 34 | { 35 | "name": "third", 36 | "matches": [ 37 | [] 38 | ], 39 | "kind": { 40 | "Open": { 41 | "growth_algo": "RandomTopo" 42 | } 43 | } 44 | } 45 | ] 46 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/same_over_idle.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "prev_over_idle_core", 4 | "comment": "prev_over_idle_core", 5 | "matches": [ 6 | [{"CommPrefix": "stress-ng"}], 7 | [{"PcommPrefix": "stress-ng"}] 8 | ], 9 | "kind": { 10 | "Confined": { 11 | "cpus_range_frac": [0.25,0.25], 12 | "min_exec_us": 100, 13 | "growth_algo": "NodeSpreadReverse", 14 | "prev_over_idle_core": true, 15 | "util_range": [ 16 | 0.8, 17 | 0.9 18 | ] 19 | } 20 | } 21 | }, 22 | { 23 | "name": "not_prev_over_idle_core", 24 | "comment": "not_prev_over_idle_core", 25 | "matches": [ 26 | [{"CommPrefix": "stress"}], 27 | [{"PcommPrefix": "stress"}] 28 | ], 29 | "kind": { 30 | "Confined": { 31 | "cpus_range_frac": [0.25,0.25], 32 | "min_exec_us": 100, 33 | "growth_algo": "NodeSpread", 34 | "util_range": [ 35 | 0.8, 36 | 0.9 37 | ], 38 | "prev_over_idle_core": false 39 | } 40 | } 41 | }, 42 | { 43 | "name": "normal", 44 | "comment": "the rest", 45 | "matches": [ 46 | [] 47 | ], 48 | "kind": { 49 | "Open": { 50 | "util_range": [ 51 | 0.8, 52 | 0.9 53 | ], 54 | "min_exec_us": 100, 55 | "allow_node_aligned": true 56 | } 57 | } 58 | } 59 | ] 60 | 61 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/examples/template.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "template", 4 | "template": { "CgroupSuffix": "common_suffix/" }, 5 | "matches": [ 6 | [{ "CgroupPrefix": "top_level/" }] 7 | ], 8 | "kind": { 9 | "Grouped": { 10 | "cpus_range_frac": [0.25, 0.5], 11 | "util_range": [0.4, 0.85], 12 | "growth_algo": "RandomTopo", 13 | "disallow_preempt_after_us": 0, 14 | "protected": true 15 | } 16 | } 17 | }, 18 | { 19 | "name": "the rest", 20 | "matches": [ 21 | [] 22 | ], 23 | "kind": { 24 | "Open": { 25 | "growth_algo": "RandomTopo" 26 | } 27 | } 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/integration/layer_llc.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S bpftrace --unsafe -q 2 | 3 | /* 4 | * Asserts that the `llc` layer config works properly by failing if the pid 5 | * passed to the script runs on any other LLC besides LLC id 0. The layered 6 | * config should restrict the pid passed to the script to run on a layer 7 | * that only runs on LLC 0. 8 | */ 9 | 10 | BEGIN 11 | { 12 | @bpftrace_pid = pid; 13 | @sig = 0; 14 | 15 | if ($1 == 0) { 16 | // exit 137 17 | @sig = 9; 18 | } 19 | 20 | // Credit to Alastair Robertson for this mapping of CPU to LLC 21 | $num_cpus = *(uint32*)kaddr("__num_online_cpus"); 22 | $per_cpu_offsets = (uint64*)kaddr("__per_cpu_offset"); 23 | $cpu_to_node_map = kaddr("x86_cpu_to_node_map"); 24 | $ci_cpu_cacheinfo = kaddr("ci_cpu_cacheinfo"); 25 | $cpuid = 0; 26 | while ($cpuid < $num_cpus && $cpuid < 500) { 27 | $numa_id = *($cpu_to_node_map + *($per_cpu_offsets + $cpuid)); 28 | @cpu_to_node[$cpuid] = $numa_id; 29 | $cpu_cacheinfo = (struct cpu_cacheinfo*)($ci_cpu_cacheinfo + *($per_cpu_offsets + $cpuid)); 30 | $l3_cacheinfo = $cpu_cacheinfo->info_list[3]; 31 | $l3_cacheid = $l3_cacheinfo.id; 32 | @cpu_to_l3[$cpuid] = $l3_cacheid; 33 | $cpuid++; 34 | } 35 | } 36 | 37 | profile:hz:1 38 | { 39 | @counts[cpu] = @counts[cpu] + 1; 40 | if (@counts[cpu] == 15) { 41 | // exit 0 42 | @sig = 15; 43 | } 44 | } 45 | 46 | rawtracepoint:sched_switch 47 | { 48 | $task = (struct task_struct *)arg1; 49 | 50 | if (($task->parent->pid == $1 && @cpu_to_l3[cpu] != 0) || 51 | ($task->real_parent->pid == $1 && @cpu_to_l3[cpu] != 0)) { 52 | // exit 137 53 | @sig = 9; 54 | } 55 | } 56 | 57 | kprobe:__x64_sys_* / @bpftrace_pid == pid / { 58 | if (@sig > 0) { 59 | signal(@sig); 60 | } 61 | } 62 | 63 | interval:s:1 { 64 | print(("bpftrace monitoring pid", $1, "signal", @sig)); 65 | } 66 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/integration/layer_node.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S bpftrace --unsafe -q 2 | 3 | /* 4 | * Asserts that the `node` layer config works properly by failing if the pid 5 | * passed to the script runs on NUMA node 1. The layered config should restrict 6 | * the pid passed to the script to run on a layer that only runs on NUMA node 0. 7 | */ 8 | 9 | BEGIN 10 | { 11 | @bpftrace_pid = pid; 12 | @sig = 0; 13 | 14 | if ($1 == 0) { 15 | // exit 137 16 | @sig = 9; 17 | } 18 | } 19 | 20 | profile:hz:1 21 | { 22 | @counts[cpu] = @counts[cpu] + 1; 23 | if (@counts[cpu] == 15) { 24 | // exit 0 25 | @sig = 15; 26 | } 27 | } 28 | 29 | rawtracepoint:sched_switch 30 | { 31 | $task = (struct task_struct *)arg1; 32 | 33 | if (($task->parent->pid == $1 && numaid == 1) || 34 | ($task->real_parent->pid == $1 && numaid == 1)) { 35 | // exit 137 36 | @sig = 9; 37 | } 38 | } 39 | 40 | kprobe:__x64_sys_* / @bpftrace_pid == pid / { 41 | if (@sig > 0) { 42 | signal(@sig); 43 | } 44 | } 45 | 46 | interval:s:1 { 47 | print(("bpftrace monitoring pid", $1, "signal", @sig)); 48 | } 49 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/integration/llc.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "llc_0", 4 | "comment": "llc0", 5 | "matches": [ 6 | [ 7 | { 8 | "CommPrefix": "stress-ng" 9 | } 10 | ], 11 | [ 12 | { 13 | "PcommPrefix": "stress-ng" 14 | } 15 | ] 16 | ], 17 | "kind": { 18 | "Confined": { 19 | "util_range": [ 20 | 0.4, 21 | 0.90 22 | ], 23 | "llcs": [ 24 | 0 25 | ] 26 | } 27 | } 28 | }, 29 | { 30 | "name": "reset", 31 | "comment": "the rest", 32 | "matches":[[]], 33 | "kind": { 34 | "Grouped": { 35 | "util_range": [ 36 | 0.05, 37 | 0.60 38 | ] 39 | } 40 | } 41 | } 42 | ] 43 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/integration/numa.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "numa_0", 4 | "comment": "numa0", 5 | "matches": [ 6 | [ 7 | { 8 | "CommPrefix": "stress-ng" 9 | } 10 | ], 11 | [ 12 | { 13 | "PcommPrefix": "stress-ng" 14 | } 15 | ] 16 | ], 17 | "kind": { 18 | "Confined": { 19 | "util_range": [ 20 | 0.4, 21 | 0.90 22 | ], 23 | "nodes": [ 24 | 0 25 | ] 26 | } 27 | } 28 | }, 29 | { 30 | "name": "reset", 31 | "comment": "the rest", 32 | "matches":[[]], 33 | "kind": { 34 | "Grouped": { 35 | "util_range": [ 36 | 0.05, 37 | 0.60 38 | ] 39 | } 40 | } 41 | } 42 | ] 43 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/integration/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | layered_bin=${1:-../../../../target/release/scx_layered} 4 | test_scripts=( "layer_node.bt" "layer_llc.bt" ) 5 | test_configs=( "numa.json" "llc.json" ) 6 | 7 | for i in "${!test_scripts[@]}"; do 8 | test_script="${test_scripts[$i]}" 9 | test_config="${test_configs[$i]}" 10 | 11 | sudo pkill -9 -f scx_layered 12 | sudo "${layered_bin}" --stats 1 "f:${test_config}" -v & 13 | layered_pid=$! 14 | 15 | echo "layered pid ${layered_pid}" 16 | sleep 2 17 | 18 | stress-ng -c 2 -f 1 -t 30 & 19 | stress_pid=$! 20 | 21 | echo "stress-ng pid ${stress_pid}" 22 | sleep 1 23 | 24 | sudo "./${test_script}" "${stress_pid}" 25 | test_exit=$? 26 | 27 | pidof scx_layered && sudo pkill -9 -f scx_layered 28 | # always cleanup stress-ng 29 | sudo pkill -9 -f stress-ng 30 | 31 | if [ $test_exit -ne 0 ]; then 32 | echo "test script ${test_script} failed: ${test_exit}" 33 | exit $test_exit; 34 | fi 35 | echo "test script ${test_script} passed: ${test_exit}" 36 | done 37 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/src/bpf/timer.bpf.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 2 | #ifndef __LAYERED_TIMER_H 3 | #define __LAYERED_TIMER_H 4 | 5 | #ifdef LSP 6 | #ifndef __bpf__ 7 | #define __bpf__ 8 | #endif 9 | #include "../../../../include/scx/common.bpf.h" 10 | #else 11 | #include 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | enum timer_consts { 19 | // kernel definitions 20 | CLOCK_BOOTTIME = 7, 21 | }; 22 | 23 | struct layered_timer { 24 | // if set to 0 the timer will only be scheduled once 25 | u64 interval_ns; 26 | u64 init_flags; 27 | int start_flags; 28 | }; 29 | 30 | enum layer_timer_callbacks { 31 | LAYERED_MONITOR, 32 | ANTISTALL_TIMER, 33 | NOOP_TIMER, 34 | MAX_TIMERS, 35 | }; 36 | 37 | bool run_timer_cb(int key); 38 | int start_layered_timers(void); 39 | 40 | extern struct layered_timer layered_timers[MAX_TIMERS]; 41 | 42 | #endif /* __LAYERED_TIMER_H */ 43 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/src/bpf/util.bpf.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 2 | #ifndef __LAYERED_UTIL_H 3 | #define __LAYERED_UTIL_H 4 | 5 | #ifdef LSP 6 | #ifndef __bpf__ 7 | #define __bpf__ 8 | #endif 9 | #include "../../../../include/scx/common.bpf.h" 10 | #else 11 | #include 12 | #endif 13 | 14 | #include 15 | #include 16 | #include 17 | 18 | extern const volatile u32 debug; 19 | 20 | #define dbg(fmt, args...) do { if (debug) bpf_printk(fmt, ##args); } while (0) 21 | #define trace(fmt, args...) do { if (debug > 1) bpf_printk(fmt, ##args); } while (0) 22 | 23 | enum MatchType { 24 | STR_PREFIX = 0, 25 | STR_SUFFIX = 1, 26 | STR_SUBSTR = 2 27 | }; 28 | 29 | bool match_str(const char *prefix, const char *str, enum MatchType match_type); 30 | char *format_cgrp_path(struct cgroup *cgrp); 31 | 32 | #endif /* __LAYERED_UTIL_H */ 33 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_layered/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_mitosis" 3 | version = "0.0.9" 4 | authors = ["Dan Schatzberg ", "Meta"] 5 | edition = "2021" 6 | description = "A dynamic affinity scheduler used within sched_ext, which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | bitvec = "1.0" 12 | cgroupfs = "0.9.0" 13 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 14 | ctrlc = { version = "3.1", features = ["termination"] } 15 | fb_procfs = "0.7" 16 | itertools = "0.13.0" 17 | lazy_static = "1.5.0" 18 | libbpf-rs = "=0.25.0-beta.1" 19 | libc = "0.2.137" 20 | log = "0.4.17" 21 | maplit = "1.0.2" 22 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 23 | serde = { version = "1.0.215", features = ["derive"] } 24 | serde_json = "1.0.133" 25 | simplelog = "0.12" 26 | 27 | [build-dependencies] 28 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 29 | 30 | [features] 31 | enable_backtrace = [] 32 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/README.md: -------------------------------------------------------------------------------- 1 | # scx_mitosis 2 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/mitosis.bpf.c", "bpf") 11 | .build() 12 | .unwrap(); 13 | } 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #ifndef __INTF_H 6 | #define __INTF_H 7 | 8 | #ifndef __KERNEL__ 9 | typedef unsigned long long u64; 10 | typedef unsigned int u32; 11 | typedef _Bool bool; 12 | #endif 13 | 14 | #ifdef LSP 15 | #define __bpf__ 16 | #include "../../../../include/scx/ravg.bpf.h" 17 | #else 18 | #include 19 | #endif 20 | 21 | enum consts { 22 | CACHELINE_SIZE = 64, 23 | MAX_CPUS_SHIFT = 9, 24 | MAX_CPUS = 1 << MAX_CPUS_SHIFT, 25 | MAX_CPUS_U8 = MAX_CPUS / 8, 26 | MAX_CELLS = 16, 27 | USAGE_HALF_LIFE = 100000000, /* 100ms */ 28 | 29 | HI_FALLBACK_DSQ = MAX_CELLS, 30 | LO_FALLBACK_DSQ = MAX_CELLS + 1, 31 | }; 32 | 33 | /* Statistics */ 34 | enum cell_stat_idx { 35 | CSTAT_LOCAL, 36 | CSTAT_GLOBAL, 37 | CSTAT_AFFN_VIOL, 38 | NR_CSTATS, 39 | }; 40 | 41 | struct cpu_ctx { 42 | u64 cstats[MAX_CELLS][NR_CSTATS]; 43 | u64 cell_cycles[MAX_CELLS]; 44 | u32 prev_cell; 45 | u32 cell; 46 | }; 47 | 48 | struct cgrp_ctx { 49 | u32 cell; 50 | bool cell_owner; 51 | }; 52 | 53 | /* 54 | * cell is the per-cell book-keeping 55 | */ 56 | struct cell { 57 | // current vtime of the cell 58 | u64 vtime_now; 59 | // which dsq the cell uses 60 | u32 dsq; 61 | // Whether or not the cell is used or not 62 | u32 in_use; 63 | }; 64 | 65 | #endif /* __INTF_H */ 66 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_mitosis/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_p2dq" 3 | version = "1.0.16" 4 | authors = ["Daniel Hodges "] 5 | edition = "2021" 6 | description = "scx_p2dq A simple pick two load balancing scheduler in BPF" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | chrono = "0.4" 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | crossbeam = "0.8.4" 14 | ctrlc = { version = "3.1", features = ["termination"] } 15 | fb_procfs = "0.7" 16 | lazy_static = "1.5.0" 17 | libbpf-rs = "=0.25.0-beta.1" 18 | libc = "0.2.137" 19 | log = "0.4.17" 20 | ordered-float = "3.4.0" 21 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 22 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 23 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 24 | serde = { version = "1.0.215", features = ["derive"] } 25 | simplelog = "0.12" 26 | sorted-vec = "0.8.3" 27 | static_assertions = "1.1.0" 28 | 29 | [build-dependencies] 30 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 31 | 32 | [features] 33 | enable_backtrace = [] 34 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/README.md: -------------------------------------------------------------------------------- 1 | # scx_p2dq 2 | 3 | ## Overview 4 | A simple pick 2 load balancing scheduler with (dumb) multi-layer queueing. 5 | 6 | The p2dq scheduler is a simple load balancing scheduler that uses a pick two 7 | algorithm for load balancing. A fixed number of DSQs are created per LLC with 8 | incremental slice intervals. If a task is able to consume the majority of the 9 | assigned slice is it dispatched to a DSQ with a longer slice. Tasks that do not 10 | consume more than half the slice are moved to shorter slice DSQs. The DSQs with 11 | the shortest slice lengths are then determined to be "interactive". All DSQs on 12 | the same LLC share the same vtime and there is special handling for 13 | (non)interactive tasks for load balancing purposes. 14 | 15 | The scheduler handles all scheduling decisions in BPF and the userspace 16 | component is only for metric reporting. 17 | 18 | ## Use Cases 19 | p2dq can perform well in a variety of workloads including interactive workloads 20 | such as gaming, batch processing and server applications. Tuning of of p2dq for 21 | each use case is required. 22 | 23 | ### Configuration 24 | The main idea behind p2dq is being able to classify which tasks are interactive 25 | and using a separate dispatch queue (DSQ) for them. Non interactive tasks 26 | can have special properties such as being able to be load balanced across 27 | LLCs/NUMA nodes. The `--autoslice` option will attempt to scale DSQ time slices 28 | based on the `--interactive-ratio`. DSQ time slices can also be set manually 29 | if the duration/distribution of tasks that are considered to be interactive is 30 | known in advance. `scxtop` can be used to get an understanding of time slice 31 | utilization so that DSQs can be properly configured. For desktop systems keeping 32 | the interactive ratio small (ex: <5) and using a small number of queues (2) will 33 | give a general performance with autoslice enabled. 34 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .add_source("src/bpf/lib/sdt_task.bpf.c") 12 | .add_source("src/bpf/lib/sdt_alloc.bpf.c") 13 | .add_source("src/bpf/lib/bitmap.bpf.c") 14 | .add_source("src/bpf/lib/cpumask.bpf.c") 15 | .add_source("src/bpf/lib/topology.bpf.c") 16 | .add_source("src/bpf/lib/arena.bpf.c") 17 | .compile_link_gen() 18 | .unwrap(); 19 | } 20 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #ifndef __P2DQ_INTF_H 6 | #define __P2DQ_INTF_H 7 | 8 | #include 9 | #ifndef __kptr 10 | #ifdef __KERNEL__ 11 | #error "__kptr_ref not defined in the kernel" 12 | #endif 13 | #define __kptr 14 | #endif 15 | 16 | #ifndef __KERNEL__ 17 | typedef unsigned char u8; 18 | typedef unsigned int u32; 19 | typedef unsigned long long u64; 20 | #endif 21 | 22 | 23 | enum consts { 24 | MAX_CPUS = 512, 25 | MAX_NUMA_NODES = 64, 26 | MAX_LLCS = 64, 27 | MAX_DSQS_PER_LLC = 8, 28 | MAX_TASK_PRIO = 39, 29 | MAX_TOPO_NODES = 1024, 30 | 31 | NSEC_PER_USEC = 1000ULL, 32 | NSEC_PER_MSEC = (1000ULL * NSEC_PER_USEC), 33 | MSEC_PER_SEC = 1000ULL, 34 | NSEC_PER_SEC = NSEC_PER_MSEC * MSEC_PER_SEC, 35 | 36 | LOAD_BALANCE_SLACK = 20ULL, 37 | 38 | // kernel definitions 39 | CLOCK_BOOTTIME = 7, 40 | 41 | STATIC_ALLOC_PAGES_GRANULARITY = 8, 42 | }; 43 | 44 | enum p2dq_timers_defs { 45 | EAGER_LOAD_BALANCER_TMR, 46 | MAX_TIMERS, 47 | }; 48 | 49 | enum stat_idx { 50 | P2DQ_STAT_DIRECT, 51 | P2DQ_STAT_DSQ_SAME, 52 | P2DQ_STAT_DSQ_CHANGE, 53 | P2DQ_STAT_IDLE, 54 | P2DQ_STAT_LB_SELECT, 55 | P2DQ_STAT_LB_DISPATCH, 56 | P2DQ_STAT_LLC_MIGRATION, 57 | P2DQ_STAT_NODE_MIGRATION, 58 | P2DQ_STAT_KEEP, 59 | P2DQ_STAT_SELECT_PICK2, 60 | P2DQ_STAT_DISPATCH_PICK2, 61 | P2DQ_STAT_WAKE_PREV, 62 | P2DQ_STAT_WAKE_LLC, 63 | P2DQ_STAT_WAKE_MIG, 64 | P2DQ_NR_STATS, 65 | }; 66 | 67 | enum scheduler_mode { 68 | MODE_PERFORMANCE, 69 | }; 70 | 71 | enum enqueue_promise_kind { 72 | P2DQ_ENQUEUE_PROMISE_COMPLETE, 73 | P2DQ_ENQUEUE_PROMISE_VTIME, 74 | P2DQ_ENQUEUE_PROMISE_FIFO, 75 | }; 76 | 77 | #endif /* __P2DQ_INTF_H */ 78 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/src/bpf/lib: -------------------------------------------------------------------------------- 1 | ../../../../../lib -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/src/bpf/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct p2dq_timer { 4 | // if set to 0 the timer will only be scheduled once 5 | u64 interval_ns; 6 | u64 init_flags; 7 | int start_flags; 8 | }; 9 | 10 | struct cpu_ctx { 11 | int id; 12 | u32 llc_id; 13 | u64 affn_dsq; 14 | u32 dsq_index; 15 | u64 dsq_id; 16 | u64 slice_ns; 17 | u32 perf; 18 | bool interactive; 19 | bool is_big; 20 | u64 ran_for; 21 | u32 node_id; 22 | u64 affn_max_vtime; 23 | u64 dsqs[MAX_DSQS_PER_LLC]; 24 | u64 max_load_dsq; 25 | }; 26 | 27 | struct llc_ctx { 28 | u32 id; 29 | u32 nr_cpus; 30 | u32 node_id; 31 | u64 vtime; 32 | u32 lb_llc_id; 33 | u64 last_period_ns; 34 | u64 load; 35 | u32 index; 36 | bool all_big; 37 | u64 affn_load; 38 | u64 affn_max_vtime; 39 | u64 dsqs[MAX_DSQS_PER_LLC]; 40 | u64 dsq_max_vtime[MAX_DSQS_PER_LLC]; 41 | u64 dsq_load[MAX_DSQS_PER_LLC]; 42 | struct bpf_cpumask __kptr *cpumask; 43 | struct bpf_cpumask __kptr *big_cpumask; 44 | struct bpf_cpumask __kptr *little_cpumask; 45 | struct bpf_cpumask __kptr *node_cpumask; 46 | }; 47 | 48 | struct node_ctx { 49 | u32 id; 50 | bool all_big; 51 | struct bpf_cpumask __kptr *cpumask; 52 | struct bpf_cpumask __kptr *big_cpumask; 53 | }; 54 | 55 | struct task_p2dq { 56 | u64 dsq_id; 57 | u64 slice_ns; 58 | int dsq_index; 59 | u32 llc_id; 60 | u32 node_id; 61 | u64 used; 62 | u64 last_dsq_id; 63 | u64 last_run_started; 64 | u64 last_run_at; 65 | u64 llc_runs; /* how many runs on the current LLC */ 66 | int last_dsq_index; 67 | 68 | /* The task is a workqueue worker thread */ 69 | bool is_kworker; 70 | 71 | /* Allowed to run on all CPUs */ 72 | bool all_cpus; 73 | }; 74 | 75 | typedef struct task_p2dq __arena task_ctx; 76 | 77 | struct enqueue_promise_vtime { 78 | u64 dsq_id; 79 | u64 enq_flags; 80 | u64 slice_ns; 81 | u64 vtime; 82 | }; 83 | 84 | struct enqueue_promise_fifo { 85 | u64 dsq_id; 86 | u64 enq_flags; 87 | u64 slice_ns; 88 | }; 89 | 90 | struct enqueue_promise { 91 | enum enqueue_promise_kind kind; 92 | union { 93 | struct enqueue_promise_vtime vtime; 94 | struct enqueue_promise_fifo fifo; 95 | }; 96 | }; 97 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_p2dq/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/.gitignore: -------------------------------------------------------------------------------- 1 | intf.h 2 | main.bpf.c 3 | bpf.rs 4 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_rlfifo" 3 | version = "1.0.12" 4 | authors = ["Andrea Righi "] 5 | edition = "2021" 6 | description = "A simple FIFO scheduler in Rust that runs in user-space" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | plain = "0.2.3" 12 | procfs = "0.17" 13 | ctrlc = { version = "3.1", features = ["termination"] } 14 | libbpf-rs = "=0.25.0-beta.1" 15 | libc = "0.2.137" 16 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 17 | scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "2.3.1" } 18 | 19 | [build-dependencies] 20 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 21 | scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "2.3.1" } 22 | 23 | [features] 24 | enable_backtrace = [] 25 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/README.md: -------------------------------------------------------------------------------- 1 | # scx_rlfifo 2 | 3 | This is a single user-defined scheduler used within [sched_ext](https://github.com/sched-ext/scx/tree/main), which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | scx_rlfifo is a simple Round-Robin scheduler runs in user-space, based on the 8 | scx_rustland_core framework. 9 | It dequeues tasks in FIFO order and assigns dynamic time slices, preempting and 10 | re-enqueuing tasks to achieve basic Round-Robin behavior. 11 | 12 | ## Typical Use Case 13 | 14 | This scheduler is provided as a simple template that can be used as a baseline 15 | to test more complex scheduling policies. 16 | 17 | ## Production Ready? 18 | 19 | Definitely not. Using this scheduler in a production environment is not 20 | recommended, unless there are specific requirements that necessitate a basic 21 | FIFO scheduling approach. Even then, it's still recommended to use the kernel's 22 | SCHED_FIFO real-time class. 23 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/build.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | fn main() { 5 | scx_rustland_core::RustLandBuilder::new() 6 | .unwrap() 7 | .build() 8 | .unwrap(); 9 | } 10 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | #![allow(non_upper_case_globals)] 5 | #![allow(non_camel_case_types)] 6 | #![allow(non_snake_case)] 7 | #![allow(dead_code)] 8 | 9 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 10 | -------------------------------------------------------------------------------- /scheds/rust/scx_rlfifo/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 5 | -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/.gitignore: -------------------------------------------------------------------------------- 1 | intf.h 2 | main.bpf.c 3 | bpf.rs 4 | -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_rustland" 3 | version = "1.0.12" 4 | authors = ["Andrea Righi "] 5 | edition = "2021" 6 | description = "A BPF component (dispatcher) that implements the low level sched-ext functionalities and a user-space counterpart (scheduler), written in Rust, that implements the actual scheduling policy. This is used within sched_ext, which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | plain = "0.2.3" 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | ctrlc = { version = "3.1", features = ["termination"] } 14 | libbpf-rs = "=0.25.0-beta.1" 15 | libc = "0.2.137" 16 | log = "0.4.17" 17 | ordered-float = "3.4.0" 18 | procfs = "0.17" 19 | serde = { version = "1.0.215", features = ["derive"] } 20 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 21 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 22 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 23 | scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "2.3.1" } 24 | simplelog = "0.12" 25 | 26 | [build-dependencies] 27 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 28 | scx_rustland_core = { path = "../../../rust/scx_rustland_core", version = "2.3.1" } 29 | 30 | [features] 31 | enable_backtrace = [] 32 | -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/build.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | fn main() { 5 | scx_rustland_core::RustLandBuilder::new() 6 | .unwrap() 7 | .build() 8 | .unwrap(); 9 | } 10 | -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | #![allow(non_upper_case_globals)] 5 | #![allow(non_camel_case_types)] 6 | #![allow(non_snake_case)] 7 | #![allow(dead_code)] 8 | 9 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 10 | -------------------------------------------------------------------------------- /scheds/rust/scx_rustland/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // This software may be used and distributed according to the terms of the 2 | // GNU General Public License version 2. 3 | 4 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 5 | -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_rusty" 3 | version = "1.0.12" 4 | authors = ["Dan Schatzberg ", "Meta"] 5 | edition = "2021" 6 | description = "A multi-domain, BPF / user space hybrid scheduler used within sched_ext, which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | chrono = "0.4" 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | crossbeam = "0.8.4" 14 | ctrlc = { version = "3.1", features = ["termination"] } 15 | fb_procfs = "0.7" 16 | libbpf-rs = "=0.25.0-beta.1" 17 | libc = "0.2.137" 18 | log = "0.4.17" 19 | ordered-float = "3.4.0" 20 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 21 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 22 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 23 | serde = { version = "1.0.215", features = ["derive"] } 24 | simplelog = "0.12" 25 | sorted-vec = "0.8.3" 26 | static_assertions = "1.1.0" 27 | 28 | [build-dependencies] 29 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 30 | 31 | [features] 32 | enable_backtrace = [] 33 | 34 | [lints.clippy] 35 | non_canonical_partial_ord_impl = "allow" 36 | -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/README.md: -------------------------------------------------------------------------------- 1 | # scx_rusty 2 | 3 | This is a single user-defined scheduler used within [sched_ext](https://github.com/sched-ext/scx/tree/main), which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | A multi-domain, BPF / user space hybrid scheduler. The BPF portion of the 8 | scheduler does a simple round robin in each domain, and the user space portion 9 | (written in Rust) calculates the load factor of each domain, and informs BPF of 10 | how tasks should be load balanced accordingly. 11 | 12 | ## How To Install 13 | 14 | Available as a [Rust crate](https://crates.io/crates/scx_rusty): `cargo add scx_rusty` 15 | 16 | ## Typical Use Case 17 | 18 | Rusty is designed to be flexible, accommodating different architectures and 19 | workloads. Various load balancing thresholds (e.g. greediness, frequency, etc), 20 | as well as how Rusty should partition the system into scheduling domains, can 21 | be tuned to achieve the optimal configuration for any given system or workload. 22 | 23 | ## Production Ready? 24 | 25 | Yes. If tuned correctly, rusty should be performant across various CPU 26 | architectures and workloads. By default, rusty creates a separate scheduling 27 | domain per-LLC, so its default configuration may be performant as well. Note 28 | however that scx_rusty does not yet disambiguate between LLCs in different NUMA 29 | nodes, so it may perform better on multi-CCX machines where all the LLCs share 30 | the same socket, as opposed to multi-socket machines. 31 | 32 | Note as well that you may run into an issue with infeasible weights, where a 33 | task with a very high weight may cause the scheduler to incorrectly leave cores 34 | idle because it thinks they're necessary to accommodate the compute for a 35 | single task. This can also happen in CFS, and should soon be addressed for 36 | scx_rusty. 37 | -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .add_source("src/bpf/sdt_task.bpf.c") 12 | .add_source("src/bpf/sdt_alloc.bpf.c") 13 | .compile_link_gen() 14 | .unwrap(); 15 | } 16 | -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_rusty/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_tickless" 3 | version = "1.0.3" 4 | authors = ["Andrea Righi ", "NVIDIA"] 5 | edition = "2021" 6 | description = "A server-oriented scheduler designed to minimize OS noise and maximize performance isolation. https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | affinity = "0.1" 11 | anyhow = "1.0.65" 12 | ctrlc = { version = "3.1", features = ["termination"] } 13 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 14 | crossbeam = "0.8.4" 15 | libbpf-rs = "=0.25.0-beta.1" 16 | log = "0.4.17" 17 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 18 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 19 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15", features = ["autopower"] } 20 | serde = { version = "1.0.215", features = ["derive"] } 21 | simplelog = "0.12" 22 | 23 | [build-dependencies] 24 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 25 | 26 | [features] 27 | enable_backtrace = [] 28 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/README.md: -------------------------------------------------------------------------------- 1 | # scx_tickless 2 | 3 | This is a single user-defined scheduler used within [sched_ext](https://github.com/sched-ext/scx/tree/main), which is a Linux kernel feature which enables implementing kernel thread schedulers in BPF and dynamically loading them. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | scx_tickless is a server-oriented scheduler designed for cloud computing, 8 | virtualization, and high-performance computing workloads. 9 | 10 | The scheduler works by routing all scheduling events through a pool of 11 | primary CPUs assigned to handle these events. This allows disabling the 12 | scheduler's tick on other CPUs, reducing OS noise. 13 | 14 | By default, only CPU 0 is included in the pool of primary CPUs. However, 15 | the pool size can be adjusted using the `--primary-domain CPUMASK` option. 16 | On systems with a large number of CPUs, allocating multiple CPUs to the 17 | primary pool may be beneficial. 18 | 19 | Tasks are placed into a global queue and the primary CPUs are responsible 20 | for distributing them to the other "tickless" CPUs. Preemption events are 21 | sent from the primary CPUs via IPC only when a "tickless" CPU is being 22 | contended by multiple tasks. 23 | 24 | The primary CPUs also perform the check for a contended CPU and the 25 | frequency of this check can be adjusted with the `--frequency FREQ` option. 26 | This effectively determines the tick frequency on the "tickless" CPUs when 27 | multiple tasks are competing for them. 28 | 29 | NOTE: in order to effectively disable ticks on the "tickless" CPUs the 30 | kernel must be booted with `nohz_full`. Keep in mind that `nohz_full` 31 | introduces syscall overhead, so this may regress latency-sensitive 32 | workloads. 33 | 34 | ## Typical Use Case 35 | 36 | Typical use cases include cloud computing, virtualization and high 37 | performance computing workloads. This scheduler is not designed for 38 | latency-sensitive workloads. 39 | 40 | ## Production Ready? 41 | 42 | This scheduler is still experimental. 43 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Andrea Righi 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "bpf") 11 | .build() 12 | .unwrap(); 13 | } 14 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/src/bpf/intf.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0 */ 2 | /* 3 | * Copyright (c) 2025 Andrea Righi 4 | * 5 | * This software may be used and distributed according to the terms of the GNU 6 | * General Public License version 2. 7 | */ 8 | #ifndef __INTF_H 9 | #define __INTF_H 10 | 11 | #ifndef __VMLINUX_H__ 12 | typedef unsigned char u8; 13 | typedef unsigned short u16; 14 | typedef unsigned int u32; 15 | typedef unsigned long u64; 16 | 17 | typedef signed char s8; 18 | typedef signed short s16; 19 | typedef signed int s32; 20 | typedef signed long s64; 21 | 22 | typedef int pid_t; 23 | #endif /* __VMLINUX_H__ */ 24 | 25 | struct cpu_arg { 26 | s32 cpu_id; 27 | }; 28 | 29 | #endif /* __INTF_H */ 30 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2025 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | #![allow(non_upper_case_globals)] 8 | #![allow(non_camel_case_types)] 9 | #![allow(non_snake_case)] 10 | #![allow(dead_code)] 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 13 | -------------------------------------------------------------------------------- /scheds/rust/scx_tickless/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0 2 | // 3 | // Copyright (c) 2025 Andrea Righi 4 | 5 | // This software may be used and distributed according to the terms of the 6 | // GNU General Public License version 2. 7 | 8 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 9 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scx_wd40" 3 | version = "1.0.12" 4 | authors = ["Emil Tsalapatis ", "Dan Schatzberg ", "Meta"] 5 | edition = "2021" 6 | description = "An experimental fork of the scx_rusty scheduler that uses BPF arenas to simplify scheduler development. Found in: https://github.com/sched-ext/scx/tree/main" 7 | license = "GPL-2.0-only" 8 | 9 | [dependencies] 10 | anyhow = "1.0.65" 11 | chrono = "0.4" 12 | clap = { version = "4.5.28", features = ["derive", "env", "unicode", "wrap_help"] } 13 | crossbeam = "0.8.4" 14 | ctrlc = { version = "3.1", features = ["termination"] } 15 | fb_procfs = "0.7" 16 | libbpf-rs = "=0.25.0-beta.1" 17 | nix = { features = ["process", "time"], default-features = false, version = "0.29" } 18 | log = "0.4.17" 19 | ordered-float = "3.4.0" 20 | scx_stats = { path = "../../../rust/scx_stats", version = "1.0.12" } 21 | scx_stats_derive = { path = "../../../rust/scx_stats/scx_stats_derive", version = "1.0.12" } 22 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 23 | serde = { version = "1.0.215", features = ["derive"] } 24 | simplelog = "0.12" 25 | sorted-vec = "0.8.3" 26 | static_assertions = "1.1.0" 27 | 28 | [build-dependencies] 29 | scx_utils = { path = "../../../rust/scx_utils", version = "1.0.15" } 30 | 31 | [features] 32 | enable_backtrace = [] 33 | 34 | [lints.clippy] 35 | non_canonical_partial_ord_impl = "allow" 36 | 37 | [workspace] 38 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/LICENSE: -------------------------------------------------------------------------------- 1 | ../../../LICENSE -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/README.md: -------------------------------------------------------------------------------- 1 | # scx_wd40 2 | 3 | An experimental fork of the scx_rusty scheduler that uses BPF arenas to simplify scheduler development. Found in the main [sched_ext](https://github.com/sched-ext/scx/tree/main) repository. [Read more about sched_ext](https://github.com/sched-ext/scx/tree/main). 4 | 5 | ## Overview 6 | 7 | A multi-domain, BPF / user space hybrid scheduler. The BPF portion of the 8 | scheduler does a simple round robin in each domain, and the user space portion 9 | (written in Rust) calculates the load factor of each domain, and informs BPF of 10 | how tasks should be load balanced accordingly. 11 | 12 | ## Goals 13 | 14 | This scheduler ultimately aims to demonstrate how to build modular BPF schedulers 15 | to enable easy code reuse between scheduler codebases. The main way of achieving 16 | this is through the use of BPF arenas that make it possible to directly share memory 17 | between the userspace and kernel scheduler components. This in turn lets us offload 18 | most of the complexity of the scheduler to userspace. Userspace components can be 19 | more easily combined, as opposed to scheduler BPF methods that are often mutually 20 | exclusive. 21 | 22 | ## Production Ready? 23 | 24 | No. This scheduler heavily uses BPF arenas and as such routinely requires a 25 | bleeding-edge kernel toolchain to even run and verify. 26 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | scx_utils::BpfBuilder::new() 8 | .unwrap() 9 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 10 | .enable_skel("src/bpf/main.bpf.c", "main") 11 | .add_source("src/bpf/lb_domain.bpf.c") 12 | .add_source("src/bpf/common.bpf.c") 13 | .add_source("src/bpf/deadline.bpf.c") 14 | .add_source("src/bpf/placement.bpf.c") 15 | .add_source("src/bpf/deadline.bpf.c") 16 | .add_source("../../../lib/arena.bpf.c") 17 | .add_source("../../../lib/bitmap.bpf.c") 18 | .add_source("../../../lib/cpumask.bpf.c") 19 | .add_source("../../../lib/sdt_task.bpf.c") 20 | .add_source("../../../lib/sdt_alloc.bpf.c") 21 | .add_source("../../../lib/topology.bpf.c") 22 | .compile_link_gen() 23 | .unwrap(); 24 | } 25 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf/common.bpf.c: -------------------------------------------------------------------------------- 1 | /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 2 | /* 3 | * This software may be used and distributed according to the terms of the 4 | * GNU General Public License version 2. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | 14 | #include 15 | 16 | #include "intf.h" 17 | #include "types.h" 18 | #include "lb_domain.h" 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | volatile u32 dom_numa_id_map[MAX_DOMS]; 27 | const volatile u32 debug; 28 | const volatile u32 load_half_life = 1000000000 /* 1s */; 29 | const volatile u32 nr_doms = 32; /* !0 for veristat, set during init */ 30 | const volatile u32 nr_nodes = 32; /* !0 for veristat, set during init */ 31 | 32 | /* base slice duration */ 33 | volatile u64 slice_ns; 34 | 35 | /* 36 | * Statistics 37 | */ 38 | struct { 39 | __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 40 | __uint(key_size, sizeof(u32)); 41 | __uint(value_size, sizeof(u64)); 42 | __uint(max_entries, RUSTY_NR_STATS); 43 | } stats SEC(".maps"); 44 | 45 | __weak 46 | int stat_add(enum stat_idx idx, u64 addend) 47 | { 48 | u32 idx_v = idx; 49 | 50 | u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx_v); 51 | if (cnt_p) 52 | (*cnt_p) += addend; 53 | 54 | return 0; 55 | } 56 | 57 | 58 | __hidden 59 | u32 dom_node_id(u32 dom_id) 60 | { 61 | const volatile u32 *nid_ptr; 62 | 63 | nid_ptr = MEMBER_VPTR(dom_numa_id_map, [dom_id]); 64 | if (!nid_ptr) { 65 | scx_bpf_error("Couldn't look up node ID for %d", dom_id); 66 | return 0; 67 | } 68 | return *nid_ptr; 69 | } 70 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf/common.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sched-ext/scx/41160ea2ab71d57f025100dd5cb3184b869d4c17/scheds/rust/scx_wd40/src/bpf/common.h -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf/deadline.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sched-ext/scx/41160ea2ab71d57f025100dd5cb3184b869d4c17/scheds/rust/scx_wd40/src/bpf/deadline.h -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf/lb_domain.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | int lb_domain_init(void); 6 | dom_ptr lb_domain_alloc(u32 dom_id); 7 | void lb_domain_free(dom_ptr domc); 8 | dom_ptr try_lookup_dom_ctx(u32 dom_id); 9 | dom_ptr lookup_dom_ctx(u32 dom_id); 10 | 11 | __weak s32 alloc_dom(u32 dom_id); 12 | __weak s32 create_dom(u32 dom_id); 13 | int dom_xfer_task(struct task_struct *p __arg_trusted, u32 new_dom_id, u64 now); 14 | 15 | extern volatile scx_bitmap_t node_data[MAX_NUMA_NODES]; 16 | extern const volatile u32 load_half_life; 17 | extern const volatile u32 debug; 18 | extern volatile u64 slice_ns; 19 | extern const volatile u32 nr_doms; 20 | extern const volatile u32 nr_nodes; 21 | 22 | #define lookup_task_ctx(p) ((task_ptr) scx_task_data(p)) 23 | u32 dom_node_id(u32 dom_id); 24 | void dom_dcycle_adj(dom_ptr domc, u32 weight, u64 now, bool runnable); 25 | 26 | static inline u64 min(u64 a, u64 b) 27 | { 28 | return a <= b ? a : b; 29 | } 30 | 31 | int stat_add(enum stat_idx idx, u64 addend); 32 | static inline u64 dom_min_vruntime(dom_ptr domc) 33 | { 34 | return READ_ONCE(domc->min_vruntime); 35 | } 36 | 37 | void place_task_dl(struct task_struct *p, task_ptr taskc, u64 enq_flags); 38 | void running_update_vtime(struct task_struct *p, task_ptr taskc); 39 | void stopping_update_vtime(struct task_struct *p); 40 | 41 | u64 update_freq(u64 freq, u64 interval); 42 | void init_vtime(struct task_struct *p, task_ptr taskc); 43 | void task_pick_and_set_domain(task_ptr taskc, 44 | struct task_struct *p, 45 | const struct cpumask *cpumask, 46 | bool init_dsq_vtime); 47 | bool task_set_domain(struct task_struct *p __arg_trusted, 48 | u32 new_dom_id, bool init_dsq_vtime); 49 | /* 50 | * Per-CPU context 51 | */ 52 | struct pcpu_ctx { 53 | u32 dom_rr_cur; /* used when scanning other doms */ 54 | dom_ptr domc; 55 | } __attribute__((aligned(CACHELINE_SIZE))); 56 | 57 | extern struct pcpu_ctx pcpu_ctx[MAX_CPUS]; 58 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf/types.h: -------------------------------------------------------------------------------- 1 | #ifndef __TYPES_H 2 | #define __TYPES_H 3 | 4 | typedef struct dom_ctx __arena *dom_ptr; 5 | struct dom_ctx; 6 | 7 | #define arena_lock_t arena_spinlock_t __arena * 8 | 9 | struct task_ctx { 10 | /* The domains this task can run on */ 11 | u64 dom_mask; 12 | u64 preferred_dom_mask; 13 | 14 | /* Arena pointer to this task's domain. */ 15 | dom_ptr domc; 16 | 17 | u32 target_dom; 18 | u32 weight; 19 | bool runnable; 20 | u64 dom_active_tasks_gen; 21 | u64 deadline; 22 | 23 | u64 sum_runtime; 24 | u64 avg_runtime; 25 | u64 last_run_at; 26 | 27 | /* frequency with which a task is blocked (consumer) */ 28 | u64 blocked_freq; 29 | u64 last_blocked_at; 30 | 31 | /* frequency with which a task wakes other tasks (producer) */ 32 | u64 waker_freq; 33 | u64 last_woke_at; 34 | 35 | /* The task is a workqueue worker thread */ 36 | bool is_kworker; 37 | 38 | /* Allowed on all CPUs and eligible for DIRECT_GREEDY optimization */ 39 | bool all_cpus; 40 | 41 | /* select_cpu() telling enqueue() to queue directly on the DSQ */ 42 | bool dispatch_local; 43 | 44 | /* For visibility from userspace, may become stale after multithreaded exec */ 45 | u32 pid; 46 | 47 | struct ravg_data dcyc_rd; 48 | 49 | scx_bitmap_t cpumask; 50 | }; 51 | 52 | typedef struct task_ctx __arena *task_ptr; 53 | 54 | struct bucket_ctx { 55 | u64 dcycle; 56 | struct ravg_data rd; 57 | }; 58 | 59 | struct dom_active_tasks { 60 | u64 gen; 61 | u64 read_idx; 62 | u64 write_idx; 63 | task_ptr tasks[MAX_DOM_ACTIVE_TPTRS]; 64 | }; 65 | 66 | struct dom_ctx { 67 | u32 id; 68 | u64 min_vruntime; 69 | 70 | u64 dbg_dcycle_printed_at; 71 | struct bucket_ctx buckets[LB_LOAD_BUCKETS]; 72 | struct dom_active_tasks active_tasks; 73 | 74 | scx_bitmap_t cpumask; 75 | scx_bitmap_t direct_greedy_cpumask; 76 | scx_bitmap_t node_cpumask; 77 | 78 | arena_lock_t vtime_lock; 79 | }; 80 | 81 | #endif /* __TYPES_H */ 82 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /scheds/rust/scx_wd40/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /scripts/bpftrace_stress_wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set +x 4 | 5 | STRESS_CMD="$1" 6 | SCHED_CMD="$2" 7 | TIMEOUT_SEC="$3" 8 | BPFTRACE_SCRIPTS="$4" 9 | KERNEL_HEADERS="$5" 10 | 11 | echo $PWD 12 | 13 | SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 14 | 15 | for a in $(echo "$BPFTRACE_SCRIPTS" | tr ',' ' '); do 16 | BPFTRACE_KERNEL_SOURCE=${KERNEL_HEADERS} bpftrace -o "$a.ci.log" $SCRIPT_DIR/$a & 17 | done 18 | 19 | $STRESS_CMD > stress_cmd.ci.log 2>&1 & 20 | 21 | STRESS_PID=$! 22 | 23 | timeout --foreground --preserve-status $TIMEOUT_SEC $SCHED_CMD > sched_output.ci.log 2>&1 & 24 | 25 | STATUS_PID=$! 26 | 27 | # wait for scheduler to exit 28 | tail --pid=$STATUS_PID -f sched_output.ci.log 29 | 30 | # wait for stress to exit, ignore err 31 | wait $STRESS_PID || true 32 | 33 | sleep 10 34 | 35 | killall -w 10s $(jobs -p) 36 | 37 | for a in $(echo "$BPFTRACE_SCRIPTS" | tr ',' ' '); do 38 | echo "$a OUTPUT" 39 | cat "$a.ci.log" 40 | echo "$a OUTPUT DONE" 41 | done 42 | 43 | echo "STRESS OUTPUT" 44 | cat stress_cmd.ci.log 45 | echo "STRESS OUTPUT DONE" 46 | 47 | # if anything isn't right, exit non 0 so we know. 48 | wait $STRESS_PID 49 | wait $STATUS_PID -------------------------------------------------------------------------------- /scripts/dsq_lat.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bpftrace 2 | // Copyright (c) Meta Platforms, Inc. and affiliates. 3 | 4 | // This software may be used and distributed according to the terms of the 5 | // GNU General Public License version 2. 6 | 7 | /* 8 | * dsq_lat.bt - Observe DSQ latencies 9 | * 10 | * Prints an average and histogram of DSQ latencies by timestamping from when a 11 | * task is enqueued to a DSQ to when a task is running. 12 | * 13 | * PIDs can be filtered by passing a parameter to dsq_lat.bt (0 for all pids): 14 | * 15 | * # filter PID 1234 16 | * $ ./dsq_lat.bt 1234 17 | * 18 | * DSQs can be filtered by passing a second parameter: 19 | * 20 | * # filter DSQ 789 21 | * $ ./dsq_lat.bt 0 789 22 | * 23 | */ 24 | 25 | kprobe:scx_bpf_dsq_insert, 26 | kprobe:scx_bpf_dispatch, 27 | kprobe:scx_bpf_dsq_insert_vtime, 28 | kprobe:scx_bpf_dispatch_vtime, 29 | { 30 | $task = (struct task_struct *)arg0; 31 | $dsq = arg1; 32 | 33 | if ($1 > 0 && $task->tgid != (int32) $1) { 34 | return; 35 | } 36 | if ($2 >= 0 && $2 != $dsq && $# == 2) { 37 | return; 38 | } 39 | 40 | if ($dsq >= 0) { 41 | @qtime[$task->pid] = nsecs; 42 | @task_dsq[$task->pid] = $dsq; 43 | } 44 | } 45 | 46 | rawtracepoint:sched_switch 47 | { 48 | $prev = (struct task_struct *)arg1; 49 | $next = (struct task_struct *)arg2; 50 | $prev_state = arg3; 51 | 52 | if ($1 > 0 && $next->tgid != (int32) $1) { 53 | return; 54 | } 55 | 56 | $start = @qtime[$next->pid]; 57 | $dsq = @task_dsq[$next->pid]; 58 | if ($2 >= 0 && $2 != $dsq && $# == 2) { 59 | delete(@qtime[$next->pid]); 60 | delete(@task_dsq[$next->pid]); 61 | return; 62 | } 63 | 64 | if ($start && $dsq >= 0 && $dsq < (uint64) 2<<16) { 65 | $dur = nsecs - $start; 66 | $usec = $dur / 1000; 67 | @lat_avg_usec[$dsq] = avg($usec); 68 | @dsq_hist_usec[$dsq] = hist($usec); 69 | @dsq_lat_avg_usec[$dsq] = avg($usec); 70 | } 71 | delete(@qtime[$next->pid]); 72 | delete(@task_dsq[$next->pid]); 73 | } 74 | 75 | interval:s:1 { 76 | print("------------------------------"); 77 | print(@lat_avg_usec); 78 | print(@dsq_hist_usec); 79 | print(@dsq_lat_avg_usec); 80 | } 81 | 82 | END { 83 | clear(@task_dsq); 84 | clear(@qtime); 85 | } 86 | -------------------------------------------------------------------------------- /scripts/freq_trace.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bpftrace 2 | // Copyright (c) Meta Platforms, Inc. and affiliates. 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | rawtracepoint:cpu_frequency 7 | { 8 | $freq = arg0; 9 | $cpu = arg1; 10 | 11 | @freq_hist = lhist($freq, 0, 6000000, 50000); 12 | @avg_freq[$cpu] = avg($freq); 13 | } 14 | 15 | interval:s:1 { 16 | print(@avg_freq); 17 | print(@freq_hist); 18 | } 19 | -------------------------------------------------------------------------------- /scripts/process_runqlat.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bpftrace 2 | 3 | /* 4 | * Copyright (c) Jose Fernandez 5 | * 6 | * process_runqlat.bt - Instrument runqueue latency for a PID and its threads. 7 | * 8 | * This script measures the runqueue latency for a specified process ID (PID) 9 | * and includes all threads spawned by that process. 10 | * 11 | * USAGE: sudo ./process_runqlat.bt 12 | * 13 | * The program output will include: 14 | * - Stats by task (count, avg latency, total latency) 15 | * - A histogram of all latency measurements 16 | * - Aggregated total stats (count, avg latency, total latency) 17 | */ 18 | 19 | #include 20 | 21 | BEGIN 22 | { 23 | if ($1 == 0) { 24 | printf("PID is missing, use `sudo ./process_runqlat.bt `\n"); 25 | exit(); 26 | } 27 | printf("Instrumenting runqueue latency for PID %d. Hit Ctrl-C to end.\n", $1); 28 | } 29 | 30 | /* 31 | * args: 32 | * - struct task_struct *p 33 | */ 34 | rawtracepoint:sched_wakeup, 35 | rawtracepoint:sched_wakeup_new, 36 | { 37 | $task = (struct task_struct *)arg0; 38 | // We filter by tgid to include all threads of the process 39 | if ($task->tgid == $1) { 40 | @qtime[$task->pid] = nsecs; 41 | } 42 | } 43 | 44 | /* 45 | * args: 46 | * - bool preempt 47 | * - struct task_struct *prev 48 | * - struct task_struct *next 49 | * - unsigned int prev_state 50 | */ 51 | rawtracepoint:sched_switch 52 | { 53 | $prev = (struct task_struct *)arg1; 54 | $next = (struct task_struct *)arg2; 55 | $prev_state = arg3; 56 | 57 | if ($next->tgid != $1) { 58 | return; 59 | } 60 | 61 | if ($prev_state == TASK_RUNNING && $prev->tgid == $1) { 62 | @qtime[$prev->pid] = nsecs; 63 | } 64 | 65 | $nsec = @qtime[$next->pid]; 66 | if ($nsec) { 67 | $usec = (nsecs - $nsec) / 1000; 68 | @usec_total_stats = stats($usec); 69 | @usec_hist = hist($usec); 70 | @tasks[$next->comm, $next->pid] = stats($usec); 71 | } 72 | delete(@qtime[$next->pid]); 73 | } 74 | 75 | END 76 | { 77 | clear(@qtime); 78 | } -------------------------------------------------------------------------------- /scripts/sched_ftrace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import time 6 | 7 | HEADER = "TASK-PID" 8 | BUFF_STARTED = "buffer started ###" 9 | TRACING_PATH = "/sys/kernel/tracing" 10 | TRACE_PIPE_PATH = os.path.join(TRACING_PATH, "trace_pipe") 11 | 12 | 13 | def ftrace_trim(stream, duration, nproc): 14 | nproc = nproc - 1 15 | seen_header = False 16 | proc_buffer_started = 0 17 | start_time = time.time() 18 | 19 | for line in stream: 20 | if time.time() - start_time >= duration: 21 | break 22 | l = line.replace("\n", "") 23 | if HEADER in l: 24 | seen_header = True 25 | print(l) 26 | if BUFF_STARTED in line: 27 | proc_buffer_started += 1 28 | continue 29 | if proc_buffer_started == nproc or not seen_header: 30 | print(l) 31 | 32 | 33 | def run_trace(duration): 34 | tracing_on_path = os.path.join(TRACING_PATH, "tracing_on") 35 | sched_switch_enable_path = os.path.join( 36 | TRACING_PATH, "events/sched/sched_switch/enable" 37 | ) 38 | 39 | # Enable tracing and sched_switch event 40 | with open(tracing_on_path, "w") as f: 41 | f.write("1") 42 | with open(sched_switch_enable_path, "w") as f: 43 | f.write("1") 44 | 45 | # Process the sched_switch events from the trace file 46 | try: 47 | with open(TRACE_PIPE_PATH, "r") as trace_pipe: 48 | ftrace_trim(trace_pipe, duration, os.cpu_count()) 49 | 50 | except KeyboardInterrupt: 51 | pass # Allow clean termination with Ctrl+C 52 | 53 | # Disable tracing and sched_switch event after the duration 54 | with open(sched_switch_enable_path, "w") as f: 55 | f.write("0") 56 | with open(tracing_on_path, "w") as f: 57 | f.write("0") 58 | 59 | 60 | def main(): 61 | duration = int(sys.argv[1]) if len(sys.argv) > 1 else 5 62 | run_trace(duration) 63 | 64 | 65 | if __name__ == "__main__": 66 | main() 67 | -------------------------------------------------------------------------------- /scripts/slicesnoop.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bpftrace 2 | // Copyright (c) Meta Platforms, Inc. and affiliates. 3 | 4 | // This software may be used and distributed according to the terms of the 5 | // GNU General Public License version 2. 6 | 7 | 8 | /* 9 | * slicesnoop - Explore the slice distribution of DSQs 10 | * 11 | * This script is used to explore the distrubtion of slice intervals for 12 | * schedulers aggregated by DSQ id. 13 | * 14 | * Processes can be filtered by passing a pid as the first parameter (0 for 15 | * all pids): 16 | * 17 | * # filter pid 1234 18 | * $ ./slicesnoop.bt 1234 19 | * # all pids (default) 20 | * $ ./slicesnoop.bt 0 21 | * 22 | * DSQs (above 0) can be filtered by passing the dsq id as the second parameter: 23 | * 24 | * # filter dsq 1234 25 | * $ ./slicesnoop.bt 0 1234 26 | */ 27 | 28 | kprobe:scx_bpf_dsq_insert, 29 | kprobe:scx_bpf_dispatch, 30 | kprobe:scx_bpf_dsq_insert_vtime, 31 | kprobe:scx_bpf_dispatch_vtime, 32 | { 33 | $task = (struct task_struct *)arg0; 34 | $dsq = arg1; 35 | $slice = arg2; 36 | 37 | if ($1 > 0 && $task->tgid != $1) { 38 | return; 39 | } 40 | if ($2 > 0 && $2 != $dsq) { 41 | return; 42 | } 43 | 44 | if ($dsq >= 0 && $slice > 0) { 45 | @dsq_slice_avg[$dsq] = avg($slice); 46 | @dsq_slice[$dsq] = hist($slice); 47 | } 48 | } 49 | 50 | 51 | interval:s:1 { 52 | print("-----------------------------------"); 53 | print(@dsq_slice); 54 | print(@dsq_slice_avg); 55 | } 56 | -------------------------------------------------------------------------------- /scripts/vtime_dist.bt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bpftrace 2 | // Copyright (c) Meta Platforms, Inc. and affiliates. 3 | 4 | // This software may be used and distributed according to the terms of the 5 | // GNU General Public License version 2. 6 | 7 | 8 | /* 9 | * vtime_dist - Show the distribution of the change in vtime 10 | * 11 | * This script is used to probe scx_bpf_dispatch_vtime and prints the 12 | * distribution of the difference in the vtime parameter The values are 13 | * aggregated by dsq id. 14 | * 15 | * Processes can be filtered by passing a pid as the first parameter (0 for 16 | * all pids): 17 | * 18 | * # filter pid 1234 19 | * $ ./dsq_lat.bt 1234 20 | * # all pids (default) 21 | * $ ./dsq_lat.bt 0 22 | * 23 | * DSQs (above 0) can be filtered by passing the dsq id as the second parameter: 24 | * 25 | * # filter dsq 1234 26 | * $ ./dsq_lat.bt 0 1234 27 | */ 28 | 29 | kprobe:scx_bpf_dsq_insert_vtime, 30 | kprobe:scx_bpf_dispatch_vtime, 31 | { 32 | $task = (struct task_struct *)arg0; 33 | $dsq = arg1; 34 | $vtime = arg3; 35 | 36 | if ($1 > 0 && $task->tgid != $1) { 37 | return; 38 | } 39 | if ($2 > 0 && $2 != $dsq) { 40 | return; 41 | } 42 | 43 | if ($dsq >= 0) { 44 | $prev_vtime = @dsq_vtime_now[$dsq]; 45 | @dsq_vtime_prev[$dsq] = $prev_vtime; 46 | @dsq_vtime_now[$dsq] = $vtime; 47 | @dsq_vtime_diff[$dsq] = avg($vtime - $prev_vtime); 48 | @dsq_vtime[$dsq] = hist($vtime - $prev_vtime); 49 | } 50 | } 51 | 52 | 53 | interval:s:1 { 54 | print("-----------------------------------"); 55 | print(@dsq_vtime); 56 | print(@dsq_vtime_diff); 57 | } 58 | -------------------------------------------------------------------------------- /services/README.md: -------------------------------------------------------------------------------- 1 | # A Quick Start Guide 2 | 3 | This guide provides instructions for running the SCX schedulers as a systemd service and checking its logs. 4 | 5 | ## Getting Started 6 | 7 | At the very beginning, configure the /etc/default/scx file: 8 | 9 | - in the SCX_SCHEDULER variable, select the scheduler you are interested in 10 | 11 | - in the SCX_FLAGS variable, specify the flags you want to add. To do this, execute and read what flags you can add. 12 | 13 | ``` 14 | scx_SCHEDNAME --help 15 | ``` 16 | 17 | To start the SCX scheduler at boot, you need to run the systemd service as root. Here are the steps: 18 | 19 | 20 | - Enable the service: 21 | 22 | ``` 23 | systemctl enable scx.service 24 | ``` 25 | 26 | - Start the service: 27 | 28 | ``` 29 | systemctl start scx.service 30 | ``` 31 | 32 | Alternatively, you can use a shortened version of these commands: 33 | 34 | ``` 35 | systemctl enable --now scx.service 36 | ``` 37 | 38 | - To check the status of the service, use the following command: 39 | 40 | ``` 41 | systemctl status scx.service 42 | ``` 43 | 44 | ## Override global configuration 45 | 46 | It is possible to override the global scx settings using systemd environment 47 | variables `SCX_SCHEDULER_OVERRIDE` and `SCX_FLAGS_OVERRIDE`. 48 | 49 | Example: 50 | 51 | ``` 52 | systemctl set-environment SCX_SCHEDULER_OVERRIDE=scx_rustland 53 | systemctl set-environment SCX_FLAGS_OVERRIDE="-s 10000" 54 | systemctl restart scx 55 | ``` 56 | 57 | If you want to restore the default value from the `/etc/default/scx` file execute: 58 | 59 | ``` 60 | systemctl unset-environment SCX_SCHEDULER_OVERRIDE 61 | systemctl unset-environment SCX_FLAGS_OVERRIDE 62 | systemctl restart scx 63 | ``` 64 | 65 | ## Checking journald Logs 66 | 67 | 68 | - To view the logs, use the following command: 69 | 70 | ``` 71 | journalctl -u scx.service 72 | ``` 73 | 74 | - To view the logs of the current session use the command 75 | 76 | ``` 77 | journalctl -u scx.service -b 0 78 | ``` 79 | 80 | -------------------------------------------------------------------------------- /services/openrc/meson.build: -------------------------------------------------------------------------------- 1 | # Install the 'scx.initrd' file to /etc/init.d/scx 2 | install_data('scx.initrd', install_dir: '/etc/init.d', rename: 'scx') 3 | 4 | # Install the 'scx' file to the '/etc/default' directory 5 | install_data('../scx', install_dir: '/etc/default') 6 | 7 | # Symlinking /etc/default/scx to /etc/conf.d/scx 8 | install_symlink( 9 | 'scx', 10 | pointing_to: '/etc/default/scx', 11 | install_dir: '/etc/conf.d' 12 | ) 13 | # meson.add_install_script( 14 | # 'sh', '-c', 15 | # 'mkdir -p $(dirname $DESTDIR/@0@/@1@)'.format(systemd_system_unit_dir, 16 | # 'multi-user.target.wants/phosphor-certificate-manager@nslcd.service'), 17 | # ) 18 | # meson.add_install_script( 19 | # 'sh', '-c', 20 | # 'ln -s @0@ $DESTDIR/@1@/@2@'.format('../phosphor-certificate-manager@.service', 21 | # systemd_system_unit_dir, 22 | # 'multi-user.target.wants/phosphor-certificate-manager@nslcd.service'), 23 | # ) 24 | -------------------------------------------------------------------------------- /services/openrc/scx.initrd: -------------------------------------------------------------------------------- 1 | #!/usr/bin/openrc-run 2 | # Powered by CachyOS Team 3 | # Distributed under the terms of the GNU General Public License v2 4 | 5 | description="Start scx_scheduler" 6 | supervisor=supervise-daemon 7 | required_files="/etc/default/scx" 8 | 9 | command="/usr/bin/$SCX_SCHEDULER" 10 | command_args="$SCX_FLAGS" 11 | command_user="${SCX_USER:-root}:${SCX_GROUP:-root}" 12 | 13 | # stop_post() { 14 | # rm -rf /var/cache/"${RC_SVCNAME}" 15 | # } 16 | -------------------------------------------------------------------------------- /services/scx: -------------------------------------------------------------------------------- 1 | # List of scx_schedulers: scx_bpfland scx_central scx_flash scx_lavd scx_layered scx_nest scx_qmap scx_rlfifo scx_rustland scx_rusty scx_simple scx_userland scx_p2dq scx_tickless 2 | SCX_SCHEDULER=scx_bpfland 3 | 4 | # Set custom flags for each scheduler, below is an example of how to use 5 | #SCX_FLAGS='-p -m performance' 6 | -------------------------------------------------------------------------------- /services/systemd/meson.build: -------------------------------------------------------------------------------- 1 | # Get the directory for systemd system unit files 2 | systemd_system_unit_dir = systemd.get_variable(pkgconfig : 'systemdsystemunitdir') 3 | 4 | # Install the 'scx.service' file to the systemd system unit directory 5 | install_data('scx.service', install_dir: systemd_system_unit_dir) 6 | 7 | # Install the 'scx_loader.service' file to the systemd system unit directory 8 | install_data('scx_loader.service', install_dir: systemd_system_unit_dir) 9 | 10 | # Install the 'org.scx.Loader.service' file to the dbus system services directory 11 | install_data('org.scx.Loader.service', install_dir: '/usr/share/dbus-1/system-services') 12 | 13 | # Install the 'scx' file to the '/etc/default' directory 14 | install_data('../scx', install_dir: '/etc/default') 15 | -------------------------------------------------------------------------------- /services/systemd/org.scx.Loader.service: -------------------------------------------------------------------------------- 1 | [D-BUS Service] 2 | Name=org.scx.Loader 3 | Exec=/usr/bin/scx_loader 4 | User=root 5 | SystemdService=scx_loader.service 6 | -------------------------------------------------------------------------------- /services/systemd/scx.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Start scx_scheduler 3 | ConditionPathIsDirectory=/sys/kernel/sched_ext 4 | StartLimitIntervalSec=30 5 | StartLimitBurst=2 6 | 7 | [Service] 8 | Type=simple 9 | EnvironmentFile=/etc/default/scx 10 | ExecStart=/bin/bash -c 'exec ${SCX_SCHEDULER_OVERRIDE:-$SCX_SCHEDULER} ${SCX_FLAGS_OVERRIDE:-$SCX_FLAGS} ' 11 | Restart=on-failure 12 | StandardError=journal 13 | 14 | [Install] 15 | WantedBy=graphical.target 16 | -------------------------------------------------------------------------------- /services/systemd/scx_loader.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=DBUS on-demand loader of sched-ext schedulers 3 | 4 | [Service] 5 | Type=dbus 6 | BusName=org.scx.Loader 7 | ExecStart=/usr/bin/scx_loader 8 | KillSignal=SIGINT 9 | 10 | [Install] 11 | WantedBy=graphical.target 12 | -------------------------------------------------------------------------------- /tools/scxctl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [[bin]] 2 | name = "scxctl" 3 | 4 | [package] 5 | name = "scxctl" 6 | version = "1.0.12" 7 | edition = "2021" 8 | description = "A cli dbus client for scx_loader" 9 | authors = ["Joe Maples "] 10 | repository = "https://github.com/sched-ext/scx" 11 | readme = "README.md" 12 | keywords = ["linux", "scx", "scx_scheds", "scx_loader", "sched_ext"] 13 | categories = ["command-line-utilities"] 14 | license = "GPL-2.0-only" 15 | 16 | [dependencies] 17 | clap = { version = "4.5.28", features = ["derive"] } 18 | colored = "3.0.0" 19 | zbus = "5.3.1" 20 | scx_loader = { path = "../../rust/scx_loader", version="1.0.12" } 21 | -------------------------------------------------------------------------------- /tools/scxctl/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /tools/scxctl/src/cli.rs: -------------------------------------------------------------------------------- 1 | use clap::{Parser, Subcommand}; 2 | use scx_loader::SchedMode; 3 | 4 | #[derive(Parser, Debug)] 5 | #[command(author, version, about, long_about = None)] 6 | pub struct Cli { 7 | #[command(subcommand)] 8 | pub command: Commands, 9 | } 10 | 11 | #[derive(Parser, Debug)] 12 | #[group(required = true)] 13 | pub struct StartArgs { 14 | #[arg(short, long, help = "Scheduler to start", required = true)] 15 | pub sched: String, 16 | #[arg( 17 | short, 18 | long, 19 | value_enum, 20 | default_value = "auto", 21 | conflicts_with = "args", 22 | help = "Mode to start in" 23 | )] 24 | pub mode: Option, 25 | #[arg( 26 | short, 27 | long, 28 | value_delimiter(','), 29 | requires = "sched", 30 | conflicts_with = "mode", 31 | help = "Arguments to run scheduler with" 32 | )] 33 | pub args: Option>, 34 | } 35 | 36 | #[derive(Parser, Debug)] 37 | #[group(required = true)] 38 | pub struct SwitchArgs { 39 | #[arg(short, long, help = "Scheduler to switch to")] 40 | pub sched: Option, 41 | #[arg( 42 | short, 43 | long, 44 | value_enum, 45 | conflicts_with = "args", 46 | help = "Mode to switch to" 47 | )] 48 | pub mode: Option, 49 | #[arg( 50 | short, 51 | long, 52 | value_delimiter(','), 53 | requires = "sched", 54 | conflicts_with = "mode", 55 | help = "Arguments to run scheduler with" 56 | )] 57 | pub args: Option>, 58 | } 59 | 60 | #[derive(Subcommand, Debug)] 61 | pub enum Commands { 62 | #[command(about = "Get the current scheduler and mode")] 63 | Get, 64 | #[command(about = "List all supported schedulers")] 65 | List, 66 | #[command(about = "Start a scheduler in a mode or with arguments")] 67 | Start { 68 | #[clap(flatten)] 69 | args: StartArgs, 70 | }, 71 | #[command(about = "Switch schedulers or modes, optionally with arguments")] 72 | Switch { 73 | #[clap(flatten)] 74 | args: SwitchArgs, 75 | }, 76 | #[command(about = "Stop the current scheduler")] 77 | Stop, 78 | } 79 | -------------------------------------------------------------------------------- /tools/scxtop/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "scxtop" 3 | version = "1.0.13" 4 | edition = "2021" 5 | authors = ["Daniel Hodges "] 6 | license = "GPL-2.0-only" 7 | repository = "https://github.com/sched-ext/scx" 8 | description = "sched_ext scheduler tool for observability" 9 | 10 | [package.metadata.scx] 11 | ci.use_clippy = true 12 | 13 | [dependencies] 14 | anyhow = "1.0.65" 15 | clap = { version = "4.5.28", features = [ 16 | "derive", 17 | "cargo", 18 | "wrap_help", 19 | "unicode", 20 | "string", 21 | "unstable-styles", 22 | ] } 23 | clap_complete = "4.5.45" 24 | config = "0.14.1" 25 | crossterm = { version = "0.28.1", features = ["serde", "event-stream"] } 26 | derive_deref = "1.1.1" 27 | directories = "5.0.1" 28 | futures = "0.3.31" 29 | glob = "0.3.2" 30 | json5 = "0.4.1" 31 | lazy_static = "1.5.0" 32 | libbpf-rs = "=0.25.0-beta.1" 33 | libc = "0.2.137" 34 | log = "0.4.17" 35 | num-format = { version = "0.4.3", features = ["with-serde", "with-system-locale"] } 36 | plain = "0.2.3" 37 | prost = "0.13.5" 38 | rand = "0.8.5" 39 | ratatui = { version = "0.29.0", features = ["serde", "macros"] } 40 | regex = "1.11.1" 41 | scx_stats = { path = "../../rust/scx_stats", version = "1.0.12" } 42 | scx_utils = { path = "../../rust/scx_utils", version = "1.0.15" } 43 | simplelog = "0.12" 44 | serde_json = "1.0.133" 45 | serde = { version = "1.0.215", features = ["derive"] } 46 | signal-hook = "0.3.17" 47 | strip-ansi-escapes = "0.2.0" 48 | tokio-util = "0.7.13" 49 | tokio = { version = "1.42.0", features = ["full"] } 50 | toml = "0.8.19" 51 | tracing = "0.1.41" 52 | tracing-error = "0.2.1" 53 | tracing-subscriber = { version = "0.3.19", features = ["env-filter", "serde"] } 54 | xdg = "2.5.2" 55 | log-panics = { version = "2", features = ["with-backtrace"]} 56 | hashbrown = "0.15.2" 57 | smartstring = { version = "1.0.1", features = ["serde"] } 58 | nix = { version = "0.29", features = ["time"] } 59 | 60 | [dev-dependencies] 61 | criterion = "0.6.0" 62 | 63 | [[bench]] 64 | name = "search_benchmark" 65 | harness = false 66 | 67 | [build-dependencies] 68 | prost-build = "0.13.5" 69 | scx_stats = { path = "../../rust/scx_stats", version = "1.0.12" } 70 | scx_utils = { path = "../../rust/scx_utils", version = "1.0.15" } 71 | -------------------------------------------------------------------------------- /tools/scxtop/build.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | fn main() { 7 | prost_build::Config::new() 8 | .compile_protos(&["src/protos/perfetto_scx.proto"], &["src/protos"]) 9 | .unwrap(); 10 | scx_utils::BpfBuilder::new() 11 | .unwrap() 12 | .enable_intf("src/bpf/intf.h", "bpf_intf.rs") 13 | .enable_skel("src/bpf/main.bpf.c", "bpf") 14 | .compile_link_gen() 15 | .unwrap(); 16 | } 17 | -------------------------------------------------------------------------------- /tools/scxtop/src/bpf_intf.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | #![allow(non_upper_case_globals)] 6 | #![allow(non_camel_case_types)] 7 | #![allow(non_snake_case)] 8 | #![allow(dead_code)] 9 | 10 | include!(concat!(env!("OUT_DIR"), "/bpf_intf.rs")); 11 | -------------------------------------------------------------------------------- /tools/scxtop/src/bpf_skel.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | // We can't directly include the generated skeleton in main.rs as it may 7 | // contain compiler attributes that can't be `include!()`ed via macro and we 8 | // can't use the `#[path = "..."]` because `concat!(env!("OUT_DIR"), 9 | // "/bpf.skel.rs")` does not work inside the path attribute yet (see 10 | // https://github.com/rust-lang/rust/pull/83366). 11 | 12 | include!(concat!(env!("OUT_DIR"), "/bpf_skel.rs")); 13 | -------------------------------------------------------------------------------- /tools/scxtop/src/bpf_stats.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use crate::bpf_intf; 7 | use crate::bpf_skel::BpfSkel; 8 | 9 | use libbpf_rs::MapCore; 10 | 11 | const STAT_DROPPED_EVENTS: usize = bpf_intf::stat_id_STAT_DROPPED_EVENTS as usize; 12 | 13 | #[derive(Default, Debug)] 14 | pub struct BpfStats { 15 | pub dropped_events: u64, 16 | } 17 | 18 | impl BpfStats { 19 | pub fn get_from_skel(skel: &BpfSkel<'_>) -> anyhow::Result { 20 | let all_cpus = skel 21 | .maps 22 | .stats 23 | .lookup_percpu(&0_u32.to_ne_bytes(), libbpf_rs::MapFlags::ANY)? 24 | .expect("BPF_MAP_TYPE_PERCPU_ARRAY"); 25 | 26 | let read_stat = |idx| { 27 | all_cpus 28 | .iter() 29 | .map(|pcpu| { 30 | // pcpu comes in as unaligned u8s. stride over 8 of them for each stat index. 31 | let val = &pcpu[idx * 8..]; 32 | u64::from_ne_bytes(val.try_into().expect("all stats are u64s")) 33 | }) 34 | .sum() 35 | }; 36 | 37 | Ok(BpfStats { 38 | dropped_events: read_stat(STAT_DROPPED_EVENTS), 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tools/scxtop/src/cpu_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use crate::EventData; 7 | use crate::PerfEvent; 8 | 9 | use std::collections::VecDeque; 10 | 11 | /// Container for per CPU data. 12 | #[derive(Clone, Debug)] 13 | pub struct CpuData { 14 | pub llc: usize, 15 | pub node: usize, 16 | pub core: usize, 17 | pub cpu: usize, 18 | pub data: EventData, 19 | pub max_data_size: usize, 20 | } 21 | 22 | impl CpuData { 23 | /// Creates a new CpuData. 24 | pub fn new(cpu: usize, core: usize, llc: usize, node: usize, max_data_size: usize) -> CpuData { 25 | let mut data = EventData::new(max_data_size); 26 | for event in PerfEvent::default_events() { 27 | data.event_data(&event.event); 28 | } 29 | Self { 30 | llc, 31 | node, 32 | core, 33 | cpu, 34 | data, 35 | max_data_size, 36 | } 37 | } 38 | 39 | /// Initializes events with default values. 40 | pub fn initialize_events(&mut self, events: &[&str]) { 41 | self.data.initialize_events(events); 42 | } 43 | 44 | /// Returns the data for an event and updates if no entry is present. 45 | pub fn event_data(&mut self, event: &str) -> &VecDeque { 46 | self.data.event_data(event) 47 | } 48 | 49 | /// Returns the data for an event and updates if no entry is present. 50 | pub fn event_data_immut(&self, event: &str) -> Vec { 51 | self.data.event_data_immut(event) 52 | } 53 | 54 | /// Adds data for an event. 55 | pub fn add_event_data(&mut self, event: &str, val: u64) { 56 | self.data.add_event_data(event, val) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /tools/scxtop/src/llc_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use crate::EventData; 7 | use crate::PerfEvent; 8 | 9 | use std::collections::VecDeque; 10 | 11 | /// Container for per LLC data. 12 | #[derive(Clone, Debug)] 13 | pub struct LlcData { 14 | pub node: usize, 15 | pub llc: usize, 16 | pub data: EventData, 17 | pub max_data_size: usize, 18 | } 19 | 20 | impl LlcData { 21 | /// Creates a new NodeData. 22 | pub fn new(llc: usize, node: usize, max_data_size: usize) -> LlcData { 23 | let mut data = EventData::new(max_data_size); 24 | for event in PerfEvent::default_events() { 25 | data.event_data(&event.event); 26 | } 27 | 28 | Self { 29 | llc, 30 | node, 31 | data, 32 | max_data_size, 33 | } 34 | } 35 | 36 | /// Initializes events with default values. 37 | pub fn initialize_events(&mut self, events: &[&str]) { 38 | self.data.initialize_events(events); 39 | } 40 | 41 | /// Returns the data for an event and updates if no entry is present. 42 | pub fn event_data(&mut self, event: &str) -> &VecDeque { 43 | self.data.event_data(event) 44 | } 45 | 46 | /// Returns the data for an event and updates if no entry is present. 47 | pub fn event_data_immut(&self, event: &str) -> Vec { 48 | self.data.event_data_immut(event) 49 | } 50 | 51 | /// Adds data for an event. 52 | pub fn add_event_data(&mut self, event: &str, val: u64) { 53 | self.data.add_event_data(event, val); 54 | } 55 | 56 | /// Adds data for a cpu by updating the first value. 57 | pub fn add_cpu_event_data(&mut self, event: &str, val: u64) { 58 | let data = self.data.event_data_mut(event); 59 | let len = data.len(); 60 | data[len - 1] += val; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /tools/scxtop/src/node_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use crate::EventData; 7 | use crate::PerfEvent; 8 | 9 | use std::collections::VecDeque; 10 | 11 | /// Container for per NUMA node data. 12 | #[derive(Clone, Debug)] 13 | pub struct NodeData { 14 | pub node: usize, 15 | pub data: EventData, 16 | pub max_data_size: usize, 17 | } 18 | 19 | impl NodeData { 20 | /// Creates a new NodeData. 21 | pub fn new(node: usize, max_data_size: usize) -> NodeData { 22 | let mut data = EventData::new(max_data_size); 23 | for event in PerfEvent::default_events() { 24 | data.event_data(&event.event); 25 | } 26 | Self { 27 | node, 28 | data, 29 | max_data_size, 30 | } 31 | } 32 | 33 | /// Initializes events with default values. 34 | pub fn initialize_events(&mut self, events: &[&str]) { 35 | self.data.initialize_events(events); 36 | } 37 | 38 | /// Returns the data for an event and updates if no entry is present. 39 | pub fn event_data(&mut self, event: &str) -> &VecDeque { 40 | self.data.event_data(event) 41 | } 42 | 43 | /// Returns the data for an event and updates if no entry is present. 44 | pub fn event_data_immut(&self, event: &str) -> Vec { 45 | self.data.event_data_immut(event) 46 | } 47 | 48 | /// Adds data for an event. 49 | pub fn add_event_data(&mut self, event: &str, val: u64) { 50 | self.data.add_event_data(event, val) 51 | } 52 | 53 | /// Adds data for a cpu by updating the first value. 54 | pub fn add_cpu_event_data(&mut self, event: &str, val: u64) { 55 | let data = self.data.event_data_mut(event); 56 | let len = data.len(); 57 | data[len - 1] += val; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tools/scxtop/src/protos/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod protos_gen { 2 | // Auto generated code, ignore clippy warnings 3 | #[allow(clippy::all)] 4 | pub mod perfetto_scx { 5 | include!(concat!(env!("OUT_DIR"), "/perfetto.protos.rs")); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tools/scxtop/src/tracer.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use crate::bpf_skel::BpfSkel; 7 | use crate::bpf_stats::BpfStats; 8 | 9 | use anyhow::Result; 10 | use libbpf_rs::Link; 11 | use log::debug; 12 | 13 | pub struct Tracer<'a> { 14 | pub skel: BpfSkel<'a>, 15 | trace_links: Vec, 16 | } 17 | 18 | impl<'a> Tracer<'a> { 19 | /// Creates a new appliation. 20 | pub fn new(skel: BpfSkel<'a>) -> Self { 21 | let trace_links = vec![]; 22 | Self { skel, trace_links } 23 | } 24 | 25 | /// Returns the BPF stats for the tracer. 26 | pub fn stats(&self) -> Result { 27 | BpfStats::get_from_skel(&self.skel) 28 | } 29 | 30 | /// Attaches any BPF programs required for perfetto traces. 31 | fn attach_trace_progs(&mut self) -> Result<()> { 32 | self.trace_links = vec![ 33 | self.skel.progs.on_softirq_entry.attach()?, 34 | self.skel.progs.on_softirq_exit.attach()?, 35 | self.skel.progs.on_ipi_send_cpu.attach()?, 36 | ]; 37 | 38 | Ok(()) 39 | } 40 | 41 | /// Starts the collection of a trace, does not stop the trace. 42 | pub async fn trace_async(&mut self, dur: std::time::Duration) -> Result<()> { 43 | self.skel.maps.data_data.sample_rate = 1; 44 | self.skel.maps.data_data.enable_bpf_events = true; 45 | self.attach_trace_progs()?; 46 | debug!( 47 | "attached {} trace progs, sample_rate: {}", 48 | self.trace_links.len(), 49 | self.skel.maps.data_data.sample_rate 50 | ); 51 | tokio::time::sleep(dur).await; 52 | self.trace_links.clear(); 53 | Ok(()) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /tools/scxtop/src/util.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This software may be used and distributed according to the terms of the 4 | // GNU General Public License version 2. 5 | 6 | use anyhow::Result; 7 | use std::fs; 8 | use std::io::Read; 9 | 10 | /// Returns the file content as a String. 11 | pub fn read_file_string(path: &str) -> Result { 12 | let mut file = fs::File::open(path)?; 13 | let mut contents = String::new(); 14 | file.read_to_string(&mut contents)?; 15 | Ok(contents) 16 | } 17 | 18 | /// Formats a value in hz to human readable. 19 | pub fn format_hz(hz: u64) -> String { 20 | match hz { 21 | 0..=999 => format!("{}Hz", hz), 22 | 1_000..=999_999 => format!("{:.0}MHz", hz as f64 / 1_000.0), 23 | 1_000_000..=999_999_999 => format!("{:.3}GHz", hz as f64 / 1_000_000.0), 24 | _ => format!("{:.3}THz", hz as f64 / 1_000_000_000.0), 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tools/vmlinux_docify/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "vmlinux_docify" 3 | version = "0.1.1" 4 | edition = "2021" 5 | authors = ["Pat S "] 6 | description = "A tool to annotate vmlinux.h with documentation from kernel sources" 7 | 8 | [dependencies] 9 | clap = { version = "4.5.28", features = ["derive"] } 10 | walkdir = "2.5" 11 | -------------------------------------------------------------------------------- /tools/vmlinux_docify/README.md: -------------------------------------------------------------------------------- 1 | # vmlinux_docify 2 | 3 | A simple tool for annotating vmlinux.h with documentation from kernel sources. 4 | 5 | To help bridge the tooling gap between writing c code for the bpf vm vs c code for err, everything else. 6 | 7 | ``` 8 | Usage: vmlinux_docify [OPTIONS] --kernel-dir --vmlinux-h 9 | 10 | Options: 11 | -k, --kernel-dir 12 | Path to the kernel source directory 13 | 14 | -v, --vmlinux-h 15 | Path to the vmlinux.h file to annotate 16 | 17 | -o, --output 18 | Path to the output file (default: vmlinux_annotated.h) 19 | 20 | [default: vmlinux_annotated.h] 21 | 22 | -h, --help 23 | Print help (see a summary with '-h') 24 | 25 | -V, --version 26 | Print version 27 | ``` 28 | 29 | --------------------------------------------------------------------------------