├── .dockerignore ├── .gitattibutes ├── .gitignore ├── .pre-commit-config.yaml ├── .prettierignore ├── .prettierrc ├── Dockerfile ├── Gemfile ├── LICENSE ├── README.md ├── _config.yml ├── _includes ├── audio.liquid ├── bib_search.liquid ├── citation.liquid ├── cv │ ├── list.liquid │ ├── list_groups.liquid │ ├── map.liquid │ ├── nested_list.liquid │ └── time_table.liquid ├── disqus.liquid ├── distill-citation.liquid ├── distill_scripts.liquid ├── figure.liquid ├── footer.liquid ├── giscus.liquid ├── head.liquid ├── header.liquid ├── latest_posts.liquid ├── metadata.liquid ├── miscellaneous.liquid ├── news.liquid ├── newsletter.liquid ├── pagination.liquid ├── projects.liquid ├── projects_horizontal.liquid ├── related_posts.liquid ├── repository │ ├── repo.liquid │ ├── repo_trophies.liquid │ └── repo_user.liquid ├── resume │ ├── awards.liquid │ ├── basics.liquid │ ├── certificates.liquid │ ├── education.liquid │ ├── interests.liquid │ ├── languages.liquid │ ├── projects.liquid │ ├── publications.liquid │ ├── references.liquid │ ├── skills.liquid │ ├── volunteer.liquid │ └── work.liquid ├── scripts.liquid ├── selected_papers.liquid ├── social.liquid └── video.liquid ├── _layouts ├── about.liquid ├── archive-category.liquid ├── archive-tag.liquid ├── archive-year.liquid ├── bib.liquid ├── cv.liquid ├── default.liquid ├── distill.liquid ├── none.liquid ├── page.liquid ├── post.liquid └── profiles.liquid ├── _plugins ├── cache-bust.rb ├── details.rb ├── download-3rd-party.rb ├── external-posts.rb ├── file-exists.rb ├── google-scholar-citations.rb ├── hide-custom-bibtex.rb ├── inspirehep-citations.rb └── remove-accents.rb ├── _sass ├── _base.scss ├── _cv.scss ├── _distill.scss ├── _layout.scss ├── _tabs.scss ├── _themes.scss ├── _typograms.scss ├── _variables.scss ├── font-awesome │ ├── _animated.scss │ ├── _bordered-pulled.scss │ ├── _core.scss │ ├── _fixed-width.scss │ ├── _functions.scss │ ├── _icons.scss │ ├── _list.scss │ ├── _mixins.scss │ ├── _rotated-flipped.scss │ ├── _screen-reader.scss │ ├── _shims.scss │ ├── _sizing.scss │ ├── _stacked.scss │ ├── _variables.scss │ ├── brands.scss │ ├── fontawesome.scss │ ├── regular.scss │ ├── solid.scss │ └── v4-shims.scss └── tabler-icons │ ├── tabler-icons-filled.scss │ ├── tabler-icons-outline.scss │ └── tabler-icons.scss ├── _scripts └── search.liquid.js ├── applied-inference.md ├── applied-training.md ├── assets ├── bibliography │ └── main.bib ├── css │ ├── academicons.min.css │ ├── bootstrap-toc.min.css │ ├── bootstrap.min.css │ ├── bootstrap.min.css.map │ ├── jekyll-pygments-themes-github.css │ ├── jekyll-pygments-themes-native.css │ ├── jekyll-pygments-themes-vs.css │ ├── jupyter-grade3.css │ ├── jupyter-monokai.css │ ├── jupyter.css │ ├── main.scss │ ├── mdb.min.css │ ├── mdb.min.css.map │ └── scholar-icons.css ├── fonts │ ├── academicons.ttf │ ├── academicons.woff │ ├── scholar-icons.ttf │ ├── scholar-icons.woff │ ├── tabler-icons-filled.ttf │ ├── tabler-icons-filled.woff │ ├── tabler-icons-filled.woff2 │ ├── tabler-icons-outline.ttf │ ├── tabler-icons-outline.woff │ ├── tabler-icons-outline.woff2 │ ├── tabler-icons.ttf │ ├── tabler-icons.woff │ └── tabler-icons.woff2 ├── gpu │ ├── all-gather.gif │ ├── all-to-all-approx.png │ ├── b200-node.png │ ├── b200-superpod.png │ ├── blackwell-sm.png │ ├── gb200-superpod.png │ ├── gpu-all-gather-bw.png │ ├── gpu-all-reduce-bw.png │ ├── gpu-diagram.png │ ├── h100-superpod.png │ ├── megatron-hparams.png │ ├── nvl-576.png │ ├── nvlink-nodes.png │ ├── nvlink4.png │ ├── pipeline-bubble.png │ ├── pipeline-transfer.png │ ├── sharded-all-gather-cost.png │ ├── sharp-algorithm.png │ ├── sharp-all-reduce-cost.png │ ├── superpod-diagram.png │ ├── tpu-all-gather-bw.png │ ├── tpu-all-reduce-bw.png │ └── warp-divergence.png ├── img │ ├── 2d-weight-stationary.png │ ├── all-collectives.png │ ├── all-gather-bandwidth.png │ ├── all-gather.gif │ ├── all-reduce-profile.png │ ├── all-to-all.gif │ ├── attn-xprof.png │ ├── b100-sm-diagram.png │ ├── batch-scaling-latency.png │ ├── batch-scaling-throughput.png │ ├── batched-prefill.png │ ├── button.png │ ├── cached-inference.png │ ├── challenge-problem.png │ ├── comms-flops-time.png │ ├── continuous-batching.gif │ ├── cores.png │ ├── data-parallelism-alt.png │ ├── data-parallelism.png │ ├── decode-stages.png │ ├── deepseek-pipeline.png │ ├── disaggregation.png │ ├── dragon.png │ ├── esta-figure.png │ ├── favicon.ico │ ├── ffw-profile.png │ ├── flash-algo.png │ ├── flops-comms-summary.png │ ├── fsdp-figure.png │ ├── fsdp.png │ ├── gmqa-sharding.png │ ├── gmqa.png │ ├── graph-viewer.png │ ├── ici-wraparound.png │ ├── interleaving.png │ ├── kv-sharing.png │ ├── latency-cost.png │ ├── llama-json.png │ ├── math-comms-time.png │ ├── matmul-flops.gif │ ├── memory-viewer.png │ ├── mixed-fsdp-comms-2.png │ ├── mixed-fsdp-model-parallelism.png │ ├── model-parallelism.png │ ├── moe.png │ ├── more-subslices.png │ ├── naive-inference.png │ ├── not-overlapped.png │ ├── overlapped.png │ ├── paged-attention.png │ ├── pcie.png │ ├── pointwise-product.gif │ ├── prefix-caching-trie.png │ ├── reduce-scatter-graph.png │ ├── reduce-scatter-xprof.png │ ├── reduce-scatter.gif │ ├── roofline-improved.png │ ├── roofline-plot-q3.png │ ├── sharding-colored1.png │ ├── sharding-colored2.png │ ├── sharding-colored3.png │ ├── sharding-colored4.png │ ├── sharding-colored5.png │ ├── sharding-colored6.png │ ├── sharding-example.png │ ├── spec-sampling1.png │ ├── spec-sampling2.png │ ├── spec-sampling3.png │ ├── subslices.png │ ├── systolic-array-bad.gif │ ├── systolic-array-pipelining.png │ ├── systolic-array.gif │ ├── systolic-array2.gif │ ├── tiling.png │ ├── tiling2.png │ ├── tpu-bandwidth.png │ ├── tpu-chip.png │ ├── tpu-rack.png │ ├── trace-viewer.png │ ├── transformer-diagram.png │ ├── transformer-ffw.png │ ├── transformer-layer.png │ ├── transformer-xprof.png │ ├── twotrees.jpg │ ├── xprof-attention.png │ ├── xprof-overview.png │ ├── xprof-problem1.png │ ├── xprof-problem2.png │ ├── xprof1.png │ ├── xprof2.png │ ├── xprof3.png │ ├── xprof4.png │ └── zoomed-in.png ├── js │ ├── bibsearch.js │ ├── bootstrap-toc.min.js │ ├── bootstrap.bundle.min.js │ ├── bootstrap.bundle.min.js.map │ ├── chartjs-setup.js │ ├── common.js │ ├── copy_code.js │ ├── cronitor-analytics-setup.js │ ├── diff2html-setup.js │ ├── distillpub │ │ ├── template.v2.js │ │ ├── template.v2.js.map │ │ ├── transforms.v2.js │ │ └── transforms.v2.js.map │ ├── echarts-setup.js │ ├── google-analytics-setup.js │ ├── highlight-search-term.js │ ├── jupyter_new_tab.js │ ├── leaflet-setup.js │ ├── masonry.js │ ├── mathjax-setup.js │ ├── mermaid-setup.js │ ├── newsletter.js │ ├── no_defer.js │ ├── open-panel-analytics-setup.js │ ├── photoswipe-setup.js │ ├── progress-bar.js │ ├── pseudocode-setup.js │ ├── search-setup.js │ ├── search │ │ ├── base-styles.min.js │ │ ├── command-score.min.js │ │ ├── hotkeys-js │ │ │ └── hotkeys.esm.min.js │ │ ├── lit-element │ │ │ └── lit-element.min.js │ │ ├── lit-html │ │ │ ├── directives │ │ │ │ ├── class-map.min.js │ │ │ │ ├── join.min.js │ │ │ │ ├── live.min.js │ │ │ │ ├── ref.min.js │ │ │ │ ├── repeat.min.js │ │ │ │ └── unsafe-html.min.js │ │ │ ├── is-server.min.js │ │ │ └── lit-html.min.js │ │ ├── lit │ │ │ ├── decorators.min.js │ │ │ ├── directives │ │ │ │ ├── class-map.min.js │ │ │ │ ├── join.min.js │ │ │ │ ├── live.min.js │ │ │ │ ├── ref.min.js │ │ │ │ ├── repeat.min.js │ │ │ │ └── unsafe-html.min.js │ │ │ └── index.min.js │ │ ├── ninja-action.min.js │ │ ├── ninja-footer.min.js │ │ ├── ninja-header.min.js │ │ ├── ninja-keys.min.js │ │ └── reactive-element │ │ │ ├── decorators │ │ │ ├── custom-element.min.js │ │ │ ├── event-options.min.js │ │ │ ├── property.min.js │ │ │ ├── query-all.min.js │ │ │ ├── query-assigned-elements.min.js │ │ │ ├── query-assigned-nodes.min.js │ │ │ ├── query-async.min.js │ │ │ ├── query.min.js │ │ │ └── state.min.js │ │ │ └── reactive-element.min.js │ ├── shortcut-key.js │ ├── tabs.min.js │ ├── theme.js │ ├── tooltips-setup.js │ ├── typograms.js │ ├── vanilla-back-to-top.min.js │ ├── vega-setup.js │ ├── venobox-setup.js │ ├── wechat.js │ └── zoom.js ├── plotly │ ├── fsdp-time.html │ ├── latency_breakdown.html │ ├── latency_breakdown_log.html │ ├── latency_throughput.html │ ├── pareto-2k.html │ ├── pareto-8k.html │ ├── pareto.html │ └── training-roofline.html └── webfonts │ ├── fa-brands-400.ttf │ ├── fa-brands-400.woff2 │ ├── fa-regular-400.ttf │ ├── fa-regular-400.woff2 │ ├── fa-solid-900.ttf │ ├── fa-solid-900.woff2 │ ├── fa-v4compatibility.ttf │ └── fa-v4compatibility.woff2 ├── bin ├── deploy └── entry_point.sh ├── conclusion.md ├── docker-compose-slim.yml ├── docker-compose.yml ├── dropdown.md ├── gpus.md ├── index.md ├── inference.md ├── jax-stuff.md ├── package.json ├── profiling.md ├── purgecss.config.js ├── requirements.txt ├── robots.txt ├── roofline.md ├── sharding.md ├── tpus.md ├── training.md └── transformers.md /.dockerignore: -------------------------------------------------------------------------------- 1 | _site/ 2 | .git/ 3 | assets/ 4 | -------------------------------------------------------------------------------- /.gitattibutes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/.gitattibutes -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/.prettierignore -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/.prettierrc -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/Dockerfile -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/Gemfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/README.md -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_config.yml -------------------------------------------------------------------------------- /_includes/audio.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/audio.liquid -------------------------------------------------------------------------------- /_includes/bib_search.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/bib_search.liquid -------------------------------------------------------------------------------- /_includes/citation.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/citation.liquid -------------------------------------------------------------------------------- /_includes/cv/list.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/cv/list.liquid -------------------------------------------------------------------------------- /_includes/cv/list_groups.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/cv/list_groups.liquid -------------------------------------------------------------------------------- /_includes/cv/map.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/cv/map.liquid -------------------------------------------------------------------------------- /_includes/cv/nested_list.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/cv/nested_list.liquid -------------------------------------------------------------------------------- /_includes/cv/time_table.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/cv/time_table.liquid -------------------------------------------------------------------------------- /_includes/disqus.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/disqus.liquid -------------------------------------------------------------------------------- /_includes/distill-citation.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/distill-citation.liquid -------------------------------------------------------------------------------- /_includes/distill_scripts.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/distill_scripts.liquid -------------------------------------------------------------------------------- /_includes/figure.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/figure.liquid -------------------------------------------------------------------------------- /_includes/footer.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/footer.liquid -------------------------------------------------------------------------------- /_includes/giscus.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/giscus.liquid -------------------------------------------------------------------------------- /_includes/head.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/head.liquid -------------------------------------------------------------------------------- /_includes/header.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/header.liquid -------------------------------------------------------------------------------- /_includes/latest_posts.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/latest_posts.liquid -------------------------------------------------------------------------------- /_includes/metadata.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/metadata.liquid -------------------------------------------------------------------------------- /_includes/miscellaneous.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/miscellaneous.liquid -------------------------------------------------------------------------------- /_includes/news.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/news.liquid -------------------------------------------------------------------------------- /_includes/newsletter.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/newsletter.liquid -------------------------------------------------------------------------------- /_includes/pagination.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/pagination.liquid -------------------------------------------------------------------------------- /_includes/projects.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/projects.liquid -------------------------------------------------------------------------------- /_includes/projects_horizontal.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/projects_horizontal.liquid -------------------------------------------------------------------------------- /_includes/related_posts.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/related_posts.liquid -------------------------------------------------------------------------------- /_includes/repository/repo.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/repository/repo.liquid -------------------------------------------------------------------------------- /_includes/repository/repo_trophies.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/repository/repo_trophies.liquid -------------------------------------------------------------------------------- /_includes/repository/repo_user.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/repository/repo_user.liquid -------------------------------------------------------------------------------- /_includes/resume/awards.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/awards.liquid -------------------------------------------------------------------------------- /_includes/resume/basics.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/basics.liquid -------------------------------------------------------------------------------- /_includes/resume/certificates.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/certificates.liquid -------------------------------------------------------------------------------- /_includes/resume/education.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/education.liquid -------------------------------------------------------------------------------- /_includes/resume/interests.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/interests.liquid -------------------------------------------------------------------------------- /_includes/resume/languages.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/languages.liquid -------------------------------------------------------------------------------- /_includes/resume/projects.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/projects.liquid -------------------------------------------------------------------------------- /_includes/resume/publications.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/publications.liquid -------------------------------------------------------------------------------- /_includes/resume/references.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/references.liquid -------------------------------------------------------------------------------- /_includes/resume/skills.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/skills.liquid -------------------------------------------------------------------------------- /_includes/resume/volunteer.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/volunteer.liquid -------------------------------------------------------------------------------- /_includes/resume/work.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/resume/work.liquid -------------------------------------------------------------------------------- /_includes/scripts.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/scripts.liquid -------------------------------------------------------------------------------- /_includes/selected_papers.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/selected_papers.liquid -------------------------------------------------------------------------------- /_includes/social.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/social.liquid -------------------------------------------------------------------------------- /_includes/video.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_includes/video.liquid -------------------------------------------------------------------------------- /_layouts/about.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/about.liquid -------------------------------------------------------------------------------- /_layouts/archive-category.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/archive-category.liquid -------------------------------------------------------------------------------- /_layouts/archive-tag.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/archive-tag.liquid -------------------------------------------------------------------------------- /_layouts/archive-year.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/archive-year.liquid -------------------------------------------------------------------------------- /_layouts/bib.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/bib.liquid -------------------------------------------------------------------------------- /_layouts/cv.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/cv.liquid -------------------------------------------------------------------------------- /_layouts/default.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/default.liquid -------------------------------------------------------------------------------- /_layouts/distill.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/distill.liquid -------------------------------------------------------------------------------- /_layouts/none.liquid: -------------------------------------------------------------------------------- 1 | {{ content }} 2 | -------------------------------------------------------------------------------- /_layouts/page.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/page.liquid -------------------------------------------------------------------------------- /_layouts/post.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/post.liquid -------------------------------------------------------------------------------- /_layouts/profiles.liquid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_layouts/profiles.liquid -------------------------------------------------------------------------------- /_plugins/cache-bust.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/cache-bust.rb -------------------------------------------------------------------------------- /_plugins/details.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/details.rb -------------------------------------------------------------------------------- /_plugins/download-3rd-party.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/download-3rd-party.rb -------------------------------------------------------------------------------- /_plugins/external-posts.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/external-posts.rb -------------------------------------------------------------------------------- /_plugins/file-exists.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/file-exists.rb -------------------------------------------------------------------------------- /_plugins/google-scholar-citations.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/google-scholar-citations.rb -------------------------------------------------------------------------------- /_plugins/hide-custom-bibtex.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/hide-custom-bibtex.rb -------------------------------------------------------------------------------- /_plugins/inspirehep-citations.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/inspirehep-citations.rb -------------------------------------------------------------------------------- /_plugins/remove-accents.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_plugins/remove-accents.rb -------------------------------------------------------------------------------- /_sass/_base.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_base.scss -------------------------------------------------------------------------------- /_sass/_cv.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_cv.scss -------------------------------------------------------------------------------- /_sass/_distill.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_distill.scss -------------------------------------------------------------------------------- /_sass/_layout.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_layout.scss -------------------------------------------------------------------------------- /_sass/_tabs.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_tabs.scss -------------------------------------------------------------------------------- /_sass/_themes.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_themes.scss -------------------------------------------------------------------------------- /_sass/_typograms.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_typograms.scss -------------------------------------------------------------------------------- /_sass/_variables.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/_variables.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_animated.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_animated.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_bordered-pulled.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_bordered-pulled.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_core.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_core.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_fixed-width.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_fixed-width.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_functions.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_functions.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_icons.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_icons.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_list.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_list.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_mixins.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_mixins.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_rotated-flipped.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_rotated-flipped.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_screen-reader.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_screen-reader.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_shims.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_shims.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_sizing.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_sizing.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_stacked.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_stacked.scss -------------------------------------------------------------------------------- /_sass/font-awesome/_variables.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/_variables.scss -------------------------------------------------------------------------------- /_sass/font-awesome/brands.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/brands.scss -------------------------------------------------------------------------------- /_sass/font-awesome/fontawesome.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/fontawesome.scss -------------------------------------------------------------------------------- /_sass/font-awesome/regular.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/regular.scss -------------------------------------------------------------------------------- /_sass/font-awesome/solid.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/solid.scss -------------------------------------------------------------------------------- /_sass/font-awesome/v4-shims.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/font-awesome/v4-shims.scss -------------------------------------------------------------------------------- /_sass/tabler-icons/tabler-icons-filled.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/tabler-icons/tabler-icons-filled.scss -------------------------------------------------------------------------------- /_sass/tabler-icons/tabler-icons-outline.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/tabler-icons/tabler-icons-outline.scss -------------------------------------------------------------------------------- /_sass/tabler-icons/tabler-icons.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_sass/tabler-icons/tabler-icons.scss -------------------------------------------------------------------------------- /_scripts/search.liquid.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/_scripts/search.liquid.js -------------------------------------------------------------------------------- /applied-inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/applied-inference.md -------------------------------------------------------------------------------- /applied-training.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/applied-training.md -------------------------------------------------------------------------------- /assets/bibliography/main.bib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/bibliography/main.bib -------------------------------------------------------------------------------- /assets/css/academicons.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/academicons.min.css -------------------------------------------------------------------------------- /assets/css/bootstrap-toc.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/bootstrap-toc.min.css -------------------------------------------------------------------------------- /assets/css/bootstrap.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/bootstrap.min.css -------------------------------------------------------------------------------- /assets/css/bootstrap.min.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/bootstrap.min.css.map -------------------------------------------------------------------------------- /assets/css/jekyll-pygments-themes-github.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jekyll-pygments-themes-github.css -------------------------------------------------------------------------------- /assets/css/jekyll-pygments-themes-native.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jekyll-pygments-themes-native.css -------------------------------------------------------------------------------- /assets/css/jekyll-pygments-themes-vs.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jekyll-pygments-themes-vs.css -------------------------------------------------------------------------------- /assets/css/jupyter-grade3.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jupyter-grade3.css -------------------------------------------------------------------------------- /assets/css/jupyter-monokai.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jupyter-monokai.css -------------------------------------------------------------------------------- /assets/css/jupyter.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/jupyter.css -------------------------------------------------------------------------------- /assets/css/main.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/main.scss -------------------------------------------------------------------------------- /assets/css/mdb.min.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/mdb.min.css -------------------------------------------------------------------------------- /assets/css/mdb.min.css.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/mdb.min.css.map -------------------------------------------------------------------------------- /assets/css/scholar-icons.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/css/scholar-icons.css -------------------------------------------------------------------------------- /assets/fonts/academicons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/academicons.ttf -------------------------------------------------------------------------------- /assets/fonts/academicons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/academicons.woff -------------------------------------------------------------------------------- /assets/fonts/scholar-icons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/scholar-icons.ttf -------------------------------------------------------------------------------- /assets/fonts/scholar-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/scholar-icons.woff -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-filled.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-filled.ttf -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-filled.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-filled.woff -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-filled.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-filled.woff2 -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-outline.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-outline.ttf -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-outline.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-outline.woff -------------------------------------------------------------------------------- /assets/fonts/tabler-icons-outline.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons-outline.woff2 -------------------------------------------------------------------------------- /assets/fonts/tabler-icons.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons.ttf -------------------------------------------------------------------------------- /assets/fonts/tabler-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons.woff -------------------------------------------------------------------------------- /assets/fonts/tabler-icons.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/fonts/tabler-icons.woff2 -------------------------------------------------------------------------------- /assets/gpu/all-gather.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/all-gather.gif -------------------------------------------------------------------------------- /assets/gpu/all-to-all-approx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/all-to-all-approx.png -------------------------------------------------------------------------------- /assets/gpu/b200-node.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/b200-node.png -------------------------------------------------------------------------------- /assets/gpu/b200-superpod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/b200-superpod.png -------------------------------------------------------------------------------- /assets/gpu/blackwell-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/blackwell-sm.png -------------------------------------------------------------------------------- /assets/gpu/gb200-superpod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/gb200-superpod.png -------------------------------------------------------------------------------- /assets/gpu/gpu-all-gather-bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/gpu-all-gather-bw.png -------------------------------------------------------------------------------- /assets/gpu/gpu-all-reduce-bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/gpu-all-reduce-bw.png -------------------------------------------------------------------------------- /assets/gpu/gpu-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/gpu-diagram.png -------------------------------------------------------------------------------- /assets/gpu/h100-superpod.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/h100-superpod.png -------------------------------------------------------------------------------- /assets/gpu/megatron-hparams.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/megatron-hparams.png -------------------------------------------------------------------------------- /assets/gpu/nvl-576.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/nvl-576.png -------------------------------------------------------------------------------- /assets/gpu/nvlink-nodes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/nvlink-nodes.png -------------------------------------------------------------------------------- /assets/gpu/nvlink4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/nvlink4.png -------------------------------------------------------------------------------- /assets/gpu/pipeline-bubble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/pipeline-bubble.png -------------------------------------------------------------------------------- /assets/gpu/pipeline-transfer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/pipeline-transfer.png -------------------------------------------------------------------------------- /assets/gpu/sharded-all-gather-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/sharded-all-gather-cost.png -------------------------------------------------------------------------------- /assets/gpu/sharp-algorithm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/sharp-algorithm.png -------------------------------------------------------------------------------- /assets/gpu/sharp-all-reduce-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/sharp-all-reduce-cost.png -------------------------------------------------------------------------------- /assets/gpu/superpod-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/superpod-diagram.png -------------------------------------------------------------------------------- /assets/gpu/tpu-all-gather-bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/tpu-all-gather-bw.png -------------------------------------------------------------------------------- /assets/gpu/tpu-all-reduce-bw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/tpu-all-reduce-bw.png -------------------------------------------------------------------------------- /assets/gpu/warp-divergence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/gpu/warp-divergence.png -------------------------------------------------------------------------------- /assets/img/2d-weight-stationary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/2d-weight-stationary.png -------------------------------------------------------------------------------- /assets/img/all-collectives.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/all-collectives.png -------------------------------------------------------------------------------- /assets/img/all-gather-bandwidth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/all-gather-bandwidth.png -------------------------------------------------------------------------------- /assets/img/all-gather.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/all-gather.gif -------------------------------------------------------------------------------- /assets/img/all-reduce-profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/all-reduce-profile.png -------------------------------------------------------------------------------- /assets/img/all-to-all.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/all-to-all.gif -------------------------------------------------------------------------------- /assets/img/attn-xprof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/attn-xprof.png -------------------------------------------------------------------------------- /assets/img/b100-sm-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/b100-sm-diagram.png -------------------------------------------------------------------------------- /assets/img/batch-scaling-latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/batch-scaling-latency.png -------------------------------------------------------------------------------- /assets/img/batch-scaling-throughput.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/batch-scaling-throughput.png -------------------------------------------------------------------------------- /assets/img/batched-prefill.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/batched-prefill.png -------------------------------------------------------------------------------- /assets/img/button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/button.png -------------------------------------------------------------------------------- /assets/img/cached-inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/cached-inference.png -------------------------------------------------------------------------------- /assets/img/challenge-problem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/challenge-problem.png -------------------------------------------------------------------------------- /assets/img/comms-flops-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/comms-flops-time.png -------------------------------------------------------------------------------- /assets/img/continuous-batching.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/continuous-batching.gif -------------------------------------------------------------------------------- /assets/img/cores.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/cores.png -------------------------------------------------------------------------------- /assets/img/data-parallelism-alt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/data-parallelism-alt.png -------------------------------------------------------------------------------- /assets/img/data-parallelism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/data-parallelism.png -------------------------------------------------------------------------------- /assets/img/decode-stages.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/decode-stages.png -------------------------------------------------------------------------------- /assets/img/deepseek-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/deepseek-pipeline.png -------------------------------------------------------------------------------- /assets/img/disaggregation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/disaggregation.png -------------------------------------------------------------------------------- /assets/img/dragon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/dragon.png -------------------------------------------------------------------------------- /assets/img/esta-figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/esta-figure.png -------------------------------------------------------------------------------- /assets/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/favicon.ico -------------------------------------------------------------------------------- /assets/img/ffw-profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/ffw-profile.png -------------------------------------------------------------------------------- /assets/img/flash-algo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/flash-algo.png -------------------------------------------------------------------------------- /assets/img/flops-comms-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/flops-comms-summary.png -------------------------------------------------------------------------------- /assets/img/fsdp-figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/fsdp-figure.png -------------------------------------------------------------------------------- /assets/img/fsdp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/fsdp.png -------------------------------------------------------------------------------- /assets/img/gmqa-sharding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/gmqa-sharding.png -------------------------------------------------------------------------------- /assets/img/gmqa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/gmqa.png -------------------------------------------------------------------------------- /assets/img/graph-viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/graph-viewer.png -------------------------------------------------------------------------------- /assets/img/ici-wraparound.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/ici-wraparound.png -------------------------------------------------------------------------------- /assets/img/interleaving.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/interleaving.png -------------------------------------------------------------------------------- /assets/img/kv-sharing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/kv-sharing.png -------------------------------------------------------------------------------- /assets/img/latency-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/latency-cost.png -------------------------------------------------------------------------------- /assets/img/llama-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/llama-json.png -------------------------------------------------------------------------------- /assets/img/math-comms-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/math-comms-time.png -------------------------------------------------------------------------------- /assets/img/matmul-flops.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/matmul-flops.gif -------------------------------------------------------------------------------- /assets/img/memory-viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/memory-viewer.png -------------------------------------------------------------------------------- /assets/img/mixed-fsdp-comms-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/mixed-fsdp-comms-2.png -------------------------------------------------------------------------------- /assets/img/mixed-fsdp-model-parallelism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/mixed-fsdp-model-parallelism.png -------------------------------------------------------------------------------- /assets/img/model-parallelism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/model-parallelism.png -------------------------------------------------------------------------------- /assets/img/moe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/moe.png -------------------------------------------------------------------------------- /assets/img/more-subslices.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/more-subslices.png -------------------------------------------------------------------------------- /assets/img/naive-inference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/naive-inference.png -------------------------------------------------------------------------------- /assets/img/not-overlapped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/not-overlapped.png -------------------------------------------------------------------------------- /assets/img/overlapped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/overlapped.png -------------------------------------------------------------------------------- /assets/img/paged-attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/paged-attention.png -------------------------------------------------------------------------------- /assets/img/pcie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/pcie.png -------------------------------------------------------------------------------- /assets/img/pointwise-product.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/pointwise-product.gif -------------------------------------------------------------------------------- /assets/img/prefix-caching-trie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/prefix-caching-trie.png -------------------------------------------------------------------------------- /assets/img/reduce-scatter-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/reduce-scatter-graph.png -------------------------------------------------------------------------------- /assets/img/reduce-scatter-xprof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/reduce-scatter-xprof.png -------------------------------------------------------------------------------- /assets/img/reduce-scatter.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/reduce-scatter.gif -------------------------------------------------------------------------------- /assets/img/roofline-improved.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/roofline-improved.png -------------------------------------------------------------------------------- /assets/img/roofline-plot-q3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/roofline-plot-q3.png -------------------------------------------------------------------------------- /assets/img/sharding-colored1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored1.png -------------------------------------------------------------------------------- /assets/img/sharding-colored2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored2.png -------------------------------------------------------------------------------- /assets/img/sharding-colored3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored3.png -------------------------------------------------------------------------------- /assets/img/sharding-colored4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored4.png -------------------------------------------------------------------------------- /assets/img/sharding-colored5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored5.png -------------------------------------------------------------------------------- /assets/img/sharding-colored6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-colored6.png -------------------------------------------------------------------------------- /assets/img/sharding-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/sharding-example.png -------------------------------------------------------------------------------- /assets/img/spec-sampling1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/spec-sampling1.png -------------------------------------------------------------------------------- /assets/img/spec-sampling2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/spec-sampling2.png -------------------------------------------------------------------------------- /assets/img/spec-sampling3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/spec-sampling3.png -------------------------------------------------------------------------------- /assets/img/subslices.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/subslices.png -------------------------------------------------------------------------------- /assets/img/systolic-array-bad.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/systolic-array-bad.gif -------------------------------------------------------------------------------- /assets/img/systolic-array-pipelining.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/systolic-array-pipelining.png -------------------------------------------------------------------------------- /assets/img/systolic-array.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/systolic-array.gif -------------------------------------------------------------------------------- /assets/img/systolic-array2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/systolic-array2.gif -------------------------------------------------------------------------------- /assets/img/tiling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/tiling.png -------------------------------------------------------------------------------- /assets/img/tiling2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/tiling2.png -------------------------------------------------------------------------------- /assets/img/tpu-bandwidth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/tpu-bandwidth.png -------------------------------------------------------------------------------- /assets/img/tpu-chip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/tpu-chip.png -------------------------------------------------------------------------------- /assets/img/tpu-rack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/tpu-rack.png -------------------------------------------------------------------------------- /assets/img/trace-viewer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/trace-viewer.png -------------------------------------------------------------------------------- /assets/img/transformer-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/transformer-diagram.png -------------------------------------------------------------------------------- /assets/img/transformer-ffw.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/transformer-ffw.png -------------------------------------------------------------------------------- /assets/img/transformer-layer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/transformer-layer.png -------------------------------------------------------------------------------- /assets/img/transformer-xprof.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/transformer-xprof.png -------------------------------------------------------------------------------- /assets/img/twotrees.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/twotrees.jpg -------------------------------------------------------------------------------- /assets/img/xprof-attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof-attention.png -------------------------------------------------------------------------------- /assets/img/xprof-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof-overview.png -------------------------------------------------------------------------------- /assets/img/xprof-problem1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof-problem1.png -------------------------------------------------------------------------------- /assets/img/xprof-problem2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof-problem2.png -------------------------------------------------------------------------------- /assets/img/xprof1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof1.png -------------------------------------------------------------------------------- /assets/img/xprof2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof2.png -------------------------------------------------------------------------------- /assets/img/xprof3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof3.png -------------------------------------------------------------------------------- /assets/img/xprof4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/xprof4.png -------------------------------------------------------------------------------- /assets/img/zoomed-in.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/img/zoomed-in.png -------------------------------------------------------------------------------- /assets/js/bibsearch.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/bibsearch.js -------------------------------------------------------------------------------- /assets/js/bootstrap-toc.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/bootstrap-toc.min.js -------------------------------------------------------------------------------- /assets/js/bootstrap.bundle.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/bootstrap.bundle.min.js -------------------------------------------------------------------------------- /assets/js/bootstrap.bundle.min.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/bootstrap.bundle.min.js.map -------------------------------------------------------------------------------- /assets/js/chartjs-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/chartjs-setup.js -------------------------------------------------------------------------------- /assets/js/common.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/common.js -------------------------------------------------------------------------------- /assets/js/copy_code.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/copy_code.js -------------------------------------------------------------------------------- /assets/js/cronitor-analytics-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/cronitor-analytics-setup.js -------------------------------------------------------------------------------- /assets/js/diff2html-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/diff2html-setup.js -------------------------------------------------------------------------------- /assets/js/distillpub/template.v2.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/distillpub/template.v2.js -------------------------------------------------------------------------------- /assets/js/distillpub/template.v2.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/distillpub/template.v2.js.map -------------------------------------------------------------------------------- /assets/js/distillpub/transforms.v2.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/distillpub/transforms.v2.js -------------------------------------------------------------------------------- /assets/js/distillpub/transforms.v2.js.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/distillpub/transforms.v2.js.map -------------------------------------------------------------------------------- /assets/js/echarts-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/echarts-setup.js -------------------------------------------------------------------------------- /assets/js/google-analytics-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/google-analytics-setup.js -------------------------------------------------------------------------------- /assets/js/highlight-search-term.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/highlight-search-term.js -------------------------------------------------------------------------------- /assets/js/jupyter_new_tab.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/jupyter_new_tab.js -------------------------------------------------------------------------------- /assets/js/leaflet-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/leaflet-setup.js -------------------------------------------------------------------------------- /assets/js/masonry.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/masonry.js -------------------------------------------------------------------------------- /assets/js/mathjax-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/mathjax-setup.js -------------------------------------------------------------------------------- /assets/js/mermaid-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/mermaid-setup.js -------------------------------------------------------------------------------- /assets/js/newsletter.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/newsletter.js -------------------------------------------------------------------------------- /assets/js/no_defer.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/no_defer.js -------------------------------------------------------------------------------- /assets/js/open-panel-analytics-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/open-panel-analytics-setup.js -------------------------------------------------------------------------------- /assets/js/photoswipe-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/photoswipe-setup.js -------------------------------------------------------------------------------- /assets/js/progress-bar.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/progress-bar.js -------------------------------------------------------------------------------- /assets/js/pseudocode-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/pseudocode-setup.js -------------------------------------------------------------------------------- /assets/js/search-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search-setup.js -------------------------------------------------------------------------------- /assets/js/search/base-styles.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/base-styles.min.js -------------------------------------------------------------------------------- /assets/js/search/command-score.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/command-score.min.js -------------------------------------------------------------------------------- /assets/js/search/hotkeys-js/hotkeys.esm.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/hotkeys-js/hotkeys.esm.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-element/lit-element.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-element/lit-element.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/class-map.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/class-map.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/join.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/join.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/live.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/live.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/ref.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/ref.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/repeat.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/repeat.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/directives/unsafe-html.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/directives/unsafe-html.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/is-server.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/is-server.min.js -------------------------------------------------------------------------------- /assets/js/search/lit-html/lit-html.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit-html/lit-html.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/decorators.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/decorators.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/class-map.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/class-map.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/join.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/join.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/live.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/live.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/ref.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/ref.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/repeat.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/repeat.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/directives/unsafe-html.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/directives/unsafe-html.min.js -------------------------------------------------------------------------------- /assets/js/search/lit/index.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/lit/index.min.js -------------------------------------------------------------------------------- /assets/js/search/ninja-action.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/ninja-action.min.js -------------------------------------------------------------------------------- /assets/js/search/ninja-footer.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/ninja-footer.min.js -------------------------------------------------------------------------------- /assets/js/search/ninja-header.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/ninja-header.min.js -------------------------------------------------------------------------------- /assets/js/search/ninja-keys.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/ninja-keys.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/custom-element.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/custom-element.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/event-options.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/event-options.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/property.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/property.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/query-all.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/query-all.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/query-assigned-elements.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/query-assigned-elements.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/query-assigned-nodes.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/query-assigned-nodes.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/query-async.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/query-async.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/query.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/query.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/decorators/state.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/decorators/state.min.js -------------------------------------------------------------------------------- /assets/js/search/reactive-element/reactive-element.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/search/reactive-element/reactive-element.min.js -------------------------------------------------------------------------------- /assets/js/shortcut-key.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/shortcut-key.js -------------------------------------------------------------------------------- /assets/js/tabs.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/tabs.min.js -------------------------------------------------------------------------------- /assets/js/theme.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/theme.js -------------------------------------------------------------------------------- /assets/js/tooltips-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/tooltips-setup.js -------------------------------------------------------------------------------- /assets/js/typograms.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/typograms.js -------------------------------------------------------------------------------- /assets/js/vanilla-back-to-top.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/vanilla-back-to-top.min.js -------------------------------------------------------------------------------- /assets/js/vega-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/vega-setup.js -------------------------------------------------------------------------------- /assets/js/venobox-setup.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/venobox-setup.js -------------------------------------------------------------------------------- /assets/js/wechat.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/wechat.js -------------------------------------------------------------------------------- /assets/js/zoom.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/js/zoom.js -------------------------------------------------------------------------------- /assets/plotly/fsdp-time.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/fsdp-time.html -------------------------------------------------------------------------------- /assets/plotly/latency_breakdown.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/latency_breakdown.html -------------------------------------------------------------------------------- /assets/plotly/latency_breakdown_log.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/latency_breakdown_log.html -------------------------------------------------------------------------------- /assets/plotly/latency_throughput.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/latency_throughput.html -------------------------------------------------------------------------------- /assets/plotly/pareto-2k.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/pareto-2k.html -------------------------------------------------------------------------------- /assets/plotly/pareto-8k.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/pareto-8k.html -------------------------------------------------------------------------------- /assets/plotly/pareto.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/pareto.html -------------------------------------------------------------------------------- /assets/plotly/training-roofline.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/plotly/training-roofline.html -------------------------------------------------------------------------------- /assets/webfonts/fa-brands-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-brands-400.ttf -------------------------------------------------------------------------------- /assets/webfonts/fa-brands-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-brands-400.woff2 -------------------------------------------------------------------------------- /assets/webfonts/fa-regular-400.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-regular-400.ttf -------------------------------------------------------------------------------- /assets/webfonts/fa-regular-400.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-regular-400.woff2 -------------------------------------------------------------------------------- /assets/webfonts/fa-solid-900.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-solid-900.ttf -------------------------------------------------------------------------------- /assets/webfonts/fa-solid-900.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-solid-900.woff2 -------------------------------------------------------------------------------- /assets/webfonts/fa-v4compatibility.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-v4compatibility.ttf -------------------------------------------------------------------------------- /assets/webfonts/fa-v4compatibility.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/assets/webfonts/fa-v4compatibility.woff2 -------------------------------------------------------------------------------- /bin/deploy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/bin/deploy -------------------------------------------------------------------------------- /bin/entry_point.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/bin/entry_point.sh -------------------------------------------------------------------------------- /conclusion.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/conclusion.md -------------------------------------------------------------------------------- /docker-compose-slim.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/docker-compose-slim.yml -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /dropdown.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/dropdown.md -------------------------------------------------------------------------------- /gpus.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/gpus.md -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/index.md -------------------------------------------------------------------------------- /inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/inference.md -------------------------------------------------------------------------------- /jax-stuff.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/jax-stuff.md -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/package.json -------------------------------------------------------------------------------- /profiling.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/profiling.md -------------------------------------------------------------------------------- /purgecss.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/purgecss.config.js -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | nbconvert 2 | -------------------------------------------------------------------------------- /robots.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/robots.txt -------------------------------------------------------------------------------- /roofline.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/roofline.md -------------------------------------------------------------------------------- /sharding.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/sharding.md -------------------------------------------------------------------------------- /tpus.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/tpus.md -------------------------------------------------------------------------------- /training.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/training.md -------------------------------------------------------------------------------- /transformers.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jax-ml/scaling-book/HEAD/transformers.md --------------------------------------------------------------------------------