22 |
37 | Click on cells to see information about instruction or the next instruction group.
38 |
39 |
40 |
43 |
44 |
2 | 
3 |
Haruspex
4 |/həˈrʌspeks/
5 |A religious official in ancient Rome who predicted the future or interpreted the meaning of events by examining the insides of birds or animals that had been sacrificed.
6 |7 | Exploration of x86-64 ISA using speculative execution. 8 |
9 | 10 | 11 | 12 | Haruspex is a project attempting to explore and audit the x86-64 instruction set by (ab)using speculative execution and certain low-level performance counters Intel CPUs provide mainly for the identification of undocumented opcodes and deducing the pipeline properties of each instruction. You can find the article explaining the methodology [here](https://blog.can.ac/2021/03/22/speculating-x86-64-isa-with-one-weird-trick/). 13 | 14 | - `/analysis` contains the Node.js server responsible for reducing the dataset and serving it as an interactive table, which you can find live at [haruspex.can.ac](https://haruspex.can.ac) 15 | . 16 | - `/kernel` contains the code responsible for producing this data, note that it is essentially "pseudo-code" due to some of the dependencies and parts of the toolchain I used to build it I cannot share, but should be pretty easy to translate it to work in your OS. 17 | - `/raw-data` contains the raw data from certain processors. 18 | 19 | # Thanks to 20 | 21 | - @JustasMasiulis for helping with the web components. 22 | - @H4vC for the name. 23 | 24 | # License 25 | 26 | Haruspex is licensed under the GNU General Public License v3. 27 | -------------------------------------------------------------------------------- /analysis/dataset.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const crypto = require("crypto"); 3 | 4 | const prefixList = [ 5 | "2e", 6 | "36", 7 | "3e", 8 | "26", 9 | "40", 10 | "41", 11 | "42", 12 | "43", 13 | "44", 14 | "45", 15 | "46", 16 | "47", 17 | "48", 18 | "49", 19 | "4a", 20 | "4b", 21 | "4c", 22 | "4d", 23 | "4e", 24 | "4f", 25 | "64", 26 | "65", 27 | "66", 28 | "67", 29 | "9b", 30 | "f0", 31 | "f2", 32 | "f3", 33 | ]; 34 | 35 | function hexb(v) { 36 | return v <= 0xf ? "0" + v.toString(16) : v.toString(16); 37 | } 38 | 39 | function parse(rawData) { 40 | // Read and parse the JSON. 41 | // 42 | const { nopBaseline, brand, faultBaseline, data, nopUops } = JSON.parse(rawData); 43 | 44 | // Parse the entries slightly. 45 | // 46 | const instructions = {}; 47 | for (const entry of data) { 48 | entry.opcode = Buffer.from(entry.opcode).toString("hex"); 49 | entry.mits = entry.uops[0] - faultBaseline.mits; 50 | entry.ms = entry.uops[2] - faultBaseline.ms; 51 | delete entry.uops; 52 | 53 | if ( 54 | entry.ms < 0 || 55 | entry.mits < 0 || 56 | entry.category.includes("_BR") || 57 | entry.category.includes("CALL_") 58 | ) { 59 | entry.branch = true; 60 | entry.serializing = null; 61 | entry.ms = 0; 62 | entry.mits = 0; 63 | } else { 64 | entry.branch = false; 65 | entry.serializing = entry.mits == 0; 66 | } 67 | entry.speculationFence = entry.outOfOrder < 4; 68 | instructions[entry.opcode] = entry; 69 | } 70 | 71 | const propertyMatch = (i1, i2) => { 72 | return ( 73 | i2.ms == i1.ms && 74 | i2.iclass == i1.iclass && 75 | i2.category == i1.category && 76 | i2.extension == i1.extension && 77 | i2.cpl == i1.cpl && 78 | i2.valid == i1.valid 79 | ); 80 | }; 81 | 82 | // Purge redundant suffixes. 83 | // 84 | let suffixPurgeCounter = 0; 85 | const keysSorted = Object.keys(instructions).sort(function (a, b) { 86 | return a.length - b.length; 87 | }); 88 | for (const key of keysSorted) { 89 | const evaluate = (k1, ki1) => { 90 | const i1 = instructions[ki1]; 91 | if (!i1) { 92 | return; 93 | } 94 | 95 | let matches = []; 96 | for (let n = 0; n <= 0xff; n++) { 97 | const k2 = k1 + hexb(n); 98 | const i2 = instructions[k2]; 99 | if (i2 && propertyMatch(i1, i2)) { 100 | matches.push(k2); 101 | } 102 | } 103 | 104 | if (matches.length < 256) { 105 | return false; 106 | } 107 | 108 | let success = false; 109 | for (const k2 of matches) { 110 | if (k2 != ki1) { 111 | delete instructions[k2]; 112 | suffixPurgeCounter++; 113 | success = true; 114 | } 115 | } 116 | return success; 117 | }; 118 | if (evaluate(key, key) || key.length > 2) { 119 | evaluate(key.substr(0, key.length - 2), key); 120 | } 121 | } 122 | 123 | // Purge redundant prefixes. 124 | // 125 | let prefixPurgeCounter = 0; 126 | for (const k1 of Object.keys(instructions)) { 127 | // Skip if already deleted. 128 | // 129 | const i1 = instructions[k1]; 130 | if (!i1) { 131 | continue; 132 | } 133 | 134 | // Iterate each prefix (apart from 0f): 135 | // 136 | for (const pfx of prefixList) { 137 | // If the instruction exists: 138 | // 139 | const k2 = pfx + k1; 140 | if (k2 in instructions) { 141 | // If the instruction has matching properties as the derived from parent, delete the entry. 142 | // 143 | const i2 = instructions[k2]; 144 | if (propertyMatch(i1, i2)) { 145 | // MITS#1 == MITS#2 can indicate same instruction if instruction halts. 146 | // Otherwise MITS#1 has to be one more than MITS#2 since it should execute one more NOP. 147 | // 148 | if (i1.mits != i2.mits) { 149 | if (i1.mits != i2.mits + nopUops) { 150 | continue; 151 | } 152 | } else if (i1.mits > faultBaseline.mits) { 153 | continue; 154 | } 155 | 156 | prefixPurgeCounter++; 157 | delete instructions[k2]; 158 | } 159 | } 160 | } 161 | } 162 | 163 | // Return the parsed entry. 164 | // 165 | return { 166 | brand, 167 | instructions, 168 | prefixPurgeCounter, 169 | suffixPurgeCounter, 170 | nopUops, 171 | }; 172 | } 173 | 174 | function load(path, noCache = false) { 175 | // Skip the body if no-caching is requested. 176 | // 177 | if (noCache) { 178 | return parse(fs.readFileSync(path)); 179 | } 180 | 181 | // Read the data and weakly hash the input. 182 | // 183 | const data = fs.readFileSync(path); 184 | const sha256 = crypto.createHash("sha256"); 185 | sha256.update(data); 186 | const hash = sha256.digest().toString("hex").substr(0, 8); 187 | 188 | // See if there is a cached instance, if so return without parsing. 189 | // 190 | const cacheName = path + ".cache" + hash; 191 | if (fs.existsSync(cacheName)) { 192 | try { 193 | return JSON.parse(fs.readFileSync(cacheName)); 194 | } catch (ex) { 195 | console.warn(`Error parsing cache entry ${cacheName}`); 196 | } 197 | } 198 | 199 | // Parse the data, write the cache entry, return the result. 200 | // 201 | const parsed = parse(data); 202 | fs.writeFileSync(cacheName, JSON.stringify(parsed)); 203 | return parsed; 204 | } 205 | 206 | module.exports = { 207 | parse, 208 | load, 209 | }; 210 | -------------------------------------------------------------------------------- /analysis/haruspex.js: -------------------------------------------------------------------------------- 1 | const dataset = require("./dataset"); 2 | const yargs = require("yargs"); 3 | const compression = require("compression"); 4 | const path = require("path"); 5 | const express = require("express"); 6 | const fs = require("fs"); 7 | const staticPath = __dirname + "/public"; 8 | 9 | // Parse the arguments. 10 | // 11 | const argv = yargs 12 | .command("process", "Produces a reduced data-set given raw data.", { 13 | dataset: { 14 | description: "path to the dataset, will be written over with the processed dataset", 15 | alias: "f", 16 | type: "string", 17 | }, 18 | }) 19 | .command("start", "Starts the interactive server.", { 20 | dataset: { 21 | description: "path to the dataset directory", 22 | alias: "f", 23 | type: "string", 24 | }, 25 | port: { 26 | description: "port number, defaults to 80", 27 | alias: "p", 28 | type: "number", 29 | }, 30 | }) 31 | .help() 32 | .alias("help", "h").argv; 33 | 34 | // If requested to process a dataset: 35 | // 36 | if (argv._.includes("process")) { 37 | fs.writeFileSync(argv.file, JSON.stringify(dataset.load(argv.dataset, true))); 38 | } 39 | 40 | // If not requested to start the server, return. 41 | // 42 | if (!argv._.includes("start")) { 43 | return; 44 | } 45 | 46 | // Discover all datasets. 47 | // 48 | const datasetdir = argv.dataset || __dirname + "/../raw-data/"; 49 | const datasetEntries = fs 50 | .readdirSync(datasetdir) 51 | .filter((k) => k.startsWith("isa-") && k.endsWith(".json")) 52 | .map((e) => ({ name: e.substr(4, e.length - 9), path: path.join(datasetdir, e) })); 53 | 54 | // Process and save all datasets to the static data path. 55 | // 56 | const datasetList = []; 57 | for (const { name, path } of datasetEntries) { 58 | const data = dataset.load(path); 59 | fs.writeFileSync(staticPath + `/dataset-${name}.json`, JSON.stringify(data)); 60 | datasetList.push({ id: name, brand: data.brand }); 61 | } 62 | 63 | // Declare the endpoints. 64 | // 65 | const app = express(); 66 | app.use(compression()); 67 | app.use("/static", express.static(staticPath)); 68 | app.get("/", (req, res) => { 69 | res.sendFile(staticPath + "/index.html"); 70 | }); 71 | app.get("/datasets.json", (req, res) => { 72 | res.send(JSON.stringify(datasetList)); 73 | }); 74 | app.get("/chip/:chip", (req, res) => { 75 | res.redirect("/chip/" + req.params.chip + "/grid/"); 76 | }); 77 | app.get("/chip/:chip/list", (req, res) => { 78 | res.sendFile(staticPath + "/list.html"); 79 | }); 80 | app.get("/chip/:chip/grid/:sub?", (req, res) => { 81 | res.sendFile(staticPath + "/grid.html"); 82 | }); 83 | app.get("*", function (req, res) { 84 | res.redirect("/"); 85 | }); 86 | 87 | // Start listening. 88 | // 89 | const port = argv.port || 80; 90 | app.listen(port, () => { 91 | console.log(`Listening at http://localhost:${port}`); 92 | }); 93 | -------------------------------------------------------------------------------- /analysis/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "haruspex", 3 | "version": "1.0.0", 4 | "description": "Analysis server for the Haruspex project for exploration of x86-64 ISA.", 5 | "main": "haruspex.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/can1357/haruspex.git" 9 | }, 10 | "author": "Can Bölük", 11 | "license": "GPL-3.0-or-later", 12 | "bugs": { 13 | "url": "https://github.com/can1357/haruspex/issues" 14 | }, 15 | "homepage": "https://github.com/can1357/haruspex#readme", 16 | "dependencies": { 17 | "compression": "^1.7.4", 18 | "express": "^4.17.1", 19 | "yargs": "^16.2.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /analysis/public/chipview.js: -------------------------------------------------------------------------------- 1 | // Write the chip table. 2 | // 3 | async function writeTable() { 4 | const datasets = await (await fetch(`/datasets.json`)).json(); 5 | console.log(datasets); 6 | 7 | const el = document.getElementById("chip-table"); 8 | for (const { id, brand } of datasets) { 9 | el.innerHTML += `${brand}`; 10 | } 11 | } 12 | writeTable(); 13 | -------------------------------------------------------------------------------- /analysis/public/grid.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 13 | 14 | 15 |" + decoding.toUpperCase() + "
";
181 | } else {
182 | item.innerHTML += "" + list.length + "
";
183 | }
184 |
185 | // Set the classes.
186 | //
187 | if (udList.length !== 0) {
188 | item.classList += " op-undocumented";
189 | }
190 |
191 | if (list.length !== 0) {
192 | if (hexBase === "" && prefixList.includes(hexb(i))) {
193 | item.classList += " color-text-link";
194 | } else {
195 | item.classList +=
196 | list.length == 1 ? " color-text-primary" : " color-text-warning";
197 | }
198 |
199 | item.classList += " op-button";
200 | item.onclick = () => {
201 | checkPopoverForClose();
202 |
203 | if (list.length > 1) {
204 | activePopover = null;
205 | visitTable(hexBase + subHex);
206 | } else {
207 | const instr = instructions[list[0]];
208 | const items = data_schema
209 | .map(
210 | ({ key, title, desc }) =>
211 | `${instr[key]}
218 |
230 |
`
231 | );
232 |
233 | activePopover = item;
234 | document.getElementById(instr.opcode).addEventListener("click", (event) => {
235 | checkPopoverForClose();
236 | event.stopPropagation();
237 | });
238 | }
239 | };
240 | } else {
241 | item.classList += " color-text-tertiary color-bg-tertiary";
242 | item.onclick = () => {};
243 | }
244 | }
245 | }
246 |
247 | document.addEventListener("click", (event) => {
248 | checkPopoverForClose(event);
249 | });
250 |
251 | // Build the table with the original location.
252 | //
253 | visitTable(opcode);
254 |
--------------------------------------------------------------------------------
/analysis/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
9 |
13 |
14 |
15 |
31 | Choose a chip below to browse the data.
32 |
33 |
34 |
36 |
37 |
38 |