Debian Codegraph @

crates/fuzz-stats/src/bin/failed-instantiations.rs

raw
1 use arbitrary::{Arbitrary, Error, Unstructured};
2 use rand::RngCore;
3 use std::sync::Arc;
4 use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering::SeqCst};
5 use wasmtime::*;
6
7 struct State {
8 engine: Engine,
9 print: bool,
10 remaining: AtomicIsize,
11 total: AtomicUsize,
12 instantiate_trap: AtomicUsize,
13 instantiate_oom: AtomicUsize,
14 }
15
16 fn main() {
17 Arc::new(State::new()).run();
18 }
19
20 // Theoretically this test can fail because it's based on random data. In
21 // practice it's expected that the "fails to instantiate" rate is <2%, so if we
22 // cross the 10% threshold that's quite bad.
23 #[test]
24 fn under_10_percent() {
25 let mut state = State::new();
26 state.print = false;
27 state.remaining.store(1000, SeqCst);
28 let state = Arc::new(state);
29 state.run();
30
31 let total = state.total.load(SeqCst);
32 let bad = state.instantiate_trap.load(SeqCst) + state.instantiate_oom.load(SeqCst);
33 assert!(
34 bad < total / 10,
35 "{bad} modules failed to instantiate out of {total}, this failure rate is too high"
36 );
37 }
38
39 impl State {
40 fn new() -> State {
41 State {
42 engine: Engine::default(),
43 print: true,
44 total: AtomicUsize::new(0),
45 remaining: AtomicIsize::new(isize::max_value()),
46 instantiate_trap: AtomicUsize::new(0),
47 instantiate_oom: AtomicUsize::new(0),
48 }
49 }
50
51 fn run(self: &Arc<Self>) {
52 let threads = (0..num_cpus::get())
53 .map(|_| {
54 let state = self.clone();
55 std::thread::spawn(move || state.run_worker())
56 })
57 .collect::<Vec<_>>();
58 for thread in threads {
59 thread.join().unwrap();
60 }
61 }
62
63 fn run_worker(&self) {
64 let mut rng = rand::rng();
65 let mut data = Vec::new();
66
67 while self.remaining.fetch_sub(1, SeqCst) >= 0 {
68 data.truncate(0);
69 data.resize(1024, 0);
70 rng.fill_bytes(&mut data);
71 loop {
72 match self.run_once(&data) {
73 Ok(()) => break,
74 Err(Error::NotEnoughData) => {
75 let cur = data.len();
76 let extra = 1024;
77 data.resize(cur + extra, 0);
78 rng.fill_bytes(&mut data[cur..]);
79 }
80 Err(e) => panic!("failed to generated module: {e}"),
81 }
82 }
83 }
84 }
85
86 /// Generates a random modules using `data`, and then attempts to
87 /// instantiate it.
88 ///
89 /// Records when instantiation fails and why it fails.
90 fn run_once(&self, data: &[u8]) -> Result<(), Error> {
91 let mut u = Unstructured::new(data);
92 // Here swarm testing is used to get hopefully a bit more coverage of
93 // interesting states, and we also forcibly disable all `start`
94 // functions for now. Not much work has gone into minimizing the traps
95 // generated from wasm functions themselves, and this shouldn't be
96 // enabled until that's been worked on.
97 let mut config = wasm_smith::Config::arbitrary(&mut u)?;
98 config.allow_start_export = false;
99
100 config.exceptions_enabled = false; // Not implemented by Wasmtime
101 config.threads_enabled = false; // not enabled by default in Wasmtime
102
103 // NB: just added "table64" support to this and wasmtime doesn't
104 // implement that yet
105 config.memory64_enabled = false;
106
107 // Wasmtime doesn't support these proposals yet.
108 config.gc_enabled = false;
109
110 let mut wasm = wasm_smith::Module::new(config, &mut u)?;
111 wasm.ensure_termination(10_000).unwrap();
112 let wasm = wasm.to_bytes();
113
114 // We install a resource limiter in the store which limits the store to
115 // 1gb of memory. That's half the default allocation of memory for
116 // libfuzzer-based fuzzers by default, and ideally we're not in a
117 // situation where most of the modules are above this threshold.
118 let module = match Module::new(&self.engine, &wasm) {
119 Ok(m) => m,
120 // NB: after bytecodealliance/wasm-tools#1426 wasm-smith is
121 // generating modules that Wasmtime can't handle until
122 // bytecodealliance/wasmtime#7996 is on crates.io, until that time
123 // ignore these errors.
124 Err(e) if format!("{e:?}").contains("unsupported init expr") => return Ok(()),
125 Err(e) => panic!("unexpected module compile error {e:?}"),
126 };
127 let mut store = Store::new(
128 &self.engine,
129 fuzz_stats::limits::StoreLimits {
130 remaining_memory: 1 << 30,
131 oom: false,
132 },
133 );
134 store.limiter(|s| s as &mut dyn ResourceLimiter);
135
136 // Synthesize dummy imports based on what the module asked for, and then
137 // instantiate!
138 let instance = fuzz_stats::dummy::dummy_imports(&mut store, &module)
139 .and_then(|imports| Instance::new(&mut store, &module, &imports));
140
141 match instance {
142 // If instantiation succeeded, we're not too interested in anything
143 // else right now. In the future we should probably run exported
144 // functions and record whether a trap happened or not.
145 Ok(_i) => {}
146
147 Err(e) => {
148 // Traps are ok if they happen during instantiation. This is an
149 // expected occurrence we want to account for.
150 if e.downcast_ref::<Trap>().is_some() {
151 std::fs::write("trap.wasm", &wasm).unwrap();
152 self.instantiate_trap.fetch_add(1, SeqCst);
153
154 // Ooms, like traps, are normal during instantiations. This
155 // can happen, for example, if a defined memory is very large.
156 } else if store.data().oom {
157 std::fs::write("oom.wasm", &wasm).unwrap();
158 self.instantiate_oom.fetch_add(1, SeqCst);
159
160 // In theory nothing else fails to instantiate. If it does, then
161 // panic.
162 } else {
163 std::fs::write("panic.wasm", &wasm).unwrap();
164 panic!("unknown: {e}");
165 }
166 }
167 }
168
169 let prev_total = self.total.fetch_add(1, SeqCst);
170 if prev_total % 10_000 == 0 && self.print {
171 self.print(prev_total + 1);
172 }
173
174 Ok(())
175 }
176
177 /// Prints summary statistics of how many modules have been instantiated so
178 /// far and how many of them have oom'd or trap'd.
179 fn print(&self, total: usize) {
180 print!("total: {total:8}");
181 let stat = |name: &str, stat: &AtomicUsize| {
182 let stat = stat.load(SeqCst);
183 if stat > 0 {
184 print!(" {} {:5.02}% ", name, (stat as f64) / (total as f64) * 100.);
185 }
186 };
187 stat("i-oom", &self.instantiate_oom);
188 stat("i-trap", &self.instantiate_trap);
189 println!();
190 }
191 }