1use crate::granule::GRANULE_SHIFT;
2use crate::granule::{set_granule, GranuleState};
3use crate::measurement::HashContext;
4use crate::realm::mm::address::GuestPhysAddr;
5use crate::realm::mm::attribute::desc_type;
6use crate::realm::mm::entry;
7use crate::realm::mm::stage2_translation::{RttAllocator, Tlbi};
8use crate::realm::mm::stage2_tte::{hipas, mapping_size, ripas, S2TTE};
9use crate::realm::mm::stage2_tte::{
10 level_mask, INVALID_UNPROTECTED, TABLE_TTE, TTE_ATTR_MASK, VALID_NS_TTE, VALID_TTE,
11};
12use crate::realm::mm::table_level;
13use crate::realm::rd::Rd;
14use crate::rmi::error::Error;
15use crate::rmi::rtt_entry_state;
16use crate::{get_granule, get_granule_if};
17use armv9a::bits_in_reg;
18use vmsa::address::PhysAddr;
19use vmsa::page_table::{Entry, Level, PageTable};
20
21pub const RTT_MIN_BLOCK_LEVEL: usize = table_level::L1Table::THIS_LEVEL;
22pub const RTT_PAGE_LEVEL: usize = table_level::L3Table::THIS_LEVEL;
23pub const RTT_STRIDE: usize = GRANULE_SHIFT - 3;
24
25const CHANGE_DESTROYED: u64 = 0x1;
26
27fn level_space_size(rd: &Rd, level: usize) -> usize {
28 rd.s2_table().lock().space_size(level)
29}
30
31fn create_pgtbl_at(
32 rtt_addr: usize,
33 flags: u64,
34 mut pa: usize,
35 map_size: usize,
36) -> Result<(), Error> {
37 let alloc = RttAllocator { base: rtt_addr };
38 let mut new_s2tte = pa as u64 | flags;
39
40 let ret = PageTable::<
41 GuestPhysAddr,
42 table_level::L3Table, entry::Entry,
44 { table_level::L3Table::NUM_ENTRIES },
45 >::new_init_in(&alloc, |entries| {
46 for e in entries.iter_mut() {
47 let _ = (*e).set(PhysAddr::from(pa), new_s2tte);
48 pa += map_size;
49 new_s2tte = pa as u64 | flags;
50 }
51 });
52
53 if ret.is_err() {
54 return Err(Error::RmiErrorRtt(0));
55 }
56 Ok(())
57}
58
59pub fn create(rd: &Rd, rtt_addr: usize, ipa: usize, level: usize) -> Result<(), Error> {
60 let mut invalidate = Tlbi::NONE;
61
62 let (parent_s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level - 1, Error::RmiErrorInput)?;
63
64 if last_level != level - 1 || parent_s2tte.is_table(last_level) {
65 return Err(Error::RmiErrorRtt(last_level));
66 }
67
68 let map_size = mapping_size(level);
69 let mut flags = parent_s2tte.get_masked(S2TTE::HIPAS);
70 if rd.addr_in_par(ipa) {
71 flags |= parent_s2tte.get_masked(S2TTE::RIPAS);
72 } else {
73 flags |= parent_s2tte.get_masked(S2TTE::NS);
74 }
75
76 if parent_s2tte.is_unassigned() || parent_s2tte.is_unassigned_ns() {
77 create_pgtbl_at(rtt_addr, flags, 0, 0)?;
78 } else {
79 if !parent_s2tte.is_assigned_invalid() {
81 if level == RTT_PAGE_LEVEL {
82 flags |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::L3_PAGE);
83 } else {
84 flags |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::L012_BLOCK);
85 }
86 invalidate = Tlbi::LEAF(rd.id());
87 }
88 let pa: usize = parent_s2tte.addr_as_block(level - 1).into(); flags |= parent_s2tte.get_masked(TTE_ATTR_MASK);
90 create_pgtbl_at(rtt_addr, flags, pa, map_size)?;
91 }
92
93 let parent_s2tte = rtt_addr as u64 | TABLE_TTE;
94 rd.s2_table().lock().ipa_to_pte_set(
95 GuestPhysAddr::from(ipa),
96 level - 1,
97 parent_s2tte,
98 invalidate,
99 )?;
100
101 Ok(())
102}
103
104pub fn destroy<F: FnMut(usize)>(
105 rd: &Rd,
106 ipa: usize,
107 level: usize,
108 mut f: F,
109) -> Result<(usize, usize), Error> {
110 let invalidate;
111 let (parent_s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level - 1, Error::RmiErrorRtt(0))?;
112
113 if (last_level != level - 1) || !parent_s2tte.is_table(last_level) {
114 let top_ipa = skip_non_live_entries(rd, ipa, last_level)?;
115 f(top_ipa);
116 return Err(Error::RmiErrorRtt(last_level));
117 }
118
119 let rtt_addr = parent_s2tte.addr_as_block(RTT_PAGE_LEVEL).into();
120
121 let mut g_rtt = get_granule_if!(rtt_addr, GranuleState::RTT)?;
122
123 if is_live_rtt(rd, ipa, level).unwrap_or(false) {
126 f(ipa);
127 return Err(Error::RmiErrorRtt(level));
128 }
129
130 let parent_s2tte = if rd.addr_in_par(ipa) {
131 invalidate = Tlbi::LEAF(rd.id());
132 bits_in_reg(S2TTE::HIPAS, hipas::UNASSIGNED)
133 | bits_in_reg(S2TTE::RIPAS, ripas::DESTROYED)
134 | bits_in_reg(S2TTE::DESC_TYPE, desc_type::LX_INVALID)
135 } else {
136 invalidate = Tlbi::BREAKDOWN(rd.id());
137 bits_in_reg(S2TTE::NS, 1)
138 | bits_in_reg(S2TTE::HIPAS, hipas::UNASSIGNED)
139 | INVALID_UNPROTECTED
140 };
141
142 rd.s2_table().lock().ipa_to_pte_set(
143 GuestPhysAddr::from(ipa),
144 level - 1,
145 parent_s2tte,
146 invalidate,
147 )?;
148
149 set_granule(&mut g_rtt, GranuleState::Delegated)?;
150
151 let top_ipa = skip_non_live_entries(rd, ipa, last_level)?;
152 Ok((rtt_addr, top_ipa))
153}
154
155pub fn init_ripas(rd: &mut Rd, base: usize, top: usize) -> Result<usize, Error> {
156 let level = RTT_PAGE_LEVEL;
158 let (s2tte, last_level) = S2TTE::get_s2tte(rd, base, level, Error::RmiErrorRtt(0))?;
159
160 let map_size = mapping_size(last_level);
161
162 let mut addr = base & !(map_size - 1);
163 if addr != base {
164 warn!("base is not aligned");
165 return Err(Error::RmiErrorRtt(last_level));
166 }
167
168 if top != (top & !(map_size - 1)) {
169 warn!("top is not aligned");
170 return Err(Error::RmiErrorRtt(last_level));
171 }
172
173 if s2tte.get_masked_value(S2TTE::HIPAS) != hipas::UNASSIGNED {
174 warn!("base is assigned already");
175 return Err(Error::RmiErrorRtt(last_level));
176 }
177
178 let space_size = level_space_size(rd, last_level);
179 let top_addr = (addr & !(space_size - 1)) + space_size;
180 while addr < top_addr {
181 let next = addr + map_size;
182 if next > top {
183 break;
184 }
185 let (s2tte, last_level) = S2TTE::get_s2tte(rd, addr, level, Error::RmiErrorRtt(0))?;
186 if s2tte.is_table(last_level) || s2tte.get_masked_value(S2TTE::HIPAS) == hipas::ASSIGNED {
187 break;
188 }
189 let new_s2tte =
190 bits_in_reg(S2TTE::HIPAS, hipas::UNASSIGNED) | bits_in_reg(S2TTE::RIPAS, ripas::RAM);
191
192 rd.s2_table().lock().ipa_to_pte_set(
193 GuestPhysAddr::from(addr),
194 last_level,
195 new_s2tte,
196 Tlbi::NONE,
197 )?;
198
199 #[cfg(not(kani))]
200 HashContext::new(rd)?.measure_ripas_granule(addr as u64, next as u64)?;
202
203 addr += map_size;
204 }
205
206 if addr > base {
207 Ok(addr)
208 } else {
209 Err(Error::RmiErrorRtt(last_level))
210 }
211}
212
213pub fn get_ripas(rd: &Rd, start: usize, end: usize) -> Result<(usize, u64), Error> {
215 let level = RTT_PAGE_LEVEL;
216 let mut addr = start;
217 let mut common_ripas = 0; let mut map_size = 0; while addr < end {
220 let (s2tte, last_level) = S2TTE::get_s2tte(rd, addr, level, Error::RmiErrorRtt(0))?;
221 if !s2tte.has_ripas(level) {
222 break;
223 }
224 let ripas = s2tte.get_ripas();
225 if addr == start {
226 common_ripas = ripas;
227 map_size = mapping_size(last_level);
228 } else if common_ripas != ripas {
229 break;
230 }
231 addr += map_size;
232 }
233 if addr == start {
234 return Err(Error::RmiErrorInput);
235 }
236 Ok((addr, common_ripas))
237}
238
239pub fn read_entry(rd: &Rd, ipa: usize, level: usize) -> Result<[usize; 4], Error> {
240 let (s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(0))?;
241
242 let r1 = last_level;
243 let (mut r2, mut r3, mut r4) = (0, 0, 0);
244
245 if s2tte.is_unassigned() {
246 r2 = rtt_entry_state::RMI_UNASSIGNED;
247 r4 = s2tte.get_masked_value(S2TTE::RIPAS) as usize;
248 } else if s2tte.is_assigned() {
249 r2 = rtt_entry_state::RMI_ASSIGNED;
250 r3 = s2tte.addr_as_block(last_level).into(); r4 = s2tte.get_masked_value(S2TTE::RIPAS) as usize;
252 } else if s2tte.is_table(last_level) {
253 r2 = rtt_entry_state::RMI_TABLE;
254 r3 = s2tte.get_masked(S2TTE::ADDR_TBL_OR_PAGE); } else if s2tte.is_unassigned_ns() {
256 r2 = rtt_entry_state::RMI_UNASSIGNED;
257 } else if s2tte.is_assigned_ns(last_level) {
258 r2 = rtt_entry_state::RMI_ASSIGNED;
259 let addr_mask: u64 = level_mask(last_level).ok_or(Error::RmiErrorRtt(0))?;
260 let mask = addr_mask | S2TTE::MEMATTR | S2TTE::S2AP;
261 r3 = s2tte.get_masked(mask);
262 } else {
263 error!("Unexpected S2TTE value retrieved!");
264 }
265 Ok([r1, r2, r3 as usize, r4])
266}
267
268pub fn map_unprotected(rd: &Rd, ipa: usize, level: usize, host_s2tte: usize) -> Result<(), Error> {
269 if rd.addr_in_par(ipa) {
270 return Err(Error::RmiErrorInput);
271 }
272
273 let (s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(0))?;
275
276 if level != last_level {
277 return Err(Error::RmiErrorRtt(last_level));
278 }
279
280 if !s2tte.is_unassigned_ns() {
281 return Err(Error::RmiErrorRtt(level));
282 }
283
284 let host_s2tte = S2TTE::new(host_s2tte as u64);
285 let mut new_s2tte = host_s2tte.get_masked(S2TTE::ADDR_TBL_OR_PAGE)
286 | host_s2tte.get_masked(S2TTE::MEMATTR)
287 | host_s2tte.get_masked(S2TTE::S2AP)
288 | bits_in_reg(S2TTE::HIPAS, hipas::ASSIGNED);
289 if level == RTT_PAGE_LEVEL {
290 new_s2tte |= VALID_NS_TTE | bits_in_reg(S2TTE::DESC_TYPE, desc_type::L3_PAGE);
291 } else {
292 new_s2tte |= VALID_NS_TTE | bits_in_reg(S2TTE::DESC_TYPE, desc_type::L012_BLOCK);
293 }
294
295 rd.s2_table().lock().ipa_to_pte_set(
296 GuestPhysAddr::from(ipa),
297 level,
298 new_s2tte,
299 Tlbi::LEAF(rd.id()),
300 )?;
301
302 Ok(())
303}
304
305pub fn unmap_unprotected<F: FnMut(usize)>(
306 rd: &Rd,
307 ipa: usize,
308 level: usize,
309 mut f: F,
310) -> Result<usize, Error> {
311 if rd.addr_in_par(ipa) {
312 return Err(Error::RmiErrorInput);
313 }
314
315 let (s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(0))?;
316
317 if level != last_level || !s2tte.is_assigned_ns(last_level) {
318 let top_ipa = skip_non_live_entries(rd, ipa, last_level)?;
319 f(top_ipa);
320 return Err(Error::RmiErrorRtt(last_level));
321 }
322
323 let new_s2tte: u64 = bits_in_reg(S2TTE::NS, 1)
324 | bits_in_reg(S2TTE::HIPAS, hipas::UNASSIGNED)
325 | INVALID_UNPROTECTED;
326
327 rd.s2_table().lock().ipa_to_pte_set(
328 GuestPhysAddr::from(ipa),
329 level,
330 new_s2tte,
331 Tlbi::LEAF(rd.id()),
332 )?;
333
334 let top_ipa = skip_non_live_entries(rd, ipa, level)?;
335 Ok(top_ipa)
336}
337
338pub fn set_ripas(rd: &Rd, base: usize, top: usize, ripas: u8, flags: u64) -> Result<usize, Error> {
339 let level = RTT_PAGE_LEVEL;
341 let (_s2tte, level) = S2TTE::get_s2tte(rd, base, RTT_PAGE_LEVEL, Error::RmiErrorRtt(level))?;
342
343 let map_size = mapping_size(level);
344
345 let mut addr = base & !(map_size - 1);
346 if addr != base || top & !(map_size - 1) != top {
347 return Err(Error::RmiErrorRtt(level));
348 }
349 let space_size = level_space_size(rd, level);
350 let table_top = (addr & !(space_size - 1)) + space_size;
351 if table_top < top {
352 debug!(
353 "table can address upto 0x{:X}, top {:X} overlimits the range",
354 table_top, top
355 );
356 }
357
358 while addr < table_top && addr < top {
359 let mut invalidate = Tlbi::NONE;
360 let (s2tte, last_level) = S2TTE::get_s2tte(rd, addr, level, Error::RmiErrorRtt(level))?;
361 let mut new_s2tte = 0;
362 let mut add_pa = false;
363
364 if s2tte.is_table(last_level) || s2tte.is_destroyed() && flags & CHANGE_DESTROYED == 0 {
365 break;
366 }
367 let pa: usize = s2tte.addr_as_block(last_level).into(); new_s2tte |= s2tte.get_masked(S2TTE::HIPAS);
369 new_s2tte |= bits_in_reg(S2TTE::RIPAS, ripas as u64);
370 if ripas == s2tte.get_masked_value(S2TTE::RIPAS) as u8 {
372 addr += map_size;
373 continue;
374 }
375 if ripas as u64 == ripas::EMPTY {
376 if s2tte.is_assigned_ram(last_level) {
377 add_pa = true;
378 invalidate = Tlbi::LEAF(rd.id());
379 } else if s2tte.is_assigned_destroyed() {
380 add_pa = true;
381 }
382 new_s2tte |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::LX_INVALID);
383 } else if ripas as u64 == ripas::RAM {
384 if s2tte.is_assigned_invalid() {
385 new_s2tte |= VALID_TTE;
386 if last_level == RTT_PAGE_LEVEL {
387 new_s2tte |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::L3_PAGE);
388 } else {
389 new_s2tte |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::L012_BLOCK);
390 }
391 add_pa = true;
392 }
393 } else {
394 unreachable!();
395 }
396 if add_pa {
397 new_s2tte |= pa as u64;
398 }
399 rd.s2_table().lock().ipa_to_pte_set(
400 GuestPhysAddr::from(addr),
401 last_level,
402 new_s2tte,
403 invalidate,
404 )?;
405
406 addr += map_size;
407 }
408 if addr > base {
409 Ok(addr)
410 } else {
411 Err(Error::RmiErrorRtt(level))
412 }
413}
414
415pub fn data_create(rd: &Rd, ipa: usize, target_pa: usize, unknown: bool) -> Result<(), Error> {
416 let level = RTT_PAGE_LEVEL;
417 let (s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(0))?;
418
419 if level != last_level {
420 return Err(Error::RmiErrorRtt(last_level));
421 }
422
423 if !s2tte.is_unassigned() {
424 return Err(Error::RmiErrorRtt(RTT_PAGE_LEVEL));
425 }
426
427 let mut new_s2tte = target_pa as u64;
428 if s2tte.is_ripas() {
429 panic!("invalid ripas");
430 }
431 let ripas = s2tte.get_ripas();
432 new_s2tte |= bits_in_reg(S2TTE::HIPAS, hipas::ASSIGNED);
434 if unknown && ripas != ripas::RAM {
435 new_s2tte |= bits_in_reg(S2TTE::RIPAS, ripas);
437 } else {
438 new_s2tte |= bits_in_reg(S2TTE::RIPAS, ripas::RAM);
440 new_s2tte |= bits_in_reg(S2TTE::DESC_TYPE, desc_type::L3_PAGE);
441 new_s2tte |= VALID_TTE;
442 }
443
444 rd.s2_table()
445 .lock()
446 .ipa_to_pte_set(GuestPhysAddr::from(ipa), level, new_s2tte, Tlbi::NONE)?;
447
448 Ok(())
449}
450
451pub fn data_destroy<F: FnMut(usize)>(
452 rd: &Rd,
453 ipa: usize,
454 mut f: F,
455) -> Result<(usize, usize), Error> {
456 let mut invalidate = Tlbi::NONE;
457 let level = RTT_PAGE_LEVEL;
458 let (s2tte, last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(level))?;
459
460 if last_level < level || !s2tte.is_assigned() {
461 let top_ipa = skip_non_live_entries(rd, ipa, last_level)?;
462 f(top_ipa);
463 return Err(Error::RmiErrorRtt(last_level));
464 }
465
466 let pa = s2tte.addr_as_block(last_level).into(); let mut new_s2tte = bits_in_reg(S2TTE::HIPAS, hipas::UNASSIGNED)
469 | bits_in_reg(S2TTE::DESC_TYPE, desc_type::LX_INVALID);
470 if s2tte.get_masked(S2TTE::RIPAS) == ripas::EMPTY {
471 new_s2tte |= bits_in_reg(S2TTE::RIPAS, ripas::EMPTY);
472 } else {
473 new_s2tte |= bits_in_reg(S2TTE::RIPAS, ripas::DESTROYED);
474 }
475 if s2tte.is_assigned_ram(RTT_PAGE_LEVEL) {
476 invalidate = Tlbi::LEAF(rd.id());
477 }
478 rd.s2_table()
479 .lock()
480 .ipa_to_pte_set(GuestPhysAddr::from(ipa), level, new_s2tte, invalidate)?;
481
482 let top_ipa = skip_non_live_entries(rd, ipa, level)?;
483
484 Ok((pa, top_ipa))
485}
486
487fn is_live_rtt(rd: &Rd, base: usize, level: usize) -> Result<bool, Error> {
488 let binding = rd.s2_table();
489 let binding = binding.lock();
490 let (entries_iter, last_level) = binding.entries(GuestPhysAddr::from(base), level)?;
491 if level != last_level {
492 error!(
493 "level doesn't match! level:{:?} last_level:{:?}",
494 level, last_level
495 );
496 }
497
498 for entry in entries_iter {
499 let s2tte = S2TTE::new(entry.pte());
500 if s2tte.is_live(level) {
501 return Ok(true);
502 }
503 }
504 Err(Error::RmiErrorRtt(level))
505}
506
507fn skip_non_live_entries(rd: &Rd, base: usize, level: usize) -> Result<usize, Error> {
508 let map_size = mapping_size(level);
509
510 let mut addr = base & !(map_size - 1);
511 let space_size = level_space_size(rd, level);
512 let mut entry0_ipa = addr & !(space_size - 1);
513
514 let binding = rd.s2_table();
515 let binding = binding.lock();
516 let (entries_iter, last_level) = binding.entries(GuestPhysAddr::from(base), level)?;
517 if level != last_level {
518 warn!(
519 "level doesn't match! level:{:?} last_level:{:?}",
520 level, last_level
521 );
522 }
523 for entry in entries_iter {
524 if entry0_ipa < base {
526 entry0_ipa += map_size;
527 continue;
528 }
529 let s2tte = S2TTE::new(entry.pte());
530 if s2tte.is_live(level) {
531 return Ok(addr);
532 }
533 addr += map_size;
534 }
535 Ok(addr)
536}
537
538pub fn fold(rd: &Rd, ipa: usize, level: usize) -> Result<usize, Error> {
539 let is_protected_ipa = rd.addr_in_par(ipa);
540 let (fold_s2tte, _last_level) = S2TTE::get_s2tte(rd, ipa, level, Error::RmiErrorRtt(level))?;
541 let (parent_s2tte, parent_level) = S2TTE::get_s2tte(rd, ipa, level - 1, Error::RmiErrorInput)?;
542 if parent_level < (level - 1) || !parent_s2tte.is_table(level - 1) {
543 return Err(Error::RmiErrorRtt(parent_level));
544 }
545 if fold_s2tte.is_table(level) {
547 warn!("Trying to fold which points another RTT");
548 return Err(Error::RmiErrorRtt(level));
549 }
550
551 let rtt_addr = parent_s2tte.get_masked(S2TTE::ADDR_TBL_OR_PAGE);
552 let mut g_rtt = get_granule_if!(rtt_addr as usize, GranuleState::RTT)?;
553
554 let binding = rd.s2_table();
557 let mut binding = binding.lock();
558 let (mut entries_iter, _) = binding.entries(GuestPhysAddr::from(ipa), level)?;
559 if !S2TTE::is_homogeneous(&mut entries_iter, level) {
560 return Err(Error::RmiErrorRtt(level));
561 }
562 let mut pa: u64 = 0;
563 let mut attr = fold_s2tte.get_masked(S2TTE::NS);
564 let hipas = fold_s2tte.get_masked(S2TTE::HIPAS);
565 let mut ripas = 0;
566 let mut desc_type = 0;
567
568 if fold_s2tte.get_masked_value(S2TTE::HIPAS) == hipas::ASSIGNED {
569 pa = fold_s2tte.addr_as_block(level).into();
570 attr |= fold_s2tte.get_masked(TTE_ATTR_MASK);
571 }
572 if is_protected_ipa {
573 ripas = fold_s2tte.get_masked(S2TTE::RIPAS);
574 }
575 if fold_s2tte.is_assigned_ram(level) || fold_s2tte.is_assigned_ns(level) {
576 desc_type = desc_type::L012_BLOCK;
577 }
578
579 let parent_s2tte = pa | attr | hipas | ripas | desc_type;
580 binding.ipa_to_pte_set(
581 GuestPhysAddr::from(ipa),
582 level - 1,
583 parent_s2tte,
584 Tlbi::BREAKDOWN(rd.id()),
585 )?;
586 set_granule(&mut g_rtt, GranuleState::Delegated)?;
588 Ok(rtt_addr as usize)
589}