2020-03-24 10:33:23 -04:00
|
|
|
const std = @import("std");
|
2021-10-04 23:47:27 -07:00
|
|
|
const builtin = @import("builtin");
|
|
|
|
|
const arch = builtin.cpu.arch;
|
|
|
|
|
const os = builtin.os.tag;
|
2022-06-10 10:25:59 +02:00
|
|
|
pub const panic = @import("common.zig").panic;
|
2020-03-24 10:33:23 -04:00
|
|
|
|
|
|
|
|
// Ported from llvm-project d32170dbd5b0d54436537b6b75beaf44324e0c28
|
|
|
|
|
|
|
|
|
|
// The compiler generates calls to __clear_cache() when creating
|
|
|
|
|
// trampoline functions on the stack for use with nested functions.
|
|
|
|
|
// It is expected to invalidate the instruction cache for the
|
|
|
|
|
// specified range.
|
|
|
|
|
|
2022-06-10 10:25:59 +02:00
|
|
|
comptime {
|
2023-05-29 05:07:17 +01:00
|
|
|
_ = &clear_cache;
|
2022-06-10 10:25:59 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn clear_cache(start: usize, end: usize) callconv(.C) void {
|
2020-03-24 10:33:23 -04:00
|
|
|
const x86 = switch (arch) {
|
2022-10-07 21:24:44 +03:30
|
|
|
.x86, .x86_64 => true,
|
2020-03-24 10:33:23 -04:00
|
|
|
else => false,
|
|
|
|
|
};
|
|
|
|
|
const arm32 = switch (arch) {
|
|
|
|
|
.arm, .armeb, .thumb, .thumbeb => true,
|
|
|
|
|
else => false,
|
|
|
|
|
};
|
|
|
|
|
const arm64 = switch (arch) {
|
std.Target.Cpu.Arch: Remove the `aarch64_32` tag.
This is a misfeature that we inherited from LLVM:
* https://reviews.llvm.org/D61259
* https://reviews.llvm.org/D61939
(`aarch64_32` and `arm64_32` are equivalent.)
I truly have no idea why this triple passed review in LLVM. It is, to date, the
*only* tag in the architecture component that is not, in fact, an architecture.
In reality, it is just an ILP32 ABI for AArch64 (*not* AArch32).
The triples that use `aarch64_32` look like `aarch64_32-apple-watchos`. Yes,
that triple is exactly what you think; it has no ABI component. They really,
seriously did this.
Since only Apple could come up with silliness like this, it should come as no
surprise that no one else uses `aarch64_32`. Later on, a GNU ILP32 ABI for
AArch64 was developed, and support was added to LLVM:
* https://reviews.llvm.org/D94143
* https://reviews.llvm.org/D104931
Here, sanity seems to have prevailed, and a triple using this ABI looks like
`aarch64-linux-gnu_ilp32` as you would expect.
As can be seen from the diffs in this commit, there was plenty of confusion
throughout the Zig codebase about what exactly `aarch64_32` was. So let's just
remove it. In its place, we'll use `aarch64-watchos-ilp32`,
`aarch64-linux-gnuilp32`, and so on. We'll then translate these appropriately
when talking to LLVM. Hence, this commit adds the `ilp32` ABI tag (we already
have `gnuilp32`).
2024-07-27 03:52:19 +02:00
|
|
|
.aarch64, .aarch64_be => true,
|
2020-03-24 10:33:23 -04:00
|
|
|
else => false,
|
|
|
|
|
};
|
2024-10-16 01:05:27 +02:00
|
|
|
const loongarch = switch (arch) {
|
|
|
|
|
.loongarch32,
|
|
|
|
|
.loongarch64,
|
|
|
|
|
=> true,
|
2024-08-10 16:24:01 +08:00
|
|
|
else => false,
|
|
|
|
|
};
|
2020-03-24 10:33:23 -04:00
|
|
|
const mips = switch (arch) {
|
|
|
|
|
.mips, .mipsel, .mips64, .mips64el => true,
|
|
|
|
|
else => false,
|
|
|
|
|
};
|
2020-03-31 12:40:28 +02:00
|
|
|
const riscv = switch (arch) {
|
|
|
|
|
.riscv32, .riscv64 => true,
|
|
|
|
|
else => false,
|
|
|
|
|
};
|
2020-03-24 10:33:23 -04:00
|
|
|
const powerpc64 = switch (arch) {
|
|
|
|
|
.powerpc64, .powerpc64le => true,
|
|
|
|
|
else => false,
|
|
|
|
|
};
|
|
|
|
|
const sparc = switch (arch) {
|
std.Target: Remove `sparcel` architecture tag.
What is `sparcel`, you might ask? Good question!
If you take a peek in the SPARC v8 manual, §2.2, it is quite explicit that SPARC
v8 is a big-endian architecture. No little-endian or mixed-endian support to be
found here.
On the other hand, the SPARC v9 manual, in §3.2.1.2, states that it has support
for mixed-endian operation, with big-endian mode being the default.
Ok, so `sparcel` must just be referring to SPARC v9 running in little-endian
mode, surely?
Nope:
* https://github.com/llvm/llvm-project/blob/40b4fd7a3e81d32b29364a1b15337bcf817659c0/llvm/lib/Target/Sparc/SparcTargetMachine.cpp#L226
* https://github.com/llvm/llvm-project/blob/40b4fd7a3e81d32b29364a1b15337bcf817659c0/llvm/lib/Target/Sparc/SparcTargetMachine.cpp#L104
So, `sparcel` in LLVM is referring to some sort of fantastical little-endian
SPARC v8 architecture. I've scoured the internet and I can find absolutely no
evidence that such a thing exists or has ever existed. In fact, I can find no
evidence that a little-endian implementation of SPARC v9 ever existed, either.
Or any SPARC version, actually!
The support was added here: https://reviews.llvm.org/D8741
Notably, there is no mention whatsoever of what CPU this might be referring to,
and no justification given for the "but some are little" comment added in the
patch.
My best guess is that this might have been some private exercise in creating a
little-endian version of SPARC that never saw the light of day. Given that SPARC
v8 explicitly doesn't support little-endian operation (let alone little-endian
instruction encoding!), and no CPU is known to be implemented as such, I think
it's very reasonable for us to just remove this support.
2024-07-30 02:31:25 +02:00
|
|
|
.sparc, .sparc64 => true,
|
2020-03-24 10:33:23 -04:00
|
|
|
else => false,
|
|
|
|
|
};
|
|
|
|
|
const apple = switch (os) {
|
2024-05-09 15:04:13 +02:00
|
|
|
.ios, .macos, .watchos, .tvos, .visionos => true,
|
2020-03-24 10:33:23 -04:00
|
|
|
else => false,
|
|
|
|
|
};
|
|
|
|
|
if (x86) {
|
|
|
|
|
// Intel processors have a unified instruction and data cache
|
|
|
|
|
// so there is nothing to do
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (os == .windows and (arm32 or arm64)) {
|
2020-03-31 10:48:48 -04:00
|
|
|
// TODO
|
2020-03-24 10:33:23 -04:00
|
|
|
// FlushInstructionCache(GetCurrentProcess(), start, end - start);
|
2020-03-31 10:48:48 -04:00
|
|
|
// exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (arm32 and !apple) {
|
2020-03-31 12:40:28 +02:00
|
|
|
switch (os) {
|
|
|
|
|
.freebsd, .netbsd => {
|
|
|
|
|
var arg = arm_sync_icache_args{
|
|
|
|
|
.addr = start,
|
|
|
|
|
.len = end - start,
|
|
|
|
|
};
|
2023-06-15 13:14:16 +06:00
|
|
|
const result = sysarch(ARM_SYNC_ICACHE, @intFromPtr(&arg));
|
2020-03-31 12:40:28 +02:00
|
|
|
std.debug.assert(result == 0);
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-31 12:40:28 +02:00
|
|
|
},
|
|
|
|
|
.linux => {
|
|
|
|
|
const result = std.os.linux.syscall3(.cacheflush, start, end, 0);
|
|
|
|
|
std.debug.assert(result == 0);
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-31 12:40:28 +02:00
|
|
|
},
|
2020-03-31 10:48:48 -04:00
|
|
|
else => {},
|
2020-03-31 00:19:19 +02:00
|
|
|
}
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (os == .linux and mips) {
|
2020-03-31 12:40:28 +02:00
|
|
|
const flags = 3; // ICACHE | DCACHE
|
2020-03-31 10:48:48 -04:00
|
|
|
const result = std.os.linux.syscall3(.cacheflush, start, end - start, flags);
|
2020-03-31 12:40:28 +02:00
|
|
|
std.debug.assert(result == 0);
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (mips and os == .openbsd) {
|
2020-03-31 10:48:48 -04:00
|
|
|
// TODO
|
2020-03-24 10:33:23 -04:00
|
|
|
//cacheflush(start, (uintptr_t)end - (uintptr_t)start, BCACHE);
|
2020-03-31 10:48:48 -04:00
|
|
|
// exportIt();
|
2020-03-31 12:40:28 +02:00
|
|
|
} else if (os == .linux and riscv) {
|
2020-03-31 10:48:48 -04:00
|
|
|
const result = std.os.linux.syscall3(.riscv_flush_icache, start, end - start, 0);
|
2020-03-31 12:40:28 +02:00
|
|
|
std.debug.assert(result == 0);
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (arm64 and !apple) {
|
|
|
|
|
// Get Cache Type Info.
|
|
|
|
|
// TODO memoize this?
|
|
|
|
|
var ctr_el0: u64 = 0;
|
|
|
|
|
asm volatile (
|
|
|
|
|
\\mrs %[x], ctr_el0
|
|
|
|
|
\\
|
2021-08-29 11:57:32 +02:00
|
|
|
: [x] "=r" (ctr_el0),
|
2020-03-24 10:33:23 -04:00
|
|
|
);
|
|
|
|
|
// The DC and IC instructions must use 64-bit registers so we don't use
|
|
|
|
|
// uintptr_t in case this runs in an IPL32 environment.
|
|
|
|
|
var addr: u64 = undefined;
|
|
|
|
|
// If CTR_EL0.IDC is set, data cache cleaning to the point of unification
|
|
|
|
|
// is not required for instruction to data coherence.
|
|
|
|
|
if (((ctr_el0 >> 28) & 0x1) == 0x0) {
|
2023-08-19 03:32:47 -04:00
|
|
|
const dcache_line_size = @as(usize, 4) << @intCast((ctr_el0 >> 16) & 15);
|
2020-03-24 10:33:23 -04:00
|
|
|
addr = start & ~(dcache_line_size - 1);
|
|
|
|
|
while (addr < end) : (addr += dcache_line_size) {
|
|
|
|
|
asm volatile ("dc cvau, %[addr]"
|
|
|
|
|
:
|
2021-08-29 11:57:32 +02:00
|
|
|
: [addr] "r" (addr),
|
2020-03-24 10:33:23 -04:00
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
asm volatile ("dsb ish");
|
|
|
|
|
// If CTR_EL0.DIC is set, instruction cache invalidation to the point of
|
|
|
|
|
// unification is not required for instruction to data coherence.
|
|
|
|
|
if (((ctr_el0 >> 29) & 0x1) == 0x0) {
|
2023-08-19 03:32:47 -04:00
|
|
|
const icache_line_size = @as(usize, 4) << @intCast((ctr_el0 >> 0) & 15);
|
2020-03-24 10:33:23 -04:00
|
|
|
addr = start & ~(icache_line_size - 1);
|
|
|
|
|
while (addr < end) : (addr += icache_line_size) {
|
|
|
|
|
asm volatile ("ic ivau, %[addr]"
|
|
|
|
|
:
|
2021-08-29 11:57:32 +02:00
|
|
|
: [addr] "r" (addr),
|
2020-03-24 10:33:23 -04:00
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
asm volatile ("isb sy");
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (powerpc64) {
|
2020-03-31 10:48:48 -04:00
|
|
|
// TODO
|
2020-03-24 10:33:23 -04:00
|
|
|
//const size_t line_size = 32;
|
|
|
|
|
//const size_t len = (uintptr_t)end - (uintptr_t)start;
|
|
|
|
|
//
|
|
|
|
|
//const uintptr_t mask = ~(line_size - 1);
|
|
|
|
|
//const uintptr_t start_line = ((uintptr_t)start) & mask;
|
|
|
|
|
//const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask;
|
|
|
|
|
//
|
|
|
|
|
//for (uintptr_t line = start_line; line < end_line; line += line_size)
|
|
|
|
|
// __asm__ volatile("dcbf 0, %0" : : "r"(line));
|
|
|
|
|
//__asm__ volatile("sync");
|
|
|
|
|
//
|
|
|
|
|
//for (uintptr_t line = start_line; line < end_line; line += line_size)
|
|
|
|
|
// __asm__ volatile("icbi 0, %0" : : "r"(line));
|
|
|
|
|
//__asm__ volatile("isync");
|
2020-03-31 10:48:48 -04:00
|
|
|
// exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (sparc) {
|
2020-03-31 10:48:48 -04:00
|
|
|
// TODO
|
2020-03-24 10:33:23 -04:00
|
|
|
//const size_t dword_size = 8;
|
|
|
|
|
//const size_t len = (uintptr_t)end - (uintptr_t)start;
|
|
|
|
|
//
|
|
|
|
|
//const uintptr_t mask = ~(dword_size - 1);
|
|
|
|
|
//const uintptr_t start_dword = ((uintptr_t)start) & mask;
|
|
|
|
|
//const uintptr_t end_dword = ((uintptr_t)start + len + dword_size - 1) & mask;
|
|
|
|
|
//
|
|
|
|
|
//for (uintptr_t dword = start_dword; dword < end_dword; dword += dword_size)
|
|
|
|
|
// __asm__ volatile("flush %0" : : "r"(dword));
|
2020-03-31 10:48:48 -04:00
|
|
|
// exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
} else if (apple) {
|
2020-03-24 11:51:13 -04:00
|
|
|
// On Darwin, sys_icache_invalidate() provides this functionality
|
|
|
|
|
sys_icache_invalidate(start, end - start);
|
2020-03-31 10:48:48 -04:00
|
|
|
exportIt();
|
2024-10-16 01:05:27 +02:00
|
|
|
} else if (os == .linux and loongarch) {
|
2024-08-10 16:24:01 +08:00
|
|
|
// See: https://github.com/llvm/llvm-project/blob/cf54cae26b65fc3201eff7200ffb9b0c9e8f9a13/compiler-rt/lib/builtins/clear_cache.c#L94-L95
|
|
|
|
|
asm volatile (
|
|
|
|
|
\\ ibar 0
|
|
|
|
|
);
|
|
|
|
|
exportIt();
|
2020-03-24 10:33:23 -04:00
|
|
|
}
|
|
|
|
|
}
|
2020-03-24 11:51:13 -04:00
|
|
|
|
2024-02-18 21:25:23 -08:00
|
|
|
const linkage = if (builtin.is_test) std.builtin.GlobalLinkage.internal else std.builtin.GlobalLinkage.weak;
|
2020-03-31 10:48:48 -04:00
|
|
|
|
2020-03-31 11:17:40 -04:00
|
|
|
fn exportIt() void {
|
2024-08-26 03:49:36 +01:00
|
|
|
@export(&clear_cache, .{ .name = "__clear_cache", .linkage = linkage });
|
2020-03-31 10:48:48 -04:00
|
|
|
}
|
|
|
|
|
|
2020-03-24 11:51:13 -04:00
|
|
|
// Darwin-only
|
|
|
|
|
extern fn sys_icache_invalidate(start: usize, len: usize) void;
|
2020-03-31 12:40:28 +02:00
|
|
|
// BSD-only
|
|
|
|
|
const arm_sync_icache_args = extern struct {
|
|
|
|
|
addr: usize, // Virtual start address
|
|
|
|
|
len: usize, // Region size
|
|
|
|
|
};
|
|
|
|
|
const ARM_SYNC_ICACHE = 0;
|
|
|
|
|
extern "c" fn sysarch(number: i32, args: usize) i32;
|
