LIR definition and Cranelift backend

Cargo.toml

@@ -1,19 +1,20 @@
 [package]
-name = "holyjit_pkg"
+name = "holyjit"
 version = "0.0.0"
 authors = [ "Nicolas B. Pierron <nicolas.b.pierron@nbp.name>" ]
 
 [workspace]
 members = [
+  "lir",
+  "codegen",
   "lib",
   "plugin",
   "bin"
 ]
 
 [dependencies]
 holyjit_lib = { path = "./lib" }
-holyjit_plugin = { path = "./plugin" }
-holyjit = { path = "./bin" }
+holyjit_bin = { path = "./bin" }
 
 # Before running any tests or examples, make sure to set the RUSTC_WRAPPER as
 # follow:

README.md

@@ -108,30 +108,32 @@ currently at a prototype stage, and most of the code & dependencies present
 today were made only as a proof of concept and not as a definitive
 implementation design.
 
-## HolyJit Roadmap for 0.0.0
+## [HolyJit Roadmap](https://github.com/nbp/holyjit/milestones)
 
-The current goal is to make a proof of concept which highlights the main
-feature, i.e. being trivial to integrate into an existing code base and to
-have a running JIT compiler.
+### v0.2.0: Unroll Interpreter loops (future)
 
-As of today, HolyJit contains a draft of what the interface might look like,
-and is able to generate code for the example present in the repository.
+Interpreters are most of the time written as a loop over a byte-code. This goal
+is about unrolling interpreter loops at JIT-compile time based on a limited set
+of guarded/assumed constant inputs.
+
+At this stage: We should have a macro interfaces to annotate variables which
+should be used as guarded or assumed constants. We should remove the dispatch
+cost of an interpreter.
+
+### v0.1.0: Cranelift + LIR (in progress)
+
+The goal is to replace the quick and dirty solution by a solution with a proper
+code generator and a proper intermediate representation.
 
-- [x] Create Rust library
-  - [x] Allocate pages and map them as executable.
-  - [x] Add a way to call either a dynamically compiled function or a
-        statically compiled function.
-  - [x] Add a `jit!` macro, to make calls transparent from the usage point of
-        view.
-  - [x] Create a JitContext class, and use it to request JIT compiled code.
-  - [x] Create a graph representation.
-  - [x] Consume the graph to generate code.
-
-- [x] Create a MIR plugin
-  - [x] Detect locations which have to be patched.
-  - [x] Find functions which have to be converted.
-  - [x] Inject a generated vector in the binary content.
-  - [x] Inject static variables as a tuple.
-  - [x] Collect static variable references.
-  - [x] Convert the MIR (from the Rust compiler) to the library graph
-        representation.
+At this stage, the LIR (Low-level Intermediate Represenation) should be
+documented, generated by the Rustc driver and consumed to generate Cranelift IR.
+The Cranelift IR should be used to generate machine code.
+
+### v0.0.0: Proof of concept
+
+This goal is to make a proof of concept which highlights the main feature, i.e.
+being trivial to integrate into an existing code base and to have a running
+dummy-JIT compiler.
+
+At this stage, HolyJit contains a draft of what the interface might look like,
+and is able to generate code for the example present in the repository.

bin/Cargo.toml

@@ -1,7 +1,11 @@
 [package]
-name = "holyjit"
+name = "holyjit_bin"
 version = "0.0.0"
 authors = ["Nicolas B. Pierron <nicolas.b.pierron@gmail.com>"]
 
 [dependencies]
 holyjit_plugin = { path = "../plugin" }
+
+[[bin]]
+name = "holyjit"
+path = "src/main.rs"

bin/src/main.rs

@@ -183,6 +183,11 @@ pub fn main() {
             .collect()
     };
 
+    // TODO: HolyJIT does not yet supports unwinding as this would require
+    // generating ELF code with eh_frame section, to encode the block address
+    // where to resume in case of unwinding. TODO: In the mean time we should
+    // enforce usage of "-C panic=abort", and tell the user about it to avoid
+    // surprises.
     let holyjit_enabled = true;
     let mut holyjit_cc = HolyJitCompilerCalls::new(holyjit_enabled);
     rustc_driver::run(move || {

codegen/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "holyjit_codegen"
+version = "0.0.0"
+authors = [ "Nicolas B. Pierron <nicolas.b.pierron@nbp.name>" ]
+
+[dependencies]
+mmap = "*"
+region = "0.3"
+cranelift-codegen = "0.18"
+cranelift-native = "0.18"
+cranelift-frontend = "0.18"
+holyjit_lir = { path = "../lir" }
+
+[lib]
+name = "holyjit_codegen"
+path = "src/lib.rs"
+

codegen/src/error.rs

@@ -0,0 +1,51 @@
+use codegen::verifier::VerifierError;
+use codegen::CodegenError;
+use mmap::MapError;
+use region;
+
+/// Simple wrapper type to wrap any lowerring error.
+pub type LowerResult<T> = Result<T, LowerError>;
+
+/// Lowering error can be any error involve either during code generation, or
+/// during the allocation of code to memory.
+#[derive(Debug)]
+pub enum LowerError {
+    CodeGen(CodegenError),
+    Verifier(VerifierError),
+    Map(MapError),
+    Protect,
+    UnitIsNotAFunction,
+    ComplexTypeNotLowered,
+    NoFixPointForVarTypes,
+}
+
+// TODO: impl Error for LowerError
+// TODO: impl Display for LowerError
+
+impl From<CodegenError> for LowerError {
+    /// Implictly convert Cranelift codegen errors into LowerError.
+    fn from(err: CodegenError) -> LowerError {
+        LowerError::CodeGen(err)
+    }
+}
+
+impl From<VerifierError> for LowerError {
+    /// Implictly convert Cranelift codegen errors into LowerError.
+    fn from(err: VerifierError) -> LowerError {
+        LowerError::Verifier(err)
+    }
+}
+
+impl From<MapError> for LowerError {
+    /// Implictly convert mmap errors into LowerError.
+    fn from(err: MapError) -> LowerError {
+        LowerError::Map(err)
+    }
+}
+
+impl From<region::Error> for LowerError {
+    /// Implictly convert region errors into LowerError.
+    fn from(_err: region::Error) -> LowerError {
+        LowerError::Protect
+    }
+}

codegen/src/exec_alloc.rs

@@ -0,0 +1,70 @@
+/// This module is used as part of the lowering to allocate memory pages, copy
+/// the code into them, patch the code based on relocation informations, and map
+/// these pages as executable.
+use mmap::*;
+use region;
+use region::Protection;
+use error::LowerResult;
+use codegen::binemit::{RelocSink, CodeOffset, Reloc, Addend};
+use codegen::ir;
+
+/// This structure hold the code while it is being written down in memory, and
+/// mutated based on relocation informations provided by Cranelift.
+pub struct WrittableCode {
+    page: MemoryMap,
+}
+
+/// This structure hold the code in a executable-only state (readable if
+/// needed), and with no way to convert it back to a writtable format. The
+/// implicit Drop trait implemented for it will discard the executable code as
+/// soon as there is no more references to it.
+pub struct ExecutableCode {
+    page: MemoryMap,
+}
+
+impl WrittableCode {
+    /// Allocate a new set of pages in which we can copy the result of a
+    /// compilation into.
+    pub fn with_capacity(size: usize) -> LowerResult<WrittableCode> {
+        Ok(WrittableCode {
+            page: MemoryMap::new(size, &[ MapOption::MapWritable ])?,
+        })
+    }
+
+    /// Get a mutable pointer to write the code into.
+    pub fn as_ptr(&mut self) -> *mut u8 {
+        // Note: Based on mmap.data(), we only require a &self argument.
+        self.page.data()
+    }
+
+    /// Map the pages as executable, and replace the write permission by an
+    /// executable permission. Return the executable code.
+    pub fn make_executable(self, _reloc: NullRelocSink, _trap: NullTrapSink) -> LowerResult<ExecutableCode> {
+        let WrittableCode { page } = self;
+        unsafe { region::protect(page.data(), page.len(), Protection::ReadExecute)?; }
+        Ok(ExecutableCode { page })
+    }
+}
+
+impl ExecutableCode {
+    pub fn as_ptr(&self) -> *const u8 {
+        self.page.data()
+    }
+}
+
+pub use codegen::binemit::NullTrapSink;
+pub struct NullRelocSink {}
+
+impl RelocSink for NullRelocSink {
+    fn reloc_ebb(&mut self, _offset: CodeOffset, _reloc: Reloc, _ebb_offset: CodeOffset) {
+        unimplemented!();
+    }
+
+    fn reloc_external(&mut self, _offset: CodeOffset, _reloc: Reloc, _name: &ir::ExternalName, _addend: Addend) {
+        unimplemented!();
+    }
+
+    fn reloc_jt(&mut self, _offset: CodeOffset, _reloc: Reloc, _jt: ir::JumpTable) {
+        unimplemented!();
+    }
+}

codegen/src/lib.rs

@@ -0,0 +1,81 @@
+extern crate mmap;
+extern crate region;
+extern crate cranelift_codegen as codegen;
+extern crate cranelift_frontend as frontend;
+extern crate cranelift_native as native;
+extern crate holyjit_lir as lir;
+
+mod lower;
+mod exec_alloc;
+pub mod error;
+
+use codegen::settings::Configurable;
+use exec_alloc::{WrittableCode, ExecutableCode};
+
+use lir::unit;
+use lir::context;
+
+/// This is a reusable code generator, which is used to compile a LIR Unit to
+/// machine code.
+pub struct CodeGenerator {
+    ctx: codegen::Context,
+    isa: Box<codegen::isa::TargetIsa>,
+}
+
+/// Result of a compiled lir::Unit.
+pub struct JitCode {
+    code: ExecutableCode,
+}
+
+impl CodeGenerator {
+    /// Create a lowering (code generator and executable page allocation)
+    /// context for the architecture on which this code is running.
+    pub fn new() -> Self {
+        // Extract configuration builders tuned for the architecture on which
+        // this code is running.
+        let (mut settings_bld, isa_bld) = native::builders().unwrap();
+        // Optimize for compilation time.
+        settings_bld.set("opt_level", "fastest").unwrap();
+        // Check the emitted Cranelift IR.
+        settings_bld.set("enable_verifier", "1").unwrap();
+        // Use Rust call convention.
+        settings_bld.set("call_conv", "system_v").unwrap();
+        // Generate position independent code.
+        settings_bld.set("is_pic", "1").unwrap();
+        // No need to generate a single return per function.
+        settings_bld.set("return_at_end", "0").unwrap();
+        // Do not attempt to avoid trap on divisions. (TODO: double check that
+        // this is what rust expects)
+        settings_bld.set("avoid_div_traps", "0").unwrap();
+        let flags = codegen::settings::Flags::new(settings_bld);
+        let isa = isa_bld.finish(flags);
+
+        Self {
+            ctx: codegen::Context::new(),
+            isa: isa,
+        }
+    }
+
+    /// Given an HolyJIT LIR Unit, convert it to a Cranelift function in order
+    /// to generate the corresponding bytes, then allocate memory pages and map
+    /// them as executable.
+    pub fn compile(&mut self, lir_ctx: &context::Context, unit: &unit::Unit) -> error::LowerResult<JitCode> {
+        let &mut CodeGenerator { ref mut ctx, ref isa, .. } = self;
+        ctx.func = lower::convert(isa.as_ref(), lir_ctx, unit)?;
+        let mut reloc_sink = exec_alloc::NullRelocSink {};
+        let mut trap_sink = exec_alloc::NullTrapSink {};
+        let code_size = ctx.compile(isa.as_ref())?;
+        let mut code = WrittableCode::with_capacity(code_size as usize)?;
+        unsafe {
+            ctx.emit_to_memory(isa.as_ref(), code.as_ptr(), &mut reloc_sink, &mut trap_sink);
+        }
+        let code = code.make_executable(reloc_sink, trap_sink)?;
+        Ok(JitCode { code })
+    }
+}
+
+impl JitCode {
+    pub unsafe fn as_ptr(&self) -> *const u8 {
+        self.code.as_ptr()
+    }
+}