new: add ui file_transfer

build.rs

@@ -26,7 +26,7 @@ fn visit_dirs(dir: &Path, output_buffer: &mut Vec<u8>) -> io::Result<()> {
                     .replace("\\", "/");
                 writeln!(
                     output_buffer,
-                    "    (\"{}\", include_str!(\"{}\")),",
+                    "    (\"{}\", include_bytes!(\"{}\")),",
                     path_str,
                     path_str_from_includes,
                 )?;
@@ -59,10 +59,10 @@ fn add_branch_name(repo: &git2::Repository) -> anyhow::Result<()> {
 
 fn main() -> anyhow::Result<()> {
     let mut output_buffer = Vec::new();
-    writeln!(&mut output_buffer, "const PATH_TO_CONTENT: &[(&str, &str)] = &[")?;
+    writeln!(&mut output_buffer, "const PATH_TO_CONTENT: &[(&str, &[u8])] = &[")?;
     writeln!(
-        output_buffer,
-        "    (\"{}\", include_str!(\"{}\")),",
+        &mut output_buffer,
+        "    (\"{}\", include_bytes!(\"{}\")),",
         "componentize.mjs",
         "componentize.mjs",
     )?;

src/new/mod.rs

@@ -1,6 +1,6 @@
 use std::{fs, path::{PathBuf, Path}, collections::HashMap};
 
-use tracing::instrument;
+use tracing::{info, instrument};
 
 include!("includes.rs");
 
@@ -150,15 +150,22 @@ pub fn execute(
         ui_infix,
         template.to_string(),
     );
-    let mut path_to_content: HashMap<String, String> = PATH_TO_CONTENT
+    let mut path_to_content: HashMap<String, Vec<u8>> = PATH_TO_CONTENT
         .iter()
         .filter_map(|(k, v)| {
             k
                 .strip_prefix(&template_prefix)
                 .or_else(|| k.strip_prefix(&ui_prefix))
                 .and_then(|stripped| {
                     let key = replace_vars(stripped, &package_name, &publisher);
-                    let val = replace_vars(v, &package_name, &publisher);
+                    let val = match std::str::from_utf8(v) {
+                        Err(_) => v.to_vec(),
+                        Ok(v) => {
+                            replace_vars(v, &package_name, &publisher)
+                                .as_bytes()
+                                .to_vec()
+                        },
+                    };
                     Some((key, val))
                 })
         })
@@ -171,11 +178,17 @@ pub fn execute(
             template.to_string(),
         ));
     }
+
+    // add componentize.mjs
     match language {
         Language::Javascript => {
             path_to_content.insert(
                 format!("{}/{}", package_name, PATH_TO_CONTENT[0].0),
-                replace_vars(PATH_TO_CONTENT[0].1, &package_name, &publisher),
+                replace_vars(
+                    std::str::from_utf8(PATH_TO_CONTENT[0].1).unwrap(),
+                    &package_name,
+                    &publisher,
+                ).as_bytes().to_vec(),
             );
         },
         _ => {},
@@ -192,6 +205,6 @@ pub fn execute(
         fs::write(new_dir.join(path), content)?;
     }
 
-    tracing::info!("Template directory created successfully at {:?}.", new_dir);
+    info!("Template directory created successfully at {:?}.", new_dir);
     Ok(())
 }

src/new/templates/rust/ui/file_transfer/.gitignore

@@ -0,0 +1,14 @@
+target
+*/target/
+pkg/*.wasm
+pkg/ui
+*.swp
+*.swo
+*/wasi_snapshot_preview1.wasm
+*/wit/
+*/process_env
+ui/dist
+ui/node_modules
+node_modules
+.vscode
+*/**/yarn-error.log

src/new/templates/rust/ui/file_transfer/pkg/manifest.json

@@ -0,0 +1,15 @@
+[
+    {
+        "process_name": "{package_name}",
+        "process_wasm_path": "/{package_name}.wasm",
+        "on_exit": "Restart",
+        "request_networking": true,
+        "request_capabilities": [
+            "net:distro:sys",
+            "vfs:distro:sys",
+            "http_server:distro:sys"
+        ],
+        "grant_capabilities": [],
+        "public": true
+    }
+]

src/new/templates/rust/ui/file_transfer/pkg/metadata.json

@@ -0,0 +1,5 @@
+{
+    "package": "{package_name}",
+    "publisher": "{publisher}",
+    "version": [0, 1, 0]
+}

src/new/templates/rust/ui/file_transfer/worker/Cargo.toml_

@@ -0,0 +1,23 @@
+[package]
+name = "worker"
+version = "0.1.0"
+edition = "2021"
+
+[profile.release]
+panic = "abort"
+opt-level = "s"
+lto = true
+
+[dependencies]
+anyhow = "1.0"
+bincode = "1.3.3"
+kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib", tag = "v0.5.9-alpha"}
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+wit-bindgen = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "efcc759" }
+
+[lib]
+crate-type = ["cdylib"]
+
+[package.metadata.component]
+package = "kinode:process"

src/new/templates/rust/ui/file_transfer/worker/src/lib.rs

@@ -0,0 +1,199 @@
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+use kinode_process_lib::{
+    await_message, get_blob, println,
+    vfs::{open_dir, open_file, Directory, File, SeekFrom},
+    Address, Message, ProcessId, Request, Response,
+};
+
+wit_bindgen::generate!({
+    path: "wit",
+    world: "process",
+    exports: {
+        world: Component,
+    },
+});
+
+const CHUNK_SIZE: u64 = 1048576; // 1MB
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum WorkerRequest {
+    Initialize {
+        name: String,
+        target_worker: Option<Address>,
+    },
+    Chunk {
+        name: String,
+        offset: u64,
+        length: u64,
+    },
+    Size(u64),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum TransferRequest {
+    Progress { name: String, progress: u64 },
+}
+
+fn handle_message(
+    our: &Address,
+    file: &mut Option<File>,
+    files_dir: &Directory,
+    size: &mut Option<u64>,
+) -> anyhow::Result<bool> {
+    let message = await_message()?;
+
+    match message {
+        Message::Request { ref body, .. } => {
+            let request = serde_json::from_slice::<WorkerRequest>(body)?;
+
+            match request {
+                WorkerRequest::Initialize {
+                    name,
+                    target_worker,
+                } => {
+                    // initialize command from main process,
+                    // sets up worker, matches on if it's a sender or receiver.
+                    // target_worker = None, we are receiver, else sender.
+
+                    // open/create empty file in both cases.
+                    let mut active_file =
+                        open_file(&format!("{}/{}", files_dir.path, &name), true)?;
+
+                    match target_worker {
+                        Some(target_worker) => {
+                            // we have a target, chunk the data, and send it.
+                            let size = active_file.metadata()?.len;
+                            let num_chunks = (size as f64 / CHUNK_SIZE as f64).ceil() as u64;
+
+                            // give the receiving worker a size request so it can track it's progress!
+                            Request::new()
+                                .body(serde_json::to_vec(&WorkerRequest::Size(size))?)
+                                .target(target_worker.clone())
+                                .send()?;
+
+                            active_file.seek(SeekFrom::Start(0))?;
+
+                            for i in 0..num_chunks {
+                                let offset = i * CHUNK_SIZE;
+                                let length = CHUNK_SIZE.min(size - offset);
+
+                                let mut buffer = vec![0; length as usize];
+                                active_file.read_at(&mut buffer)?;
+
+                                Request::new()
+                                    .body(serde_json::to_vec(&WorkerRequest::Chunk {
+                                        name: name.clone(),
+                                        offset,
+                                        length,
+                                    })?)
+                                    .target(target_worker.clone())
+                                    .blob_bytes(buffer)
+                                    .send()?;
+                            }
+                            Response::new().body(serde_json::to_vec(&"Done")?).send()?;
+                            return Ok(true);
+                        }
+                        None => {
+                            // waiting for response, store created empty file.
+                            *file = Some(active_file);
+                            Response::new()
+                                .body(serde_json::to_vec(&"Started")?)
+                                .send()?;
+                        }
+                    }
+                }
+                // someone sending a chunk to us!
+                WorkerRequest::Chunk {
+                    name,
+                    offset,
+                    length,
+                } => {
+                    let file = match file {
+                        Some(file) => file,
+                        None => {
+                            return Err(anyhow::anyhow!(
+                                "kino_files: receive error: no file initialized"
+                            ));
+                        }
+                    };
+
+                    let bytes = match get_blob() {
+                        Some(blob) => blob.bytes,
+                        None => {
+                            return Err(anyhow::anyhow!("kino_files: receive error: no blob"));
+                        }
+                    };
+
+                    // file.seek(SeekFrom::Start(offset))?; seek not necessary if the sends come in order.
+                    file.write_all(&bytes)?;
+                    // if sender has sent us a size, give a progress update to main transfer!
+                    if let Some(size) = size {
+                        let progress = ((offset + length) as f64 / *size as f64 * 100.0) as u64;
+
+                        // send update to main process
+                        let main_app = Address {
+                            node: our.node.clone(),
+                            process: ProcessId::from_str(
+                                "kino_files:kino_files:gloriainexcelsisdeo.os",
+                            )?,
+                        };
+
+                        Request::new()
+                            .body(serde_json::to_vec(&TransferRequest::Progress {
+                                name,
+                                progress,
+                            })?)
+                            .target(&main_app)
+                            .send()?;
+
+                        if progress >= 100 {
+                            return Ok(true);
+                        }
+                    }
+                }
+                WorkerRequest::Size(incoming_size) => {
+                    *size = Some(incoming_size);
+                }
+            }
+        }
+        _ => {
+            println!("kino_files worker: got something else than request...");
+        }
+    }
+    Ok(false)
+}
+
+struct Component;
+impl Guest for Component {
+    fn init(our: String) {
+        println!("kino_files worker: begin");
+        let start = std::time::Instant::now();
+
+        let our = Address::from_str(&our).unwrap();
+
+        let drive_path = format!("{}/files", our.package_id());
+        let files_dir = open_dir(&drive_path, false).unwrap();
+
+        let mut file: Option<File> = None;
+        let mut size: Option<u64> = None;
+
+        loop {
+            match handle_message(&our, &mut file, &files_dir, &mut size) {
+                Ok(exit) => {
+                    if exit {
+                        println!(
+                            "kino_files worker done: exiting, took {:?}",
+                            start.elapsed()
+                        );
+                        break;
+                    }
+                }
+                Err(e) => {
+                    println!("kino_files: worker error: {:?}", e);
+                }
+            };
+        }
+    }
+}

src/new/templates/rust/ui/file_transfer/{package_name}/Cargo.toml_

@@ -0,0 +1,24 @@
+[package]
+name = "{package_name}"
+version = "0.1.0"
+edition = "2021"
+
+[profile.release]
+panic = "abort"
+opt-level = "s"
+lto = true
+
+[dependencies]
+anyhow = "1.0"
+bincode = "1.3.3"
+kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib", tag = "v0.5.9-alpha" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+wit-bindgen = { git = "https://github.com/bytecodealliance/wit-bindgen", rev = "efcc759" }
+multipart = "0.18.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[package.metadata.component]
+package = "kinode:process"