Add ability to create standalone binaries with qjs

Ref: https://github.com/quickjs-ng/quickjs/issues/438
Closes: https://github.com/quickjs-ng/quickjs/pull/441

.github/workflows/ci.yml

@@ -160,6 +160,11 @@ jobs:
           git submodule update --init --checkout --depth 1
           time make test262
 
+      - name: test standalone
+        run: |
+          ./build/qjs -c examples/hello.js -o hello
+          ./hello
+
   windows-msvc:
     runs-on: windows-latest
     strategy:
@@ -184,6 +189,10 @@ jobs:
           build\${{matrix.buildType}}\qjs.exe examples\test_point.js
           build\${{matrix.buildType}}\run-test262.exe -c tests.conf
           build\${{matrix.buildType}}\function_source.exe
+      - name: test standalone
+        run: |
+          build\${{matrix.buildType}}\qjs.exe -c examples\hello.js -o hello.exe
+          .\hello.exe
       - name: Set up Visual Studio shell
         uses: egor-tensin/vs-shell@v2
         with:
@@ -351,6 +360,10 @@ jobs:
     - name: test
       run: |
         make test
+    - name: test standalone
+      run: |
+        ./build/qjs -c examples/hello.js -o hello.exe
+        ./hello
   windows-mingw-shared:
     runs-on: windows-latest
     defaults:

CMakeLists.txt

@@ -259,6 +259,7 @@ target_link_libraries(qjsc qjs)
 
 add_executable(qjs_exe
     gen/repl.c
+    gen/standalone.c
     qjs.c
 )
 add_qjs_libc_if_needed(qjs_exe)

Makefile

@@ -70,6 +70,7 @@ clean:
 
 codegen: $(QJSC)
 	$(QJSC) -ss -o gen/repl.c -m repl.js
+	$(QJSC) -ss -o gen/standalone.c -m standalone.js
 	$(QJSC) -e -o gen/function_source.c tests/function_source.js
 	$(QJSC) -e -o gen/hello.c examples/hello.js
 	$(QJSC) -e -o gen/hello_module.c -m examples/hello_module.js

docs/docs/cli.md

@@ -18,10 +18,13 @@ usage: qjs [options] [file [args]]
 -m  --module       load as ES6 module (default=autodetect)
     --script       load as ES6 script (default=autodetect)
 -I  --include file include an additional file
-    --std          make 'std' and 'os' available to the loaded script
+    --std          make 'std', 'os' and 'bjson' available to script
 -T  --trace        trace memory allocation
 -d  --dump         dump the memory usage stats
 -D  --dump-flags   flags for dumping debug data (see DUMP_* defines)
+-c  --compile FILE compile the given JS file as a standalone executable
+-o  --out FILE     output file for standalone executables
+    --exe          select the executable to use as the base, defaults to the current one
     --memory-limit n       limit the memory usage to 'n' Kbytes
     --stack-size n         limit the stack size to 'n' Kbytes
     --unhandled-rejection  dump unhandled promise rejections
@@ -52,6 +55,37 @@ DUMP_ATOMS         0x40000  /* dump atoms in JS_FreeRuntime */
 DUMP_SHAPES        0x80000  /* dump shapes in JS_FreeRuntime */
 ```
 
+### Creating standalone executables
+
+With the `qjs` CLI it's possible to create standalone executables that will bundle the given JavaScript file
+alongside the binary.
+
+```
+$ qjs -c app.js -o app --exe qjs
+```
+
+The resulting `app` binary will have the same runtime dependencies as the `qjs` binary. This is acomplished
+by compiling the target JavaScript file to bytecode and adding it a copy of the executable, with a little
+trailer to help locate it.
+
+Rather than using the current executable, it's possible to use the `--exe` switch to create standalone
+executables for other platforms.
+
+No JavaScript bundling is performed, the specified JS file cannot depend on other files. A bundler such
+as `esbuild` can be used to generate an app bundle which can then be turned into the executable.
+
+```
+npx esbuild my-app/index.js \
+    --bundle \
+    --outfile=app.js \
+    --external:qjs:* \
+    --minify \
+    --target=es2023 \
+    --platform=neutral \
+    --format=esm \
+    --main-fields=main,module
+```
+
 ## `qjsc` - The QuickJS JavaScript compiler
 
 The `qjsc` executable runs the JavaScript compiler, it can generate bytecode from

qjs.c

@@ -45,11 +45,65 @@
 
 extern const uint8_t qjsc_repl[];
 extern const uint32_t qjsc_repl_size;
+extern const uint8_t qjsc_standalone[];
+extern const uint32_t qjsc_standalone_size;
+
+// Must match standalone.js
+#define TRAILER_SIZE 12
+static const char trailer_magic[] = "quickjs2";
+static const int trailer_magic_size = sizeof(trailer_magic) - 1;
+static const int trailer_size = TRAILER_SIZE;
 
 static int qjs__argc;
 static char **qjs__argv;
 
 
+static BOOL is_standalone(const char *exe)
+{
+    FILE *exe_f = fopen(exe, "rb");
+    if (!exe_f)
+        return FALSE;
+    if (fseek(exe_f, -trailer_size, SEEK_END) < 0)
+        goto fail;
+    uint8_t buf[TRAILER_SIZE];
+    if (fread(buf, 1, trailer_size, exe_f) != trailer_size)
+        goto fail;
+    fclose(exe_f);
+    return !memcmp(buf, trailer_magic, trailer_magic_size);
+fail:
+    fclose(exe_f);
+    return FALSE;
+}
+
+static JSValue load_standalone_module(JSContext *ctx)
+{
+    JSModuleDef *m;
+    JSValue obj, val;
+    obj = JS_ReadObject(ctx, qjsc_standalone, qjsc_standalone_size, JS_READ_OBJ_BYTECODE);
+    if (JS_IsException(obj))
+        goto exception;
+    assert(JS_VALUE_GET_TAG(obj) == JS_TAG_MODULE);
+    if (JS_ResolveModule(ctx, obj) < 0) {
+        JS_FreeValue(ctx, obj);
+        goto exception;
+    }
+    js_module_set_import_meta(ctx, obj, FALSE, TRUE);
+    val = JS_EvalFunction(ctx, JS_DupValue(ctx, obj));
+    val = js_std_await(ctx, val);
+
+    if (JS_IsException(val)) {
+        JS_FreeValue(ctx, obj);
+    exception:
+        js_std_dump_error(ctx);
+        exit(1);
+    }
+    JS_FreeValue(ctx, val);
+
+    m = JS_VALUE_GET_PTR(obj);
+    JS_FreeValue(ctx, obj);
+    return JS_GetModuleNamespace(ctx, m);
+}
+
 static int eval_buf(JSContext *ctx, const void *buf, int buf_len,
                     const char *filename, int eval_flags)
 {
@@ -333,6 +387,9 @@ void help(void)
            "-T  --trace        trace memory allocation\n"
            "-d  --dump         dump the memory usage stats\n"
            "-D  --dump-flags   flags for dumping debug data (see DUMP_* defines)\n"
+           "-c  --compile FILE compile the given JS file as a standalone executable\n"
+           "-o  --out FILE     output file for standalone executables\n"
+           "    --exe          select the executable to use as the base, defaults to the current one\n"
            "    --memory-limit n       limit the memory usage to 'n' Kbytes\n"
            "    --stack-size n         limit the stack size to 'n' Kbytes\n"
            "    --unhandled-rejection  dump unhandled promise rejections\n"
@@ -344,11 +401,15 @@ int main(int argc, char **argv)
 {
     JSRuntime *rt;
     JSContext *ctx;
-    JSValue ret;
+    JSValue ret = JS_UNDEFINED;
     struct trace_malloc_data trace_data = { NULL };
-    int optind;
+    int optind = 1;
+    char *compile_file = NULL;
+    char *exe = NULL;
     char *expr = NULL;
     char *dump_flags_str = NULL;
+    char *out = NULL;
+    int standalone = 0;
     int interactive = 0;
     int dump_memory = 0;
     int dump_flags = 0;
@@ -366,12 +427,16 @@ int main(int argc, char **argv)
     qjs__argc = argc;
     qjs__argv = argv;
 
+    if (is_standalone(argv[0])) {
+        standalone = 1;
+        goto start;
+    }
+
     dump_flags_str = getenv("QJS_DUMP_FLAGS");
     dump_flags = dump_flags_str ? strtol(dump_flags_str, NULL, 16) : 0;
 
     /* cannot use getopt because we want to pass the command line to
        the script */
-    optind = 1;
     while (optind < argc && *argv[optind] == '-') {
         char *arg = argv[optind] + 1;
         const char *longopt = "";
@@ -405,7 +470,7 @@ int main(int argc, char **argv)
                 if (!opt_arg) {
                     if (optind >= argc) {
                         fprintf(stderr, "qjs: missing expression for -e\n");
-                        exit(2);
+                        exit(1);
                     }
                     opt_arg = argv[optind++];
                 }
@@ -482,6 +547,39 @@ int main(int argc, char **argv)
                 stack_size = parse_limit(opt_arg);
                 break;
             }
+            if (opt == 'c' || !strcmp(longopt, "compile")) {
+                if (!opt_arg) {
+                    if (optind >= argc) {
+                        fprintf(stderr, "qjs: missing file for -c\n");
+                        exit(1);
+                    }
+                    opt_arg = argv[optind++];
+                }
+                compile_file = opt_arg;
+                break;
+            }
+            if (opt == 'o' || !strcmp(longopt, "out")) {
+                if (!opt_arg) {
+                    if (optind >= argc) {
+                        fprintf(stderr, "qjs: missing file for -o\n");
+                        exit(1);
+                    }
+                    opt_arg = argv[optind++];
+                }
+                out = opt_arg;
+                break;
+            }
+            if (!strcmp(longopt, "exe")) {
+                if (!opt_arg) {
+                    if (optind >= argc) {
+                        fprintf(stderr, "qjs: missing file for --exe\n");
+                        exit(1);
+                    }
+                    opt_arg = argv[optind++];
+                }
+                exe = opt_arg;
+                break;
+            }
             if (opt) {
                 fprintf(stderr, "qjs: unknown option '-%c'\n", opt);
             } else {
@@ -491,6 +589,11 @@ int main(int argc, char **argv)
         }
     }
 
+    if (compile_file && !out)
+        help();
+
+start:
+
     if (trace_memory) {
         js_trace_malloc_init(&trace_data);
         rt = JS_NewRuntime2(&trace_mf, &trace_data);
@@ -547,11 +650,37 @@ int main(int argc, char **argv)
                 goto fail;
         }
 
-        if (expr) {
+        if (standalone) {
+            JSValue ns = load_standalone_module(ctx);
+            if (JS_IsException(ns))
+                goto fail;
+            JSValue func = JS_GetPropertyStr(ctx, ns, "runStandalone");
+            JS_FreeValue(ctx, ns);
+            if (JS_IsException(func))
+                goto fail;
+            ret = JS_Call(ctx, func, JS_UNDEFINED, 0, NULL);
+            JS_FreeValue(ctx, func);
+        } else if (compile_file) {
+            JSValue ns = load_standalone_module(ctx);
+            if (JS_IsException(ns))
+                goto fail;
+            JSValue func = JS_GetPropertyStr(ctx, ns, "compileStandalone");
+            JS_FreeValue(ctx, ns);
+            if (JS_IsException(func))
+                goto fail;
+            JSValue args[3];
+            args[0] = JS_NewString(ctx, compile_file);
+            args[1] = JS_NewString(ctx, out);
+            args[2] = JS_NewString(ctx, exe != NULL ? exe : argv[0]);
+            ret = JS_Call(ctx, func, JS_UNDEFINED, countof(args), args);
+            JS_FreeValue(ctx, func);
+            JS_FreeValue(ctx, args[0]);
+            JS_FreeValue(ctx, args[1]);
+            JS_FreeValue(ctx, args[2]);
+        } else if (expr) {
             if (eval_buf(ctx, expr, strlen(expr), "<cmdline>", 0))
                 goto fail;
-        } else
+        } else if (optind >= argc) {
-        if (optind >= argc) {
             /* interactive mode */
             interactive = 1;
         } else {
@@ -563,7 +692,16 @@ int main(int argc, char **argv)
         if (interactive) {
             js_std_eval_binary(ctx, qjsc_repl, qjsc_repl_size, 0);
         }
-        ret = js_std_loop(ctx);
+        if (standalone || compile_file) {
+            if (JS_IsException(ret)) {
+                ret = JS_GetException(ctx);
+            } else {
+                JS_FreeValue(ctx, ret);
+                ret = js_std_loop(ctx);
+            }
+        } else {
+            ret = js_std_loop(ctx);
+        }
         if (!JS_IsUndefined(ret)) {
             js_std_dump_error1(ctx, ret);
             JS_FreeValue(ctx, ret);

standalone.js

@@ -0,0 +1,123 @@
+import * as std from "qjs:std";
+import * as os from "qjs:os";
+import * as bjson from "qjs:bjson";
+
+// See quickjs.h
+const JS_READ_OBJ_BYTECODE = 1 << 0;
+const JS_READ_OBJ_REFERENCE = 1 << 3;
+const JS_WRITE_OBJ_BYTECODE = 1 << 0;
+const JS_WRITE_OBJ_REFERENCE = 1 << 3;
+const JS_WRITE_OBJ_STRIP_SOURCE = 1 << 4;
+
+/**
+ * Trailer for standalone binaries. When some code gets bundled with the qjs
+ * executable we add a 12 byte trailer. The first 8 bytes are the magic
+ * string that helps us understand this is a standalone binary, and the
+ * remaining 4 are the offset (from the beginning of the binary) where the
+ * bundled data is located.
+ *
+ * The offset is stored as a 32bit little-endian number.
+ */
+const Trailer = {
+    Magic: 'quickjs2',
+    MagicSize: 8,
+    DataSize: 4,
+    Size: 12
+};
+
+function encodeAscii(txt) {
+  return new Uint8Array(txt.split('').map(c => c.charCodeAt(0)));
+}
+
+function decodeAscii(buf) {
+  return Array.from(buf).map(c => String.fromCharCode(c)).join('')
+}
+
+export function compileStandalone(inFile, outFile, targetExe) {
+  // Step 1: compile the source file to bytecode
+  const js = std.loadFile(inFile);
+
+  if (!js) {
+    throw new Error(`failed to open ${inFile}`);
+  }
+
+  const code = std.evalScript(js, {
+    compile_only: true,
+    compile_module: true
+  });
+  const bytecode = new Uint8Array(bjson.write(code, JS_WRITE_OBJ_BYTECODE | JS_WRITE_OBJ_REFERENCE | JS_WRITE_OBJ_STRIP_SOURCE));
+
+  // Step 2: copy the bytecode to the end of the executable and add a marker.
+  const exe = std.loadFile(targetExe ?? globalThis.argv0, { binary: true });
+  const exeSize = exe.length;
+  const newBuffer = exe.buffer.transfer(exeSize + bytecode.length + Trailer.Size);
+  const newExe = new Uint8Array(newBuffer);
+
+  newExe.set(bytecode, exeSize);
+  newExe.set(encodeAscii(Trailer.Magic), exeSize + bytecode.length);
+
+  const dw = new DataView(newBuffer, exeSize + bytecode.length + Trailer.MagicSize, Trailer.DataSize);
+
+  dw.setUint32(0, exeSize, true /* little-endian */);
+
+  // We use os.open() so we can set the permissions mask.
+  const newFd = os.open(outFile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755);
+
+  if (newFd < 0) {
+    throw new Error(`failed to create ${outFile}`);
+  }
+  if (os.write(newFd, newBuffer, 0, newBuffer.byteLength) < 0) {
+    os.close(newFd);
+    throw new Error(`failed to write to output file`);
+  }
+  os.close(newFd);
+}
+
+export function runStandalone() {
+  const file = globalThis.argv0;
+  const exe = std.open(file, 'rb');
+
+  if (!exe) {
+    throw new Error(`failed to open executable: ${file}`);
+  }
+
+  let r = exe.seek(-Trailer.Size, std.SEEK_END);
+  if (r < 0) {
+    throw new Error(`seek error: ${-r}`);
+  }
+
+  const trailer = new Uint8Array(Trailer.Size);
+
+  exe.read(trailer.buffer, 0, Trailer.Size);
+
+  const magic = new Uint8Array(trailer.buffer, 0, Trailer.MagicSize);
+
+  // Shouldn't happen since qjs.c checks for it.
+  if (decodeAscii(magic) !== Trailer.Magic) {
+    exe.close();
+    throw new Error('corrupted binary, magic mismatch');
+  }
+
+  const dw = new DataView(trailer.buffer, Trailer.MagicSize, Trailer.DataSize);
+  const offset = dw.getUint32(0, true /* little-endian */);
+  const bytecode = new Uint8Array(offset - Trailer.Size);
+
+  r = exe.seek(offset, std.SEEK_SET);
+  if (r < 0) {
+    exe.close();
+    throw new Error(`seek error: ${-r}`);
+  }
+
+  exe.read(bytecode.buffer, 0, bytecode.length);
+  if (exe.error()) {
+    exe.close();
+    throw new Error('read error');
+  }
+  exe.close();
+
+  const code = bjson.read(bytecode.buffer, 0, bytecode.length, JS_READ_OBJ_BYTECODE | JS_READ_OBJ_REFERENCE);
+
+  return std.evalScript(code, {
+    eval_module: true
+  });
+}