http.request Lua syscall

lib/plugos/syscalls/fetch.ts

@@ -1,7 +1,7 @@
 import type { SysCallMapping } from "../system.ts";
 import type {
-  ProxyFetchRequest,
+  ProxyFetchRequest64,
-  ProxyFetchResponse,
+  ProxyFetchResponse64,
 } from "../../proxy_fetch.ts";
 import { base64Encode } from "../../crypto.ts";
 
@@ -10,8 +10,8 @@ export function sandboxFetchSyscalls(): SysCallMapping {
     "sandboxFetch.fetch": async (
       _ctx,
       url: string,
-      options: ProxyFetchRequest,
+      options: ProxyFetchRequest64,
-    ): Promise<ProxyFetchResponse> => {
+    ): Promise<ProxyFetchResponse64> => {
       // console.log("Got sandbox fetch ", url);
       const resp = await fetch(url, options);
       return {

lib/plugos/worker_runtime.ts

@@ -1,6 +1,9 @@
 // This is the runtime imported from the compiled plug worker code
 import type { ControllerMessage, WorkerMessage } from "./protocol.ts";
-import type { ProxyFetchRequest, ProxyFetchResponse } from "../proxy_fetch.ts";
+import type {
+  ProxyFetchRequest64,
+  ProxyFetchResponse64,
+} from "../proxy_fetch.ts";
 
 declare global {
   function syscall(name: string, ...args: any[]): Promise<any>;
@@ -157,8 +160,8 @@ export function base64Encode(buffer: Uint8Array | string): string {
 
 export async function sandboxFetch(
   reqInfo: RequestInfo,
-  options?: ProxyFetchRequest,
+  options?: ProxyFetchRequest64,
-): Promise<ProxyFetchResponse> {
+): Promise<ProxyFetchResponse64> {
   if (typeof reqInfo !== "string") {
     const body = new Uint8Array(await reqInfo.arrayBuffer());
     const encodedBody = body.length > 0 ? base64Encode(body) : undefined;

lib/proxy_fetch.ts

@@ -1,12 +1,12 @@
 import { base64Decode, base64Encode } from "./crypto.ts";
 
-export type ProxyFetchRequest = {
+export type ProxyFetchRequest64 = {
   method?: string;
   headers?: Record<string, string>;
   base64Body?: string;
 };
 
-export type ProxyFetchResponse = {
+export type ProxyFetchResponse64 = {
   ok: boolean;
   status: number;
   headers: Record<string, string>;
@@ -14,10 +14,24 @@ export type ProxyFetchResponse = {
   base64Body: string;
 };
 
+export type ProxyFetchRequest = {
+  method?: string;
+  headers?: Record<string, string>;
+  body?: Uint8Array | string | any;
+};
+
+export type ProxyFetchResponse = {
+  ok: boolean;
+  status: number;
+  headers: Record<string, string>;
+  // We base64 encode the body because the body can be binary data that we have to push through the worker boundary
+  body: Uint8Array | string | any;
+};
+
 export async function performLocalFetch(
   url: string,
-  req: ProxyFetchRequest,
+  req: ProxyFetchRequest64,
-): Promise<ProxyFetchResponse> {
+): Promise<ProxyFetchResponse64> {
   const result = await fetch(
     url,
     req && {

server/http_server.ts

@@ -718,6 +718,7 @@ export class HttpServer {
         console.log("Proxying to", url);
         try {
           const safeRequestHeaders = new Headers();
+          // List all headers
           for (
             const headerName of ["Authorization", "Accept", "Content-Type"]
           ) {
@@ -728,6 +729,15 @@ export class HttpServer {
               );
             }
           }
+          // List all headers starting with X-Proxy-Header-, remove the prefix and add to the safe headers
+          for (const [key, value] of Object.entries(req.header())) {
+            if (key.startsWith("x-proxy-header-")) {
+              safeRequestHeaders.set(
+                key.slice("x-proxy-header-".length), // corrected casing of header prefix
+                value,
+              );
+            }
+          }
           const body = await req.arrayBuffer();
           const fetchReq = await fetch(url, {
             method: req.method,

web/syscalls/fetch.ts

@@ -2,8 +2,10 @@ import type { SysCallMapping } from "$lib/plugos/system.ts";
 import {
   performLocalFetch,
   type ProxyFetchRequest,
+  type ProxyFetchRequest64,
   type ProxyFetchResponse,
-} from "../../lib/proxy_fetch.ts";
+  type ProxyFetchResponse64,
+} from "$lib/proxy_fetch.ts";
 import type { Client } from "../client.ts";
 import { base64Decode, base64Encode } from "$lib/crypto.ts";
 
@@ -11,11 +13,59 @@ export function sandboxFetchSyscalls(
   client: Client,
 ): SysCallMapping {
   return {
+    // For use in Lua
+    "http.request": async (
+      _ctx,
+      url: string,
+      options: ProxyFetchRequest = {},
+    ): Promise<ProxyFetchResponse> => {
+      url = url.replace(/^https?:\/\//, "");
+      // JSONify any non-serializable body
+      if (
+        options?.body && typeof options.body !== "string" &&
+        !(options.body instanceof Uint8Array)
+      ) {
+        options.body = JSON.stringify(options.body);
+      }
+      const fetchOptions = options
+        ? {
+          method: options.method,
+          headers: options.headers,
+          body: options.body,
+        }
+        : {};
+      fetchOptions.headers = { "X-Proxy-Request": "true" };
+      // Copy the headers from the options prefixed with X-Proxy-Header
+      if (options.headers) {
+        for (const [k, v] of Object.entries(options.headers)) {
+          fetchOptions.headers[`X-Proxy-Header-${k}`] = v;
+        }
+      }
+      const resp = await client.httpSpacePrimitives.authenticatedFetch(
+        `${client.httpSpacePrimitives.url}/!${url}`,
+        fetchOptions,
+      );
+      // Do sensible things with the body based on the content type
+      let body: any;
+      if (resp.headers.get("Content-Type")?.startsWith("application/json")) {
+        body = await resp.json();
+      } else if (resp.headers.get("Content-Type")?.startsWith("text/")) {
+        body = await resp.text();
+      } else {
+        body = new Uint8Array(await resp.arrayBuffer());
+      }
+      return {
+        ok: resp.ok,
+        status: resp.status,
+        headers: Object.fromEntries(resp.headers.entries()),
+        body: body,
+      };
+    },
     "sandboxFetch.fetch": async (
       _ctx,
       url: string,
-      options?: ProxyFetchRequest,
+      options?: ProxyFetchRequest64,
-    ): Promise<ProxyFetchResponse> => {
+    ): Promise<ProxyFetchResponse64> => {
       // console.log("Got sandbox fetch ", url, op);
       url = url.replace(/^https?:\/\//, "");
       const fetchOptions = options

website/API/http.md

@@ -0,0 +1,18 @@
+HTTP APIs.
+
+### http.request(url, options?)
+Performs a HTTP call, proxied via the server (to avoid CORS issues).
+
+Options:
+* `method`: GET, POST, PUT, DELETE (GET is default)
+* `headers`: table with header -> value mappings
+* `body`: either a string or table (which will be JSON stringified)
+
+Returns:
+* `ok`: boolean if the request went ok
+* `status`: HTTP status code
+* `headers`: HTTP headers
+* `body`: for content types:
+  * `text/*`: string
+  * `application/json`: parsed JSON object
+  * anything else: UInt8Array