feat: implement go support (#571)

* progress

* progress

* all in one

* frontend

* small nits

* go job test

* go.sum is optional

* add golang-go to backend test image

Co-authored-by: sqwishy <somebody@froghat.ca>

.github/DockerfileBackendTests

@@ -13,7 +13,8 @@ RUN apt-get -y update \
     libnl-route-3-dev=3.4.* \
     make=4.2.* \
     pkg-config=0.29-6 \
-    protobuf-compiler=3.6.*
+    protobuf-compiler=3.6.* \
+    golang-go
 
 RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
     && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800

Dockerfile

@@ -76,7 +76,7 @@ RUN apt-get update \
     make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
     libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
     libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
-    libv8-dev tesseract-ocr \
+    libv8-dev tesseract-ocr golang-go \
     && rm -rf /var/lib/apt/lists/*
 
 ENV TZ=Etc/UTC

backend/Cargo.lock

@@ -4134,6 +4134,12 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
 
+[[package]]
+name = "unicode-general-category"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1218098468b8085b19a2824104c70d976491d247ce194bbd9dc77181150cdfd6"
+
 [[package]]
 name = "unicode-id"
 version = "0.3.2"
@@ -4520,6 +4526,7 @@ dependencies = [
  "tracing",
  "tracing-subscriber",
  "ulid",
+ "unicode-general-category",
  "url",
  "urlencoding",
  "uuid",

backend/Cargo.toml

@@ -58,6 +58,7 @@ async-recursion = "^1"
 swc_common = "^0"
 swc_ecma_parser = "^0"
 swc_ecma_ast = "^0"
+unicode-general-category = "^0"
 
 sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
 dotenv = "^0"

backend/migrations/20220905175719_go_lang.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20220905175719_go_lang.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE 'go';

backend/openapi.yaml

@@ -1742,7 +1742,7 @@ paths:
                     type: string
                 language:
                   type: string
-                  enum: [python3, deno]
+                  enum: [python3, deno, go]
                 kind:
                   type: string
                   enum: [script, failure, trigger, command]
@@ -1802,6 +1802,27 @@ paths:
               schema:
                 $ref: "#/components/schemas/MainArgSignature"
 
+  /scripts/go/tojsonschema:
+    post:
+      summary: inspect go code to infer jsonschema of arguments
+      operationId: goToJsonschema
+      tags:
+        - script
+      requestBody:
+        description: go code with the main function
+        required: true
+        content:
+          application/json:
+            schema:
+              type: string
+      responses:
+        "200":
+          description: parsed args
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/MainArgSignature"
+
   /w/{workspace}/scripts/archive/p/{path}:
     post:
       summary: archive script by path
@@ -3240,7 +3261,7 @@ components:
           type: string
         language:
           type: string
-          enum: [python3, deno]
+          enum: [python3, deno, go]
         kind:
           type: string
           enum: [script, failure, trigger, command]
@@ -3331,7 +3352,7 @@ components:
           type: boolean
         language:
           type: string
-          enum: [python3, deno]
+          enum: [python3, deno, go]
       required:
         - id
         - running
@@ -3409,7 +3430,7 @@ components:
           type: boolean
         language:
           type: string
-          enum: [python3, deno]
+          enum: [python3, deno, go]
         is_skipped:
           type: boolean
       required:
@@ -3781,7 +3802,7 @@ components:
           $ref: "#/components/schemas/ScriptArgs"
         language:
           type: string
-          enum: [python3, deno]
+          enum: [python3, deno, go]
 
       required:
         - content

backend/src/jobs.rs

@@ -1100,7 +1100,7 @@ pub enum JobPayload {
     ScriptHub { path: String },
     ScriptHash { hash: ScriptHash, path: String },
     Code(RawCode),
-    Dependencies { hash: ScriptHash, dependencies: Vec<String> },
+    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
     Flow(String),
     RawFlow { value: FlowValue, path: Option<String> },
 }
@@ -1240,13 +1240,13 @@ pub async fn push<'c>(
             None,
             Some(language),
         ),
-        JobPayload::Dependencies { hash, dependencies } => (
+        JobPayload::Dependencies { hash, dependencies, language } => (
             Some(hash.0),
             None,
-            Some(dependencies.join("\n")),
+            Some(dependencies),
             JobKind::Dependencies,
             None,
-            Some(ScriptLang::Python3),
+            Some(language),
         ),
         JobPayload::RawFlow { value, path } => {
             (None, path, None, JobKind::FlowPreview, Some(value), None)

backend/src/lib.rs

@@ -37,6 +37,10 @@ mod js_eval;
 mod more_serde;
 mod oauth2;
 mod parser;
+mod parser_go;
+mod parser_go_ast;
+mod parser_go_scanner;
+mod parser_go_token;
 mod parser_py;
 mod parser_ts;
 mod resources;

backend/src/main.rs

@@ -89,11 +89,15 @@ async fn main() -> anyhow::Result<()> {
                     .ok()
                     .and_then(|x| x.parse::<bool>().ok())
                     .unwrap_or(false);
+                let keep_job_dir = std::env::var("KEEP_JOB_DIR")
+                    .ok()
+                    .and_then(|x| x.parse::<bool>().ok())
+                    .unwrap_or(false);
 
                 tracing::info!(
                     "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \
                      {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \
-                     {timeout}"
+                     {timeout}, KEEP_JOB_DIR: {keep_job_dir}"
                 );
                 windmill::run_workers(
                     db.clone(),
@@ -101,7 +105,13 @@ async fn main() -> anyhow::Result<()> {
                     timeout,
                     num_workers,
                     sleep_queue,
-                    WorkerConfig { disable_nsjail, disable_nuser, base_internal_url, base_url },
+                    WorkerConfig {
+                        disable_nsjail,
+                        disable_nuser,
+                        base_internal_url,
+                        base_url,
+                        keep_job_dir,
+                    },
                     rx.resubscribe(),
                 )
                 .await?;

backend/src/parser.rs

@@ -42,6 +42,7 @@ pub enum Typ {
 #[derive(Serialize, Clone, Debug, PartialEq)]
 pub struct Arg {
     pub name: String,
+    pub otyp: Option<String>,
     pub typ: Typ,
     pub default: Option<serde_json::Value>,
     pub has_default: bool,

backend/src/parser_go_ast.rs

@@ -0,0 +1,347 @@
+#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box.
+
+use crate::parser_go_token::{Position, Token};
+use std::collections::BTreeMap;
+
+// https://pkg.go.dev/go/ast#CommentGroup
+#[derive(Debug)]
+pub struct CommentGroup {
+    // List []*Comment // len(List) > 0
+}
+
+// https://pkg.go.dev/go/ast#FieldList
+#[derive(Debug)]
+pub struct FieldList<'a> {
+    pub opening: Option<Position<'a>>, // position of opening parenthesis/brace, if any
+    pub list: Vec<Field<'a>>,          // field list; or nil
+    pub closing: Option<Position<'a>>, // position of closing parenthesis/brace, if any
+}
+
+// https://pkg.go.dev/go/ast#Field
+#[derive(Debug)]
+pub struct Field<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub names: Option<Vec<Ident<'a>>>, // field/method/(type) parameter names, or type "type"; or nil
+    pub type_: Option<Expr<'a>>,       // field/method/parameter type, type list type; or nil
+    pub tag: Option<BasicLit<'a>>,     // field tag; or nil
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#File
+#[derive(Debug)]
+pub struct File<'a> {
+    // package name
+    pub decls: Vec<Decl<'a>>, // top-level declarations; or nil // list of all comments in the source file
+}
+
+// https://pkg.go.dev/go/ast#FuncDecl
+#[derive(Debug)]
+pub struct FuncDecl<'a> {
+    pub doc: Option<CommentGroup>,   // associated documentation; or nil
+    pub recv: Option<FieldList<'a>>, // receiver (methods); or nil (functions)
+    pub name: Ident<'a>,             // function/method name
+    pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword
+    pub body: Option<BlockStmt<'a>>, // function body; or nil for external (non-Go) function
+}
+
+// https://pkg.go.dev/go/ast#BlockStmt
+#[derive(Debug)]
+pub struct BlockStmt<'a> {
+    pub lbrace: Position<'a>, // position of "{"
+    pub list: Vec<Stmt>,
+    pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error)
+}
+
+// https://pkg.go.dev/go/ast#FuncType
+#[derive(Debug)]
+pub struct FuncType<'a> {
+    pub func: Option<Position<'a>>, // position of "func" keyword (token.NoPos if there is no "func")
+    pub params: FieldList<'a>,      // (incoming) parameters; non-nil
+    pub results: Option<FieldList<'a>>, // (outgoing) results; or nil
+}
+
+// https://pkg.go.dev/go/ast#Ident
+#[derive(Debug)]
+pub struct Ident<'a> {
+    pub name_pos: Position<'a>,       // identifier position
+    pub name: &'a str,                // identifier name
+    pub obj: Option<Box<Object<'a>>>, // denoted object; or nil
+}
+
+// https://pkg.go.dev/go/ast#ValueSpec
+#[derive(Debug)]
+pub struct ValueSpec<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub names: Vec<Ident<'a>>,         // value names (len(Names) > 0)
+    pub type_: Option<Expr<'a>>,       // value type; or nil
+    pub values: Option<Vec<Expr<'a>>>, // initial values; or nil
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#BasicLit
+#[derive(Debug)]
+pub struct BasicLit<'a> {
+    pub value_pos: Position<'a>, // literal position
+    pub kind: Token,             // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
+    pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
+}
+
+// https://pkg.go.dev/go/ast#Object
+#[derive(Debug)]
+pub struct Object<'a> {
+    pub kind: ObjKind,
+    pub name: &'a str,         // declared name
+    pub decl: Option<ObjDecl>, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
+    pub data: Option<usize>,   // object-specific data; or nil
+    pub type_: Option<()>,     // placeholder for type information; may be nil
+}
+
+// https://pkg.go.dev/go/ast#Ellipsis
+#[derive(Debug)]
+pub struct Ellipsis<'a> {
+    pub ellipsis: Position<'a>,     // position of "..."
+    pub elt: Option<Box<Expr<'a>>>, // ellipsis element type (parameter lists only); or nil
+}
+
+// https://pkg.go.dev/go/ast#Ellipsis
+#[derive(Debug)]
+pub struct TypeAssertExpr<'a> {
+    pub x: Box<Expr<'a>>,     // expression
+    pub lparen: Position<'a>, // position of "("
+    pub type_: Box<Expr<'a>>, // asserted type; nil means type switch X.(type)
+    pub rparen: Position<'a>, // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#SliceExpr
+#[derive(Debug)]
+pub struct SliceExpr<'a> {
+    pub x: Box<Expr<'a>>,            // expression
+    pub lbrack: Position<'a>,        // position of "["
+    pub low: Option<Box<Expr<'a>>>,  // begin of slice range; or nil
+    pub high: Option<Box<Expr<'a>>>, // end of slice range; or nil
+    pub max: Option<Box<Expr<'a>>>,  // maximum capacity of slice; or nil
+    pub slice3: bool,                // true if 3-index slice (2 colons present)
+    pub rbrack: Position<'a>,        // position of "]"
+}
+
+// https://pkg.go.dev/go/ast#ObjKind
+#[derive(Debug)]
+pub enum ObjKind {}
+
+#[derive(Debug)]
+pub enum ObjDecl {}
+
+// https://pkg.go.dev/go/ast#Decl
+#[derive(Debug)]
+pub enum Decl<'a> {
+    FuncDecl(FuncDecl<'a>),
+}
+
+// https://pkg.go.dev/go/ast#Scope
+#[derive(Debug)]
+pub struct Scope<'a> {
+    pub outer: Option<Box<Scope<'a>>>,
+    pub objects: BTreeMap<&'a str, Object<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#GenDecl
+#[derive(Debug)]
+pub struct GenDecl<'a> {
+    pub doc: Option<CommentGroup>,    // associated documentation; or nil
+    pub tok_pos: Position<'a>,        // position of Tok
+    pub tok: Token,                   // IMPORT, CONST, TYPE, or VAR
+    pub lparen: Option<Position<'a>>, // position of '(', if any
+    pub specs: Vec<Spec>,
+    pub rparen: Option<Position<'a>>, // position of ')', if any
+}
+
+// https://pkg.go.dev/go/ast#AssignStmt
+#[derive(Debug)]
+pub struct AssignStmt<'a> {
+    pub lhs: Vec<Expr<'a>>,
+    pub tok_pos: Position<'a>, // position of Tok
+    pub tok: Token,            // assignment token, DEFINE
+    pub rhs: Vec<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#BinaryExpr
+#[derive(Debug)]
+pub struct BinaryExpr<'a> {
+    pub x: Box<Expr<'a>>,     // left operand
+    pub op_pos: Position<'a>, // position of Op
+    pub op: Token,            // operator
+    pub y: Box<Expr<'a>>,     // right operand
+}
+
+// https://pkg.go.dev/go/ast#ReturnStmt
+#[derive(Debug)]
+pub struct ReturnStmt<'a> {
+    pub return_: Position<'a>,  // position of "return" keyword
+    pub results: Vec<Expr<'a>>, // result expressions; or nil
+}
+
+// https://pkg.go.dev/go/ast#TypeSpec
+#[derive(Debug)]
+pub struct TypeSpec<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub name: Option<Ident<'a>>,       // type name
+    pub assign: Option<Position<'a>>,  // position of '=', if any
+    pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#StructType
+#[derive(Debug)]
+pub struct StructType<'a> {
+    pub struct_: Position<'a>,         // position of "struct" keyword
+    pub fields: Option<FieldList<'a>>, // list of field declarations
+    pub incomplete: bool,              // true if (source) fields are missing in the Fields list
+}
+
+// https://pkg.go.dev/go/ast#StarExpr
+#[derive(Debug)]
+pub struct StarExpr<'a> {
+    pub star: Position<'a>, // position of "*"
+    pub x: Box<Expr<'a>>,   // operand
+}
+
+// https://pkg.go.dev/go/ast#InterfaceType
+#[derive(Debug)]
+pub struct InterfaceType<'a> {
+    pub interface: Position<'a>,        // position of "interface" keyword
+    pub methods: Option<FieldList<'a>>, // list of embedded interfaces, methods, or types
+    pub incomplete: bool, // true if (source) methods or types are missing in the Methods list
+}
+
+// https://pkg.go.dev/go/ast#UnaryExpr
+#[derive(Debug)]
+pub struct UnaryExpr<'a> {
+    pub op_pos: Position<'a>, // position of Op
+    pub op: Token,            // operator
+    pub x: Box<Expr<'a>>,     // operand
+}
+
+// https://pkg.go.dev/go/ast#CallExpr
+#[derive(Debug)]
+pub struct CallExpr<'a> {
+    pub fun: Box<Expr<'a>>,             // function expression
+    pub lparen: Position<'a>,           // position of "("
+    pub args: Option<Vec<Expr<'a>>>,    // function arguments; or nil
+    pub ellipsis: Option<Position<'a>>, // position of "..." (token.NoPos if there is no "...")
+    pub rparen: Position<'a>,           // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#SelectorExpr
+#[derive(Debug)]
+pub struct SelectorExpr<'a> {
+    pub x: Box<Expr<'a>>, // expression
+    pub sel: Ident<'a>,   // field selector
+}
+
+// https://pkg.go.dev/go/ast#ParenExpr
+#[derive(Debug)]
+pub struct ParenExpr<'a> {
+    pub lparen: Position<'a>, // position of "("
+    pub x: Box<Expr<'a>>,     // parenthesized expression
+    pub rparen: Position<'a>, // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#FuncLit
+#[derive(Debug)]
+pub struct FuncLit<'a> {
+    pub type_: FuncType<'a>, // function type
+    pub body: BlockStmt<'a>, // function body
+}
+
+// https://pkg.go.dev/go/ast#ChanType
+#[derive(Debug)]
+pub struct ChanType<'a> {
+    pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first)
+    pub arrow: Option<Position<'a>>, // position of "<-" (token.NoPos if there is no "<-")
+    pub dir: u8,             // channel direction
+    pub value: Box<Expr<'a>>, // value type
+}
+
+// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr
+#[derive(Debug)]
+pub struct IndexExpr<'a> {
+    pub x: Box<Expr<'a>>,     // expression
+    pub lbrack: Position<'a>, // position of "["
+    pub index: Box<Expr<'a>>, // index expression
+    pub rbrack: Position<'a>, // position of "]"
+}
+
+// https://pkg.go.dev/go/ast#MapType
+#[derive(Debug)]
+pub struct MapType<'a> {
+    pub map: Position<'a>,
+    pub key: Box<Expr<'a>>,
+    pub value: Box<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#CompositeLit
+#[derive(Debug)]
+pub struct CompositeLit<'a> {
+    pub type_: Box<Expr<'a>>,        // literal type; or nil
+    pub lbrace: Position<'a>,        // position of "{"
+    pub elts: Option<Vec<Expr<'a>>>, // list of composite elements; or nil
+    pub rbrace: Position<'a>,        // position of "}"
+    pub incomplete: bool,            // true if (source) expressions are missing in the Elts list
+}
+
+// https://pkg.go.dev/go/ast#KeyValueExpr
+#[derive(Debug)]
+pub struct KeyValueExpr<'a> {
+    pub key: Box<Expr<'a>>,
+    pub colon: Position<'a>, // position of ":"
+    pub value: Box<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#ArrayType
+#[derive(Debug)]
+pub struct ArrayType<'a> {
+    pub lbrack: Position<'a>,       // position of "["
+    pub len: Option<Box<Expr<'a>>>, // Ellipsis node for [...]T array types, nil for slice types
+    pub elt: Box<Expr<'a>>,         // element type
+}
+
+// https://pkg.go.dev/go/ast#ChanDir
+#[derive(Debug)]
+pub enum ChanDir {
+    SEND = 1 << 0,
+    RECV = 1 << 1,
+}
+
+// https://pkg.go.dev/go/ast#Spec
+#[derive(Debug)]
+pub enum Spec {}
+
+// https://pkg.go.dev/go/ast#Expr
+#[derive(Debug)]
+pub enum Expr<'a> {
+    ArrayType(ArrayType<'a>),
+    BasicLit(BasicLit<'a>),
+    BinaryExpr(BinaryExpr<'a>),
+    CallExpr(CallExpr<'a>),
+    ChanType(ChanType<'a>),
+    CompositeLit(CompositeLit<'a>),
+    Ellipsis(Ellipsis<'a>),
+    FuncLit(FuncLit<'a>),
+    FuncType(FuncType<'a>),
+    Ident(Ident<'a>),
+    IndexExpr(IndexExpr<'a>),
+    InterfaceType(InterfaceType<'a>),
+    KeyValueExpr(KeyValueExpr<'a>),
+    MapType(MapType<'a>),
+    ParenExpr(ParenExpr<'a>),
+    SelectorExpr(SelectorExpr<'a>),
+    SliceExpr(SliceExpr<'a>),
+    StarExpr(StarExpr<'a>),
+    StructType(StructType<'a>),
+    TypeAssertExpr(TypeAssertExpr<'a>),
+    UnaryExpr(UnaryExpr<'a>),
+}
+
+// https://pkg.go.dev/go/ast#Stmt
+#[derive(Debug)]
+pub enum Stmt {}

backend/src/parser_go_scanner.rs

@@ -0,0 +1,948 @@
+// https://golang.org/ref/spec#Lexical_elements
+
+use crate::parser_go_token::{Position, Token};
+use phf::{phf_map, Map};
+use std::fmt;
+use unicode_general_category::{get_general_category, GeneralCategory};
+
+pub type Step<'a> = (Position<'a>, Token, &'a str);
+
+#[derive(Debug)]
+pub enum ScannerError {
+    HexadecimalNotFound,
+    OctalNotFound,
+    UnterminatedComment,
+    UnterminatedEscapedChar,
+    UnterminatedRune,
+    UnterminatedString,
+    InvalidDirective,
+}
+
+impl std::error::Error for ScannerError {}
+
+impl fmt::Display for ScannerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "scanner error: {:?}", self)
+    }
+}
+
+pub type Result<T> = std::result::Result<T, ScannerError>;
+
+#[derive(Debug)]
+pub struct Scanner<'a> {
+    directory: &'a str,
+    file: &'a str,
+    buffer: &'a str,
+    //
+    chars: std::iter::Peekable<std::str::Chars<'a>>,
+    current_char: Option<char>,
+    current_char_len: usize,
+    //
+    offset: usize,
+    line: usize,
+    column: usize,
+    start_offset: usize,
+    start_line: usize,
+    start_column: usize,
+    //
+    hide_column: bool,
+    insert_semi: bool,
+    pending_line_info: Option<LineInfo<'a>>,
+}
+
+type LineInfo<'a> = (Option<&'a str>, usize, Option<usize>, bool);
+
+impl<'a> Scanner<'a> {
+    pub fn new(filename: &'a str, buffer: &'a str) -> Self {
+        let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename));
+        let mut s = Scanner {
+            directory,
+            file,
+            buffer,
+            //
+            chars: buffer.chars().peekable(),
+            current_char: None,
+            current_char_len: 0,
+            //
+            offset: 0,
+            line: 1,
+            column: 1,
+            start_offset: 0,
+            start_line: 1,
+            start_column: 1,
+            //
+            hide_column: false,
+            insert_semi: false,
+            pending_line_info: None,
+        };
+        s.next(); // read the first character
+        s
+    }
+
+    #[allow(clippy::cognitive_complexity)] // Allow complex scan function
+    pub fn scan(&mut self) -> Result<Step<'a>> {
+        let insert_semi = self.insert_semi;
+        self.insert_semi = false;
+
+        while let Some(c) = self.current_char {
+            self.reset_start();
+
+            match c {
+                ' ' | '\t' | '\r' => {
+                    self.next();
+                }
+
+                '\n' => {
+                    self.next();
+                    if insert_semi {
+                        return Ok((self.position(), Token::SEMICOLON, "\n"));
+                    }
+                }
+
+                _ => break,
+            }
+        }
+
+        if let Some(c) = self.current_char {
+            match c {
+                '+' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::ADD_ASSIGN, ""));
+                        }
+                        Some('+') => {
+                            self.insert_semi = true;
+                            self.next();
+                            return Ok((self.position(), Token::INC, ""));
+                        }
+                        _ => return Ok((self.position(), Token::ADD, "")),
+                    }
+                }
+
+                '-' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::SUB_ASSIGN, ""));
+                        }
+                        Some('-') => {
+                            self.insert_semi = true;
+                            self.next();
+                            return Ok((self.position(), Token::DEC, ""));
+                        }
+                        _ => return Ok((self.position(), Token::SUB, "")),
+                    }
+                }
+
+                '*' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::MUL_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::MUL, "")),
+                    }
+                }
+
+                '/' => match self.peek() {
+                    Some('=') => {
+                        self.next();
+                        self.next();
+                        return Ok((self.position(), Token::QUO_ASSIGN, ""));
+                    }
+                    Some('/') => {
+                        if insert_semi {
+                            return Ok((self.position(), Token::SEMICOLON, "\n"));
+                        }
+                        return self.scan_line_comment();
+                    }
+                    Some('*') => {
+                        if insert_semi && self.find_line_end() {
+                            return Ok((self.position(), Token::SEMICOLON, "\n"));
+                        }
+                        return self.scan_general_comment();
+                    }
+                    _ => {
+                        self.next();
+                        return Ok((self.position(), Token::QUO, ""));
+                    }
+                },
+
+                '%' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::REM_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::REM, "")),
+                    }
+                }
+
+                '&' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::AND_ASSIGN, ""));
+                        }
+                        Some('&') => {
+                            self.next();
+                            return Ok((self.position(), Token::LAND, ""));
+                        }
+                        Some('^') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::AND_NOT_ASSIGN, ""));
+                                }
+                                _ => return Ok((self.position(), Token::AND_NOT, "")),
+                            }
+                        }
+                        _ => return Ok((self.position(), Token::AND, "")),
+                    }
+                }
+
+                '|' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::OR_ASSIGN, ""));
+                        }
+                        Some('|') => {
+                            self.next();
+                            return Ok((self.position(), Token::LOR, ""));
+                        }
+                        _ => return Ok((self.position(), Token::OR, "")),
+                    }
+                }
+
+                '^' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::XOR_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::XOR, "")),
+                    }
+                }
+
+                '<' => {
+                    self.next();
+                    match self.current_char {
+                        Some('<') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::SHL_ASSIGN, ""));
+                                }
+                                _ => return Ok((self.position(), Token::SHL, "")),
+                            }
+                        }
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::LEQ, ""));
+                        }
+                        Some('-') => {
+                            self.next();
+                            return Ok((self.position(), Token::ARROW, ""));
+                        }
+                        _ => return Ok((self.position(), Token::LSS, "")),
+                    }
+                }
+
+                '>' => {
+                    self.next();
+                    match self.current_char {
+                        Some('>') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::SHR_ASSIGN, ""));
+                                }
+                                _ => {
+                                    return Ok((self.position(), Token::SHR, ""));
+                                }
+                            }
+                        }
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::GEQ, ""));
+                        }
+                        _ => return Ok((self.position(), Token::GTR, "")),
+                    }
+                }
+
+                ':' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::DEFINE, ""));
+                        }
+                        _ => return Ok((self.position(), Token::COLON, "")),
+                    }
+                }
+
+                '!' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::NEQ, ""));
+                        }
+                        _ => return Ok((self.position(), Token::NOT, "")),
+                    }
+                }
+
+                ',' => {
+                    self.next();
+                    return Ok((self.position(), Token::COMMA, ""));
+                }
+
+                '(' => {
+                    self.next();
+                    return Ok((self.position(), Token::LPAREN, ""));
+                }
+
+                ')' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RPAREN, ""));
+                }
+
+                '[' => {
+                    self.next();
+                    return Ok((self.position(), Token::LBRACK, ""));
+                }
+
+                ']' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RBRACK, ""));
+                }
+
+                '{' => {
+                    self.next();
+                    return Ok((self.position(), Token::LBRACE, ""));
+                }
+
+                '}' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RBRACE, ""));
+                }
+
+                ';' => {
+                    self.next();
+                    return Ok((self.position(), Token::SEMICOLON, ";"));
+                }
+
+                '.' => {
+                    self.next();
+                    match self.current_char {
+                        Some('0'..='9') => return self.scan_int_or_float_or_imag(true),
+                        Some('.') => match self.peek() {
+                            Some('.') => {
+                                self.next();
+                                self.next();
+                                return Ok((self.position(), Token::ELLIPSIS, ""));
+                            }
+                            _ => return Ok((self.position(), Token::PERIOD, "")),
+                        },
+                        _ => return Ok((self.position(), Token::PERIOD, "")),
+                    }
+                }
+
+                '=' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::EQL, ""));
+                        }
+                        _ => return Ok((self.position(), Token::ASSIGN, "")),
+                    }
+                }
+
+                '0'..='9' => return self.scan_int_or_float_or_imag(false),
+                '\'' => return self.scan_rune(),
+                '"' => return self.scan_interpreted_string(),
+                '`' => return self.scan_raw_string(),
+                _ => return self.scan_pkg_or_keyword_or_ident(),
+            };
+        }
+
+        self.reset_start();
+        if insert_semi {
+            Ok((self.position(), Token::SEMICOLON, "\n"))
+        } else {
+            Ok((self.position(), Token::EOF, ""))
+        }
+    }
+
+    // https://golang.org/ref/spec#Keywords
+    // https://golang.org/ref/spec#Identifiers
+    fn scan_pkg_or_keyword_or_ident(&mut self) -> Result<Step<'a>> {
+        self.next();
+
+        while let Some(c) = self.current_char {
+            if !(is_letter(c) || is_unicode_digit(c)) {
+                break;
+            }
+            self.next()
+        }
+
+        let pos = self.position();
+        let literal = self.literal();
+
+        if literal.len() > 1 {
+            if let Some(&token) = KEYWORDS.get(literal) {
+                self.insert_semi = matches!(
+                    token,
+                    Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN
+                );
+                return Ok((pos, token, literal));
+            }
+        }
+
+        self.insert_semi = true;
+        Ok((pos, Token::IDENT, literal))
+    }
+
+    // https://golang.org/ref/spec#Integer_literals
+    // https://golang.org/ref/spec#Floating-point_literals
+    // https://golang.org/ref/spec#Imaginary_literals
+    fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result<Step<'a>> {
+        self.insert_semi = true;
+
+        let mut token = Token::INT;
+        let mut digits = "_0123456789";
+        let mut exp = "eE";
+
+        if !preceding_dot {
+            if matches!(self.current_char, Some('0')) {
+                self.next();
+                match self.current_char {
+                    Some('b' | 'B') => {
+                        digits = "_01";
+                        exp = "";
+                        self.next();
+                    }
+                    Some('o' | 'O') => {
+                        digits = "_01234567";
+                        exp = "";
+                        self.next();
+                    }
+                    Some('x' | 'X') => {
+                        digits = "_0123456789abcdefABCDEF";
+                        exp = "pP";
+                        self.next();
+                    }
+                    _ => {}
+                };
+            }
+
+            while let Some(c) = self.current_char {
+                if !digits.contains(c) {
+                    break;
+                }
+                self.next();
+            }
+        }
+
+        if preceding_dot || matches!(self.current_char, Some('.')) {
+            token = Token::FLOAT;
+            self.next();
+            while let Some(c) = self.current_char {
+                if !digits.contains(c) {
+                    break;
+                }
+                self.next();
+            }
+        }
+
+        if !exp.is_empty() {
+            if let Some(c) = self.current_char {
+                if exp.contains(c) {
+                    token = Token::FLOAT;
+                    self.next();
+                    if matches!(self.current_char, Some('-' | '+')) {
+                        self.next();
+                    }
+                    while let Some(c) = self.current_char {
+                        if !matches!(c, '_' | '0'..='9') {
+                            break;
+                        }
+                        self.next();
+                    }
+                }
+            }
+        }
+
+        if matches!(self.current_char, Some('i')) {
+            token = Token::IMAG;
+            self.next();
+        }
+
+        Ok((self.position(), token, self.literal()))
+    }
+
+    // https://golang.org/ref/spec#Rune_literals
+    fn scan_rune(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        match self.current_char {
+            Some('\\') => self.require_escaped_char::<'\''>()?,
+            Some(_) => self.next(),
+            _ => return Err(ScannerError::UnterminatedRune),
+        }
+
+        if matches!(self.current_char, Some('\'')) {
+            self.next();
+            return Ok((self.position(), Token::CHAR, self.literal()));
+        }
+
+        Err(ScannerError::UnterminatedRune)
+    }
+
+    // https://golang.org/ref/spec#String_literals
+    fn scan_interpreted_string(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '"' => {
+                    self.next();
+                    return Ok((self.position(), Token::STRING, self.literal()));
+                }
+                '\\' => self.require_escaped_char::<'"'>()?,
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedString)
+    }
+
+    // https://golang.org/ref/spec#String_literals
+    fn scan_raw_string(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '`' => {
+                    self.next();
+                    return Ok((self.position(), Token::STRING, self.literal()));
+                }
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedString)
+    }
+
+    // https://golang.org/ref/spec#Comments
+    fn scan_general_comment(&mut self) -> Result<Step<'a>> {
+        self.next();
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '*' => {
+                    self.next();
+                    if matches!(self.current_char, Some('/')) {
+                        self.next();
+
+                        let pos = self.position();
+                        let lit = self.literal();
+
+                        // look for compiler directives
+                        self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?;
+
+                        return Ok((pos, Token::COMMENT, lit));
+                    }
+                }
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedComment)
+    }
+
+    // https://golang.org/ref/spec#Comments
+    fn scan_line_comment(&mut self) -> Result<Step<'a>> {
+        self.next();
+        self.next();
+
+        while let Some(c) = self.current_char {
+            if is_newline(c) {
+                break;
+            }
+            self.next();
+        }
+
+        let pos = self.position();
+        let lit = self.literal();
+
+        // look for compiler directives (at the beginning of line)
+        if self.start_column == 1 {
+            self.directive(lit["//".len()..].trim_end(), false)?;
+        }
+
+        Ok((pos, Token::COMMENT, self.literal()))
+    }
+
+    // https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives
+    fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> {
+        if let Some(line_directive) = input.strip_prefix("line ") {
+            self.pending_line_info = self.parse_line_directive(line_directive)?;
+            if immediate {
+                self.consume_pending_line_info();
+            }
+        }
+        Ok(())
+    }
+
+    fn parse_line_directive(&mut self, line_directive: &'a str) -> Result<Option<LineInfo<'a>>> {
+        if let Some((file, line)) = line_directive.rsplit_once(':') {
+            let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?;
+
+            if let Some((file, l)) = file.rsplit_once(':') {
+                if let Ok(l) = l.parse() {
+                    //line :line:col
+                    //line filename:line:col
+                    /*line :line:col*/
+                    /*line filename:line:col*/
+                    let file = if !file.is_empty() { Some(file) } else { None };
+                    let col = Some(line);
+                    let line = l;
+                    let hide_column = false;
+                    return Ok(Some((file, line, col, hide_column)));
+                }
+            }
+
+            //line :line
+            //line filename:line
+            /*line :line*/
+            /*line filename:line*/
+            Ok(Some((Some(file), line, None, true)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    const fn find_line_end(&self) -> bool {
+        let buffer = self.buffer.as_bytes();
+        let mut in_comment = true;
+
+        let mut i = self.offset;
+        let max = self.buffer.len();
+        while i < max {
+            let c = buffer[i] as char;
+
+            if i < max - 1 {
+                let n = buffer[i + 1] as char;
+
+                if !in_comment && c == '/' && n == '/' {
+                    return true;
+                }
+
+                if c == '/' && n == '*' {
+                    i += 2;
+                    in_comment = true;
+                    continue;
+                }
+
+                if c == '*' && n == '/' {
+                    i += 2;
+                    in_comment = false;
+                    continue;
+                }
+            }
+
+            if is_newline(c) {
+                return true;
+            }
+
+            if !in_comment && !matches!(c, ' ' | '\t' | '\r') {
+                return false;
+            }
+
+            i += 1;
+        }
+
+        !in_comment
+    }
+
+    fn consume_pending_line_info(&mut self) {
+        if let Some(line_info) = self.pending_line_info.take() {
+            if let Some(file) = line_info.0 {
+                self.file = file;
+            }
+
+            self.line = line_info.1;
+
+            if let Some(column) = line_info.2 {
+                self.column = column;
+            }
+
+            self.hide_column = line_info.3;
+        }
+    }
+
+    fn peek(&mut self) -> Option<char> {
+        self.chars.peek().copied()
+    }
+
+    fn next(&mut self) {
+        self.offset += self.current_char_len;
+        self.column += self.current_char_len;
+        let last_char = self.current_char;
+
+        self.current_char = self.chars.next();
+        if let Some(c) = self.current_char {
+            self.current_char_len = c.len_utf8();
+            if matches!(last_char, Some('\n')) {
+                self.line += 1;
+                self.column = 1;
+                self.consume_pending_line_info();
+            }
+        } else {
+            self.current_char_len = 0
+        }
+    }
+
+    const fn position(&self) -> Position<'a> {
+        Position {
+            directory: self.directory,
+            file: self.file,
+            offset: self.start_offset,
+            line: self.start_line,
+            column: if self.hide_column {
+                0
+            } else {
+                self.start_column
+            },
+        }
+    }
+
+    fn reset_start(&mut self) {
+        self.start_offset = self.offset;
+        self.start_line = self.line;
+        self.start_column = self.column;
+    }
+
+    fn literal(&self) -> &'a str {
+        &self.buffer[self.start_offset..self.offset]
+    }
+
+    fn require_escaped_char<const DELIM: char>(&mut self) -> Result<()> {
+        self.next();
+
+        let c = self
+            .current_char
+            .ok_or(ScannerError::UnterminatedEscapedChar)?;
+
+        // TODO: move this to the match when const generics can be referenced in patterns
+        if c == DELIM {
+            self.next();
+            return Ok(());
+        }
+
+        match c {
+            'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(),
+            'x' => {
+                self.next();
+                self.require_hex_digits::<2>()?
+            }
+            'u' => {
+                self.next();
+                self.require_hex_digits::<4>()?;
+            }
+            'U' => {
+                self.next();
+                self.require_hex_digits::<8>()?;
+            }
+            '0'..='7' => self.require_octal_digits::<3>()?,
+            _ => return Err(ScannerError::UnterminatedEscapedChar),
+        }
+
+        Ok(())
+    }
+
+    fn require_octal_digits<const COUNT: usize>(&mut self) -> Result<()> {
+        for _ in 0..COUNT {
+            let c = self.current_char.ok_or(ScannerError::OctalNotFound)?;
+
+            if !is_octal_digit(c) {
+                return Err(ScannerError::OctalNotFound);
+            }
+
+            self.next();
+        }
+
+        Ok(())
+    }
+
+    fn require_hex_digits<const COUNT: usize>(&mut self) -> Result<()> {
+        for _ in 0..COUNT {
+            let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?;
+
+            if !is_hex_digit(c) {
+                return Err(ScannerError::HexadecimalNotFound);
+            }
+
+            self.next();
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a> IntoIterator for Scanner<'a> {
+    type Item = Result<Step<'a>>;
+    type IntoIter = IntoIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        Self::IntoIter::new(self)
+    }
+}
+
+pub struct IntoIter<'a> {
+    scanner: Scanner<'a>,
+    done: bool,
+}
+
+impl<'a> IntoIter<'a> {
+    const fn new(scanner: Scanner<'a>) -> Self {
+        Self { scanner, done: false }
+    }
+}
+
+impl<'a> Iterator for IntoIter<'a> {
+    type Item = Result<Step<'a>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+
+        match self.scanner.scan() {
+            Ok((pos, tok, lit)) => {
+                if tok == Token::EOF {
+                    self.done = true;
+                }
+                Some(Ok((pos, tok, lit)))
+            }
+            Err(err) => {
+                self.done = true;
+                Some(Err(err))
+            }
+        }
+    }
+}
+
+// https://golang.org/ref/spec#Letters_and_digits
+
+fn is_letter(c: char) -> bool {
+    c == '_' || is_unicode_letter(c)
+}
+
+//const fn is_decimal_digit(c: char) -> bool {
+//matches!(c, '0'..='9')
+//}
+
+//const fn is_binary_digit(c: char) -> bool {
+//matches!(c, '0'..='1')
+//}
+
+const fn is_octal_digit(c: char) -> bool {
+    matches!(c, '0'..='7')
+}
+
+const fn is_hex_digit(c: char) -> bool {
+    matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f')
+}
+
+// https://golang.org/ref/spec#Characters
+
+const fn is_newline(c: char) -> bool {
+    c == '\n'
+}
+
+//const fn is_unicode_char(c: char) -> bool {
+//c != '\n'
+//}
+
+fn is_unicode_letter(c: char) -> bool {
+    matches!(
+        get_general_category(c),
+        GeneralCategory::UppercaseLetter
+            | GeneralCategory::LowercaseLetter
+            | GeneralCategory::TitlecaseLetter
+            | GeneralCategory::ModifierLetter
+            | GeneralCategory::OtherLetter
+    )
+}
+
+fn is_unicode_digit(c: char) -> bool {
+    get_general_category(c) == GeneralCategory::DecimalNumber
+}
+
+// https://golang.org/ref/spec#Keywords
+
+static KEYWORDS: Map<&'static str, Token> = phf_map! {
+  "break" => Token::BREAK,
+  "case" => Token::CASE,
+  "chan" => Token::CHAN,
+  "const" => Token::CONST,
+  "continue" => Token::CONTINUE,
+
+  "default" => Token::DEFAULT,
+  "defer" => Token::DEFER,
+  "else" => Token::ELSE,
+  "fallthrough" => Token::FALLTHROUGH,
+  "for" => Token::FOR,
+
+  "func" => Token::FUNC,
+  "go" => Token::GO,
+  "goto" => Token::GOTO,
+  "if" => Token::IF,
+  "import" => Token::IMPORT,
+
+  "interface" => Token::INTERFACE,
+  "map" => Token::MAP,
+  "package" => Token::PACKAGE,
+  "range" => Token::RANGE,
+  "return" => Token::RETURN,
+
+  "select" => Token::SELECT,
+  "struct" => Token::STRUCT,
+  "switch" => Token::SWITCH,
+  "type" => Token::TYPE,
+  "var" => Token::VAR,
+};
+
+#[cfg(test)]
+mod tests {
+    use super::Scanner;
+
+    #[test] // fuzz
+    fn it_should_return_an_error_on_missing_line_number() {
+        let input = "/*line :*/";
+        let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect();
+        assert!(out.pop().unwrap().is_err());
+    }
+}

backend/src/parser_go_token.rs

@@ -0,0 +1,273 @@
+// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go
+
+#![allow(non_camel_case_types)] // For consistency with the Go tokens
+
+use std::fmt;
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Position<'a> {
+    pub directory: &'a str,
+    pub file: &'a str,
+    pub offset: usize,
+    pub line: usize,
+    pub column: usize,
+}
+
+impl<'a> fmt::Display for Position<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.file.is_empty() {
+            write!(f, ":{}:{}", self.line, self.column)
+        } else if self.file.starts_with('/') {
+            write!(f, "{}:{}:{}", self.file, self.line, self.column)
+        } else {
+            write!(
+                f,
+                "{}/{}:{}:{}",
+                self.directory, self.file, self.line, self.column
+            )
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum Token {
+    EOF,
+    COMMENT,
+
+    IDENT,  // main
+    INT,    // 12345
+    FLOAT,  // 123.45
+    IMAG,   // 123.45i
+    CHAR,   // 'a'
+    STRING, // "abc"
+
+    ADD, // +
+    SUB, // -
+    MUL, // *
+    QUO, // /
+    REM, // %
+
+    AND,     // &
+    OR,      // |
+    XOR,     // ^
+    SHL,     // <<
+    SHR,     // >>
+    AND_NOT, // &^
+
+    ADD_ASSIGN, // +=
+    SUB_ASSIGN, // -=
+    MUL_ASSIGN, // *=
+    QUO_ASSIGN, // /=
+    REM_ASSIGN, // %=
+
+    AND_ASSIGN,     // &=
+    OR_ASSIGN,      // |=
+    XOR_ASSIGN,     // ^=
+    SHL_ASSIGN,     // <<=
+    SHR_ASSIGN,     // >>=
+    AND_NOT_ASSIGN, // &^=
+
+    LAND,  // &&
+    LOR,   // ||
+    ARROW, // <-
+    INC,   // ++
+    DEC,   // --
+
+    EQL,    // ==
+    LSS,    // <
+    GTR,    // >
+    ASSIGN, // =
+    NOT,    // !
+
+    NEQ,      // !=
+    LEQ,      // <=
+    GEQ,      // >=
+    DEFINE,   // :=
+    ELLIPSIS, // ...
+
+    LPAREN, // (
+    LBRACK, // [
+    LBRACE, // {
+    COMMA,  // ,
+    PERIOD, // .
+
+    RPAREN,    // )
+    RBRACK,    // ]
+    RBRACE,    // }
+    SEMICOLON, // ;
+    COLON,     // :
+
+    BREAK,
+    CASE,
+    CHAN,
+    CONST,
+    CONTINUE,
+
+    DEFAULT,
+    DEFER,
+    ELSE,
+    FALLTHROUGH,
+    FOR,
+
+    FUNC,
+    GO,
+    GOTO,
+    IF,
+    IMPORT,
+
+    INTERFACE,
+    MAP,
+    PACKAGE,
+    RANGE,
+    RETURN,
+
+    SELECT,
+    STRUCT,
+    SWITCH,
+    TYPE,
+    VAR,
+}
+
+impl Token {
+    pub const fn is_assign_op(&self) -> bool {
+        use Token::*;
+        matches!(
+            self,
+            ADD_ASSIGN
+                | SUB_ASSIGN
+                | MUL_ASSIGN
+                | QUO_ASSIGN
+                | REM_ASSIGN
+                | AND_ASSIGN
+                | OR_ASSIGN
+                | XOR_ASSIGN
+                | SHL_ASSIGN
+                | SHR_ASSIGN
+                | AND_NOT_ASSIGN
+        )
+    }
+
+    // https://go.dev/ref/spec#Operator_precedence
+    pub fn precedence(&self) -> u8 {
+        use Token::*;
+        match self {
+            MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5,
+            ADD | SUB | OR | XOR => 4,
+            EQL | NEQ | LSS | LEQ | GTR | GEQ => 3,
+            LAND => 2,
+            LOR => 1,
+            _ => unreachable!(
+                "precedence() is only supported for binary operators, called with: {:?}",
+                self
+            ),
+        }
+    }
+
+    pub const fn lowest_precedence() -> u8 {
+        0
+    }
+}
+
+impl From<&Token> for &'static str {
+    fn from(token: &Token) -> Self {
+        use Token::*;
+
+        match token {
+            EOF => "EOF",
+            COMMENT => "COMMENT",
+
+            IDENT => "IDENT",
+            INT => "INT",
+            FLOAT => "FLOAT",
+            IMAG => "IMAG",
+            CHAR => "CHAR",
+            STRING => "STRING",
+
+            ADD => "+",
+            SUB => "-",
+            MUL => "*",
+            QUO => "/",
+            REM => "%",
+
+            AND => "&",
+            OR => "|",
+            XOR => "^",
+            SHL => "<<",
+            SHR => ">>",
+            AND_NOT => "&^",
+
+            ADD_ASSIGN => "+=",
+            SUB_ASSIGN => "-=",
+            MUL_ASSIGN => "*=",
+            QUO_ASSIGN => "/=",
+            REM_ASSIGN => "%=",
+
+            AND_ASSIGN => "&=",
+            OR_ASSIGN => "|=",
+            XOR_ASSIGN => "^=",
+            SHL_ASSIGN => "<<=",
+            SHR_ASSIGN => ">>=",
+            AND_NOT_ASSIGN => "&^=",
+
+            LAND => "&&",
+            LOR => "||",
+            ARROW => "<-",
+            INC => "++",
+            DEC => "--",
+
+            EQL => "==",
+            LSS => "<",
+            GTR => ">",
+            ASSIGN => "=",
+            NOT => "!",
+
+            NEQ => "!=",
+            LEQ => "<=",
+            GEQ => ">=",
+            DEFINE => ":=",
+            ELLIPSIS => "...",
+
+            LPAREN => "(",
+            LBRACK => "[",
+            LBRACE => "{",
+            COMMA => ",",
+            PERIOD => ".",
+
+            RPAREN => ")",
+            RBRACK => "]",
+            RBRACE => "}",
+            SEMICOLON => ";",
+            COLON => ":",
+
+            BREAK => "break",
+            CASE => "case",
+            CHAN => "chan",
+            CONST => "const",
+            CONTINUE => "continue",
+
+            DEFAULT => "default",
+            DEFER => "defer",
+            ELSE => "else",
+            FALLTHROUGH => "fallthrough",
+            FOR => "for",
+
+            FUNC => "func",
+            GO => "go",
+            GOTO => "goto",
+            IF => "if",
+            IMPORT => "import",
+
+            INTERFACE => "interface",
+            MAP => "map",
+            PACKAGE => "package",
+            RANGE => "range",
+            RETURN => "return",
+
+            SELECT => "select",
+            STRUCT => "struct",
+            SWITCH => "switch",
+            TYPE => "type",
+            VAR => "var",
+        }
+    }
+}

backend/src/parser_py.rs

@@ -101,6 +101,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
                         None
                     };
                     Arg {
+                        otyp: None,
                         name: x.arg,
                         typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
                             Located { location: _, node: ExpressionType::Identifier { name } } => {
@@ -269,18 +270,21 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
                 star_kwargs: false,
                 args: vec![
                     Arg {
+                        otyp: None,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Unknown,
                         default: Some(json!("<function call>")),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
@@ -316,18 +320,21 @@ def main(test1: str,
                 star_kwargs: false,
                 args: vec![
                     Arg {
+                        otyp: None,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Unknown,
                         default: Some(json!("<function call>")),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
@@ -359,18 +366,21 @@ def main(test1: str,
                 star_kwargs: false,
                 args: vec![
                     Arg {
+                        otyp: None,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Unknown,
                         default: Some(json!("<function call>")),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),

backend/src/parser_ts.rs

@@ -68,6 +68,7 @@ pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
                     Pat::Ident(ident) => {
                         let (name, typ, nullable) = binding_ident_to_arg(&ident);
                         Ok(Arg {
+                            otyp: None,
                             name,
                             typ,
                             default: None,
@@ -106,7 +107,7 @@ pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
                         if typ == Typ::Unknown && default.is_some() {
                             typ = json_to_typ(default.as_ref().unwrap());
                         }
-                        Ok(Arg { name, typ, default, has_default: true })
+                        Ok(Arg { otyp: None, name, typ, default, has_default: true })
                     }
                     _ => Err(error::Error::ExecutionErr(format!(
                         "parameter syntax unsupported: `{}`",
@@ -288,72 +289,84 @@ export function main(test1?: string, test2: string = \"burkina\",
                 star_kwargs: false,
                 args: vec![
                     Arg {
+                        otyp: None,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "test2".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("burkina")),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "test3".to_string(),
                         typ: Typ::Resource("postgres".to_string()),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "b64".to_string(),
                         typ: Typ::Bytes,
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "ls".to_string(),
                         typ: Typ::List(Box::new(Typ::Bytes)),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "email".to_string(),
                         typ: Typ::Email,
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "literal".to_string(),
                         typ: Typ::Str(Some(vec!["test".to_string()])),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "literal_union".to_string(),
                         typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
                         default: None,
                         has_default: false
                     },
                     Arg {
+                        otyp: None,
                         name: "opt_type".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "opt_type_union".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "opt_type_union_union2".to_string(),
                         typ: Typ::Str(None),
                         default: None,
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "min_object".to_string(),
                         typ: Typ::Object(vec![
                             ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
@@ -388,36 +401,42 @@ export function main(test2 = \"burkina\",
                 star_kwargs: false,
                 args: vec![
                     Arg {
+                        otyp: None,
                         name: "test2".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("burkina")),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "bool".to_string(),
                         typ: Typ::Bool,
                         default: Some(json!(true)),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "float".to_string(),
                         typ: Typ::Float,
                         default: Some(json!(4.2)),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "int".to_string(),
                         typ: Typ::Int,
                         default: Some(json!(42)),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "ls".to_string(),
                         typ: Typ::List(Box::new(Typ::Str(None))),
                         default: Some(json!(["test"])),
                         has_default: true
                     },
                     Arg {
+                        otyp: None,
                         name: "min_object".to_string(),
                         typ: Typ::Object(vec![
                             ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },

backend/src/scripts.rs

@@ -14,7 +14,7 @@ use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{to_anyhow, Error, JsonResult, Result},
-    jobs, parser, parser_py, parser_ts,
+    jobs, parser, parser_go, parser_py, parser_ts,
     users::{owner_to_token_owner, truncate_token, Authed, Tokened},
     utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
 };
@@ -43,6 +43,7 @@ pub fn global_service() -> Router {
             post(parse_python_code_to_jsonschema),
         )
         .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema))
+        .route("/go/tojsonschema", post(parse_go_code_to_jsonschema))
         .route("/hub/list", get(list_hub_scripts))
         .route("/hub/get/*path", get(get_hub_script_by_path))
 }
@@ -68,6 +69,7 @@ pub fn workspaced_service() -> Router {
 pub enum ScriptLang {
     Deno,
     Python3,
+    Go,
 }
 
 impl ScriptLang {
@@ -75,6 +77,7 @@ impl ScriptLang {
         match self {
             ScriptLang::Deno => "deno",
             ScriptLang::Python3 => "python3",
+            ScriptLang::Go => "go",
         }
     }
 }
@@ -436,12 +439,15 @@ async fn create_script(
     .execute(&mut tx)
     .await?;
 
-    let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
-        let dependencies = parser_py::parse_python_imports(&ns.content)?;
+    let mut tx = if ns.lock.is_none() && ns.language != ScriptLang::Deno {
+        let dependencies = match ns.language {
+            ScriptLang::Python3 => parser_py::parse_python_imports(&ns.content)?.join("\n"),
+            _ => ns.content,
+        };
         let (_, tx) = jobs::push(
             tx,
             &w_id,
-            jobs::JobPayload::Dependencies { hash, dependencies },
+            jobs::JobPayload::Dependencies { hash, dependencies, language: ns.language },
             None,
             &authed.username,
             owner_to_token_owner(&authed.username, false),
@@ -772,6 +778,11 @@ async fn parse_deno_code_to_jsonschema(
 ) -> JsonResult<parser::MainArgSignature> {
     parser_ts::parse_deno_signature(&code).map(Json)
 }
+async fn parse_go_code_to_jsonschema(
+    Json(code): Json<String>,
+) -> JsonResult<parser::MainArgSignature> {
+    parser_go::parse_go_sig(&code).map(Json)
+}
 
 pub fn to_i64(s: &str) -> Result<i64> {
     let v = hex::decode(s)?;

frontend/src/lib/components/Editor.svelte

@@ -47,8 +47,7 @@
 	let divEl: HTMLDivElement | null = null
 	let editor: monaco.editor.IStandaloneCodeEditor
 
-	export let deno = false
-	export let lang = deno ? 'typescript' : 'python'
+	export let lang: 'typescript' | 'python' | 'go'
 	export let code: string = ''
 	export let hash: string = randomHash()
 	export let cmdEnterAction: (() => void) | undefined = undefined
@@ -139,201 +138,190 @@
 
 	export async function reloadWebsocket() {
 		await closeWebsockets()
-		if (lang == 'python' || deno) {
-			const { MonacoLanguageClient } = await import('monaco-languageclient')
-			const { CloseAction, ErrorAction } = await import('vscode-languageclient')
-			const { toSocket, WebSocketMessageReader, WebSocketMessageWriter } = await import(
-				'vscode-ws-jsonrpc'
-			)
-			const vscode = await import('vscode')
-			const { RequestType } = await import('vscode-jsonrpc')
-			// install Monaco language client services
-			const { MonacoServices } = await import('monaco-languageclient')
+		const { MonacoLanguageClient } = await import('monaco-languageclient')
+		const { CloseAction, ErrorAction } = await import('vscode-languageclient')
+		const { toSocket, WebSocketMessageReader, WebSocketMessageWriter } = await import(
+			'vscode-ws-jsonrpc'
+		)
+		const vscode = await import('vscode')
+		const { RequestType } = await import('vscode-jsonrpc')
+		// install Monaco language client services
+		const { MonacoServices } = await import('monaco-languageclient')
 
-			monacoServices = MonacoServices.install()
+		monacoServices = MonacoServices.install()
 
-			function createLanguageClient(
-				transports: MessageTransports,
-				name: string,
-				initializationOptions?: any
-			) {
-				const client = new MonacoLanguageClient({
-					name: name,
-					clientOptions: {
-						documentSelector: deno ? ['typescript'] : ['python'],
-						errorHandler: {
-							error: () => ({ action: ErrorAction.Continue }),
-							closed: () => ({
-								action: CloseAction.Restart
-							})
-						},
-						markdown: {
-							isTrusted: true
-						},
-
-						// workspaceFolder: { uri: Uri.parse(`/tmp/${name}`), name: 'tmp', index: 0 },
-						initializationOptions,
-						middleware: {
-							workspace: {
-								configuration: (params, token, configuration) => {
-									return [
-										{
-											enable: true
-										}
-									]
-								}
-							}
-						}
-					},
-					connectionProvider: {
-						get: () => {
-							return Promise.resolve(transports)
-						}
-					}
-				})
-				return client
-			}
-
-			async function connectToLanguageServer(url: string, name: string, options?: any) {
-				try {
-					const webSocket = new WebSocket(url)
-
-					webSocket.onopen = async () => {
-						const socket = toSocket(webSocket)
-						const reader = new WebSocketMessageReader(socket)
-						const writer = new WebSocketMessageWriter(socket)
-						const languageClient = createLanguageClient({ reader, writer }, name, options)
-						websockets.push([languageClient, webSocket])
-
-						reader.onClose(async () => {
-							try {
-								console.log('CLOSE')
-								websocketAlive[name] = false
-								await languageClient.stop()
-							} catch (err) {
-								console.error(err)
-							}
+		function createLanguageClient(
+			transports: MessageTransports,
+			name: string,
+			initializationOptions?: any
+		) {
+			const client = new MonacoLanguageClient({
+				name: name,
+				clientOptions: {
+					documentSelector: [lang],
+					errorHandler: {
+						error: () => ({ action: ErrorAction.Continue }),
+						closed: () => ({
+							action: CloseAction.Restart
 						})
-						socket.onClose((_code, _reason) => {
-							websocketAlive[name] = false
-						})
-
-						try {
-							console.log('started client')
-							await languageClient.start()
-						} catch (err) {
-							console.log('err at client')
-							console.error(err)
-							throw new Error(err)
-						}
-
-						lastWsAttempt = new Date()
-						nbWsAttempt = 0
-						if (name == 'deno') {
-							command && command.dispose()
-							command = undefined
-							command = vscode.commands.registerCommand(
-								'deno.cache',
-								(uris: DocumentUri[] = []) => {
-									languageClient.sendRequest(new RequestType('deno/cache'), {
-										referrer: { uri },
-										uris: uris.map((uri) => ({ uri }))
-									})
-								}
-							)
-						}
-
-						websocketAlive[name] = true
-					}
-				} catch (err) {
-					console.error(`connection to ${name} language server failed`)
-				}
-			}
-
-			if (deno) {
-				await connectToLanguageServer(`wss://${$page.url.host}/ws/deno`, 'deno', {
-					certificateStores: null,
-					enablePaths: [],
-					config: null,
-					importMap: null,
-					internalDebug: false,
-					lint: false,
-					path: null,
-					tlsCertificate: null,
-					unsafelyIgnoreCertificateErrors: null,
-					unstable: true,
-					enable: true,
-					cache: null,
-					codeLens: {
-						implementations: true,
-						references: true
 					},
-					suggest: {
-						autoImports: true,
-						completeFunctionCalls: false,
-						names: true,
-						paths: true,
-						imports: {
-							autoDiscover: true,
-							hosts: {
-								'https://deno.land': true
+					markdown: {
+						isTrusted: true
+					},
+
+					// workspaceFolder: { uri: Uri.parse(`/tmp/${name}`), name: 'tmp', index: 0 },
+					initializationOptions,
+					middleware: {
+						workspace: {
+							configuration: (params, token, configuration) => {
+								return [
+									{
+										enable: true
+									}
+								]
 							}
 						}
 					}
-				})
-			} else {
-				await connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', {
-					executionEnvironments: [
-						{
-							root: '/tmp/pyright',
-							pythonVersion: '3.7',
-							pythonPlatform: 'platform',
-							extraPaths: []
-						}
-					]
-				})
-
-				connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', {
-					formatters: {
-						black: {
-							command: 'black',
-							args: ['--quiet', '-']
-						}
-					},
-					formatFiletypes: {
-						python: 'black'
-					}
-				})
-			}
-
-			websocketInterval && clearInterval(websocketInterval)
-			websocketInterval = setInterval(() => {
-				console.log(
-					websocketInterval,
-					document.visibilityState,
-					new Date().getTime() - lastWsAttempt.getTime(),
-					nbWsAttempt
-				)
-				if (document.visibilityState == 'visible') {
-					if (
-						!lastWsAttempt ||
-						(new Date().getTime() - lastWsAttempt.getTime() > 60000 && nbWsAttempt < 2)
-					) {
-						if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) {
-							console.log('reconnecting to language servers')
-							lastWsAttempt = new Date()
-							nbWsAttempt++
-							reloadWebsocket()
-						} else {
-							if (nbWsAttempt >= 2) {
-								sendUserToast('Giving up on establishing smart assistant connection', true)
-								clearInterval(websocketInterval)
-							}
-						}
+				},
+				connectionProvider: {
+					get: () => {
+						return Promise.resolve(transports)
 					}
 				}
-			}, 5000)
+			})
+			return client
 		}
+
+		async function connectToLanguageServer(url: string, name: string, options?: any) {
+			try {
+				const webSocket = new WebSocket(url)
+
+				webSocket.onopen = async () => {
+					const socket = toSocket(webSocket)
+					const reader = new WebSocketMessageReader(socket)
+					const writer = new WebSocketMessageWriter(socket)
+					const languageClient = createLanguageClient({ reader, writer }, name, options)
+					websockets.push([languageClient, webSocket])
+
+					reader.onClose(async () => {
+						try {
+							console.log('CLOSE')
+							websocketAlive[name] = false
+							await languageClient.stop()
+						} catch (err) {
+							console.error(err)
+						}
+					})
+					socket.onClose((_code, _reason) => {
+						websocketAlive[name] = false
+					})
+
+					try {
+						console.log('started client')
+						await languageClient.start()
+					} catch (err) {
+						console.log('err at client')
+						console.error(err)
+						throw new Error(err)
+					}
+
+					lastWsAttempt = new Date()
+					nbWsAttempt = 0
+					if (name == 'deno') {
+						command && command.dispose()
+						command = undefined
+						command = vscode.commands.registerCommand('deno.cache', (uris: DocumentUri[] = []) => {
+							languageClient.sendRequest(new RequestType('deno/cache'), {
+								referrer: { uri },
+								uris: uris.map((uri) => ({ uri }))
+							})
+						})
+					}
+
+					websocketAlive[name] = true
+				}
+			} catch (err) {
+				console.error(`connection to ${name} language server failed`)
+			}
+		}
+
+		if (lang == 'typescript') {
+			await connectToLanguageServer(`wss://${$page.url.host}/ws/deno`, 'deno', {
+				certificateStores: null,
+				enablePaths: [],
+				config: null,
+				importMap: null,
+				internalDebug: false,
+				lint: false,
+				path: null,
+				tlsCertificate: null,
+				unsafelyIgnoreCertificateErrors: null,
+				unstable: true,
+				enable: true,
+				cache: null,
+				codeLens: {
+					implementations: true,
+					references: true
+				},
+				suggest: {
+					autoImports: true,
+					completeFunctionCalls: false,
+					names: true,
+					paths: true,
+					imports: {
+						autoDiscover: true,
+						hosts: {
+							'https://deno.land': true
+						}
+					}
+				}
+			})
+		} else if (lang === 'python') {
+			await connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', {
+				executionEnvironments: [
+					{
+						root: '/tmp/pyright',
+						pythonVersion: '3.7',
+						pythonPlatform: 'platform',
+						extraPaths: []
+					}
+				]
+			})
+
+			connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', {
+				formatters: {
+					black: {
+						command: 'black',
+						args: ['--quiet', '-']
+					}
+				},
+				formatFiletypes: {
+					python: 'black'
+				}
+			})
+		}
+
+		websocketInterval && clearInterval(websocketInterval)
+		websocketInterval = setInterval(() => {
+			if (document.visibilityState == 'visible') {
+				if (
+					!lastWsAttempt ||
+					(new Date().getTime() - lastWsAttempt.getTime() > 60000 && nbWsAttempt < 2)
+				) {
+					if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) {
+						console.log('reconnecting to language servers')
+						lastWsAttempt = new Date()
+						nbWsAttempt++
+						reloadWebsocket()
+					} else {
+						if (nbWsAttempt >= 2) {
+							sendUserToast('Giving up on establishing smart assistant connection', true)
+							clearInterval(websocketInterval)
+						}
+					}
+				}
+			}
+		}, 5000)
 	}
 
 	async function closeWebsockets() {
@@ -388,15 +376,13 @@
 
 		editor.onDidFocusEditorText(() => {
 			dispatch('focus')
-			if (deno || lang == 'typescript') {
-				if (
-					!websocketAlive.black &&
-					!websocketAlive.deno &&
-					!websocketAlive.pyright &&
-					!websocketInterval
-				) {
-					reloadWebsocket()
-				}
+			if (
+				!websocketAlive.black &&
+				!websocketAlive.deno &&
+				!websocketAlive.pyright &&
+				!websocketInterval
+			) {
+				reloadWebsocket()
 			}
 		})
 
@@ -404,9 +390,7 @@
 			dispatch('blur')
 		})
 
-		if (lang == 'python' || deno) {
-			reloadWebsocket()
-		}
+		reloadWebsocket()
 
 		return () => {
 			try {

frontend/src/lib/components/EditorBar.svelte

@@ -14,7 +14,7 @@
 	import ResourceEditor from './ResourceEditor.svelte'
 	import VariableEditor from './VariableEditor.svelte'
 
-	export let lang: 'python3' | 'deno'
+	export let lang: 'python3' | 'deno' | 'go'
 	export let editor: Editor
 	export let websocketAlive: { pyright: boolean; black: boolean; deno: boolean }
 
@@ -25,7 +25,7 @@
 	let resourceEditor: ResourceEditor
 
 	let codeViewer: Modal
-	let codeLang: 'python3' | 'deno' = 'deno'
+	let codeLang: 'python3' | 'deno' | 'go' = 'deno'
 	let codeContent: string = ''
 
 	async function loadVariables() {
@@ -85,7 +85,7 @@
 		if (!path) {
 			if (lang == 'deno') {
 				editor.insertAtCursor(`Deno.env.get('${name}')`)
-			} else {
+			} else if (lang == 'python3') {
 				if (!editor.getCode().includes('import os')) {
 					editor.insertAtBeginning('import os\n')
 				}
@@ -100,7 +100,7 @@
 					)
 				}
 				editor.insertAtCursor(`(await wmill.getVariable('${path}'))`)
-			} else {
+			} else if (lang == 'python3') {
 				if (!editor.getCode().includes('import wmill')) {
 					editor.insertAtBeginning('import wmill\n')
 				}
@@ -139,7 +139,7 @@
 				)
 			}
 			editor.insertAtCursor(`(await wmill.getResource('${path}'))`)
-		} else {
+		} else if (lang == 'python3') {
 			if (!editor.getCode().includes('import wmill')) {
 				editor.insertAtBeginning('import wmill\n')
 			}

frontend/src/lib/components/FlowModulesViewer.svelte

@@ -1,15 +1,12 @@
 <script lang="ts">
 	import { scriptPathToHref } from '$lib/utils'
 
-	import Highlight from 'svelte-highlight'
-	import python from 'svelte-highlight/languages/python'
-	import typescript from 'svelte-highlight/languages/typescript'
-
 	import { slide } from 'svelte/transition'
 
 	import InputTransformsViewer from './InputTransformsViewer.svelte'
 	import IconedPath from './IconedPath.svelte'
 	import type { FlowModule } from '$lib/gen'
+	import HighlightCode from './HighlightCode.svelte'
 
 	export let modules: FlowModule[]
 
@@ -82,8 +79,8 @@
 									{#if open[i]}
 										<div transition:slide class="border border-black p-2 bg-gray-50 w-full">
 											<InputTransformsViewer inputTransforms={mod?.input_transforms} />
-											<Highlight
-												language={mod?.value?.language == 'deno' ? typescript : python}
+											<HighlightCode
+												language={mod?.value?.language ?? 'deno'}
 												code={mod?.value?.content}
 											/>
 										</div>

frontend/src/lib/components/HighlightCode.svelte

@@ -0,0 +1,24 @@
+<script lang="ts">
+	import Highlight from 'svelte-highlight'
+	import python from 'svelte-highlight/languages/python'
+	import typescript from 'svelte-highlight/languages/typescript'
+	import go from 'svelte-highlight/languages/go'
+
+	export let code: string = ''
+	export let language: 'python3' | 'deno' | 'go' | undefined
+
+	function getLang() {
+		switch (language) {
+			case 'python3':
+				return python
+			case 'deno':
+				return typescript
+			case 'go':
+				return go
+			default:
+				return python
+		}
+	}
+</script>
+
+<Highlight language={getLang()} {code} />

frontend/src/lib/components/ModuleStep.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { RawScript, type FlowModule } from '$lib/gen'
+	import type { FlowModule } from '$lib/gen'
 	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
 	import Icon from 'svelte-awesome'
 	import Editor from './Editor.svelte'
@@ -23,6 +23,7 @@
 	import type { Schema } from '$lib/common'
 	import { flowStateStore, type FlowModuleSchema } from './flows/flowState'
 	import { stepOpened } from './flows/stepOpenedStore'
+	import { scriptLangToEditorLang } from '$lib/utils'
 
 	export let indexes: number[]
 	export let mod: FlowModule
@@ -109,7 +110,7 @@
 							bind:this={editor}
 							class="{bigEditor ? 'h-2/3' : 'h-80'} border p-2 rounded"
 							bind:code={mod.value.content}
-							deno={mod.value.language === RawScript.language.DENO}
+							lang={scriptLangToEditorLang(mod.value.language)}
 							automaticLayout={true}
 							formatAction={() => reload(mod)}
 						/>

frontend/src/lib/components/ScriptBuilder.svelte

@@ -35,7 +35,7 @@
 	}
 
 	function initContent(
-		language: 'deno' | 'python3',
+		language: 'deno' | 'python3' | 'go',
 		kind: Script.kind,
 		template: 'pgsql' | 'script'
 	) {
@@ -190,7 +190,8 @@
 						label="Language"
 						options={[
 							['Typescript (Deno)', 'deno'],
-							['Python 3.10', 'python3']
+							['Python 3.10', 'python3'],
+							['Go', 'go']
 						]}
 						on:change={(e) => initContent(e.detail, script.kind, template)}
 						bind:value={script.language}

frontend/src/lib/components/ScriptEditor.svelte

@@ -2,7 +2,7 @@
 	import type { Schema } from '$lib/common'
 	import { CompletedJob, Job, JobService } from '$lib/gen'
 	import { userStore, workspaceStore } from '$lib/stores'
-	import { classNames, emptySchema } from '$lib/utils'
+	import { classNames, emptySchema, scriptLangToEditorLang } from '$lib/utils'
 	import {
 		faCheck,
 		faExclamationTriangle,
@@ -253,7 +253,7 @@
 							inferSchema()
 						}}
 						class="flex flex-1 h-full"
-						deno={lang == 'deno'}
+						lang={scriptLangToEditorLang(lang)}
 						automaticLayout={true}
 					/>
 				</div>

frontend/src/lib/components/ScriptPicker.svelte

@@ -24,7 +24,7 @@
 	let itemPicker: ItemPicker
 	let modalViewer: Modal
 	let code: string = ''
-	let lang: 'deno' | 'python3' | undefined
+	let lang: 'deno' | 'python3' | 'go' | undefined
 
 	let options: [[string, any]] = [['Script', 'script']]
 	allowHub && options.unshift(['Hub', 'hub'])

frontend/src/lib/components/flows/FlowInputs.svelte

@@ -52,6 +52,14 @@
 			on:click={() =>
 				dispatch('new', { language: RawScript.language.DENO, kind: 'script', subkind: 'flow' })}
 		/>
+
+		<FlowScriptPicker
+			label="New Go script"
+			icon={faCode}
+			iconColor="text-blue-700"
+			on:click={() =>
+				dispatch('new', { language: RawScript.language.GO, kind: 'script', subkind: 'flow' })}
+		/>
 	</div>
 
 	{#if !shouldDisableTriggerScripts}

frontend/src/lib/components/flows/FlowModuleHeader.svelte

@@ -12,9 +12,7 @@
 	import { Button } from 'flowbite-svelte'
 	import { createEventDispatcher } from 'svelte'
 	import Icon from 'svelte-awesome'
-	import { Highlight } from 'svelte-highlight'
-	import python from 'svelte-highlight/languages/python'
-	import typescript from 'svelte-highlight/languages/typescript'
+	import HighlightCode from '../HighlightCode.svelte'
 	import IconedPath from '../IconedPath.svelte'
 	import Modal from '../Modal.svelte'
 	import { isEmptyFlowModule } from './flowStateUtils'
@@ -27,7 +25,7 @@
 
 	let modalViewer: Modal
 	let modalViewerContent = ''
-	let modalViewerLanguage: 'deno' | 'python3' = 'deno'
+	let modalViewerLanguage: 'deno' | 'python3' | 'go' = 'deno'
 
 	async function viewCode() {
 		if (mod.value.type == 'script') {
@@ -113,10 +111,6 @@
 <Modal bind:this={modalViewer}>
 	<div slot="title">Script {'path' in mod?.value ? mod?.value.path : ''}</div>
 	<div slot="content">
-		{#if modalViewerLanguage === 'python3'}
-			<Highlight language={python} code={modalViewerContent} />
-		{:else if modalViewerLanguage === 'deno'}
-			<Highlight language={typescript} code={modalViewerContent} />
-		{/if}
+		<HighlightCode language={modalViewerLanguage} code={modalViewerContent} />
 	</div>
 </Modal>

frontend/src/lib/components/script_editor/LogPanel.svelte

@@ -13,8 +13,9 @@
 	import TableCustom from '../TableCustom.svelte'
 	import Drawer from '../common/drawer/Drawer.svelte'
 	import { Highlight } from 'svelte-highlight'
-	import { json, python, typescript } from 'svelte-highlight/languages'
+	import { json } from 'svelte-highlight/languages'
 	import DrawerContent from '../common/drawer/DrawerContent.svelte'
+	import HighlightCode from '../HighlightCode.svelte'
 
 	export let path: string | undefined
 	export let lang: Preview.language
@@ -51,10 +52,8 @@
 			<pre class="overflow-x-auto break-all relative h-full m-2 text-xs bg-white shadow-inner p-2">
 				{drawerContent?.content}
 			</pre>
-		{:else if drawerContent?.mode === 'deno'}
-			<Highlight language={typescript} code={drawerContent?.content} />
-		{:else if drawerContent?.mode === 'python3'}
-			<Highlight language={python} code={drawerContent?.content} />
+		{:else if drawerContent?.mode === 'deno' || drawerContent?.mode === 'python3' || drawerContent?.mode === 'go'}}
+			<HighlightCode language={drawerContent?.mode} code={drawerContent?.content} />
 		{/if}
 	</DrawerContent>
 </Drawer>

frontend/src/lib/infer.ts

@@ -2,7 +2,7 @@ import { ScriptService, type MainArgSignature } from '$lib/gen'
 import type { Schema, SchemaProperty } from './common.js'
 
 export async function inferArgs(
-	language: 'python3' | 'deno',
+	language: 'python3' | 'deno' | 'go',
 	code: string,
 	schema: Schema
 ): Promise<void> {
@@ -16,6 +16,10 @@ export async function inferArgs(
 		inferedSchema = await ScriptService.denoToJsonschema({
 			requestBody: code
 		})
+	} else if (language == 'go') {
+		inferedSchema = await ScriptService.goToJsonschema({
+			requestBody: code
+		})
 	} else {
 		return
 	}

frontend/src/lib/script_helpers.ts

@@ -49,6 +49,18 @@ export async function main(
 }
 `
 
+export const GO_INIT_CODE = `import (
+	"fmt"
+	"rsc.io/quote"
+)
+
+func main(x string) (interface{}, error) {
+	fmt.Println("Hello, World")
+	fmt.Println(quote.Opt())
+  return x, nil
+}
+`
+
 export const DENO_INIT_CODE_CLEAR = `// import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"
 
 export async function main() {
@@ -111,7 +123,7 @@ export function isInitialCode(content: string): boolean {
   return false
 }
 
-export function initialCode(language: 'deno' | 'python3', kind: Script.kind, subkind: 'pgsql' | 'flow' | 'script' | undefined): string {
+export function initialCode(language: 'deno' | 'python3' | 'go', kind: Script.kind, subkind: 'pgsql' | 'flow' | 'script' | undefined): string {
   if (language === 'deno') {
     if (kind === 'trigger') {
       return DENO_INIT_CODE_TRIGGER
@@ -126,11 +138,13 @@ export function initialCode(language: 'deno' | 'python3', kind: Script.kind, sub
     } else {
       return DENO_INIT_CODE
     }
-  } else {
+  } else if (language === 'python3') {
     if (subkind === 'flow') {
       return PYTHON_INIT_CODE_CLEAR
     } else {
       return PYTHON_INIT_CODE
     }
+  } else {
+    return GO_INIT_CODE
   }
 }

frontend/src/lib/utils.ts

@@ -475,7 +475,7 @@ export function scriptPathToHref(path: string): string {
 
 export async function getScriptByPath(path: string): Promise<{
 	content: string
-	language: 'deno' | 'python3'
+	language: 'deno' | 'python3' | 'go'
 }> {
 	if (path.startsWith('hub/')) {
 		const content = await ScriptService.getHubScriptContentByPath({ path })
@@ -560,3 +560,14 @@ export function scriptToHubUrl(
 export function classNames(...classes: string[]): string {
 	return classes.filter(Boolean).join(' ')
 }
+
+
+export function scriptLangToEditorLang(lang: Script.language): 'typescript' | 'python' | 'go' {
+	if (lang == 'deno') {
+		return 'typescript'
+	} else if (lang == 'python3') {
+		return 'python'
+	} else {
+		return lang
+	}
+}

frontend/src/routes/run/[...run].svelte

@@ -40,15 +40,13 @@
 	import Tooltip from '$lib/components/Tooltip.svelte'
 	import DisplayResult from '$lib/components/DisplayResult.svelte'
 
-	import Highlight from 'svelte-highlight'
-	import typescript from 'svelte-highlight/languages/typescript'
-	import python from 'svelte-highlight/languages/python'
 	import { userStore, workspaceStore } from '$lib/stores'
 	import CenteredPage from '$lib/components/CenteredPage.svelte'
 	import FlowStatusViewer from '$lib/components/FlowStatusViewer.svelte'
 	import JobStatus from '$lib/components/JobStatus.svelte'
 	import TableCustom from '$lib/components/TableCustom.svelte'
 	import ArgInfo from '$lib/components/ArgInfo.svelte'
+	import HighlightCode from '$lib/components/HighlightCode.svelte'
 
 	let workspace_id_query: string | undefined = $page.url.searchParams.get('workspace') ?? undefined
 	let workspace_id: string | undefined
@@ -438,11 +436,7 @@
 					{:else}Loading...{/if}
 				{:else if viewTab == 'code'}
 					{#if job && 'raw_code' in job && job.raw_code}
-						{#if job.language == 'python3'}
-							<Highlight language={python} code={job.raw_code} />
-						{:else if job.language == 'deno'}
-							<Highlight language={typescript} code={job.raw_code} />
-						{/if}
+						<HighlightCode language={job.language} code={job.raw_code} />
 					{:else if job}No code is available
 					{:else}Loading...{/if}
 				{:else if job && 'result' in job && job.result}<DisplayResult result={job.result} />

frontend/src/routes/scripts/get/[...hash].svelte

@@ -30,9 +30,6 @@
 		faGlobe,
 		faCodeFork
 	} from '@fortawesome/free-solid-svg-icons'
-	import Highlight from 'svelte-highlight'
-	import typescript from 'svelte-highlight/languages/typescript'
-	import python from 'svelte-highlight/languages/python'
 
 	import Tooltip from '$lib/components/Tooltip.svelte'
 	import ShareModal from '$lib/components/ShareModal.svelte'
@@ -43,6 +40,7 @@
 	import Dropdown from '$lib/components/Dropdown.svelte'
 	import CenteredPage from '$lib/components/CenteredPage.svelte'
 	import { onDestroy } from 'svelte'
+	import HighlightCode from '$lib/components/HighlightCode.svelte'
 
 	let script: Script | undefined
 	let topHash: string | undefined
@@ -384,11 +382,7 @@
 			</div>
 			<div>
 				<h3 class="text-gray-700 pb-1 mb-3 border-b">Code</h3>
-				{#if script.language == 'python3'}
-					<Highlight language={python} code={script.content} />
-				{:else if script.language == 'deno'}
-					<Highlight language={typescript} code={script.content} />
-				{/if}
+				<HighlightCode language={script.language} code={script.content} />
 			</div>
 			<div>
 				<h3 class="text-gray-700 pb-1 mb-3 border-b">Dependencies lock file</h3>

nsjail/download.py.config.proto

@@ -84,11 +84,6 @@ mount {
     options: "size=500000000"
 }
 
-mount {
-    src: "{JOB_DIR}/download.config.proto"
-    dst: "/user/download.config.proto"
-    is_bind: true
-}
 
 mount {
     src: "{JOB_DIR}/requirements.txt"
@@ -105,7 +100,7 @@ mount {
 
 
 mount {
-    src: "{WORKER_DIR}/download_deps.sh"
+    src: "{WORKER_DIR}/download_deps.py.sh"
     dst: "/download_deps.sh"
     is_bind: true
 }

nsjail/run.go.config.proto

@@ -0,0 +1,126 @@
+name: "go run script"
+
+mode: ONCE
+hostname: "go"
+log_level: ERROR
+time_limit: 300
+
+rlimit_as: 2048
+rlimit_cpu: 1000
+rlimit_fsize: 1024
+rlimit_nofile: 64
+
+cwd: "/tmp/go"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+keep_caps: false
+keep_env: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+}
+
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+
+mount {
+	src: "/dev/null"
+	dst: "/dev/null"
+	is_bind: true
+	rw: true
+}
+
+mount {
+	dst: "/tmp"
+	fstype: "tmpfs"
+	rw: true
+    options: "size=500000000"
+}
+
+mount {
+    src: "{JOB_DIR}/go.sum"
+    dst: "/tmp/go/go.sum"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "{JOB_DIR}/go.mod"
+    dst: "/tmp/go/go.mod"
+    is_bind: true
+}
+
+
+mount {
+    src: "{JOB_DIR}/mymod"
+    dst: "/tmp/go/mymod"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/args.json"
+    dst: "/tmp/go/args.json"
+    is_bind: true
+}
+
+
+mount {
+    src: "/etc/ssl"
+    dst: "/etc/ssl"
+	is_bind: true
+}
+
+mount {
+    src: "/etc/pki"
+    dst: "/etc/pki"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/resolv.conf"
+    dst: "/etc/resolv.conf"
+	is_bind: true
+}
+
+mount {
+    src: "/dev/random"
+    dst: "/dev/random"
+    is_bind: true
+}
+
+iface_no_lo: true
+
+mount {
+    src: "{CACHE_DIR}"
+    dst: "/tmp/.cache/go"
+    is_bind: true
+    rw: true
+    mandatory: false
+}
+
+envar: "GOPATH=/tmp/.cache/go"
+envar: "HOME=/tmp/go"
+
+

openflow.openapi.yaml

@@ -147,6 +147,7 @@ components:
           enum:
             - deno
             - python3
+            - go
         path:
           type: string
         type: